From c837f16b5b05b1b7a67a331381adbdc90e71ab69 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 09:59:26 +0000 Subject: [PATCH 001/153] test(distributed): share suite containers, isolate specs by database Starting a Postgres and a NATS container per spec cost roughly 48 minutes of startup across the 213 specs behind SetupInfra, which is why this suite was never wired into CI. Containers move to BeforeSuite and isolation comes from CREATE DATABASE, which the dbName argument already described. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/dbname_test.go | 33 +++++ tests/e2e/distributed/testhelpers_test.go | 167 +++++++++++++++++----- 2 files changed, 166 insertions(+), 34 deletions(-) create mode 100644 tests/e2e/distributed/dbname_test.go diff --git a/tests/e2e/distributed/dbname_test.go b/tests/e2e/distributed/dbname_test.go new file mode 100644 index 000000000000..76f1030747fa --- /dev/null +++ b/tests/e2e/distributed/dbname_test.go @@ -0,0 +1,33 @@ +package distributed_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Test database naming", Label("Distributed"), func() { + Describe("sanitizeDBName", func() { + It("lowercases and replaces characters Postgres will not accept unquoted", func() { + Expect(sanitizeDBName("LocalAI-Test.Suite")).To(Equal("localai_test_suite")) + }) + + It("truncates to fit the 63-byte identifier limit with room for a suffix", func() { + long := "" + for i := 0; i < 100; i++ { + long += "a" + } + Expect(len(sanitizeDBName(long))).To(BeNumerically("<=", 50)) + }) + + It("never produces an empty name", func() { + Expect(sanitizeDBName("---")).ToNot(BeEmpty()) + }) + }) + + Describe("replaceDBName", func() { + It("swaps the database in a testcontainers DSN and keeps the query string", func() { + dsn := "postgres://test:test@127.0.0.1:32768/localai_suite?sslmode=disable" + Expect(replaceDBName(dsn, "spec_7")).To(Equal("postgres://test:test@127.0.0.1:32768/spec_7?sslmode=disable")) + }) + }) +}) diff --git a/tests/e2e/distributed/testhelpers_test.go b/tests/e2e/distributed/testhelpers_test.go index 68cf537e30bd..564e8c40603a 100644 --- a/tests/e2e/distributed/testhelpers_test.go +++ b/tests/e2e/distributed/testhelpers_test.go @@ -2,6 +2,10 @@ package distributed_test import ( "context" + "fmt" + "net/url" + "strings" + "sync/atomic" "time" "github.com/mudler/LocalAI/core/services/messaging" @@ -13,6 +17,9 @@ import ( tcnats "github.com/testcontainers/testcontainers-go/modules/nats" tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" "github.com/testcontainers/testcontainers-go/wait" + "gorm.io/driver/postgres" + "gorm.io/gorm" + gormlogger "gorm.io/gorm/logger" ) // TestInfra holds shared test containers and connection strings. @@ -25,71 +32,167 @@ type TestInfra struct { NC *messaging.Client } -// SetupInfra starts PostgreSQL and NATS containers and connects a messaging client. -// Call in BeforeEach. Use DeferCleanup or call Teardown in AfterEach. -func SetupInfra(dbName string) *TestInfra { - GinkgoHelper() +// Containers are suite-scoped, not spec-scoped. Starting a Postgres (~10s) and a +// NATS (~3.5s) per spec cost roughly 48 minutes of pure startup across the 213 +// specs behind SetupInfra, which is why this suite was never wired into CI. +// Isolation now comes from a database per spec (~67ms), which is what the dbName +// argument was always describing. +// +// Plain BeforeSuite rather than SynchronizedBeforeSuite is deliberate: under +// `ginkgo -p` each process gets its own container pair, which keeps NATS subjects +// isolated per process. A single shared NATS across parallel processes would let +// specs on different processes see each other's messages on the same subject. +var ( + suitePG *tcpostgres.PostgresContainer + suiteNATS *tcnats.NATSContainer + suitePGDSN string + suiteNatsURL string + dbCounter atomic.Int64 +) - infra := &TestInfra{Ctx: context.Background()} +var _ = BeforeSuite(func() { + ctx := context.Background() var err error - // Start PostgreSQL container - infra.PGContainer, err = tcpostgres.Run(infra.Ctx, "postgres:16-alpine", - tcpostgres.WithDatabase(dbName), + suitePG, err = tcpostgres.Run(ctx, "postgres:16-alpine", + tcpostgres.WithDatabase("localai_suite"), tcpostgres.WithUsername("test"), tcpostgres.WithPassword("test"), testcontainers.WithWaitStrategy( wait.ForLog("database system is ready to accept connections"). WithOccurrence(2). - WithStartupTimeout(30*time.Second), + WithStartupTimeout(90*time.Second), ), ) Expect(err).ToNot(HaveOccurred()) - infra.PGURL, err = infra.PGContainer.ConnectionString(infra.Ctx, "sslmode=disable") + suitePGDSN, err = suitePG.ConnectionString(ctx, "sslmode=disable") + Expect(err).ToNot(HaveOccurred()) + + suiteNATS, err = tcnats.Run(ctx, "nats:2-alpine") + Expect(err).ToNot(HaveOccurred()) + + suiteNatsURL, err = suiteNATS.ConnectionString(ctx) Expect(err).ToNot(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + ctx := context.Background() + if suitePG != nil { + _ = suitePG.Terminate(ctx) + } + if suiteNATS != nil { + _ = suiteNATS.Terminate(ctx) + } +}) + +// sanitizeDBName maps a spec-supplied label onto a legal unquoted Postgres +// identifier, leaving headroom for the uniqueness suffix appended by SetupInfra. +func sanitizeDBName(name string) string { + var b strings.Builder + for _, r := range strings.ToLower(name) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '_': + b.WriteRune(r) + default: + b.WriteRune('_') + } + } + out := strings.Trim(b.String(), "_") + if out == "" { + out = "spec" + } + // Postgres identifiers cap at 63 bytes; reserve the rest for "_". + if len(out) > 50 { + out = out[:50] + } + return out +} - // Start NATS container - infra.NATSContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine") +// replaceDBName swaps the database component of a DSN, preserving credentials, +// host, port and query parameters. +func replaceDBName(dsn, name string) string { + GinkgoHelper() + u, err := url.Parse(dsn) Expect(err).ToNot(HaveOccurred()) + u.Path = "/" + name + return u.String() +} - infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx) +// adminDB opens a short-lived connection to the suite's maintenance database. +// CREATE/DROP DATABASE cannot run inside a transaction or against the target +// database itself, so every call gets its own connection and closes it. +func adminDB() *gorm.DB { + GinkgoHelper() + db, err := gorm.Open(postgres.Open(suitePGDSN), &gorm.Config{Logger: gormlogger.Discard}) Expect(err).ToNot(HaveOccurred()) + return db +} + +func closeDB(db *gorm.DB) { + if db == nil { + return + } + if sqlDB, err := db.DB(); err == nil { + _ = sqlDB.Close() + } +} - // Connect messaging client +// SetupInfra provisions a dedicated database on the suite-scoped Postgres and +// returns a client connected to the suite-scoped NATS. Call in BeforeEach; +// cleanup is registered with DeferCleanup. +func SetupInfra(dbName string) *TestInfra { + GinkgoHelper() + Expect(suitePG).ToNot(BeNil(), "SetupInfra called before BeforeSuite started the shared containers") + + infra := &TestInfra{ + Ctx: context.Background(), + PGContainer: suitePG, + NATSContainer: suiteNATS, + NatsURL: suiteNatsURL, + } + + db := fmt.Sprintf("%s_%d", sanitizeDBName(dbName), dbCounter.Add(1)) + + admin := adminDB() + Expect(admin.Exec(fmt.Sprintf("CREATE DATABASE %q", db)).Error).To(Succeed()) + closeDB(admin) + + infra.PGURL = replaceDBName(suitePGDSN, db) + + var err error infra.NC, err = messaging.New(infra.NatsURL) Expect(err).ToNot(HaveOccurred()) - // Register cleanup in LIFO order DeferCleanup(func() { if infra.NC != nil { infra.NC.Close() } - if infra.PGContainer != nil { - infra.PGContainer.Terminate(context.Background()) - } - if infra.NATSContainer != nil { - infra.NATSContainer.Terminate(context.Background()) + // FORCE terminates any connection the spec left open (Postgres 13+). + // Failure to drop must not fail the spec: the container dies at AfterSuite. + drop := adminDB() + if err := drop.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %q WITH (FORCE)", db)).Error; err != nil { + AddReportEntry("drop database failed", fmt.Sprintf("%s: %v", db, err)) } + closeDB(drop) }) return infra } -// SetupNATSOnly starts only a NATS container and connects a messaging client. -// Useful for tests that don't need PostgreSQL. +// SetupNATSOnly returns a client on the suite-scoped NATS for specs that need no +// database. func SetupNATSOnly() *TestInfra { GinkgoHelper() + Expect(suiteNATS).ToNot(BeNil(), "SetupNATSOnly called before BeforeSuite started the shared containers") - infra := &TestInfra{Ctx: context.Background()} - var err error - - infra.NATSContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine") - Expect(err).ToNot(HaveOccurred()) - - infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx) - Expect(err).ToNot(HaveOccurred()) + infra := &TestInfra{ + Ctx: context.Background(), + NATSContainer: suiteNATS, + NatsURL: suiteNatsURL, + } + var err error infra.NC, err = messaging.New(infra.NatsURL) Expect(err).ToNot(HaveOccurred()) @@ -97,16 +200,12 @@ func SetupNATSOnly() *TestInfra { if infra.NC != nil { infra.NC.Close() } - if infra.NATSContainer != nil { - infra.NATSContainer.Terminate(context.Background()) - } }) return infra } // FlushNATS ensures all subscriptions are registered server-side before publishing. -// Replaces time.Sleep(100ms) after Subscribe calls. func FlushNATS(nc *messaging.Client) { GinkgoHelper() Expect(nc.Conn().Flush()).To(Succeed()) From d0136191c0fe2566c0a6c6e44233bf1831ef9ba0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 10:08:17 +0000 Subject: [PATCH 002/153] test(distributed): stop leaking admin pools when database setup fails A failed CREATE DATABASE panics out of the assertion before closeDB runs, leaking a pgx pool per attempt. With --flake-attempts 5 that exhausts postgres:16-alpine's 100 connection slots, at which point the cleanup path's own Expect fails the spec and one hiccup cascades across the suite. Scope the admin handle so the panic unwinds through defer closeDB, and let cleanup use a fallible tryAdminDB that reports rather than asserts. Register DeferCleanup immediately after CREATE so a later failure cannot leave the database behind, and warn on TestInfra that the container handles are now suite-wide. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/testhelpers_test.go | 54 +++++++++++++++++------ 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/tests/e2e/distributed/testhelpers_test.go b/tests/e2e/distributed/testhelpers_test.go index 564e8c40603a..17ee72ce2e89 100644 --- a/tests/e2e/distributed/testhelpers_test.go +++ b/tests/e2e/distributed/testhelpers_test.go @@ -23,6 +23,11 @@ import ( ) // TestInfra holds shared test containers and connection strings. +// +// PGContainer and NATSContainer are the SUITE-WIDE containers, shared by every +// spec. Never call Terminate or Stop on them from a spec: it ends the run for +// everything after it. They are exposed only because nats_jwt_helpers_test.go +// builds its own TestInfra around a dedicated NATS container. type TestInfra struct { Ctx context.Context PGContainer *tcpostgres.PostgresContainer @@ -119,12 +124,24 @@ func replaceDBName(dsn, name string) string { return u.String() } -// adminDB opens a short-lived connection to the suite's maintenance database. +// tryAdminDB opens a short-lived connection to the suite's maintenance +// database. Cleanup paths use this rather than adminDB: once connections are +// scarce, a fatal assertion here would convert one Postgres hiccup into a +// suite-wide cascade that buries the original failure. +// // CREATE/DROP DATABASE cannot run inside a transaction or against the target // database itself, so every call gets its own connection and closes it. +func tryAdminDB() (*gorm.DB, error) { + db, err := gorm.Open(postgres.Open(suitePGDSN), &gorm.Config{Logger: gormlogger.Discard}) + if err != nil { + return nil, fmt.Errorf("connecting to the suite maintenance database: %w", err) + } + return db, nil +} + func adminDB() *gorm.DB { GinkgoHelper() - db, err := gorm.Open(postgres.Open(suitePGDSN), &gorm.Config{Logger: gormlogger.Discard}) + db, err := tryAdminDB() Expect(err).ToNot(HaveOccurred()) return db } @@ -154,29 +171,38 @@ func SetupInfra(dbName string) *TestInfra { db := fmt.Sprintf("%s_%d", sanitizeDBName(dbName), dbCounter.Add(1)) - admin := adminDB() - Expect(admin.Exec(fmt.Sprintf("CREATE DATABASE %q", db)).Error).To(Succeed()) - closeDB(admin) - - infra.PGURL = replaceDBName(suitePGDSN, db) - - var err error - infra.NC, err = messaging.New(infra.NatsURL) - Expect(err).ToNot(HaveOccurred()) + // Scoped so a failed CREATE cannot leak the pool: the assertion panics, and a + // leaked pgx pool per failing spec exhausts the server's connection limit. + func() { + admin := adminDB() + defer closeDB(admin) + Expect(admin.Exec(fmt.Sprintf("CREATE DATABASE %q", db)).Error).To(Succeed()) + }() + // Registered before anything else can fail: a NATS connect error below would + // otherwise leave the database behind for the rest of the suite. DeferCleanup(func() { if infra.NC != nil { infra.NC.Close() } + drop, err := tryAdminDB() + if err != nil { + AddReportEntry("drop database skipped", fmt.Sprintf("%s: %v", db, err)) + return + } + defer closeDB(drop) // FORCE terminates any connection the spec left open (Postgres 13+). - // Failure to drop must not fail the spec: the container dies at AfterSuite. - drop := adminDB() if err := drop.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %q WITH (FORCE)", db)).Error; err != nil { AddReportEntry("drop database failed", fmt.Sprintf("%s: %v", db, err)) } - closeDB(drop) }) + infra.PGURL = replaceDBName(suitePGDSN, db) + + var err error + infra.NC, err = messaging.New(infra.NatsURL) + Expect(err).ToNot(HaveOccurred()) + return infra } From 4c8b8c3441713859ff83b8bf881d84087082ee14 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 10:31:51 +0000 Subject: [PATCH 003/153] test(distributed): wait for the log subscriber instead of racing it The WebSocket log handler writes its "initial" batch before it calls Subscribe, so a line appended the instant that batch arrives lands in the circular buffer with no subscriber to receive it. Three backend-logs specs append exactly there and then wait out a 5s read deadline; once a gorilla read hits its deadline the connection is unusable, so the spec cannot retry. `--focus='Worker WebSocket log streaming' --repeat=25` failed on attempt 17 with nothing else running, which is far too often to wire into CI. Add BackendLogStore.SubscriberCount, resolving a model ID by the same exact-key and replica-prefix rules Subscribe uses, and have the specs poll it until the handler has attached. Nothing in production calls it and no assertion is weakened; the handler's own snapshot/subscribe window is left as it is, being a production streaming question rather than a test one. Verified with 60 repeats of the WebSocket specs and three consecutive --randomize-all runs of the whole distributed suite, all at --flake-attempts 1: 239 of 240 specs pass in about 80 seconds. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- pkg/model/backend_log_store.go | 39 ++++++++++++++++++++++ pkg/model/backend_log_store_test.go | 32 ++++++++++++++++++ tests/e2e/distributed/backend_logs_test.go | 18 ++++++++++ 3 files changed, 89 insertions(+) diff --git a/pkg/model/backend_log_store.go b/pkg/model/backend_log_store.go index c5b5253ddc40..30268fde47de 100644 --- a/pkg/model/backend_log_store.go +++ b/pkg/model/backend_log_store.go @@ -344,3 +344,42 @@ func (s *BackendLogStore) Subscribe(modelID string) (chan BackendLogLine, func() return ch, unsubscribe } + +// SubscriberCount reports how many live subscriptions exist for modelID, +// resolving the ID with the same exact-key / replica-prefix rules as Subscribe. +// +// Streaming handlers send a GetLines snapshot before they call Subscribe, so a +// line appended between those two calls reaches the buffer but no channel. A +// caller that has to observe a line it appends itself must therefore wait for +// the subscription to exist rather than assume the handler got there first. +func (s *BackendLogStore) SubscriberCount(modelID string) int { + s.mu.RLock() + exact, exactOK := s.buffers[modelID] + var replicas []*backendLogBuffer + if !strings.Contains(modelID, replicaSeparator) { + prefix := modelID + replicaSeparator + for k, b := range s.buffers { + if strings.HasPrefix(k, prefix) { + replicas = append(replicas, b) + } + } + } + s.mu.RUnlock() + + // Counted after releasing s.mu: no other path takes s.mu and a buffer lock + // together, and keeping it that way costs nothing here. + count := func(buf *backendLogBuffer) int { + buf.mu.Lock() + defer buf.mu.Unlock() + return len(buf.subscribers) + } + + total := 0 + if exactOK { + total += count(exact) + } + for _, b := range replicas { + total += count(b) + } + return total +} diff --git a/pkg/model/backend_log_store_test.go b/pkg/model/backend_log_store_test.go index 775e07cdb561..593bcf5a23fa 100644 --- a/pkg/model/backend_log_store_test.go +++ b/pkg/model/backend_log_store_test.go @@ -76,6 +76,38 @@ var _ = Describe("BackendLogStore", func() { }) }) + Describe("SubscriberCount", func() { + It("reports zero before anyone subscribes and drops back after unsubscribe", func() { + s.AppendLine("model-a", "stderr", "preload") + Expect(s.SubscriberCount("model-a")).To(Equal(0)) + + _, unsubscribe := s.Subscribe("model-a") + Expect(s.SubscriberCount("model-a")).To(Equal(1)) + + unsubscribe() + Expect(s.SubscriberCount("model-a")).To(Equal(0)) + }) + + // Subscribe resolves a bare model ID across every replica buffer, so the + // count has to follow the same rule or a caller waiting on it would give + // up while a perfectly good subscription was in place. + It("sums the replica buffers a bare model ID resolves to", func() { + s.AppendLine("model-a#0", "stderr", "preload-r0") + s.AppendLine("model-a#1", "stderr", "preload-r1") + + _, unsubscribe := s.Subscribe("model-a") + defer unsubscribe() + + Expect(s.SubscriberCount("model-a")).To(Equal(2)) + Expect(s.SubscriberCount("model-a#0")).To(Equal(1)) + Expect(s.SubscriberCount("model-b")).To(Equal(0)) + }) + + It("returns zero for a model that has no buffer at all", func() { + Expect(s.SubscriberCount("never-seen")).To(Equal(0)) + }) + }) + Describe("Subscribe", func() { // Confirms the WebSocket streaming path (the live tail UI) receives // lines from every replica when the caller subscribes by bare modelID. diff --git a/tests/e2e/distributed/backend_logs_test.go b/tests/e2e/distributed/backend_logs_test.go index 79dea3902d01..473dad54f111 100644 --- a/tests/e2e/distributed/backend_logs_test.go +++ b/tests/e2e/distributed/backend_logs_test.go @@ -25,6 +25,21 @@ import ( "gorm.io/gorm/logger" ) +// waitForLogSubscriber blocks until the worker's WebSocket log handler has +// registered its subscription on the store. +// +// The handler writes the "initial" batch first and subscribes only afterwards, +// so a line appended the instant that batch lands is buffered but never +// streamed, and the spec then waits out its full read deadline. Measured at +// roughly one run in seventeen with `--repeat`, which is far too often for CI. +// Waiting on the subscription removes the race from the spec; the handler's own +// snapshot/subscribe window is a separate production question. +func waitForLogSubscriber(logStore *model.BackendLogStore, modelID string) { + GinkgoHelper() + Eventually(func() int { return logStore.SubscriberCount(modelID) }, "10s", "5ms"). + Should(BeNumerically(">", 0), "the WebSocket handler never subscribed to %q", modelID) +} + var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func() { Context("Worker HTTP log endpoints", func() { @@ -212,6 +227,7 @@ var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func Expect(initialLines[1].Text).To(Equal("line-2")) // Now append a new line and verify it arrives via WebSocket + waitForLogSubscriber(logStore, "ws-model") logStore.AppendLine("ws-model", "stdout", "line-3-realtime") conn.SetReadDeadline(time.Now().Add(5 * time.Second)) @@ -280,6 +296,7 @@ var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func Expect(conn.ReadJSON(&initialMsg)).To(Succeed()) // Append line to a different model + waitForLogSubscriber(logStore, "ws-model") logStore.AppendLine("other-model", "stdout", "should not appear") // Append line to our model logStore.AppendLine("ws-model", "stdout", "should appear") @@ -475,6 +492,7 @@ var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func Expect(initialLines[0].Text).To(Equal("initial line from worker")) // Append a new line on the worker's log store + waitForLogSubscriber(logStore, "proxy-model") logStore.AppendLine("proxy-model", "stderr", "realtime via proxy") // Read the streamed line through the proxy From 1dc79985202c5e0fc5b392d56c587f4f6ad1bfb2 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 10:42:49 +0000 Subject: [PATCH 004/153] test(distributed): scope the log-subscriber wait and mark the race it works around MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three corrections from review of the previous commit. The lock-order comment on SubscriberCount claimed no path takes s.mu and a buffer lock together. Subscribe does exactly that, holding s.mu.RLock across replica registrations that take buf.mu. State the rule that is actually true — s.mu precedes any buffer lock, so counting after releasing it preserves the order — and say what follows from it: the total is a sample, not a snapshot. waitForLogSubscriber read as general-purpose but unblocks on the first registered subscription. Subscribe attaches the exact-key buffer and each replica buffer one at a time, so for a replicated model the count goes positive while later replicas are still unattached and the race survives. Rename it waitForSingleLogSubscriber, document that it holds only where Subscribe resolves to one buffer, and assert on exactly 1: misuse then fails loudly on the count rather than going quietly back to being flaky. Taking the expected count as a parameter was the alternative, but that makes callers predict a store-internal number and an under-count fails the same silent way as the original bug. The snapshot-then-subscribe race had no artifact outside a report, and review found a second site carrying it. Mark both handlers identically, including the point that swapping the two calls duplicates rather than drops and so is not the fix. The race itself is left alone; this branch stays test infrastructure. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/localai/backend_logs.go | 7 ++++++ core/services/nodes/file_transfer_server.go | 7 ++++++ pkg/model/backend_log_store.go | 7 ++++-- tests/e2e/distributed/backend_logs_test.go | 24 +++++++++++++++------ 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/core/http/endpoints/localai/backend_logs.go b/core/http/endpoints/localai/backend_logs.go index 6072b8483708..8b5f99d66f78 100644 --- a/core/http/endpoints/localai/backend_logs.go +++ b/core/http/endpoints/localai/backend_logs.go @@ -121,6 +121,13 @@ func BackendLogsWebSocketEndpoint(ml *model.ModelLoader) echo.HandlerFunc { conn := &backendLogsConn{Conn: ws} + // KNOWN RACE: the snapshot is sent before the subscription is registered, so + // a line appended in that window is never streamed. A viewer attaching while + // a model loads (when a backend is at its noisiest) can silently miss lines; + // they stay in the buffer, so a reload shows them. Fixing it needs an atomic + // snapshot-plus-subscribe under the store lock, not a reorder of these two + // calls, which would duplicate instead of drop. + // Send existing lines as initial batch existingLines := ml.BackendLogs().GetLines(modelID) initialMsg := map[string]any{ diff --git a/core/services/nodes/file_transfer_server.go b/core/services/nodes/file_transfer_server.go index 486753b77828..34f2d97582e1 100644 --- a/core/services/nodes/file_transfer_server.go +++ b/core/services/nodes/file_transfer_server.go @@ -1210,6 +1210,13 @@ func handleBackendLogsWS(w http.ResponseWriter, r *http.Request, logStore *model conn := &backendLogsWSConn{Conn: ws} + // KNOWN RACE: the snapshot is sent before the subscription is registered, so + // a line appended in that window is never streamed. A viewer attaching while + // a model loads (when a backend is at its noisiest) can silently miss lines; + // they stay in the buffer, so a reload shows them. Fixing it needs an atomic + // snapshot-plus-subscribe under the store lock, not a reorder of these two + // calls, which would duplicate instead of drop. + // Send existing lines as initial batch existingLines := logStore.GetLines(modelID) initialMsg := map[string]any{ diff --git a/pkg/model/backend_log_store.go b/pkg/model/backend_log_store.go index 30268fde47de..3c60f34a3736 100644 --- a/pkg/model/backend_log_store.go +++ b/pkg/model/backend_log_store.go @@ -366,8 +366,11 @@ func (s *BackendLogStore) SubscriberCount(modelID string) int { } s.mu.RUnlock() - // Counted after releasing s.mu: no other path takes s.mu and a buffer lock - // together, and keeping it that way costs nothing here. + // Lock order in this type is always s.mu before any buffer lock — Subscribe + // holds s.mu.RLock across its replica registrations, which take buf.mu — so + // counting after releasing s.mu keeps that order rather than inverting it. + // The total is therefore a sample, not a snapshot: a concurrent Subscribe + // can register a further buffer while this loop runs. count := func(buf *backendLogBuffer) int { buf.mu.Lock() defer buf.mu.Unlock() diff --git a/tests/e2e/distributed/backend_logs_test.go b/tests/e2e/distributed/backend_logs_test.go index 473dad54f111..82e8ac156401 100644 --- a/tests/e2e/distributed/backend_logs_test.go +++ b/tests/e2e/distributed/backend_logs_test.go @@ -25,7 +25,7 @@ import ( "gorm.io/gorm/logger" ) -// waitForLogSubscriber blocks until the worker's WebSocket log handler has +// waitForSingleLogSubscriber blocks until the worker's WebSocket log handler has // registered its subscription on the store. // // The handler writes the "initial" batch first and subscribes only afterwards, @@ -33,11 +33,21 @@ import ( // streamed, and the spec then waits out its full read deadline. Measured at // roughly one run in seventeen with `--repeat`, which is far too often for CI. // Waiting on the subscription removes the race from the spec; the handler's own -// snapshot/subscribe window is a separate production question. -func waitForLogSubscriber(logStore *model.BackendLogStore, modelID string) { +// snapshot/subscribe window is a separate production question, marked at both +// production sites. +// +// Only valid where BackendLogStore.Subscribe resolves modelID to exactly ONE +// buffer: a bare model ID with no "#N" replica buffers in the store, or +// a full process key. Subscribe registers the exact-key buffer and each replica +// buffer one at a time, so for a model that does have replicas the count goes +// positive while later replicas are still unattached and the race survives. +// Hence the assertion is on exactly 1 rather than "at least 1": a spec that +// misapplies this to a replicated model fails loudly on the count instead of +// going quietly back to being flaky. +func waitForSingleLogSubscriber(logStore *model.BackendLogStore, modelID string) { GinkgoHelper() Eventually(func() int { return logStore.SubscriberCount(modelID) }, "10s", "5ms"). - Should(BeNumerically(">", 0), "the WebSocket handler never subscribed to %q", modelID) + Should(Equal(1), "the WebSocket handler never subscribed to %q exactly once", modelID) } var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func() { @@ -227,7 +237,7 @@ var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func Expect(initialLines[1].Text).To(Equal("line-2")) // Now append a new line and verify it arrives via WebSocket - waitForLogSubscriber(logStore, "ws-model") + waitForSingleLogSubscriber(logStore, "ws-model") logStore.AppendLine("ws-model", "stdout", "line-3-realtime") conn.SetReadDeadline(time.Now().Add(5 * time.Second)) @@ -296,7 +306,7 @@ var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func Expect(conn.ReadJSON(&initialMsg)).To(Succeed()) // Append line to a different model - waitForLogSubscriber(logStore, "ws-model") + waitForSingleLogSubscriber(logStore, "ws-model") logStore.AppendLine("other-model", "stdout", "should not appear") // Append line to our model logStore.AppendLine("ws-model", "stdout", "should appear") @@ -492,7 +502,7 @@ var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func Expect(initialLines[0].Text).To(Equal("initial line from worker")) // Append a new line on the worker's log store - waitForLogSubscriber(logStore, "proxy-model") + waitForSingleLogSubscriber(logStore, "proxy-model") logStore.AppendLine("proxy-model", "stderr", "realtime via proxy") // Read the streamed line through the proxy From 05bc51908b0a0c1654930c763c402e18b177c87c Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 10:49:54 +0000 Subject: [PATCH 005/153] ci(distributed): run the distributed e2e suite on PRs The suite has never run in CI, so 239 specs across 32 files were verified only by hand. Path-filtered to distributed code, advisory until it earns a track record, and with flake retries at 1 rather than 5 so nondeterminism surfaces instead of being retried away. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .github/workflows/tests-e2e-distributed.yml | 73 +++++++++++++++++++++ Makefile | 8 ++- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tests-e2e-distributed.yml diff --git a/.github/workflows/tests-e2e-distributed.yml b/.github/workflows/tests-e2e-distributed.yml new file mode 100644 index 000000000000..4afc76dc05e1 --- /dev/null +++ b/.github/workflows/tests-e2e-distributed.yml @@ -0,0 +1,73 @@ +--- +name: 'E2E Distributed Tests' + +on: + pull_request: + paths: + - 'core/services/nodes/**' + - 'core/services/worker/**' + - 'core/services/messaging/**' + - 'core/services/syncstate/**' + - 'core/services/jobs/**' + - 'core/services/agents/**' + - 'core/services/agentpool/**' + - 'core/services/galleryop/**' + - 'core/http/routes/nodes.go' + - 'core/http/endpoints/localai/nodes.go' + - 'core/http/endpoints/openresponses/**' + - 'core/application/distributed.go' + - 'core/config/distributed_config.go' + - 'pkg/natsauth/**' + - 'tests/e2e/distributed/**' + - '.github/workflows/tests-e2e-distributed.yml' + - 'Makefile' + push: + branches: + - master + +concurrency: + group: ci-tests-e2e-distributed-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + tests-e2e-distributed: + runs-on: ubuntu-latest + # Advisory while the suite builds a track record. Flip to a required check + # only after it has run clean for two weeks; a heavy suite made required on + # day one gets disabled instead of fixed. + continue-on-error: true + timeout-minutes: 45 + steps: + - name: Clone + uses: actions/checkout@v7 + with: + submodules: true + - name: Configure apt mirror on runner + uses: ./.github/actions/configure-apt-mirror + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.25.x' + cache: false + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential libopus-dev + - name: Proto Dependencies + run: | + curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \ + unzip -j -d /usr/local/bin protoc.zip bin/protoc && \ + rm protoc.zip + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + PATH="$PATH:$HOME/go/bin" make protogen-go + - name: Pre-pull test images + # Pulling here rather than inside the suite keeps container-start timing + # out of the spec timeouts and makes a registry outage read as a + # setup failure instead of a test failure. + run: | + docker pull postgres:16-alpine + docker pull nats:2-alpine + - name: Distributed E2E + run: | + PATH="$PATH:$HOME/go/bin" make test-e2e-distributed diff --git a/Makefile b/Makefile index e39c5dfa008b..dd878a3e3ea3 100644 --- a/Makefile +++ b/Makefile @@ -342,12 +342,18 @@ run-e2e-aio: protogen-go @echo 'Running e2e AIO tests' $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e-aio +# Flake retries for the distributed suite. Defaults to 1, unlike TEST_FLAKES: +# this suite exists to catch nondeterministic cluster behaviour, and retrying +# hides exactly the failures it is meant to surface. Raise it locally if you are +# bisecting something unrelated. +DISTRIBUTED_TEST_FLAKES?=1 + # Distributed architecture e2e (PostgreSQL + NATS via testcontainers). # Includes NatsJWT specs (JWT-enabled NATS). Requires Docker. # VLLMMultinode is excluded here; use test-e2e-vllm-multinode for that. test-e2e-distributed: protogen-go @echo 'Running distributed e2e tests (label Distributed, incl. NatsJWT)' - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e/distributed + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode && !Cluster' --flake-attempts $(DISTRIBUTED_TEST_FLAKES) --timeout=40m -v -r ./tests/e2e/distributed # vLLM multi-node DP smoke (CPU). Builds local-ai:tests and the # cpu-vllm backend from the current working tree, then drives a From f5e90f205b5b881249d931eed5359142daf9ad37 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 11:02:13 +0000 Subject: [PATCH 006/153] ci(distributed): widen the trigger and drop the mid-suite image pull The path allowlist covered 13 of the 99 packages the suite reaches. Commit 1dc3aeef8 touched core/config, core/services/modeladmin and core/backend and matched no entry, so it would have merged without running the very specs that cover it. Use the paths-ignore denylist tests-e2e.yml already uses. Disable the testcontainers reaper: the runner is ephemeral, so the reaper buys nothing and its unpinned image was pulled mid-suite, defeating the pre-pull. Drop continue-on-error, which no other workflow uses and which reports a failed run as green. The job is advisory by staying out of branch protection instead. Pin Go to 1.26.0 to match go.mod, and add the tmate-on-failure step. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .github/workflows/tests-e2e-distributed.yml | 56 ++++++++++++--------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/.github/workflows/tests-e2e-distributed.yml b/.github/workflows/tests-e2e-distributed.yml index 4afc76dc05e1..4b41f72b5014 100644 --- a/.github/workflows/tests-e2e-distributed.yml +++ b/.github/workflows/tests-e2e-distributed.yml @@ -3,24 +3,15 @@ name: 'E2E Distributed Tests' on: pull_request: - paths: - - 'core/services/nodes/**' - - 'core/services/worker/**' - - 'core/services/messaging/**' - - 'core/services/syncstate/**' - - 'core/services/jobs/**' - - 'core/services/agents/**' - - 'core/services/agentpool/**' - - 'core/services/galleryop/**' - - 'core/http/routes/nodes.go' - - 'core/http/endpoints/localai/nodes.go' - - 'core/http/endpoints/openresponses/**' - - 'core/application/distributed.go' - - 'core/config/distributed_config.go' - - 'pkg/natsauth/**' - - 'tests/e2e/distributed/**' - - '.github/workflows/tests-e2e-distributed.yml' - - 'Makefile' + # The suite's dependency graph is 99 packages, so an allowlist of paths + # silently stops guarding the moment code moves. At ~75s the job is cheap + # enough to run unless the diff is confined to paths it provably cannot + # reach. See .agents/ci-caching.md. + paths-ignore: + - 'gallery/**' + - 'docs/**' + - 'examples/**' + - '**/*.md' push: branches: - master @@ -32,10 +23,11 @@ concurrency: jobs: tests-e2e-distributed: runs-on: ubuntu-latest - # Advisory while the suite builds a track record. Flip to a required check - # only after it has run clean for two weeks; a heavy suite made required on - # day one gets disabled instead of fixed. - continue-on-error: true + # Advisory because it is deliberately not in branch protection, so a failure + # is a visible red X rather than a blocked merge. Promoting it to a required + # check is a repository-settings change, to be made once it has a track + # record; a heavy suite made required on day one gets disabled instead of + # fixed. timeout-minutes: 45 steps: - name: Clone @@ -47,7 +39,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '1.25.x' + go-version: '1.26.0' cache: false - name: Dependencies run: | @@ -64,10 +56,26 @@ jobs: - name: Pre-pull test images # Pulling here rather than inside the suite keeps container-start timing # out of the spec timeouts and makes a registry outage read as a - # setup failure instead of a test failure. + # setup failure instead of a test failure. These two are the only images + # the suite needs once the testcontainers reaper is disabled below. run: | docker pull postgres:16-alpine docker pull nats:2-alpine - name: Distributed E2E + # TESTCONTAINERS_RYUK_DISABLED keeps the pre-pull above meaningful. The + # reaper exists to clean up leaked containers on a long-lived host, but + # this runner is ephemeral and every container dies with the VM. Leaving + # it enabled would pull a third, unpinned image (testcontainers/ryuk) + # from Docker Hub mid-suite: exactly the registry dependency the + # pre-pull step exists to remove. + env: + TESTCONTAINERS_RYUK_DISABLED: "true" run: | PATH="$PATH:$HOME/go/bin" make test-e2e-distributed + - name: Setup tmate session if tests fail + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3.23 + with: + detached: true + connect-timeout-seconds: 180 + limit-access-to-actor: true From 8533a10082e0815ffdda126d6833088103255d8d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 11:10:04 +0000 Subject: [PATCH 007/153] test(distributed): add a process-level cluster harness Runs local-ai as real child processes, one per frontend replica and one per worker, against containerised infrastructure. The in-process suites cannot express frontend-replica failure: there is no process to kill and no real HTTP boundary between a worker and the frontend it registered with. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/cluster/cluster.go | 329 ++++++++++++++++++ .../distributed/cluster/cluster_suite_test.go | 13 + tests/e2e/distributed/cluster/cluster_test.go | 37 ++ 3 files changed, 379 insertions(+) create mode 100644 tests/e2e/distributed/cluster/cluster.go create mode 100644 tests/e2e/distributed/cluster/cluster_suite_test.go create mode 100644 tests/e2e/distributed/cluster/cluster_test.go diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go new file mode 100644 index 000000000000..306980e00d2b --- /dev/null +++ b/tests/e2e/distributed/cluster/cluster.go @@ -0,0 +1,329 @@ +// Package cluster runs LocalAI as real child processes for end-to-end tests. +// +// The in-process suites cannot express frontend-replica failure: there is no +// process to kill, no second replica to race, and no real HTTP boundary between +// a worker and the frontend it registered with. This package starts the same +// binary an operator runs, one process per frontend replica and one per worker, +// against containerised Postgres and NATS. +package cluster + +import ( + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/mudler/LocalAI/pkg/httpclient" + + "github.com/phayes/freeport" +) + +// Options configures a cluster. Every field without a default is required. +type Options struct { + // Binary is the path to a built local-ai. + Binary string + // MockBackend is the path to the mock-backend binary. When set it is copied + // into each worker's backends directory as "mock-backend", which is the name + // model YAML refers to (see tests/e2e/e2e_suite_test.go:75). + MockBackend string + // PGDSN and NatsURL point at infrastructure the caller already started. + PGDSN string + NatsURL string + // LogDir receives one file per process. Never empty: a cluster failure is + // unreadable without them. + LogDir string + + RegistrationToken string // default "e2e-token" + AdminEmail string // default "admin@e2e.local" + + Frontends int + Workers int +} + +// Process is one running local-ai. +type Process struct { + Name string + Cmd *exec.Cmd + Port int + LogPath string + + logFile *os.File + // exited closes once the reaper has collected the process. Only the reaper + // calls Cmd.Wait, so nothing else may: a second Wait on the same Cmd races + // the first and corrupts ProcessState. + exited chan struct{} + waitErr error +} + +// Cluster is a running set of frontend and worker processes. +type Cluster struct { + opts Options + frontends []*Process + workers []*Process + baseDir string +} + +const ( + defaultRegistrationToken = "e2e-token" + defaultAdminEmail = "admin@e2e.local" + readinessTimeout = 90 * time.Second + readinessPoll = 200 * time.Millisecond +) + +func (o *Options) applyDefaults() { + if o.RegistrationToken == "" { + o.RegistrationToken = defaultRegistrationToken + } + if o.AdminEmail == "" { + o.AdminEmail = defaultAdminEmail + } +} + +func (o Options) validate() error { + if o.Frontends < 1 { + return fmt.Errorf("cluster needs at least one frontend, got %d", o.Frontends) + } + if o.LogDir == "" { + return fmt.Errorf("cluster needs a LogDir: process logs are the only way to read a cluster failure") + } + if st, err := os.Stat(o.Binary); err != nil || st.IsDir() { + return fmt.Errorf("local-ai binary not found at %q (run: make build)", o.Binary) + } + return nil +} + +// Start brings up the cluster and blocks until every frontend answers /readyz +// and every worker has registered. +func Start(opts Options) (*Cluster, error) { + opts.applyDefaults() + if err := opts.validate(); err != nil { + return nil, err + } + + baseDir, err := os.MkdirTemp("", "localai-cluster-*") + if err != nil { + return nil, fmt.Errorf("creating cluster work dir: %w", err) + } + + c := &Cluster{opts: opts, baseDir: baseDir} + + for i := 0; i < opts.Frontends; i++ { + p, err := c.startFrontend(i) + if err != nil { + c.Stop() + return nil, err + } + c.frontends = append(c.frontends, p) + } + for i := 0; i < opts.Workers; i++ { + p, err := c.startWorker(i) + if err != nil { + c.Stop() + return nil, err + } + c.workers = append(c.workers, p) + } + return c, nil +} + +func (c *Cluster) startFrontend(i int) (*Process, error) { + port, err := freeport.GetFreePort() + if err != nil { + return nil, fmt.Errorf("allocating frontend port: %w", err) + } + name := fmt.Sprintf("frontend-%d", i) + dir := filepath.Join(c.baseDir, name) + if err := os.MkdirAll(filepath.Join(dir, "models"), 0o755); err != nil { + return nil, fmt.Errorf("creating %s dirs: %w", name, err) + } + if err := os.MkdirAll(filepath.Join(dir, "backends"), 0o755); err != nil { + return nil, fmt.Errorf("creating %s dirs: %w", name, err) + } + + cmd := exec.Command(c.opts.Binary, "run", + "--address", fmt.Sprintf("127.0.0.1:%d", port), + "--models-path", filepath.Join(dir, "models"), + "--backends-path", filepath.Join(dir, "backends"), + ) + // Cmd.Environ() is the parent environment this Cmd would already run with; + // the children need PATH, HOME and the Go/CI environment intact. + cmd.Env = append(cmd.Environ(), + "LOCALAI_DISTRIBUTED=true", + "LOCALAI_NATS_URL="+c.opts.NatsURL, + "LOCALAI_AUTH=true", + "LOCALAI_AUTH_DATABASE_URL="+c.opts.PGDSN, + "LOCALAI_ADMIN_EMAIL="+c.opts.AdminEmail, + "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, + "LOCALAI_AUTO_APPROVE_NODES=true", + "DEBUG=true", + ) + + p, err := c.spawn(name, cmd, port) + if err != nil { + return nil, err + } + if err := waitReady(p, fmt.Sprintf("http://127.0.0.1:%d/readyz", port)); err != nil { + // The caller never sees this process, so Stop() will never reach it: + // reap it here or it outlives the suite holding a port and a log handle. + p.terminate() + return nil, fmt.Errorf("%s never became ready (see %s): %w", name, p.LogPath, err) + } + return p, nil +} + +func (c *Cluster) startWorker(i int) (*Process, error) { + // Two independent free ports: the worker's file-transfer server defaults to + // basePort-1, which freeport never reserved and which is basePort of another + // worker whenever two allocations land adjacent. + ports, err := freeport.GetFreePorts(2) + if err != nil { + return nil, fmt.Errorf("allocating worker ports: %w", err) + } + grpcPort, httpPort := ports[0], ports[1] + name := fmt.Sprintf("worker-%d", i) + dir := filepath.Join(c.baseDir, name) + backends := filepath.Join(dir, "backends") + if err := os.MkdirAll(filepath.Join(dir, "models"), 0o755); err != nil { + return nil, fmt.Errorf("creating %s dirs: %w", name, err) + } + if err := os.MkdirAll(backends, 0o755); err != nil { + return nil, fmt.Errorf("creating %s dirs: %w", name, err) + } + if c.opts.MockBackend != "" { + if err := copyExecutable(c.opts.MockBackend, filepath.Join(backends, "mock-backend")); err != nil { + return nil, fmt.Errorf("installing mock backend for %s: %w", name, err) + } + } + + cmd := exec.Command(c.opts.Binary, "worker", + "--models-path", filepath.Join(dir, "models"), + "--backends-path", backends, + ) + cmd.Env = append(cmd.Environ(), + fmt.Sprintf("LOCALAI_SERVE_ADDR=127.0.0.1:%d", grpcPort), + fmt.Sprintf("LOCALAI_ADVERTISE_ADDR=127.0.0.1:%d", grpcPort), + fmt.Sprintf("LOCALAI_HTTP_ADDR=127.0.0.1:%d", httpPort), + fmt.Sprintf("LOCALAI_ADVERTISE_HTTP_ADDR=127.0.0.1:%d", httpPort), + "LOCALAI_REGISTER_TO="+c.FrontendURL(0), + "LOCALAI_NODE_NAME="+name, + "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, + "LOCALAI_NATS_URL="+c.opts.NatsURL, + "DEBUG=true", + ) + + return c.spawn(name, cmd, grpcPort) +} + +func (c *Cluster) spawn(name string, cmd *exec.Cmd, port int) (*Process, error) { + logPath := filepath.Join(c.opts.LogDir, name+".log") + f, err := os.Create(logPath) + if err != nil { + return nil, fmt.Errorf("creating log file for %s: %w", name, err) + } + cmd.Stdout = f + cmd.Stderr = f + if err := cmd.Start(); err != nil { + _ = f.Close() + return nil, fmt.Errorf("starting %s: %w", name, err) + } + p := &Process{Name: name, Cmd: cmd, Port: port, LogPath: logPath, logFile: f, exited: make(chan struct{})} + // One reaper per process, joined by terminate(): a child that dies on its own + // is collected immediately, so waiters learn about it instead of polling a + // dead port until the readiness timeout. + go func() { + p.waitErr = cmd.Wait() + close(p.exited) + }() + return p, nil +} + +// terminate kills the process, waits for the reaper, and releases the log +// handle. Safe to call more than once and on a process that already exited. +func (p *Process) terminate() { + if p == nil || p.Cmd == nil || p.Cmd.Process == nil { + return + } + _ = p.Cmd.Process.Kill() + <-p.exited + if p.logFile != nil { + _ = p.logFile.Close() + } +} + +// FrontendURL is the base URL of frontend i. +func (c *Cluster) FrontendURL(i int) string { + return fmt.Sprintf("http://127.0.0.1:%d", c.frontends[i].Port) +} + +// WorkerName is the node name worker i registered under. +func (c *Cluster) WorkerName(i int) string { + return c.workers[i].Name +} + +// Stop terminates every process and removes the work directory. Logs survive in +// LogDir, which the caller owns. +func (c *Cluster) Stop() { + for _, p := range append(append([]*Process{}, c.workers...), c.frontends...) { + p.terminate() + } + if c.baseDir != "" { + _ = os.RemoveAll(c.baseDir) + } +} + +// DumpLogs writes every process log to stdout. Call from an AfterEach guarded by +// CurrentSpecReport().Failed(). +func (c *Cluster) DumpLogs() { + for _, p := range append(append([]*Process{}, c.frontends...), c.workers...) { + if p == nil { + continue + } + data, err := os.ReadFile(p.LogPath) + if err != nil { + fmt.Printf("=== %s: log unreadable: %v\n", p.Name, err) + continue + } + fmt.Printf("=== %s (%s) ===\n%s\n", p.Name, p.LogPath, string(data)) + } +} + +func waitReady(p *Process, url string) error { + deadline := time.Now().Add(readinessTimeout) + client := httpclient.NewWithTimeout(2 * time.Second) + var last error + for time.Now().Before(deadline) { + select { + case <-p.exited: + if p.waitErr == nil { + return fmt.Errorf("process exited cleanly before becoming ready") + } + return fmt.Errorf("process exited before becoming ready: %w", p.waitErr) + default: + } + resp, err := client.Get(url) + if err == nil { + _ = resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return nil + } + last = fmt.Errorf("status %d", resp.StatusCode) + } else { + last = err + } + time.Sleep(readinessPoll) + } + return fmt.Errorf("not ready within %s: %w", readinessTimeout, last) +} + +func copyExecutable(src, dst string) error { + data, err := os.ReadFile(src) + if err != nil { + return fmt.Errorf("reading %s: %w", src, err) + } + if err := os.WriteFile(dst, data, 0o755); err != nil { + return fmt.Errorf("writing %s: %w", dst, err) + } + return nil +} diff --git a/tests/e2e/distributed/cluster/cluster_suite_test.go b/tests/e2e/distributed/cluster/cluster_suite_test.go new file mode 100644 index 000000000000..69f785ba4cad --- /dev/null +++ b/tests/e2e/distributed/cluster/cluster_suite_test.go @@ -0,0 +1,13 @@ +package cluster_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestCluster(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Cluster Harness Suite") +} diff --git a/tests/e2e/distributed/cluster/cluster_test.go b/tests/e2e/distributed/cluster/cluster_test.go new file mode 100644 index 000000000000..4d48f1ecb246 --- /dev/null +++ b/tests/e2e/distributed/cluster/cluster_test.go @@ -0,0 +1,37 @@ +package cluster_test + +import ( + "os" + "path/filepath" + + "github.com/mudler/LocalAI/tests/e2e/distributed/cluster" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Cluster options", Label("Distributed"), func() { + It("rejects a binary path that does not exist", func() { + _, err := cluster.Start(cluster.Options{ + Binary: filepath.Join(os.TempDir(), "definitely-not-local-ai"), + PGDSN: "postgres://test:test@127.0.0.1:5432/x?sslmode=disable", + NatsURL: "nats://127.0.0.1:4222", + LogDir: GinkgoT().TempDir(), + Frontends: 1, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("local-ai binary")) + }) + + It("rejects a cluster with no frontends", func() { + _, err := cluster.Start(cluster.Options{ + Binary: "/bin/true", + PGDSN: "postgres://test:test@127.0.0.1:5432/x?sslmode=disable", + NatsURL: "nats://127.0.0.1:4222", + LogDir: GinkgoT().TempDir(), + Frontends: 0, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("at least one frontend")) + }) +}) From 0cf3981800e7f03171ee6ebe7d5f9453a24d0199 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 11:18:49 +0000 Subject: [PATCH 008/153] test(distributed): make the cluster harness survive a restart Restarting a frontend replica must not move it: workers read LOCALAI_REGISTER_TO once at boot and never re-resolve it, so a replica that returns on a fresh port is unreachable by the workers that registered with it. startFrontend now takes the port, with <= 0 meaning "allocate". Process logs are opened for append rather than truncated, so a restarted process cannot erase the log of the instance that died, which is the log a failover post-mortem needs. The post-SIGKILL wait is bounded, so one stuck child no longer becomes a suite-wide timeout that names nothing. Stop is nil-safe because Start returns a nil cluster after stopping itself. Start's doc comment no longer claims to wait for worker registration; that needs an authenticated admin session, so it now says callers must poll /api/nodes themselves. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/cluster/cluster.go | 47 +++++++++++++++++++----- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 306980e00d2b..0445fe3868e9 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -44,7 +44,10 @@ type Options struct { // Process is one running local-ai. type Process struct { - Name string + Name string + // Cmd is exposed for signalling only. Never call Cmd.Wait on it: the reaper + // goroutine started in spawn owns it, a second Wait races the first, and + // waitErr is only safe to read after <-p.exited. Cmd *exec.Cmd Port int LogPath string @@ -70,6 +73,10 @@ const ( defaultAdminEmail = "admin@e2e.local" readinessTimeout = 90 * time.Second readinessPoll = 200 * time.Millisecond + // processExitTimeout bounds the post-SIGKILL wait in terminate. An unbounded + // wait turns one stuck child (D state, or a Wait that never returns) into a + // suite-wide Ginkgo timeout that names nothing. + processExitTimeout = 10 * time.Second ) func (o *Options) applyDefaults() { @@ -94,8 +101,10 @@ func (o Options) validate() error { return nil } -// Start brings up the cluster and blocks until every frontend answers /readyz -// and every worker has registered. +// Start brings up the cluster. It blocks until every frontend answers /readyz +// and every worker process has been spawned. It does NOT wait for workers to +// register: that needs an authenticated admin session, so a caller that depends +// on registration must poll /api/nodes itself. func Start(opts Options) (*Cluster, error) { opts.applyDefaults() if err := opts.validate(); err != nil { @@ -110,7 +119,7 @@ func Start(opts Options) (*Cluster, error) { c := &Cluster{opts: opts, baseDir: baseDir} for i := 0; i < opts.Frontends; i++ { - p, err := c.startFrontend(i) + p, err := c.startFrontend(i, 0) if err != nil { c.Stop() return nil, err @@ -128,10 +137,17 @@ func Start(opts Options) (*Cluster, error) { return c, nil } -func (c *Cluster) startFrontend(i int) (*Process, error) { - port, err := freeport.GetFreePort() - if err != nil { - return nil, fmt.Errorf("allocating frontend port: %w", err) +// startFrontend starts frontend i. A port <= 0 allocates a fresh one; a pinned +// port exists for restart: workers take LOCALAI_REGISTER_TO once at boot and +// never re-resolve it, so a replica that comes back on a new port is +// unreachable by exactly the workers that registered with it. +func (c *Cluster) startFrontend(i int, port int) (*Process, error) { + if port <= 0 { + allocated, err := freeport.GetFreePort() + if err != nil { + return nil, fmt.Errorf("allocating frontend port: %w", err) + } + port = allocated } name := fmt.Sprintf("frontend-%d", i) dir := filepath.Join(c.baseDir, name) @@ -218,7 +234,9 @@ func (c *Cluster) startWorker(i int) (*Process, error) { func (c *Cluster) spawn(name string, cmd *exec.Cmd, port int) (*Process, error) { logPath := filepath.Join(c.opts.LogDir, name+".log") - f, err := os.Create(logPath) + // Append rather than truncate: a restarted process reopens the same path, and + // the log of the instance that died is the one a failover post-mortem needs. + f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) if err != nil { return nil, fmt.Errorf("creating log file for %s: %w", name, err) } @@ -246,7 +264,11 @@ func (p *Process) terminate() { return } _ = p.Cmd.Process.Kill() - <-p.exited + select { + case <-p.exited: + case <-time.After(processExitTimeout): + fmt.Printf("warning: %s did not exit within %s after SIGKILL; continuing teardown\n", p.Name, processExitTimeout) + } if p.logFile != nil { _ = p.logFile.Close() } @@ -265,6 +287,11 @@ func (c *Cluster) WorkerName(i int) string { // Stop terminates every process and removes the work directory. Logs survive in // LogDir, which the caller owns. func (c *Cluster) Stop() { + // Start returns (nil, err) after stopping itself, so a spec that defers + // c.Stop before asserting the error would otherwise nil-deref. + if c == nil { + return + } for _, p := range append(append([]*Process{}, c.workers...), c.frontends...) { p.terminate() } From 84ac07df7d8fb110871777686726a7116433b425 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 11:25:47 +0000 Subject: [PATCH 009/153] test(distributed): add admin session helper to the cluster harness The register handler answers 201 both for "user created, here is your session" and for "this email already exists", so the status code cannot tell a fresh registration from a repeat one. Key on the session cookie instead and fall through to login when it is absent. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/cluster/admin.go | 168 ++++++++++++++++++ tests/e2e/distributed/cluster/cluster_test.go | 22 +++ 2 files changed, 190 insertions(+) create mode 100644 tests/e2e/distributed/cluster/admin.go diff --git a/tests/e2e/distributed/cluster/admin.go b/tests/e2e/distributed/cluster/admin.go new file mode 100644 index 000000000000..b1371f532271 --- /dev/null +++ b/tests/e2e/distributed/cluster/admin.go @@ -0,0 +1,168 @@ +package cluster + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/cookiejar" + "net/url" + "time" + + "github.com/mudler/LocalAI/pkg/httpclient" +) + +const ( + // adminPassword must satisfy core/http/auth's policy (>= 12 chars and a + // zxcvbn score of 3 against hints that include "admin" and "localai"). + // This one scores 3 today; acknowledgeWeakPassword is sent alongside it so + // a future tightening of the policy cannot silently break every failover + // spec at setup time. + adminPassword = "e2e-admin-password" + // sessionCookieName mirrors the unexported constant in core/http/auth. + // The register handler returns 201 both for "user created, here is your + // session" and for "this email already exists" (a deliberate account + // enumeration defence), so the status code alone cannot tell the two + // apart: the presence of this cookie is the only reliable signal. + sessionCookieName = "session" + // authRequestTimeout bounds one register/login round trip. + authRequestTimeout = 30 * time.Second + // bodyExcerptLimit caps how much of an error response is quoted back. + bodyExcerptLimit = 512 +) + +// ForTestingEmpty returns a Cluster with no processes. It exists so the package's +// own argument-validation specs do not need to start anything. +func ForTestingEmpty() *Cluster { + return &Cluster{} +} + +// AdminSession registers the admin user on frontend i and returns a client +// carrying the resulting session cookie. The email matches LOCALAI_ADMIN_EMAIL, +// which core/http/auth exempts from the approval gate and assigns the admin +// role, so registration alone yields an active admin session. +// +// Call this ONCE per cluster and share the client. Two reasons: the auth +// endpoints are rate limited to 5 requests per minute per client IP, and every +// e2e request arrives from 127.0.0.1; and the returned client is already good +// for every frontend, because sessions live in the shared Postgres auth DB and +// Go's cookie jar keys cookies by host without the port. +func (c *Cluster) AdminSession(i int) (*http.Client, error) { + base, err := c.frontendBaseURL(i) + if err != nil { + return nil, err + } + + jar, err := cookiejar.New(nil) + if err != nil { + return nil, fmt.Errorf("creating cookie jar: %w", err) + } + // httpclient hardens the transport and refuses redirects; the jar is the one + // thing it does not configure, and a session cookie is the whole point here. + client := httpclient.NewWithTimeout(authRequestTimeout) + client.Jar = jar + + credentials := map[string]any{ + "email": c.opts.AdminEmail, + "password": adminPassword, + } + registration := map[string]any{ + "email": c.opts.AdminEmail, + "password": adminPassword, + "name": "E2E Admin", + "acknowledge_weak_password": true, + } + + registerStatus, registerBody, err := postJSON(client, base+"/api/auth/register", registration) + if err != nil { + return nil, fmt.Errorf("registering admin on frontend %d: %w", i, err) + } + if hasSessionCookie(jar, base) { + return client, nil + } + + // No cookie means the user already existed (a repeat call against the same + // Postgres), or registration was rejected. Log in; on failure the + // registration response is the diagnosis, so carry it into the error. + loginStatus, loginBody, err := postJSON(client, base+"/api/auth/login", credentials) + if err != nil { + return nil, fmt.Errorf("logging in admin on frontend %d: %w", i, err) + } + if loginStatus != http.StatusOK { + return nil, fmt.Errorf( + "admin login on frontend %d returned %d (%s); registration had returned %d (%s)", + i, loginStatus, loginBody, registerStatus, registerBody) + } + if !hasSessionCookie(jar, base) { + return nil, fmt.Errorf("admin login on frontend %d returned 200 but set no %q cookie: %s", i, sessionCookieName, loginBody) + } + return client, nil +} + +// GetJSON performs an authenticated GET against a frontend and decodes the body. +func (c *Cluster) GetJSON(client *http.Client, frontend int, path string, out any) error { + base, err := c.frontendBaseURL(frontend) + if err != nil { + return err + } + resp, err := client.Get(base + path) + if err != nil { + return fmt.Errorf("GET %s on frontend %d: %w", path, frontend, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("GET %s on frontend %d returned %d: %s", path, frontend, resp.StatusCode, excerpt(resp.Body)) + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("decoding %s from frontend %d: %w", path, frontend, err) + } + return nil +} + +// frontendBaseURL validates the index before FrontendURL indexes the slice: a +// bare index panic in a helper every failover spec calls is far harder to read +// than a named error. +func (c *Cluster) frontendBaseURL(i int) (string, error) { + if i < 0 || i >= len(c.frontends) { + return "", fmt.Errorf("frontend %d out of range (cluster has %d)", i, len(c.frontends)) + } + return c.FrontendURL(i), nil +} + +// postJSON sends body as JSON and returns the status plus an excerpt of the +// response, closing the body in every path. +func postJSON(client *http.Client, endpoint string, body any) (int, string, error) { + encoded, err := json.Marshal(body) + if err != nil { + return 0, "", fmt.Errorf("marshalling request body: %w", err) + } + resp, err := client.Post(endpoint, "application/json", bytes.NewReader(encoded)) + if err != nil { + return 0, "", err + } + defer func() { _ = resp.Body.Close() }() + return resp.StatusCode, excerpt(resp.Body), nil +} + +// hasSessionCookie reports whether the jar holds a usable session for base. +func hasSessionCookie(jar *cookiejar.Jar, base string) bool { + u, err := url.Parse(base) + if err != nil { + return false + } + for _, cookie := range jar.Cookies(u) { + if cookie.Name == sessionCookieName && cookie.Value != "" { + return true + } + } + return false +} + +func excerpt(r io.Reader) string { + data, err := io.ReadAll(io.LimitReader(r, bodyExcerptLimit)) + if err != nil { + return fmt.Sprintf("", err) + } + return string(bytes.TrimSpace(data)) +} diff --git a/tests/e2e/distributed/cluster/cluster_test.go b/tests/e2e/distributed/cluster/cluster_test.go index 4d48f1ecb246..807c275c1c46 100644 --- a/tests/e2e/distributed/cluster/cluster_test.go +++ b/tests/e2e/distributed/cluster/cluster_test.go @@ -3,7 +3,9 @@ package cluster_test import ( "os" "path/filepath" + "time" + "github.com/mudler/LocalAI/pkg/httpclient" "github.com/mudler/LocalAI/tests/e2e/distributed/cluster" . "github.com/onsi/ginkgo/v2" @@ -35,3 +37,23 @@ var _ = Describe("Cluster options", Label("Distributed"), func() { Expect(err.Error()).To(ContainSubstring("at least one frontend")) }) }) + +// The HTTP flow inside AdminSession and GetJSON cannot run here: it needs a +// built local-ai plus real Postgres and NATS, which arrive with the failover +// suites. These specs cover the argument validation that would otherwise panic +// on an out-of-range slice index inside a helper every later spec calls. +var _ = Describe("Admin session", Label("Distributed"), func() { + It("reports a clear error when the frontend index is out of range", func() { + c := cluster.ForTestingEmpty() + _, err := c.AdminSession(3) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("frontend 3")) + }) + + It("reports a clear error when GetJSON names a frontend that does not exist", func() { + c := cluster.ForTestingEmpty() + err := c.GetJSON(httpclient.NewWithTimeout(time.Second), 1, "/api/nodes", &struct{}{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("frontend 1")) + }) +}) From 1d746756751cef6d58e64a5517750a890fef86e7 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 14:03:58 +0000 Subject: [PATCH 010/153] test(distributed): give each frontend its own data dir and one pinned secret Session rows are keyed by an HMAC of the token under a secret generated per instance into {DataPath}/.hmac_secret. The replicas shared that secret only because they shared a working directory, and that directory was the source tree. Give each frontend LOCALAI_DATA_PATH under its own baseDir and pin LOCALAI_AUTH_HMAC_SECRET, so a session minted at one replica resolves at every other one by construction. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/cluster/admin.go | 28 +++++++++++++++--------- tests/e2e/distributed/cluster/cluster.go | 25 +++++++++++++++++++-- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/tests/e2e/distributed/cluster/admin.go b/tests/e2e/distributed/cluster/admin.go index b1371f532271..617bcef99086 100644 --- a/tests/e2e/distributed/cluster/admin.go +++ b/tests/e2e/distributed/cluster/admin.go @@ -14,11 +14,12 @@ import ( ) const ( - // adminPassword must satisfy core/http/auth's policy (>= 12 chars and a - // zxcvbn score of 3 against hints that include "admin" and "localai"). - // This one scores 3 today; acknowledgeWeakPassword is sent alongside it so - // a future tightening of the policy cannot silently break every failover - // spec at setup time. + // adminPassword is sent with "acknowledge_weak_password": true, which sets + // PasswordPolicy{AllowWeak: true} and skips the length floor and the zxcvbn + // score entirely (core/http/auth/password.go). Only the technical + // invariants still apply: non-empty, at most 72 bytes, no NUL. The + // acknowledgement is deliberate rather than incidental, so a future + // tightening of the policy cannot break every failover spec at setup time. adminPassword = "e2e-admin-password" // sessionCookieName mirrors the unexported constant in core/http/auth. // The register handler returns 201 both for "user created, here is your @@ -43,11 +44,18 @@ func ForTestingEmpty() *Cluster { // which core/http/auth exempts from the approval gate and assigns the admin // role, so registration alone yields an active admin session. // -// Call this ONCE per cluster and share the client. Two reasons: the auth -// endpoints are rate limited to 5 requests per minute per client IP, and every -// e2e request arrives from 127.0.0.1; and the returned client is already good -// for every frontend, because sessions live in the shared Postgres auth DB and -// Go's cookie jar keys cookies by host without the port. +// Call this ONCE per cluster and share the client. Two reasons: +// +// One, a single rate limiter of 5 requests per minute per client IP guards +// POST /api/auth/token-login, POST /api/auth/register, POST /api/auth/login AND +// PUT /api/auth/password (core/http/routes/auth.go:190). They share one budget, +// and every e2e request arrives from 127.0.0.1, so a spec that changes a +// password spends from the same five. +// +// Two, the returned client is already good for every frontend: sessions live in +// the shared Postgres auth DB, the harness pins one HMAC secret across replicas +// so the session row resolves at any of them, and Go's cookie jar keys cookies +// by host without the port. func (c *Cluster) AdminSession(i int) (*http.Client, error) { base, err := c.frontendBaseURL(i) if err != nil { diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 0445fe3868e9..224018833da0 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -71,8 +71,11 @@ type Cluster struct { const ( defaultRegistrationToken = "e2e-token" defaultAdminEmail = "admin@e2e.local" - readinessTimeout = 90 * time.Second - readinessPoll = 200 * time.Millisecond + // testHMACSecret is shared by every frontend so a session minted at one + // replica validates at all of them. See the note in startFrontend. + testHMACSecret = "e2e-cluster-hmac-secret" + readinessTimeout = 90 * time.Second + readinessPoll = 200 * time.Millisecond // processExitTimeout bounds the post-SIGKILL wait in terminate. An unbounded // wait turns one stuck child (D state, or a Wait that never returns) into a // suite-wide Ginkgo timeout that names nothing. @@ -157,6 +160,14 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { if err := os.MkdirAll(filepath.Join(dir, "backends"), 0o755); err != nil { return nil, fmt.Errorf("creating %s dirs: %w", name, err) } + // Without an explicit LOCALAI_DATA_PATH every child resolves DataPath to + // ${cwd}/data (core/cli/run.go:48), which under `go test` is inside the + // source tree and shared by every replica: one collectiondb, one task and + // job store for processes that are meant to be independent. + dataPath := filepath.Join(dir, "data") + if err := os.MkdirAll(dataPath, 0o750); err != nil { + return nil, fmt.Errorf("creating %s dirs: %w", name, err) + } cmd := exec.Command(c.opts.Binary, "run", "--address", fmt.Sprintf("127.0.0.1:%d", port), @@ -171,6 +182,16 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { "LOCALAI_AUTH=true", "LOCALAI_AUTH_DATABASE_URL="+c.opts.PGDSN, "LOCALAI_ADMIN_EMAIL="+c.opts.AdminEmail, + "LOCALAI_DATA_PATH="+dataPath, + // Session rows are keyed by HMAC-SHA256(token, APIKeyHMACSecret), and + // the secret is generated per instance into {DataPath}/.hmac_secret + // unless pinned (core/application/startup.go:141-148). Now that each + // replica owns its data directory, an unpinned secret would differ per + // replica, so the cookie minted at frontend 0 would hash to a session + // row that does not exist at frontend 1 and every post-failover + // /api/nodes call would 401 with nothing in the logs to explain it. + // Pinning makes the cross-replica session a property of the harness. + "LOCALAI_AUTH_HMAC_SECRET="+testHMACSecret, "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, "LOCALAI_AUTO_APPROVE_NODES=true", "DEBUG=true", From 941150a0a2b6995b593fa6fd95ee1b2ab224ee63 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 14:12:35 +0000 Subject: [PATCH 011/153] test(distributed): add kill and restart primitives to the cluster harness The point of running LocalAI as real child processes is to be able to take one away. Add KillFrontend (SIGKILL, the lost replica), StopFrontendGracefully (SIGTERM, the rolling update), KillWorker, RestartFrontend and FrontendAlive. RestartFrontend pins the dead replica's original port. Workers read LOCALAI_REGISTER_TO once at boot and never re-resolve it, so a replica that returns on a fresh port is unreachable by exactly the workers that registered with it and the failover under test never happens. It also wipes the replica's data directory, so the process comes back with empty local state and has to rehydrate node, session and job state from the shared Postgres and NATS. Reusing the directory would model a pod with a persistent volume and hide the class of bug these tests exist to find. That is only safe because the harness pins LOCALAI_AUTH_HMAC_SECRET; otherwise the wipe would take {DataPath}/.hmac_secret with it and every session minted before the restart would 401 afterwards. FrontendAlive consults the reaper's exited channel before signal 0: a child that has died but has not yet been waited on is a zombie, and signal 0 to a zombie succeeds, which would report a dead replica as alive. The new specs cover argument validation only. Killing, stopping and restarting a live process needs a built binary plus Postgres and NATS, so those paths stay unexecuted until the failover suites land. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/cluster/cluster.go | 6 +- tests/e2e/distributed/cluster/cluster_test.go | 28 ++++ tests/e2e/distributed/cluster/failure.go | 144 ++++++++++++++++++ 3 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/distributed/cluster/failure.go diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 224018833da0..811a94c8c51b 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -152,8 +152,8 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { } port = allocated } - name := fmt.Sprintf("frontend-%d", i) - dir := filepath.Join(c.baseDir, name) + name := frontendName(i) + dir := c.frontendDir(i) if err := os.MkdirAll(filepath.Join(dir, "models"), 0o755); err != nil { return nil, fmt.Errorf("creating %s dirs: %w", name, err) } @@ -164,7 +164,7 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { // ${cwd}/data (core/cli/run.go:48), which under `go test` is inside the // source tree and shared by every replica: one collectiondb, one task and // job store for processes that are meant to be independent. - dataPath := filepath.Join(dir, "data") + dataPath := c.frontendDataDir(i) if err := os.MkdirAll(dataPath, 0o750); err != nil { return nil, fmt.Errorf("creating %s dirs: %w", name, err) } diff --git a/tests/e2e/distributed/cluster/cluster_test.go b/tests/e2e/distributed/cluster/cluster_test.go index 807c275c1c46..2f63d0a28e76 100644 --- a/tests/e2e/distributed/cluster/cluster_test.go +++ b/tests/e2e/distributed/cluster/cluster_test.go @@ -57,3 +57,31 @@ var _ = Describe("Admin session", Label("Distributed"), func() { Expect(err.Error()).To(ContainSubstring("frontend 1")) }) }) + +// Like the admin specs above, these cover argument validation only. Killing, +// stopping and restarting a real replica needs a built local-ai plus Postgres +// and NATS, so those paths stay unexecuted until the failover suites land. +var _ = Describe("Failure primitives", Label("Distributed"), func() { + It("rejects an out-of-range frontend index rather than panicking", func() { + c := cluster.ForTestingEmpty() + Expect(c.KillFrontend(0)).To(MatchError(ContainSubstring("frontend 0 out of range"))) + Expect(c.StopFrontendGracefully(2)).To(MatchError(ContainSubstring("frontend 2 out of range"))) + Expect(c.KillWorker(1)).To(MatchError(ContainSubstring("worker 1 out of range"))) + }) + + It("rejects a restart of a frontend index that does not exist", func() { + Expect(cluster.ForTestingEmpty().RestartFrontend(0)). + To(MatchError(ContainSubstring("frontend 0 out of range"))) + }) + + It("rejects a negative index without treating it as an offset from the end", func() { + c := cluster.ForTestingEmpty() + Expect(c.KillFrontend(-1)).To(MatchError(ContainSubstring("frontend -1 out of range"))) + Expect(c.KillWorker(-1)).To(MatchError(ContainSubstring("worker -1 out of range"))) + Expect(c.FrontendAlive(-1)).To(BeFalse()) + }) + + It("reports a frontend that was never started as not alive", func() { + Expect(cluster.ForTestingEmpty().FrontendAlive(0)).To(BeFalse()) + }) +}) diff --git a/tests/e2e/distributed/cluster/failure.go b/tests/e2e/distributed/cluster/failure.go new file mode 100644 index 000000000000..9e8141e316d4 --- /dev/null +++ b/tests/e2e/distributed/cluster/failure.go @@ -0,0 +1,144 @@ +package cluster + +import ( + "fmt" + "os" + "path/filepath" + "syscall" +) + +// KillFrontend SIGKILLs frontend i. This is the "replica died" case: no drain, +// no graceful deregistration, sockets drop without a FIN from the application. +// +// The signal is delivered but not waited on, because a spec that asserts on the +// cluster's reaction wants to observe the window between the death and the +// survivors noticing it. Poll FrontendAlive with Eventually to join the exit. +func (c *Cluster) KillFrontend(i int) error { + if err := c.checkFrontendIndex(i); err != nil { + return err + } + return signalProcess(c.frontends[i], syscall.SIGKILL) +} + +// StopFrontendGracefully SIGTERMs frontend i. This is the rolling-update case: +// the process gets a chance to drain and deregister. Like KillFrontend it does +// not wait; the point of the distinction between the two is what the process +// does with the time between the signal and its exit. +func (c *Cluster) StopFrontendGracefully(i int) error { + if err := c.checkFrontendIndex(i); err != nil { + return err + } + return signalProcess(c.frontends[i], syscall.SIGTERM) +} + +// KillWorker SIGKILLs worker i. +func (c *Cluster) KillWorker(i int) error { + if i < 0 || i >= len(c.workers) { + return fmt.Errorf("worker %d out of range (cluster has %d)", i, len(c.workers)) + } + return signalProcess(c.workers[i], syscall.SIGKILL) +} + +// RestartFrontend brings frontend i back on its original port with an empty +// data directory, modelling a replaced pod rather than a resumed one. +// +// The port is pinned rather than reallocated: workers read LOCALAI_REGISTER_TO +// once at boot and never re-resolve it, so a replica that returns on a new port +// is unreachable by exactly the workers that registered with it, and the +// failover the spec means to observe never happens. Rebinding is safe because +// the previous listener is fully closed before the new process starts (see the +// terminate below) and Go's listeners set SO_REUSEADDR, so a lingering +// TIME_WAIT on an accepted connection does not block the bind. +// +// The data directory is wiped so the replica must rehydrate node, session and +// job state from the shared Postgres and NATS. Keeping it would model a pod +// with a persistent volume and would hide the very class of bug these tests +// exist to find. This is only safe because startFrontend pins +// LOCALAI_AUTH_HMAC_SECRET: the secret otherwise lives at +// {DataPath}/.hmac_secret, and wiping it would make every session minted before +// the restart hash to a row the restarted replica cannot find, turning a +// failover assertion into an unexplained 401. +func (c *Cluster) RestartFrontend(i int) error { + if err := c.checkFrontendIndex(i); err != nil { + return err + } + old := c.frontends[i] + if old == nil { + return fmt.Errorf("frontend %d was never started, nothing to restart", i) + } + // The old process may still be running (a restart with no preceding kill) or + // already dead but unreaped. terminate is idempotent, bounds its wait, and + // releases the log handle the replacement is about to reopen; without it the + // replacement races the old listener for the port and leaks a file + // descriptor per restart. + old.terminate() + + if err := os.RemoveAll(c.frontendDataDir(i)); err != nil { + return fmt.Errorf("wiping data dir of frontend %d: %w", i, err) + } + + p, err := c.startFrontend(i, old.Port) + if err != nil { + return fmt.Errorf("restarting frontend %d: %w", i, err) + } + c.frontends[i] = p + return nil +} + +// FrontendAlive reports whether frontend i's process is still running. +func (c *Cluster) FrontendAlive(i int) bool { + if i < 0 || i >= len(c.frontends) { + return false + } + return c.frontends[i].alive() +} + +// alive reports whether the process is still running. The reaper's exited +// channel is authoritative and is consulted first: between a child's death and +// the reaper's Wait returning, the child is a zombie, and signal 0 to a zombie +// succeeds, which would report a dead replica as alive. +func (p *Process) alive() bool { + if p == nil || p.Cmd == nil || p.Cmd.Process == nil { + return false + } + select { + case <-p.exited: + return false + default: + } + // Signal 0 tests for existence without delivering anything. + return p.Cmd.Process.Signal(syscall.Signal(0)) == nil +} + +func signalProcess(p *Process, sig syscall.Signal) error { + if p == nil || p.Cmd == nil || p.Cmd.Process == nil { + return fmt.Errorf("process is not running") + } + if err := p.Cmd.Process.Signal(sig); err != nil { + return fmt.Errorf("signalling %s with %v: %w", p.Name, sig, err) + } + return nil +} + +// checkFrontendIndex keeps the out-of-range wording identical across every +// primitive, so a failing spec reads the same whichever one tripped. +func (c *Cluster) checkFrontendIndex(i int) error { + if i < 0 || i >= len(c.frontends) { + return fmt.Errorf("frontend %d out of range (cluster has %d)", i, len(c.frontends)) + } + return nil +} + +func frontendName(i int) string { + return fmt.Sprintf("frontend-%d", i) +} + +func (c *Cluster) frontendDir(i int) string { + return filepath.Join(c.baseDir, frontendName(i)) +} + +// frontendDataDir is LOCALAI_DATA_PATH for frontend i. RestartFrontend wipes it, +// so it must be the exact path startFrontend hands the child. +func (c *Cluster) frontendDataDir(i int) string { + return filepath.Join(c.frontendDir(i), "data") +} From 6e78ea954ab169fef076787d7ac70735393af6a6 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 14:22:35 +0000 Subject: [PATCH 012/153] test(distributed): correct the failure-primitive comments and guard the wipe Review round 1. Comments only, plus one guard. The note on Process.alive claimed the exited check closed the zombie window. It does not. The reaper closes exited only after Cmd.Wait returns, and Wait marks the os.Process done before returning, so exited being closed implies signal 0 already errors and the branch cannot fire earlier than the one it precedes. The window between the child exiting and waitid collecting it stays open in both versions, and the only real mitigation is for callers to poll with Eventually rather than sample once. Keep the check as hygiene, say what it actually does, and say it again on the exited field, so nobody reads the old claim and drops the Eventually. Record what the cold wipe destroys. The harness sets no LOCALAI_STORAGE_URL, so the object store is a directory under DataPath, and quantization and fine-tune outputs live there too. Postgres keeps the job row; the artifact it points at does not survive the restart. A spec that asserts otherwise will fail for a storage reason wearing a failover costume. Tell callers to let a graceful stop finish before restarting: RestartFrontend terminates with SIGKILL, so pairing it straight after StopFrontendGracefully cuts the drain short and silently converts the rolling-update case into the crash case. Refuse to wipe when the cluster has no work dir. frontendDataDir is relative when baseDir is empty, so a Cluster built by some future test helper without one would have RemoveAll walking frontend-N/data inside the source tree. The guard sits before terminate, so a refusal leaves the cluster as it was. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/cluster/cluster.go | 5 +++ tests/e2e/distributed/cluster/failure.go | 41 +++++++++++++++++++++--- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 811a94c8c51b..6cf15f349cdc 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -56,6 +56,11 @@ type Process struct { // exited closes once the reaper has collected the process. Only the reaper // calls Cmd.Wait, so nothing else may: a second Wait on the same Cmd races // the first and corrupts ProcessState. + // + // A closed exited proves the child is gone. An open one proves nothing: it + // is still open for the whole interval between the child exiting and waitid + // collecting it, during which the child is a zombie that signal 0 reports as + // alive. Anything asserting on a process being dead must poll, not sample. exited chan struct{} waitErr error } diff --git a/tests/e2e/distributed/cluster/failure.go b/tests/e2e/distributed/cluster/failure.go index 9e8141e316d4..3f2a1f42ba28 100644 --- a/tests/e2e/distributed/cluster/failure.go +++ b/tests/e2e/distributed/cluster/failure.go @@ -58,6 +58,25 @@ func (c *Cluster) KillWorker(i int) error { // {DataPath}/.hmac_secret, and wiping it would make every session minted before // the restart hash to a row the restarted replica cannot find, turning a // failover assertion into an unexplained 401. +// +// The wipe also destroys state that nothing can rebuild. This harness sets no +// LOCALAI_STORAGE_URL, so the distributed object store is a directory under +// {DataPath} (core/application/distributed.go:146), and quantization and +// fine-tune jobs write their outputs to {DataPath}/quantization and +// {DataPath}/fine-tune (core/services/{quantization,finetune}/service.go:95); +// agent state, router-corpus, the voiceprofile store and {DataPath}/traces go +// the same way. Postgres keeps the job row, the artifact it points at is gone. +// So a spec that finishes a quantization or fine-tune on a replica, restarts +// it, and then asserts the artifact is retrievable fails for a storage reason +// dressed up as a failover one. No spec does that today; this note is here so +// the first one that tries does not spend a day on it. +// +// After StopFrontendGracefully, wait for the process to actually go +// (Eventually(c.FrontendAlive).Should(BeFalse())) before restarting. Restart +// terminates whatever is still running with SIGKILL, so restarting straight +// after a SIGTERM cuts the drain short and quietly turns the rolling-update +// case into the crash case, which is the opposite of what pairing those two +// calls is meant to express. func (c *Cluster) RestartFrontend(i int) error { if err := c.checkFrontendIndex(i); err != nil { return err @@ -66,13 +85,18 @@ func (c *Cluster) RestartFrontend(i int) error { if old == nil { return fmt.Errorf("frontend %d was never started, nothing to restart", i) } + // frontendDataDir is relative when baseDir is empty, and this deletes it: + // a Cluster assembled by a future test helper without a work dir would have + // RemoveAll walking "frontend-N/data" under the package source directory. + if c.baseDir == "" { + return fmt.Errorf("refusing to wipe the data dir of frontend %d: cluster has no work dir", i) + } // The old process may still be running (a restart with no preceding kill) or // already dead but unreaped. terminate is idempotent, bounds its wait, and // releases the log handle the replacement is about to reopen; without it the // replacement races the old listener for the port and leaks a file // descriptor per restart. old.terminate() - if err := os.RemoveAll(c.frontendDataDir(i)); err != nil { return fmt.Errorf("wiping data dir of frontend %d: %w", i, err) } @@ -93,10 +117,17 @@ func (c *Cluster) FrontendAlive(i int) bool { return c.frontends[i].alive() } -// alive reports whether the process is still running. The reaper's exited -// channel is authoritative and is consulted first: between a child's death and -// the reaper's Wait returning, the child is a zombie, and signal 0 to a zombie -// succeeds, which would report a dead replica as alive. +// alive reports whether the process is still running. +// +// The exited check is cheap hygiene, not a fix for the zombie window. The +// reaper closes exited only after Cmd.Wait returns, and Wait marks the +// os.Process done before it returns (runtime/os pidfd path), so by the time +// exited is closed signal 0 already errors: this branch cannot fire earlier +// than the one it precedes. The window that stays open is the other one, +// between the child exiting and waitid collecting it: there the child is a +// zombie, signal 0 to a zombie succeeds, and alive reports true for a process +// that is already dead. There is no local fix; the caller's is to poll, +// Eventually(c.FrontendAlive).Should(BeFalse()), rather than assert once. func (p *Process) alive() bool { if p == nil || p.Cmd == nil || p.Cmd.Process == nil { return false From 3d2a96d925f6318f8c90d3e18063af91a947d6dd Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 14:38:12 +0000 Subject: [PATCH 013/153] test(distributed): prove the cluster harness with a two-replica baseline Tasks 4 to 6 built a harness that runs local-ai as real child processes, but none of it had ever started a process: every spec so far returned inside argument validation. These two specs are the first to run it against a real binary, a real Postgres and a real NATS. Two frontends against one database both see a worker that registered through only one of them. Every failover spec assumes this, so it is asserted first. One admin session is minted at frontend 0 and reused for both replicas rather than registering per frontend. The auth routes share a five-per-minute-per-IP limiter and all e2e traffic is 127.0.0.1, so a session per frontend would exhaust the budget as soon as a spec needs a third one. Reuse is sound because sessions live in the shared Postgres and the harness pins one HMAC secret across replicas; frontend 1 answering /api/nodes with 200 on a cookie minted at frontend 0 is what proves it. The binaries are resolved before SetupInfra so a missing build skips without first provisioning a database the skip would then have to tear down. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .../e2e/distributed/cluster_baseline_test.go | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 tests/e2e/distributed/cluster_baseline_test.go diff --git a/tests/e2e/distributed/cluster_baseline_test.go b/tests/e2e/distributed/cluster_baseline_test.go new file mode 100644 index 000000000000..45101aaec29a --- /dev/null +++ b/tests/e2e/distributed/cluster_baseline_test.go @@ -0,0 +1,148 @@ +package distributed_test + +import ( + "net/http" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/tests/e2e/distributed/cluster" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// nodeRosterTimeout bounds the wait for a worker to appear healthy in +// /api/nodes. Registration is an HTTP call the worker retries, followed by a +// heartbeat that has to land before the frontend calls the node healthy, so the +// budget covers several retry intervals rather than a single round trip. +const ( + nodeRosterTimeout = "90s" + nodeRosterPoll = "1s" +) + +// node is the subset of the /api/nodes payload these specs assert on. +type node struct { + Name string `json:"name"` + Status string `json:"status"` +} + +// localAIBinary resolves the built binary, skipping rather than failing when it +// is absent so `make test-e2e-distributed` still runs for someone who has not +// built it. CI always builds it, so CI never skips. +func localAIBinary() string { + GinkgoHelper() + path := os.Getenv("LOCALAI_E2E_BINARY") + if path == "" { + wd, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + path = filepath.Join(wd, "..", "..", "..", "local-ai") + } + if _, err := os.Stat(path); err != nil { + Skip("local-ai binary not found at " + path + "; run `make build` or set LOCALAI_E2E_BINARY") + } + return path +} + +func mockBackendBinary() string { + GinkgoHelper() + wd, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + path := filepath.Join(wd, "..", "mock-backend", "mock-backend") + if _, err := os.Stat(path); err != nil { + Skip("mock-backend not found at " + path + "; run `make build-mock-backend`") + } + return path +} + +// startCluster brings up a cluster against a freshly provisioned database and +// registers cleanup, including a log dump on failure. +func startCluster(frontends, workers int) *cluster.Cluster { + GinkgoHelper() + + // Resolved before SetupInfra so a missing binary skips without having paid + // for a database that the skip would then leave to DeferCleanup. + binary := localAIBinary() + mockBackend := mockBackendBinary() + + infra := SetupInfra("cluster") + + // The log directory must be predictable so CI can upload it as an artifact. + // GinkgoT().TempDir() lands under TMPDIR, which on a GitHub runner is not + // /tmp, so an artifact glob would silently match nothing. + logDir := os.Getenv("LOCALAI_E2E_LOG_DIR") + if logDir == "" { + logDir = GinkgoT().TempDir() + } else { + logDir = filepath.Join(logDir, sanitizeDBName(CurrentSpecReport().LeafNodeText)) + Expect(os.MkdirAll(logDir, 0o755)).To(Succeed()) + } + + c, err := cluster.Start(cluster.Options{ + Binary: binary, + MockBackend: mockBackend, + PGDSN: infra.PGURL, + NatsURL: infra.NatsURL, + LogDir: logDir, + Frontends: frontends, + Workers: workers, + }) + Expect(err).ToNot(HaveOccurred()) + + DeferCleanup(func() { + if CurrentSpecReport().Failed() { + c.DumpLogs() + } + c.Stop() + }) + return c +} + +// healthyNodeNames polls one frontend's roster. It returns nil on any error so +// Eventually keeps retrying: the roster is unreachable for the first moments of +// a replica's life, and a hard failure there would only re-report a startup race. +func healthyNodeNames(c *cluster.Cluster, client *http.Client, frontend int) []string { + var nodes []node + if err := c.GetJSON(client, frontend, "/api/nodes", &nodes); err != nil { + return nil + } + names := []string{} + for _, n := range nodes { + if n.Status == "healthy" { + names = append(names, n.Name) + } + } + return names +} + +var _ = Describe("Cluster baseline", Label("Distributed"), Label("Cluster"), func() { + It("brings up a frontend and a worker, and the worker appears in the roster", func() { + c := startCluster(1, 1) + + client, err := c.AdminSession(0) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() []string { + return healthyNodeNames(c, client, 0) + }, nodeRosterTimeout, nodeRosterPoll).Should(ContainElement(c.WorkerName(0))) + }) + + It("runs two frontends against one database and both see the same worker", func() { + c := startCluster(2, 1) + + // One session for the whole cluster, minted at frontend 0. Registering or + // logging in again per frontend would spend from the same five-per-minute + // budget the auth routes share per client IP, and every request here comes + // from 127.0.0.1. The single client is valid at both replicas: sessions + // live in the shared Postgres, the harness pins one HMAC secret so the + // row resolves anywhere, and Go's cookie jar keys by host without port. + client, err := c.AdminSession(0) + Expect(err).ToNot(HaveOccurred()) + + for _, frontend := range []int{0, 1} { + Eventually(func() []string { + return healthyNodeNames(c, client, frontend) + }, nodeRosterTimeout, nodeRosterPoll).Should(ContainElement(c.WorkerName(0)), + "frontend %d should see the worker registered through frontend 0", frontend) + } + }) +}) From 9862d9eb4d04bc9820e8b0e1ee98e2d4d176027a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 14:54:23 +0000 Subject: [PATCH 014/153] test(distributed): fail rather than skip the cluster specs in CI The Cluster label partition is these two specs and nothing else, so a missing binary skipped the entire job. Ginkgo exits 0 on skips, so a build step that broke or moved its output would have left the job reporting "0 Passed | 2 Skipped" and going green without ever starting a cluster: the silent pass this suite exists to make impossible. Skipping stays the local default, which is the right courtesy for someone who has not run `make build`, but LOCALAI_E2E_REQUIRE_BINARIES turns it into a failure that names the missing path and the target that builds it. A value that is set but unparseable counts as on, since reading it as off would restore the very skip it disables. Failures also name themselves now. The roster poll kept returning a bare nil on error, so a 401 at the second replica, a decode failure and "the worker never registered" all presented identically as an empty list. It now retains the last error and the last roster and reports whichever happened, through a lazily evaluated Gomega description that costs nothing until something fails. Finally, the two-frontend spec no longer depends on the harness to mean what it says. It asserts an unauthenticated GET /api/nodes at frontend 1 is refused, which observes the admin gate instead of assuming it, and it compares the worker's registration id across the two replicas rather than its name. A future harness that registered every worker with every frontend would have kept a name-only assertion green while it quietly stopped proving anything about shared state. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .../e2e/distributed/cluster_baseline_test.go | 173 +++++++++++++++--- 1 file changed, 143 insertions(+), 30 deletions(-) diff --git a/tests/e2e/distributed/cluster_baseline_test.go b/tests/e2e/distributed/cluster_baseline_test.go index 45101aaec29a..e25739b71ba4 100644 --- a/tests/e2e/distributed/cluster_baseline_test.go +++ b/tests/e2e/distributed/cluster_baseline_test.go @@ -1,34 +1,80 @@ package distributed_test import ( + "fmt" "net/http" "os" "path/filepath" + "strconv" + "strings" + "time" + "github.com/mudler/LocalAI/pkg/httpclient" "github.com/mudler/LocalAI/tests/e2e/distributed/cluster" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -// nodeRosterTimeout bounds the wait for a worker to appear healthy in -// /api/nodes. Registration is an HTTP call the worker retries, followed by a -// heartbeat that has to land before the frontend calls the node healthy, so the -// budget covers several retry intervals rather than a single round trip. const ( + // nodeRosterTimeout bounds the wait for a worker to appear healthy in + // /api/nodes. Registration is an HTTP call the worker retries, followed by a + // heartbeat that has to land before the frontend calls the node healthy, so + // the budget covers several retry intervals rather than a single round trip. nodeRosterTimeout = "90s" nodeRosterPoll = "1s" + // authProbeTimeout bounds the single unauthenticated request that checks the + // admin gate is actually closed. One round trip against a ready local + // process; anything slower is a defect, not slowness. + authProbeTimeout = 30 * time.Second ) -// node is the subset of the /api/nodes payload these specs assert on. +// node is the subset of the /api/nodes payload these specs assert on. ID is the +// registration identity the worker minted, which is what distinguishes "the +// same node row seen from a second replica" from "a second registration that +// happens to share a name". type node struct { + ID string `json:"id"` Name string `json:"name"` Status string `json:"status"` } -// localAIBinary resolves the built binary, skipping rather than failing when it -// is absent so `make test-e2e-distributed` still runs for someone who has not -// built it. CI always builds it, so CI never skips. +// requireBinaries reports whether a missing binary must fail the spec instead of +// skipping it. +// +// Skipping is the right courtesy locally: someone who has not run `make build` +// should get a clear note, not a wall of red. In CI it is the opposite. The +// whole Cluster label partition is these two specs, so if the workflow's build +// step breaks or moves its output, a skip would leave the job reporting +// "0 Passed | 2 Skipped" and exiting 0. Ginkgo exits 0 on skips, so that job +// goes green having never started a cluster, which is precisely the silent pass +// this suite exists to make impossible. CI sets this variable (Task 9) and gets +// a failure instead. +func requireBinaries() bool { + value := strings.TrimSpace(os.Getenv("LOCALAI_E2E_REQUIRE_BINARIES")) + if value == "" { + return false + } + if parsed, err := strconv.ParseBool(value); err == nil { + return parsed + } + // Set but unparseable means someone meant to turn this on. Reading it as + // false would quietly restore the silent skip the flag exists to prevent. + return true +} + +// missingBinary skips or fails, naming the path and the target that builds it. +func missingBinary(what, path, makeTarget string) { + GinkgoHelper() + message := fmt.Sprintf("%s not found at %s; run `%s`", what, path, makeTarget) + if requireBinaries() { + Fail(message + " (LOCALAI_E2E_REQUIRE_BINARIES is set, so this fails rather than skips: " + + "in CI a skipped cluster spec is indistinguishable from a passing one)") + } + Skip(message) +} + +// localAIBinary resolves the built binary. func localAIBinary() string { GinkgoHelper() path := os.Getenv("LOCALAI_E2E_BINARY") @@ -38,7 +84,7 @@ func localAIBinary() string { path = filepath.Join(wd, "..", "..", "..", "local-ai") } if _, err := os.Stat(path); err != nil { - Skip("local-ai binary not found at " + path + "; run `make build` or set LOCALAI_E2E_BINARY") + missingBinary("local-ai binary", path, "make build") } return path } @@ -49,7 +95,7 @@ func mockBackendBinary() string { Expect(err).ToNot(HaveOccurred()) path := filepath.Join(wd, "..", "mock-backend", "mock-backend") if _, err := os.Stat(path); err != nil { - Skip("mock-backend not found at " + path + "; run `make build-mock-backend`") + missingBinary("mock-backend", path, "make build-mock-backend") } return path } @@ -97,16 +143,39 @@ func startCluster(frontends, workers int) *cluster.Cluster { return c } -// healthyNodeNames polls one frontend's roster. It returns nil on any error so -// Eventually keeps retrying: the roster is unreachable for the first moments of -// a replica's life, and a hard failure there would only re-report a startup race. -func healthyNodeNames(c *cluster.Cluster, client *http.Client, frontend int) []string { - var nodes []node - if err := c.GetJSON(client, frontend, "/api/nodes", &nodes); err != nil { +// rosterProbe polls one frontend's node roster. +// +// It keeps the last error and the last roster it saw so a failing Eventually can +// name the cause. Returning a bare nil on error makes a 401 at the second +// replica, a JSON decode failure and "the worker never registered" all present +// identically as an empty list, which is the least useful thing a failover +// suite can say when it goes red. +type rosterProbe struct { + cluster *cluster.Cluster + client *http.Client + frontend int + + lastErr error + lastSeen []node +} + +func newRosterProbe(c *cluster.Cluster, client *http.Client, frontend int) *rosterProbe { + return &rosterProbe{cluster: c, client: client, frontend: frontend} +} + +// healthyNames returns nil on any error so Eventually keeps retrying: the roster +// is unreachable for the first moments of a replica's life, and failing hard +// there would only re-report a startup race. +func (p *rosterProbe) healthyNames() []string { + var roster []node + if err := p.cluster.GetJSON(p.client, p.frontend, "/api/nodes", &roster); err != nil { + p.lastErr = err return nil } + p.lastErr = nil + p.lastSeen = roster names := []string{} - for _, n := range nodes { + for _, n := range roster { if n.Status == "healthy" { names = append(names, n.Name) } @@ -114,6 +183,27 @@ func healthyNodeNames(c *cluster.Cluster, client *http.Client, frontend int) []s return names } +// idOf returns the registration ID the roster last reported for a node name. +func (p *rosterProbe) idOf(name string) string { + for _, n := range p.lastSeen { + if n.Name == name { + return n.ID + } + } + return "" +} + +// describe is handed to Should as the failure message. Gomega calls a +// func() string description lazily, so this runs only on failure and reports +// whichever of the two distinct causes actually occurred. +func (p *rosterProbe) describe() string { + if p.lastErr != nil { + return fmt.Sprintf("frontend %d: the last GET /api/nodes failed: %v", p.frontend, p.lastErr) + } + return fmt.Sprintf("frontend %d: GET /api/nodes succeeded but the roster held %d node(s): %+v", + p.frontend, len(p.lastSeen), p.lastSeen) +} + var _ = Describe("Cluster baseline", Label("Distributed"), Label("Cluster"), func() { It("brings up a frontend and a worker, and the worker appears in the roster", func() { c := startCluster(1, 1) @@ -121,28 +211,51 @@ var _ = Describe("Cluster baseline", Label("Distributed"), Label("Cluster"), fun client, err := c.AdminSession(0) Expect(err).ToNot(HaveOccurred()) - Eventually(func() []string { - return healthyNodeNames(c, client, 0) - }, nodeRosterTimeout, nodeRosterPoll).Should(ContainElement(c.WorkerName(0))) + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), probe.describe) }) It("runs two frontends against one database and both see the same worker", func() { c := startCluster(2, 1) + // Observe that frontend 1 really is gated before proving a session opens + // it. Without this, "the cookie minted at frontend 0 works here" is + // indistinguishable from "this endpoint needs no auth at all". The probe + // is free: it touches no auth route, so it spends nothing from the + // five-per-minute-per-IP budget those routes share. + anonymous := httpclient.NewWithTimeout(authProbeTimeout) + refused, err := anonymous.Get(c.FrontendURL(1) + "/api/nodes") + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = refused.Body.Close() }() + Expect(refused.StatusCode).To(Equal(http.StatusUnauthorized), + "an unauthenticated GET /api/nodes must be refused, otherwise this spec proves nothing about sessions") + // One session for the whole cluster, minted at frontend 0. Registering or // logging in again per frontend would spend from the same five-per-minute - // budget the auth routes share per client IP, and every request here comes - // from 127.0.0.1. The single client is valid at both replicas: sessions - // live in the shared Postgres, the harness pins one HMAC secret so the - // row resolves anywhere, and Go's cookie jar keys by host without port. + // budget, and every request here comes from 127.0.0.1. The single client + // is valid at both replicas: sessions live in the shared Postgres, the + // harness pins one HMAC secret so the row resolves anywhere, and Go's + // cookie jar keys by host without port. client, err := c.AdminSession(0) Expect(err).ToNot(HaveOccurred()) - for _, frontend := range []int{0, 1} { - Eventually(func() []string { - return healthyNodeNames(c, client, frontend) - }, nodeRosterTimeout, nodeRosterPoll).Should(ContainElement(c.WorkerName(0)), - "frontend %d should see the worker registered through frontend 0", frontend) - } + // The worker is pointed at frontend 0 alone (the harness sets + // LOCALAI_REGISTER_TO to frontend 0), so read its identity there first. + at0 := newRosterProbe(c, client, 0) + Eventually(at0.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), at0.describe) + registeredID := at0.idOf(c.WorkerName(0)) + Expect(registeredID).ToNot(BeEmpty(), "frontend 0 reported the worker without a registration ID") + + // Then assert frontend 1 serves the same row, not merely a node with the + // same name. Comparing IDs pins the topology inside the spec: a future + // harness that registered every worker with every frontend would keep a + // name-only assertion green while it quietly stopped testing shared state. + at1 := newRosterProbe(c, client, 1) + Eventually(at1.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), at1.describe) + Expect(at1.idOf(c.WorkerName(0))).To(Equal(registeredID), + "frontend 1 must serve the same node row that registered through frontend 0, not a separate registration") }) }) From 9b0d3827b275def22efef15c43e2cbfc58d9af97 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 15:06:54 +0000 Subject: [PATCH 015/153] test(distributed): require the cluster binaries by default under CI The previous round made a missing binary fail instead of skip, but only when a workflow remembered to set LOCALAI_E2E_REQUIRE_BINARIES. That leaves the silent pass one forgotten line away: the Cluster label partition is two specs, Ginkgo exits 0 on skips, and a job that skips both reports "0 Passed | 2 Skipped" and goes green having never started a cluster. So the polarity is inverted. Binaries are required whenever CI is set, which GitHub Actions always does, and the flag now exists to force the requirement OFF rather than to be remembered ON. A local developer sees no change, since CI is unset in an ordinary shell and a missing binary still skips with a message naming the path and how to build it. off, no, n and disabled are honoured as off; ParseBool rejects them, and reading a word that unambiguous as its opposite would be a worse trap than the one this removes. Also correct a claim the previous commit message got wrong. Comparing the worker's registration id across the two replicas does not pin the topology: NodeRegistry.Register looks a node up by name and preserves the existing id, and both replicas read one Postgres, so registering the worker with every frontend would yield identical ids too. The assertion is still worth keeping for what it does catch, a replica answering from its own registry or database instead of the shared one, and the comment now says that and nothing more. The topology fact moves to where someone would break it: a note on LOCALAI_REGISTER_TO recording that workers register with frontend 0 only, that the cross-replica specs depend on it, and that nothing in those specs can detect a change to it. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/cluster/cluster.go | 12 ++++ .../e2e/distributed/cluster_baseline_test.go | 55 +++++++++++++------ 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 6cf15f349cdc..0077c21cefe6 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -248,6 +248,18 @@ func (c *Cluster) startWorker(i int) (*Process, error) { fmt.Sprintf("LOCALAI_ADVERTISE_ADDR=127.0.0.1:%d", grpcPort), fmt.Sprintf("LOCALAI_HTTP_ADDR=127.0.0.1:%d", httpPort), fmt.Sprintf("LOCALAI_ADVERTISE_HTTP_ADDR=127.0.0.1:%d", httpPort), + // Workers register with frontend 0 ONLY, never with every replica, and + // the cross-replica session specs depend on that. They prove a session + // minted at frontend 0 resolves at frontend 1 by reading a node that + // only frontend 0 was ever told about; register the worker everywhere + // and they still pass while proving nothing. + // + // Nothing in those specs can detect the change. The registry keys nodes + // by name and preserves ids across the shared Postgres + // (core/services/nodes/registry.go:522-527), so a roster read at + // frontend 1 looks identical either way. Anyone pointing workers at + // more than one replica must revisit + // tests/e2e/distributed/cluster_baseline_test.go by hand. "LOCALAI_REGISTER_TO="+c.FrontendURL(0), "LOCALAI_NODE_NAME="+name, "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, diff --git a/tests/e2e/distributed/cluster_baseline_test.go b/tests/e2e/distributed/cluster_baseline_test.go index e25739b71ba4..662f93b38835 100644 --- a/tests/e2e/distributed/cluster_baseline_test.go +++ b/tests/e2e/distributed/cluster_baseline_test.go @@ -40,7 +40,7 @@ type node struct { } // requireBinaries reports whether a missing binary must fail the spec instead of -// skipping it. +// skipping it. It defaults to ON under CI. // // Skipping is the right courtesy locally: someone who has not run `make build` // should get a clear note, not a wall of red. In CI it is the opposite. The @@ -48,28 +48,42 @@ type node struct { // step breaks or moves its output, a skip would leave the job reporting // "0 Passed | 2 Skipped" and exiting 0. Ginkgo exits 0 on skips, so that job // goes green having never started a cluster, which is precisely the silent pass -// this suite exists to make impossible. CI sets this variable (Task 9) and gets -// a failure instead. +// this suite exists to make impossible. +// +// Hence the polarity: the safe behaviour is the default, keyed off CI (GitHub +// Actions always sets it), and LOCALAI_E2E_REQUIRE_BINARIES exists to be forced +// OFF rather than to be remembered ON. A future workflow author cannot reach +// the green-on-nothing state by forgetting a line, only by writing one that +// explicitly asks for it. A local developer sees no change: CI is unset in an +// ordinary shell, so a missing binary still skips. func requireBinaries() bool { value := strings.TrimSpace(os.Getenv("LOCALAI_E2E_REQUIRE_BINARIES")) if value == "" { + return os.Getenv("CI") != "" + } + // ParseBool rejects these, and the fallback below reads anything it rejects + // as ON. Someone writing "off" plainly means off, and silently inverting + // them would be a worse trap than the one this flag removes. + switch strings.ToLower(value) { + case "off", "no", "n", "disabled": return false } if parsed, err := strconv.ParseBool(value); err == nil { return parsed } - // Set but unparseable means someone meant to turn this on. Reading it as - // false would quietly restore the silent skip the flag exists to prevent. + // Set to something meaningless means someone meant to turn this on. Reading + // it as false would quietly restore the silent skip the flag guards against. return true } -// missingBinary skips or fails, naming the path and the target that builds it. -func missingBinary(what, path, makeTarget string) { +// missingBinary skips or fails, naming the path and how to produce it. +func missingBinary(what, path, remedy string) { GinkgoHelper() - message := fmt.Sprintf("%s not found at %s; run `%s`", what, path, makeTarget) + message := fmt.Sprintf("%s not found at %s; %s", what, path, remedy) if requireBinaries() { - Fail(message + " (LOCALAI_E2E_REQUIRE_BINARIES is set, so this fails rather than skips: " + - "in CI a skipped cluster spec is indistinguishable from a passing one)") + Fail(message + " (binaries are required here, either under CI or via " + + "LOCALAI_E2E_REQUIRE_BINARIES, so this fails rather than skips: a skipped " + + "cluster spec is indistinguishable from a passing one)") } Skip(message) } @@ -84,7 +98,7 @@ func localAIBinary() string { path = filepath.Join(wd, "..", "..", "..", "local-ai") } if _, err := os.Stat(path); err != nil { - missingBinary("local-ai binary", path, "make build") + missingBinary("local-ai binary", path, "run `make build` or set LOCALAI_E2E_BINARY") } return path } @@ -95,7 +109,7 @@ func mockBackendBinary() string { Expect(err).ToNot(HaveOccurred()) path := filepath.Join(wd, "..", "mock-backend", "mock-backend") if _, err := os.Stat(path); err != nil { - missingBinary("mock-backend", path, "make build-mock-backend") + missingBinary("mock-backend", path, "run `make build-mock-backend`") } return path } @@ -248,14 +262,21 @@ var _ = Describe("Cluster baseline", Label("Distributed"), Label("Cluster"), fun registeredID := at0.idOf(c.WorkerName(0)) Expect(registeredID).ToNot(BeEmpty(), "frontend 0 reported the worker without a registration ID") - // Then assert frontend 1 serves the same row, not merely a node with the - // same name. Comparing IDs pins the topology inside the spec: a future - // harness that registered every worker with every frontend would keep a - // name-only assertion green while it quietly stopped testing shared state. + // Then assert frontend 1 serves the same row, by id and not merely by name. + // + // Be precise about what this proves. It does NOT pin the topology: + // NodeRegistry.Register looks a node up by name and preserves the + // existing id (core/services/nodes/registry.go:522-527), and both + // replicas read one Postgres, so a harness that registered the worker + // with every frontend would yield identical ids here too. What it does + // catch is a frontend answering from its own registry or its own + // database rather than the shared one, which is a different regression + // and just as silent. The topology fact is not asserted anywhere; it is + // recorded next to LOCALAI_REGISTER_TO in cluster.go. at1 := newRosterProbe(c, client, 1) Eventually(at1.healthyNames, nodeRosterTimeout, nodeRosterPoll). Should(ContainElement(c.WorkerName(0)), at1.describe) Expect(at1.idOf(c.WorkerName(0))).To(Equal(registeredID), - "frontend 1 must serve the same node row that registered through frontend 0, not a separate registration") + "frontend 1 must resolve the same node row as frontend 0; a differing id means it is not reading the shared state") }) }) From 8990c511513d47518d2c6b9ece84d1f733366fec Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 16:28:10 +0000 Subject: [PATCH 016/153] test(distributed): cover replica death, rolling restart and racing replicas Four scenarios with no prior equivalent: killing a replica must not disturb a worker that never depended on it, a cold-restarted replica must rehydrate the roster from shared state and keep accepting the worker's heartbeats, a dead worker must settle to offline on every replica, and two replicas registering a worker each must converge on one roster. The timings are measured, not assumed. Node liveness is heartbeat freshness, so the only eviction path is StaleNodeThreshold (60s) plus one HealthCheckInterval tick (15s), and neither is reachable from the CLI. A worker whose registrar was killed was observed going offline at 74.2s. Every window here is sized to outlast that, because an assertion that expires before the system could have reacted proves nothing. Two assertions are deliberately unlike the obvious form. Statuses are compared for equality against a probe that returns a sentinel on error, rather than asserting a name is absent from the healthy list: the list probe returns nil on any error, and "does not contain" is satisfied by nil, so a 401 at the second replica would have passed while observing nothing. And a killed worker is required to settle to exactly offline, because it first flaps to unhealthy at ~8s and back to healthy at ~14s, which any not-healthy matcher would accept. SpreadWorkerRegistrations is new, off by default, and exists so the racing spec is a race: the harness otherwise points every worker at frontend 0, which would have left that scenario asserting on two sequential writes through one process. The default is unchanged because the baseline specs depend on it. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/cluster/cluster.go | 52 +++- .../e2e/distributed/cluster_baseline_test.go | 15 +- .../e2e/distributed/cluster_failover_test.go | 280 ++++++++++++++++++ 3 files changed, 335 insertions(+), 12 deletions(-) create mode 100644 tests/e2e/distributed/cluster_failover_test.go diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 0077c21cefe6..198ff5cae735 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -40,6 +40,17 @@ type Options struct { Frontends int Workers int + + // SpreadWorkerRegistrations sends worker i to frontend i%Frontends instead + // of sending every worker to frontend 0. + // + // Off by default, and deliberately so: the cross-replica session specs read + // a node at frontend 1 that only frontend 0 was ever told about, and + // spreading registrations would leave them passing while proving nothing. + // It exists for the racing-replicas spec, which is about two replicas + // writing to one roster concurrently and cannot express that at all while + // every worker registers through the same process. + SpreadWorkerRegistrations bool } // Process is one running local-ai. @@ -248,19 +259,25 @@ func (c *Cluster) startWorker(i int) (*Process, error) { fmt.Sprintf("LOCALAI_ADVERTISE_ADDR=127.0.0.1:%d", grpcPort), fmt.Sprintf("LOCALAI_HTTP_ADDR=127.0.0.1:%d", httpPort), fmt.Sprintf("LOCALAI_ADVERTISE_HTTP_ADDR=127.0.0.1:%d", httpPort), - // Workers register with frontend 0 ONLY, never with every replica, and - // the cross-replica session specs depend on that. They prove a session - // minted at frontend 0 resolves at frontend 1 by reading a node that - // only frontend 0 was ever told about; register the worker everywhere - // and they still pass while proving nothing. + // Workers register with frontend 0 ONLY unless the caller opts into + // SpreadWorkerRegistrations, and the cross-replica session specs depend + // on that default. They prove a session minted at frontend 0 resolves at + // frontend 1 by reading a node that only frontend 0 was ever told about; + // register the worker everywhere and they still pass while proving + // nothing. // // Nothing in those specs can detect the change. The registry keys nodes // by name and preserves ids across the shared Postgres // (core/services/nodes/registry.go:522-527), so a roster read at - // frontend 1 looks identical either way. Anyone pointing workers at - // more than one replica must revisit - // tests/e2e/distributed/cluster_baseline_test.go by hand. - "LOCALAI_REGISTER_TO="+c.FrontendURL(0), + // frontend 1 looks identical either way. Anyone changing the default + // here must revisit tests/e2e/distributed/cluster_baseline_test.go by + // hand. + // + // The registrar also fixes where this worker's heartbeats go for the + // rest of its life: the loop posts to the URL it was given at boot and + // never re-resolves it (core/cli/workerregistry/client.go), so killing a + // worker's registrar orphans that worker rather than failing it over. + "LOCALAI_REGISTER_TO="+c.FrontendURL(c.registrarFor(i)), "LOCALAI_NODE_NAME="+name, "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, "LOCALAI_NATS_URL="+c.opts.NatsURL, @@ -322,6 +339,23 @@ func (c *Cluster) WorkerName(i int) string { return c.workers[i].Name } +// registrarFor is the frontend index worker i registers and heartbeats with. +// +// It is read at spawn time and baked into the worker's environment, so it is +// also the answer to "which replica's death orphans this worker". +func (c *Cluster) registrarFor(worker int) int { + if !c.opts.SpreadWorkerRegistrations || c.opts.Frontends < 1 { + return 0 + } + return worker % c.opts.Frontends +} + +// WorkerRegistrar is registrarFor, exported so a spec can say which replica it +// is about to kill relative to a worker instead of re-deriving the rule. +func (c *Cluster) WorkerRegistrar(worker int) int { + return c.registrarFor(worker) +} + // Stop terminates every process and removes the work directory. Logs survive in // LogDir, which the caller owns. func (c *Cluster) Stop() { diff --git a/tests/e2e/distributed/cluster_baseline_test.go b/tests/e2e/distributed/cluster_baseline_test.go index 662f93b38835..6cbaaa0ea915 100644 --- a/tests/e2e/distributed/cluster_baseline_test.go +++ b/tests/e2e/distributed/cluster_baseline_test.go @@ -116,7 +116,11 @@ func mockBackendBinary() string { // startCluster brings up a cluster against a freshly provisioned database and // registers cleanup, including a log dump on failure. -func startCluster(frontends, workers int) *cluster.Cluster { +// +// customise runs against the assembled Options immediately before Start, for +// the one spec that needs a non-default topology. It is variadic so every +// existing caller keeps the plain two-argument form and the default shape. +func startCluster(frontends, workers int, customise ...func(*cluster.Options)) *cluster.Cluster { GinkgoHelper() // Resolved before SetupInfra so a missing binary skips without having paid @@ -137,7 +141,7 @@ func startCluster(frontends, workers int) *cluster.Cluster { Expect(os.MkdirAll(logDir, 0o755)).To(Succeed()) } - c, err := cluster.Start(cluster.Options{ + options := cluster.Options{ Binary: binary, MockBackend: mockBackend, PGDSN: infra.PGURL, @@ -145,7 +149,12 @@ func startCluster(frontends, workers int) *cluster.Cluster { LogDir: logDir, Frontends: frontends, Workers: workers, - }) + } + for _, apply := range customise { + apply(&options) + } + + c, err := cluster.Start(options) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { diff --git a/tests/e2e/distributed/cluster_failover_test.go b/tests/e2e/distributed/cluster_failover_test.go new file mode 100644 index 000000000000..d854baa66c7c --- /dev/null +++ b/tests/e2e/distributed/cluster_failover_test.go @@ -0,0 +1,280 @@ +package distributed_test + +import ( + "fmt" + + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/tests/e2e/distributed/cluster" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The two sentinels below are returned by statusOf in place of a node status. +// Neither can ever equal one of the nodes.Status* constants, which is the whole +// point: every assertion in this file compares against a real status with +// Equal, so an unreachable frontend or a vanished row fails the assertion and +// names itself rather than quietly satisfying it. +// +// This is not hypothetical. The obvious way to write "the dead worker is gone" +// is ShouldNot(ContainElement(name)) over a list of healthy names, and the +// probe returns an empty list on any error, so an expired session, a 401 at the +// second replica or a decode failure all satisfy that matcher. The spec would +// go green having observed nothing at all. +const ( + statusUnreachable = "" + statusAbsent = "" +) + +const ( + // orphanEvictionWindow is how long a spec watches a roster to be sure the + // system had a real chance to evict a node and chose not to. + // + // It is sized from the only eviction path there is. Node liveness is + // heartbeat freshness: the health monitor wakes every HealthCheckInterval + // (15s) and marks any node whose last heartbeat is older than + // StaleNodeThreshold (60s) offline (core/services/nodes/health.go, defaults + // in core/config/distributed_config.go). Neither is settable from the CLI, + // so 60s + one 15s tick = 75s is the worst case and cannot be shortened. + // + // Measured rather than assumed: a worker whose registrar was killed goes + // offline at both surviving replicas at t=74.2s. A window shorter than that + // would be the classic false green, a Consistently that passes because + // nothing has had time to happen yet. 100s clears the measured latency by a + // third. + orphanEvictionWindow = "100s" + rosterPollInterval = "2s" + + // workerDeathTimeout bounds the wait for a killed worker to be marked + // offline. Roughly twice the measured 74.3s, which absorbs a health tick + // landing just before the kill plus a slow CI runner. + workerDeathTimeout = "150s" + + // settledStatusWindow is how long an observed status has to hold before the + // spec believes it. A killed worker does not go straight to offline: it + // flaps to unhealthy at ~8s and back to healthy at ~14s (see the note in + // the worker-death spec), so a status has to outlast that transient and two + // further 15s health ticks to count as the settled state. + settledStatusWindow = "45s" + + // restartRehydrationTimeout bounds the wait for a cold-restarted replica to + // answer with the roster. The restart itself measured 1.0s; the budget is + // for a loaded CI runner, not for a slow code path. + restartRehydrationTimeout = "60s" + + // frontendExitTimeout bounds the wait for a signalled frontend to be + // collected. SIGTERM measured 0.2s. It is polled rather than sampled + // because FrontendAlive reports true for the zombie window between the + // child exiting and the reaper calling waitid. + frontendExitTimeout = "30s" + frontendExitPoll = "200ms" +) + +// statusOf refreshes the roster at the probe's frontend and returns the status +// that frontend reports for name. +// +// It shares rosterProbe's lastErr/lastSeen so describe() still explains a +// failure, but unlike healthyNames it never collapses an error into an empty +// result: the caller is comparing against an exact status, so an error has to +// be a value that no assertion can accept. +func (p *rosterProbe) statusOf(name string) string { + var roster []node + if err := p.cluster.GetJSON(p.client, p.frontend, "/api/nodes", &roster); err != nil { + p.lastErr = err + return statusUnreachable + } + p.lastErr = nil + p.lastSeen = roster + for _, n := range roster { + if n.Name == name { + return n.Status + } + } + return statusAbsent +} + +// explain builds a lazy failure description. +// +// Gomega formats a (string, args...) description as soon as the assertion is +// constructed, which for an Eventually or a Consistently is before anything has +// gone wrong; the roster it quoted would be the one from before the wait. A +// func() string is called only on failure, so describe() reports the last +// observation the assertion actually made. +func (p *rosterProbe) explain(format string, args ...any) func() string { + return func() string { + return fmt.Sprintf(format, args...) + ": " + p.describe() + } +} + +var _ = Describe("Cluster failover", Label("Distributed"), Label("Cluster"), func() { + It("keeps a healthy worker in the roster when a peer replica dies", func() { + // Two replicas, one worker. The worker registers and heartbeats with + // frontend 0 only (the harness default), so frontend 1 is a replica it + // has never spoken to. + c := startCluster(2, 1) + worker := c.WorkerName(0) + + // One session for the whole cluster: register/login/token-login/password + // share a five-per-minute-per-IP budget at every frontend + // (core/http/routes/auth.go:190) and everything here comes from + // 127.0.0.1. The cookie is valid at both replicas because sessions live + // in the shared Postgres and the harness pins one HMAC secret. + client, err := c.AdminSession(0) + Expect(err).ToNot(HaveOccurred()) + + survivor := newRosterProbe(c, client, 0) + Eventually(survivor.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(worker), survivor.describe) + + // Kill the replica this worker never registered with. That choice is + // what makes the assertion below mean anything. + // + // Killing frontend 0 instead would sever the worker's only heartbeat + // path, because the heartbeat loop posts to the URL it was handed at + // boot and never re-resolves it; the worker is then genuinely orphaned + // and IS evicted, at a measured 74.2s. A spec written that way can only + // pass by watching for less time than the eviction takes. + Expect(c.WorkerRegistrar(0)).ToNot(Equal(1), + "this spec kills frontend 1 precisely because worker 0 does not depend on it") + Expect(c.KillFrontend(1)).To(Succeed()) + Eventually(func() bool { return c.FrontendAlive(1) }, frontendExitTimeout, frontendExitPoll). + Should(BeFalse(), "frontend 1 did not die, so nothing below is a failover assertion") + + // The survivor must keep answering, and must keep the worker healthy. + // + // A GET that returns 200 with a decodable roster is the "keeps serving" + // half; statusUnreachable would fail this matcher. The window outlasts + // the full 75s stale-plus-one-tick eviction path, so an implementation + // that reacted to a dead peer by sweeping its nodes, by resetting + // heartbeats, or by marking the whole roster stale would be caught + // whether it reacted immediately or on a health tick. + Consistently(survivor.statusOf, orphanEvictionWindow, rosterPollInterval). + WithArguments(worker). + Should(Equal(nodes.StatusHealthy), + survivor.explain("killing a peer replica must not disturb a worker that never depended on it")) + }) + + It("rediscovers the worker from shared state after a cold rolling restart", func() { + c := startCluster(2, 1) + worker := c.WorkerName(0) + + client, err := c.AdminSession(0) + Expect(err).ToNot(HaveOccurred()) + + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(worker), probe.describe) + registeredID := probe.idOf(worker) + Expect(registeredID).ToNot(BeEmpty(), "frontend 0 reported the worker without a registration ID") + + // The rolling-update shape: drain, wait for the process to actually go, + // then bring the replacement up. Restarting without that wait would + // SIGKILL the replica mid-drain and silently turn this into the crash + // case. + Expect(c.StopFrontendGracefully(0)).To(Succeed()) + Eventually(func() bool { return c.FrontendAlive(0) }, frontendExitTimeout, frontendExitPoll). + Should(BeFalse(), "frontend 0 ignored SIGTERM, so the restart below would be a SIGKILL mid-drain") + + // RestartFrontend wipes the replica's data directory, so the process + // that comes back has no local memory of the cluster. Everything the + // assertions below observe has to come out of the shared Postgres. + Expect(c.RestartFrontend(0)).To(Succeed()) + + restarted := newRosterProbe(c, client, 0) + Eventually(restarted.statusOf, restartRehydrationTimeout, nodeRosterPoll). + WithArguments(worker). + Should(Equal(nodes.StatusHealthy), + restarted.explain("a replica with an empty data directory must rehydrate the roster from shared state")) + Expect(restarted.idOf(worker)).To(Equal(registeredID), + "the restarted replica invented a new row for the worker instead of resolving the shared one") + + // Rehydration alone is a weak claim: the row was written before the + // restart and would still read healthy for up to 75s even if the + // replacement never accepted another heartbeat. Holding it past that + // window is what proves the worker's heartbeats are landing again, + // which is the part a restart can plausibly break (a replacement on a + // different port, or one that rejects the node id it did not issue). + Consistently(restarted.statusOf, orphanEvictionWindow, rosterPollInterval). + WithArguments(worker). + Should(Equal(nodes.StatusHealthy), + restarted.explain("the worker went stale after the restart, so its heartbeats are not reaching the replacement")) + }) + + It("settles a dead worker to offline on every replica", func() { + c := startCluster(2, 1) + worker := c.WorkerName(0) + + client, err := c.AdminSession(0) + Expect(err).ToNot(HaveOccurred()) + + at0 := newRosterProbe(c, client, 0) + at1 := newRosterProbe(c, client, 1) + Eventually(at0.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(worker), at0.describe) + + Expect(c.KillWorker(0)).To(Succeed()) + + // Assert the settled status, not the absence of a healthy name. + // + // A killed worker does not move monotonically. Measured: healthy until + // ~8s, unhealthy at 8s, healthy again at 14s, offline from 74s. The + // unhealthy blip comes from a liveness probe; the health monitor's + // "heartbeat is still fresh" branch then marks it healthy again + // (core/services/nodes/health.go), and only the stale-heartbeat branch + // reaches MarkOffline. So requiring exactly offline is what pins this to + // the stale-detection path rather than to the transient, which any + // not-healthy or not-present matcher would accept at t=8s. + for _, probe := range []*rosterProbe{at0, at1} { + Eventually(probe.statusOf, workerDeathTimeout, rosterPollInterval). + WithArguments(worker). + Should(Equal(nodes.StatusOffline), + probe.explain("frontend %d never settled the dead worker to offline", probe.frontend)) + } + + // And it has to stay offline. Nothing may resurrect a row for a process + // that no longer exists, and this window covers three health ticks. + for _, probe := range []*rosterProbe{at0, at1} { + Consistently(probe.statusOf, settledStatusWindow, rosterPollInterval). + WithArguments(worker). + Should(Equal(nodes.StatusOffline), + probe.explain("frontend %d flipped the dead worker away from offline", probe.frontend)) + } + }) + + It("converges on one roster when two replicas register a worker each", func() { + // SpreadWorkerRegistrations is what makes this a race between replicas + // rather than two sequential writes through one: worker 0 registers with + // frontend 0 and worker 1 with frontend 1, both during Start, so two + // processes insert into the shared roster at the same moment. + c := startCluster(2, 2, func(o *cluster.Options) { + o.SpreadWorkerRegistrations = true + }) + Expect(c.WorkerRegistrar(0)).ToNot(Equal(c.WorkerRegistrar(1)), + "both workers registered through the same replica, so this spec is not testing a race") + + client, err := c.AdminSession(0) + Expect(err).ToNot(HaveOccurred()) + + at0 := newRosterProbe(c, client, 0) + at1 := newRosterProbe(c, client, 1) + expected := []string{c.WorkerName(0), c.WorkerName(1)} + + // ConsistOf, not ContainElements: it fails on a third entry, which is + // how a registration that raced into two rows would show up. + Eventually(at0.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ConsistOf(expected), at0.describe) + Eventually(at1.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ConsistOf(expected), at1.describe) + + // Same names is not the same roster. Compare the registration ids, which + // is the only way to tell "both replicas read one set of rows" from + // "each replica has its own row per worker that happens to share a + // name". + for _, name := range expected { + id := at0.idOf(name) + Expect(id).ToNot(BeEmpty(), fmt.Sprintf("frontend 0 reported %s without a registration ID", name)) + Expect(at1.idOf(name)).To(Equal(id), + fmt.Sprintf("the replicas disagree on the identity of %s, so they are not sharing one roster", name)) + } + }) +}) From 4e079c964af50b421d456ed82d490ab356e76011 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 19:40:01 +0000 Subject: [PATCH 017/153] test(distributed): prove health checking was alive during the failover windows The two specs that assert a healthy worker stays healthy were pure negatives: they say nothing happened. A cluster whose health checking had wedged, by leaking the advisory lock the monitor takes at health.go:110, would freeze the roster and satisfy both while observing a corpse. Kill the worker once the window closes and require the roster to settle it to offline, so the preceding Consistently is a statement about behaviour rather than about a stopped clock. Applied to the cold-restart spec as well as the peer-death one: a restart is exactly the event that could leave a replacement unable to check anything. Document the hazard that can make an offline assertion hang. The staleness branch skips a node already marked unhealthy (health.go:153-155), a skip meant for nodes an operator took down, which also swallows the flap: an unhealthy mark landing after the heartbeat goes stale means MarkOffline is never called and the node stays unhealthy forever. Name the file and line at the assertion, and have the failure message say so when the roster shows a node stuck there, so a timeout sends the reader to LocalAI rather than to the harness. Stop calling the two-replica registration spec a race. Start spawns workers sequentially and the registrations land about a second apart; it is a shared-roster identity test, and saying otherwise invites someone to trust it for something it does not check. WorkerRegistrar now bound-checks its index like every other index-taking method here. It answered 0 for an out-of-range worker, and 0 is a real frontend index, so the failure mode was a spec killing the wrong replica. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/cluster/cluster.go | 14 ++- tests/e2e/distributed/cluster/failure.go | 13 ++- .../e2e/distributed/cluster_failover_test.go | 102 ++++++++++++++++-- 3 files changed, 117 insertions(+), 12 deletions(-) diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 198ff5cae735..1783d71e16f0 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -352,8 +352,18 @@ func (c *Cluster) registrarFor(worker int) int { // WorkerRegistrar is registrarFor, exported so a spec can say which replica it // is about to kill relative to a worker instead of re-deriving the rule. -func (c *Cluster) WorkerRegistrar(worker int) int { - return c.registrarFor(worker) +// +// It returns an error rather than indexing blindly, like every other exported +// method here that takes an index. Gomega treats the trailing error as one that +// must be nil, so Expect(c.WorkerRegistrar(0)).To(...) reads unchanged at the +// call site while an out-of-range index fails the spec by name instead of +// silently answering 0, which is a real frontend index and would send a spec +// off to kill the wrong replica. +func (c *Cluster) WorkerRegistrar(worker int) (int, error) { + if err := c.checkWorkerIndex(worker); err != nil { + return 0, err + } + return c.registrarFor(worker), nil } // Stop terminates every process and removes the work directory. Logs survive in diff --git a/tests/e2e/distributed/cluster/failure.go b/tests/e2e/distributed/cluster/failure.go index 3f2a1f42ba28..919376f4a550 100644 --- a/tests/e2e/distributed/cluster/failure.go +++ b/tests/e2e/distributed/cluster/failure.go @@ -33,8 +33,8 @@ func (c *Cluster) StopFrontendGracefully(i int) error { // KillWorker SIGKILLs worker i. func (c *Cluster) KillWorker(i int) error { - if i < 0 || i >= len(c.workers) { - return fmt.Errorf("worker %d out of range (cluster has %d)", i, len(c.workers)) + if err := c.checkWorkerIndex(i); err != nil { + return err } return signalProcess(c.workers[i], syscall.SIGKILL) } @@ -160,6 +160,15 @@ func (c *Cluster) checkFrontendIndex(i int) error { return nil } +// checkWorkerIndex is checkFrontendIndex for workers, and exists for the same +// reason: one wording, whichever primitive tripped. +func (c *Cluster) checkWorkerIndex(i int) error { + if i < 0 || i >= len(c.workers) { + return fmt.Errorf("worker %d out of range (cluster has %d)", i, len(c.workers)) + } + return nil +} + func frontendName(i int) string { return fmt.Sprintf("frontend-%d", i) } diff --git a/tests/e2e/distributed/cluster_failover_test.go b/tests/e2e/distributed/cluster_failover_test.go index d854baa66c7c..3e980cb3de75 100644 --- a/tests/e2e/distributed/cluster_failover_test.go +++ b/tests/e2e/distributed/cluster_failover_test.go @@ -106,6 +106,54 @@ func (p *rosterProbe) explain(format string, args ...any) func() string { } } +// explainStuckOffline is explain with one extra diagnosis attached. +// +// An assertion waiting for offline has a failure mode that looks like a harness +// bug and is not one, so the message names it rather than leaving the reader to +// find it. See proveHealthCheckingIsAlive and the comment on the dead-worker +// spec for the mechanism. +func (p *rosterProbe) explainStuckOffline(worker, format string, args ...any) func() string { + return func() string { + message := fmt.Sprintf(format, args...) + ": " + p.describe() + for _, n := range p.lastSeen { + if n.Name != worker || n.Status != nodes.StatusUnhealthy { + continue + } + message += "\n\nThe node is stuck at unhealthy, which is a LocalAI defect rather than " + + "a harness one: core/services/nodes/health.go:153-155 skips MarkOffline for a node " + + "already marked unhealthy, so a node whose unhealthy mark lands after its heartbeat " + + "has gone stale never reaches offline at all. Start there, not here." + } + return message + } +} + +// proveHealthCheckingIsAlive kills a worker and waits for the roster to settle +// it to offline. +// +// It is the terminating positive control for the two specs that assert a +// healthy worker STAYS healthy. On their own those are pure negative +// assertions: a cluster whose health checking had wedged entirely, say by +// leaking the Postgres advisory lock the monitor takes (health.go:110), would +// freeze the roster and satisfy them while observing a corpse. Killing a worker +// afterwards and requiring the roster to react proves the monitor was running +// for the whole window, which is the only thing that makes the preceding +// Consistently a statement about behaviour rather than about a stopped clock. +// +// It costs a full detection cycle, which is why it is a shared helper: the +// wall-clock price should be paid once per spec and explained once. +func proveHealthCheckingIsAlive(c *cluster.Cluster, probe *rosterProbe, workerIndex int) { + GinkgoHelper() + worker := c.WorkerName(workerIndex) + Expect(c.KillWorker(workerIndex)).To(Succeed()) + Eventually(probe.statusOf, workerDeathTimeout, rosterPollInterval). + WithArguments(worker). + Should(Equal(nodes.StatusOffline), + probe.explainStuckOffline(worker, + "frontend %d never reacted to a killed worker, so health checking was not running during the window above and the assertion before this one proved nothing", + probe.frontend)) +} + var _ = Describe("Cluster failover", Label("Distributed"), Label("Cluster"), func() { It("keeps a healthy worker in the roster when a peer replica dies", func() { // Two replicas, one worker. The worker registers and heartbeats with @@ -152,6 +200,10 @@ var _ = Describe("Cluster failover", Label("Distributed"), Label("Cluster"), fun WithArguments(worker). Should(Equal(nodes.StatusHealthy), survivor.explain("killing a peer replica must not disturb a worker that never depended on it")) + + // Everything above is a negative: nothing happened. Prove that the + // survivor was capable of making something happen the whole time. + proveHealthCheckingIsAlive(c, survivor, 0) }) It("rediscovers the worker from shared state after a cold rolling restart", func() { @@ -198,6 +250,12 @@ var _ = Describe("Cluster failover", Label("Distributed"), Label("Cluster"), fun WithArguments(worker). Should(Equal(nodes.StatusHealthy), restarted.explain("the worker went stale after the restart, so its heartbeats are not reaching the replacement")) + + // Same hole as the peer-death spec, and it is worse here: a cold + // restart is exactly the event that could leave a replacement unable to + // run health checks at all, and a frozen roster reads identically to a + // healthy one. This is the assertion that tells the two apart. + proveHealthCheckingIsAlive(c, restarted, 0) }) It("settles a dead worker to offline on every replica", func() { @@ -224,11 +282,26 @@ var _ = Describe("Cluster failover", Label("Distributed"), Label("Cluster"), fun // reaches MarkOffline. So requiring exactly offline is what pins this to // the stale-detection path rather than to the transient, which any // not-healthy or not-present matcher would accept at t=8s. + // + // KNOWN HAZARD, read this before blaming the harness for a timeout here. + // The staleness branch skips a node that is already unhealthy + // (core/services/nodes/health.go:153-155, `if node.Status == + // StatusOffline || node.Status == StatusUnhealthy { continue }`). The + // skip exists to stop the monitor re-logging nodes an operator took + // down, but it applies to the flap too: if the transient unhealthy mark + // lands AFTER the heartbeat has already gone stale, rather than at the + // ~8s observed here, MarkOffline is never called and this node stays + // unhealthy forever. This spec would then hang to workerDeathTimeout + // and fail with a roster that looks perfectly ordinary. The ordering + // that triggers it did not occur in any run so far, but nothing + // prevents it, so explainStuckOffline says so in the failure message + // when it sees a node stuck at unhealthy. Fixing it is LocalAI work, + // not test work. for _, probe := range []*rosterProbe{at0, at1} { Eventually(probe.statusOf, workerDeathTimeout, rosterPollInterval). WithArguments(worker). Should(Equal(nodes.StatusOffline), - probe.explain("frontend %d never settled the dead worker to offline", probe.frontend)) + probe.explainStuckOffline(worker, "frontend %d never settled the dead worker to offline", probe.frontend)) } // And it has to stay offline. Nothing may resurrect a row for a process @@ -242,15 +315,28 @@ var _ = Describe("Cluster failover", Label("Distributed"), Label("Cluster"), fun }) It("converges on one roster when two replicas register a worker each", func() { - // SpreadWorkerRegistrations is what makes this a race between replicas - // rather than two sequential writes through one: worker 0 registers with - // frontend 0 and worker 1 with frontend 1, both during Start, so two - // processes insert into the shared roster at the same moment. + // SpreadWorkerRegistrations sends worker 0 to frontend 0 and worker 1 to + // frontend 1, so the roster is written through two different replicas. + // + // This is a shared-roster identity test, NOT a concurrency test, and the + // distinction matters because the obvious reading of the spec name is + // the wrong one. Start spawns workers one after another and waits for + // neither, and the registrations land about a second apart in practice; + // there is no synchronisation point and nothing here is tuned to make + // the two writes collide. What it does establish is that a roster + // written through two replicas is one roster and not two: same rows, + // same identities, read back from either process. A genuine concurrent + // registration test would need workers released together against a + // shared barrier, and does not exist yet. c := startCluster(2, 2, func(o *cluster.Options) { o.SpreadWorkerRegistrations = true }) - Expect(c.WorkerRegistrar(0)).ToNot(Equal(c.WorkerRegistrar(1)), - "both workers registered through the same replica, so this spec is not testing a race") + registrar0, err := c.WorkerRegistrar(0) + Expect(err).ToNot(HaveOccurred()) + registrar1, err := c.WorkerRegistrar(1) + Expect(err).ToNot(HaveOccurred()) + Expect(registrar0).ToNot(Equal(registrar1), + "both workers registered through the same replica, so nothing below says anything about two replicas sharing a roster") client, err := c.AdminSession(0) Expect(err).ToNot(HaveOccurred()) @@ -260,7 +346,7 @@ var _ = Describe("Cluster failover", Label("Distributed"), Label("Cluster"), fun expected := []string{c.WorkerName(0), c.WorkerName(1)} // ConsistOf, not ContainElements: it fails on a third entry, which is - // how a registration that raced into two rows would show up. + // how a duplicated row for one worker would show up. Eventually(at0.healthyNames, nodeRosterTimeout, nodeRosterPoll). Should(ConsistOf(expected), at0.describe) Eventually(at1.healthyNames, nodeRosterTimeout, nodeRosterPoll). From c3458e91da0c1f8f38e92713e4e2d8f47a60fb60 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 19:49:19 +0000 Subject: [PATCH 018/153] ci(distributed): run the process-level cluster suite Add test-e2e-cluster and a second CI job that runs it. The cluster specs spawn local-ai as real child processes and kill them, so they need a built binary; keeping them in their own job means the fast in-process suite is not held behind that build. The binary is built with a stubbed core/http/react-ui/dist. A single index.html satisfies the go:embed in core/http/app.go, and this suite drives the HTTP API only, so the job skips a Node and Vite install entirely. The job runs serial and pins --flake-attempts 1. Each Ginkgo process would otherwise get its own PostgreSQL and NATS container while every spec spawns two or three children, and a retry would hide exactly the nondeterminism the suite exists to catch. Measured at 8m39s over three runs, hence a 25 minute job timeout and a 20 minute Ginkgo timeout. LOCALAI_E2E_LOG_DIR points inside the workspace so the per-process logs upload as an artifact on failure; they are the only way to read a cluster failure. LOCALAI_E2E_REQUIRE_BINARIES is set explicitly even though CI already implies it, because a skipped cluster spec is indistinguishable from a passing one and this job's whole value is that it cannot go green without starting a cluster. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .github/workflows/tests-e2e-distributed.yml | 102 ++++++++++++++++++++ Makefile | 23 +++++ 2 files changed, 125 insertions(+) diff --git a/.github/workflows/tests-e2e-distributed.yml b/.github/workflows/tests-e2e-distributed.yml index 4b41f72b5014..3cb4268063d8 100644 --- a/.github/workflows/tests-e2e-distributed.yml +++ b/.github/workflows/tests-e2e-distributed.yml @@ -79,3 +79,105 @@ jobs: detached: true connect-timeout-seconds: 180 limit-access-to-actor: true + + tests-e2e-cluster: + runs-on: ubuntu-latest + # Advisory for the same reason as the job above: master has no branch + # protection, so a failure here is a visible red X rather than a blocked + # merge. That is a repository-settings property, not a YAML key. The key + # that looks like it says "advisory" instead flips the run's conclusion to + # success, which hides the failure rather than flagging it, so it appears in + # none of this repo's workflows and must not be added here. + # + # Separate job from tests-e2e-distributed so the fast in-process suite is + # not held behind a Go build of local-ai. Serial on purpose: each Ginkgo + # process would get its own PostgreSQL and NATS container and each spec + # spawns two or three local-ai children, so --procs on an unmeasured runner + # is a change to make with numbers, not by default. + # + # Measured at 8m39s over three consecutive runs (509.1s / 509.8s / 512.3s). + # Three specs sit at ~167s each because they wait out a 60s staleness + # threshold plus a 15s reconcile tick. Do not shorten those windows to make + # this job faster: the wait is what stops the assertions from passing before + # the system could have reacted, which was a real false green earlier on. + timeout-minutes: 25 + steps: + - name: Clone + uses: actions/checkout@v7 + with: + submodules: true + - name: Configure apt mirror on runner + uses: ./.github/actions/configure-apt-mirror + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.26.0' + cache: false + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential libopus-dev + - name: Proto Dependencies + run: | + curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \ + unzip -j -d /usr/local/bin protoc.zip bin/protoc && \ + rm protoc.zip + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + PATH="$PATH:$HOME/go/bin" make protogen-go + - name: Stub the embedded React UI + # core/http/react-ui/dist is gitignored and built by Node, but this + # suite drives the HTTP API and never the UI, which has its own e2e + # suite. A single index.html satisfies the //go:embed react-ui/dist/* + # in core/http/app.go, so the job skips a full Node and Vite install. + run: | + mkdir -p core/http/react-ui/dist + printf 'stub\n' > core/http/react-ui/dist/index.html + - name: Build local-ai + # Not `make build`: that target pulls in the React UI build. The specs + # exec this binary directly via LOCALAI_E2E_BINARY. + run: | + PATH="$PATH:$HOME/go/bin" go build -o local-ai ./cmd/local-ai + - name: Pre-pull test images + # Same reasoning as the job above: pulling here keeps container-start + # timing out of the spec timeouts and makes a registry outage read as a + # setup failure rather than a test failure. + run: | + docker pull postgres:16-alpine + docker pull nats:2-alpine + - name: Cluster E2E + env: + LOCALAI_E2E_BINARY: ${{ github.workspace }}/local-ai + # Must live under the workspace so the upload step below can reach it. + # The harness defaults to GinkgoT().TempDir(), which lands under + # TMPDIR and would leave the artifact glob matching nothing. + LOCALAI_E2E_LOG_DIR: ${{ github.workspace }}/cluster-logs + # Belt and braces: the harness already fails rather than skips when CI + # is set, and GitHub Actions always sets CI. Stating it here means a + # future edit to that default cannot silently turn this job into one + # that passes without ever starting a cluster, since a skipped cluster + # spec is indistinguishable from a passing one. + LOCALAI_E2E_REQUIRE_BINARIES: "true" + # See the job above: the runner is ephemeral, so the reaper buys + # nothing and would pull a third, unpinned Docker Hub image mid-suite. + TESTCONTAINERS_RYUK_DISABLED: "true" + run: | + PATH="$PATH:$HOME/go/bin" make test-e2e-cluster + - name: Upload process logs + # The per-process logs are the only way to read a cluster failure: the + # Ginkgo output says which assertion failed, not what the four child + # processes were doing. Without this a red job is undebuggable. + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: cluster-process-logs + path: cluster-logs/**/*.log + if-no-files-found: ignore + retention-days: 7 + - name: Setup tmate session if tests fail + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3.23 + with: + detached: true + connect-timeout-seconds: 180 + limit-access-to-actor: true diff --git a/Makefile b/Makefile index dd878a3e3ea3..65b826b5e1e8 100644 --- a/Makefile +++ b/Makefile @@ -351,10 +351,33 @@ DISTRIBUTED_TEST_FLAKES?=1 # Distributed architecture e2e (PostgreSQL + NATS via testcontainers). # Includes NatsJWT specs (JWT-enabled NATS). Requires Docker. # VLLMMultinode is excluded here; use test-e2e-vllm-multinode for that. +# Cluster is excluded too and runs in test-e2e-cluster below, which needs a +# built binary. The argument-validation specs under tests/e2e/distributed/cluster +# carry Label("Distributed") only, so they run here and not there, on purpose. test-e2e-distributed: protogen-go @echo 'Running distributed e2e tests (label Distributed, incl. NatsJWT)' $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode && !Cluster' --flake-attempts $(DISTRIBUTED_TEST_FLAKES) --timeout=40m -v -r ./tests/e2e/distributed +# Cluster e2e: runs local-ai as real child processes (frontend replicas + +# workers) against PostgreSQL and NATS, and kills them to assert failover. +# Needs a built ./local-ai (or LOCALAI_E2E_BINARY) plus the mock backend. +# +# The label split is deliberate and asymmetric. --label-filter='Cluster' +# selects only the specs that spawn processes: the baseline and failover specs +# in tests/e2e/distributed. The argument-validation specs in +# tests/e2e/distributed/cluster carry Label("Distributed") alone, so they stay +# in test-e2e-distributed above, where they belong: they exercise the harness's +# own option handling, need no binary, no PostgreSQL and no NATS, and run in +# milliseconds. Ginkgo therefore reports 0 of 8 specs run for that package here. +# That zero is correct, not a filter bug. +# +# --flake-attempts is pinned to 1 rather than $(DISTRIBUTED_TEST_FLAKES), and +# should stay there: this suite exists to catch nondeterministic cluster +# behaviour, and a retry turns exactly that signal into a green run. +test-e2e-cluster: protogen-go build-mock-backend + @echo 'Running cluster e2e tests (label Cluster, real local-ai processes)' + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Cluster' --flake-attempts 1 --timeout=20m -v -r ./tests/e2e/distributed + # vLLM multi-node DP smoke (CPU). Builds local-ai:tests and the # cpu-vllm backend from the current working tree, then drives a # head + headless follower via testcontainers-go and asserts a chat From 8e8a5d3a2bb12086cc429e575f4703695a5f3e34 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 20:00:14 +0000 Subject: [PATCH 019/153] ci(distributed): fail the cluster job when it selects no specs Ginkgo exits 0 when a label filter matches nothing, so a refactor that renamed or dropped Label("Cluster") would have left the job reporting "Test Suite Passed" having started no cluster. LOCALAI_E2E_REQUIRE_BINARIES does not cover that case: it only fires inside a spec that is already running. Add --fail-on-empty to both distributed targets. Drop -r from test-e2e-cluster while here. All six Cluster specs live in the top-level package, and the cluster subpackage contributes nothing under this filter by design, so recursing only widened the blast radius. test-e2e- distributed keeps -r: it must reach the eight argument-validation specs in that subpackage. Raise the cluster job to 45 minutes, matching its sibling. The 20 minute Ginkgo timeout bounds the suite alone; the job timeout must also cover setup, which is the larger and more variable half here: cold-cache module download, protoc and protogen-go, a full build of ./cmd/local-ai and a separate test compile, realistically 8-12 minutes on a 4-vCPU runner. At 25 minutes the runner would have hard-killed the job before Ginkgo could report which spec hung, which is the red-with-no-evidence outcome that gets suites disabled. Also move upload-artifact to @v7 with the rest of the repo, and note on the react-ui stub step that it must go if a spec ever asserts on a UI asset, since a developer box has a real dist/ and would not catch that locally. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .github/workflows/tests-e2e-distributed.yml | 32 ++++++++++++++++----- Makefile | 25 +++++++++------- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests-e2e-distributed.yml b/.github/workflows/tests-e2e-distributed.yml index 3cb4268063d8..c69ae7924553 100644 --- a/.github/workflows/tests-e2e-distributed.yml +++ b/.github/workflows/tests-e2e-distributed.yml @@ -95,12 +95,26 @@ jobs: # spawns two or three local-ai children, so --procs on an unmeasured runner # is a change to make with numbers, not by default. # - # Measured at 8m39s over three consecutive runs (509.1s / 509.8s / 512.3s). - # Three specs sit at ~167s each because they wait out a 60s staleness - # threshold plus a 15s reconcile tick. Do not shorten those windows to make - # this job faster: the wait is what stops the assertions from passing before - # the system could have reacted, which was a real false green earlier on. - timeout-minutes: 25 + # The two timeouts bound different things and are not alternatives. Ginkgo's + # --timeout=20m bounds the SUITE only; this job timeout must additionally + # cover setup, which here is the larger and more variable half: submodule + # checkout, apt, protoc plus two go installs plus protogen-go, a cold-cache + # module download (cache: false), a full go build of ./cmd/local-ai, and a + # separate ginkgo test compile. That build alone is ~316s of CPU, so on a + # 4-vCPU runner setup is realistically 8-12 minutes. + # + # 45 minutes therefore, matching the sibling job. A tighter number does not + # make a hang fail faster, it just moves the kill from Ginkgo, which prints + # which spec hung, to the runner, which prints nothing: a red job with no + # evidence, which is how a suite gets disabled rather than fixed. + # + # The suite itself is 8m39s over three consecutive runs (509.1s / 509.8s / + # 512.3s) on a developer box, and will be slower here. Three specs sit at + # ~167s each because they wait out a 60s staleness threshold plus a 15s + # reconcile tick. Do not shorten those windows to make this job faster: the + # wait is what stops the assertions from passing before the system could + # have reacted, which was a real false green earlier on. + timeout-minutes: 45 steps: - name: Clone uses: actions/checkout@v7 @@ -130,6 +144,10 @@ jobs: # suite drives the HTTP API and never the UI, which has its own e2e # suite. A single index.html satisfies the //go:embed react-ui/dist/* # in core/http/app.go, so the job skips a full Node and Vite install. + # If a cluster spec ever asserts on a UI asset, this step must go and + # the real build come back: a developer box has a real dist/, so such a + # spec would pass locally and fail only here, or worse be served the + # stub and pass in both places. run: | mkdir -p core/http/react-ui/dist printf 'stub\n' > core/http/react-ui/dist/index.html @@ -168,7 +186,7 @@ jobs: # Ginkgo output says which assertion failed, not what the four child # processes were doing. Without this a red job is undebuggable. if: ${{ failure() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: cluster-process-logs path: cluster-logs/**/*.log diff --git a/Makefile b/Makefile index 65b826b5e1e8..85a2509951a2 100644 --- a/Makefile +++ b/Makefile @@ -354,29 +354,34 @@ DISTRIBUTED_TEST_FLAKES?=1 # Cluster is excluded too and runs in test-e2e-cluster below, which needs a # built binary. The argument-validation specs under tests/e2e/distributed/cluster # carry Label("Distributed") only, so they run here and not there, on purpose. +# -r stays because of those: they are in a subpackage this target must reach. +# --fail-on-empty because ginkgo exits 0 when a label filter matches nothing, so +# without it a rename of the label would turn this target into a silent no-op +# that still reports "Test Suite Passed". test-e2e-distributed: protogen-go @echo 'Running distributed e2e tests (label Distributed, incl. NatsJWT)' - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode && !Cluster' --flake-attempts $(DISTRIBUTED_TEST_FLAKES) --timeout=40m -v -r ./tests/e2e/distributed + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode && !Cluster' --fail-on-empty --flake-attempts $(DISTRIBUTED_TEST_FLAKES) --timeout=40m -v -r ./tests/e2e/distributed # Cluster e2e: runs local-ai as real child processes (frontend replicas + # workers) against PostgreSQL and NATS, and kills them to assert failover. # Needs a built ./local-ai (or LOCALAI_E2E_BINARY) plus the mock backend. # -# The label split is deliberate and asymmetric. --label-filter='Cluster' -# selects only the specs that spawn processes: the baseline and failover specs -# in tests/e2e/distributed. The argument-validation specs in -# tests/e2e/distributed/cluster carry Label("Distributed") alone, so they stay -# in test-e2e-distributed above, where they belong: they exercise the harness's -# own option handling, need no binary, no PostgreSQL and no NATS, and run in -# milliseconds. Ginkgo therefore reports 0 of 8 specs run for that package here. -# That zero is correct, not a filter bug. +# The argument-validation specs in tests/e2e/distributed/cluster deliberately +# stay in test-e2e-distributed above: they need no binary, no PostgreSQL and no +# NATS, so no -r here and that package is simply out of scope. +# +# --fail-on-empty is load-bearing, not tidiness. Ginkgo exits 0 when a label +# filter selects nothing, so without it a refactor that renames or drops +# Label("Cluster") leaves this target reporting "Test Suite Passed" having +# started no cluster at all. LOCALAI_E2E_REQUIRE_BINARIES does not cover this: +# it only fires inside a spec that is actually running. # # --flake-attempts is pinned to 1 rather than $(DISTRIBUTED_TEST_FLAKES), and # should stay there: this suite exists to catch nondeterministic cluster # behaviour, and a retry turns exactly that signal into a green run. test-e2e-cluster: protogen-go build-mock-backend @echo 'Running cluster e2e tests (label Cluster, real local-ai processes)' - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Cluster' --flake-attempts 1 --timeout=20m -v -r ./tests/e2e/distributed + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Cluster' --fail-on-empty --flake-attempts 1 --timeout=20m -v ./tests/e2e/distributed # vLLM multi-node DP smoke (CPU). Builds local-ai:tests and the # cpu-vllm backend from the current working tree, then drives a From 1dae0ceefce66e3b8e96a86907bde9ea63ddffda Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 20:11:49 +0000 Subject: [PATCH 020/153] docs: document the distributed and cluster e2e suites Two Make targets, a flake-budget variable and two environment variables landed with no way to discover them. CONTRIBUTING.md now tells a contributor how to run both suites, what each costs and which variables steer the cluster one. .agents/building-and-testing.md records the decisions that are easy to undo by accident: suite-scoped containers, the shared NATS bus and what that means for a new spec, BeforeSuite over SynchronizedBeforeSuite, the label split, --fail-on-empty, the binary gate, the flake budget of 1, the coverage exclusion, and why the cluster suite's long waits must not be shortened. .agents/ci-caching.md lists tests-e2e-distributed.yml in its paths-ignore inventory; the workflow already pointed readers there, so the cross-reference was dangling. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .agents/building-and-testing.md | 19 +++++++++++++++++++ .agents/ci-caching.md | 4 ++-- CONTRIBUTING.md | 31 +++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 021d555ec993..f82445f7e5fa 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -45,3 +45,22 @@ Rules (both gates): - **Don't weaken the gate:** never hand-lower a baseline or widen a tolerance to turn a red gate green. The ratchet only moves up. - If a change drops coverage, **add tests** (sort `coverage-summary.json` by line% ascending to find untested code) rather than editing the baseline. When coverage legitimately rises, commit the regenerated baseline (`make test-coverage-baseline` / `test-ui-coverage-baseline`). - The Go gate is **strict — no tolerance**; `covermode=atomic` keeps it deterministic. The UI gate keeps a small tolerance only because its e2e coverage isn't. + +## Distributed-mode test suites + +Two suites cover distributed mode (frontend replicas, worker nodes, PostgreSQL, NATS), split by a Ginkgo label: + +- `make test-e2e-distributed` runs `Distributed && !VLLMMultinode && !Cluster` over `./tests/e2e/distributed` recursively. Services are wired directly into the test binary. ~240 specs, ~75s. +- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. 6 specs, 8m39s measured over three consecutive runs (509.1s / 509.8s / 512.3s). + +Both jobs live in `.github/workflows/tests-e2e-distributed.yml`, PR-triggered with a `paths-ignore` filter (see [.agents/ci-caching.md](ci-caching.md)) and `timeout-minutes: 45` each. They are advisory only because `master` carries no branch protection, which is a repository setting and not a YAML key: `continue-on-error: true` would flip the run's *conclusion* to success and hide the failure, so it is not used. + +- **Containers are suite-scoped, not spec-scoped.** `SetupInfra` used to start a PostgreSQL (~10s) and a NATS (~3.5s) per spec. Across the 213 specs behind it that was roughly **48 minutes of pure container startup per run**, which is why this suite was never in CI. Containers now start once in `BeforeSuite` and each spec gets its own database via `CREATE DATABASE` (~67ms), which is what the `dbName` argument was always describing. Adding a spec needs no change: call `SetupInfra("some-name")` as before, the name is a prefix and a counter keeps it unique. +- **Consequence for new specs:** the NATS bus is now *shared* within a Ginkgo process, so a wildcard subscriber can observe another spec's traffic. Filter assertions on an identifier your spec owns (a node ID, a job ID) instead of counting everything on `jobs.*.progress`, and verify the spec with `--randomize-all`. +- **`BeforeSuite`, not `SynchronizedBeforeSuite`.** Under `ginkgo -p` each process then gets its own container pair, keeping NATS subjects isolated per process. A single shared NATS across parallel processes would let specs on different processes see each other's messages on the same subject. +- **The label split.** The 8 argument-validation specs under `tests/e2e/distributed/cluster/` carry `Label("Distributed")` only, on purpose: they need no binary, no PostgreSQL and no NATS, so they belong in the fast job. That is why `test-e2e-distributed` keeps `-r` (it must reach the subpackage) and `test-e2e-cluster` deliberately does **not** (the subpackage is out of its scope). +- **`--fail-on-empty` is load-bearing on both targets.** Ginkgo exits 0 when a label filter selects nothing, so without it a refactor that renames or drops `Label("Cluster")` leaves the target reporting "Test Suite Passed" having started no cluster at all. `LOCALAI_E2E_REQUIRE_BINARIES` does not cover this case: it only fires inside a spec that is actually running. +- **The binary gate.** `localAIBinary()` and `mockBackendBinary()` **fail** rather than skip when `CI` is set, or when `LOCALAI_E2E_REQUIRE_BINARIES` is truthy; `LOCALAI_E2E_REQUIRE_BINARIES=0` (also `off`/`no`/`disabled`) forces skipping even under CI. The polarity is deliberate: in CI a skipped cluster spec is indistinguishable from a passing one, because Ginkgo exits 0 on skips. Locally a missing binary still just skips, since `CI` is unset in an ordinary shell. +- **Flake budget:** `DISTRIBUTED_TEST_FLAKES` defaults to **1**, not the repo-wide `TEST_FLAKES=5`, and `test-e2e-cluster` pins `--flake-attempts 1` outright. These suites exist to surface nondeterminism, so a retry converts exactly that signal into a green run. Raise it locally when bisecting something unrelated, not in the Makefile. +- **Coverage:** `tests/e2e/distributed` is excluded from the coverage roots (`COVERAGE_E2E_ROOTS = ./tests/e2e`, run non-recursively), and so is the `cluster` helper package beneath it. Neither suite moves the baseline, so production code that these suites are the only cover for reads as **uncovered**. Unit tests for such code belong under `./core/...` with `testutil.SetupTestDB()`. +- **Do not shorten the cluster suite's waits.** Three of its six specs sit at ~167s each because they wait out a 60s staleness threshold plus a 15s health-check tick. That wait is what stops the assertions from passing before the system could have reacted, which was a real false green earlier on. If the job has to get faster, the levers are CI concurrency or making the thresholds configurable, not shorter waits. diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 6742049e68ff..244bb4549fcb 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -153,7 +153,7 @@ This is worth more than it looks. Measured over the week to 2026-07-30, **97% of The volume is real: 13 gallery-only PRs merged that week with 10 open at once, and 78 of the 137 PRs opened were bot-generated. -`paths-ignore` on the PR trigger of `image-pr.yml` (7 jobs), `build-test.yaml` (3), `lint.yml` (2) and `tests-e2e.yml` (1) drops 13 of those 20. The excluded set: +`paths-ignore` on the PR trigger of `image-pr.yml` (7 jobs), `build-test.yaml` (3), `lint.yml` (2) and `tests-e2e.yml` (1) drops 13 of those 20. `tests-e2e-distributed.yml` (2 jobs) landed after that measurement and carries the same exclusion set for the same reason: its dependency graph is 99 packages, so an allowlist of paths would silently stop guarding the moment code moved, while a diff confined to the paths below provably cannot reach it. The excluded set: | Path | Why no image or Go build can see it | |---|---| @@ -192,7 +192,7 @@ What still runs, and why it has to: Two properties this relies on: - `paths-ignore` skips a run only when **every** changed file matches, so a PR touching the gallery *and* Go code still runs everything. That is what makes the exclusion safe rather than a hole. -- `master` carries no branch protection and no rulesets, so a skipped workflow reports no status and nothing waits on it. If required status checks are ever introduced, these four entries must be excluded from the required set or PRs will hang on "Expected — Waiting for status to be reported". +- `master` carries no branch protection and no rulesets, so a skipped workflow reports no status and nothing waits on it. If required status checks are ever introduced, these five entries must be excluded from the required set or PRs will hang on "Expected — Waiting for status to be reported". ### `image.yml` on master push is gated too, by a job rather than a path filter diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 655695cfcc1b..cebe3661e70c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -265,6 +265,37 @@ The e2e tests run LocalAI in a Docker container and exercise the API: make test-e2e ``` +### Running distributed-mode tests + +Distributed mode (several frontend replicas, worker nodes, PostgreSQL and NATS) has two suites. Both bring up their PostgreSQL and NATS with testcontainers, so Docker has to be available: + +```bash +make test-e2e-distributed # in-process: services wired directly into the test binary +make test-e2e-cluster # process-level: real local-ai child processes +``` + +`make test-e2e-distributed` is the fast one (around 240 specs in roughly 75 seconds). It starts one PostgreSQL and one NATS for the whole run and gives each spec its own database. It retries a failing spec once (`DISTRIBUTED_TEST_FLAKES`, default 1, deliberately lower than the repo-wide `TEST_FLAKES=5`), because this suite exists to catch nondeterministic cluster behaviour and retrying hides exactly that. + +`make test-e2e-cluster` runs `local-ai` as real child processes, one per frontend replica and one per worker, so a spec can kill a replica and assert what the survivors do. Budget about 8m40s: three of its six specs wait out real staleness and health-check windows. It needs a built binary and the mock backend: + +```bash +make build build-mock-backend +make test-e2e-cluster +``` + +Two environment variables steer it: + +| Variable | Purpose | +|---|---| +| `LOCALAI_E2E_BINARY` | path to the `local-ai` binary (default: `local-ai` in the repository root) | +| `LOCALAI_E2E_LOG_DIR` | directory for the per-process logs (default: a Ginkgo temp dir) | + +Set `LOCALAI_E2E_LOG_DIR` when debugging. A cluster failure is unreadable without the individual frontend and worker logs, and Ginkgo only tells you which assertion failed. + +A missing binary skips the cluster specs locally but fails them whenever `CI` is set, so a build problem cannot turn the CI job green without ever starting a cluster. `LOCALAI_E2E_REQUIRE_BINARIES=1` forces that failing behaviour anywhere; `LOCALAI_E2E_REQUIRE_BINARIES=0` forces the skip back on even under CI. + +Both suites run on pull requests via `.github/workflows/tests-e2e-distributed.yml`. + ### React UI tests and coverage The React UI (`core/http/react-ui/`) is covered by Playwright e2e specs, gated by a **monotonic line-coverage ratchet** (`make test-ui-coverage-check`, run in CI). The metric is non-deterministic — a fast local box reads higher than a slow CI runner for the same code — so a small tolerance is unavoidable. From 235057b4c9807b060afda58271e135070b818799 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 20:21:36 +0000 Subject: [PATCH 021/153] docs: correct the flake-attempts semantics and fill the review gaps --flake-attempts is total attempts, not retries: ginkgo v2.29.0 sets maxAttempts = FlakeAttempts and loops attempt < maxAttempts, and the flag's usage string reads "0 - failed tests are not retried". At 1 there is no retry at all, so "retries a failing spec once" was false in CONTRIBUTING.md and implied in .agents/building-and-testing.md. Both now say each spec runs once, and cite the source so the next reader need not re-derive it. Also restores the React-UI stub rationale, which is load-bearing because a spec asserting on a UI asset passes locally against a real dist/ and is served the stub in CI; explains why 213 and ~240 differ; records that the workflow also triggers on master pushes, where paths-ignore does not apply; and completes the LOCALAI_E2E_REQUIRE_BINARIES value table, including that any unparseable value reads as ON. In .agents/ci-caching.md the stale "13 of those 20" figure now carries its qualifier inline rather than in the following sentence. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .agents/building-and-testing.md | 9 +++++---- .agents/ci-caching.md | 2 +- CONTRIBUTING.md | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index f82445f7e5fa..874e49f8b3b7 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -53,14 +53,15 @@ Two suites cover distributed mode (frontend replicas, worker nodes, PostgreSQL, - `make test-e2e-distributed` runs `Distributed && !VLLMMultinode && !Cluster` over `./tests/e2e/distributed` recursively. Services are wired directly into the test binary. ~240 specs, ~75s. - `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. 6 specs, 8m39s measured over three consecutive runs (509.1s / 509.8s / 512.3s). -Both jobs live in `.github/workflows/tests-e2e-distributed.yml`, PR-triggered with a `paths-ignore` filter (see [.agents/ci-caching.md](ci-caching.md)) and `timeout-minutes: 45` each. They are advisory only because `master` carries no branch protection, which is a repository setting and not a YAML key: `continue-on-error: true` would flip the run's *conclusion* to success and hide the failure, so it is not used. +Both jobs live in `.github/workflows/tests-e2e-distributed.yml`, with `timeout-minutes: 45` each. They trigger on pull requests *and* on every push to `master`; the `paths-ignore` filter (see [.agents/ci-caching.md](ci-caching.md)) sits on the pull-request trigger only, so a master push always runs both. They are advisory only because `master` carries no branch protection, which is a repository setting and not a YAML key: `continue-on-error: true` would flip the run's *conclusion* to success and hide the failure, so it is not used. -- **Containers are suite-scoped, not spec-scoped.** `SetupInfra` used to start a PostgreSQL (~10s) and a NATS (~3.5s) per spec. Across the 213 specs behind it that was roughly **48 minutes of pure container startup per run**, which is why this suite was never in CI. Containers now start once in `BeforeSuite` and each spec gets its own database via `CREATE DATABASE` (~67ms), which is what the `dbName` argument was always describing. Adding a spec needs no change: call `SetupInfra("some-name")` as before, the name is a prefix and a counter keeps it unique. +- **Containers are suite-scoped, not spec-scoped.** `SetupInfra` used to start a PostgreSQL (~10s) and a NATS (~3.5s) per spec. Across the 213 specs behind it that was roughly **48 minutes of pure container startup per run**, which is why this suite was never in CI. (213 rather than the ~240 above: the larger number is everything the label filter selects, the smaller one is just the specs that call `SetupInfra`.) Containers now start once in `BeforeSuite` and each spec gets its own database via `CREATE DATABASE` (~67ms), which is what the `dbName` argument was always describing. Adding a spec needs no change: call `SetupInfra("some-name")` as before, the name is a prefix and a counter keeps it unique. - **Consequence for new specs:** the NATS bus is now *shared* within a Ginkgo process, so a wildcard subscriber can observe another spec's traffic. Filter assertions on an identifier your spec owns (a node ID, a job ID) instead of counting everything on `jobs.*.progress`, and verify the spec with `--randomize-all`. - **`BeforeSuite`, not `SynchronizedBeforeSuite`.** Under `ginkgo -p` each process then gets its own container pair, keeping NATS subjects isolated per process. A single shared NATS across parallel processes would let specs on different processes see each other's messages on the same subject. - **The label split.** The 8 argument-validation specs under `tests/e2e/distributed/cluster/` carry `Label("Distributed")` only, on purpose: they need no binary, no PostgreSQL and no NATS, so they belong in the fast job. That is why `test-e2e-distributed` keeps `-r` (it must reach the subpackage) and `test-e2e-cluster` deliberately does **not** (the subpackage is out of its scope). - **`--fail-on-empty` is load-bearing on both targets.** Ginkgo exits 0 when a label filter selects nothing, so without it a refactor that renames or drops `Label("Cluster")` leaves the target reporting "Test Suite Passed" having started no cluster at all. `LOCALAI_E2E_REQUIRE_BINARIES` does not cover this case: it only fires inside a spec that is actually running. -- **The binary gate.** `localAIBinary()` and `mockBackendBinary()` **fail** rather than skip when `CI` is set, or when `LOCALAI_E2E_REQUIRE_BINARIES` is truthy; `LOCALAI_E2E_REQUIRE_BINARIES=0` (also `off`/`no`/`disabled`) forces skipping even under CI. The polarity is deliberate: in CI a skipped cluster spec is indistinguishable from a passing one, because Ginkgo exits 0 on skips. Locally a missing binary still just skips, since `CI` is unset in an ordinary shell. -- **Flake budget:** `DISTRIBUTED_TEST_FLAKES` defaults to **1**, not the repo-wide `TEST_FLAKES=5`, and `test-e2e-cluster` pins `--flake-attempts 1` outright. These suites exist to surface nondeterminism, so a retry converts exactly that signal into a green run. Raise it locally when bisecting something unrelated, not in the Makefile. +- **The binary gate.** `localAIBinary()` and `mockBackendBinary()` **fail** rather than skip when `CI` is set, or when `LOCALAI_E2E_REQUIRE_BINARIES` is truthy; `LOCALAI_E2E_REQUIRE_BINARIES=0` (also `off`, `no`, `n`, `disabled`, and anything `strconv.ParseBool` reads as false) forces skipping even under CI. **Any value that parses as neither reads as ON**, not off: setting the variable to something meaningless means someone meant to turn the gate on, and reading it as false would quietly restore the silent skip the flag exists to remove. The whole polarity is deliberate, because in CI a skipped cluster spec is indistinguishable from a passing one: Ginkgo exits 0 on skips. Locally a missing binary still just skips, since `CI` is unset in an ordinary shell. +- **Flake budget: no retries at all.** `--flake-attempts` is *total attempts*, not retries (ginkgo v2.29.0 `internal/group.go` sets `maxAttempts = FlakeAttempts` and loops `attempt < maxAttempts`; the flag's own usage string reads "0 - failed tests are not retried"). `DISTRIBUTED_TEST_FLAKES` defaults to **1**, so each spec runs once and a failure is a failure, and `test-e2e-cluster` pins `--flake-attempts 1` outright rather than reading the variable. The repo-wide `TEST_FLAKES=5` means up to five attempts, so up to four retries. These suites exist to surface nondeterminism, and a retry converts exactly that signal into a green run. Raise it locally when bisecting something unrelated, not in the Makefile. - **Coverage:** `tests/e2e/distributed` is excluded from the coverage roots (`COVERAGE_E2E_ROOTS = ./tests/e2e`, run non-recursively), and so is the `cluster` helper package beneath it. Neither suite moves the baseline, so production code that these suites are the only cover for reads as **uncovered**. Unit tests for such code belong under `./core/...` with `testutil.SetupTestDB()`. +- **The cluster job builds against a stubbed React UI.** `core/http/react-ui/dist` is gitignored and built by Node, so the workflow writes a one-line `index.html` there to satisfy the `//go:embed react-ui/dist/*` in `core/http/app.go` and skips a full Node and Vite install. That holds only while the suite drives the HTTP API and never the UI, which has its own e2e suite. A spec that ever asserts on a UI asset would pass locally, where a real `dist/` exists, and be served the stub in CI: if you write one, the stub step has to go and the real build come back. - **Do not shorten the cluster suite's waits.** Three of its six specs sit at ~167s each because they wait out a 60s staleness threshold plus a 15s health-check tick. That wait is what stops the assertions from passing before the system could have reacted, which was a real false green earlier on. If the job has to get faster, the levers are CI concurrency or making the thresholds configurable, not shorter waits. diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 244bb4549fcb..8dc243747384 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -153,7 +153,7 @@ This is worth more than it looks. Measured over the week to 2026-07-30, **97% of The volume is real: 13 gallery-only PRs merged that week with 10 open at once, and 78 of the 137 PRs opened were bot-generated. -`paths-ignore` on the PR trigger of `image-pr.yml` (7 jobs), `build-test.yaml` (3), `lint.yml` (2) and `tests-e2e.yml` (1) drops 13 of those 20. `tests-e2e-distributed.yml` (2 jobs) landed after that measurement and carries the same exclusion set for the same reason: its dependency graph is 99 packages, so an allowlist of paths would silently stop guarding the moment code moved, while a diff confined to the paths below provably cannot reach it. The excluded set: +`paths-ignore` on the PR trigger of `image-pr.yml` (7 jobs), `build-test.yaml` (3), `lint.yml` (2) and `tests-e2e.yml` (1) drops 13 of those 20, measured before `tests-e2e-distributed.yml` (2 jobs) landed. That workflow carries the same exclusion set for the same reason: its dependency graph is 99 packages, so an allowlist of paths would silently stop guarding the moment code moved, while a diff confined to the paths below provably cannot reach it. The excluded set: | Path | Why no image or Go build can see it | |---|---| diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cebe3661e70c..2de815bd3b9b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -274,7 +274,7 @@ make test-e2e-distributed # in-process: services wired directly into the test make test-e2e-cluster # process-level: real local-ai child processes ``` -`make test-e2e-distributed` is the fast one (around 240 specs in roughly 75 seconds). It starts one PostgreSQL and one NATS for the whole run and gives each spec its own database. It retries a failing spec once (`DISTRIBUTED_TEST_FLAKES`, default 1, deliberately lower than the repo-wide `TEST_FLAKES=5`), because this suite exists to catch nondeterministic cluster behaviour and retrying hides exactly that. +`make test-e2e-distributed` is the fast one (around 240 specs in roughly 75 seconds). It starts one PostgreSQL and one NATS for the whole run and gives each spec its own database. It runs each spec exactly once, with no retry: `DISTRIBUTED_TEST_FLAKES` defaults to 1 and feeds ginkgo's `--flake-attempts`, which counts *total attempts*, not retries. That is deliberately below the repo-wide `TEST_FLAKES=5`, because this suite exists to catch nondeterministic cluster behaviour and a retry hides exactly the failure it is meant to catch. Raise it locally when bisecting something unrelated. `make test-e2e-cluster` runs `local-ai` as real child processes, one per frontend replica and one per worker, so a spec can kill a replica and assert what the survivors do. Budget about 8m40s: three of its six specs wait out real staleness and health-check windows. It needs a built binary and the mock backend: @@ -294,7 +294,7 @@ Set `LOCALAI_E2E_LOG_DIR` when debugging. A cluster failure is unreadable withou A missing binary skips the cluster specs locally but fails them whenever `CI` is set, so a build problem cannot turn the CI job green without ever starting a cluster. `LOCALAI_E2E_REQUIRE_BINARIES=1` forces that failing behaviour anywhere; `LOCALAI_E2E_REQUIRE_BINARIES=0` forces the skip back on even under CI. -Both suites run on pull requests via `.github/workflows/tests-e2e-distributed.yml`. +Both suites run in `.github/workflows/tests-e2e-distributed.yml`, on pull requests and on every push to `master`. The `paths-ignore` filter is on the pull-request trigger only, so a master push always runs them. ### React UI tests and coverage From 1e5f5dbe52d9e56a67dc81779c8cacef8e58de7f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 20:47:33 +0000 Subject: [PATCH 022/153] test(distributed): correct the claims the e2e comments make Review of the whole branch found five comments that would send a reader to the wrong place, plus three smaller inaccuracies. Nothing here changes behaviour. The KNOWN RACE note on both backend-log WebSocket handlers said the fix needs an atomic snapshot-plus-subscribe "under the store lock". It does not: BackendLogStore.mu guards only the buffers map, and AppendLine enqueues and fans out under the per-buffer buf.mu. Whoever took the store lock would ship and the race would survive, so both notes now name buf.mu and say what s.mu does and does not exclude. Two comments in the cluster harness quoted Eventually(c.FrontendAlive) .Should(BeFalse()). FrontendAlive takes an index, so Gomega rejects that with "requested 1 arguments but received 0". Both now quote the closure form the specs actually use, and say why the closure is needed. proveHealthCheckingIsAlive claimed to prove the health monitor ran for the whole preceding window. It proves the monitor was alive at the end of it, and inferring backwards needs any wedge to be sticky. In the peer-replica-death spec that inverts: health checks are single-flighted by a session-scoped pg_try_advisory_lock, the spec SIGKILLs the replica that may hold it, and until Postgres reaps the session the survivor acquires nothing and checks nothing silently. Consistently(healthy) can then pass because nothing was checking, with the positive control still succeeding once the lock frees. The doc now states what is proven, names that gap, and says the assertion is a floor rather than a proof. The Makefile still called DISTRIBUTED_TEST_FLAKES a retry count, which is what seeded that error into the two docs just corrected against it, and the workflow called the 15s window a reconcile tick when the mechanism is HealthCheckInterval in the node health monitor. Also: the cluster suite measured 509.1s / 509.8s / 512.3s, so about 8m30s and not the 8m39s/8m40s three files claimed; the dead-worker spec title implied two independent detectors when both probes read one advisory-lock-serialised verdict out of the same row; and the sanitizeDBName length assertion used <= 50, which an empty string also satisfies, where the invariant for an over-long input is exactly 50. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .agents/building-and-testing.md | 2 +- .github/workflows/tests-e2e-distributed.yml | 9 ++-- CONTRIBUTING.md | 2 +- Makefile | 9 ++-- core/http/endpoints/localai/backend_logs.go | 8 +++- core/services/nodes/file_transfer_server.go | 8 +++- tests/e2e/distributed/cluster/failure.go | 18 ++++++-- .../e2e/distributed/cluster_failover_test.go | 46 ++++++++++++++++--- tests/e2e/distributed/dbname_test.go | 3 +- 9 files changed, 80 insertions(+), 25 deletions(-) diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 874e49f8b3b7..eee5123ac82c 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -51,7 +51,7 @@ Rules (both gates): Two suites cover distributed mode (frontend replicas, worker nodes, PostgreSQL, NATS), split by a Ginkgo label: - `make test-e2e-distributed` runs `Distributed && !VLLMMultinode && !Cluster` over `./tests/e2e/distributed` recursively. Services are wired directly into the test binary. ~240 specs, ~75s. -- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. 6 specs, 8m39s measured over three consecutive runs (509.1s / 509.8s / 512.3s). +- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. 6 specs, about 8m30s measured over three consecutive runs (509.1s / 509.8s / 512.3s, so 8m29s to 8m32s). Both jobs live in `.github/workflows/tests-e2e-distributed.yml`, with `timeout-minutes: 45` each. They trigger on pull requests *and* on every push to `master`; the `paths-ignore` filter (see [.agents/ci-caching.md](ci-caching.md)) sits on the pull-request trigger only, so a master push always runs both. They are advisory only because `master` carries no branch protection, which is a repository setting and not a YAML key: `continue-on-error: true` would flip the run's *conclusion* to success and hide the failure, so it is not used. diff --git a/.github/workflows/tests-e2e-distributed.yml b/.github/workflows/tests-e2e-distributed.yml index c69ae7924553..d0cf24875c14 100644 --- a/.github/workflows/tests-e2e-distributed.yml +++ b/.github/workflows/tests-e2e-distributed.yml @@ -108,10 +108,11 @@ jobs: # which spec hung, to the runner, which prints nothing: a red job with no # evidence, which is how a suite gets disabled rather than fixed. # - # The suite itself is 8m39s over three consecutive runs (509.1s / 509.8s / - # 512.3s) on a developer box, and will be slower here. Three specs sit at - # ~167s each because they wait out a 60s staleness threshold plus a 15s - # reconcile tick. Do not shorten those windows to make this job faster: the + # The suite itself is about 8m30s over three consecutive runs (509.1s / + # 509.8s / 512.3s, so 8m29s to 8m32s) on a developer box, and will be slower + # here. Three specs sit at ~167s each because they wait out a 60s staleness + # threshold plus a 15s health-check tick (HealthCheckInterval, in + # core/services/nodes/health.go, not one of the reconcilers). Do not shorten those windows to make this job faster: the # wait is what stops the assertions from passing before the system could # have reacted, which was a real false green earlier on. timeout-minutes: 45 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2de815bd3b9b..69ddbd6f2fc7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -276,7 +276,7 @@ make test-e2e-cluster # process-level: real local-ai child processes `make test-e2e-distributed` is the fast one (around 240 specs in roughly 75 seconds). It starts one PostgreSQL and one NATS for the whole run and gives each spec its own database. It runs each spec exactly once, with no retry: `DISTRIBUTED_TEST_FLAKES` defaults to 1 and feeds ginkgo's `--flake-attempts`, which counts *total attempts*, not retries. That is deliberately below the repo-wide `TEST_FLAKES=5`, because this suite exists to catch nondeterministic cluster behaviour and a retry hides exactly the failure it is meant to catch. Raise it locally when bisecting something unrelated. -`make test-e2e-cluster` runs `local-ai` as real child processes, one per frontend replica and one per worker, so a spec can kill a replica and assert what the survivors do. Budget about 8m40s: three of its six specs wait out real staleness and health-check windows. It needs a built binary and the mock backend: +`make test-e2e-cluster` runs `local-ai` as real child processes, one per frontend replica and one per worker, so a spec can kill a replica and assert what the survivors do. Budget about 8m30s (measured 509.1s / 509.8s / 512.3s over three consecutive runs): three of its six specs wait out real staleness and health-check windows. It needs a built binary and the mock backend: ```bash make build build-mock-backend diff --git a/Makefile b/Makefile index 85a2509951a2..debc5406ff51 100644 --- a/Makefile +++ b/Makefile @@ -342,10 +342,11 @@ run-e2e-aio: protogen-go @echo 'Running e2e AIO tests' $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e-aio -# Flake retries for the distributed suite. Defaults to 1, unlike TEST_FLAKES: -# this suite exists to catch nondeterministic cluster behaviour, and retrying -# hides exactly the failures it is meant to surface. Raise it locally if you are -# bisecting something unrelated. +# Total ginkgo attempts per spec for the distributed suite: --flake-attempts counts +# attempts, not retries. Defaults to 1, so each spec runs once and is never retried, +# unlike TEST_FLAKES=5. This suite exists to catch nondeterministic cluster behaviour, +# and a retry hides exactly the failures it is meant to surface. Raise it locally if +# you are bisecting something unrelated. DISTRIBUTED_TEST_FLAKES?=1 # Distributed architecture e2e (PostgreSQL + NATS via testcontainers). diff --git a/core/http/endpoints/localai/backend_logs.go b/core/http/endpoints/localai/backend_logs.go index 8b5f99d66f78..f2112502d253 100644 --- a/core/http/endpoints/localai/backend_logs.go +++ b/core/http/endpoints/localai/backend_logs.go @@ -125,8 +125,12 @@ func BackendLogsWebSocketEndpoint(ml *model.ModelLoader) echo.HandlerFunc { // a line appended in that window is never streamed. A viewer attaching while // a model loads (when a backend is at its noisiest) can silently miss lines; // they stay in the buffer, so a reload shows them. Fixing it needs an atomic - // snapshot-plus-subscribe under the store lock, not a reorder of these two - // calls, which would duplicate instead of drop. + // snapshot-plus-subscribe held under the buffer's own lock (buf.mu in + // pkg/model/backend_log_store.go), because that is the lock AppendLine takes + // while it enqueues and fans out to subscribers. The store-level s.mu guards + // only the buffers map and excludes nothing an appender does, so taking it + // leaves this race exactly where it is. Reordering these two calls is not a + // fix either: it would duplicate instead of drop. // Send existing lines as initial batch existingLines := ml.BackendLogs().GetLines(modelID) diff --git a/core/services/nodes/file_transfer_server.go b/core/services/nodes/file_transfer_server.go index 34f2d97582e1..676cb84bd63f 100644 --- a/core/services/nodes/file_transfer_server.go +++ b/core/services/nodes/file_transfer_server.go @@ -1214,8 +1214,12 @@ func handleBackendLogsWS(w http.ResponseWriter, r *http.Request, logStore *model // a line appended in that window is never streamed. A viewer attaching while // a model loads (when a backend is at its noisiest) can silently miss lines; // they stay in the buffer, so a reload shows them. Fixing it needs an atomic - // snapshot-plus-subscribe under the store lock, not a reorder of these two - // calls, which would duplicate instead of drop. + // snapshot-plus-subscribe held under the buffer's own lock (buf.mu in + // pkg/model/backend_log_store.go), because that is the lock AppendLine takes + // while it enqueues and fans out to subscribers. The store-level s.mu guards + // only the buffers map and excludes nothing an appender does, so taking it + // leaves this race exactly where it is. Reordering these two calls is not a + // fix either: it would duplicate instead of drop. // Send existing lines as initial batch existingLines := logStore.GetLines(modelID) diff --git a/tests/e2e/distributed/cluster/failure.go b/tests/e2e/distributed/cluster/failure.go index 919376f4a550..9a461eed702a 100644 --- a/tests/e2e/distributed/cluster/failure.go +++ b/tests/e2e/distributed/cluster/failure.go @@ -71,8 +71,15 @@ func (c *Cluster) KillWorker(i int) error { // dressed up as a failover one. No spec does that today; this note is here so // the first one that tries does not spend a day on it. // -// After StopFrontendGracefully, wait for the process to actually go -// (Eventually(c.FrontendAlive).Should(BeFalse())) before restarting. Restart +// After StopFrontendGracefully, wait for the process to actually go before +// restarting: +// +// Eventually(func() bool { return c.FrontendAlive(i) }, "20s", "500ms"). +// Should(BeFalse()) +// +// FrontendAlive takes an index, so it has to be wrapped in a closure; handing +// Gomega the method value directly fails with "requested 1 arguments but +// received 0". Restart // terminates whatever is still running with SIGKILL, so restarting straight // after a SIGTERM cuts the drain short and quietly turns the rolling-update // case into the crash case, which is the opposite of what pairing those two @@ -126,8 +133,11 @@ func (c *Cluster) FrontendAlive(i int) bool { // than the one it precedes. The window that stays open is the other one, // between the child exiting and waitid collecting it: there the child is a // zombie, signal 0 to a zombie succeeds, and alive reports true for a process -// that is already dead. There is no local fix; the caller's is to poll, -// Eventually(c.FrontendAlive).Should(BeFalse()), rather than assert once. +// that is already dead. There is no local fix; the caller's is to poll rather +// than assert once, wrapping the index-taking FrontendAlive in a closure: +// +// Eventually(func() bool { return c.FrontendAlive(i) }, "20s", "500ms"). +// Should(BeFalse()) func (p *Process) alive() bool { if p == nil || p.Cmd == nil || p.Cmd.Process == nil { return false diff --git a/tests/e2e/distributed/cluster_failover_test.go b/tests/e2e/distributed/cluster_failover_test.go index 3e980cb3de75..2256a16ac187 100644 --- a/tests/e2e/distributed/cluster_failover_test.go +++ b/tests/e2e/distributed/cluster_failover_test.go @@ -134,11 +134,36 @@ func (p *rosterProbe) explainStuckOffline(worker, format string, args ...any) fu // It is the terminating positive control for the two specs that assert a // healthy worker STAYS healthy. On their own those are pure negative // assertions: a cluster whose health checking had wedged entirely, say by -// leaking the Postgres advisory lock the monitor takes (health.go:110), would -// freeze the roster and satisfy them while observing a corpse. Killing a worker -// afterwards and requiring the roster to react proves the monitor was running -// for the whole window, which is the only thing that makes the preceding -// Consistently a statement about behaviour rather than about a stopped clock. +// leaking the Postgres advisory lock the monitor takes +// (core/services/nodes/health.go:112), would freeze the roster and satisfy them +// while observing a corpse. +// +// WHAT IT ACTUALLY PROVES, which is less than it looks like. Killing a worker +// afterwards and requiring the roster to react proves the monitor was alive at +// the END of the preceding window. It does not observe the window itself. The +// inference back across it holds only if a wedge would have been sticky, i.e. +// still present when this helper ran. +// +// THE RESIDUAL GAP, and it is not hypothetical in the peer-replica-death spec. +// Health checks are single-flighted across replicas by a session-scoped +// pg_try_advisory_lock (advisorylock.TryWithLockCtx, non-blocking: a replica +// that does not get the lock returns immediately and checks nothing, silently, +// because checkAll discards the acquired flag). That spec SIGKILLs frontend 1, +// which may have been holding the lock at the moment it died. Postgres releases +// a session-level advisory lock only when it reaps the dead backend, so until +// then frontend 0's ticks acquire nothing and no check runs. The roster freezes, +// Consistently(healthy) passes BECAUSE NOTHING WAS CHECKING, and this helper +// still succeeds afterwards once the session is reaped and the lock comes free. +// That wedge is transient rather than permanent, which is exactly the shape the +// backwards inference cannot see. Low probability, real, and unbounded only by +// how fast Postgres notices a dead connection. +// +// So treat this as a floor and not a proof: it rules out a health monitor that +// is permanently dead, which is the failure that would otherwise make the +// preceding Consistently a statement about a stopped clock, and it does not rule +// out a monitor that was idle for part of the window. Closing the gap needs a +// positive observation from inside the window (a log or metric assertion that a +// check ran), not a stronger assertion here. // // It costs a full detection cycle, which is why it is a shared helper: the // wall-clock price should be paid once per spec and explained once. @@ -258,7 +283,7 @@ var _ = Describe("Cluster failover", Label("Distributed"), Label("Cluster"), fun proveHealthCheckingIsAlive(c, restarted, 0) }) - It("settles a dead worker to offline on every replica", func() { + It("settles a dead worker to offline and both replicas report it offline", func() { c := startCluster(2, 1) worker := c.WorkerName(0) @@ -297,6 +322,15 @@ var _ = Describe("Cluster failover", Label("Distributed"), Label("Cluster"), fun // prevents it, so explainStuckOffline says so in the failure message // when it sees a node stuck at unhealthy. Fixing it is LocalAI work, // not test work. + // Reading the same verdict at both replicas proves shared-verdict + // propagation, NOT two independent detectors. Health checks are + // single-flighted by the advisory lock (see proveHealthCheckingIsAlive), + // so exactly one replica ran the check that wrote the offline status, and + // both probes then read that one Postgres row back. What this rules out + // is a replica that keeps a private roster, or one that reads the shared + // row and reports something else. A spec claiming both replicas can + // detect death on their own would have to isolate them from each other, + // which the shared database makes impossible by design. for _, probe := range []*rosterProbe{at0, at1} { Eventually(probe.statusOf, workerDeathTimeout, rosterPollInterval). WithArguments(worker). diff --git a/tests/e2e/distributed/dbname_test.go b/tests/e2e/distributed/dbname_test.go index 76f1030747fa..33305533d3da 100644 --- a/tests/e2e/distributed/dbname_test.go +++ b/tests/e2e/distributed/dbname_test.go @@ -16,7 +16,8 @@ var _ = Describe("Test database naming", Label("Distributed"), func() { for i := 0; i < 100; i++ { long += "a" } - Expect(len(sanitizeDBName(long))).To(BeNumerically("<=", 50)) + Expect(len(sanitizeDBName(long))).To(Equal(50), + "an over-long name must be truncated to exactly the 50-byte budget; <= 50 would also accept an empty name") }) It("never produces an empty name", func() { From 15e325c07e2dcbf498aae827e46c025593fcd66b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 20:54:25 +0000 Subject: [PATCH 023/153] docs(e2e): correct three claims in the distributed e2e comments The closure note in cluster/failure.go quoted a Gomega error that Gomega does not emit. Describe the argument-count failure and the Eventually().WithArguments() hint instead, so nobody greps for a string that never appears. The advisory-lock note in cluster_failover_test.go called the wedge window unbounded. A SIGKILLed local child closes its socket at once, the Postgres backend reads EOF and is reaped in milliseconds, so the mechanism bounds the window tightly. Say bounded, and keep the low probability but real framing, which was right. The workflow comment attributed HealthCheckInterval to core/services/nodes/health.go. It is declared in core/config/distributed_config.go:64; health.go only carries the ticker on the unexported checkInterval. Point a debugger at the right file. Comments only, no behaviour change. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .github/workflows/tests-e2e-distributed.yml | 8 +++++--- tests/e2e/distributed/cluster/failure.go | 5 +++-- tests/e2e/distributed/cluster_failover_test.go | 5 +++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests-e2e-distributed.yml b/.github/workflows/tests-e2e-distributed.yml index d0cf24875c14..8f17e72f169a 100644 --- a/.github/workflows/tests-e2e-distributed.yml +++ b/.github/workflows/tests-e2e-distributed.yml @@ -112,9 +112,11 @@ jobs: # 509.8s / 512.3s, so 8m29s to 8m32s) on a developer box, and will be slower # here. Three specs sit at ~167s each because they wait out a 60s staleness # threshold plus a 15s health-check tick (HealthCheckInterval, in - # core/services/nodes/health.go, not one of the reconcilers). Do not shorten those windows to make this job faster: the - # wait is what stops the assertions from passing before the system could - # have reacted, which was a real false green earlier on. + # core/config/distributed_config.go; core/services/nodes/health.go runs the + # ticker on the unexported checkInterval, not one of the reconcilers). Do + # not shorten those windows to make this job faster: the wait is what stops + # the assertions from passing before the system could have reacted, which + # was a real false green earlier on. timeout-minutes: 45 steps: - name: Clone diff --git a/tests/e2e/distributed/cluster/failure.go b/tests/e2e/distributed/cluster/failure.go index 9a461eed702a..fe808401cd56 100644 --- a/tests/e2e/distributed/cluster/failure.go +++ b/tests/e2e/distributed/cluster/failure.go @@ -78,8 +78,9 @@ func (c *Cluster) KillWorker(i int) error { // Should(BeFalse()) // // FrontendAlive takes an index, so it has to be wrapped in a closure; handing -// Gomega the method value directly fails with "requested 1 arguments but -// received 0". Restart +// Gomega the method value directly fails immediately: Eventually reports that +// the function it was given takes one argument and none were provided, and +// points at Eventually().WithArguments(). Restart // terminates whatever is still running with SIGKILL, so restarting straight // after a SIGTERM cuts the drain short and quietly turns the rolling-update // case into the crash case, which is the opposite of what pairing those two diff --git a/tests/e2e/distributed/cluster_failover_test.go b/tests/e2e/distributed/cluster_failover_test.go index 2256a16ac187..d007a8ae6ac8 100644 --- a/tests/e2e/distributed/cluster_failover_test.go +++ b/tests/e2e/distributed/cluster_failover_test.go @@ -155,8 +155,9 @@ func (p *rosterProbe) explainStuckOffline(worker, format string, args ...any) fu // Consistently(healthy) passes BECAUSE NOTHING WAS CHECKING, and this helper // still succeeds afterwards once the session is reaped and the lock comes free. // That wedge is transient rather than permanent, which is exactly the shape the -// backwards inference cannot see. Low probability, real, and unbounded only by -// how fast Postgres notices a dead connection. +// backwards inference cannot see. Low probability, real, and bounded by how +// fast Postgres reaps the dead backend, usually immediate on a local socket +// close. // // So treat this as a floor and not a proof: it rules out a health monitor that // is permanently dead, which is the failure that would otherwise make the From c72baa4ad99579bcd3f90361a2f7be5d372e788d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 21:56:27 +0000 Subject: [PATCH 024/153] feat(cluster): record frontend replicas in a shared instances table Replicas need to find each other to relay worker traffic, and nothing in the tree recorded a replica's address. The advertised address is discovered by opening a UDP socket toward PostgreSQL and reading back the local address, which yields the interface every replica demonstrably shares without asking an operator to configure one. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/cluster_suite_test.go | 13 ++ core/services/cluster/instance.go | 189 ++++++++++++++++++++ core/services/cluster/instance_test.go | 101 +++++++++++ core/services/nodes/registry.go | 3 +- 4 files changed, 305 insertions(+), 1 deletion(-) create mode 100644 core/services/cluster/cluster_suite_test.go create mode 100644 core/services/cluster/instance.go create mode 100644 core/services/cluster/instance_test.go diff --git a/core/services/cluster/cluster_suite_test.go b/core/services/cluster/cluster_suite_test.go new file mode 100644 index 000000000000..d821487298fc --- /dev/null +++ b/core/services/cluster/cluster_suite_test.go @@ -0,0 +1,13 @@ +package cluster_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestCluster(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Cluster Package Suite") +} diff --git a/core/services/cluster/instance.go b/core/services/cluster/instance.go new file mode 100644 index 000000000000..7d4347204135 --- /dev/null +++ b/core/services/cluster/instance.go @@ -0,0 +1,189 @@ +// Package cluster records the frontend replicas that make up one LocalAI +// deployment and, later, the links between them. It is deliberately free of +// dependencies on core/services/nodes: nodes migrates and consumes the models +// declared here, so an import in the other direction would be a cycle. +package cluster + +import ( + "context" + "errors" + "fmt" + "net" + "net/url" + "strconv" + "strings" + "time" + + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// ErrInstanceNotFound reports that no row exists for the requested instance ID. +// Callers distinguish it from a transport failure to decide whether to +// re-register or to retry. +var ErrInstanceNotFound = errors.New("cluster: instance not found") + +// Instance is one live frontend replica, keyed by the ID that replica chose for +// itself. Column sizes mirror nodes.BackendNode so both tables agree on what an +// ID and a host:port look like. +type Instance struct { + ID string `gorm:"primaryKey;size:36" json:"id"` + AdvertisedAddr string `gorm:"size:255" json:"advertised_addr"` // host:port other replicas dial + Version string `gorm:"size:64" json:"version"` + LastSeen time.Time `gorm:"index" json:"last_seen"` +} + +// Registry reads and writes the instances table. +type Registry struct { + db *gorm.DB +} + +// NewRegistry returns a Registry over db. Migration is the caller's job; the +// nodes registry owns the AutoMigrate for every table in this deployment so +// that a single advisory lock covers them all. +func NewRegistry(db *gorm.DB) *Registry { + return &Registry{db: db} +} + +// Register records this replica's address, refreshing LastSeen. It upserts on +// the primary key rather than deleting and re-inserting, so a concurrent Live +// never observes a live replica as missing. +func (r *Registry) Register(ctx context.Context, id, addr, version string) error { + inst := Instance{ + ID: id, + AdvertisedAddr: addr, + Version: version, + LastSeen: time.Now(), + } + if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "id"}}, + DoUpdates: clause.AssignmentColumns([]string{"advertised_addr", "version", "last_seen"}), + }).Create(&inst).Error; err != nil { + return fmt.Errorf("registering instance %q: %w", id, err) + } + return nil +} + +// Heartbeat refreshes LastSeen for an already-registered instance. An unknown +// ID is an error rather than an insert: a heartbeat carries no address, so +// inserting would publish a replica nobody can reach. +func (r *Registry) Heartbeat(ctx context.Context, id string) error { + // gorm reports no error when a Where matches nothing, so the miss has to be + // read off RowsAffected. + res := r.db.WithContext(ctx).Model(&Instance{}). + Where("id = ?", id). + Update("last_seen", time.Now()) + if res.Error != nil { + return fmt.Errorf("heartbeating instance %q: %w", id, res.Error) + } + if res.RowsAffected == 0 { + return fmt.Errorf("heartbeating instance %q: %w", id, ErrInstanceNotFound) + } + return nil +} + +// Live returns the instances whose LastSeen is newer than now-within. +func (r *Registry) Live(ctx context.Context, within time.Duration) ([]Instance, error) { + var out []Instance + if err := r.db.WithContext(ctx). + Where("last_seen > ?", time.Now().Add(-within)). + Order("id"). + Find(&out).Error; err != nil { + return nil, fmt.Errorf("listing live instances: %w", err) + } + return out, nil +} + +// Get returns one instance, or ErrInstanceNotFound if it is not registered. +func (r *Registry) Get(ctx context.Context, id string) (*Instance, error) { + var inst Instance + err := r.db.WithContext(ctx).Where("id = ?", id).First(&inst).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, fmt.Errorf("getting instance %q: %w", id, ErrInstanceNotFound) + } + if err != nil { + return nil, fmt.Errorf("getting instance %q: %w", id, err) + } + return &inst, nil +} + +// DiscoverAdvertisedAddr determines the address this replica should advertise +// to its peers, with no operator configuration. +// +// Every replica in a deployment reaches the same PostgreSQL server, so the +// local interface that routes to PostgreSQL is on a network all the replicas +// demonstrably share. Opening a UDP socket toward the database sends no packet; +// it only asks the kernel to pick a source address for that route, which is the +// address to advertise. The caller supplies the port, since the frontend's +// listening port has nothing to do with the database's. +// +// Failures are returned rather than papered over with a fallback such as +// 127.0.0.1, which would publish an address no peer can dial. +func DiscoverAdvertisedAddr(dsn string, port int) (string, error) { + host, dbPort, err := dsnHostPort(dsn) + if err != nil { + return "", err + } + conn, err := net.Dial("udp", net.JoinHostPort(host, dbPort)) + if err != nil { + return "", fmt.Errorf("resolving route to database host %q: %w", host, err) + } + // Nothing was ever sent on this socket, so a close failure carries no + // information about the address we just read. + defer func() { _ = conn.Close() }() + local, ok := conn.LocalAddr().(*net.UDPAddr) + if !ok || local.IP == nil || local.IP.IsUnspecified() { + return "", fmt.Errorf("no local address on the route to database host %q", host) + } + return net.JoinHostPort(local.IP.String(), strconv.Itoa(port)), nil +} + +// dsnHostPort extracts the host and port from either DSN form gorm's postgres +// driver accepts: a URL ("postgres://user:pass@host:5432/db") or libpq keyword +// pairs ("host=... port=..."). +func dsnHostPort(dsn string) (string, string, error) { + const defaultPort = "5432" + dsn = strings.TrimSpace(dsn) + if dsn == "" { + return "", "", errors.New("empty database DSN") + } + + if strings.HasPrefix(dsn, "postgres://") || strings.HasPrefix(dsn, "postgresql://") { + u, err := url.Parse(dsn) + if err != nil { + return "", "", fmt.Errorf("parsing database DSN: %w", err) + } + host := u.Hostname() + if host == "" { + return "", "", errors.New("database DSN has no host") + } + port := u.Port() + if port == "" { + port = defaultPort + } + return host, port, nil + } + + host, port := "", defaultPort + for _, field := range strings.Fields(dsn) { + key, value, found := strings.Cut(field, "=") + if !found { + continue + } + switch key { + case "host": + host = value + case "port": + port = value + } + } + if host == "" { + return "", "", errors.New("database DSN has no host") + } + // A Unix socket directory tells us nothing about which interface reaches + // the database, so there is no address to derive. + if strings.HasPrefix(host, "/") { + return "", "", fmt.Errorf("database DSN uses a unix socket (%q); no routable address to advertise", host) + } + return host, port, nil +} diff --git a/core/services/cluster/instance_test.go b/core/services/cluster/instance_test.go new file mode 100644 index 000000000000..112cc485123e --- /dev/null +++ b/core/services/cluster/instance_test.go @@ -0,0 +1,101 @@ +package cluster_test + +import ( + "context" + "net" + "time" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +var _ = Describe("Instance registry", func() { + var ( + db *gorm.DB + reg *cluster.Registry + ctx context.Context + ) + + BeforeEach(func() { + db = testutil.SetupTestDB() + Expect(db.AutoMigrate(&cluster.Instance{})).To(Succeed()) + reg = cluster.NewRegistry(db) + ctx = context.Background() + }) + + It("registers an instance and reads it back", func() { + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + + got, err := reg.Get(ctx, "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(got.AdvertisedAddr).To(Equal("10.0.0.1:8080")) + Expect(got.Version).To(Equal("v1")) + }) + + It("re-registering the same id updates the address instead of duplicating", func() { + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.9:9090", "v2")).To(Succeed()) + + live, err := reg.Live(ctx, time.Hour) + Expect(err).ToNot(HaveOccurred()) + Expect(live).To(HaveLen(1)) + Expect(live[0].AdvertisedAddr).To(Equal("10.0.0.9:9090")) + }) + + It("reports a missing instance distinguishably", func() { + _, err := reg.Get(ctx, "nope") + Expect(err).To(MatchError(cluster.ErrInstanceNotFound)) + }) + + It("excludes instances whose heartbeat has aged out", func() { + Expect(reg.Register(ctx, "stale", "10.0.0.1:8080", "v1")).To(Succeed()) + // Age the row directly; sleeping in a spec is forbidden. + Expect(db.Model(&cluster.Instance{}).Where("id = ?", "stale"). + Update("last_seen", time.Now().Add(-10*time.Minute)).Error).To(Succeed()) + + live, err := reg.Live(ctx, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(live).To(BeEmpty()) + }) + + It("brings a stale instance back with a heartbeat", func() { + Expect(reg.Register(ctx, "revive", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(db.Model(&cluster.Instance{}).Where("id = ?", "revive"). + Update("last_seen", time.Now().Add(-10*time.Minute)).Error).To(Succeed()) + Expect(reg.Heartbeat(ctx, "revive")).To(Succeed()) + + live, err := reg.Live(ctx, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(live).To(HaveLen(1)) + }) + + It("heartbeating an unknown instance is an error, not a silent insert", func() { + Expect(reg.Heartbeat(ctx, "ghost")).To(MatchError(cluster.ErrInstanceNotFound)) + }) +}) + +var _ = Describe("Advertised address discovery", func() { + // The address itself depends on host networking and is deliberately not + // asserted. What is portable is the shape: whatever interface routes to the + // database, the port must be the one the caller asked for, not the + // database's. + It("combines a local interface with the caller's port", func() { + addr, err := cluster.DiscoverAdvertisedAddr("postgres://198.51.100.1:5432/testdb", 8080) + if err != nil { + Skip("no route to a database host on this machine: " + err.Error()) + } + host, port, splitErr := net.SplitHostPort(addr) + Expect(splitErr).ToNot(HaveOccurred()) + Expect(port).To(Equal("8080")) + Expect(net.ParseIP(host)).ToNot(BeNil()) + }) + + It("refuses a DSN it cannot derive an address from", func() { + _, err := cluster.DiscoverAdvertisedAddr("", 8080) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index 186cf378480a..1b31bb8bf116 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -11,6 +11,7 @@ import ( "github.com/google/uuid" "github.com/mudler/LocalAI/core/services/advisorylock" + "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/vrambudget" "github.com/mudler/xlog" @@ -487,7 +488,7 @@ const heartbeatMaterialDelta = 256 << 20 // 256 MiB // when multiple instances (frontend + workers) start at the same time. func NewNodeRegistry(db *gorm.DB) (*NodeRegistry, error) { if err := advisorylock.WithLockCtx(context.Background(), db, advisorylock.KeySchemaMigrate, func() error { - return db.AutoMigrate(&BackendNode{}, &NodeModel{}, &NodeLabel{}, &ModelSchedulingConfig{}, &PendingBackendOp{}, &ModelLoadInfo{}, &ModelLoadJob{}, &ModelConfigState{}) + return db.AutoMigrate(&BackendNode{}, &NodeModel{}, &NodeLabel{}, &ModelSchedulingConfig{}, &PendingBackendOp{}, &ModelLoadInfo{}, &ModelLoadJob{}, &ModelConfigState{}, &cluster.Instance{}) }); err != nil { return nil, fmt.Errorf("migrating node tables: %w", err) } From 42be69d4693d5516edbb853e2679c6fa7c9ae44c Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 22:08:42 +0000 Subject: [PATCH 025/153] fix(cluster): stamp liveness on the database clock and refuse undialable addresses DiscoverAdvertisedAddr promised to return an error rather than a fallback no peer can dial, but only rejected an unspecified address. With PostgreSQL on the same host or pod as a replica, which is compose, single-node and any sidecar layout, the route to it is loopback, so every replica advertised 127.0.0.1 and a peer dialling that reached itself. Loopback, link-local and zoned source addresses are now rejected with an error naming the remedy, and a port outside 1-65535 is rejected before it becomes an undialable address. Liveness was also measured on each replica's own clock: Register and Heartbeat stamped last_seen from the Go process, and Live compared those rows against the reading replica's time.Now(). Skew therefore shrank or stretched the window by writerBehind+readerAhead, evicting healthy peers or keeping dead ones. Both sides now use the database clock, which is the one clock every replica demonstrably shares. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/instance.go | 60 +++++++++++++++++++------- core/services/cluster/instance_test.go | 14 ++++++ 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/core/services/cluster/instance.go b/core/services/cluster/instance.go index 7d4347204135..027b0db3b0d9 100644 --- a/core/services/cluster/instance.go +++ b/core/services/cluster/instance.go @@ -49,16 +49,24 @@ func NewRegistry(db *gorm.DB) *Registry { // the primary key rather than deleting and re-inserting, so a concurrent Live // never observes a live replica as missing. func (r *Registry) Register(ctx context.Context, id, addr, version string) error { - inst := Instance{ - ID: id, - AdvertisedAddr: addr, - Version: version, - LastSeen: time.Now(), - } - if err := r.db.WithContext(ctx).Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "id"}}, - DoUpdates: clause.AssignmentColumns([]string{"advertised_addr", "version", "last_seen"}), - }).Create(&inst).Error; err != nil { + // last_seen is stamped by the database, never by this process. Liveness is + // compared across replicas, so it has to be measured on the one clock they + // all share; with per-replica clocks the effective Live window becomes + // `within - writerBehind - readerAhead`, which either evicts healthy peers + // or keeps dead ones alive. + if err := r.db.WithContext(ctx).Model(&Instance{}).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "id"}}, + DoUpdates: clause.Assignments(map[string]any{ + "advertised_addr": addr, + "version": version, + "last_seen": gorm.Expr("now()"), + }), + }).Create(map[string]any{ + "id": id, + "advertised_addr": addr, + "version": version, + "last_seen": gorm.Expr("now()"), + }).Error; err != nil { return fmt.Errorf("registering instance %q: %w", id, err) } return nil @@ -72,7 +80,7 @@ func (r *Registry) Heartbeat(ctx context.Context, id string) error { // read off RowsAffected. res := r.db.WithContext(ctx).Model(&Instance{}). Where("id = ?", id). - Update("last_seen", time.Now()) + Update("last_seen", gorm.Expr("now()")) if res.Error != nil { return fmt.Errorf("heartbeating instance %q: %w", id, res.Error) } @@ -85,8 +93,10 @@ func (r *Registry) Heartbeat(ctx context.Context, id string) error { // Live returns the instances whose LastSeen is newer than now-within. func (r *Registry) Live(ctx context.Context, within time.Duration) ([]Instance, error) { var out []Instance + // The cutoff is computed by the database for the same reason Register stamps + // there: a reader's clock must not decide whether another replica is alive. if err := r.db.WithContext(ctx). - Where("last_seen > ?", time.Now().Add(-within)). + Where("last_seen > now() - make_interval(secs => ?)", within.Seconds()). Order("id"). Find(&out).Error; err != nil { return nil, fmt.Errorf("listing live instances: %w", err) @@ -117,9 +127,19 @@ func (r *Registry) Get(ctx context.Context, id string) (*Instance, error) { // address to advertise. The caller supplies the port, since the frontend's // listening port has nothing to do with the database's. // -// Failures are returned rather than papered over with a fallback such as -// 127.0.0.1, which would publish an address no peer can dial. +// The discovery only holds while the database is a shared, remote host. When +// PostgreSQL runs on this same host or pod (compose, single-node, any sidecar +// layout) the route to it is loopback, and advertising 127.0.0.1 would make a +// peer dialling this replica reach itself instead. So an unspecified, loopback, +// or link-local source address is rejected with an error telling the operator to +// configure the advertised address explicitly, rather than returned. There is no +// fallback string: no address is better than a wrong one. func DiscoverAdvertisedAddr(dsn string, port int) (string, error) { + // A port of 0 (or out of range) would produce an address nothing can dial, + // and the caller is likelier to have passed an unset field than to mean it. + if port < 1 || port > 65535 { + return "", fmt.Errorf("advertised port %d is out of range 1-65535", port) + } host, dbPort, err := dsnHostPort(dsn) if err != nil { return "", err @@ -133,7 +153,17 @@ func DiscoverAdvertisedAddr(dsn string, port int) (string, error) { defer func() { _ = conn.Close() }() local, ok := conn.LocalAddr().(*net.UDPAddr) if !ok || local.IP == nil || local.IP.IsUnspecified() { - return "", fmt.Errorf("no local address on the route to database host %q", host) + return "", fmt.Errorf("no local address on the route to database host %q; set the advertised address explicitly", host) + } + if local.IP.IsLoopback() { + return "", fmt.Errorf("the route to database host %q is loopback (%s), so the database is local to this replica and its peer-reachable address cannot be discovered; set the advertised address explicitly", host, local.IP) + } + // A zone is only ever attached to a scoped (link-local) address, so this is + // the same rejection stated twice; the Zone check keeps the guarantee if a + // platform ever hands back a scoped address of another class, because + // IP.String() would silently drop the %iface and yield an undialable host. + if local.IP.IsLinkLocalUnicast() || local.Zone != "" { + return "", fmt.Errorf("the route to database host %q is link-local (%s), which peers on other hosts cannot dial; set the advertised address explicitly", host, local.IP) } return net.JoinHostPort(local.IP.String(), strconv.Itoa(port)), nil } diff --git a/core/services/cluster/instance_test.go b/core/services/cluster/instance_test.go index 112cc485123e..8e74e930e3b9 100644 --- a/core/services/cluster/instance_test.go +++ b/core/services/cluster/instance_test.go @@ -98,4 +98,18 @@ var _ = Describe("Advertised address discovery", func() { _, err := cluster.DiscoverAdvertisedAddr("", 8080) Expect(err).To(HaveOccurred()) }) + + // A database on this same host routes over loopback on every platform, so + // this is deterministic rather than host-dependent. Returning 127.0.0.1 + // would make a peer dialling this replica reach itself. + It("refuses a loopback route instead of advertising an address peers cannot use", func() { + addr, err := cluster.DiscoverAdvertisedAddr("postgres://user@127.0.0.1:5432/testdb", 8080) + Expect(addr).To(BeEmpty()) + Expect(err).To(MatchError(ContainSubstring("loopback"))) + }) + + It("refuses a port that cannot be dialled", func() { + _, err := cluster.DiscoverAdvertisedAddr("postgres://198.51.100.1:5432/testdb", 0) + Expect(err).To(MatchError(ContainSubstring("out of range"))) + }) }) From 6caceb4b1d73e05b1000092fe031c6eb7c4eb037 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 22:19:05 +0000 Subject: [PATCH 026/153] feat(cluster): add the bidirectional splice used by the relay and tunnel Returns on the first direction to finish and closes both sides so the other unblocks; a sequential copy deadlocks on any protocol where the far side speaks first. EOF and use-of-closed are normal termination, not errors. The fourth spec covers a peer that stops reading mid-body, the case where a copy is parked in Write rather than in Read. The other three tear down an idle splice and pass even against a Splice that closes only one side. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/splice.go | 68 ++++++++++++++++ core/services/cluster/splice_test.go | 114 +++++++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 core/services/cluster/splice.go create mode 100644 core/services/cluster/splice_test.go diff --git a/core/services/cluster/splice.go b/core/services/cluster/splice.go new file mode 100644 index 000000000000..a31f0475109c --- /dev/null +++ b/core/services/cluster/splice.go @@ -0,0 +1,68 @@ +package cluster + +import ( + "errors" + "io" + "net" +) + +// Splice joins two streams and copies bytes between them in both directions +// until one direction finishes, then closes both so the other unblocks and +// returns. It is the primitive under the inter-replica relay and the worker +// tunnel, so it carries gRPC: both directions can be live at once and either +// peer may speak first, which is why the copies run concurrently. A sequential +// io.Copy then io.Copy would deadlock waiting for a request on a stream whose +// far side is waiting for a response. +// +// The error reported is the one from the direction that finished first, with +// the endings that mean "someone closed" mapped to nil. The other direction's +// error is discarded: by then it is only an echo of the Close done here. +func Splice(a, b io.ReadWriteCloser) error { + errs := make(chan error, 2) + go func() { errs <- copyStream(b, a) }() + go func() { errs <- copyStream(a, b) }() + + first := <-errs + + // Closing both ends is what releases the other direction, whether it is + // parked in Read or halfway through a Write nobody is draining. Each end + // is closed exactly once, here and nowhere else, so a stream that reports + // an error on a second Close (yamux does) never sees one. + closeErrA := a.Close() + closeErrB := b.Close() + + // Wait for the second direction so no copy is still touching either + // stream once Splice has returned. + <-errs + + if first != nil { + return first + } + // A close that fails on a stream that was otherwise healthy is worth + // reporting; a close of an already-dead stream is not. + if err := normalizeStreamErr(closeErrA); err != nil { + return err + } + return normalizeStreamErr(closeErrB) +} + +// copyStream moves one direction and reports only genuine transport failures. +func copyStream(dst io.Writer, src io.Reader) error { + _, err := io.Copy(dst, src) + return normalizeStreamErr(err) +} + +// normalizeStreamErr drops the endings that mean the conversation is over +// rather than broken. io.EOF is the clean end of a stream, net.ErrClosed is +// what a socket or yamux stream reports once it or its peer has been closed, +// and io.ErrClosedPipe is the same condition on an in-memory pipe. Anything +// else is a real failure the caller should see. +func normalizeStreamErr(err error) error { + if err == nil || + errors.Is(err, io.EOF) || + errors.Is(err, net.ErrClosed) || + errors.Is(err, io.ErrClosedPipe) { + return nil + } + return err +} diff --git a/core/services/cluster/splice_test.go b/core/services/cluster/splice_test.go new file mode 100644 index 000000000000..9e3529e65e1a --- /dev/null +++ b/core/services/cluster/splice_test.go @@ -0,0 +1,114 @@ +package cluster_test + +import ( + "errors" + "io" + "net" + "time" + + "github.com/mudler/LocalAI/core/services/cluster" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Splice", func() { + // pipePair returns two connected in-memory conns. + newPair := func() (net.Conn, net.Conn) { return net.Pipe() } + + It("copies bytes in both directions", func() { + aLeft, aRight := newPair() + bLeft, bRight := newPair() + + done := make(chan error, 1) + go func() { done <- cluster.Splice(aRight, bLeft) }() + + go func() { + _, _ = aLeft.Write([]byte("ping")) + }() + buf := make([]byte, 4) + Expect(bRight.SetReadDeadline(time.Now().Add(5 * time.Second))).To(Succeed()) + _, err := io.ReadFull(bRight, buf) + Expect(err).ToNot(HaveOccurred()) + Expect(string(buf)).To(Equal("ping")) + + go func() { + _, _ = bRight.Write([]byte("pong")) + }() + Expect(aLeft.SetReadDeadline(time.Now().Add(5 * time.Second))).To(Succeed()) + _, err = io.ReadFull(aLeft, buf) + Expect(err).ToNot(HaveOccurred()) + Expect(string(buf)).To(Equal("pong")) + + Expect(aLeft.Close()).To(Succeed()) + Eventually(done, "5s").Should(Receive()) + }) + + It("returns when one side closes, and closes the other", func() { + aLeft, aRight := newPair() + bLeft, bRight := newPair() + + done := make(chan error, 1) + go func() { done <- cluster.Splice(aRight, bLeft) }() + + Expect(aLeft.Close()).To(Succeed()) + Eventually(done, "5s").Should(Receive(BeNil())) + + // The far side must have been closed too, so a read there fails + // rather than blocking forever. The read runs in a goroutine and is + // polled instead of carrying a read deadline: net.Pipe refuses to set + // a deadline once *either* end is closed, so a deadline here would + // fail exactly when Splice did its job. + reads := make(chan error, 1) + go func() { + _, err := bRight.Read(make([]byte, 1)) + reads <- err + }() + var err error + Eventually(reads, "5s").Should(Receive(&err)) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, io.EOF)).To(BeTrue()) + }) + + It("does not leak a goroutine when both sides close", func() { + aLeft, aRight := newPair() + bLeft, bRight := newPair() + + done := make(chan error, 1) + go func() { done <- cluster.Splice(aRight, bLeft) }() + + Expect(aLeft.Close()).To(Succeed()) + Expect(bRight.Close()).To(Succeed()) + Eventually(done, "5s").Should(Receive()) + }) + + // The three specs above only ever tear down an idle splice: at the moment + // of Close no copy is parked inside a Write. A relayed inference response + // is the opposite case, a reader that walks away mid-body while 50MB is + // still being pushed at it, so this covers the direction that is blocked + // in Write rather than in Read when its peer disappears. + It("returns when the reader disappears while a write is in flight", func() { + aLeft, aRight := newPair() + bLeft, bRight := newPair() + + done := make(chan error, 1) + go func() { done <- cluster.Splice(aRight, bLeft) }() + + // Nothing ever reads from bRight, so the a->b direction parks inside + // Write on an unbuffered pipe with the payload half-delivered. + payload := make([]byte, 1<<20) + writes := make(chan error, 1) + go func() { + _, err := aLeft.Write(payload) + writes <- err + }() + + Expect(bRight.Close()).To(Succeed()) + Eventually(done, "5s").Should(Receive()) + + // The abandoned writer must be released as well, and only Splice + // closing its end can do that: no deadline is set on aLeft, so a + // splice that forgot to close would leave this write parked forever. + Eventually(writes, "5s").Should(Receive(HaveOccurred())) + }) +}) From c05246a366a045e4ce07660211608a0098dfeb0a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 22:33:54 +0000 Subject: [PATCH 027/153] fix(cluster): treat a yamux teardown in Splice as a normal ending go-yamux/v5 matches none of its errors against net.ErrClosed, so the classifier reported an ordinary teardown as a failure: when the session has gone away, the FIN that Splice's own Close writes returns ErrSessionShutdown, and a stream torn down under a live copy surfaces as ErrStreamClosed or a reset. Splice owns that Close, so it owns the errors it produces; the sentinels are named here rather than injected by the caller, which would make a forgotten classifier reintroduce the same bug silently. Cover the error half of the contract, which no in-memory pipe could reach: a scripted stream now feeds Splice a genuine transport failure and each closed-stream ending in turn. Replacing the tail of Splice with "return nil" passed every previous spec. Also assert that Splice does not return until the second direction has finished, rename a spec that promised a leak check it never made, and correct two comments that claimed more than the code did. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/splice.go | 43 ++++++-- core/services/cluster/splice_test.go | 145 ++++++++++++++++++++++++++- go.mod | 2 +- 3 files changed, 181 insertions(+), 9 deletions(-) diff --git a/core/services/cluster/splice.go b/core/services/cluster/splice.go index a31f0475109c..3f7bd8fbf7d1 100644 --- a/core/services/cluster/splice.go +++ b/core/services/cluster/splice.go @@ -4,6 +4,8 @@ import ( "errors" "io" "net" + + "github.com/libp2p/go-yamux/v5" ) // Splice joins two streams and copies bytes between them in both directions @@ -14,9 +16,16 @@ import ( // io.Copy then io.Copy would deadlock waiting for a request on a stream whose // far side is waiting for a response. // +// EOF in one direction therefore truncates whatever is still in flight in the +// other. That is right for gRPC, HTTP/2 and yamux, which end a stream in both +// directions at once, but a future caller relaying raw TCP with a half-close +// would lose the response body still arriving after the request's CloseWrite. +// // The error reported is the one from the direction that finished first, with // the endings that mean "someone closed" mapped to nil. The other direction's -// error is discarded: by then it is only an echo of the Close done here. +// error is dropped; most of the time it is an echo of the Close below, but it +// can also be a genuine failure that lost the race, so a Splice error means +// "one direction failed", never "only this failed". func Splice(a, b io.ReadWriteCloser) error { errs := make(chan error, 2) go func() { errs <- copyStream(b, a) }() @@ -31,8 +40,13 @@ func Splice(a, b io.ReadWriteCloser) error { closeErrA := a.Close() closeErrB := b.Close() - // Wait for the second direction so no copy is still touching either - // stream once Splice has returned. + // Wait for the second direction so no copy is still touching either stream + // once Splice has returned. This is load-bearing: it assumes Close unblocks + // a copy parked in Read or Write, and a stream where that is false hangs + // here rather than leaking a goroutine. Both callers satisfy it. net.Conn + // does, and so does go-yamux/v5, whose Close sets readErr and calls + // notifyWaiting to wake a parked Read while a parked Write returns + // ErrStreamClosed. <-errs if first != nil { @@ -54,14 +68,29 @@ func copyStream(dst io.Writer, src io.Reader) error { // normalizeStreamErr drops the endings that mean the conversation is over // rather than broken. io.EOF is the clean end of a stream, net.ErrClosed is -// what a socket or yamux stream reports once it or its peer has been closed, -// and io.ErrClosedPipe is the same condition on an in-memory pipe. Anything -// else is a real failure the caller should see. +// what a socket reports once it or its peer has been closed, and +// io.ErrClosedPipe is the same condition on an in-memory pipe. +// +// The yamux sentinels are here because Splice owns the Close that produces +// them: closing a stream whose session has already gone away returns +// ErrSessionShutdown from the FIN write, and a stream torn down under a live +// copy surfaces as ErrStreamClosed or a reset. None of yamux's error types +// match net.ErrClosed, so each has to be named. Matching ErrStreamReset covers +// every *StreamError and *GoAwayError, both of which report themselves as a +// reset; ErrStreamClosed is a plain sentinel and matches only itself. +// +// Anything else is a real failure the caller should see. Note that a peer +// resetting a socket mid-stream (ECONNRESET, EPIPE) is deliberately not in +// this set: whether an abandoned request is routine is the caller's policy, +// not this primitive's. func normalizeStreamErr(err error) error { if err == nil || errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) || - errors.Is(err, io.ErrClosedPipe) { + errors.Is(err, io.ErrClosedPipe) || + errors.Is(err, yamux.ErrStreamClosed) || + errors.Is(err, yamux.ErrStreamReset) || + errors.Is(err, yamux.ErrSessionShutdown) { return nil } return err diff --git a/core/services/cluster/splice_test.go b/core/services/cluster/splice_test.go index 9e3529e65e1a..5b69eef720ef 100644 --- a/core/services/cluster/splice_test.go +++ b/core/services/cluster/splice_test.go @@ -4,8 +4,12 @@ import ( "errors" "io" "net" + "sync" + "sync/atomic" "time" + "github.com/libp2p/go-yamux/v5" + "github.com/mudler/LocalAI/core/services/cluster" . "github.com/onsi/ginkgo/v2" @@ -70,7 +74,7 @@ var _ = Describe("Splice", func() { Expect(errors.Is(err, io.EOF)).To(BeTrue()) }) - It("does not leak a goroutine when both sides close", func() { + It("returns when both sides close", func() { aLeft, aRight := newPair() bLeft, bRight := newPair() @@ -111,4 +115,143 @@ var _ = Describe("Splice", func() { // splice that forgot to close would leave this write parked forever. Eventually(writes, "5s").Should(Receive(HaveOccurred())) }) + // net.Pipe can only ever end in EOF or a closed pipe, so the error half of + // the contract needs a stream that can be told how to fail. + Context("when a stream fails rather than closing", func() { + errBoom := errors.New("transport exploded") + + It("reports a genuine transport error", func() { + failing := &scriptedStream{readErr: errBoom} + idle := &scriptedStream{} + + done := make(chan error, 1) + go func() { done <- cluster.Splice(failing, idle) }() + + var err error + Eventually(done, "5s").Should(Receive(&err)) + Expect(errors.Is(err, errBoom)).To(BeTrue()) + + // Closed exactly once each: a second Close is what makes a yamux + // stream complain about a teardown that went fine. + Expect(failing.closes()).To(Equal(int32(1))) + Expect(idle.closes()).To(Equal(int32(1))) + }) + + It("reports a genuine failure from its own Close", func() { + failing := &scriptedStream{closeErr: errBoom} + idle := &scriptedStream{} + + done := make(chan error, 1) + go func() { done <- cluster.Splice(idle, failing) }() + + Expect(idle.Close()).To(Succeed()) + var err error + Eventually(done, "5s").Should(Receive(&err)) + Expect(errors.Is(err, errBoom)).To(BeTrue()) + }) + + // Every one of these means "a stream we were copying through was + // closed". The yamux entries are the teardown Splice itself provokes: + // none of them matches net.ErrClosed, so each has to be classified by + // name or a normal relayed request ends up reported as a failure. + DescribeTable("treats a closed stream as normal termination", + func(ending error) { + ended := &scriptedStream{readErr: ending} + idle := &scriptedStream{} + + done := make(chan error, 1) + go func() { done <- cluster.Splice(ended, idle) }() + + Eventually(done, "5s").Should(Receive(BeNil())) + }, + Entry("EOF", io.EOF), + Entry("a closed socket", net.ErrClosed), + Entry("a closed in-memory pipe", io.ErrClosedPipe), + Entry("a closed yamux stream", yamux.ErrStreamClosed), + Entry("a reset yamux stream", yamux.ErrStreamReset), + Entry("a shut-down yamux session", yamux.ErrSessionShutdown), + Entry("a stream reset by the remote", &yamux.StreamError{ErrorCode: 1, Remote: true}), + Entry("a go-away from the remote", yamux.ErrRemoteGoAway), + ) + + // The tunnel's own teardown: the local backend finishes normally while + // the yamux session has already gone away, so the FIN that Splice's + // Close writes fails. Nothing went wrong and nothing may be reported. + It("does not report a shut-down session on its own Close", func() { + stream := &scriptedStream{closeErr: yamux.ErrSessionShutdown} + backend := &scriptedStream{readErr: io.EOF} + + done := make(chan error, 1) + go func() { done <- cluster.Splice(stream, backend) }() + + Eventually(done, "5s").Should(Receive(BeNil())) + }) + + // The anti-leak guarantee, which the pipe specs cannot see because + // their parked copy is released too quickly to catch Splice in the + // act. Waking a copy is asynchronous on a real stream (yamux's Close + // notifies the reader, which then has to be scheduled), so this stream + // splits the two: Close records itself, and the spec decides when the + // parked Read actually returns. + It("does not return until the second direction has finished", func() { + parked := &scriptedStream{holdReadPastClose: true} + ending := &scriptedStream{readErr: io.EOF} + + done := make(chan error, 1) + go func() { done <- cluster.Splice(ending, parked) }() + + Eventually(parked.closes, "5s").Should(Equal(int32(1))) + Consistently(done, "200ms").ShouldNot(Receive()) + + parked.release() + Eventually(done, "5s").Should(Receive(BeNil())) + }) + }) }) + +// scriptedStream is an io.ReadWriteCloser whose endings the spec dictates, so +// Splice can be fed failures no in-memory pipe can produce. With no readErr it +// parks in Read until Close, standing in for an idle half of a live stream. +type scriptedStream struct { + readErr error + closeErr error + // holdReadPastClose keeps a parked Read blocked until release is called, + // standing in for the gap between a Close waking a reader and that reader + // running. Without it, Close releases the Read as a real stream does. + holdReadPastClose bool + + releaseOnce sync.Once + released chan struct{} + initOnce sync.Once + closeN atomic.Int32 +} + +func (s *scriptedStream) gate() chan struct{} { + s.initOnce.Do(func() { s.released = make(chan struct{}) }) + return s.released +} + +func (s *scriptedStream) release() { + gate := s.gate() + s.releaseOnce.Do(func() { close(gate) }) +} + +func (s *scriptedStream) Read(p []byte) (int, error) { + if s.readErr != nil { + return 0, s.readErr + } + <-s.gate() + return 0, io.EOF +} + +func (s *scriptedStream) Write(p []byte) (int, error) { return len(p), nil } + +func (s *scriptedStream) Close() error { + s.closeN.Add(1) + if !s.holdReadPastClose { + s.release() + } + return s.closeErr +} + +func (s *scriptedStream) closes() int32 { return s.closeN.Load() } diff --git a/go.mod b/go.mod index 1d4025b89e5f..a74e4200a310 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/klauspost/cpuid/v2 v2.3.0 github.com/labstack/echo/v4 v4.15.1 github.com/libp2p/go-libp2p v0.48.0 + github.com/libp2p/go-yamux/v5 v5.1.0 github.com/lithammer/fuzzysearch v1.1.8 github.com/mholt/archiver/v3 v3.5.1 github.com/microcosm-cc/bluemonday v1.0.27 @@ -329,7 +330,6 @@ require ( github.com/jeandeaual/go-locale v0.0.0-20250612000132-0ef82f21eade // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/jsummers/gobmp v0.0.0-20230614200233-a9de23ed2e25 // indirect - github.com/libp2p/go-yamux/v5 v5.1.0 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect From a0deca63855ad5f028e48badcfc8edd2feee38b0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 22:49:57 +0000 Subject: [PATCH 028/153] fix(cluster): report a dead yamux session instead of swallowing it Matching yamux errors with errors.Is was too broad. Session.close hands every live stream ErrStreamReset wrapped around whatever killed the connection, so a keepalive timeout, a broken TCP connection or a peer that simply vanished all matched, and a relayed request that died reported a clean ending. Nothing upstream would have retried or logged it. Match the plain sentinels by identity, since only identity separates a stream that was reset from the wrapped form that means the session died. Treat a StreamError as a per-stream reset, and a GoAwayError as normal only when it carries the no-error code, read off ErrRemoteGoAway because the constant is unexported. ErrSessionShutdown needs no entry of its own; it is a GoAwayError with that code. Order matters as much as the matching: session death wraps its cause, which is routinely io.EOF or a closed socket, so the mux checks run before the generic endings. Reversing them alone puts a vanished peer back to nil. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/splice.go | 93 ++++++++++++++++++++++------ core/services/cluster/splice_test.go | 67 ++++++++++++++++++++ 2 files changed, 140 insertions(+), 20 deletions(-) diff --git a/core/services/cluster/splice.go b/core/services/cluster/splice.go index 3f7bd8fbf7d1..ba5ac28af294 100644 --- a/core/services/cluster/splice.go +++ b/core/services/cluster/splice.go @@ -35,8 +35,10 @@ func Splice(a, b io.ReadWriteCloser) error { // Closing both ends is what releases the other direction, whether it is // parked in Read or halfway through a Write nobody is draining. Each end - // is closed exactly once, here and nowhere else, so a stream that reports - // an error on a second Close (yamux does) never sees one. + // is closed exactly once, here and nowhere else, which keeps the error + // below meaningful: a second Close of a yamux stream that was reset + // returns the error that killed it, and Splice would have no way to tell + // that from a fresh failure. closeErrA := a.Close() closeErrB := b.Close() @@ -71,27 +73,78 @@ func copyStream(dst io.Writer, src io.Reader) error { // what a socket reports once it or its peer has been closed, and // io.ErrClosedPipe is the same condition on an in-memory pipe. // -// The yamux sentinels are here because Splice owns the Close that produces -// them: closing a stream whose session has already gone away returns -// ErrSessionShutdown from the FIN write, and a stream torn down under a live -// copy surfaces as ErrStreamClosed or a reset. None of yamux's error types -// match net.ErrClosed, so each has to be named. Matching ErrStreamReset covers -// every *StreamError and *GoAwayError, both of which report themselves as a -// reset; ErrStreamClosed is a plain sentinel and matches only itself. -// -// Anything else is a real failure the caller should see. Note that a peer -// resetting a socket mid-stream (ECONNRESET, EPIPE) is deliberately not in -// this set: whether an abandoned request is routine is the caller's policy, -// not this primitive's. +// The mux checks run first, and that ordering is load-bearing: a dying yamux +// session hands every live stream its own cause wrapped up (session.go:330), +// and that cause is routinely io.EOF or a closed-socket error, so consulting +// the generic endings first would report a peer that vanished mid-request as a +// clean completion. func normalizeStreamErr(err error) error { - if err == nil || - errors.Is(err, io.EOF) || + if err == nil { + return nil + } + if isMuxSessionFailure(err) { + return err + } + if isMuxStreamTeardown(err) { + return nil + } + if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) || - errors.Is(err, io.ErrClosedPipe) || - errors.Is(err, yamux.ErrStreamClosed) || - errors.Is(err, yamux.ErrStreamReset) || - errors.Is(err, yamux.ErrSessionShutdown) { + errors.Is(err, io.ErrClosedPipe) { return nil } return err } + +// normalGoAwayCode is yamux's "no error" go-away code, read off a sentinel +// declared with it because the constant itself is unexported. +var normalGoAwayCode = yamux.ErrRemoteGoAway.ErrorCode + +// isMuxSessionFailure reports whether err is the yamux session underneath a +// stream dying, as opposed to a single stream being torn down. The distinction +// matters because Splice must stay quiet about the teardown it provokes itself +// while still reporting a dead peer: a keepalive timeout, a broken TCP +// connection or a protocol error under a relayed request has to reach the +// caller, or a failed inference looks like a finished one. +func isMuxSessionFailure(err error) bool { + // A go-away ends the whole session. Only the "no error" code is a normal + // ending; a protocol or internal error go-away is a real failure. + var goAway *yamux.GoAwayError + if errors.As(err, &goAway) { + return goAway.ErrorCode != normalGoAwayCode + } + // A stream error is scoped to one stream, whatever killed it. + var streamErr *yamux.StreamError + if errors.As(err, &streamErr) { + return false + } + // Session.close gives every stream it kills ErrStreamReset wrapped around + // the cause, so the bare sentinel means this stream was reset and a + // wrapped one means the session died under it. Identity is what separates + // them; errors.Is cannot. + return errors.Is(err, yamux.ErrStreamReset) && err != yamux.ErrStreamReset +} + +// isMuxStreamTeardown reports whether err is yamux ending one stream, which +// Splice mostly provokes itself: closing a stream whose session has already +// shut down normally returns ErrSessionShutdown from the FIN write, and a copy +// parked on a stream that gets closed comes back with ErrStreamClosed or a +// reset. A reset does not only mean that, though; the same sentinel heads the +// error a dying session hands its streams, which is why isMuxSessionFailure +// runs first. None of yamux's error types match net.ErrClosed, so all of this +// has to be recognised by shape. +func isMuxStreamTeardown(err error) bool { + // Sentinels by identity, never errors.Is: the wrapped forms belong to a + // dead session and are reported instead. ErrSessionShutdown is absent on + // purpose rather than by oversight, being a *GoAwayError carrying the + // normal code, which the last check below covers. + if err == yamux.ErrStreamClosed || err == yamux.ErrStreamReset { + return true + } + var streamErr *yamux.StreamError + if errors.As(err, &streamErr) { + return true + } + var goAway *yamux.GoAwayError + return errors.As(err, &goAway) && goAway.ErrorCode == normalGoAwayCode +} diff --git a/core/services/cluster/splice_test.go b/core/services/cluster/splice_test.go index 5b69eef720ef..f18618b6b2cd 100644 --- a/core/services/cluster/splice_test.go +++ b/core/services/cluster/splice_test.go @@ -2,6 +2,7 @@ package cluster_test import ( "errors" + "fmt" "io" "net" "sync" @@ -174,6 +175,72 @@ var _ = Describe("Splice", func() { Entry("a go-away from the remote", yamux.ErrRemoteGoAway), ) + // sessionDeath is the exact shape Session.close hands every live + // stream when the session dies for a non-go-away reason + // (session.go:330). It matters that these are wrapped: the cause it + // carries is routinely io.EOF or a closed socket, so a classifier that + // looked at the cause would call a vanished peer a clean ending. + sessionDeath := func(cause error) error { + return fmt.Errorf("%w: connection closed: %w", yamux.ErrStreamReset, cause) + } + + // A dead peer under a relayed inference request has to reach the + // caller. If it arrives as nil, a failed request looks like a finished + // one and nothing upstream retries or logs it. + DescribeTable("reports the session dying under a stream", + func(ending error) { + dead := &scriptedStream{readErr: ending} + idle := &scriptedStream{} + + done := make(chan error, 1) + go func() { done <- cluster.Splice(dead, idle) }() + + var err error + Eventually(done, "5s").Should(Receive(&err)) + Expect(err).To(MatchError(ending)) + }, + Entry("a keepalive timeout", sessionDeath(yamux.ErrKeepAliveTimeout)), + Entry("a broken connection", sessionDeath(errors.New("read tcp 10.0.0.1:4000: broken pipe"))), + Entry("a peer that vanished", sessionDeath(io.EOF)), + Entry("a protocol-error go-away", &yamux.GoAwayError{Remote: true, ErrorCode: 1}), + Entry("an internal-error go-away", &yamux.GoAwayError{Remote: true, ErrorCode: 2}), + ) + + // The distinction the classifier turns on, in one spec: yamux uses the + // same sentinel for "this stream was reset", which Splice provokes + // itself and must stay quiet about, and as the head of the wrapped + // error meaning "the session died", which it must report. Only + // identity separates them. + It("separates a bare reset from a session that died wrapping one", func() { + spliceEnding := func(ending error) error { + done := make(chan error, 1) + go func() { + done <- cluster.Splice(&scriptedStream{readErr: ending}, &scriptedStream{}) + }() + var err error + EventuallyWithOffset(1, done, "5s").Should(Receive(&err)) + return err + } + + Expect(spliceEnding(yamux.ErrStreamReset)).To(BeNil()) + Expect(spliceEnding(sessionDeath(yamux.ErrKeepAliveTimeout))).ToNot(BeNil()) + }) + + // Session death also arrives through the Close Splice makes itself, on + // a stream whose session died while the other side was finishing. That + // is not the quiet teardown ErrSessionShutdown describes. + It("reports a session that died, even from its own Close", func() { + stream := &scriptedStream{closeErr: sessionDeath(yamux.ErrKeepAliveTimeout)} + backend := &scriptedStream{readErr: io.EOF} + + done := make(chan error, 1) + go func() { done <- cluster.Splice(stream, backend) }() + + var err error + Eventually(done, "5s").Should(Receive(&err)) + Expect(err).To(MatchError(yamux.ErrKeepAliveTimeout)) + }) + // The tunnel's own teardown: the local backend finishes normally while // the yamux session has already gone away, so the FIN that Splice's // Close writes fails. Nothing went wrong and nothing may be reported. From c2162f712a2cdbd4c4134a63251c3b3cb0e7b0cf Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 23:14:37 +0000 Subject: [PATCH 029/153] feat(cluster): accept authenticated peer links on /api/cluster/peer Upgrades to a WebSocket, wraps it as a yamux server session and hands it to the caller. Rejects before upgrading so an unauthenticated dial sees a 401 rather than a WebSocket error, which is what the route-coverage test asserts. The adapter keeps the reader of a partially consumed message across Read calls. yamux reads through a 4 KiB bufio.Reader, so a small-payload test cannot see a dropped message tail; the framing specs drive the adapter directly with buffers smaller than the message. An empty configured token authorizes nobody here, unlike the worker file transfer server's check: this route is registered in every deployment, so failing open would publish an unauthenticated mux. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/auth/public_routes.go | 9 +- .../endpoints/cluster/cluster_suite_test.go | 13 ++ core/http/endpoints/cluster/peer.go | 104 +++++++++ core/http/endpoints/cluster/peer_test.go | 117 ++++++++++ core/http/endpoints/cluster/wsconn.go | 129 +++++++++++ core/http/endpoints/cluster/wsconn_test.go | 216 ++++++++++++++++++ 6 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 core/http/endpoints/cluster/cluster_suite_test.go create mode 100644 core/http/endpoints/cluster/peer.go create mode 100644 core/http/endpoints/cluster/peer_test.go create mode 100644 core/http/endpoints/cluster/wsconn.go create mode 100644 core/http/endpoints/cluster/wsconn_test.go diff --git a/core/http/auth/public_routes.go b/core/http/auth/public_routes.go index c2509e9a9f6c..e1a79b3ab12f 100644 --- a/core/http/auth/public_routes.go +++ b/core/http/auth/public_routes.go @@ -5,6 +5,8 @@ package auth import ( "net/http" "strings" + + clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" ) type publicRouteRule struct { @@ -83,5 +85,10 @@ func isPublicRoute(method, path string) bool { // usesAlternativeAuthentication identifies requests whose credentials are // validated by route-group middleware instead of the global auth middleware. func usesAlternativeAuthentication(path string) bool { - return strings.HasPrefix(path, "/api/node/") + // The peer link carries the cluster token in an Authorization header that + // no browser session ever sets, and its handler checks that token itself. + // The prefix comes from the endpoints package so the route and the + // exemption cannot drift apart. + return strings.HasPrefix(path, "/api/node/") || + strings.HasPrefix(path, clusterep.AlternativeAuthPrefix) } diff --git a/core/http/endpoints/cluster/cluster_suite_test.go b/core/http/endpoints/cluster/cluster_suite_test.go new file mode 100644 index 000000000000..00f45e37a672 --- /dev/null +++ b/core/http/endpoints/cluster/cluster_suite_test.go @@ -0,0 +1,13 @@ +package cluster_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestClusterEndpoints(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Cluster Endpoints Suite") +} diff --git a/core/http/endpoints/cluster/peer.go b/core/http/endpoints/cluster/peer.go new file mode 100644 index 000000000000..4b6ef542caf4 --- /dev/null +++ b/core/http/endpoints/cluster/peer.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: MIT + +// Package cluster serves the replica-to-replica link that a LocalAI frontend +// uses to reach a worker tunnel it does not own. A peer dials +// GET /api/cluster/peer, the connection becomes one multiplexed yamux session, +// and the relay opens a stream on it per request. +package cluster + +import ( + "crypto/subtle" + "net/http" + + "github.com/gorilla/websocket" + "github.com/labstack/echo/v4" + "github.com/libp2p/go-yamux/v5" + "github.com/mudler/xlog" +) + +// AlternativeAuthPrefix is the path prefix whose credentials are checked by +// this package rather than by the global session middleware. The auth layer +// consults this same constant, so the two cannot drift apart and leave every +// peer dial answering 401. +const AlternativeAuthPrefix = "/api/cluster/" + +// PeerPath is the route a peer replica dials. +const PeerPath = AlternativeAuthPrefix + "peer" + +// RegisterClusterRoutes registers the peer link. onPeer receives every +// authenticated session; see PeerHandler for what it is expected to do with it. +func RegisterClusterRoutes(e *echo.Echo, token string, onPeer func(string, *yamux.Session)) { + e.GET(PeerPath, PeerHandler(token, onPeer)) +} + +// PeerHandler upgrades an authenticated peer dial to a WebSocket, wraps it as +// a yamux server session and hands it to onSession. +// +// onSession runs on the request goroutine, so it must return promptly; the +// session outlives the handler because the upgrade hijacks the connection, and +// closing it is the caller's job. +func PeerHandler(token string, onSession func(peerID string, sess *yamux.Session)) echo.HandlerFunc { + // gorilla's default CheckOrigin already restricts a browser to same-origin + // and lets a header-less client (which every peer is) through, so the + // zero value is what this link wants. + upgrader := websocket.Upgrader{} + + return func(c echo.Context) error { + // Reject before upgrading. Upgrading and then closing would give the + // dialer a WebSocket error in place of an HTTP status, and both the + // route-coverage test and a peer's own retry logic read the status. + if !authorizedPeer(c.Request(), token) { + return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") + } + + peerID := c.QueryParam("id") + if peerID == "" { + return echo.NewHTTPError(http.StatusBadRequest, "missing peer id") + } + + ws, err := upgrader.Upgrade(c.Response(), c.Request(), nil) + if err != nil { + // Upgrade has already written its own failure to the client. + xlog.Debug("cluster peer link upgrade failed", "peer", peerID, "error", err) + return nil + } + + // Server side of the mux: the dialing peer is the client, so it owns + // the odd stream IDs and this side the even ones. + sess, err := yamux.Server(WebsocketConn(ws), nil, nil) + if err != nil { + xlog.Error("cluster peer link session setup failed", "peer", peerID, "error", err) + _ = ws.Close() + return nil + } + + if onSession == nil { + // Nothing will ever read from this session, so do not leave the + // peer believing it has a live link. + _ = sess.Close() + return nil + } + + xlog.Debug("cluster peer link established", "peer", peerID, "remote", ws.RemoteAddr().String()) + onSession(peerID, sess) + return nil + } +} + +// authorizedPeer compares the request's bearer token with the cluster token in +// constant time, matching the check the worker file-transfer server makes. +// +// Unlike that one, an empty configured token authorizes nobody: this route is +// registered in every deployment, so failing open would publish an +// unauthenticated mux to any caller that can reach the port. +func authorizedPeer(r *http.Request, expected string) bool { + if expected == "" { + return false + } + const prefix = "Bearer " + header := r.Header.Get("Authorization") + if len(header) < len(prefix) || header[:len(prefix)] != prefix { + return false + } + return subtle.ConstantTimeCompare([]byte(header[len(prefix):]), []byte(expected)) == 1 +} diff --git a/core/http/endpoints/cluster/peer_test.go b/core/http/endpoints/cluster/peer_test.go new file mode 100644 index 000000000000..b4aa98356c13 --- /dev/null +++ b/core/http/endpoints/cluster/peer_test.go @@ -0,0 +1,117 @@ +package cluster_test + +import ( + "net/http" + "net/http/httptest" + "strings" + + clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" + + "github.com/gorilla/websocket" + "github.com/labstack/echo/v4" + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Peer link handler", func() { + var ( + srv *httptest.Server + sessions chan *yamux.Session + ) + + BeforeEach(func() { + sessions = make(chan *yamux.Session, 1) + e := echo.New() + clusterep.RegisterClusterRoutes(e, "peer-token", func(_ string, s *yamux.Session) { + sessions <- s + }) + srv = httptest.NewServer(e) + DeferCleanup(srv.Close) + }) + + wsURL := func(s *httptest.Server) string { + return "ws" + strings.TrimPrefix(s.URL, "http") + "/api/cluster/peer?id=peer-1" + } + + It("rejects a connection with no token", func() { + _, resp, err := websocket.DefaultDialer.Dial(wsURL(srv), nil) + Expect(err).To(HaveOccurred()) + Expect(resp).ToNot(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + + It("rejects a connection with the wrong token", func() { + h := http.Header{} + h.Set("Authorization", "Bearer wrong") + _, resp, err := websocket.DefaultDialer.Dial(wsURL(srv), h) + Expect(err).To(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + + It("accepts an authenticated peer and yields a usable yamux session", func() { + h := http.Header{} + h.Set("Authorization", "Bearer peer-token") + conn, _, err := websocket.DefaultDialer.Dial(wsURL(srv), h) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + + var serverSess *yamux.Session + Eventually(sessions, "5s").Should(Receive(&serverSess)) + Expect(serverSess).ToNot(BeNil()) + + // The client wraps its side as a yamux CLIENT and opens a stream; the + // server must accept it. This proves the WebSocket was adapted into a + // stream-oriented conn correctly, which is the part most likely to be + // subtly wrong. + clientSess, err := yamux.Client(clusterep.WebsocketConn(conn), nil, nil) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = clientSess.Close() }) + + go func() { + defer GinkgoRecover() + st, e := clientSess.OpenStream(GinkgoT().Context()) + if e == nil { + _, _ = st.Write([]byte("hello")) + } + }() + + accepted := make(chan []byte, 1) + go func() { + defer GinkgoRecover() + st, e := serverSess.AcceptStream() + if e != nil { + return + } + buf := make([]byte, 5) + if _, e := st.Read(buf); e == nil { + accepted <- buf + } + }() + Eventually(accepted, "10s").Should(Receive(Equal([]byte("hello")))) + }) + + It("reports the peer id it was given", func() { + h := http.Header{} + h.Set("Authorization", "Bearer peer-token") + ids := make(chan string, 1) + e := echo.New() + clusterep.RegisterClusterRoutes(e, "peer-token", func(id string, _ *yamux.Session) { ids <- id }) + s2 := httptest.NewServer(e) + DeferCleanup(s2.Close) + + conn, _, err := websocket.DefaultDialer.Dial(wsURL(s2), h) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + + Eventually(ids, "5s").Should(Receive(Equal("peer-1"))) + }) +}) + +var _ = Describe("Peer link auth prefix", func() { + It("is covered by the alternative-authentication prefix list", func() { + // /api/cluster/ authenticates with the cluster token, not the global + // session middleware, so it must be listed or every peer dial 401s. + Expect(clusterep.AlternativeAuthPrefix).To(Equal("/api/cluster/")) + }) +}) diff --git a/core/http/endpoints/cluster/wsconn.go b/core/http/endpoints/cluster/wsconn.go new file mode 100644 index 000000000000..1f1f5306ccd2 --- /dev/null +++ b/core/http/endpoints/cluster/wsconn.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: MIT + +package cluster + +import ( + "fmt" + "io" + "net" + "sync" + "time" + + "github.com/gorilla/websocket" +) + +// WebsocketConn adapts a gorilla WebSocket into the net.Conn that a yamux +// session drives. +// +// The two disagree about framing: WebSocket delivers whole messages, yamux +// wants an undelimited byte stream. The adapter therefore keeps the reader of +// the message it is part-way through between calls, so a Read whose buffer is +// smaller than the message hands back a prefix now and the rest next time +// instead of dropping the tail. That case is not hypothetical: yamux reads +// through a 4 KiB bufio.Reader while a single stream write can put a much +// larger data frame on the wire in one Write, so any message above the buffer +// size is read in pieces. +// +// The returned conn is safe for one reader and one writer concurrently, which +// is all yamux uses: its recvLoop reads and its sendLoop writes. It is not a +// general-purpose net.Conn. +func WebsocketConn(ws *websocket.Conn) net.Conn { + return &wsConn{ws: ws} +} + +type wsConn struct { + ws *websocket.Conn + + // readMu guards frame, which carries a partially consumed message across + // Read calls. gorilla allows a single concurrent reader, and this keeps + // the adapter to that contract even if a caller reads from two goroutines. + readMu sync.Mutex + frame io.Reader + + // writeMu keeps to gorilla's one-concurrent-writer contract. + writeMu sync.Mutex +} + +func (c *wsConn) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + + c.readMu.Lock() + defer c.readMu.Unlock() + + for { + if c.frame == nil { + messageType, r, err := c.ws.NextReader() + if err != nil { + return 0, translateReadErr(err) + } + // Binary is the only type this link speaks. Skipping an unexpected + // text message would silently desynchronise the yamux framing, so + // it is reported instead. + if messageType != websocket.BinaryMessage { + return 0, fmt.Errorf("cluster: peer link received websocket message type %d, want binary", messageType) + } + c.frame = r + } + + n, err := c.frame.Read(p) + if err == io.EOF { + // End of one message, not end of the stream: drop the reader so + // the next call pulls the next message. Passing io.EOF up would + // end the yamux session at an arbitrary message boundary. + c.frame = nil + err = nil + } + if n > 0 || err != nil { + return n, err + } + // A zero-length message yields nothing to return, and (0, nil) reads + // look like a stalled stream to some callers, so wait for the next one. + } +} + +func (c *wsConn) Write(p []byte) (int, error) { + c.writeMu.Lock() + defer c.writeMu.Unlock() + + if err := c.ws.WriteMessage(websocket.BinaryMessage, p); err != nil { + return 0, err + } + return len(p), nil +} + +// Close drops the underlying network connection without negotiating a +// WebSocket close handshake. yamux has already sent its own go-away by this +// point, and a close frame would need the write lock that a blocked sendLoop +// may still hold. +func (c *wsConn) Close() error { + return c.ws.Close() +} + +func (c *wsConn) LocalAddr() net.Addr { return c.ws.LocalAddr() } +func (c *wsConn) RemoteAddr() net.Addr { return c.ws.RemoteAddr() } + +func (c *wsConn) SetDeadline(t time.Time) error { + if err := c.ws.SetReadDeadline(t); err != nil { + return err + } + return c.ws.SetWriteDeadline(t) +} + +func (c *wsConn) SetReadDeadline(t time.Time) error { return c.ws.SetReadDeadline(t) } +func (c *wsConn) SetWriteDeadline(t time.Time) error { return c.ws.SetWriteDeadline(t) } + +// translateReadErr maps a peer hanging up cleanly onto io.EOF, which is how a +// yamux session recognises a normal ending. Any other close code, and any +// transport error, is passed through so the session reports a real failure. +func translateReadErr(err error) error { + if websocket.IsCloseError(err, + websocket.CloseNormalClosure, + websocket.CloseGoingAway, + websocket.CloseNoStatusReceived, + ) { + return io.EOF + } + return err +} diff --git a/core/http/endpoints/cluster/wsconn_test.go b/core/http/endpoints/cluster/wsconn_test.go new file mode 100644 index 000000000000..f3ce4ddf0261 --- /dev/null +++ b/core/http/endpoints/cluster/wsconn_test.go @@ -0,0 +1,216 @@ +package cluster_test + +import ( + "bytes" + "crypto/rand" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "time" + + clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" + + "github.com/gorilla/websocket" + "github.com/labstack/echo/v4" + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// wsPair returns the two ends of one live WebSocket connection. +func wsPair() (clientSide, serverSide *websocket.Conn) { + GinkgoHelper() + + upgrader := websocket.Upgrader{} + accepted := make(chan *websocket.Conn, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ws, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + accepted <- ws + })) + DeferCleanup(srv.Close) + + c, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = c.Close() }) + + var s *websocket.Conn + Eventually(accepted, "5s").Should(Receive(&s)) + DeferCleanup(func() { _ = s.Close() }) + + return c, s +} + +var _ = Describe("WebsocketConn framing", func() { + // The specs below exist because the brief's end-to-end yamux spec cannot + // catch a lost message tail: yamux reads through a 4 KiB bufio.Reader, so + // every small message arrives whole no matter how the adapter behaves. + // These drive the adapter directly with buffers smaller than the message. + + It("returns the rest of a message on the following Read", func() { + clientWS, serverWS := wsPair() + writer := clusterep.WebsocketConn(clientWS) + reader := clusterep.WebsocketConn(serverWS) + + // A lost tail would otherwise park the reassembly below forever; with a + // deadline it fails as a timeout on the read that has nothing left. + Expect(reader.SetReadDeadline(time.Now().Add(10 * time.Second))).To(Succeed()) + + payload := []byte("0123456789abcdefghijklmnopqrstuvwxyz") + n, err := writer.Write(payload) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(len(payload))) + + // Deliberately smaller than the message: a naive adapter that starts a + // fresh NextReader on every call drops everything past the first 7 + // bytes, and this reassembly fails. + got := make([]byte, 0, len(payload)) + buf := make([]byte, 7) + for len(got) < len(payload) { + read, err := reader.Read(buf) + Expect(err).ToNot(HaveOccurred()) + Expect(read).To(BeNumerically(">", 0)) + Expect(read).To(BeNumerically("<=", len(buf))) + got = append(got, buf[:read]...) + } + Expect(got).To(Equal(payload)) + }) + + It("streams a message larger than the yamux read buffer without loss or reordering", func() { + clientWS, serverWS := wsPair() + writer := clusterep.WebsocketConn(clientWS) + reader := clusterep.WebsocketConn(serverWS) + + Expect(reader.SetReadDeadline(time.Now().Add(20 * time.Second))).To(Succeed()) + + payload := make([]byte, 256*1024) + _, err := rand.Read(payload) + Expect(err).ToNot(HaveOccurred()) + + go func() { + defer GinkgoRecover() + _, _ = writer.Write(payload) + }() + + // 4096 is the buffer yamux's bufio.Reader actually hands down. + got := make([]byte, len(payload)) + _, err = io.ReadFull(reader, got) + Expect(err).ToNot(HaveOccurred()) + Expect(bytes.Equal(got, payload)).To(BeTrue()) + }) + + It("presents consecutive messages as one continuous byte stream", func() { + clientWS, serverWS := wsPair() + writer := clusterep.WebsocketConn(clientWS) + reader := clusterep.WebsocketConn(serverWS) + + Expect(reader.SetReadDeadline(time.Now().Add(10 * time.Second))).To(Succeed()) + + for _, chunk := range []string{"abc", "", "de", "fghij"} { + _, err := writer.Write([]byte(chunk)) + Expect(err).ToNot(HaveOccurred()) + } + + // A read spanning three messages must be satisfied, and the empty + // message must not surface as a premature (0, nil) or an EOF. + got := make([]byte, 10) + _, err := io.ReadFull(reader, got) + Expect(err).ToNot(HaveOccurred()) + Expect(string(got)).To(Equal("abcdefghij")) + }) + + It("reports a clean peer close as io.EOF", func() { + clientWS, serverWS := wsPair() + reader := clusterep.WebsocketConn(serverWS) + + Expect(clientWS.WriteMessage(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))).To(Succeed()) + + _, err := reader.Read(make([]byte, 8)) + Expect(err).To(MatchError(io.EOF)) + }) + + It("refuses a text message rather than desynchronising the stream", func() { + clientWS, serverWS := wsPair() + reader := clusterep.WebsocketConn(serverWS) + + Expect(clientWS.WriteMessage(websocket.TextMessage, []byte("not a frame"))).To(Succeed()) + + _, err := reader.Read(make([]byte, 32)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("want binary")) + }) + + It("satisfies net.Conn, including the deadlines yamux sets on every write", func() { + clientWS, _ := wsPair() + var conn net.Conn = clusterep.WebsocketConn(clientWS) + + Expect(conn.LocalAddr()).ToNot(BeNil()) + Expect(conn.RemoteAddr()).ToNot(BeNil()) + // yamux's sendLoop calls SetWriteDeadline before every flush, so an + // adapter that dropped the call would let a stalled peer block the + // session forever instead of failing it. + Expect(conn.SetWriteDeadline(time.Now().Add(time.Minute))).To(Succeed()) + Expect(conn.SetReadDeadline(time.Now().Add(time.Minute))).To(Succeed()) + Expect(conn.SetDeadline(time.Time{})).To(Succeed()) + }) +}) + +var _ = Describe("Peer link payloads", func() { + It("carries a payload far larger than one yamux frame end to end", func() { + sessions := make(chan *yamux.Session, 1) + e := echo.New() + clusterep.RegisterClusterRoutes(e, "peer-token", func(_ string, s *yamux.Session) { sessions <- s }) + srv := httptest.NewServer(e) + DeferCleanup(srv.Close) + + h := http.Header{} + h.Set("Authorization", "Bearer peer-token") + conn, _, err := websocket.DefaultDialer.Dial( + "ws"+strings.TrimPrefix(srv.URL, "http")+"/api/cluster/peer?id=peer-1", h) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + + var serverSess *yamux.Session + Eventually(sessions, "5s").Should(Receive(&serverSess)) + + clientSess, err := yamux.Client(clusterep.WebsocketConn(conn), nil, nil) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = clientSess.Close() }) + + payload := make([]byte, 1<<20) + _, err = rand.Read(payload) + Expect(err).ToNot(HaveOccurred()) + + go func() { + defer GinkgoRecover() + st, e := clientSess.OpenStream(GinkgoT().Context()) + if e != nil { + return + } + defer func() { _ = st.Close() }() + _, _ = io.Copy(st, bytes.NewReader(payload)) + }() + + received := make(chan []byte, 1) + go func() { + defer GinkgoRecover() + st, e := serverSess.AcceptStream() + if e != nil { + return + } + buf := make([]byte, len(payload)) + if _, e := io.ReadFull(st, buf); e == nil { + received <- buf + } + }() + + var got []byte + Eventually(received, "30s").Should(Receive(&got)) + Expect(bytes.Equal(got, payload)).To(BeTrue()) + }) +}) From 73da8089d6cd011c2a1d5d1d1e38e7031a3b94ec Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 23:19:25 +0000 Subject: [PATCH 030/153] fix(cluster): stop reading a bare EOF as a clean ending A dead yamux session does not always arrive wrapped. Session.close publishes shutdownErr and closes shutdownCh before it force-closes the streams, so a Write or Close landing in that window gets the raw cause back instead (session.go:507-510, 528-533), and for a peer that vanished the raw cause is a bare io.EOF. The generic io.EOF clause then reported the dead session as a clean completion. Remove the clause. A clean read-side EOF never reached it anyway: io.Copy consumes that and reports nil, and neither *yamux.Stream nor *net.TCPConn takes a WriteTo/ReadFrom path that would hand one back. Every existing spec still passes, the io.EOF entry in the normal-termination table included, which is what showed the branch was dead for legitimate endings and live only for the bug. Add a spec driving a real yamux session end to end. Every mux shape until now was a synthesized error, which is exactly why a race inside the real library stayed invisible. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/splice.go | 31 +++++++--- core/services/cluster/splice_test.go | 88 +++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 10 deletions(-) diff --git a/core/services/cluster/splice.go b/core/services/cluster/splice.go index ba5ac28af294..5df97e890da0 100644 --- a/core/services/cluster/splice.go +++ b/core/services/cluster/splice.go @@ -69,15 +69,22 @@ func copyStream(dst io.Writer, src io.Reader) error { } // normalizeStreamErr drops the endings that mean the conversation is over -// rather than broken. io.EOF is the clean end of a stream, net.ErrClosed is -// what a socket reports once it or its peer has been closed, and -// io.ErrClosedPipe is the same condition on an in-memory pipe. +// rather than broken: net.ErrClosed is what a socket reports once it or its +// peer has been closed, and io.ErrClosedPipe is the same condition on an +// in-memory pipe. +// +// io.EOF is deliberately absent. A clean read-side EOF never gets this far, +// because io.Copy consumes it and reports nil, and neither *yamux.Stream nor +// *net.TCPConn takes a WriteTo/ReadFrom path that would hand one back. So a +// bare io.EOF arriving here came from a failing Write or Close, where it means +// the peer is gone, and yamux produces exactly that when a Write races its +// session's shutdown (see isMuxSessionFailure). // // The mux checks run first, and that ordering is load-bearing: a dying yamux -// session hands every live stream its own cause wrapped up (session.go:330), -// and that cause is routinely io.EOF or a closed-socket error, so consulting -// the generic endings first would report a peer that vanished mid-request as a -// clean completion. +// session usually hands every live stream its own cause wrapped up +// (session.go:330), and that cause is routinely a closed-socket error, so +// consulting the generic endings first would report a peer that vanished +// mid-request as a clean completion. func normalizeStreamErr(err error) error { if err == nil { return nil @@ -88,8 +95,7 @@ func normalizeStreamErr(err error) error { if isMuxStreamTeardown(err) { return nil } - if errors.Is(err, io.EOF) || - errors.Is(err, net.ErrClosed) || + if errors.Is(err, net.ErrClosed) || errors.Is(err, io.ErrClosedPipe) { return nil } @@ -122,6 +128,13 @@ func isMuxSessionFailure(err error) bool { // the cause, so the bare sentinel means this stream was reset and a // wrapped one means the session died under it. Identity is what separates // them; errors.Is cannot. + // + // Wrapped is not the only way a dead session shows up, though. close() + // publishes shutdownErr and closes shutdownCh before it force-closes the + // streams, so a Write or Close landing in that window gets the raw cause + // back instead (session.go:507-510, 528-533). That form is unrecognisable + // as yamux at all, which is why normalizeStreamErr no longer forgives a + // bare io.EOF: for a peer that vanished, the raw cause is precisely io.EOF. return errors.Is(err, yamux.ErrStreamReset) && err != yamux.ErrStreamReset } diff --git a/core/services/cluster/splice_test.go b/core/services/cluster/splice_test.go index f18618b6b2cd..7bbe3246de05 100644 --- a/core/services/cluster/splice_test.go +++ b/core/services/cluster/splice_test.go @@ -1,6 +1,7 @@ package cluster_test import ( + "context" "errors" "fmt" "io" @@ -206,6 +207,23 @@ var _ = Describe("Splice", func() { Entry("an internal-error go-away", &yamux.GoAwayError{Remote: true, ErrorCode: 2}), ) + // A bare io.EOF can only reach Splice from a failing Write. io.Copy + // never surfaces a clean read-side EOF, and yamux hands out the raw + // cause rather than the wrapped one when a Write or Close races + // Session.close's shutdown window (session.go:507-510), so for a + // vanished peer this IS the dead session, arriving unwrapped. + It("reports a write that fails with a bare EOF", func() { + sink := &scriptedStream{writeErr: io.EOF} + source := &scriptedStream{feeds: true} + + done := make(chan error, 1) + go func() { done <- cluster.Splice(sink, source) }() + + var err error + Eventually(done, "5s").Should(Receive(&err)) + Expect(err).To(MatchError(io.EOF)) + }) + // The distinction the classifier turns on, in one spec: yamux uses the // same sentinel for "this stream was reset", which Splice provokes // itself and must stay quiet about, and as the head of the wrapped @@ -254,6 +272,57 @@ var _ = Describe("Splice", func() { Eventually(done, "5s").Should(Receive(BeNil())) }) + // Everything above feeds Splice a synthesized error. This one drives a + // real yamux session, because the shapes a live library produces are + // not always the ones its source suggests: the bug this spec was added + // alongside was a race inside Session.close that no synthesized error + // could show. It asserts only that a dead session is reported, not how + // it is spelled, since which of the two forms arrives is a race. + It("reports a real yamux session dying under a live stream", func() { + clientConn, serverConn := net.Pipe() + client, err := yamux.Client(clientConn, nil, nil) + Expect(err).ToNot(HaveOccurred()) + server, err := yamux.Server(serverConn, nil, nil) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { + _ = client.Close() + _ = server.Close() + }) + + accepted := make(chan *yamux.Stream, 1) + go func() { + defer GinkgoRecover() + far, err := server.AcceptStream() + if err != nil { + close(accepted) + return + } + accepted <- far + }() + + stream, err := client.OpenStream(context.Background()) + Expect(err).ToNot(HaveOccurred()) + // Push a byte so the stream is established on both sides before + // the session is killed. + _, err = stream.Write([]byte("x")) + Expect(err).ToNot(HaveOccurred()) + var far *yamux.Stream + Eventually(accepted, "10s").Should(Receive(&far)) + _, err = far.Read(make([]byte, 1)) + Expect(err).ToNot(HaveOccurred()) + + done := make(chan error, 1) + go func() { done <- cluster.Splice(stream, &scriptedStream{}) }() + + // The peer's process disappears: the connection carrying the + // session goes away, which kills every stream riding on it. + Expect(serverConn.Close()).To(Succeed()) + + var spliceErr error + Eventually(done, "10s").Should(Receive(&spliceErr)) + Expect(spliceErr).To(HaveOccurred()) + }) + // The anti-leak guarantee, which the pipe specs cannot see because // their parked copy is released too quickly to catch Splice in the // act. Waking a copy is asynchronous on a real stream (yamux's Close @@ -281,7 +350,11 @@ var _ = Describe("Splice", func() { // parks in Read until Close, standing in for an idle half of a live stream. type scriptedStream struct { readErr error + writeErr error closeErr error + // feeds makes Read produce bytes instead of parking, so a spec can keep a + // direction copying until its destination fails. + feeds bool // holdReadPastClose keeps a parked Read blocked until release is called, // standing in for the gap between a Close waking a reader and that reader // running. Without it, Close releases the Read as a real stream does. @@ -307,11 +380,24 @@ func (s *scriptedStream) Read(p []byte) (int, error) { if s.readErr != nil { return 0, s.readErr } + if s.feeds { + select { + case <-s.gate(): + return 0, io.EOF + default: + return len(p), nil + } + } <-s.gate() return 0, io.EOF } -func (s *scriptedStream) Write(p []byte) (int, error) { return len(p), nil } +func (s *scriptedStream) Write(p []byte) (int, error) { + if s.writeErr != nil { + return 0, s.writeErr + } + return len(p), nil +} func (s *scriptedStream) Close() error { s.closeN.Add(1) From 9fdd0bef55e893f7bf547d392484e732bb9f59d6 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 31 Aug 2026 23:44:11 +0000 Subject: [PATCH 031/153] fix(cluster): own the peer auth prefix in auth, and pin what the specs claimed The prefix constant moves to core/http/auth beside the check that uses it, and the endpoints package derives its route from there. Seven sibling endpoint packages already import auth, so the previous direction would have deadlocked the build as soon as this one registered in RouteFeatureRegistry, and it was dragging echo, gorilla/websocket and yamux into unrelated service packages. Four properties were argued in comments and held by nothing. Flipping the empty-token check to fail open, making SetWriteDeadline a no-op, returning a zero-length read for a zero-length message, and dropping the recover around the callback all left the suite green. Each now fails a spec that asserts the behaviour rather than the setter's return value. SetWriteDeadline takes the write mutex because gorilla keeps that deadline in a plain struct field applied at the next flush; SetReadDeadline must not take the read mutex, since it goes straight to the net.Conn and would otherwise block behind the read it exists to unblock. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/auth/public_routes.go | 16 ++-- core/http/endpoints/cluster/peer.go | 28 +++++-- core/http/endpoints/cluster/peer_test.go | 76 ++++++++++++++++++ core/http/endpoints/cluster/wsconn.go | 26 ++++-- core/http/endpoints/cluster/wsconn_test.go | 92 +++++++++++++++++++--- 5 files changed, 211 insertions(+), 27 deletions(-) diff --git a/core/http/auth/public_routes.go b/core/http/auth/public_routes.go index e1a79b3ab12f..25bb25177ad3 100644 --- a/core/http/auth/public_routes.go +++ b/core/http/auth/public_routes.go @@ -5,8 +5,6 @@ package auth import ( "net/http" "strings" - - clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" ) type publicRouteRule struct { @@ -82,13 +80,17 @@ func isPublicRoute(method, path string) bool { return false } +// ClusterPathPrefix is the replica-to-replica namespace. Its handlers check the +// cluster token in the Authorization header themselves, so the check below lets +// them through the global session middleware. The cluster endpoints build their +// route paths from this same constant, which is why it lives here beside the +// check rather than beside the handlers: the exemption and the route it exempts +// cannot then be changed independently. +const ClusterPathPrefix = "/api/cluster/" + // usesAlternativeAuthentication identifies requests whose credentials are // validated by route-group middleware instead of the global auth middleware. func usesAlternativeAuthentication(path string) bool { - // The peer link carries the cluster token in an Authorization header that - // no browser session ever sets, and its handler checks that token itself. - // The prefix comes from the endpoints package so the route and the - // exemption cannot drift apart. return strings.HasPrefix(path, "/api/node/") || - strings.HasPrefix(path, clusterep.AlternativeAuthPrefix) + strings.HasPrefix(path, ClusterPathPrefix) } diff --git a/core/http/endpoints/cluster/peer.go b/core/http/endpoints/cluster/peer.go index 4b6ef542caf4..5ff916f6c93e 100644 --- a/core/http/endpoints/cluster/peer.go +++ b/core/http/endpoints/cluster/peer.go @@ -9,18 +9,25 @@ package cluster import ( "crypto/subtle" "net/http" + "strings" "github.com/gorilla/websocket" "github.com/labstack/echo/v4" "github.com/libp2p/go-yamux/v5" + "github.com/mudler/LocalAI/core/http/auth" "github.com/mudler/xlog" ) // AlternativeAuthPrefix is the path prefix whose credentials are checked by -// this package rather than by the global session middleware. The auth layer -// consults this same constant, so the two cannot drift apart and leave every -// peer dial answering 401. -const AlternativeAuthPrefix = "/api/cluster/" +// this package rather than by the global session middleware. +// +// It is the auth package's own constant rather than a second copy: auth decides +// which paths bypass the session middleware, so owning the prefix there and +// deriving the route from it here means a change to either one moves both. The +// dependency runs endpoints -> auth, the direction the rest of core/http flows; +// pointing it the other way would deadlock the build as soon as this package +// needs anything from auth, which registering in RouteFeatureRegistry will. +const AlternativeAuthPrefix = auth.ClusterPathPrefix // PeerPath is the route a peer replica dials. const PeerPath = AlternativeAuthPrefix + "peer" @@ -80,6 +87,16 @@ func PeerHandler(token string, onSession func(peerID string, sess *yamux.Session } xlog.Debug("cluster peer link established", "peer", peerID, "remote", ws.RemoteAddr().String()) + // net/http recovers a panic from this goroutine but does not close a + // hijacked connection afterwards, so a panicking callback would leave + // the peer holding a link nobody accepts streams on: its opens would + // fill the 256-deep backlog and then hang without an error. + defer func() { + if r := recover(); r != nil { + _ = sess.Close() + panic(r) + } + }() onSession(peerID, sess) return nil } @@ -95,9 +112,10 @@ func authorizedPeer(r *http.Request, expected string) bool { if expected == "" { return false } + // RFC 7235 makes the scheme case-insensitive; the token after it is not. const prefix = "Bearer " header := r.Header.Get("Authorization") - if len(header) < len(prefix) || header[:len(prefix)] != prefix { + if len(header) < len(prefix) || !strings.EqualFold(header[:len(prefix)], prefix) { return false } return subtle.ConstantTimeCompare([]byte(header[len(prefix):]), []byte(expected)) == 1 diff --git a/core/http/endpoints/cluster/peer_test.go b/core/http/endpoints/cluster/peer_test.go index b4aa98356c13..a9f027ff0c56 100644 --- a/core/http/endpoints/cluster/peer_test.go +++ b/core/http/endpoints/cluster/peer_test.go @@ -106,6 +106,82 @@ var _ = Describe("Peer link handler", func() { Eventually(ids, "5s").Should(Receive(Equal("peer-1"))) }) + It("rejects every dial when no cluster token is configured", func() { + // The route is registered in every deployment, so an empty configured + // token must authorize nobody. Failing open the way the worker + // file-transfer server's checkBearerToken does would publish an + // unauthenticated yamux multiplexer to anyone who can reach the port. + e := echo.New() + accepted := make(chan *yamux.Session, 1) + clusterep.RegisterClusterRoutes(e, "", func(_ string, sess *yamux.Session) { accepted <- sess }) + s2 := httptest.NewServer(e) + DeferCleanup(s2.Close) + + for _, header := range []http.Header{nil, {"Authorization": []string{"Bearer "}}, {"Authorization": []string{"Bearer anything"}}} { + _, resp, err := websocket.DefaultDialer.Dial(wsURL(s2), header) + Expect(err).To(HaveOccurred()) + Expect(resp).ToNot(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + } + Expect(accepted).ToNot(Receive()) + }) + + It("accepts the bearer scheme in any case", func() { + // RFC 7235 makes the scheme case-insensitive. The token after it is not. + h := http.Header{} + h.Set("Authorization", "bearer peer-token") + conn, _, err := websocket.DefaultDialer.Dial(wsURL(srv), h) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + Eventually(sessions, "5s").Should(Receive()) + }) + + It("closes the session when the callback panics", func() { + // net/http recovers the panic but leaves the hijacked socket open, so + // without the handler's own recover the peer would keep a link nobody + // ever accepts streams on. + e := echo.New() + clusterep.RegisterClusterRoutes(e, "peer-token", func(_ string, _ *yamux.Session) { + panic("callback exploded") + }) + s2 := httptest.NewServer(e) + DeferCleanup(func() { + // A hijacked connection the handler never closed would park + // httptest's Close forever, turning the assertion below into a + // suite hang. Forcing the conns shut keeps the failure legible. + s2.CloseClientConnections() + s2.Close() + }) + + h := http.Header{} + h.Set("Authorization", "Bearer peer-token") + conn, _, err := websocket.DefaultDialer.Dial(wsURL(s2), h) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + + clientSess, err := yamux.Client(clusterep.WebsocketConn(conn), nil, nil) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = clientSess.Close() }) + + // Asserting on OpenStream would hang rather than fail: yamux only + // acknowledges a stream once the peer accepts it, and the leak this + // pins is precisely that nobody ever will. The session's own liveness + // is the observable that answers in both directions. + Eventually(clientSess.IsClosed, "10s").Should(BeTrue()) + }) + + It("rejects an authenticated dial that names no peer", func() { + // The session is keyed by peer id, so a nameless link could never be + // looked up again; refusing it is cheaper than leaking it. + h := http.Header{} + h.Set("Authorization", "Bearer peer-token") + _, resp, err := websocket.DefaultDialer.Dial( + "ws"+strings.TrimPrefix(srv.URL, "http")+"/api/cluster/peer", h) + Expect(err).To(HaveOccurred()) + Expect(resp).ToNot(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + Expect(sessions).ToNot(Receive()) + }) }) var _ = Describe("Peer link auth prefix", func() { diff --git a/core/http/endpoints/cluster/wsconn.go b/core/http/endpoints/cluster/wsconn.go index 1f1f5306ccd2..ca932d741cc8 100644 --- a/core/http/endpoints/cluster/wsconn.go +++ b/core/http/endpoints/cluster/wsconn.go @@ -24,8 +24,9 @@ import ( // larger data frame on the wire in one Write, so any message above the buffer // size is read in pieces. // -// The returned conn is safe for one reader and one writer concurrently, which -// is all yamux uses: its recvLoop reads and its sendLoop writes. It is not a +// The returned conn is safe for one reader and one writer concurrently, plus a +// third goroutine setting deadlines, which is what the relay needs: yamux's +// sendLoop writes while a supervisor arms an idle deadline. It is not a // general-purpose net.Conn. func WebsocketConn(ws *websocket.Conn) net.Conn { return &wsConn{ws: ws} @@ -105,14 +106,27 @@ func (c *wsConn) LocalAddr() net.Addr { return c.ws.LocalAddr() } func (c *wsConn) RemoteAddr() net.Addr { return c.ws.RemoteAddr() } func (c *wsConn) SetDeadline(t time.Time) error { - if err := c.ws.SetReadDeadline(t); err != nil { + if err := c.SetReadDeadline(t); err != nil { return err } - return c.ws.SetWriteDeadline(t) + return c.SetWriteDeadline(t) } -func (c *wsConn) SetReadDeadline(t time.Time) error { return c.ws.SetReadDeadline(t) } -func (c *wsConn) SetWriteDeadline(t time.Time) error { return c.ws.SetWriteDeadline(t) } +// SetReadDeadline needs no lock, and must not take readMu: gorilla passes the +// read deadline straight to the underlying net.Conn, whose deadline setters are +// safe to call from another goroutine, and taking readMu would block behind the +// parked Read this call exists to unblock. +func (c *wsConn) SetReadDeadline(t time.Time) error { return c.ws.SetReadDeadline(t) } + +// SetWriteDeadline takes writeMu because gorilla stores the write deadline in a +// plain struct field (conn.go:787) and applies it when it next flushes, so +// setting it while a write is in flight is a data race, not merely a late bound. +func (c *wsConn) SetWriteDeadline(t time.Time) error { + c.writeMu.Lock() + defer c.writeMu.Unlock() + + return c.ws.SetWriteDeadline(t) +} // translateReadErr maps a peer hanging up cleanly onto io.EOF, which is how a // yamux session recognises a normal ending. Any other close code, and any diff --git a/core/http/endpoints/cluster/wsconn_test.go b/core/http/endpoints/cluster/wsconn_test.go index f3ce4ddf0261..5ef3fcf3b1e2 100644 --- a/core/http/endpoints/cluster/wsconn_test.go +++ b/core/http/endpoints/cluster/wsconn_test.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "net/http/httptest" + "os" "strings" "time" @@ -115,14 +116,34 @@ var _ = Describe("WebsocketConn framing", func() { Expect(err).ToNot(HaveOccurred()) } - // A read spanning three messages must be satisfied, and the empty - // message must not surface as a premature (0, nil) or an EOF. + // A read spanning several messages must be satisfied: a message + // boundary is not the end of the stream. got := make([]byte, 10) _, err := io.ReadFull(reader, got) Expect(err).ToNot(HaveOccurred()) Expect(string(got)).To(Equal("abcdefghij")) }) + It("never hands back a zero-length read for a zero-length message", func() { + clientWS, serverWS := wsPair() + writer := clusterep.WebsocketConn(clientWS) + reader := clusterep.WebsocketConn(serverWS) + + Expect(reader.SetReadDeadline(time.Now().Add(10 * time.Second))).To(Succeed()) + + // An empty message carries nothing to return. Handing back (0, nil) + // would be legal for io.Reader but reads as a stalled stream to callers + // that loop on n, so the adapter waits for the next message instead. + _, err := writer.Write(nil) + Expect(err).ToNot(HaveOccurred()) + _, err = writer.Write([]byte("xy")) + Expect(err).ToNot(HaveOccurred()) + + n, err := reader.Read(make([]byte, 8)) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(2)) + }) + It("reports a clean peer close as io.EOF", func() { clientWS, serverWS := wsPair() reader := clusterep.WebsocketConn(serverWS) @@ -145,18 +166,71 @@ var _ = Describe("WebsocketConn framing", func() { Expect(err.Error()).To(ContainSubstring("want binary")) }) - It("satisfies net.Conn, including the deadlines yamux sets on every write", func() { + It("satisfies net.Conn", func() { clientWS, _ := wsPair() var conn net.Conn = clusterep.WebsocketConn(clientWS) Expect(conn.LocalAddr()).ToNot(BeNil()) Expect(conn.RemoteAddr()).ToNot(BeNil()) - // yamux's sendLoop calls SetWriteDeadline before every flush, so an - // adapter that dropped the call would let a stalled peer block the - // session forever instead of failing it. - Expect(conn.SetWriteDeadline(time.Now().Add(time.Minute))).To(Succeed()) - Expect(conn.SetReadDeadline(time.Now().Add(time.Minute))).To(Succeed()) - Expect(conn.SetDeadline(time.Time{})).To(Succeed()) + }) + + It("enforces a write deadline, which yamux arms before every flush", func() { + clientWS, _ := wsPair() + conn := clusterep.WebsocketConn(clientWS) + + // Asserting that the setter returns nil would prove nothing: gorilla + // only records the deadline and applies it at the next flush. The write + // below is what shows the deadline reached the socket, and an adapter + // that swallowed the call would let a stalled peer block yamux's send + // loop forever instead of failing it. + Expect(conn.SetWriteDeadline(time.Now().Add(-time.Second))).To(Succeed()) + _, err := conn.Write([]byte("x")) + Expect(err).To(HaveOccurred()) + Expect(os.IsTimeout(err)).To(BeTrue(), "want a timeout, got %v", err) + }) + + It("enforces a read deadline, which is how a parked reader is unblocked", func() { + clientWS, _ := wsPair() + conn := clusterep.WebsocketConn(clientWS) + + Expect(conn.SetReadDeadline(time.Now().Add(-time.Second))).To(Succeed()) + _, err := conn.Read(make([]byte, 8)) + Expect(err).To(HaveOccurred()) + Expect(os.IsTimeout(err)).To(BeTrue(), "want a timeout, got %v", err) + }) + + It("arms both directions from SetDeadline", func() { + clientWS, _ := wsPair() + conn := clusterep.WebsocketConn(clientWS) + + Expect(conn.SetDeadline(time.Now().Add(-time.Second))).To(Succeed()) + + _, err := conn.Read(make([]byte, 8)) + Expect(os.IsTimeout(err)).To(BeTrue(), "read: want a timeout, got %v", err) + _, err = conn.Write([]byte("x")) + Expect(os.IsTimeout(err)).To(BeTrue(), "write: want a timeout, got %v", err) + }) + + It("lets a deadline be armed while another goroutine writes", func() { + // Task 5's relay arms an idle deadline from a supervisor goroutine while + // yamux's send loop writes. gorilla keeps the write deadline in a plain + // struct field, so this is a data race unless the adapter serialises it; + // the spec is here to be run under -race, where it would report one. + clientWS, _ := wsPair() + conn := clusterep.WebsocketConn(clientWS) + + done := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(done) + for i := 0; i < 200; i++ { + _, _ = conn.Write([]byte("ping")) + } + }() + for i := 0; i < 200; i++ { + _ = conn.SetWriteDeadline(time.Now().Add(time.Minute)) + } + Eventually(done, "20s").Should(BeClosed()) }) }) From c7cb7e1ca548c695bcd6e2f2f1bbde2c48a9ff6a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 00:09:15 +0000 Subject: [PATCH 032/153] feat(cluster): dial and pool yamux links to peer replicas Distinguishes a peer missing from the registry from a peer that will not answer: the second must never be readable as node absence, or a network hiccup between replicas evicts healthy workers. The distinction is a property of the error type rather than of the call sites. The unreachable error formats its cause into its message and keeps it out of its unwrap chain, so an ErrInstanceNotFound picked up on the dial path cannot reach a caller's absence check. One yamux session is cached per peer and re-dialled when OpenStream on it fails, which is how both a dead transport and a graceful remote go-away arrive. A reset of one stream never reaches the pool, so an abandoned request cannot cost every other worker its link. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/peerlink.go | 302 ++++++++++++++++++ .../cluster/peerlink_internal_test.go | 41 +++ core/services/cluster/peerlink_test.go | 221 +++++++++++++ 3 files changed, 564 insertions(+) create mode 100644 core/services/cluster/peerlink.go create mode 100644 core/services/cluster/peerlink_internal_test.go create mode 100644 core/services/cluster/peerlink_test.go diff --git a/core/services/cluster/peerlink.go b/core/services/cluster/peerlink.go new file mode 100644 index 000000000000..d4a21b4c59b7 --- /dev/null +++ b/core/services/cluster/peerlink.go @@ -0,0 +1,302 @@ +package cluster + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/gorilla/websocket" + "github.com/libp2p/go-yamux/v5" + clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" + "github.com/mudler/xlog" +) + +// ErrPeerUnreachable reports that a peer this deployment knows about could not +// be reached: the dial failed, the peer refused the credentials, or its +// multiplexer would not carry a stream. +// +// It is deliberately NOT a form of ErrInstanceNotFound, and the two must stay +// unmixable. A caller that sees absence is entitled to conclude a node is gone +// and reclaim what it was running; a caller that sees unreachability may only +// retry. Collapsing the two means a network hiccup between two healthy +// replicas evicts healthy workers. See unreachableError for how that is +// enforced rather than merely documented. +var ErrPeerUnreachable = errors.New("cluster: peer unreachable") + +// unreachableError reports a peer that could not be reached, keeping the +// underlying cause in its message and out of its unwrap chain. +// +// Withholding the cause from errors.Is is the point. The dial path resolves a +// peer's address through the registry, so ErrInstanceNotFound is a cause this +// error can genuinely be built over: a row deleted between two attempts, for +// one. If the cause were unwrapped, that failure would satisfy both sentinels +// at once and every caller's absence check would fire on a transport problem. +// The guarantee therefore belongs to the type: no call site can leak absence +// through it, because there is no path by which absence gets out. +type unreachableError struct { + peerID string + cause error +} + +func (e *unreachableError) Error() string { + return fmt.Sprintf("cluster: peer %q unreachable: %v", e.peerID, e.cause) +} + +// Unwrap reports only ErrPeerUnreachable. The cause reaches a human through +// Error() and reaches no error-matching caller at all. +func (e *unreachableError) Unwrap() error { return ErrPeerUnreachable } + +func unreachablePeer(peerID string, cause error) error { + return &unreachableError{peerID: peerID, cause: cause} +} + +// ErrPoolClosed reports an Open on a pool that has been shut down. It is a +// third condition on purpose: the pool being closed is a fact about this +// process and says nothing about whether the peer exists or answers. +var ErrPoolClosed = errors.New("cluster: peer pool is closed") + +const ( + // peerLinkHandshakeTimeout bounds the WebSocket upgrade. It also bounds + // how long Close can wait behind an in-flight dial, since a dial holds the + // per-peer lock Close needs to reach the cached session. + peerLinkHandshakeTimeout = 10 * time.Second + + // peerLinkInitialWindow is the per-stream receive window every stream on a + // peer link starts at, raised from yamux's 256 KiB default. + // + // yamux already bounds head-of-line blocking with MaxMessageSize (64 KiB + // by default), so one stream cannot monopolise the connection whatever the + // window is. What the small default costs is the ramp: a stream carrying a + // multi-megabyte gRPC message spends its first megabytes window-parked, + // paying a round trip per doubling (stream.go:229) before it reaches full + // rate. On a link that is also carrying token streams for other workers, + // that ramp is pure added latency on the bulk transfer for no benefit. + peerLinkInitialWindow = 4 * 1024 * 1024 + + // peerLinkMaxWindow is the ceiling the auto-tuner may grow a stream to, + // raised from yamux's 16 MiB default to cover the bandwidth-delay product + // of a fast cross-zone link (roughly 31 MiB at 10 Gbps and 25 ms). + // + // The window is a cap on data received but not yet read, so the worst case + // a peer can make this replica buffer is MaxIncomingStreams times this, + // which is why MaxIncomingStreams is left at yamux's default rather than + // raised alongside it. + peerLinkMaxWindow = 32 * 1024 * 1024 +) + +// peerLinkConfig returns the yamux configuration for a replica-to-replica link. +func peerLinkConfig() *yamux.Config { + cfg := yamux.DefaultConfig() + cfg.InitialStreamWindowSize = peerLinkInitialWindow + cfg.MaxStreamWindowSize = peerLinkMaxWindow + return cfg +} + +// PeerPool dials peer replicas and keeps one multiplexed session per peer. +// +// A peer link carries traffic for every worker that peer owns, so it is pooled +// rather than dialled per request: a dial per relayed request would add a +// WebSocket handshake to every inference. +// +// The pool needs no knowledge of yamux error shapes to keep its cache honest. +// The two conditions worth reacting to arrive as OpenStream failures and are +// handled by the same retry: a peer that shut down gracefully hands its +// session ErrRemoteGoAway and closes it, and a session whose transport died +// hands out its shutdown error. Conditions scoped to a single stream, such as +// a peer resetting one request, never reach the pool at all, which is right: +// dropping the session over one reset request would tear down every other +// worker's traffic on that link. +type PeerPool struct { + selfID string + token string + reg *Registry + + dialer *websocket.Dialer + + mu sync.Mutex + links map[string]*peerLink + closed bool +} + +// peerLink is the cached session for one peer, plus the lock that serialises +// dialling it. The lock is per-peer so a slow or hanging dial to one peer does +// not hold up opens to any other. +type peerLink struct { + mu sync.Mutex + sess *yamux.Session +} + +// NewPeerPool returns a pool that dials peers as selfID, authenticating with +// the deployment's cluster token. +func NewPeerPool(selfID, token string, reg *Registry) *PeerPool { + return &PeerPool{ + selfID: selfID, + token: token, + reg: reg, + dialer: &websocket.Dialer{ + HandshakeTimeout: peerLinkHandshakeTimeout, + // No Proxy: a peer link is replica-to-replica inside one + // deployment, and honouring HTTP_PROXY would route it through + // whatever egress proxy the environment happens to name. + }, + links: map[string]*peerLink{}, + } +} + +// Open returns a stream to peerID, dialling and caching the session on first +// use. +// +// The errors are three distinct conditions and callers act differently on +// them: ErrInstanceNotFound means the peer is not part of this deployment, +// ErrPeerUnreachable means it is but will not answer, and ErrPoolClosed means +// this process is shutting down. Only the first is node absence. +func (p *PeerPool) Open(ctx context.Context, peerID string) (net.Conn, error) { + l, err := p.link(peerID) + if err != nil { + return nil, err + } + + l.mu.Lock() + defer l.mu.Unlock() + + if l.sess != nil { + st, err := l.sess.OpenStream(ctx) + if err == nil { + return st, nil + } + // A caller whose own context expired must not cost every other worker + // its link: the session is fine, this request is not. + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + // A session that died between calls is the common case, not an + // exception, so this is a debug line and not a warning. + xlog.Debug("cluster peer link session unusable, re-dialling", "peer", peerID, "error", err) + _ = l.sess.Close() + l.sess = nil + } + + sess, err := p.dial(ctx, peerID) + if err != nil { + return nil, err + } + + st, err := sess.OpenStream(ctx) + if err != nil { + // The peer answered and completed a handshake but will not carry a + // stream, which is a transport condition and never absence. + _ = sess.Close() + return nil, unreachablePeer(peerID, err) + } + + l.sess = sess + return st, nil +} + +// link returns the per-peer entry, creating it on first use. +func (p *PeerPool) link(peerID string) (*peerLink, error) { + p.mu.Lock() + defer p.mu.Unlock() + + if p.closed { + return nil, ErrPoolClosed + } + l, ok := p.links[peerID] + if !ok { + l = &peerLink{} + p.links[peerID] = l + } + return l, nil +} + +// dial resolves the peer's advertised address and brings up one yamux client +// session over an authenticated WebSocket. +// +// A registry miss is returned unchanged so ErrInstanceNotFound reaches the +// caller; everything after it is wrapped as unreachable. +func (p *PeerPool) dial(ctx context.Context, peerID string) (*yamux.Session, error) { + inst, err := p.reg.Get(ctx, peerID) + if err != nil { + return nil, err + } + if inst.AdvertisedAddr == "" { + // A registered replica with no address is reachable by nobody. It is + // present, so this is not absence. + return nil, unreachablePeer(peerID, errors.New("peer has no advertised address")) + } + + endpoint := url.URL{ + // Plain ws: replica-to-replica TLS is not part of this phase, and the + // link is authenticated by the cluster token rather than by transport. + Scheme: "ws", + Host: inst.AdvertisedAddr, + Path: clusterep.PeerPath, + RawQuery: url.Values{"id": []string{p.selfID}}.Encode(), + } + header := http.Header{} + header.Set("Authorization", "Bearer "+p.token) + + ws, resp, err := p.dialer.DialContext(ctx, endpoint.String(), header) + if resp != nil && resp.Body != nil { + // gorilla hands back the failed handshake's response so a caller can + // read the status; nothing here needs the body, but it has to be + // drained or the connection is not returned to the transport. + _ = resp.Body.Close() + } + if err != nil { + return nil, unreachablePeer(peerID, err) + } + + // Client side of the mux: the dialling replica owns the odd stream IDs, + // matching the yamux.Server the peer handler puts on its end. + sess, err := yamux.Client(clusterep.WebsocketConn(ws), peerLinkConfig(), nil) + if err != nil { + _ = ws.Close() + return nil, unreachablePeer(peerID, err) + } + + // Close raced this dial. Handing the session back would leak it, since + // Close has already walked the map. + p.mu.Lock() + closed := p.closed + p.mu.Unlock() + if closed { + _ = sess.Close() + return nil, ErrPoolClosed + } + + xlog.Debug("cluster peer link dialled", "peer", peerID, "addr", inst.AdvertisedAddr) + return sess, nil +} + +// Close closes every cached session. It is safe to call twice, and an Open +// after it reports ErrPoolClosed rather than anything a caller could read as +// node absence. +func (p *PeerPool) Close() { + p.mu.Lock() + if p.closed { + p.mu.Unlock() + return + } + p.closed = true + links := p.links + p.links = nil + p.mu.Unlock() + + // Each session is closed under its own peer lock rather than under p.mu, + // so closing the pool cannot deadlock against an Open that is mid-dial and + // about to take p.mu to re-check p.closed. + for _, l := range links { + l.mu.Lock() + if l.sess != nil { + _ = l.sess.Close() + l.sess = nil + } + l.mu.Unlock() + } +} diff --git a/core/services/cluster/peerlink_internal_test.go b/core/services/cluster/peerlink_internal_test.go new file mode 100644 index 000000000000..e61a4a4bd2de --- /dev/null +++ b/core/services/cluster/peerlink_internal_test.go @@ -0,0 +1,41 @@ +package cluster + +// These specs are in-package because the property they pin is a property of +// the error TYPE, not of any call site. Asserting it only from outside would +// re-check the paths peerlink_test.go already drives, which leaves the type +// free to start leaking its cause the moment a new call site is added. +// +// The other direction of the rule (a node absent from the registry is not +// merely unreachable) is driven end to end by peerlink_test.go through the +// real Registry, so it is not restated here. + +import ( + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Peer unreachability is not node absence", func() { + It("stays a transport error even when its cause is an absence error", func() { + // The dial path resolves the peer's address through the registry, so + // an ErrInstanceNotFound is genuinely reachable as a dial cause (a row + // deleted between the lookup and a retry, say). If the type let that + // through, a peer that merely would not answer would read as an absent + // node, and a replica acting on absence evicts healthy workers. + err := unreachablePeer("peer-1", fmt.Errorf("resolving: %w", ErrInstanceNotFound)) + + Expect(errors.Is(err, ErrPeerUnreachable)).To(BeTrue()) + Expect(errors.Is(err, ErrInstanceNotFound)).To(BeFalse(), + "the unreachable error must not unwrap to its cause, or absence leaks through it") + }) + + It("keeps the cause legible in its message", func() { + // Withholding the cause from errors.Is must not withhold it from a + // human reading a log line. + err := unreachablePeer("peer-1", errors.New("connection refused")) + Expect(err.Error()).To(ContainSubstring("peer-1")) + Expect(err.Error()).To(ContainSubstring("connection refused")) + }) +}) diff --git a/core/services/cluster/peerlink_test.go b/core/services/cluster/peerlink_test.go new file mode 100644 index 000000000000..4b145194af06 --- /dev/null +++ b/core/services/cluster/peerlink_test.go @@ -0,0 +1,221 @@ +package cluster_test + +import ( + "context" + "io" + "net/http/httptest" + "strings" + + clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" + + "github.com/labstack/echo/v4" + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +var _ = Describe("Peer pool", func() { + var ( + db *gorm.DB + reg *cluster.Registry + pool *cluster.PeerPool + srv *httptest.Server + accepted chan *yamux.Session + ctx context.Context + ) + + // startPeer stands up a real peer server and registers it under peerID. + startPeer := func(peerID string) *httptest.Server { + e := echo.New() + clusterep.RegisterClusterRoutes(e, "peer-token", func(_ string, s *yamux.Session) { + accepted <- s + }) + ts := httptest.NewServer(e) + addr := strings.TrimPrefix(ts.URL, "http://") + Expect(reg.Register(ctx, peerID, addr, "test")).To(Succeed()) + return ts + } + + BeforeEach(func() { + ctx = context.Background() + db = testutil.SetupTestDB() + Expect(db.AutoMigrate(&cluster.Instance{})).To(Succeed()) + reg = cluster.NewRegistry(db) + accepted = make(chan *yamux.Session, 4) + pool = cluster.NewPeerPool("self", "peer-token", reg) + DeferCleanup(pool.Close) + srv = startPeer("peer-1") + DeferCleanup(srv.Close) + }) + + It("opens a working stream to a live peer", func() { + st, err := pool.Open(ctx, "peer-1") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = st.Close() }) + + var serverSess *yamux.Session + Eventually(accepted, "10s").Should(Receive(&serverSess)) + + go func() { + defer GinkgoRecover() + _, _ = st.Write([]byte("ping")) + }() + + got := make(chan []byte, 1) + go func() { + defer GinkgoRecover() + in, e := serverSess.AcceptStream() + if e != nil { + return + } + buf := make([]byte, 4) + if _, e := io.ReadFull(in, buf); e == nil { + got <- buf + } + }() + Eventually(got, "10s").Should(Receive(Equal([]byte("ping")))) + }) + + It("identifies itself to the peer by its own instance id", func() { + // The peer records which replica is on the far end of the link, so a + // pool that sent the peer's id (or nothing) would leave every inbound + // link anonymous and indistinguishable from every other. + ids := make(chan string, 1) + e := echo.New() + clusterep.RegisterClusterRoutes(e, "peer-token", func(id string, _ *yamux.Session) { ids <- id }) + ts := httptest.NewServer(e) + DeferCleanup(ts.Close) + Expect(reg.Register(ctx, "peer-named", strings.TrimPrefix(ts.URL, "http://"), "test")).To(Succeed()) + + st, err := pool.Open(ctx, "peer-named") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = st.Close() }) + Eventually(ids, "10s").Should(Receive(Equal("self"))) + }) + + It("reuses one session across opens rather than dialling per stream", func() { + a, err := pool.Open(ctx, "peer-1") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = a.Close() }) + Eventually(accepted, "10s").Should(Receive()) + + b, err := pool.Open(ctx, "peer-1") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = b.Close() }) + + // A second dial would deliver a second server session. One session + // serving both streams is the property under test: peer links are + // pooled, not per-stream. + Consistently(accepted, "2s", "200ms").ShouldNot(Receive()) + }) + + It("returns ErrPeerUnreachable when the peer is registered but not listening", func() { + dead := startPeer("peer-dead") + dead.Close() + + _, err := pool.Open(ctx, "peer-dead") + Expect(err).To(MatchError(cluster.ErrPeerUnreachable), + "a peer that will not answer must be a transport error, never node absence") + Expect(err).ToNot(MatchError(cluster.ErrInstanceNotFound), + "an unreachable peer must never be readable as an absent node; a replica acting on absence evicts healthy workers") + }) + + It("returns ErrPeerUnreachable when the peer answers but rejects the credentials", func() { + // A token mismatch is a live peer refusing the link, not a missing + // row. Reporting absence here would evict every worker behind a peer + // that was merely rolled out with a stale secret. + e := echo.New() + clusterep.RegisterClusterRoutes(e, "a-different-token", func(_ string, s *yamux.Session) { accepted <- s }) + ts := httptest.NewServer(e) + DeferCleanup(ts.Close) + Expect(reg.Register(ctx, "peer-strict", strings.TrimPrefix(ts.URL, "http://"), "test")).To(Succeed()) + + _, err := pool.Open(ctx, "peer-strict") + Expect(err).To(MatchError(cluster.ErrPeerUnreachable)) + Expect(err).ToNot(MatchError(cluster.ErrInstanceNotFound)) + }) + + It("returns ErrInstanceNotFound when the peer is not in the registry", func() { + _, err := pool.Open(ctx, "never-registered") + Expect(err).To(MatchError(cluster.ErrInstanceNotFound)) + Expect(err).ToNot(MatchError(cluster.ErrPeerUnreachable), + "a node that was never registered is absent, not merely unreachable") + }) + + It("re-dials after the cached session dies", func() { + first, err := pool.Open(ctx, "peer-1") + Expect(err).ToNot(HaveOccurred()) + Expect(first.Close()).To(Succeed()) + + var serverSess *yamux.Session + Eventually(accepted, "10s").Should(Receive(&serverSess)) + Expect(serverSess.Close()).To(Succeed()) + srv.Close() + + // A replacement peer comes back on a new address under the same id, + // which is what a restarted replica looks like. + replacement := startPeer("peer-1") + DeferCleanup(replacement.Close) + + Eventually(func() error { + st, e := pool.Open(ctx, "peer-1") + if e == nil { + _ = st.Close() + } + return e + }, "15s", "500ms").Should(Succeed()) + + // The replacement's own session proves the pool re-dialled the address + // it re-read from the registry rather than resurrecting the dead one. + Eventually(accepted, "10s").Should(Receive()) + }) + + It("does not drop the pooled session when a single stream is reset by the peer", func() { + // A peer-initiated stream reset is scoped to one request. Dropping the + // session on it would tear down every other worker's traffic on the + // same link, so the pool must keep the session and hand out a fresh + // stream on it. + st, err := pool.Open(ctx, "peer-1") + Expect(err).ToNot(HaveOccurred()) + + var serverSess *yamux.Session + Eventually(accepted, "10s").Should(Receive(&serverSess)) + + go func() { + defer GinkgoRecover() + _, _ = st.Write([]byte("x")) + }() + var inbound *yamux.Stream + Eventually(func() error { + s, e := serverSess.AcceptStream() + inbound = s + return e + }, "10s").Should(Succeed()) + // Reset, not a graceful close: this is the *StreamError{Remote:true} + // the far end sends when it abandons a request. + Expect(inbound.Reset()).To(Succeed()) + Eventually(func() error { + _, e := st.Write([]byte("y")) + return e + }, "10s", "100ms").Should(HaveOccurred()) + + next, err := pool.Open(ctx, "peer-1") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = next.Close() }) + Consistently(accepted, "2s", "200ms").ShouldNot(Receive(), + "a reset stream must not cost the whole peer link") + }) + + It("refuses to open after Close and is safe to close twice", func() { + pool.Close() + pool.Close() + + _, err := pool.Open(ctx, "peer-1") + Expect(err).To(HaveOccurred()) + Expect(err).ToNot(MatchError(cluster.ErrInstanceNotFound), + "a locally closed pool says nothing about whether the node exists") + }) +}) From 393797e7e318821062de8cfb0cd19a4263a6b15a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 00:29:23 +0000 Subject: [PATCH 033/153] refactor(cluster): make the cluster service a leaf and blame the caller's deadline The peer link's WebSocket adapter and route constant lived in core/http/endpoints/cluster, so the dialler in core/services/cluster had to import an HTTP endpoints package to reach them. That pulled echo, core/http/auth and core/config into a package whose doc says it is deliberately free of such dependencies, and it made core/services/nodes reach an endpoints package transitively. It also has no way forward: the worker-connect handler needs the tunnel registry and the node token store, both of which are cycles from there. Move WebsocketConn and PeerPath into core/services/cluster and let the endpoints package import it, which is the direction the rest of core/http flows. The route and the auth exemption still cannot drift apart, now asserted where both are visible rather than by a const reference across the boundary, and the assertion is stronger than the one it replaces: it pins the route under the prefix instead of pinning the prefix's spelling. Also guard the fresh-dial path with ctx.Err(), mirroring the cached path. A caller with a 300ms deadline dialling a live, listening peer was told the peer was unreachable, which would be enough for one impatient client to get a healthy replica routed around once the relay consults these errors. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/cluster/peer.go | 26 +++---- core/http/endpoints/cluster/peer_test.go | 21 ++++-- core/services/cluster/peerlink.go | 56 ++++++++++++-- core/services/cluster/peerlink_test.go | 74 +++++++++++++++++++ .../endpoints => services}/cluster/wsconn.go | 0 .../cluster/wsconn_test.go | 33 +++++---- 6 files changed, 165 insertions(+), 45 deletions(-) rename core/{http/endpoints => services}/cluster/wsconn.go (100%) rename core/{http/endpoints => services}/cluster/wsconn_test.go (91%) diff --git a/core/http/endpoints/cluster/peer.go b/core/http/endpoints/cluster/peer.go index 5ff916f6c93e..85985eee5762 100644 --- a/core/http/endpoints/cluster/peer.go +++ b/core/http/endpoints/cluster/peer.go @@ -14,28 +14,20 @@ import ( "github.com/gorilla/websocket" "github.com/labstack/echo/v4" "github.com/libp2p/go-yamux/v5" - "github.com/mudler/LocalAI/core/http/auth" + clustersvc "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/xlog" ) -// AlternativeAuthPrefix is the path prefix whose credentials are checked by -// this package rather than by the global session middleware. -// -// It is the auth package's own constant rather than a second copy: auth decides -// which paths bypass the session middleware, so owning the prefix there and -// deriving the route from it here means a change to either one moves both. The -// dependency runs endpoints -> auth, the direction the rest of core/http flows; -// pointing it the other way would deadlock the build as soon as this package -// needs anything from auth, which registering in RouteFeatureRegistry will. -const AlternativeAuthPrefix = auth.ClusterPathPrefix - -// PeerPath is the route a peer replica dials. -const PeerPath = AlternativeAuthPrefix + "peer" - // RegisterClusterRoutes registers the peer link. onPeer receives every // authenticated session; see PeerHandler for what it is expected to do with it. +// +// The route is core/services/cluster's own constant, so the handler and the +// dialler cannot be registered and dialled at different paths. That the path +// also falls under auth.ClusterPathPrefix, and so bypasses the session +// middleware, is asserted by a spec in this package: it is the only place that +// can see both, since core/services/cluster must not import core/http/auth. func RegisterClusterRoutes(e *echo.Echo, token string, onPeer func(string, *yamux.Session)) { - e.GET(PeerPath, PeerHandler(token, onPeer)) + e.GET(clustersvc.PeerPath, PeerHandler(token, onPeer)) } // PeerHandler upgrades an authenticated peer dial to a WebSocket, wraps it as @@ -72,7 +64,7 @@ func PeerHandler(token string, onSession func(peerID string, sess *yamux.Session // Server side of the mux: the dialing peer is the client, so it owns // the odd stream IDs and this side the even ones. - sess, err := yamux.Server(WebsocketConn(ws), nil, nil) + sess, err := yamux.Server(clustersvc.WebsocketConn(ws), nil, nil) if err != nil { xlog.Error("cluster peer link session setup failed", "peer", peerID, "error", err) _ = ws.Close() diff --git a/core/http/endpoints/cluster/peer_test.go b/core/http/endpoints/cluster/peer_test.go index a9f027ff0c56..a86ab8add8bd 100644 --- a/core/http/endpoints/cluster/peer_test.go +++ b/core/http/endpoints/cluster/peer_test.go @@ -5,7 +5,9 @@ import ( "net/http/httptest" "strings" + "github.com/mudler/LocalAI/core/http/auth" clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" + clustersvc "github.com/mudler/LocalAI/core/services/cluster" "github.com/gorilla/websocket" "github.com/labstack/echo/v4" @@ -64,7 +66,7 @@ var _ = Describe("Peer link handler", func() { // server must accept it. This proves the WebSocket was adapted into a // stream-oriented conn correctly, which is the part most likely to be // subtly wrong. - clientSess, err := yamux.Client(clusterep.WebsocketConn(conn), nil, nil) + clientSess, err := yamux.Client(clustersvc.WebsocketConn(conn), nil, nil) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = clientSess.Close() }) @@ -159,7 +161,7 @@ var _ = Describe("Peer link handler", func() { Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = conn.Close() }) - clientSess, err := yamux.Client(clusterep.WebsocketConn(conn), nil, nil) + clientSess, err := yamux.Client(clustersvc.WebsocketConn(conn), nil, nil) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = clientSess.Close() }) @@ -185,9 +187,16 @@ var _ = Describe("Peer link handler", func() { }) var _ = Describe("Peer link auth prefix", func() { - It("is covered by the alternative-authentication prefix list", func() { - // /api/cluster/ authenticates with the cluster token, not the global - // session middleware, so it must be listed or every peer dial 401s. - Expect(clusterep.AlternativeAuthPrefix).To(Equal("/api/cluster/")) + It("keeps the peer route inside the alternative-authentication prefix", func() { + // The peer route authenticates with the cluster token, not the global + // session middleware, which only holds while the route sits under the + // prefix auth exempts. Moving either one alone 401s every peer dial. + // + // This lives here because it is the only package that can see both: + // core/services/cluster owns the route and must stay free of any + // core/http dependency, and core/http/auth owns the exemption. + Expect(strings.HasPrefix(clustersvc.PeerPath, auth.ClusterPathPrefix)).To(BeTrue(), + "peer route %q is no longer under the auth-exempt prefix %q", + clustersvc.PeerPath, auth.ClusterPathPrefix) }) }) diff --git a/core/services/cluster/peerlink.go b/core/services/cluster/peerlink.go index d4a21b4c59b7..961c1a3fa3d9 100644 --- a/core/services/cluster/peerlink.go +++ b/core/services/cluster/peerlink.go @@ -12,10 +12,20 @@ import ( "github.com/gorilla/websocket" "github.com/libp2p/go-yamux/v5" - clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" "github.com/mudler/xlog" ) +// PeerPath is the route a replica dials to open a peer link, and the route the +// HTTP layer registers the handler on. It lives here, with the dialler and the +// WebSocket adapter, so that core/services/cluster stays a leaf: the HTTP +// endpoints package imports this one, never the other way round. +// +// The literal is spelled out rather than derived from auth.ClusterPathPrefix +// because importing core/http/auth is exactly the dependency this package must +// not have. The two are kept from drifting apart by a spec in the endpoints +// package, which can see both. +const PeerPath = "/api/cluster/peer" + // ErrPeerUnreachable reports that a peer this deployment knows about could not // be reached: the dial failed, the peer refused the credentials, or its // multiplexer would not carry a stream. @@ -83,9 +93,13 @@ const ( // of a fast cross-zone link (roughly 31 MiB at 10 Gbps and 25 ms). // // The window is a cap on data received but not yet read, so the worst case - // a peer can make this replica buffer is MaxIncomingStreams times this, - // which is why MaxIncomingStreams is left at yamux's default rather than - // raised alongside it. + // a peer can make this replica buffer is MaxIncomingStreams times this. + // At yamux's default MaxIncomingStreams of 1000 that ceiling goes from + // about 15.6 GiB to about 31 GiB per peer session, which is the figure to + // size a replica against; it is why MaxIncomingStreams is left at the + // default rather than raised alongside the window. Both are ceilings on + // unread data and not allocations: yamux grows a stream's receive buffer + // as data arrives. peerLinkMaxWindow = 32 * 1024 * 1024 ) @@ -183,6 +197,17 @@ func (p *PeerPool) Open(ctx context.Context, peerID string) (net.Conn, error) { sess, err := p.dial(ctx, peerID) if err != nil { + // Same rule as above, on the path that has no cached session to + // protect: a dial that ran out of the caller's time says nothing about + // the peer, which may be listening and perfectly healthy. Blaming it + // would let one impatient client get a good replica routed around. + // + // This also swallows a genuine ErrInstanceNotFound when the context + // happened to expire at the same moment, which is the safe direction: + // a timeout must never be able to manufacture absence. + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } return nil, err } @@ -191,6 +216,9 @@ func (p *PeerPool) Open(ctx context.Context, peerID string) (net.Conn, error) { // The peer answered and completed a handshake but will not carry a // stream, which is a transport condition and never absence. _ = sess.Close() + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } return nil, unreachablePeer(peerID, err) } @@ -199,6 +227,14 @@ func (p *PeerPool) Open(ctx context.Context, peerID string) (net.Conn, error) { } // link returns the per-peer entry, creating it on first use. +// +// Entries are never pruned: a peer id opened once keeps its entry, and any +// session cached on it, until Close. The cost is not the map entry. A peer that +// has left the deployment but is still listening keeps a live WebSocket and the +// two yamux loop goroutines behind it for as long as this process runs; a peer +// that is genuinely gone is reclaimed by the 30s keepalive default, so the real +// exposure is narrow. There is no Forget because nothing yet knows which peers +// have left; Task 5's ownership work is where that knowledge appears. func (p *PeerPool) link(peerID string) (*peerLink, error) { p.mu.Lock() defer p.mu.Unlock() @@ -219,6 +255,14 @@ func (p *PeerPool) link(peerID string) (*peerLink, error) { // // A registry miss is returned unchanged so ErrInstanceNotFound reaches the // caller; everything after it is wrapped as unreachable. +// +// The address is only read here, so a peer that re-registers on a new address +// while its current session is still alive keeps being reached over that +// session until it dies. That is deliberate: an address change without a +// session break means the peer is still answering on the old one, and dropping +// a working link to chase a registry write would interrupt live requests for +// nothing. A replica that actually moved breaks its sessions in the process, +// and the re-dial above picks the new address up on the next Open. func (p *PeerPool) dial(ctx context.Context, peerID string) (*yamux.Session, error) { inst, err := p.reg.Get(ctx, peerID) if err != nil { @@ -235,7 +279,7 @@ func (p *PeerPool) dial(ctx context.Context, peerID string) (*yamux.Session, err // link is authenticated by the cluster token rather than by transport. Scheme: "ws", Host: inst.AdvertisedAddr, - Path: clusterep.PeerPath, + Path: PeerPath, RawQuery: url.Values{"id": []string{p.selfID}}.Encode(), } header := http.Header{} @@ -254,7 +298,7 @@ func (p *PeerPool) dial(ctx context.Context, peerID string) (*yamux.Session, err // Client side of the mux: the dialling replica owns the odd stream IDs, // matching the yamux.Server the peer handler puts on its end. - sess, err := yamux.Client(clusterep.WebsocketConn(ws), peerLinkConfig(), nil) + sess, err := yamux.Client(WebsocketConn(ws), peerLinkConfig(), nil) if err != nil { _ = ws.Close() return nil, unreachablePeer(peerID, err) diff --git a/core/services/cluster/peerlink_test.go b/core/services/cluster/peerlink_test.go index 4b145194af06..cfaa4dc69392 100644 --- a/core/services/cluster/peerlink_test.go +++ b/core/services/cluster/peerlink_test.go @@ -3,8 +3,11 @@ package cluster_test import ( "context" "io" + "net" "net/http/httptest" "strings" + "sync" + "time" clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" "github.com/mudler/LocalAI/core/services/cluster" @@ -209,6 +212,77 @@ var _ = Describe("Peer pool", func() { "a reset stream must not cost the whole peer link") }) + It("blames the caller's deadline, not the peer, when a dial runs out of time", func() { + // A listener that completes the TCP connection and then says nothing, + // which is what a peer under load or behind a wedged proxy looks like. + // The peer is not unreachable; the caller is impatient. Reporting + // ErrPeerUnreachable here would make an impatient client enough to get + // a healthy replica routed around. + ln, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = ln.Close() }) + // The accept loop owns the connections it holds and closes them when + // the listener goes away, so nothing is shared with the spec goroutine. + go func() { + defer GinkgoRecover() + var held []net.Conn + defer func() { + for _, c := range held { + _ = c.Close() + } + }() + for { + c, e := ln.Accept() + if e != nil { + return + } + // Hold the connection open without ever answering the upgrade. + held = append(held, c) + } + }() + Expect(reg.Register(ctx, "peer-silent", ln.Addr().String(), "test")).To(Succeed()) + + deadlined, cancel := context.WithTimeout(ctx, 300*time.Millisecond) + DeferCleanup(cancel) + _, err = pool.Open(deadlined, "peer-silent") + Expect(err).To(MatchError(context.DeadlineExceeded)) + Expect(err).ToNot(MatchError(cluster.ErrPeerUnreachable), + "the caller ran out of time; the peer never got a verdict") + Expect(err).ToNot(MatchError(cluster.ErrInstanceNotFound)) + }) + + It("dials once when many callers open the same peer at the same time", func() { + // Without a per-peer lock held across the dial, every concurrent + // caller races to dial and all but one of the resulting sessions is + // dropped on the floor still holding a live WebSocket. + const callers = 16 + streams := make(chan net.Conn, callers) + var wg sync.WaitGroup + for range callers { + wg.Add(1) + go func() { + defer wg.Done() + defer GinkgoRecover() + st, err := pool.Open(context.Background(), "peer-1") + Expect(err).ToNot(HaveOccurred()) + streams <- st + }() + } + wg.Wait() + close(streams) + + count := 0 + for st := range streams { + count++ + DeferCleanup(func(c net.Conn) { _ = c.Close() }, st) + } + Expect(count).To(Equal(callers)) + + Eventually(accepted, "10s").Should(Receive()) + Consistently(accepted, "2s", "200ms").ShouldNot(Receive(), + "concurrent opens must share one dial, not race to dial per caller") + }) + It("refuses to open after Close and is safe to close twice", func() { pool.Close() pool.Close() diff --git a/core/http/endpoints/cluster/wsconn.go b/core/services/cluster/wsconn.go similarity index 100% rename from core/http/endpoints/cluster/wsconn.go rename to core/services/cluster/wsconn.go diff --git a/core/http/endpoints/cluster/wsconn_test.go b/core/services/cluster/wsconn_test.go similarity index 91% rename from core/http/endpoints/cluster/wsconn_test.go rename to core/services/cluster/wsconn_test.go index 5ef3fcf3b1e2..305ce99b3c79 100644 --- a/core/http/endpoints/cluster/wsconn_test.go +++ b/core/services/cluster/wsconn_test.go @@ -12,6 +12,7 @@ import ( "time" clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" + "github.com/mudler/LocalAI/core/services/cluster" "github.com/gorilla/websocket" "github.com/labstack/echo/v4" @@ -54,8 +55,8 @@ var _ = Describe("WebsocketConn framing", func() { It("returns the rest of a message on the following Read", func() { clientWS, serverWS := wsPair() - writer := clusterep.WebsocketConn(clientWS) - reader := clusterep.WebsocketConn(serverWS) + writer := cluster.WebsocketConn(clientWS) + reader := cluster.WebsocketConn(serverWS) // A lost tail would otherwise park the reassembly below forever; with a // deadline it fails as a timeout on the read that has nothing left. @@ -83,8 +84,8 @@ var _ = Describe("WebsocketConn framing", func() { It("streams a message larger than the yamux read buffer without loss or reordering", func() { clientWS, serverWS := wsPair() - writer := clusterep.WebsocketConn(clientWS) - reader := clusterep.WebsocketConn(serverWS) + writer := cluster.WebsocketConn(clientWS) + reader := cluster.WebsocketConn(serverWS) Expect(reader.SetReadDeadline(time.Now().Add(20 * time.Second))).To(Succeed()) @@ -106,8 +107,8 @@ var _ = Describe("WebsocketConn framing", func() { It("presents consecutive messages as one continuous byte stream", func() { clientWS, serverWS := wsPair() - writer := clusterep.WebsocketConn(clientWS) - reader := clusterep.WebsocketConn(serverWS) + writer := cluster.WebsocketConn(clientWS) + reader := cluster.WebsocketConn(serverWS) Expect(reader.SetReadDeadline(time.Now().Add(10 * time.Second))).To(Succeed()) @@ -126,8 +127,8 @@ var _ = Describe("WebsocketConn framing", func() { It("never hands back a zero-length read for a zero-length message", func() { clientWS, serverWS := wsPair() - writer := clusterep.WebsocketConn(clientWS) - reader := clusterep.WebsocketConn(serverWS) + writer := cluster.WebsocketConn(clientWS) + reader := cluster.WebsocketConn(serverWS) Expect(reader.SetReadDeadline(time.Now().Add(10 * time.Second))).To(Succeed()) @@ -146,7 +147,7 @@ var _ = Describe("WebsocketConn framing", func() { It("reports a clean peer close as io.EOF", func() { clientWS, serverWS := wsPair() - reader := clusterep.WebsocketConn(serverWS) + reader := cluster.WebsocketConn(serverWS) Expect(clientWS.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))).To(Succeed()) @@ -157,7 +158,7 @@ var _ = Describe("WebsocketConn framing", func() { It("refuses a text message rather than desynchronising the stream", func() { clientWS, serverWS := wsPair() - reader := clusterep.WebsocketConn(serverWS) + reader := cluster.WebsocketConn(serverWS) Expect(clientWS.WriteMessage(websocket.TextMessage, []byte("not a frame"))).To(Succeed()) @@ -168,7 +169,7 @@ var _ = Describe("WebsocketConn framing", func() { It("satisfies net.Conn", func() { clientWS, _ := wsPair() - var conn net.Conn = clusterep.WebsocketConn(clientWS) + var conn net.Conn = cluster.WebsocketConn(clientWS) Expect(conn.LocalAddr()).ToNot(BeNil()) Expect(conn.RemoteAddr()).ToNot(BeNil()) @@ -176,7 +177,7 @@ var _ = Describe("WebsocketConn framing", func() { It("enforces a write deadline, which yamux arms before every flush", func() { clientWS, _ := wsPair() - conn := clusterep.WebsocketConn(clientWS) + conn := cluster.WebsocketConn(clientWS) // Asserting that the setter returns nil would prove nothing: gorilla // only records the deadline and applies it at the next flush. The write @@ -191,7 +192,7 @@ var _ = Describe("WebsocketConn framing", func() { It("enforces a read deadline, which is how a parked reader is unblocked", func() { clientWS, _ := wsPair() - conn := clusterep.WebsocketConn(clientWS) + conn := cluster.WebsocketConn(clientWS) Expect(conn.SetReadDeadline(time.Now().Add(-time.Second))).To(Succeed()) _, err := conn.Read(make([]byte, 8)) @@ -201,7 +202,7 @@ var _ = Describe("WebsocketConn framing", func() { It("arms both directions from SetDeadline", func() { clientWS, _ := wsPair() - conn := clusterep.WebsocketConn(clientWS) + conn := cluster.WebsocketConn(clientWS) Expect(conn.SetDeadline(time.Now().Add(-time.Second))).To(Succeed()) @@ -217,7 +218,7 @@ var _ = Describe("WebsocketConn framing", func() { // struct field, so this is a data race unless the adapter serialises it; // the spec is here to be run under -race, where it would report one. clientWS, _ := wsPair() - conn := clusterep.WebsocketConn(clientWS) + conn := cluster.WebsocketConn(clientWS) done := make(chan struct{}) go func() { @@ -252,7 +253,7 @@ var _ = Describe("Peer link payloads", func() { var serverSess *yamux.Session Eventually(sessions, "5s").Should(Receive(&serverSess)) - clientSess, err := yamux.Client(clusterep.WebsocketConn(conn), nil, nil) + clientSess, err := yamux.Client(cluster.WebsocketConn(conn), nil, nil) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = clientSess.Close() }) From edc7313f3ba606471dacd630a09213b01091968d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 00:50:11 +0000 Subject: [PATCH 034/153] feat(cluster): fence worker-connection ownership with a monotonic epoch A worker whose link is silently broken reconnects to another replica while the old owner's socket has not yet noticed. Without a fence both believe they own it. Claim is a single atomic upsert returning the new epoch, and a release must match both owner and epoch so a stale owner cannot delete a live claim. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/ownership.go | 103 +++++++++++++ core/services/cluster/ownership_test.go | 186 ++++++++++++++++++++++++ core/services/nodes/registry.go | 2 +- 3 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 core/services/cluster/ownership.go create mode 100644 core/services/cluster/ownership_test.go diff --git a/core/services/cluster/ownership.go b/core/services/cluster/ownership.go new file mode 100644 index 000000000000..143bc859bbc8 --- /dev/null +++ b/core/services/cluster/ownership.go @@ -0,0 +1,103 @@ +package cluster + +import ( + "context" + "errors" + "fmt" + "time" + + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// ErrNoConnection reports that no tunnel is recorded for the requested node, or +// that the claim a caller named is no longer the live one. Callers distinguish +// it from a transport failure to decide whether to relay through an owner, to +// answer "this worker is not connected here", or to retry. +var ErrNoConnection = errors.New("cluster: no connection recorded for node") + +// NodeConnection records which frontend replica currently holds a worker's +// tunnel. There is at most one row per node: a worker holds exactly one link, +// and whoever wrote the row last owns it. +// +// Epoch is the fence. A worker whose link is silently broken reconnects and may +// land on another replica before the previous owner's socket has noticed, so +// for a while two replicas both believe they own it. Every claim gets a higher +// epoch than any claim before it, so the loser can be told apart from the +// winner by a number both of them hold, without either having to detect the +// broken socket first. +type NodeConnection struct { + NodeID string `gorm:"primaryKey;size:36" json:"node_id"` + OwnerInstanceID string `gorm:"size:36;index;not null" json:"owner_instance_id"` + Epoch int64 `gorm:"not null" json:"epoch"` + ConnectedAt time.Time `gorm:"not null;default:now()" json:"connected_at"` + LastSeen time.Time `gorm:"index;not null;default:now()" json:"last_seen"` +} + +// Claim records ownerID as the owner of nodeID's tunnel and returns the new +// epoch, which is strictly greater than the epoch of every earlier claim on the +// same node. +// +// It is one statement on purpose. A read-then-write would let two replicas read +// the same epoch and hand out the same fence token, which is exactly the case +// the fence exists to rule out; PostgreSQL serializes concurrent +// INSERT ... ON CONFLICT DO UPDATE on the conflicting row, so the increment is +// computed by the database against the row version the winner just wrote. +func (r *Registry) Claim(ctx context.Context, nodeID, ownerID string) (int64, error) { + // Timestamps are stamped by the database, never by this process, for the + // same reason instance liveness is: they are compared across replicas, so + // they have to be measured on the one clock every replica shares. On insert + // that is the column default; on conflict it is the assignment below. + conn := NodeConnection{NodeID: nodeID, OwnerInstanceID: ownerID, Epoch: 1} + if err := r.db.WithContext(ctx).Clauses( + clause.OnConflict{ + Columns: []clause.Column{{Name: "node_id"}}, + DoUpdates: clause.Assignments(map[string]any{ + "owner_instance_id": ownerID, + "epoch": gorm.Expr(`"node_connections"."epoch" + 1`), + "connected_at": gorm.Expr("now()"), + "last_seen": gorm.Expr("now()"), + }), + }, + clause.Returning{Columns: []clause.Column{{Name: "epoch"}}}, + ).Create(&conn).Error; err != nil { + return 0, fmt.Errorf("claiming connection for node %q as %q: %w", nodeID, ownerID, err) + } + return conn.Epoch, nil +} + +// Owner returns the replica that holds nodeID's tunnel and the epoch of that +// claim, or ErrNoConnection when the node has no recorded connection. +func (r *Registry) Owner(ctx context.Context, nodeID string) (string, int64, error) { + var conn NodeConnection + err := r.db.WithContext(ctx).Where("node_id = ?", nodeID).First(&conn).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return "", 0, fmt.Errorf("looking up owner of node %q: %w", nodeID, ErrNoConnection) + } + if err != nil { + return "", 0, fmt.Errorf("looking up owner of node %q: %w", nodeID, err) + } + return conn.OwnerInstanceID, conn.Epoch, nil +} + +// Release drops the claim identified by ownerID and epoch. Both are in the +// WHERE so a replica that has only just noticed its dead socket cannot delete +// the claim a later reconnect established elsewhere: the row it is trying to +// clean up no longer exists, and deleting the live one would strand a worker +// that is in fact connected. A claim that is no longer the live one is reported +// as ErrNoConnection rather than silently ignored, because the caller learning +// it has been fenced out is the point. +func (r *Registry) Release(ctx context.Context, nodeID, ownerID string, epoch int64) error { + // gorm reports no error when a Where matches nothing, so the miss has to be + // read off RowsAffected. + res := r.db.WithContext(ctx). + Where("node_id = ? AND owner_instance_id = ? AND epoch = ?", nodeID, ownerID, epoch). + Delete(&NodeConnection{}) + if res.Error != nil { + return fmt.Errorf("releasing connection for node %q held by %q at epoch %d: %w", nodeID, ownerID, epoch, res.Error) + } + if res.RowsAffected == 0 { + return fmt.Errorf("releasing connection for node %q held by %q at epoch %d: %w", nodeID, ownerID, epoch, ErrNoConnection) + } + return nil +} diff --git a/core/services/cluster/ownership_test.go b/core/services/cluster/ownership_test.go new file mode 100644 index 000000000000..cb34b105bff3 --- /dev/null +++ b/core/services/cluster/ownership_test.go @@ -0,0 +1,186 @@ +package cluster_test + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + gormlogger "gorm.io/gorm/logger" +) + +// sqlRecorder captures the statements gorm actually sends, so a spec can assert +// on the SQL rather than on gorm's intent. gorm silently drops clauses it +// cannot apply to a given destination, and such a drop turns an atomic upsert +// into something that still passes every sequential expectation. +type sqlRecorder struct { + gormlogger.Interface + mu sync.Mutex + statements []string +} + +func newSQLRecorder() *sqlRecorder { + return &sqlRecorder{Interface: gormlogger.Default.LogMode(gormlogger.Silent)} +} + +func (r *sqlRecorder) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) { + sql, _ := fc() + r.mu.Lock() + r.statements = append(r.statements, sql) + r.mu.Unlock() +} + +// only returns the single recorded statement, failing the spec if the call +// under test issued anything other than exactly one. +func (r *sqlRecorder) only() string { + r.mu.Lock() + defer r.mu.Unlock() + ExpectWithOffset(1, r.statements).To(HaveLen(1), "expected exactly one statement, got: %v", r.statements) + return r.statements[0] +} + +var _ = Describe("Connection ownership", func() { + var ( + db *gorm.DB + reg *cluster.Registry + ctx context.Context + ) + + BeforeEach(func() { + db = testutil.SetupTestDB() + Expect(db.AutoMigrate(&cluster.NodeConnection{})).To(Succeed()) + reg = cluster.NewRegistry(db) + ctx = context.Background() + }) + + It("increments the epoch on every claim", func() { + e1, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + e2, err := reg.Claim(ctx, "w1", "inst-b") + Expect(err).ToNot(HaveOccurred()) + Expect(e2).To(BeNumerically(">", e1)) + }) + + It("reports the latest owner", func() { + _, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + _, err = reg.Claim(ctx, "w1", "inst-b") + Expect(err).ToNot(HaveOccurred()) + + owner, epoch, err := reg.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("inst-b")) + Expect(epoch).To(BeNumerically("==", 2)) + }) + + It("distinguishes an unknown connection", func() { + _, _, err := reg.Owner(ctx, "ghost") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + }) + + It("refuses a release from a stale owner", func() { + e1, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + _, err = reg.Claim(ctx, "w1", "inst-b") + Expect(err).ToNot(HaveOccurred()) + + // inst-a tries to clean up after losing the claim. + Expect(reg.Release(ctx, "w1", "inst-a", e1)).ToNot(Succeed()) + + owner, _, err := reg.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("inst-b"), "a stale owner must not be able to delete a live claim") + }) + + It("refuses a release that names the live owner but a stale epoch", func() { + // The same replica can reconnect a worker to itself; only the epoch + // separates the dead link from the live one. + e1, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + _, err = reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + + Expect(reg.Release(ctx, "w1", "inst-a", e1)).ToNot(Succeed()) + + owner, _, err := reg.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("inst-a")) + }) + + It("lets the current owner release its own claim", func() { + e, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "w1", "inst-a", e)).To(Succeed()) + + _, _, err = reg.Owner(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + }) + + It("claims in one statement that increments in SQL and stamps on the database clock", func() { + rec := newSQLRecorder() + recording := cluster.NewRegistry(db.Session(&gorm.Session{Logger: rec})) + + _, err := recording.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + + sql := strings.ToLower(rec.only()) + // A read-then-write would show up here as two statements; the length + // check in only() is what rules that out. The rest pins the parts a + // silently dropped clause would remove. + Expect(sql).To(ContainSubstring("on conflict")) + Expect(sql).To(ContainSubstring(`"node_connections"."epoch" + 1`), + "the epoch must be incremented by the database, not by this process") + Expect(sql).To(ContainSubstring("returning")) + Expect(sql).To(ContainSubstring(`"epoch"`)) + // Timestamps are compared across replicas, so they must be measured on + // the one clock every replica shares. A Go-side time.Now() would appear + // as a bound parameter instead. + Expect(sql).To(ContainSubstring("now()")) + Expect(sql).ToNot(MatchRegexp(`connected_at"?\s*=\s*'`), + "connected_at must not be a literal timestamp from this process's clock") + }) + + It("gives every concurrent claimant a distinct epoch and leaves exactly one winner", func() { + const claimants = 8 + epochs := make(chan int64, claimants) + var wg sync.WaitGroup + for i := 0; i < claimants; i++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + defer GinkgoRecover() + e, err := reg.Claim(context.Background(), "w-race", fmt.Sprintf("inst-%d", n)) + Expect(err).ToNot(HaveOccurred()) + epochs <- e + }(i) + } + wg.Wait() + close(epochs) + + seen := map[int64]bool{} + for e := range epochs { + Expect(seen[e]).To(BeFalse(), "epoch %d handed out twice; the fence is not atomic", e) + seen[e] = true + } + Expect(seen).To(HaveLen(claimants)) + + // Exactly one row, and its epoch is the highest handed out. + var rows []cluster.NodeConnection + Expect(db.Where("node_id = ?", "w-race").Find(&rows).Error).To(Succeed()) + Expect(rows).To(HaveLen(1)) + var max int64 + for e := range seen { + if e > max { + max = e + } + } + Expect(rows[0].Epoch).To(Equal(max)) + }) +}) diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index 1b31bb8bf116..fd1872861559 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -488,7 +488,7 @@ const heartbeatMaterialDelta = 256 << 20 // 256 MiB // when multiple instances (frontend + workers) start at the same time. func NewNodeRegistry(db *gorm.DB) (*NodeRegistry, error) { if err := advisorylock.WithLockCtx(context.Background(), db, advisorylock.KeySchemaMigrate, func() error { - return db.AutoMigrate(&BackendNode{}, &NodeModel{}, &NodeLabel{}, &ModelSchedulingConfig{}, &PendingBackendOp{}, &ModelLoadInfo{}, &ModelLoadJob{}, &ModelConfigState{}, &cluster.Instance{}) + return db.AutoMigrate(&BackendNode{}, &NodeModel{}, &NodeLabel{}, &ModelSchedulingConfig{}, &PendingBackendOp{}, &ModelLoadInfo{}, &ModelLoadJob{}, &ModelConfigState{}, &cluster.Instance{}, &cluster.NodeConnection{}) }); err != nil { return nil, fmt.Errorf("migrating node tables: %w", err) } From 6f53701f4c3c968fad97f616078aec7bfc16716d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 01:09:49 +0000 Subject: [PATCH 035/153] fix(cluster): draw connection epochs from a sequence so none is ever reused Release deletes the row, so a per-row `epoch + 1` restarted the numbering at 1 for the next claim. A replica could then be handed an epoch it already held: claim w1 at epoch 1, lose the link silently, watch another replica claim and release, reclaim and be handed 1 again, and its delayed cleanup for the first dead link would match the live claim and delete it. The fence has to be unique per node over time, not per row lifetime. Every claim now draws nextval from a dedicated sequence on both the insert and the conflict paths, so an epoch is never issued twice. The draw still happens after the row lock on the conflict path, so the winning claim still holds the highest epoch handed out. Also drop last_seen. Nothing maintained it and it was always equal to connected_at, but an indexed column named that way invites a second liveness clock; whether the owner is alive is Instance.LastSeen, and whether a claim is current is the epoch. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/ownership.go | 78 +++++++++++++++++++------ core/services/cluster/ownership_test.go | 48 +++++++++++++-- core/services/nodes/registry.go | 9 ++- 3 files changed, 109 insertions(+), 26 deletions(-) diff --git a/core/services/cluster/ownership.go b/core/services/cluster/ownership.go index 143bc859bbc8..ccddd69b9bbb 100644 --- a/core/services/cluster/ownership.go +++ b/core/services/cluster/ownership.go @@ -16,54 +16,94 @@ import ( // answer "this worker is not connected here", or to retry. var ErrNoConnection = errors.New("cluster: no connection recorded for node") +// epochSequence is the PostgreSQL sequence every claim draws its epoch from. +// A sequence rather than a per-row counter because a released row is deleted: +// with `epoch = epoch + 1` the numbering restarts at 1 for the next claim, so a +// replica that claimed, lost the worker, and claimed again could be handed an +// epoch it already held, and a delayed cleanup from the first claim would then +// match, and delete, the live one. +const epochSequence = "node_connection_epochs" + // NodeConnection records which frontend replica currently holds a worker's // tunnel. There is at most one row per node: a worker holds exactly one link, // and whoever wrote the row last owns it. // // Epoch is the fence. A worker whose link is silently broken reconnects and may // land on another replica before the previous owner's socket has noticed, so -// for a while two replicas both believe they own it. Every claim gets a higher -// epoch than any claim before it, so the loser can be told apart from the -// winner by a number both of them hold, without either having to detect the -// broken socket first. +// for a while two replicas both believe they own it. Every claim draws a fresh, +// never-reused epoch, so the loser can be told apart from the winner by a number +// both of them hold, without either having to detect the broken socket first. +// +// There is deliberately no last-seen column here. Whether the owning replica is +// alive is answered by Instance.LastSeen, and whether a claim is still the live +// one is answered by the epoch; a second liveness clock for the same fact would +// only drift from the first. type NodeConnection struct { NodeID string `gorm:"primaryKey;size:36" json:"node_id"` OwnerInstanceID string `gorm:"size:36;index;not null" json:"owner_instance_id"` Epoch int64 `gorm:"not null" json:"epoch"` ConnectedAt time.Time `gorm:"not null;default:now()" json:"connected_at"` - LastSeen time.Time `gorm:"index;not null;default:now()" json:"last_seen"` +} + +// EnsureEpochSequence creates the sequence Claim draws epochs from. It lives +// here, beside the model that needs it, because gorm's AutoMigrate models +// tables and columns but has no notion of a sequence; the caller that owns the +// migration advisory lock calls it so that concurrently starting replicas do +// not race on the DDL. It is safe to call repeatedly. +// +// The sequence is not attached as a column DEFAULT on purpose: AutoMigrate +// compares the struct's declared default against the one PostgreSQL reports +// (`nextval('...'::regclass)`), and a mismatch there makes every startup ALTER +// the column. Naming the sequence in the statement keeps the schema stable. +func EnsureEpochSequence(ctx context.Context, db *gorm.DB) error { + if err := db.WithContext(ctx).Exec(`CREATE SEQUENCE IF NOT EXISTS ` + epochSequence + ` AS bigint`).Error; err != nil { + return fmt.Errorf("creating connection epoch sequence: %w", err) + } + return nil } // Claim records ownerID as the owner of nodeID's tunnel and returns the new -// epoch, which is strictly greater than the epoch of every earlier claim on the -// same node. +// epoch, which is greater than every epoch handed out for that node before it +// and is never handed out again. // // It is one statement on purpose. A read-then-write would let two replicas read // the same epoch and hand out the same fence token, which is exactly the case // the fence exists to rule out; PostgreSQL serializes concurrent -// INSERT ... ON CONFLICT DO UPDATE on the conflicting row, so the increment is -// computed by the database against the row version the winner just wrote. +// INSERT ... ON CONFLICT DO UPDATE on the conflicting row, so the losing writers +// block until the winner commits and only then draw their own epoch, in the +// order they took the row lock. func (r *Registry) Claim(ctx context.Context, nodeID, ownerID string) (int64, error) { - // Timestamps are stamped by the database, never by this process, for the - // same reason instance liveness is: they are compared across replicas, so - // they have to be measured on the one clock every replica shares. On insert - // that is the column default; on conflict it is the assignment below. - conn := NodeConnection{NodeID: nodeID, OwnerInstanceID: ownerID, Epoch: 1} - if err := r.db.WithContext(ctx).Clauses( + // connected_at is stamped by the database, never by this process, for the + // same reason instance liveness is: it is compared across replicas, so it + // has to be measured on the one clock every replica shares. + nextEpoch := gorm.Expr("nextval('" + epochSequence + "')") + values := map[string]any{ + "node_id": nodeID, + "owner_instance_id": ownerID, + "epoch": nextEpoch, + "connected_at": gorm.Expr("now()"), + } + if err := r.db.WithContext(ctx).Model(&NodeConnection{}).Clauses( clause.OnConflict{ Columns: []clause.Column{{Name: "node_id"}}, DoUpdates: clause.Assignments(map[string]any{ "owner_instance_id": ownerID, - "epoch": gorm.Expr(`"node_connections"."epoch" + 1`), + "epoch": nextEpoch, "connected_at": gorm.Expr("now()"), - "last_seen": gorm.Expr("now()"), }), }, clause.Returning{Columns: []clause.Column{{Name: "epoch"}}}, - ).Create(&conn).Error; err != nil { + ).Create(values).Error; err != nil { return 0, fmt.Errorf("claiming connection for node %q as %q: %w", nodeID, ownerID, err) } - return conn.Epoch, nil + // gorm scans RETURNING back over the map it was handed. If that ever stops + // happening the entry is still the expression we passed in, and returning a + // bogus epoch would hand out a fence token the database never issued. + epoch, ok := values["epoch"].(int64) + if !ok { + return 0, fmt.Errorf("claiming connection for node %q as %q: epoch not returned by the database (got %T)", nodeID, ownerID, values["epoch"]) + } + return epoch, nil } // Owner returns the replica that holds nodeID's tunnel and the epoch of that diff --git a/core/services/cluster/ownership_test.go b/core/services/cluster/ownership_test.go index cb34b105bff3..4e33b056163b 100644 --- a/core/services/cluster/ownership_test.go +++ b/core/services/cluster/ownership_test.go @@ -24,6 +24,7 @@ type sqlRecorder struct { gormlogger.Interface mu sync.Mutex statements []string + errs []error } func newSQLRecorder() *sqlRecorder { @@ -31,10 +32,17 @@ func newSQLRecorder() *sqlRecorder { } func (r *sqlRecorder) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) { - sql, _ := fc() + sql, rows := fc() r.mu.Lock() r.statements = append(r.statements, sql) + if err != nil { + r.errs = append(r.errs, err) + } r.mu.Unlock() + // Delegate so a failing statement is still reported the way gorm would + // report it. An instrument used to prove what the SQL does must not be the + // one thing that hides a statement erroring. + r.Interface.Trace(ctx, begin, func() (string, int64) { return sql, rows }, err) } // only returns the single recorded statement, failing the spec if the call @@ -42,6 +50,7 @@ func (r *sqlRecorder) Trace(ctx context.Context, begin time.Time, fc func() (str func (r *sqlRecorder) only() string { r.mu.Lock() defer r.mu.Unlock() + ExpectWithOffset(1, r.errs).To(BeEmpty(), "the recorded statement failed") ExpectWithOffset(1, r.statements).To(HaveLen(1), "expected exactly one statement, got: %v", r.statements) return r.statements[0] } @@ -55,9 +64,10 @@ var _ = Describe("Connection ownership", func() { BeforeEach(func() { db = testutil.SetupTestDB() + ctx = context.Background() Expect(db.AutoMigrate(&cluster.NodeConnection{})).To(Succeed()) + Expect(cluster.EnsureEpochSequence(ctx, db)).To(Succeed()) reg = cluster.NewRegistry(db) - ctx = context.Background() }) It("increments the epoch on every claim", func() { @@ -71,13 +81,13 @@ var _ = Describe("Connection ownership", func() { It("reports the latest owner", func() { _, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) - _, err = reg.Claim(ctx, "w1", "inst-b") + e2, err := reg.Claim(ctx, "w1", "inst-b") Expect(err).ToNot(HaveOccurred()) owner, epoch, err := reg.Owner(ctx, "w1") Expect(err).ToNot(HaveOccurred()) Expect(owner).To(Equal("inst-b")) - Expect(epoch).To(BeNumerically("==", 2)) + Expect(epoch).To(Equal(e2), "the stored epoch must be the one the winning claim was handed") }) It("distinguishes an unknown connection", func() { @@ -114,6 +124,32 @@ var _ = Describe("Connection ownership", func() { Expect(owner).To(Equal("inst-a")) }) + It("never hands a node the same epoch twice, so a delayed cleanup cannot delete a live claim", func() { + // The scenario the fence exists for, with a release in the middle of it: + // inst-a claims and its link then dies silently. + eA1, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + // The worker reconnects to inst-b, which later releases cleanly. + eB, err := reg.Claim(ctx, "w1", "inst-b") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "w1", "inst-b", eB)).To(Succeed()) + // The worker comes back to inst-a, which is the same process throughout, + // so the owner id alone cannot separate this claim from the dead one. + eA2, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + + // inst-a finally notices the first link is dead and cleans up after it. + // The harm is asserted before the cause, so a regression fails on the + // live claim disappearing rather than on the epoch arithmetic. + Expect(reg.Release(ctx, "w1", "inst-a", eA1)).ToNot(Succeed()) + + owner, epoch, err := reg.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred(), "the delayed cleanup deleted the live claim") + Expect(owner).To(Equal("inst-a")) + Expect(epoch).To(Equal(eA2)) + Expect(eA2).ToNot(Equal(eA1), "an epoch handed out before a release must never be handed out again") + }) + It("lets the current owner release its own claim", func() { e, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) @@ -135,8 +171,8 @@ var _ = Describe("Connection ownership", func() { // check in only() is what rules that out. The rest pins the parts a // silently dropped clause would remove. Expect(sql).To(ContainSubstring("on conflict")) - Expect(sql).To(ContainSubstring(`"node_connections"."epoch" + 1`), - "the epoch must be incremented by the database, not by this process") + Expect(sql).To(MatchRegexp(`(?i)nextval\s*\(\s*'node_connection_epochs'\s*\)`), + "the epoch must be drawn by the database, not computed by this process") Expect(sql).To(ContainSubstring("returning")) Expect(sql).To(ContainSubstring(`"epoch"`)) // Timestamps are compared across replicas, so they must be measured on diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index fd1872861559..c3dc878ac409 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -488,7 +488,14 @@ const heartbeatMaterialDelta = 256 << 20 // 256 MiB // when multiple instances (frontend + workers) start at the same time. func NewNodeRegistry(db *gorm.DB) (*NodeRegistry, error) { if err := advisorylock.WithLockCtx(context.Background(), db, advisorylock.KeySchemaMigrate, func() error { - return db.AutoMigrate(&BackendNode{}, &NodeModel{}, &NodeLabel{}, &ModelSchedulingConfig{}, &PendingBackendOp{}, &ModelLoadInfo{}, &ModelLoadJob{}, &ModelConfigState{}, &cluster.Instance{}, &cluster.NodeConnection{}) + if err := db.AutoMigrate(&BackendNode{}, &NodeModel{}, &NodeLabel{}, &ModelSchedulingConfig{}, &PendingBackendOp{}, &ModelLoadInfo{}, &ModelLoadJob{}, &ModelConfigState{}, &cluster.Instance{}, &cluster.NodeConnection{}); err != nil { + return err + } + // AutoMigrate models tables and columns but has no notion of a + // sequence, and the connection-ownership fence draws its epochs from + // one. It runs under this same lock so concurrently starting replicas + // do not race on the DDL. + return cluster.EnsureEpochSequence(context.Background(), db) }); err != nil { return nil, fmt.Errorf("migrating node tables: %w", err) } From e9514334fa29857c29afc4f09e5fdec903502e13 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 01:28:27 +0000 Subject: [PATCH 036/153] fix(cluster): keep the connection schema migratable on SQLite The NodeConnection model carried `default:now()`, which is PostgreSQL syntax reaching the DDL, so AutoMigrate failed on the single-binary SQLite path and took every SQLite caller of nodes.NewNodeRegistry down with it. Stamp the database clock as an expression inside Claim instead, the way Register already does, and leave the column plain. CREATE SEQUENCE is Postgres-only for the same reason, so it is skipped on another dialect, and Claim refuses that dialect outright: a fence that cannot draw a token must say so rather than fail later as a missing function. Also correct a claim the previous commit made in both the doc comment and its message. An epoch is unique and never reissued, but it is not ordered: the insert path draws its sequence value before taking the row lock, so a claim that inserts after a release can be handed a lower number than one already issued. Uniqueness is what Release needs, since it matches by equality; callers must never compare epochs for order. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/ownership.go | 52 +++++++++++++++++++++---- core/services/cluster/ownership_test.go | 38 +++++++++++++++++- 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/core/services/cluster/ownership.go b/core/services/cluster/ownership.go index ccddd69b9bbb..a2e2ba9cb5ae 100644 --- a/core/services/cluster/ownership.go +++ b/core/services/cluster/ownership.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "time" "gorm.io/gorm" @@ -16,6 +17,16 @@ import ( // answer "this worker is not connected here", or to retry. var ErrNoConnection = errors.New("cluster: no connection recorded for node") +// isPostgres reports whether the gorm dialect is PostgreSQL. The connection +// fence is built out of PostgreSQL-only pieces (a sequence, ON CONFLICT +// RETURNING), and the single-binary path runs on SQLite. advisorylock has an +// identical private check; this one is copied rather than imported because it +// is a one-line comparison on gorm's own dialector name and cluster is +// deliberately a leaf package. +func isPostgres(db *gorm.DB) bool { + return strings.Contains(db.Dialector.Name(), "postgres") +} + // epochSequence is the PostgreSQL sequence every claim draws its epoch from. // A sequence rather than a per-row counter because a released row is deleted: // with `epoch = epoch + 1` the numbering restarts at 1 for the next claim, so a @@ -39,10 +50,13 @@ const epochSequence = "node_connection_epochs" // one is answered by the epoch; a second liveness clock for the same fact would // only drift from the first. type NodeConnection struct { - NodeID string `gorm:"primaryKey;size:36" json:"node_id"` - OwnerInstanceID string `gorm:"size:36;index;not null" json:"owner_instance_id"` - Epoch int64 `gorm:"not null" json:"epoch"` - ConnectedAt time.Time `gorm:"not null;default:now()" json:"connected_at"` + NodeID string `gorm:"primaryKey;size:36" json:"node_id"` + OwnerInstanceID string `gorm:"size:36;index;not null" json:"owner_instance_id"` + Epoch int64 `gorm:"not null" json:"epoch"` + // No column DEFAULT: now() is PostgreSQL syntax and would reach the DDL, + // which breaks AutoMigrate on the SQLite single-binary path. Claim writes + // the database clock as an expression instead, the way Register does. + ConnectedAt time.Time `gorm:"not null" json:"connected_at"` } // EnsureEpochSequence creates the sequence Claim draws epochs from. It lives @@ -56,15 +70,32 @@ type NodeConnection struct { // (`nextval('...'::regclass)`), and a mismatch there makes every startup ALTER // the column. Naming the sequence in the statement keeps the schema stable. func EnsureEpochSequence(ctx context.Context, db *gorm.DB) error { + // CREATE SEQUENCE is PostgreSQL-only, and the same migration path runs + // against SQLite in single-binary mode. Nothing there can claim a + // connection (Claim refuses the dialect outright), so there is nothing to + // create. + if !isPostgres(db) { + return nil + } if err := db.WithContext(ctx).Exec(`CREATE SEQUENCE IF NOT EXISTS ` + epochSequence + ` AS bigint`).Error; err != nil { return fmt.Errorf("creating connection epoch sequence: %w", err) } return nil } -// Claim records ownerID as the owner of nodeID's tunnel and returns the new -// epoch, which is greater than every epoch handed out for that node before it -// and is never handed out again. +// Claim records ownerID as the owner of nodeID's tunnel and returns the epoch +// of the claim. +// +// The epoch is UNIQUE and never reissued: no other claim, for this node or any +// other, is ever handed the same value. It is NOT ordered, and callers must not +// treat it as a version number. The sequence value on the insert path is drawn +// while the tuple is built, before the row lock, so a claim that inserts after +// a Release can be handed a number lower than one already issued elsewhere. +// Compare epochs for equality only; never compare them for order. +// +// Uniqueness is all the fence needs: Release matches owner and epoch exactly, +// so a stale claim's token cannot match a live claim's row whichever way the +// two numbers happen to compare. // // It is one statement on purpose. A read-then-write would let two replicas read // the same epoch and hand out the same fence token, which is exactly the case @@ -73,6 +104,13 @@ func EnsureEpochSequence(ctx context.Context, db *gorm.DB) error { // block until the winner commits and only then draw their own epoch, in the // order they took the row lock. func (r *Registry) Claim(ctx context.Context, nodeID, ownerID string) (int64, error) { + // Refused rather than attempted on a dialect with no sequence. The + // statement would fail anyway, but with a driver-level "no such function: + // nextval" that reads like a missing migration; and a fence that cannot + // issue a token must not look like one that did. + if !isPostgres(r.db) { + return 0, fmt.Errorf("claiming connection for node %q as %q: connection ownership requires PostgreSQL, this deployment runs on %q", nodeID, ownerID, r.db.Dialector.Name()) + } // connected_at is stamped by the database, never by this process, for the // same reason instance liveness is: it is compared across replicas, so it // has to be measured on the one clock every replica shares. diff --git a/core/services/cluster/ownership_test.go b/core/services/cluster/ownership_test.go index 4e33b056163b..ba634ebcd3eb 100644 --- a/core/services/cluster/ownership_test.go +++ b/core/services/cluster/ownership_test.go @@ -3,6 +3,7 @@ package cluster_test import ( "context" "fmt" + "path/filepath" "strings" "sync" "time" @@ -12,6 +13,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gorm.io/driver/sqlite" "gorm.io/gorm" gormlogger "gorm.io/gorm/logger" ) @@ -207,7 +209,11 @@ var _ = Describe("Connection ownership", func() { } Expect(seen).To(HaveLen(claimants)) - // Exactly one row, and its epoch is the highest handed out. + // Exactly one row, and its epoch is the highest handed out. That last + // part holds here because no Release intervenes: every claim after the + // first blocks on the row lock and draws its epoch after taking it, in + // commit order. It is not a general guarantee about epochs, which are + // unique but unordered. var rows []cluster.NodeConnection Expect(db.Where("node_id = ?", "w-race").Find(&rows).Error).To(Succeed()) Expect(rows).To(HaveLen(1)) @@ -220,3 +226,33 @@ var _ = Describe("Connection ownership", func() { Expect(rows[0].Epoch).To(Equal(max)) }) }) + +var _ = Describe("Connection ownership on a non-PostgreSQL dialect", func() { + var ( + db *gorm.DB + ctx context.Context + ) + + BeforeEach(func() { + var err error + ctx = context.Background() + db, err = gorm.Open(sqlite.Open(filepath.Join(GinkgoT().TempDir(), "cluster.db")), &gorm.Config{}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("migrates, because the single-binary path shares this schema", func() { + // A PostgreSQL-only column DEFAULT here breaks AutoMigrate for every + // SQLite caller of nodes.NewNodeRegistry, which is how this regressed. + Expect(db.AutoMigrate(&cluster.NodeConnection{})).To(Succeed()) + Expect(cluster.EnsureEpochSequence(ctx, db)).To(Succeed()) + }) + + It("refuses to claim, rather than pretending to fence", func() { + Expect(db.AutoMigrate(&cluster.NodeConnection{})).To(Succeed()) + Expect(cluster.EnsureEpochSequence(ctx, db)).To(Succeed()) + + _, err := cluster.NewRegistry(db).Claim(ctx, "w1", "inst-a") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("requires PostgreSQL")) + }) +}) From dca8ce714916d31837e5148cc71015779bc68592 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 02:28:42 +0000 Subject: [PATCH 037/153] feat(cluster): give phase 1 a call site, and prove it against real replicas Tasks 1 to 5 built an instances table, a splice, both halves of a peer link and an epoch fence, and nothing in the tree called any of it: no replica registered, no route was mounted, no sweeper ran. Proving phase 1 end to end therefore had to start by wiring it. A frontend in distributed mode now publishes the address its peers dial, heartbeats it, and sweeps replicas that stopped answering along with the connection rows they owned, in one pass so the two can never disagree about who is alive. It serves the peer link and owns the sessions peers dial in, refusing streams on them until phase 2 installs a relay: a session nobody accepts on does not fail a peer's Open, it hangs it. The address is the one peers use, not the one the process binds, and it is derived from the route to PostgreSQL. That derivation only holds while the database is remote, so LOCALAI_DISTRIBUTED_ADVERTISE_ADDR sets it explicitly and a replica that can determine neither warns and keeps serving rather than failing to start. Three e2e scenarios run against real local-ai processes, real PostgreSQL and real dials: replicas publish addresses that can actually be connected to; a sibling opens a stream over the peer link and is refused without the cluster token; and a killed replica is reported unreachable, never absent, loses the claim it held, and takes no worker with it. Each was verified by mutation: eight injected defects, each failing the scenario that claims to catch it. Also moves RegisterClusterRoutes to core/http/routes beside every other registrar, folds AutoMigrate and the epoch sequence into one cluster.Migrate, and turns the peer route's auth-coverage spec into a real assertion: it drives the request through the actual auth middleware instead of comparing two string constants, which the old spec would have passed even with the exemption deleted. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 73 +++++ core/cli/run.go | 4 + core/config/distributed_config.go | 19 +- core/http/app.go | 8 + core/http/endpoints/cluster/peer.go | 12 - core/http/endpoints/cluster/peer_test.go | 110 ++++++-- core/http/routes/cluster.go | 25 ++ core/services/cluster/instance_test.go | 4 +- core/services/cluster/membership.go | 175 ++++++++++++ core/services/cluster/membership_test.go | 116 ++++++++ core/services/cluster/ownership.go | 25 +- core/services/cluster/ownership_test.go | 9 +- core/services/cluster/peerlink_test.go | 18 +- core/services/cluster/sessions.go | 140 ++++++++++ core/services/cluster/sessions_test.go | 122 ++++++++ core/services/cluster/wsconn_test.go | 3 +- core/services/nodes/registry.go | 12 +- docs/content/features/distributed-mode.md | 22 ++ tests/e2e/distributed/cluster/cluster.go | 16 ++ .../e2e/distributed/cluster_baseline_test.go | 10 +- .../e2e/distributed/cluster_peerlink_test.go | 263 ++++++++++++++++++ 21 files changed, 1119 insertions(+), 67 deletions(-) create mode 100644 core/http/routes/cluster.go create mode 100644 core/services/cluster/membership.go create mode 100644 core/services/cluster/membership_test.go create mode 100644 core/services/cluster/sessions.go create mode 100644 core/services/cluster/sessions_test.go create mode 100644 tests/e2e/distributed/cluster_peerlink_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index 4c6214e57255..616d8a2ebc81 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "io" + "net" + "strconv" "strings" "sync" "time" @@ -12,6 +14,7 @@ import ( "github.com/google/uuid" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/agents" + "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/LocalAI/core/services/distributed" "github.com/mudler/LocalAI/core/services/jobs" "github.com/mudler/LocalAI/core/services/messaging" @@ -19,6 +22,7 @@ import ( "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/nodes/prefixcache" "github.com/mudler/LocalAI/core/services/storage" + "github.com/mudler/LocalAI/internal" "github.com/mudler/LocalAI/pkg/distributedhdr" "github.com/mudler/LocalAI/pkg/sanitize" "github.com/mudler/xlog" @@ -44,6 +48,16 @@ type DistributedServices struct { Unloader *nodes.RemoteUnloaderAdapter ModelCleanup *nodes.ModelCleanupService + // Cluster is the replica-membership registry: which frontend replicas are + // alive, at which address, and which of them holds a given worker's tunnel. + Cluster *cluster.Registry + // Membership publishes this replica's row and reaps the dead. Nil when no + // peer-reachable address could be determined, which leaves this replica + // invisible to its peers but otherwise fully functional. + Membership *cluster.Membership + // PeerSessions owns the peer links other replicas dialled into this one. + PeerSessions *cluster.SessionStore + shutdownOnce sync.Once } @@ -54,6 +68,15 @@ func (ds *DistributedServices) Shutdown() { return } ds.shutdownOnce.Do(func() { + // Peer state first: a replica that is going away should stop claiming + // to be alive before it stops answering, so peers re-home rather than + // dial a process in teardown. + if ds.Membership != nil { + ds.Membership.Stop() + } + if ds.PeerSessions != nil { + ds.PeerSessions.CloseAll() + } if ds.Health != nil { ds.Health.Stop() } @@ -174,6 +197,29 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade xlog.Warn("Control-plane database metrics unavailable", "error", err) } + // Replica membership. NewNodeRegistry has just migrated the tables this + // reads, so it has to come after it. + clusterRegistry := cluster.NewRegistry(authDB) + // Accepted peer links are held with no stream handler: this replica has + // somewhere to put a link a peer dials, and refuses the streams on it, + // because nothing relays worker traffic yet. + peerSessions := cluster.NewSessionStore(nil) + var membership *cluster.Membership + if advertised, err := advertisedPeerAddr(cfg); err != nil { + // Not fatal. A replica that cannot publish an address still serves + // every request that reaches it directly; what it cannot do is have + // another replica relay to it. Failing startup here would take out + // every existing single-host deployment, whose route to a local + // database is loopback. + xlog.Warn("This replica will not be reachable by its peers: no advertised address", + "error", err, "knob", "LOCALAI_DISTRIBUTED_ADVERTISE_ADDR") + } else { + membership = cluster.NewMembership(clusterRegistry, cfg.Distributed.InstanceID, advertised, internal.PrintableVersion()) + if err := membership.Start(cfg.Context); err != nil { + return nil, fmt.Errorf("registering this replica in the cluster: %w", err) + } + } + // Let scheduling rules be keyed by a model alias. The registry resolves a // rule's name through the config loader to find the model it governs, so an // operator can pin placement to a stable name like "production" and have it @@ -479,9 +525,36 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade ModelAdapter: modelAdapter, Unloader: remoteUnloader, ModelCleanup: modelCleanup, + Cluster: clusterRegistry, + Membership: membership, + PeerSessions: peerSessions, }, nil } +// advertisedPeerAddr is the host:port peers dial to reach this replica. +// +// The operator's value wins outright. Otherwise it is derived from the port +// this process serves on and the local address that routes to PostgreSQL, which +// is only a peer-reachable answer when the database is on another host; +// DiscoverAdvertisedAddr refuses rather than guessing when it is not. +func advertisedPeerAddr(cfg *config.ApplicationConfig) (string, error) { + if cfg.Distributed.AdvertiseAddr != "" { + return cfg.Distributed.AdvertiseAddr, nil + } + if cfg.APIAddress == "" { + return "", fmt.Errorf("no API address to derive a peer port from") + } + _, port, err := net.SplitHostPort(cfg.APIAddress) + if err != nil { + return "", fmt.Errorf("reading the peer port out of API address %q: %w", cfg.APIAddress, err) + } + portNumber, err := strconv.Atoi(port) + if err != nil { + return "", fmt.Errorf("API address %q has a non-numeric port: %w", cfg.APIAddress, err) + } + return cluster.DiscoverAdvertisedAddr(cfg.Auth.DatabaseURL, portNumber) +} + func isPostgresURL(url string) bool { return strings.HasPrefix(url, "postgres://") || strings.HasPrefix(url, "postgresql://") } diff --git a/core/cli/run.go b/core/cli/run.go index 3dc165ecc1b4..f613e1da85d8 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -165,6 +165,7 @@ type RunCMD struct { Distributed bool `env:"LOCALAI_DISTRIBUTED" default:"false" help:"Enable distributed mode (requires PostgreSQL + NATS)" group:"distributed"` InstanceID string `env:"LOCALAI_INSTANCE_ID" help:"Unique instance ID for distributed mode (auto-generated UUID if empty)" group:"distributed"` NatsURL string `env:"LOCALAI_NATS_URL" help:"NATS server URL (e.g., nats://localhost:4222)" group:"distributed"` + DistributedAdvertiseAddr string `env:"LOCALAI_DISTRIBUTED_ADVERTISE_ADDR" help:"host:port other frontend replicas dial to reach this one (peer link). Empty = derived from the local address that routes to PostgreSQL, which only works when the database is on another host." group:"distributed"` StorageURL string `env:"LOCALAI_STORAGE_URL" help:"S3-compatible storage endpoint URL (e.g., http://minio:9000)" group:"distributed"` StorageBucket string `env:"LOCALAI_STORAGE_BUCKET" default:"localai" help:"S3 bucket name for object storage" group:"distributed"` StorageRegion string `env:"LOCALAI_STORAGE_REGION" default:"us-east-1" help:"S3 region" group:"distributed"` @@ -360,6 +361,9 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { if r.InstanceID != "" { opts = append(opts, config.WithDistributedInstanceID(r.InstanceID)) } + if r.DistributedAdvertiseAddr != "" { + opts = append(opts, config.WithDistributedAdvertiseAddr(r.DistributedAdvertiseAddr)) + } if r.NatsURL != "" { opts = append(opts, config.WithNatsURL(r.NatsURL)) } diff --git a/core/config/distributed_config.go b/core/config/distributed_config.go index ef7e01bfd18f..92e31df6df34 100644 --- a/core/config/distributed_config.go +++ b/core/config/distributed_config.go @@ -13,8 +13,15 @@ import ( // DistributedConfig holds configuration for horizontal scaling mode. // When Enabled is true, PostgreSQL and NATS are required. type DistributedConfig struct { - Enabled bool // --distributed / LOCALAI_DISTRIBUTED - InstanceID string // --instance-id / LOCALAI_INSTANCE_ID (auto-generated UUID if empty) + Enabled bool // --distributed / LOCALAI_DISTRIBUTED + InstanceID string // --instance-id / LOCALAI_INSTANCE_ID (auto-generated UUID if empty) + // AdvertiseAddr is the host:port OTHER REPLICAS dial to reach this one, + // which is not the address this process binds: a replica behind a service + // or a NAT binds one and is reached at another. Empty means "work it out", + // by asking the kernel which local address routes to PostgreSQL; that + // answer is only usable when the database is remote, so a deployment with + // a local or sidecar database has to set this. + AdvertiseAddr string // LOCALAI_DISTRIBUTED_ADVERTISE_ADDR NatsURL string // --nats-url / LOCALAI_NATS_URL StorageURL string // --storage-url / LOCALAI_STORAGE_URL (S3 endpoint) RegistrationToken string // --registration-token / LOCALAI_REGISTRATION_TOKEN (required token for node registration) @@ -197,6 +204,14 @@ func WithDistributedInstanceID(id string) AppOption { } } +// WithDistributedAdvertiseAddr pins the host:port peers dial to reach this +// replica, overriding the route-based discovery. +func WithDistributedAdvertiseAddr(addr string) AppOption { + return func(o *ApplicationConfig) { + o.Distributed.AdvertiseAddr = addr + } +} + func WithNatsURL(url string) AppOption { return func(o *ApplicationConfig) { o.Distributed.NatsURL = url diff --git a/core/http/app.go b/core/http/app.go index f03522a45934..c5fc5d54ad54 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -550,6 +550,14 @@ func API(application *application.Application) (*echo.Echo, error) { routes.RegisterNodeSelfServiceRoutes(e, registry, distCfg.RegistrationToken, distCfg.AutoApproveNodes, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret, natsCfg) routes.RegisterNodeAdminRoutes(e, registry, remoteUnloader, application.GalleryService(), opcache, application.ApplicationConfig(), adminMiddleware, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret, application.ApplicationConfig().Distributed.RegistrationToken, natsCfg) + // Replica-to-replica peer link. Registered only in distributed mode: in + // single-node mode there are no peers, and the route authenticates with the + // registration token, so publishing it unconditionally would put a + // multiplexer on every single-binary install. + if d := application.Distributed(); d != nil && d.PeerSessions != nil { + routes.RegisterClusterRoutes(e, distCfg.RegistrationToken, d.PeerSessions.Accept) + } + // Distributed SSE routes (job progress + agent events via NATS) if d := application.Distributed(); d != nil { if d.Dispatcher != nil { diff --git a/core/http/endpoints/cluster/peer.go b/core/http/endpoints/cluster/peer.go index 85985eee5762..f0fb96ebc16e 100644 --- a/core/http/endpoints/cluster/peer.go +++ b/core/http/endpoints/cluster/peer.go @@ -18,18 +18,6 @@ import ( "github.com/mudler/xlog" ) -// RegisterClusterRoutes registers the peer link. onPeer receives every -// authenticated session; see PeerHandler for what it is expected to do with it. -// -// The route is core/services/cluster's own constant, so the handler and the -// dialler cannot be registered and dialled at different paths. That the path -// also falls under auth.ClusterPathPrefix, and so bypasses the session -// middleware, is asserted by a spec in this package: it is the only place that -// can see both, since core/services/cluster must not import core/http/auth. -func RegisterClusterRoutes(e *echo.Echo, token string, onPeer func(string, *yamux.Session)) { - e.GET(clustersvc.PeerPath, PeerHandler(token, onPeer)) -} - // PeerHandler upgrades an authenticated peer dial to a WebSocket, wraps it as // a yamux server session and hands it to onSession. // diff --git a/core/http/endpoints/cluster/peer_test.go b/core/http/endpoints/cluster/peer_test.go index a86ab8add8bd..dc8cba3da9e6 100644 --- a/core/http/endpoints/cluster/peer_test.go +++ b/core/http/endpoints/cluster/peer_test.go @@ -5,8 +5,9 @@ import ( "net/http/httptest" "strings" + "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/auth" - clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" + "github.com/mudler/LocalAI/core/http/routes" clustersvc "github.com/mudler/LocalAI/core/services/cluster" "github.com/gorilla/websocket" @@ -16,6 +17,11 @@ import ( . "github.com/onsi/gomega" ) +// wsPeerURL is the peer route on a test server, named as peer-1. +func wsPeerURL(s *httptest.Server) string { + return "ws" + strings.TrimPrefix(s.URL, "http") + clustersvc.PeerPath + "?id=peer-1" +} + var _ = Describe("Peer link handler", func() { var ( srv *httptest.Server @@ -25,19 +31,15 @@ var _ = Describe("Peer link handler", func() { BeforeEach(func() { sessions = make(chan *yamux.Session, 1) e := echo.New() - clusterep.RegisterClusterRoutes(e, "peer-token", func(_ string, s *yamux.Session) { + routes.RegisterClusterRoutes(e, "peer-token", func(_ string, s *yamux.Session) { sessions <- s }) srv = httptest.NewServer(e) DeferCleanup(srv.Close) }) - wsURL := func(s *httptest.Server) string { - return "ws" + strings.TrimPrefix(s.URL, "http") + "/api/cluster/peer?id=peer-1" - } - It("rejects a connection with no token", func() { - _, resp, err := websocket.DefaultDialer.Dial(wsURL(srv), nil) + _, resp, err := websocket.DefaultDialer.Dial(wsPeerURL(srv), nil) Expect(err).To(HaveOccurred()) Expect(resp).ToNot(BeNil()) Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) @@ -46,7 +48,7 @@ var _ = Describe("Peer link handler", func() { It("rejects a connection with the wrong token", func() { h := http.Header{} h.Set("Authorization", "Bearer wrong") - _, resp, err := websocket.DefaultDialer.Dial(wsURL(srv), h) + _, resp, err := websocket.DefaultDialer.Dial(wsPeerURL(srv), h) Expect(err).To(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) }) @@ -54,7 +56,7 @@ var _ = Describe("Peer link handler", func() { It("accepts an authenticated peer and yields a usable yamux session", func() { h := http.Header{} h.Set("Authorization", "Bearer peer-token") - conn, _, err := websocket.DefaultDialer.Dial(wsURL(srv), h) + conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(srv), h) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = conn.Close() }) @@ -98,11 +100,11 @@ var _ = Describe("Peer link handler", func() { h.Set("Authorization", "Bearer peer-token") ids := make(chan string, 1) e := echo.New() - clusterep.RegisterClusterRoutes(e, "peer-token", func(id string, _ *yamux.Session) { ids <- id }) + routes.RegisterClusterRoutes(e, "peer-token", func(id string, _ *yamux.Session) { ids <- id }) s2 := httptest.NewServer(e) DeferCleanup(s2.Close) - conn, _, err := websocket.DefaultDialer.Dial(wsURL(s2), h) + conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(s2), h) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = conn.Close() }) @@ -115,12 +117,12 @@ var _ = Describe("Peer link handler", func() { // unauthenticated yamux multiplexer to anyone who can reach the port. e := echo.New() accepted := make(chan *yamux.Session, 1) - clusterep.RegisterClusterRoutes(e, "", func(_ string, sess *yamux.Session) { accepted <- sess }) + routes.RegisterClusterRoutes(e, "", func(_ string, sess *yamux.Session) { accepted <- sess }) s2 := httptest.NewServer(e) DeferCleanup(s2.Close) for _, header := range []http.Header{nil, {"Authorization": []string{"Bearer "}}, {"Authorization": []string{"Bearer anything"}}} { - _, resp, err := websocket.DefaultDialer.Dial(wsURL(s2), header) + _, resp, err := websocket.DefaultDialer.Dial(wsPeerURL(s2), header) Expect(err).To(HaveOccurred()) Expect(resp).ToNot(BeNil()) Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) @@ -132,7 +134,7 @@ var _ = Describe("Peer link handler", func() { // RFC 7235 makes the scheme case-insensitive. The token after it is not. h := http.Header{} h.Set("Authorization", "bearer peer-token") - conn, _, err := websocket.DefaultDialer.Dial(wsURL(srv), h) + conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(srv), h) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = conn.Close() }) Eventually(sessions, "5s").Should(Receive()) @@ -143,7 +145,7 @@ var _ = Describe("Peer link handler", func() { // without the handler's own recover the peer would keep a link nobody // ever accepts streams on. e := echo.New() - clusterep.RegisterClusterRoutes(e, "peer-token", func(_ string, _ *yamux.Session) { + routes.RegisterClusterRoutes(e, "peer-token", func(_ string, _ *yamux.Session) { panic("callback exploded") }) s2 := httptest.NewServer(e) @@ -157,7 +159,7 @@ var _ = Describe("Peer link handler", func() { h := http.Header{} h.Set("Authorization", "Bearer peer-token") - conn, _, err := websocket.DefaultDialer.Dial(wsURL(s2), h) + conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(s2), h) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = conn.Close() }) @@ -186,17 +188,69 @@ var _ = Describe("Peer link handler", func() { }) }) -var _ = Describe("Peer link auth prefix", func() { - It("keeps the peer route inside the alternative-authentication prefix", func() { - // The peer route authenticates with the cluster token, not the global - // session middleware, which only holds while the route sits under the - // prefix auth exempts. Moving either one alone 401s every peer dial. - // - // This lives here because it is the only package that can see both: - // core/services/cluster owns the route and must stay free of any - // core/http dependency, and core/http/auth owns the exemption. - Expect(strings.HasPrefix(clustersvc.PeerPath, auth.ClusterPathPrefix)).To(BeTrue(), - "peer route %q is no longer under the auth-exempt prefix %q", - clustersvc.PeerPath, auth.ClusterPathPrefix) +var _ = Describe("Peer link auth coverage", func() { + // These specs put the REAL global auth middleware in front of the REAL + // registrar and prove a peer dial reaches the handler anyway. The peer link + // authenticates with the cluster token, not a session, so it only works + // while its path sits under the prefix auth exempts; moving either one + // alone 401s every peer dial, and the two live in packages that must not + // import each other. + // + // The predicate that grants the exemption is unexported, so this asserts on + // its effect rather than on it: what a caller can observe is whether the + // request reaches the handler. + var ( + srv *httptest.Server + sessions chan *yamux.Session + ) + + BeforeEach(func() { + sessions = make(chan *yamux.Session, 1) + e := echo.New() + // A nil DB with one legacy API key is the cheapest configuration that + // turns the middleware ON without a database. With neither, Middleware + // short-circuits to next() and every assertion below would pass against + // a server that has no auth at all. + e.Use(auth.Middleware(nil, &config.ApplicationConfig{ApiKeys: []string{"an-api-key"}})) + routes.RegisterClusterRoutes(e, "peer-token", func(_ string, s *yamux.Session) { sessions <- s }) + // A route outside the cluster prefix, registered on the same server, is + // the control: it proves the middleware in front of both is live. + e.GET("/api/nodes", func(c echo.Context) error { return c.NoContent(http.StatusOK) }) + srv = httptest.NewServer(e) + DeferCleanup(srv.Close) + }) + + It("refuses an uncredentialed request to a route outside the cluster prefix", func() { + resp, err := http.Get(srv.URL + "/api/nodes") + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = resp.Body.Close() }() + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized), + "the global auth middleware is not actually guarding this server, so the peer-route assertions below would prove nothing") + }) + + It("lets a peer dial reach the handler, which is the only thing that can authenticate it", func() { + // The cluster token is not one of the API keys the middleware knows, so + // a 400 from the handler's own missing-id check can only mean the + // request was let through unauthenticated by the middleware. + req, err := http.NewRequestWithContext(GinkgoT().Context(), http.MethodGet, srv.URL+clustersvc.PeerPath, nil) + Expect(err).ToNot(HaveOccurred()) + req.Header.Set("Authorization", "Bearer peer-token") + + resp, err := http.DefaultClient.Do(req) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = resp.Body.Close() }() + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest), + "a peer dial must reach the handler; 401 here means the peer route left the auth-exempt prefix %q", auth.ClusterPathPrefix) + }) + + It("completes a full peer handshake through the guarded server", func() { + // The status-code assertion above cannot see the upgrade, and the + // upgrade is what a peer actually does. + h := http.Header{} + h.Set("Authorization", "Bearer peer-token") + conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(srv), h) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + Eventually(sessions, "5s").Should(Receive()) }) }) diff --git a/core/http/routes/cluster.go b/core/http/routes/cluster.go new file mode 100644 index 000000000000..3e462db1cd75 --- /dev/null +++ b/core/http/routes/cluster.go @@ -0,0 +1,25 @@ +package routes + +import ( + clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" + clustersvc "github.com/mudler/LocalAI/core/services/cluster" + + "github.com/labstack/echo/v4" + "github.com/libp2p/go-yamux/v5" +) + +// RegisterClusterRoutes registers the replica-to-replica peer link. onPeer +// receives every authenticated session; see clusterep.PeerHandler for what it +// is expected to do with it. +// +// The path is core/services/cluster's own constant, so the handler and the +// dialler cannot be registered and dialled at different paths. That the path +// also falls under auth.ClusterPathPrefix, and so bypasses the global session +// middleware, is asserted by driving a request through that middleware in +// core/http/endpoints/cluster/peer_test.go. +// +// The route carries no auth middleware: it authenticates itself against the +// cluster token, because a peer replica has no session and no user. +func RegisterClusterRoutes(e *echo.Echo, token string, onPeer func(string, *yamux.Session)) { + e.GET(clustersvc.PeerPath, clusterep.PeerHandler(token, onPeer)) +} diff --git a/core/services/cluster/instance_test.go b/core/services/cluster/instance_test.go index 8e74e930e3b9..d8075a7a3cad 100644 --- a/core/services/cluster/instance_test.go +++ b/core/services/cluster/instance_test.go @@ -22,9 +22,9 @@ var _ = Describe("Instance registry", func() { BeforeEach(func() { db = testutil.SetupTestDB() - Expect(db.AutoMigrate(&cluster.Instance{})).To(Succeed()) - reg = cluster.NewRegistry(db) ctx = context.Background() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg = cluster.NewRegistry(db) }) It("registers an instance and reads it back", func() { diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go new file mode 100644 index 000000000000..3b1182b17458 --- /dev/null +++ b/core/services/cluster/membership.go @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: MIT + +package cluster + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/mudler/xlog" + "gorm.io/gorm" +) + +const ( + // InstanceHeartbeat is how often a replica refreshes its own row. + InstanceHeartbeat = 5 * time.Second + + // InstanceLiveness is how long a replica may go without a heartbeat before + // its peers treat it as gone: six consecutive misses. + // + // The window is generous on purpose. Declaring a replica dead deletes the + // connection rows it owned, and a worker whose row is deleted while its + // owner is merely slow has to be re-homed for nothing. The cost of waiting + // is bounded and symmetric: traffic for that worker is retried, not lost. + InstanceLiveness = 30 * time.Second +) + +// Membership publishes this replica's address and keeps the instances table +// free of replicas that have stopped answering. +// +// It is the only writer of this replica's row and the only sweeper of anyone +// else's, which is what keeps one fact on one clock: whether a replica is +// alive is answered by its last_seen and by nothing else. +type Membership struct { + reg *Registry + id string + addr string + version string + + interval time.Duration + liveness time.Duration + + stop chan struct{} + done chan struct{} + stopOnce sync.Once +} + +// NewMembership returns the membership loop for one replica. The address is +// what peers will dial, so it must be reachable from another host, not the +// address this process binds. +func NewMembership(reg *Registry, id, addr, version string) *Membership { + return &Membership{ + reg: reg, + id: id, + addr: addr, + version: version, + interval: InstanceHeartbeat, + liveness: InstanceLiveness, + stop: make(chan struct{}), + done: make(chan struct{}), + } +} + +// Start registers this replica and begins heartbeating and sweeping. The first +// registration is synchronous and its failure is returned: a replica whose +// address never reaches the table is invisible to its peers, and starting +// anyway would hide that behind a background log line. +func (m *Membership) Start(ctx context.Context) error { + if err := m.reg.Register(ctx, m.id, m.addr, m.version); err != nil { + return err + } + xlog.Info("Cluster instance registered", "id", m.id, "addr", m.addr) + go m.loop(ctx) + return nil +} + +// Stop ends the loop and waits for it. Safe to call more than once. +func (m *Membership) Stop() { + if m == nil { + return + } + m.stopOnce.Do(func() { close(m.stop) }) + <-m.done +} + +func (m *Membership) loop(ctx context.Context) { + defer close(m.done) + + ticker := time.NewTicker(m.interval) + defer ticker.Stop() + + for { + select { + case <-m.stop: + return + case <-ctx.Done(): + return + case <-ticker.C: + m.tick(ctx) + } + } +} + +// tick refreshes this replica's row and sweeps the dead. +// +// Every replica sweeps, rather than one elected sweeper. The deletes are +// idempotent and cheap, and an elected sweeper is one more thing that has to be +// alive for the cluster to notice that something is not. +func (m *Membership) tick(ctx context.Context) { + err := m.reg.Heartbeat(ctx, m.id) + if errors.Is(err, ErrInstanceNotFound) { + // Another replica swept this row while this process was stalled long + // enough to look dead. Re-register rather than heartbeat: a heartbeat + // carries no address, so the row has to be rebuilt from scratch. + xlog.Warn("Cluster instance row was reaped, re-registering", "id", m.id) + if err := m.reg.Register(ctx, m.id, m.addr, m.version); err != nil { + xlog.Error("Re-registering cluster instance failed", "id", m.id, "error", err) + } + } else if err != nil { + xlog.Warn("Cluster instance heartbeat failed", "id", m.id, "error", err) + } + + instances, connections, err := m.reg.ReapStale(ctx, m.id, m.liveness) + if err != nil { + xlog.Warn("Reaping stale cluster instances failed", "error", err) + return + } + if instances > 0 || connections > 0 { + xlog.Info("Reaped cluster state left by dead replicas", "instances", instances, "connections", connections) + } +} + +// ReapStale deletes the replicas that have not heartbeated within the liveness +// window, and the connection rows whose owner is no longer among the survivors. +// +// The two deletes are one sweeper on purpose. A connection row is only ever +// orphaned by its owner dying, so the moment that is decided is the moment to +// clean up after it; a second sweeper with its own schedule would either lag +// this one or race it, and would need its own answer to "is that replica +// alive", which is the one fact this table already owns. +// +// self is never reaped. This process may fail to heartbeat for longer than the +// window (a long stall, a database blip) and still be serving: deleting its own +// row would then delete the connections of workers that are, at that moment, +// connected to it. The stall is recovered by the re-register in tick instead. +// +// PostgreSQL only, like Live: distributed mode requires it, and the interval +// arithmetic is measured on the database's clock because liveness is compared +// across replicas. +func (r *Registry) ReapStale(ctx context.Context, self string, within time.Duration) (instances int64, connections int64, err error) { + err = r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + res := tx.Where("id <> ? AND last_seen <= now() - make_interval(secs => ?)", self, within.Seconds()). + Delete(&Instance{}) + if res.Error != nil { + return fmt.Errorf("deleting stale instances: %w", res.Error) + } + instances = res.RowsAffected + + // Whatever survived the delete above is the live set, so this needs no + // second liveness rule and cannot disagree with the first one. + res = tx.Where("owner_instance_id NOT IN (SELECT id FROM instances)"). + Delete(&NodeConnection{}) + if res.Error != nil { + return fmt.Errorf("deleting orphaned node connections: %w", res.Error) + } + connections = res.RowsAffected + return nil + }) + if err != nil { + return 0, 0, fmt.Errorf("reaping stale cluster state: %w", err) + } + return instances, connections, nil +} diff --git a/core/services/cluster/membership_test.go b/core/services/cluster/membership_test.go new file mode 100644 index 000000000000..a00a08c0b954 --- /dev/null +++ b/core/services/cluster/membership_test.go @@ -0,0 +1,116 @@ +package cluster_test + +import ( + "context" + "time" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +var _ = Describe("Reaping dead replicas", func() { + var ( + db *gorm.DB + reg *cluster.Registry + ctx context.Context + ) + + // age pushes a replica's heartbeat into the past. Sleeping in a spec is + // forbidden, and the liveness window is measured in tens of seconds. + age := func(id string, by time.Duration) { + GinkgoHelper() + Expect(db.Model(&cluster.Instance{}).Where("id = ?", id). + Update("last_seen", gorm.Expr("now() - make_interval(secs => ?)", by.Seconds())).Error).To(Succeed()) + } + + BeforeEach(func() { + db = testutil.SetupTestDB() + ctx = context.Background() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg = cluster.NewRegistry(db) + }) + + It("deletes a replica that stopped heartbeating, and the connections it owned", func() { + Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "dead") + Expect(err).ToNot(HaveOccurred()) + age("dead", time.Hour) + + instances, connections, err := reg.ReapStale(ctx, "live", time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(instances).To(Equal(int64(1))) + Expect(connections).To(Equal(int64(1)), + "a worker whose owner no longer exists is recorded as connected to nothing") + + _, _, err = reg.Owner(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + }) + + It("leaves the connections of a live replica alone", func() { + Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "other", "10.0.0.2:8080", "v1")).To(Succeed()) + epoch, err := reg.Claim(ctx, "w1", "other") + Expect(err).ToNot(HaveOccurred()) + + _, connections, err := reg.ReapStale(ctx, "live", time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(connections).To(BeZero()) + + owner, stored, err := reg.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("other")) + Expect(stored).To(Equal(epoch)) + }) + + It("never reaps the sweeper itself, however stale its own row looks", func() { + // A replica whose heartbeat stalled longer than the window is still + // serving the workers connected to it. Reaping its own row would delete + // their connection rows in the same pass, re-homing workers that never + // went anywhere. + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "me") + Expect(err).ToNot(HaveOccurred()) + age("me", time.Hour) + + instances, connections, err := reg.ReapStale(ctx, "me", time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(instances).To(BeZero()) + Expect(connections).To(BeZero()) + + owner, _, err := reg.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("me")) + }) + + It("keeps this replica's row alive and reaps the dead while it runs", func() { + Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1")).To(Succeed()) + age("dead", time.Hour) + + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + Expect(membership.Start(ctx)).To(Succeed()) + DeferCleanup(membership.Stop) + + // Rows, not live rows: an aged-out replica drops out of Live + // immediately, and what the sweeper adds is deleting it. Asserting on + // Live here would pass with no sweeper at all. + rows := func() int64 { + var n int64 + if err := db.Model(&cluster.Instance{}).Count(&n).Error; err != nil { + return -1 + } + return n + } + Expect(rows()).To(Equal(int64(2)), "the stale row is still in the table until a sweep deletes it") + + Eventually(rows, 3*cluster.InstanceHeartbeat, time.Second).Should(Equal(int64(1))) + live, err := reg.Live(ctx, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(live).To(HaveLen(1)) + Expect(live[0].ID).To(Equal("me"), "the sweeper deleted the wrong row") + }) +}) diff --git a/core/services/cluster/ownership.go b/core/services/cluster/ownership.go index a2e2ba9cb5ae..3b6b6cb622e4 100644 --- a/core/services/cluster/ownership.go +++ b/core/services/cluster/ownership.go @@ -59,17 +59,34 @@ type NodeConnection struct { ConnectedAt time.Time `gorm:"not null" json:"connected_at"` } -// EnsureEpochSequence creates the sequence Claim draws epochs from. It lives +// Migrate creates every table and sequence this package owns. It is the one +// call a caller has to remember: gorm's AutoMigrate models tables and columns +// but has no notion of a sequence, and the connection fence draws its epochs +// from one, so a caller that knew only about AutoMigrate would leave a schema +// that looks complete and cannot claim. Safe to call repeatedly. +// +// It does not take the migration advisory lock itself. The caller holds it +// across every table in the deployment, and taking a second one here would +// either nest inside that one or, worse, be the reason someone stops holding +// the outer one. +func Migrate(ctx context.Context, db *gorm.DB) error { + if err := db.WithContext(ctx).AutoMigrate(&Instance{}, &NodeConnection{}); err != nil { + return fmt.Errorf("migrating cluster tables: %w", err) + } + return ensureEpochSequence(ctx, db) +} + +// ensureEpochSequence creates the sequence Claim draws epochs from. It lives // here, beside the model that needs it, because gorm's AutoMigrate models // tables and columns but has no notion of a sequence; the caller that owns the -// migration advisory lock calls it so that concurrently starting replicas do -// not race on the DDL. It is safe to call repeatedly. +// migration advisory lock calls Migrate so that concurrently starting replicas +// do not race on the DDL. It is safe to call repeatedly. // // The sequence is not attached as a column DEFAULT on purpose: AutoMigrate // compares the struct's declared default against the one PostgreSQL reports // (`nextval('...'::regclass)`), and a mismatch there makes every startup ALTER // the column. Naming the sequence in the statement keeps the schema stable. -func EnsureEpochSequence(ctx context.Context, db *gorm.DB) error { +func ensureEpochSequence(ctx context.Context, db *gorm.DB) error { // CREATE SEQUENCE is PostgreSQL-only, and the same migration path runs // against SQLite in single-binary mode. Nothing there can claim a // connection (Claim refuses the dialect outright), so there is nothing to diff --git a/core/services/cluster/ownership_test.go b/core/services/cluster/ownership_test.go index ba634ebcd3eb..5744e07cd170 100644 --- a/core/services/cluster/ownership_test.go +++ b/core/services/cluster/ownership_test.go @@ -67,8 +67,7 @@ var _ = Describe("Connection ownership", func() { BeforeEach(func() { db = testutil.SetupTestDB() ctx = context.Background() - Expect(db.AutoMigrate(&cluster.NodeConnection{})).To(Succeed()) - Expect(cluster.EnsureEpochSequence(ctx, db)).To(Succeed()) + Expect(cluster.Migrate(ctx, db)).To(Succeed()) reg = cluster.NewRegistry(db) }) @@ -243,13 +242,11 @@ var _ = Describe("Connection ownership on a non-PostgreSQL dialect", func() { It("migrates, because the single-binary path shares this schema", func() { // A PostgreSQL-only column DEFAULT here breaks AutoMigrate for every // SQLite caller of nodes.NewNodeRegistry, which is how this regressed. - Expect(db.AutoMigrate(&cluster.NodeConnection{})).To(Succeed()) - Expect(cluster.EnsureEpochSequence(ctx, db)).To(Succeed()) + Expect(cluster.Migrate(ctx, db)).To(Succeed()) }) It("refuses to claim, rather than pretending to fence", func() { - Expect(db.AutoMigrate(&cluster.NodeConnection{})).To(Succeed()) - Expect(cluster.EnsureEpochSequence(ctx, db)).To(Succeed()) + Expect(cluster.Migrate(ctx, db)).To(Succeed()) _, err := cluster.NewRegistry(db).Claim(ctx, "w1", "inst-a") Expect(err).To(HaveOccurred()) diff --git a/core/services/cluster/peerlink_test.go b/core/services/cluster/peerlink_test.go index cfaa4dc69392..692f2ee6c22a 100644 --- a/core/services/cluster/peerlink_test.go +++ b/core/services/cluster/peerlink_test.go @@ -20,6 +20,16 @@ import ( "gorm.io/gorm" ) +// servePeerRoute mounts the peer handler on the route both sides agree on. +// +// It deliberately does not call routes.RegisterClusterRoutes: that registrar +// lives in core/http/routes, which imports half the server, and these specs are +// about the handler and the dialler rather than about the route table. The path +// comes from the same constant the registrar uses, so the two cannot drift. +func servePeerRoute(e *echo.Echo, token string, onPeer func(string, *yamux.Session)) { + e.GET(cluster.PeerPath, clusterep.PeerHandler(token, onPeer)) +} + var _ = Describe("Peer pool", func() { var ( db *gorm.DB @@ -33,7 +43,7 @@ var _ = Describe("Peer pool", func() { // startPeer stands up a real peer server and registers it under peerID. startPeer := func(peerID string) *httptest.Server { e := echo.New() - clusterep.RegisterClusterRoutes(e, "peer-token", func(_ string, s *yamux.Session) { + servePeerRoute(e, "peer-token", func(_ string, s *yamux.Session) { accepted <- s }) ts := httptest.NewServer(e) @@ -45,7 +55,7 @@ var _ = Describe("Peer pool", func() { BeforeEach(func() { ctx = context.Background() db = testutil.SetupTestDB() - Expect(db.AutoMigrate(&cluster.Instance{})).To(Succeed()) + Expect(cluster.Migrate(ctx, db)).To(Succeed()) reg = cluster.NewRegistry(db) accepted = make(chan *yamux.Session, 4) pool = cluster.NewPeerPool("self", "peer-token", reg) @@ -88,7 +98,7 @@ var _ = Describe("Peer pool", func() { // link anonymous and indistinguishable from every other. ids := make(chan string, 1) e := echo.New() - clusterep.RegisterClusterRoutes(e, "peer-token", func(id string, _ *yamux.Session) { ids <- id }) + servePeerRoute(e, "peer-token", func(id string, _ *yamux.Session) { ids <- id }) ts := httptest.NewServer(e) DeferCleanup(ts.Close) Expect(reg.Register(ctx, "peer-named", strings.TrimPrefix(ts.URL, "http://"), "test")).To(Succeed()) @@ -131,7 +141,7 @@ var _ = Describe("Peer pool", func() { // row. Reporting absence here would evict every worker behind a peer // that was merely rolled out with a stale secret. e := echo.New() - clusterep.RegisterClusterRoutes(e, "a-different-token", func(_ string, s *yamux.Session) { accepted <- s }) + servePeerRoute(e, "a-different-token", func(_ string, s *yamux.Session) { accepted <- s }) ts := httptest.NewServer(e) DeferCleanup(ts.Close) Expect(reg.Register(ctx, "peer-strict", strings.TrimPrefix(ts.URL, "http://"), "test")).To(Succeed()) diff --git a/core/services/cluster/sessions.go b/core/services/cluster/sessions.go new file mode 100644 index 000000000000..7ee5fa3327af --- /dev/null +++ b/core/services/cluster/sessions.go @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: MIT + +package cluster + +import ( + "net" + "sync" + + "github.com/libp2p/go-yamux/v5" + "github.com/mudler/xlog" +) + +// SessionStore holds the peer links this replica has ACCEPTED, which is the +// mirror image of PeerPool: the pool owns the sessions this replica dialled, +// this owns the ones its peers dialled into it. +// +// Something has to own an accepted session. The HTTP handler cannot: it returns +// as soon as the upgrade is done, and the hijacked connection outlives it. And +// something has to accept the streams that arrive on it, because yamux only +// acknowledges a stream once the far side accepts it, so a session nobody +// accepts on does not fail a peer's Open, it hangs it. +type SessionStore struct { + // onStream handles one accepted stream and owns closing it. A nil handler + // closes the stream immediately, which is what a replica with no relay + // installed should do: refuse promptly rather than leave a peer parked. + onStream func(peerID string, stream net.Conn) + + mu sync.Mutex + sessions map[string]*yamux.Session + closed bool +} + +// NewSessionStore returns a store whose accepted streams are handled by +// onStream. Pass nil to refuse every stream, closing it at once. +func NewSessionStore(onStream func(peerID string, stream net.Conn)) *SessionStore { + return &SessionStore{onStream: onStream, sessions: map[string]*yamux.Session{}} +} + +// Accept takes ownership of a session a peer dialled in. It is the callback +// shape RegisterClusterRoutes wants, and it returns promptly: the serving loop +// runs on its own goroutine, because the handler's return is what completes the +// hijack. +func (s *SessionStore) Accept(peerID string, sess *yamux.Session) { + if sess == nil { + return + } + + s.mu.Lock() + if s.closed { + s.mu.Unlock() + // Shutdown raced the dial. Leaving the session open would keep the peer + // believing it has a live link into a process that is going away. + _ = sess.Close() + return + } + previous := s.sessions[peerID] + s.sessions[peerID] = sess + s.mu.Unlock() + + // A peer that dials again has lost its previous link, whether or not this + // side has noticed. Keeping both would leave a session nothing can ever be + // routed to, since the map holds one per peer. + if previous != nil { + xlog.Debug("cluster peer re-dialled, dropping its previous link", "peer", peerID) + _ = previous.Close() + } + + go s.serve(peerID, sess) +} + +// Get returns the session this replica accepted from peerID. The second result +// is false when no link from that peer is held, which a caller must not read as +// the peer being absent: it may be about to dial, or dialling this replica may +// simply not be its job. +func (s *SessionStore) Get(peerID string) (*yamux.Session, bool) { + s.mu.Lock() + defer s.mu.Unlock() + sess, ok := s.sessions[peerID] + return sess, ok +} + +// serve accepts streams until the session dies, then forgets it. +func (s *SessionStore) serve(peerID string, sess *yamux.Session) { + defer func() { + s.forget(peerID, sess) + _ = sess.Close() + }() + + for { + stream, err := sess.AcceptStream() + if err != nil { + // A peer link ending is ordinary: a rolling update closes every + // session it holds. The error is the session's, not one stream's, + // so there is nothing to recover to. + xlog.Debug("cluster peer link ended", "peer", peerID, "error", err) + return + } + if s.onStream == nil { + // No relay installed. Closing is deliberate and is not the same as + // ignoring: a stream nobody answers parks the peer's request until + // its own deadline, and reports nothing about why. + xlog.Debug("cluster peer stream refused: no relay installed", "peer", peerID) + _ = stream.Close() + continue + } + // One goroutine per stream: the handler relays a whole request, and + // serving them from the accept loop would let one request stall every + // other stream on the link. + go s.onStream(peerID, stream) + } +} + +// forget drops the entry only if it still names this session. A peer that +// re-dialled has already replaced it, and deleting blindly would evict the live +// link when the old one finally noticed it was dead. +func (s *SessionStore) forget(peerID string, sess *yamux.Session) { + s.mu.Lock() + defer s.mu.Unlock() + if s.sessions[peerID] == sess { + delete(s.sessions, peerID) + } +} + +// CloseAll drops every held session. An Accept after it closes the session +// rather than storing it, so a dial racing shutdown cannot leak a link. +func (s *SessionStore) CloseAll() { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + s.closed = true + held := s.sessions + s.sessions = map[string]*yamux.Session{} + s.mu.Unlock() + + for _, sess := range held { + _ = sess.Close() + } +} diff --git a/core/services/cluster/sessions_test.go b/core/services/cluster/sessions_test.go new file mode 100644 index 000000000000..f3fb44ca8cae --- /dev/null +++ b/core/services/cluster/sessions_test.go @@ -0,0 +1,122 @@ +package cluster_test + +import ( + "net" + "time" + + "github.com/mudler/LocalAI/core/services/cluster" + + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// yamuxPair returns a client and a server session over an in-memory pipe. It +// stands in for a dialled peer link: everything the store does with a session +// is transport-agnostic, and the WebSocket half is covered where it is used. +func yamuxPair() (client *yamux.Session, server *yamux.Session) { + GinkgoHelper() + a, b := net.Pipe() + var err error + server, err = yamux.Server(a, nil, nil) + Expect(err).ToNot(HaveOccurred()) + client, err = yamux.Client(b, nil, nil) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { + _ = client.Close() + _ = server.Close() + }) + return client, server +} + +var _ = Describe("Accepted peer sessions", func() { + It("accepts and refuses a stream rather than leaving the peer parked", func() { + // yamux only acknowledges a stream once the far side accepts it, so a + // store that held the session without accepting would not fail a peer's + // Open, it would hang it, and every relayed request behind it. + store := cluster.NewSessionStore(nil) + DeferCleanup(store.CloseAll) + client, server := yamuxPair() + store.Accept("peer-1", server) + + stream, err := client.OpenStream(GinkgoT().Context()) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = stream.Close() }) + + Expect(stream.SetReadDeadline(time.Now().Add(10 * time.Second))).To(Succeed()) + _, err = stream.Read(make([]byte, 1)) + Expect(err).To(HaveOccurred(), "a refused stream must end, not hang") + }) + + It("hands a stream to the relay when one is installed", func() { + streams := make(chan net.Conn, 1) + store := cluster.NewSessionStore(func(_ string, stream net.Conn) { streams <- stream }) + DeferCleanup(store.CloseAll) + client, server := yamuxPair() + store.Accept("peer-1", server) + + stream, err := client.OpenStream(GinkgoT().Context()) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = stream.Close() }) + + var relayed net.Conn + Eventually(streams, "10s").Should(Receive(&relayed)) + go func() { + defer GinkgoRecover() + _, _ = stream.Write([]byte("hello")) + }() + buf := make([]byte, 5) + Expect(relayed.SetReadDeadline(time.Now().Add(10 * time.Second))).To(Succeed()) + _, err = relayed.Read(buf) + Expect(err).ToNot(HaveOccurred()) + Expect(string(buf)).To(Equal("hello")) + }) + + It("replaces a peer's link when it dials again, and closes the one it lost", func() { + // A peer only re-dials because its previous link is gone from where it + // stands. Keeping both would leave a session nothing can be routed to, + // since the store holds one per peer. + store := cluster.NewSessionStore(nil) + DeferCleanup(store.CloseAll) + _, first := yamuxPair() + _, second := yamuxPair() + + store.Accept("peer-1", first) + store.Accept("peer-1", second) + + held, ok := store.Get("peer-1") + Expect(ok).To(BeTrue()) + Expect(held).To(BeIdenticalTo(second)) + Eventually(first.IsClosed, "10s").Should(BeTrue()) + Expect(second.IsClosed()).To(BeFalse(), "the link the peer is actually using was dropped") + }) + + It("forgets a session that ended, without evicting the one that replaced it", func() { + store := cluster.NewSessionStore(nil) + DeferCleanup(store.CloseAll) + client, server := yamuxPair() + store.Accept("peer-1", server) + + Expect(client.Close()).To(Succeed()) + Eventually(func() bool { + _, ok := store.Get("peer-1") + return ok + }, "10s").Should(BeFalse()) + }) + + It("closes every held link on shutdown, and refuses to store one afterwards", func() { + store := cluster.NewSessionStore(nil) + _, server := yamuxPair() + store.Accept("peer-1", server) + + store.CloseAll() + Eventually(server.IsClosed, "10s").Should(BeTrue()) + + _, late := yamuxPair() + store.Accept("peer-late", late) + _, ok := store.Get("peer-late") + Expect(ok).To(BeFalse()) + Eventually(late.IsClosed, "10s").Should(BeTrue(), + "a dial racing shutdown must not be left believing it holds a live link") + }) +}) diff --git a/core/services/cluster/wsconn_test.go b/core/services/cluster/wsconn_test.go index 305ce99b3c79..d7213fd87779 100644 --- a/core/services/cluster/wsconn_test.go +++ b/core/services/cluster/wsconn_test.go @@ -11,7 +11,6 @@ import ( "strings" "time" - clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" "github.com/mudler/LocalAI/core/services/cluster" "github.com/gorilla/websocket" @@ -239,7 +238,7 @@ var _ = Describe("Peer link payloads", func() { It("carries a payload far larger than one yamux frame end to end", func() { sessions := make(chan *yamux.Session, 1) e := echo.New() - clusterep.RegisterClusterRoutes(e, "peer-token", func(_ string, s *yamux.Session) { sessions <- s }) + servePeerRoute(e, "peer-token", func(_ string, s *yamux.Session) { sessions <- s }) srv := httptest.NewServer(e) DeferCleanup(srv.Close) diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index c3dc878ac409..0e2195cdaecd 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -488,14 +488,14 @@ const heartbeatMaterialDelta = 256 << 20 // 256 MiB // when multiple instances (frontend + workers) start at the same time. func NewNodeRegistry(db *gorm.DB) (*NodeRegistry, error) { if err := advisorylock.WithLockCtx(context.Background(), db, advisorylock.KeySchemaMigrate, func() error { - if err := db.AutoMigrate(&BackendNode{}, &NodeModel{}, &NodeLabel{}, &ModelSchedulingConfig{}, &PendingBackendOp{}, &ModelLoadInfo{}, &ModelLoadJob{}, &ModelConfigState{}, &cluster.Instance{}, &cluster.NodeConnection{}); err != nil { + if err := db.AutoMigrate(&BackendNode{}, &NodeModel{}, &NodeLabel{}, &ModelSchedulingConfig{}, &PendingBackendOp{}, &ModelLoadInfo{}, &ModelLoadJob{}, &ModelConfigState{}); err != nil { return err } - // AutoMigrate models tables and columns but has no notion of a - // sequence, and the connection-ownership fence draws its epochs from - // one. It runs under this same lock so concurrently starting replicas - // do not race on the DDL. - return cluster.EnsureEpochSequence(context.Background(), db) + // The cluster package owns its own tables AND the sequence its + // ownership fence draws epochs from, which AutoMigrate cannot express. + // It runs under this same lock so concurrently starting replicas do not + // race on the DDL. + return cluster.Migrate(context.Background(), db) }); err != nil { return nil, fmt.Errorf("migrating node tables: %w", err) } diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index f8a06539a20f..bede4de44380 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -64,6 +64,7 @@ The frontend is a standard LocalAI instance with distributed mode enabled. These | `--distributed` | `LOCALAI_DISTRIBUTED` | `false` | Enable distributed mode | | `--instance-id` | `LOCALAI_INSTANCE_ID` | auto UUID | Unique instance ID for this frontend | | `--nats-url` | `LOCALAI_NATS_URL` | *(required)* | NATS server URL (e.g., `nats://localhost:4222`) | +| `--distributed-advertise-addr` | `LOCALAI_DISTRIBUTED_ADVERTISE_ADDR` | *(derived)* | `host:port` the **other frontend replicas** dial to reach this one. See [Replica peer links](#replica-peer-links). | | `--registration-token` | `LOCALAI_REGISTRATION_TOKEN` | *(empty)* | Token that workers must provide to register | | `--registration-require-auth` | `LOCALAI_REGISTRATION_REQUIRE_AUTH` | `false` | Fail startup when distributed mode is enabled but the registration token is empty (node endpoints and worker file-transfer would otherwise be unauthenticated) | | `--distributed-require-auth` | `LOCALAI_DISTRIBUTED_REQUIRE_AUTH` | `false` | **Umbrella switch.** Implies both `--nats-require-auth` and `--registration-require-auth` - one knob to lock down the NATS bus *and* the registration/file-transfer layer. Set this in production instead of the two granular flags. | @@ -80,6 +81,27 @@ The frontend is a standard LocalAI instance with distributed mode enabled. These | `--stale-node-threshold` | `LOCALAI_STALE_NODE_THRESHOLD` | `5m` | How long a node may go without a **durable** heartbeat before the health monitor marks it `offline`. Because `--node-heartbeat-checkpoint` holds back a beat that only carries a fresher timestamp, this has to stay comfortably wider than that interval: raising the checkpoint without raising this marks healthy, beating nodes offline. Neither the per-model gRPC health check nor request-time failure reads `last_heartbeat`, so neither is affected by this knob. See [Heartbeat writes and stale-node detection](#heartbeat-writes-and-stale-node-detection). | | `--expose-node-header` | `LOCALAI_EXPOSE_NODE_HEADER` | `false` | When enabled, inference responses carry an `X-LocalAI-Node` header with the ID of the worker node that served the request. Coverage spans the OpenAI-compatible endpoints (chat completions, completions, embeddings, audio transcriptions, audio speech / TTS, image generations, image inpainting), the Jina rerank endpoint (`/v1/rerank`), the VAD endpoints (`/v1/vad`, `/vad`), and the Anthropic Messages (`/v1/messages`) and Ollama (`/api/chat`, `/api/generate`, `/api/embed`) shims. Useful for debugging, observability and load-balancer attribution. Off by default: the node ID reveals internal cluster topology and should not be exposed on a public endpoint. Best-effort: under heavy concurrency for the same model across multiple replicas, the header may reflect a recent routing decision rather than this exact request's. Acceptable for observability and debugging. | +### Replica peer links + +Frontend replicas record themselves in an `instances` table and open direct links to each other, so that a request arriving at one replica can be served by state another replica holds. Each replica publishes one address for this, and every other replica dials it: it is the address **peers** use, which is not necessarily the address the process binds. A replica behind a Kubernetes Service, a load balancer or a NAT binds one and is reached at another. + +When `LOCALAI_DISTRIBUTED_ADVERTISE_ADDR` is unset, the address is derived: LocalAI asks the kernel which local address routes to PostgreSQL, and pairs it with the port it serves on. Every replica reaches the same database, so that address is on a network they demonstrably share. + +That only holds while the database is on **another host**. If PostgreSQL runs on the same host or pod (compose, single-node, a sidecar), the route to it is loopback, and advertising a loopback address would send every peer to itself. LocalAI refuses to guess in that case and logs: + +``` +This replica will not be reachable by its peers: no advertised address +``` + +The replica keeps serving every request that reaches it directly; what it cannot do is have another replica reach it. Set the address explicitly to fix it: + +```yaml +environment: + LOCALAI_DISTRIBUTED_ADVERTISE_ADDR: "10.0.1.7:8080" # or the pod IP, service DNS name, etc. +``` + +The peer link is served at `/api/cluster/peer` and authenticates with `LOCALAI_REGISTRATION_TOKEN`, the same shared secret workers register with. Replicas that disagree about it cannot link. A replica that stops heartbeating for 30 seconds is dropped from the table by the others, along with the worker-connection rows it owned. + ### The model load deadline scales with the checkpoint The `LoadModel` deadline starts *after* the backend is installed and the model files are staged, so it covers only the worker backend's own checkpoint read and pipeline init. That work is proportional to the bytes on disk, which makes any fixed deadline a model-size cliff rather than a timeout: a 70 GB video checkpoint on a Jetson Thor worker failed reproducibly against the old fixed 5m default (`rpc error: code = DeadlineExceeded` after 953.5s of wall clock, roughly 11m of which was backend install and staging), and simply raising the constant would only move the cliff to the next larger model while making a genuinely wedged *small* model hang for the whole inflated duration. diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 1783d71e16f0..521882efe962 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -209,6 +209,14 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { // Pinning makes the cross-replica session a property of the harness. "LOCALAI_AUTH_HMAC_SECRET="+testHMACSecret, "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, + // Every replica here shares one host, so the address a peer dials is + // this process's own loopback address. It has to be said explicitly: + // the automatic discovery asks which local address routes to + // PostgreSQL, and this suite's PostgreSQL is a container published on + // 127.0.0.1, so the discovery refuses (correctly) rather than + // advertising a loopback address that would mean "yourself" on a + // multi-host deployment. + fmt.Sprintf("LOCALAI_DISTRIBUTED_ADVERTISE_ADDR=127.0.0.1:%d", port), "LOCALAI_AUTO_APPROVE_NODES=true", "DEBUG=true", ) @@ -334,6 +342,14 @@ func (c *Cluster) FrontendURL(i int) string { return fmt.Sprintf("http://127.0.0.1:%d", c.frontends[i].Port) } +// RegistrationToken is the shared secret this cluster was started with. It +// authenticates worker registration AND the replica-to-replica peer link, so a +// spec acting as a peer needs it rather than a second literal that can drift +// from Options. +func (c *Cluster) RegistrationToken() string { + return c.opts.RegistrationToken +} + // WorkerName is the node name worker i registered under. func (c *Cluster) WorkerName(i int) string { return c.workers[i].Name diff --git a/tests/e2e/distributed/cluster_baseline_test.go b/tests/e2e/distributed/cluster_baseline_test.go index 6cbaaa0ea915..63995b419156 100644 --- a/tests/e2e/distributed/cluster_baseline_test.go +++ b/tests/e2e/distributed/cluster_baseline_test.go @@ -122,6 +122,14 @@ func mockBackendBinary() string { // existing caller keeps the plain two-argument form and the default shape. func startCluster(frontends, workers int, customise ...func(*cluster.Options)) *cluster.Cluster { GinkgoHelper() + c, _ := startClusterOnFreshDB(frontends, workers, customise...) + return c +} + +// startClusterOnFreshDB is startCluster plus the DSN of the database it was +// given, for a spec that has to read a table no endpoint exposes. +func startClusterOnFreshDB(frontends, workers int, customise ...func(*cluster.Options)) (*cluster.Cluster, string) { + GinkgoHelper() // Resolved before SetupInfra so a missing binary skips without having paid // for a database that the skip would then leave to DeferCleanup. @@ -163,7 +171,7 @@ func startCluster(frontends, workers int, customise ...func(*cluster.Options)) * } c.Stop() }) - return c + return c, infra.PGURL } // rosterProbe polls one frontend's node roster. diff --git a/tests/e2e/distributed/cluster_peerlink_test.go b/tests/e2e/distributed/cluster_peerlink_test.go new file mode 100644 index 000000000000..8055ba297a67 --- /dev/null +++ b/tests/e2e/distributed/cluster_peerlink_test.go @@ -0,0 +1,263 @@ +package distributed_test + +import ( + "context" + "fmt" + "net" + "strings" + "time" + + clustersvc "github.com/mudler/LocalAI/core/services/cluster" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/driver/postgres" + "gorm.io/gorm" + gormlogger "gorm.io/gorm/logger" +) + +const ( + // instanceRosterTimeout bounds the wait for a replica's row to appear. + // Registration is synchronous in startup, so this only has to cover the gap + // between /readyz answering and this spec's first query. + instanceRosterTimeout = "30s" + instanceRosterPoll = "500ms" + + // deadReplicaTimeout bounds the wait for a survivor to reap a replica that + // was killed: the liveness window plus a sweep interval plus slack. It is + // deliberately derived from the constants rather than a round number, so + // tightening the window shortens the spec instead of leaving it passing for + // the wrong reason. + deadReplicaTimeout = clustersvc.InstanceLiveness + 4*clustersvc.InstanceHeartbeat + + // peerDialTimeout bounds one peer dial. Every replica here is a local + // process, so a dial that needs longer has failed, not slowed. + peerDialTimeout = 20 * time.Second +) + +// openClusterDB connects to the database the cluster was given, so a spec can +// read the tables the peer link keeps. Nothing serves them over HTTP: they are +// replica-to-replica state, not an admin surface, and inventing an endpoint to +// observe them would be a bigger change than the thing under test. +func openClusterDB(dsn string) *gorm.DB { + GinkgoHelper() + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: gormlogger.Discard}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { closeDB(db) }) + return db +} + +// hostPortOf strips the scheme off a frontend URL, giving the form the +// instances table stores. +func hostPortOf(url string) string { + return strings.TrimPrefix(strings.TrimPrefix(url, "http://"), "https://") +} + +// instanceRoster reads the live replica rows, keeping the last error so a +// failing Eventually can name it. +type instanceRoster struct { + registry *clustersvc.Registry + ctx context.Context + + lastErr error + lastSaw []clustersvc.Instance +} + +func newInstanceRoster(db *gorm.DB) *instanceRoster { + return &instanceRoster{registry: clustersvc.NewRegistry(db), ctx: context.Background()} +} + +// addresses returns the advertised address of every live replica, or nil on a +// query error so Eventually keeps trying. +func (r *instanceRoster) addresses() []string { + live, err := r.registry.Live(r.ctx, clustersvc.InstanceLiveness) + if err != nil { + r.lastErr = err + return nil + } + r.lastErr = nil + r.lastSaw = live + addrs := []string{} + for _, instance := range live { + addrs = append(addrs, instance.AdvertisedAddr) + } + return addrs +} + +// idAt returns the id of the live replica advertising addr, or "" if no such +// row is present yet. +func (r *instanceRoster) idAt(addr string) string { + for _, instance := range r.lastSaw { + if instance.AdvertisedAddr == addr { + return instance.ID + } + } + return "" +} + +func (r *instanceRoster) describe() string { + if r.lastErr != nil { + return fmt.Sprintf("the last read of the instances table failed: %v", r.lastErr) + } + return fmt.Sprintf("the instances table held %d live replica(s): %+v", len(r.lastSaw), r.lastSaw) +} + +// awaitReplicas waits for every frontend of c to publish its address and +// returns the roster, positioned on that reading. +func awaitReplicas(roster *instanceRoster, addrs ...string) { + GinkgoHelper() + Eventually(roster.addresses, instanceRosterTimeout, instanceRosterPoll). + Should(ConsistOf(addrs), roster.describe) +} + +var _ = Describe("Cluster peer link", Label("Distributed"), Label("Cluster"), func() { + It("publishes an address for every replica that peers can actually dial", func() { + // A wrong implementation registers nothing (the whole of phase 1 had no + // call site until this spec), registers one row for two replicas, or + // records an address nothing can connect to: the bind address of a + // replica behind a service, or the loopback address the route to a + // co-located database would suggest. + c, dsn := startClusterOnFreshDB(2, 0) + + roster := newInstanceRoster(openClusterDB(dsn)) + awaitReplicas(roster, hostPortOf(c.FrontendURL(0)), hostPortOf(c.FrontendURL(1))) + + // "Routable" is not a property of the string. Connect to each address, + // which is the only check that would have caught a replica publishing + // the port it was configured with rather than the one it serves on. + for _, instance := range roster.lastSaw { + conn, err := net.DialTimeout("tcp", instance.AdvertisedAddr, peerDialTimeout) + Expect(err).ToNot(HaveOccurred(), + "replica %s advertises %q, which nothing can connect to", instance.ID, instance.AdvertisedAddr) + Expect(conn.Close()).To(Succeed()) + } + }) + + It("carries a peer stream between two replicas, and refuses one without the cluster token", func() { + // A wrong implementation fails here on WebSocket framing, which is the + // likeliest defect in the peer link: the adapter has to turn + // message-oriented WebSocket frames into the undelimited byte stream + // yamux drives. It also fails if the route was never registered on the + // real server, or if the global session middleware answers it: a peer + // carries no session and no user, only the cluster token. + // + // The stream is opened with the production dialler, resolving the peer + // through the production registry, over a real socket to a real + // process. This spec plays the sibling replica, because phase 1 has + // nothing that makes a frontend dial one on its own. + c, dsn := startClusterOnFreshDB(2, 0) + + roster := newInstanceRoster(openClusterDB(dsn)) + awaitReplicas(roster, hostPortOf(c.FrontendURL(0)), hostPortOf(c.FrontendURL(1))) + + peerID := roster.idAt(hostPortOf(c.FrontendURL(1))) + Expect(peerID).ToNot(BeEmpty()) + + ctx, cancel := context.WithTimeout(context.Background(), peerDialTimeout) + defer cancel() + + pool := clustersvc.NewPeerPool("e2e-peer", c.RegistrationToken(), roster.registry) + DeferCleanup(pool.Close) + + // OpenStream is only acknowledged once the far side accepts, so this + // returning at all proves the frontend is accepting streams on the + // session it took, in addition to proving the handshake. + stream, err := pool.Open(ctx, peerID) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = stream.Close() }) + + // Phase 1 installs no relay, so the accepted stream is refused at once. + // What matters here is that the refusal arrives: a replica that held + // the session without accepting on it would leave this read parked + // until the deadline, which is the failure mode a live cluster would + // experience as every relayed request hanging. + Expect(stream.SetReadDeadline(time.Now().Add(peerDialTimeout))).To(Succeed()) + _, err = stream.Read(make([]byte, 1)) + Expect(err).To(HaveOccurred(), + "the peer accepted the stream and then neither answered nor closed it") + Expect(err).ToNot(MatchError(context.DeadlineExceeded)) + + // The same dial with the wrong credentials must be refused, otherwise + // the success above says nothing about authentication. + impostor := clustersvc.NewPeerPool("e2e-peer", "not-the-cluster-token", roster.registry) + DeferCleanup(impostor.Close) + _, err = impostor.Open(ctx, peerID) + Expect(err).To(MatchError(clustersvc.ErrPeerUnreachable)) + Expect(err).ToNot(MatchError(clustersvc.ErrInstanceNotFound), + "a peer refusing credentials is a live peer; reading it as absence is how a replica evicts healthy workers") + }) + + It("reports a killed replica as unreachable, reaps what it owned, and evicts no worker", func() { + // This is the absence rule, pinned before phase 2 can depend on it. A + // wrong implementation lets a peer that will not answer surface as node + // absence, and a caller entitled to act on absence then reclaims what + // the peer was running: a network hiccup between two healthy replicas + // evicts healthy workers. + // + // It also pins the reaper: the connection rows a dead replica owned are + // swept by the same sweeper that decides the replica is dead, so the + // two can never disagree about who is alive. + c, dsn := startClusterOnFreshDB(2, 1) + + client, err := c.AdminSession(0) + Expect(err).ToNot(HaveOccurred()) + + // The worker registers with frontend 0, so frontend 1 is the replica + // that can die without taking the worker's registrar with it. + registrar, err := c.WorkerRegistrar(0) + Expect(err).ToNot(HaveOccurred()) + Expect(registrar).To(Equal(0), "this spec kills frontend 1 and needs the worker to have registered elsewhere") + + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), probe.describe) + workerID := probe.idOf(c.WorkerName(0)) + Expect(workerID).ToNot(BeEmpty()) + + roster := newInstanceRoster(openClusterDB(dsn)) + awaitReplicas(roster, hostPortOf(c.FrontendURL(0)), hostPortOf(c.FrontendURL(1))) + survivorID := roster.idAt(hostPortOf(c.FrontendURL(0))) + doomedID := roster.idAt(hostPortOf(c.FrontendURL(1))) + Expect(survivorID).ToNot(BeEmpty()) + Expect(doomedID).ToNot(BeEmpty()) + + // Give frontend 1 the worker's tunnel. Phase 2 makes the worker do this + // by dialling; here the claim is written directly, because the point + // under test is what happens to the claim when its owner dies. + ctx := context.Background() + epoch, err := roster.registry.Claim(ctx, workerID, doomedID) + Expect(err).ToNot(HaveOccurred()) + Expect(epoch).ToNot(BeZero()) + + Expect(c.KillFrontend(1)).To(Succeed()) + Eventually(func() bool { return c.FrontendAlive(1) }, "20s", "500ms").Should(BeFalse()) + + // The row is still there for the whole liveness window, so this is the + // case that matters: the peer is KNOWN and will not answer. + dialCtx, cancel := context.WithTimeout(ctx, peerDialTimeout) + defer cancel() + pool := clustersvc.NewPeerPool("e2e-peer", c.RegistrationToken(), roster.registry) + DeferCleanup(pool.Close) + _, err = pool.Open(dialCtx, doomedID) + Expect(err).To(MatchError(clustersvc.ErrPeerUnreachable)) + Expect(err).ToNot(MatchError(clustersvc.ErrInstanceNotFound), + "a dead replica whose row is still present is unreachable, not absent") + + // The survivor sweeps the dead replica and, in the same pass, the claim + // it left behind. + Eventually(roster.addresses, deadReplicaTimeout, instanceRosterPoll). + Should(ConsistOf(hostPortOf(c.FrontendURL(0))), roster.describe) + ownerErr := func() error { + _, _, err := roster.registry.Owner(ctx, workerID) + return err + } + Eventually(ownerErr, deadReplicaTimeout, instanceRosterPoll). + Should(MatchError(clustersvc.ErrNoConnection), + "the claim held by a replica that no longer exists was never reaped") + + // And the worker itself is untouched throughout. Nothing about a peer + // dying may reach the node roster. + Consistently(probe.healthyNames, "6s", "1s"). + Should(ContainElement(c.WorkerName(0)), probe.describe) + }) +}) From bb4719e67dbef2f3f3b91cffcb487a65f5cb8326 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 03:08:33 +0000 Subject: [PATCH 038/153] fix(cluster): hold the guarantees phase 1's comments were claiming Review found the recurring class: assertions that a wrong implementation also satisfies. The "refuse promptly, never park the peer" guarantee was stated in three places and tested in none. Removing the Close from the no-relay branch left the whole cluster suite green, because the specs asserted only that some error arrived and yamux reports a read deadline as ErrTimeout: a parked stream satisfied that as well as a refused one. Both specs now require an ENDING, EOF or a reset, inside a deadline short enough that parking is unmistakable, and both go red when the Close is removed. Deregistration existed only in a comment. Membership.Stop ended the loop and left the row behind, so every clean rolling restart had peers dialling a corpse for the full liveness window; the shutdown comment described the opposite. Registry.Deregister deletes the row and the connections that replica owned, in one transaction, for the reason the sweeper does both, and an e2e spec pins departure inside a budget shorter than the liveness window so it cannot pass on the sweeper doing the work. Before: the spec times out with both replicas still live. After: 3.6s. The configured advertised address bypassed every check discovery makes, so the one value most likely to be copied between hosts, 127.0.0.1, was taken verbatim and would make every peer dial itself. Both paths now share one rejection rule: unparseable is refused, "this host" is warned about once and honoured, because a single-host deployment uses it correctly. Two comments claimed more than the code does. The sweeper said a stalled replica recovers via re-register; only its instance row does, while the connections another replica reaped stay gone and the sockets stay held here - phase 2 must re-claim, on re-register, every connection a replica still holds locally. And Owner became OwnerRow, documenting that the owner it names may be dead for up to InstanceLiveness plus a heartbeat and that any caller acting on it must join instances itself, so the deferred constraint lives at the call site rather than in a report; the plain name is left free for the joining version. Minors: warn once when the peer link mounts with no registration token, so an operator sees the cause rather than 401s; Stop no longer blocks forever when Start was never called; corrected the NewRegistry migration doc and an e2e comment that described a 6s window as "throughout". Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 17 +++- core/http/app.go | 9 ++ core/services/cluster/instance.go | 66 +++++++++++++-- core/services/cluster/instance_test.go | 46 ++++++++++ core/services/cluster/membership.go | 84 ++++++++++++++++++- core/services/cluster/membership_test.go | 69 ++++++++++++++- core/services/cluster/ownership.go | 19 ++++- core/services/cluster/ownership_test.go | 12 +-- core/services/cluster/sessions_test.go | 16 +++- .../e2e/distributed/cluster_peerlink_test.go | 72 +++++++++++++--- 10 files changed, 369 insertions(+), 41 deletions(-) diff --git a/core/application/distributed.go b/core/application/distributed.go index 616d8a2ebc81..0021907de612 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -538,8 +538,21 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // is only a peer-reachable answer when the database is on another host; // DiscoverAdvertisedAddr refuses rather than guessing when it is not. func advertisedPeerAddr(cfg *config.ApplicationConfig) (string, error) { - if cfg.Distributed.AdvertiseAddr != "" { - return cfg.Distributed.AdvertiseAddr, nil + if configured := cfg.Distributed.AdvertiseAddr; configured != "" { + // A configured address skips discovery, so it also skips every check + // discovery makes. Unusable is refused; merely questionable (a + // loopback address, correct on one host and wrong on three) is said + // once and honoured, because refusing it would refuse single-host + // deployments that use it correctly. + reason, err := cluster.CheckAdvertisedAddr(configured) + if err != nil { + return "", err + } + if reason != "" { + xlog.Warn("Configured peer address is not one another host can dial", + "address", configured, "reason", reason, "knob", "LOCALAI_DISTRIBUTED_ADVERTISE_ADDR") + } + return configured, nil } if cfg.APIAddress == "" { return "", fmt.Errorf("no API address to derive a peer port from") diff --git a/core/http/app.go b/core/http/app.go index c5fc5d54ad54..1efc7da41a3f 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -28,6 +28,7 @@ import ( "github.com/mudler/LocalAI/core/application" "github.com/mudler/LocalAI/core/schema" + clustersvc "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/LocalAI/core/services/distributed" "github.com/mudler/LocalAI/core/services/finetune" "github.com/mudler/LocalAI/core/services/galleryop" @@ -555,6 +556,14 @@ func API(application *application.Application) (*echo.Echo, error) { // registration token, so publishing it unconditionally would put a // multiplexer on every single-binary install. if d := application.Distributed(); d != nil && d.PeerSessions != nil { + if distCfg.RegistrationToken == "" { + // The handler fails closed on an empty token, which is right and + // invisible: without this line an operator sees only 401s on a + // route they never configured, and nothing connecting them to the + // token they did not set. + xlog.Warn("Replica peer link will refuse every dial: no registration token is configured", + "route", clustersvc.PeerPath, "knob", "LOCALAI_REGISTRATION_TOKEN") + } routes.RegisterClusterRoutes(e, distCfg.RegistrationToken, d.PeerSessions.Accept) } diff --git a/core/services/cluster/instance.go b/core/services/cluster/instance.go index 027b0db3b0d9..d7e7afa71740 100644 --- a/core/services/cluster/instance.go +++ b/core/services/cluster/instance.go @@ -38,9 +38,10 @@ type Registry struct { db *gorm.DB } -// NewRegistry returns a Registry over db. Migration is the caller's job; the -// nodes registry owns the AutoMigrate for every table in this deployment so -// that a single advisory lock covers them all. +// NewRegistry returns a Registry over db. Migration is the caller's job: this +// package's tables and sequence are created by Migrate, which the nodes +// registry calls under the one advisory lock that covers every table in the +// deployment. func NewRegistry(db *gorm.DB) *Registry { return &Registry{db: db} } @@ -152,20 +153,67 @@ func DiscoverAdvertisedAddr(dsn string, port int) (string, error) { // information about the address we just read. defer func() { _ = conn.Close() }() local, ok := conn.LocalAddr().(*net.UDPAddr) - if !ok || local.IP == nil || local.IP.IsUnspecified() { + if !ok || local.IP == nil { return "", fmt.Errorf("no local address on the route to database host %q; set the advertised address explicitly", host) } - if local.IP.IsLoopback() { - return "", fmt.Errorf("the route to database host %q is loopback (%s), so the database is local to this replica and its peer-reachable address cannot be discovered; set the advertised address explicitly", host, local.IP) + if reason := unroutableReason(local.IP, local.Zone); reason != "" { + return "", fmt.Errorf("the route to database host %q is %s; set the advertised address explicitly", host, reason) } + return net.JoinHostPort(local.IP.String(), strconv.Itoa(port)), nil +} + +// unroutableReason says why ip cannot serve as an address other hosts dial, or +// "" when it can. It is the one place that decides, so the discovered address +// and the configured one are held to the same rule; they differ only in what +// they do with the answer. +func unroutableReason(ip net.IP, zone string) string { + switch { + case ip == nil || ip.IsUnspecified(): + return fmt.Sprintf("unspecified (%s), which is a bind address rather than one anything can connect to", ip) + case ip.IsLoopback(): + return fmt.Sprintf("loopback (%s), which means \"this host\" to whoever dials it, so every peer would reach itself", ip) // A zone is only ever attached to a scoped (link-local) address, so this is // the same rejection stated twice; the Zone check keeps the guarantee if a // platform ever hands back a scoped address of another class, because // IP.String() would silently drop the %iface and yield an undialable host. - if local.IP.IsLinkLocalUnicast() || local.Zone != "" { - return "", fmt.Errorf("the route to database host %q is link-local (%s), which peers on other hosts cannot dial; set the advertised address explicitly", host, local.IP) + case ip.IsLinkLocalUnicast() || zone != "": + return fmt.Sprintf("link-local (%s), which peers on other hosts cannot dial", ip) } - return net.JoinHostPort(local.IP.String(), strconv.Itoa(port)), nil + return "" +} + +// CheckAdvertisedAddr validates an address an operator configured, returning a +// reason it is questionable, or an error if it is unusable. +// +// A configured address bypasses every check DiscoverAdvertisedAddr performs, +// and the value most likely to be copied is the one that works on a single +// host: "127.0.0.1:8080" on three hosts makes every peer dial itself, which +// presents as a relay loop rather than as a configuration error. +// +// The split between error and reason is deliberate. An address that cannot be +// parsed into host and port is an error, because nothing can dial it at all. An +// address that merely means "this host" is a reason to warn and no more: a +// single-host deployment, including this repository's own e2e cluster, uses one +// correctly, and refusing it would be refusing a supported topology. +func CheckAdvertisedAddr(addr string) (reason string, err error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return "", fmt.Errorf("advertised address %q is not host:port: %w", addr, err) + } + if host == "" { + return "", fmt.Errorf("advertised address %q names no host, so peers have nothing to dial", addr) + } + portNumber, err := strconv.Atoi(port) + if err != nil || portNumber < 1 || portNumber > 65535 { + return "", fmt.Errorf("advertised address %q has no usable port (want 1-65535)", addr) + } + // A name is resolved by whoever dials it, and may resolve differently + // there, so its presence is all this side can check. + ip := net.ParseIP(host) + if ip == nil { + return "", nil + } + return unroutableReason(ip, ""), nil } // dsnHostPort extracts the host and port from either DSN form gorm's postgres diff --git a/core/services/cluster/instance_test.go b/core/services/cluster/instance_test.go index d8075a7a3cad..d27043e03c46 100644 --- a/core/services/cluster/instance_test.go +++ b/core/services/cluster/instance_test.go @@ -113,3 +113,49 @@ var _ = Describe("Advertised address discovery", func() { Expect(err).To(MatchError(ContainSubstring("out of range"))) }) }) + +var _ = Describe("Checking a configured advertised address", func() { + // The configured address bypasses discovery entirely, so it bypasses every + // rejection discovery makes. These are the checks that put back the ones + // that can be made without a route to look at. + It("accepts an address on a network other hosts can reach", func() { + reason, err := cluster.CheckAdvertisedAddr("10.0.0.7:8080") + Expect(err).ToNot(HaveOccurred()) + Expect(reason).To(BeEmpty()) + }) + + It("accepts a name, because the dialler is what resolves it", func() { + reason, err := cluster.CheckAdvertisedAddr("localai-frontend.default.svc:8080") + Expect(err).ToNot(HaveOccurred()) + Expect(reason).To(BeEmpty()) + }) + + It("refuses an address with no port, which nothing could dial", func() { + _, err := cluster.CheckAdvertisedAddr("10.0.0.7") + Expect(err).To(HaveOccurred()) + }) + + It("refuses a port outside the dialable range", func() { + _, err := cluster.CheckAdvertisedAddr("10.0.0.7:0") + Expect(err).To(MatchError(ContainSubstring("port"))) + }) + + It("refuses an address that names no host", func() { + _, err := cluster.CheckAdvertisedAddr(":8080") + Expect(err).To(MatchError(ContainSubstring("no host"))) + }) + + It("reports loopback without refusing it, because one host is a supported topology", func() { + // Correct on a single host, and the value most likely to be copied + // onto three, where every peer would then dial itself. + reason, err := cluster.CheckAdvertisedAddr("127.0.0.1:8080") + Expect(err).ToNot(HaveOccurred()) + Expect(reason).To(ContainSubstring("loopback")) + }) + + It("reports a bind address, which is not an address at all", func() { + reason, err := cluster.CheckAdvertisedAddr("0.0.0.0:8080") + Expect(err).ToNot(HaveOccurred()) + Expect(reason).To(ContainSubstring("unspecified")) + }) +}) diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index 3b1182b17458..f3f71a321acd 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -25,6 +25,10 @@ const ( // owner is merely slow has to be re-homed for nothing. The cost of waiting // is bounded and symmetric: traffic for that worker is retried, not lost. InstanceLiveness = 30 * time.Second + + // deregisterTimeout bounds the deregistration Stop performs. Shutdown is + // not the place to wait on a database. + deregisterTimeout = 5 * time.Second ) // Membership publishes this replica's address and keeps the instances table @@ -45,6 +49,10 @@ type Membership struct { stop chan struct{} done chan struct{} stopOnce sync.Once + + // mu guards started, which tells Stop whether there is a loop to join. + mu sync.Mutex + started bool } // NewMembership returns the membership loop for one replica. The address is @@ -72,17 +80,49 @@ func (m *Membership) Start(ctx context.Context) error { return err } xlog.Info("Cluster instance registered", "id", m.id, "addr", m.addr) + m.mu.Lock() + m.started = true + m.mu.Unlock() go m.loop(ctx) return nil } -// Stop ends the loop and waits for it. Safe to call more than once. +// Stop ends the loop, waits for it, and removes this replica's row. +// +// Deregistering is what makes a rolling restart quick for everyone else: a +// replica that just closes its sockets is indistinguishable from one that +// crashed, so its peers keep dialling it for the whole liveness window. It is +// best-effort by nature (a killed process never gets here), which is why the +// sweeper still exists. +// +// Safe to call more than once, and on a Membership that was never started. func (m *Membership) Stop() { if m == nil { return } - m.stopOnce.Do(func() { close(m.stop) }) - <-m.done + m.mu.Lock() + started := m.started + m.mu.Unlock() + if started { + m.stopOnce.Do(func() { close(m.stop) }) + // Only a started Membership ever closes done. Waiting on one that was + // never started, or whose Start failed, would block forever. + <-m.done + } + + // Deliberately NOT the context Start was given: that one is the + // application's, and by the time anything calls Stop it has usually been + // cancelled already, so deregistering on it would fail every time. The + // bound is here instead, because shutdown must not hang on a database that + // went away before the process using it. + ctx, cancel := context.WithTimeout(context.Background(), deregisterTimeout) + defer cancel() + if err := m.reg.Deregister(ctx, m.id); err != nil { + xlog.Warn("Deregistering this replica failed; peers will drop it when its heartbeat ages out", + "id", m.id, "within", m.liveness, "error", err) + return + } + xlog.Info("Cluster instance deregistered", "id", m.id) } func (m *Membership) loop(ctx context.Context) { @@ -114,6 +154,10 @@ func (m *Membership) tick(ctx context.Context) { // Another replica swept this row while this process was stalled long // enough to look dead. Re-register rather than heartbeat: a heartbeat // carries no address, so the row has to be rebuilt from scratch. + // + // This rebuilds the instance row ONLY. The sweep that removed it also + // removed the connections this replica owned, and re-claiming those + // needs the tunnel registry phase 2 introduces; see ReapStale. xlog.Warn("Cluster instance row was reaped, re-registering", "id", m.id) if err := m.reg.Register(ctx, m.id, m.addr, m.version); err != nil { xlog.Error("Re-registering cluster instance failed", "id", m.id, "error", err) @@ -132,6 +176,30 @@ func (m *Membership) tick(ctx context.Context) { } } +// Deregister removes one replica and the connections it owned. +// +// It deletes both, in one transaction, for the same reason ReapStale does: a +// replica that is gone owns nothing, and leaving its connection rows behind +// would point every reader at an owner that no longer exists. This is the +// announced form of what the sweeper does by inference, and the two must not +// disagree about what "gone" removes. +func (r *Registry) Deregister(ctx context.Context, id string) error { + if err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Where("owner_instance_id = ?", id).Delete(&NodeConnection{}).Error; err != nil { + return fmt.Errorf("deleting connections owned by %q: %w", id, err) + } + // No RowsAffected check: deregistering a row another replica already + // swept is the normal outcome of a slow shutdown, not an error. + if err := tx.Where("id = ?", id).Delete(&Instance{}).Error; err != nil { + return fmt.Errorf("deleting instance %q: %w", id, err) + } + return nil + }); err != nil { + return fmt.Errorf("deregistering instance %q: %w", id, err) + } + return nil +} + // ReapStale deletes the replicas that have not heartbeated within the liveness // window, and the connection rows whose owner is no longer among the survivors. // @@ -144,7 +212,15 @@ func (m *Membership) tick(ctx context.Context) { // self is never reaped. This process may fail to heartbeat for longer than the // window (a long stall, a database blip) and still be serving: deleting its own // row would then delete the connections of workers that are, at that moment, -// connected to it. The stall is recovered by the re-register in tick instead. +// connected to it. +// +// That protection is one-sided, and only the instance row recovers on its own. +// A replica that stalls long enough is reaped BY ANOTHER replica, taking its +// connection rows with it, and the re-register in tick rebuilds the instance +// row and nothing else: the sockets are still held here while the table says +// nobody holds them. Phase 2 closes this by re-claiming, on re-register, every +// connection this replica still holds locally, which needs the tunnel registry +// that owns those sockets. // // PostgreSQL only, like Live: distributed mode requires it, and the interval // arithmetic is measured on the database's clock because liveness is compared diff --git a/core/services/cluster/membership_test.go b/core/services/cluster/membership_test.go index a00a08c0b954..d490d529dd47 100644 --- a/core/services/cluster/membership_test.go +++ b/core/services/cluster/membership_test.go @@ -47,7 +47,7 @@ var _ = Describe("Reaping dead replicas", func() { Expect(connections).To(Equal(int64(1)), "a worker whose owner no longer exists is recorded as connected to nothing") - _, _, err = reg.Owner(ctx, "w1") + _, _, err = reg.OwnerRow(ctx, "w1") Expect(err).To(MatchError(cluster.ErrNoConnection)) }) @@ -61,7 +61,7 @@ var _ = Describe("Reaping dead replicas", func() { Expect(err).ToNot(HaveOccurred()) Expect(connections).To(BeZero()) - owner, stored, err := reg.Owner(ctx, "w1") + owner, stored, err := reg.OwnerRow(ctx, "w1") Expect(err).ToNot(HaveOccurred()) Expect(owner).To(Equal("other")) Expect(stored).To(Equal(epoch)) @@ -82,11 +82,74 @@ var _ = Describe("Reaping dead replicas", func() { Expect(instances).To(BeZero()) Expect(connections).To(BeZero()) - owner, _, err := reg.Owner(ctx, "w1") + owner, _, err := reg.OwnerRow(ctx, "w1") Expect(err).ToNot(HaveOccurred()) Expect(owner).To(Equal("me")) }) + It("deregisters a replica and the connections it owned, so peers drop it at once", func() { + // Without this a cleanly stopped replica is indistinguishable from a + // crashed one, and every peer keeps dialling it for the whole liveness + // window. + Expect(reg.Register(ctx, "leaving", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "staying", "10.0.0.1:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "leaving") + Expect(err).ToNot(HaveOccurred()) + _, err = reg.Claim(ctx, "w2", "staying") + Expect(err).ToNot(HaveOccurred()) + + Expect(reg.Deregister(ctx, "leaving")).To(Succeed()) + + live, err := reg.Live(ctx, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(live).To(HaveLen(1)) + Expect(live[0].ID).To(Equal("staying")) + + // The same rule the sweeper applies: a replica that is gone owns + // nothing, and a claim naming it would point every reader at an owner + // that no longer exists. + _, _, err = reg.OwnerRow(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + owner, _, err := reg.OwnerRow(ctx, "w2") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("staying"), "deregistering one replica took another replica's claim") + }) + + It("deregisters when the membership loop stops", func() { + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + Expect(membership.Start(ctx)).To(Succeed()) + + live, err := reg.Live(ctx, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(live).To(HaveLen(1)) + + membership.Stop() + + live, err = reg.Live(ctx, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(live).To(BeEmpty(), "a replica that shut down cleanly left its row behind for peers to dial") + }) + + It("tolerates a repeated deregistration, because a sweeper may have got there first", func() { + Expect(reg.Register(ctx, "gone", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Deregister(ctx, "gone")).To(Succeed()) + Expect(reg.Deregister(ctx, "gone")).To(Succeed()) + }) + + It("stops safely when it was never started", func() { + // Nothing calls this today. It exists because the loop channel is only + // ever closed by a started loop, so joining an unstarted one blocks + // forever, and phase 2 adds callers to this shutdown path. + membership := cluster.NewMembership(reg, "never-started", "10.0.0.1:8080", "v1") + done := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(done) + membership.Stop() + }() + Eventually(done, "10s").Should(BeClosed()) + }) + It("keeps this replica's row alive and reaps the dead while it runs", func() { Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1")).To(Succeed()) age("dead", time.Hour) diff --git a/core/services/cluster/ownership.go b/core/services/cluster/ownership.go index 3b6b6cb622e4..131b3d0b976d 100644 --- a/core/services/cluster/ownership.go +++ b/core/services/cluster/ownership.go @@ -161,9 +161,22 @@ func (r *Registry) Claim(ctx context.Context, nodeID, ownerID string) (int64, er return epoch, nil } -// Owner returns the replica that holds nodeID's tunnel and the epoch of that -// claim, or ErrNoConnection when the node has no recorded connection. -func (r *Registry) Owner(ctx context.Context, nodeID string) (string, int64, error) { +// OwnerRow returns the row recording which replica holds nodeID's tunnel, and +// the epoch of that claim, or ErrNoConnection when the node has no recorded +// connection. +// +// It answers "what does the table say", NOT "who holds this tunnel". The owner +// it names may be dead: a replica that dies stops heartbeating, and its rows +// survive until another replica's sweep removes them, which is up to +// InstanceLiveness plus one InstanceHeartbeat later. Any caller that ACTS on +// the answer must join instances itself and treat a non-live owner as +// ErrNoConnection; relaying to the row without that check is relaying into a +// process that is gone. +// +// The name says row on purpose, so that the joining version can take the plain +// name when phase 2 introduces the first caller that needs it. Nothing in +// phase 1 reads this outside tests, which is why the join is not here yet. +func (r *Registry) OwnerRow(ctx context.Context, nodeID string) (string, int64, error) { var conn NodeConnection err := r.db.WithContext(ctx).Where("node_id = ?", nodeID).First(&conn).Error if errors.Is(err, gorm.ErrRecordNotFound) { diff --git a/core/services/cluster/ownership_test.go b/core/services/cluster/ownership_test.go index 5744e07cd170..6d6fdf58bc7e 100644 --- a/core/services/cluster/ownership_test.go +++ b/core/services/cluster/ownership_test.go @@ -85,14 +85,14 @@ var _ = Describe("Connection ownership", func() { e2, err := reg.Claim(ctx, "w1", "inst-b") Expect(err).ToNot(HaveOccurred()) - owner, epoch, err := reg.Owner(ctx, "w1") + owner, epoch, err := reg.OwnerRow(ctx, "w1") Expect(err).ToNot(HaveOccurred()) Expect(owner).To(Equal("inst-b")) Expect(epoch).To(Equal(e2), "the stored epoch must be the one the winning claim was handed") }) It("distinguishes an unknown connection", func() { - _, _, err := reg.Owner(ctx, "ghost") + _, _, err := reg.OwnerRow(ctx, "ghost") Expect(err).To(MatchError(cluster.ErrNoConnection)) }) @@ -105,7 +105,7 @@ var _ = Describe("Connection ownership", func() { // inst-a tries to clean up after losing the claim. Expect(reg.Release(ctx, "w1", "inst-a", e1)).ToNot(Succeed()) - owner, _, err := reg.Owner(ctx, "w1") + owner, _, err := reg.OwnerRow(ctx, "w1") Expect(err).ToNot(HaveOccurred()) Expect(owner).To(Equal("inst-b"), "a stale owner must not be able to delete a live claim") }) @@ -120,7 +120,7 @@ var _ = Describe("Connection ownership", func() { Expect(reg.Release(ctx, "w1", "inst-a", e1)).ToNot(Succeed()) - owner, _, err := reg.Owner(ctx, "w1") + owner, _, err := reg.OwnerRow(ctx, "w1") Expect(err).ToNot(HaveOccurred()) Expect(owner).To(Equal("inst-a")) }) @@ -144,7 +144,7 @@ var _ = Describe("Connection ownership", func() { // live claim disappearing rather than on the epoch arithmetic. Expect(reg.Release(ctx, "w1", "inst-a", eA1)).ToNot(Succeed()) - owner, epoch, err := reg.Owner(ctx, "w1") + owner, epoch, err := reg.OwnerRow(ctx, "w1") Expect(err).ToNot(HaveOccurred(), "the delayed cleanup deleted the live claim") Expect(owner).To(Equal("inst-a")) Expect(epoch).To(Equal(eA2)) @@ -156,7 +156,7 @@ var _ = Describe("Connection ownership", func() { Expect(err).ToNot(HaveOccurred()) Expect(reg.Release(ctx, "w1", "inst-a", e)).To(Succeed()) - _, _, err = reg.Owner(ctx, "w1") + _, _, err = reg.OwnerRow(ctx, "w1") Expect(err).To(MatchError(cluster.ErrNoConnection)) }) diff --git a/core/services/cluster/sessions_test.go b/core/services/cluster/sessions_test.go index f3fb44ca8cae..29ca9b537d42 100644 --- a/core/services/cluster/sessions_test.go +++ b/core/services/cluster/sessions_test.go @@ -1,6 +1,7 @@ package cluster_test import ( + "io" "net" "time" @@ -29,6 +30,11 @@ func yamuxPair() (client *yamux.Session, server *yamux.Session) { return client, server } +// refusalDeadline bounds how long a refused stream may take to end. A refusal +// is one frame from a peer that already decided, so anything near this is the +// hang it exists to detect. +const refusalDeadline = 2 * time.Second + var _ = Describe("Accepted peer sessions", func() { It("accepts and refuses a stream rather than leaving the peer parked", func() { // yamux only acknowledges a stream once the far side accepts it, so a @@ -43,9 +49,15 @@ var _ = Describe("Accepted peer sessions", func() { Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = stream.Close() }) - Expect(stream.SetReadDeadline(time.Now().Add(10 * time.Second))).To(Succeed()) + // The deadline is short and is NOT the thing being asserted: yamux + // reports a deadline as ErrTimeout, and requiring an ending instead + // (EOF from the peer's Close, or a reset) is what separates "refused" + // from "parked". An earlier version asserted only that some error + // arrived, which a parked stream satisfies just as well. + Expect(stream.SetReadDeadline(time.Now().Add(refusalDeadline))).To(Succeed()) _, err = stream.Read(make([]byte, 1)) - Expect(err).To(HaveOccurred(), "a refused stream must end, not hang") + Expect(err).To(SatisfyAny(MatchError(io.EOF), MatchError(yamux.ErrStreamReset)), + "a refused stream must END within %s; %v means the peer accepted it and then left it parked", refusalDeadline, err) }) It("hands a stream to the relay when one is installed", func() { diff --git a/tests/e2e/distributed/cluster_peerlink_test.go b/tests/e2e/distributed/cluster_peerlink_test.go index 8055ba297a67..d99fcf70cfd8 100644 --- a/tests/e2e/distributed/cluster_peerlink_test.go +++ b/tests/e2e/distributed/cluster_peerlink_test.go @@ -3,12 +3,15 @@ package distributed_test import ( "context" "fmt" + "io" "net" "strings" "time" clustersvc "github.com/mudler/LocalAI/core/services/cluster" + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "gorm.io/driver/postgres" @@ -33,6 +36,17 @@ const ( // peerDialTimeout bounds one peer dial. Every replica here is a local // process, so a dial that needs longer has failed, not slowed. peerDialTimeout = 20 * time.Second + + // gracefulDepartureTimeout bounds the wait for a cleanly stopped replica to + // leave the table. It must stay well under InstanceLiveness, which the spec + // asserts: a budget that reached the window would pass on the sweeper doing + // the work and prove nothing about deregistration. + gracefulDepartureTimeout = 15 * time.Second + + // peerRefusalTimeout bounds how long a refused stream may take to end. It + // is short on purpose: the refusal is one frame from a replica that has + // already decided, so a stream still open at this point is parked. + peerRefusalTimeout = 5 * time.Second ) // openClusterDB connects to the database the cluster was given, so a spec can @@ -166,16 +180,16 @@ var _ = Describe("Cluster peer link", Label("Distributed"), Label("Cluster"), fu Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = stream.Close() }) - // Phase 1 installs no relay, so the accepted stream is refused at once. - // What matters here is that the refusal arrives: a replica that held - // the session without accepting on it would leave this read parked - // until the deadline, which is the failure mode a live cluster would - // experience as every relayed request hanging. - Expect(stream.SetReadDeadline(time.Now().Add(peerDialTimeout))).To(Succeed()) + // Phase 1 installs no relay, so the accepted stream must be refused at + // once: an ENDING (EOF from the peer's Close, or a reset), not merely + // an error. A replica that accepted the stream and then left it parked + // would fail this read too, but with yamux's ErrTimeout, and that is + // the failure a live cluster experiences as every relayed request + // hanging until its own deadline. + Expect(stream.SetReadDeadline(time.Now().Add(peerRefusalTimeout))).To(Succeed()) _, err = stream.Read(make([]byte, 1)) - Expect(err).To(HaveOccurred(), - "the peer accepted the stream and then neither answered nor closed it") - Expect(err).ToNot(MatchError(context.DeadlineExceeded)) + Expect(err).To(SatisfyAny(MatchError(io.EOF), MatchError(yamux.ErrStreamReset)), + "the peer accepted the stream and then neither answered nor ended it: %v", err) // The same dial with the wrong credentials must be refused, otherwise // the success above says nothing about authentication. @@ -187,6 +201,38 @@ var _ = Describe("Cluster peer link", Label("Distributed"), Label("Cluster"), fu "a peer refusing credentials is a live peer; reading it as absence is how a replica evicts healthy workers") }) + It("stops being dialled as soon as a replica shuts down cleanly", func() { + // The crash case below is handled by the sweeper, at the cost of a + // whole liveness window of peers dialling a corpse. A rolling update is + // not a crash: the replica knows it is leaving and says so. Without + // deregistration the two are indistinguishable, and every rolling + // restart spends that window failing peer dials for no reason. + c, dsn := startClusterOnFreshDB(2, 0) + + roster := newInstanceRoster(openClusterDB(dsn)) + awaitReplicas(roster, hostPortOf(c.FrontendURL(0)), hostPortOf(c.FrontendURL(1))) + departingID := roster.idAt(hostPortOf(c.FrontendURL(1))) + Expect(departingID).ToNot(BeEmpty()) + + Expect(c.StopFrontendGracefully(1)).To(Succeed()) + Eventually(func() bool { return c.FrontendAlive(1) }, "20s", "500ms").Should(BeFalse()) + + // The budget is deliberately shorter than the liveness window: passing + // it proves the replica announced its departure rather than aged out. + Expect(gracefulDepartureTimeout).To(BeNumerically("<", clustersvc.InstanceLiveness)) + Eventually(roster.addresses, gracefulDepartureTimeout, instanceRosterPoll). + Should(ConsistOf(hostPortOf(c.FrontendURL(0))), roster.describe) + + // And absence is the RIGHT answer here, unlike the killed case: the + // replica said it was going. A caller may act on this. + ctx, cancel := context.WithTimeout(context.Background(), peerDialTimeout) + defer cancel() + pool := clustersvc.NewPeerPool("e2e-peer", c.RegistrationToken(), roster.registry) + DeferCleanup(pool.Close) + _, err := pool.Open(ctx, departingID) + Expect(err).To(MatchError(clustersvc.ErrInstanceNotFound)) + }) + It("reports a killed replica as unreachable, reaps what it owned, and evicts no worker", func() { // This is the absence rule, pinned before phase 2 can depend on it. A // wrong implementation lets a peer that will not answer surface as node @@ -248,15 +294,17 @@ var _ = Describe("Cluster peer link", Label("Distributed"), Label("Cluster"), fu Eventually(roster.addresses, deadReplicaTimeout, instanceRosterPoll). Should(ConsistOf(hostPortOf(c.FrontendURL(0))), roster.describe) ownerErr := func() error { - _, _, err := roster.registry.Owner(ctx, workerID) + _, _, err := roster.registry.OwnerRow(ctx, workerID) return err } Eventually(ownerErr, deadReplicaTimeout, instanceRosterPoll). Should(MatchError(clustersvc.ErrNoConnection), "the claim held by a replica that no longer exists was never reaped") - // And the worker itself is untouched throughout. Nothing about a peer - // dying may reach the node roster. + // And the worker survives the sweep that removed its owner. This is a + // window after the reaping, not a watch over the whole scenario: + // Consistently starts here, so what it rules out is the sweep, or + // anything reacting to it, taking the worker with it. Consistently(probe.healthyNames, "6s", "1s"). Should(ContainElement(c.WorkerName(0)), probe.describe) }) From 3daf889f5e9e771fbe3e1004617b20e514884990 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 03:26:36 +0000 Subject: [PATCH 039/153] fix(cluster): share one lock order, and correct the phase 1 comments ReapStale deleted from instances then node_connections while Deregister took them the other way round, both inside one transaction and both running concurrently by design: a replica shuts down while a peer sweeps it. Opposite orders let each hold the row the other waits for. PostgreSQL breaks the cycle by aborting one side, so the cost today is a warning rather than lost data, but the inversion costs nothing to remove. Deregister now deletes the instance row first. That is the order ReapStale is forced into anyway, since its connection delete asks which instance rows survived, so the sweeper is the fixed side. Both functions say the order is deliberate and shared, and name the other. A spec records the statements each path issues and asserts they delete from the same two tables in the same order; racing two transactions until they really deadlock would be flaky and could pass for the wrong reason. The rest is comment and spec accuracy, deferred from the phase 1 task reviews: - co-location does not imply loopback. Compose's usual host=postgres resolves to a bridge address and discovery works there; it is a DSN that NAMES localhost that yields a loopback source address. Corrected in the DiscoverAdvertisedAddr doc and in the spec comment that repeated it. - unroutableReason labelled every scoped address "link-local", including the class the check exists for, and formatted the IP with %s, which drops the %iface, so the reported address was not the one being rejected. Split into two cases, both rendered with their zone. CheckAdvertisedAddr passed zone "" and net.ParseIP rejects fe80::1%eth0, so a scoped literal looked like a name and collected no warning at all; the zone is now split off before parsing. - Splice's "Both callers satisfy it" claimed callers that still do not exist. It now names the two stream types the wake-on-Close property was verified against and says a phase 2 caller over anything else has to check it. - restored, short, why a socket-level ECONNRESET stays reported while a yamux reset does not: the yamux endings are the teardown Splice's own Close provokes, and whether an aborted request is routine is the relay's policy. - the real-yamux spec's far.Read had no deadline, so a stall parked the suite rather than failing it. - gorilla's SetWriteDeadline is conn.go:796, not 787. - ClusterPathPrefix is no longer derived from: the peer route spells its path out, because core/services/cluster must not import core/http/auth. The comment now points at the spec that holds them together instead of claiming a derivation the move removed. - the epoch spec asserted e2 > e1, an ordering Claim's doc tells callers not to rely on. It asserts uniqueness, which is what the fence guarantees, and is named for that. A sibling spec still described the epoch as incrementing in SQL when it is drawn from a sequence. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/auth/public_routes.go | 12 +++-- core/services/cluster/instance.go | 59 ++++++++++++++++++------ core/services/cluster/instance_test.go | 18 ++++++-- core/services/cluster/membership.go | 19 ++++++-- core/services/cluster/membership_test.go | 25 ++++++++++ core/services/cluster/ownership_test.go | 34 ++++++++++++-- core/services/cluster/splice.go | 18 ++++++-- core/services/cluster/splice_test.go | 3 ++ core/services/cluster/wsconn.go | 2 +- 9 files changed, 158 insertions(+), 32 deletions(-) diff --git a/core/http/auth/public_routes.go b/core/http/auth/public_routes.go index 25bb25177ad3..98ef48672f8b 100644 --- a/core/http/auth/public_routes.go +++ b/core/http/auth/public_routes.go @@ -82,10 +82,14 @@ func isPublicRoute(method, path string) bool { // ClusterPathPrefix is the replica-to-replica namespace. Its handlers check the // cluster token in the Authorization header themselves, so the check below lets -// them through the global session middleware. The cluster endpoints build their -// route paths from this same constant, which is why it lives here beside the -// check rather than beside the handlers: the exemption and the route it exempts -// cannot then be changed independently. +// them through the global session middleware. +// +// The cluster routes do NOT derive their paths from this constant: they are +// registered from core/services/cluster's own literal, because that package +// must not import core/http/auth. Nothing in the compiler holds the two +// together, so a spec does instead, driving a peer request through this +// middleware in core/http/endpoints/cluster/peer_test.go. Moving either string +// without the other turns that spec red, which is the whole reason it exists. const ClusterPathPrefix = "/api/cluster/" // usesAlternativeAuthentication identifies requests whose credentials are diff --git a/core/services/cluster/instance.go b/core/services/cluster/instance.go index d7e7afa71740..4140ce6bc796 100644 --- a/core/services/cluster/instance.go +++ b/core/services/cluster/instance.go @@ -128,13 +128,16 @@ func (r *Registry) Get(ctx context.Context, id string) (*Instance, error) { // address to advertise. The caller supplies the port, since the frontend's // listening port has nothing to do with the database's. // -// The discovery only holds while the database is a shared, remote host. When -// PostgreSQL runs on this same host or pod (compose, single-node, any sidecar -// layout) the route to it is loopback, and advertising 127.0.0.1 would make a -// peer dialling this replica reach itself instead. So an unspecified, loopback, -// or link-local source address is rejected with an error telling the operator to -// configure the advertised address explicitly, rather than returned. There is no -// fallback string: no address is better than a wrong one. +// What defeats the discovery is a DSN that NAMES loopback, not the database +// being co-located. Co-location is fine as long as the DSN names something +// routable: compose's usual `host=postgres` resolves to a bridge address, so +// the kernel picks this container's own bridge IP as the source, which is the +// address a peer on that network dials. It is `host=localhost` (or 127.0.0.1, +// or ::1) that makes the route loopback, and advertising 127.0.0.1 would make +// a peer dialling this replica reach itself instead. So an unspecified, +// loopback, or scoped source address is rejected with an error telling the +// operator to configure the advertised address explicitly, rather than +// returned. There is no fallback string: no address is better than a wrong one. func DiscoverAdvertisedAddr(dsn string, port int) (string, error) { // A port of 0 (or out of range) would produce an address nothing can dial, // and the caller is likelier to have passed an unset field than to mean it. @@ -172,16 +175,29 @@ func unroutableReason(ip net.IP, zone string) string { return fmt.Sprintf("unspecified (%s), which is a bind address rather than one anything can connect to", ip) case ip.IsLoopback(): return fmt.Sprintf("loopback (%s), which means \"this host\" to whoever dials it, so every peer would reach itself", ip) - // A zone is only ever attached to a scoped (link-local) address, so this is - // the same rejection stated twice; the Zone check keeps the guarantee if a - // platform ever hands back a scoped address of another class, because - // IP.String() would silently drop the %iface and yield an undialable host. - case ip.IsLinkLocalUnicast() || zone != "": - return fmt.Sprintf("link-local (%s), which peers on other hosts cannot dial", ip) + case ip.IsLinkLocalUnicast(): + return fmt.Sprintf("link-local (%s), which peers on other hosts cannot dial", withZone(ip, zone)) + // A zone is normally attached only to a link-local address, which the case + // above already rejects. This one stays for the scoped address of some + // other class a platform may hand back, and says so rather than repeating + // the link-local label: the two have different cures, and an operator told + // the wrong one looks in the wrong place. + case zone != "": + return fmt.Sprintf("scoped to interface %q (%s), and the zone is dropped by the time an address is stored, leaving a host nothing can dial", zone, withZone(ip, zone)) } return "" } +// withZone renders the address the way it has to be dialled. IP.String() drops +// the %iface, so an unadorned %s in a rejection reports an address that differs +// from the one being rejected. +func withZone(ip net.IP, zone string) string { + if zone == "" { + return ip.String() + } + return ip.String() + "%" + zone +} + // CheckAdvertisedAddr validates an address an operator configured, returning a // reason it is questionable, or an error if it is unusable. // @@ -207,13 +223,28 @@ func CheckAdvertisedAddr(addr string) (reason string, err error) { if err != nil || portNumber < 1 || portNumber > 65535 { return "", fmt.Errorf("advertised address %q has no usable port (want 1-65535)", addr) } + // The zone is split off before parsing because net.ParseIP rejects + // "fe80::1%eth0" outright. Left joined, a scoped literal would look like a + // name and collect no warning at all, which is the one case where the + // address is guaranteed not to work for a peer. + host, zone := splitZone(host) // A name is resolved by whoever dials it, and may resolve differently // there, so its presence is all this side can check. ip := net.ParseIP(host) if ip == nil { return "", nil } - return unroutableReason(ip, ""), nil + return unroutableReason(ip, zone), nil +} + +// splitZone separates an IPv6 scope from the address it qualifies. A name +// never carries one, so a host with no "%" comes back unchanged. +func splitZone(host string) (string, string) { + addr, zone, found := strings.Cut(host, "%") + if !found { + return host, "" + } + return addr, zone } // dsnHostPort extracts the host and port from either DSN form gorm's postgres diff --git a/core/services/cluster/instance_test.go b/core/services/cluster/instance_test.go index d27043e03c46..aa0349496161 100644 --- a/core/services/cluster/instance_test.go +++ b/core/services/cluster/instance_test.go @@ -99,9 +99,11 @@ var _ = Describe("Advertised address discovery", func() { Expect(err).To(HaveOccurred()) }) - // A database on this same host routes over loopback on every platform, so - // this is deterministic rather than host-dependent. Returning 127.0.0.1 - // would make a peer dialling this replica reach itself. + // A DSN that NAMES loopback routes over loopback on every platform, so this + // is deterministic rather than host-dependent. Co-location is not the + // trigger: compose's `host=postgres` resolves to a bridge address and + // discovery works there. Returning 127.0.0.1 would make a peer dialling + // this replica reach itself. It("refuses a loopback route instead of advertising an address peers cannot use", func() { addr, err := cluster.DiscoverAdvertisedAddr("postgres://user@127.0.0.1:5432/testdb", 8080) Expect(addr).To(BeEmpty()) @@ -158,4 +160,14 @@ var _ = Describe("Checking a configured advertised address", func() { Expect(err).ToNot(HaveOccurred()) Expect(reason).To(ContainSubstring("unspecified")) }) + + It("reports a scoped literal, which net.ParseIP alone would wave through as a name", func() { + // The zone has to be split off before parsing, or this address is + // indistinguishable from a hostname and collects no warning at all. + reason, err := cluster.CheckAdvertisedAddr("[fe80::1%eth0]:8080") + Expect(err).ToNot(HaveOccurred()) + Expect(reason).ToNot(BeEmpty(), "a scoped address peers cannot dial was accepted in silence") + Expect(reason).To(ContainSubstring("fe80::1%eth0"), + "the reported address must carry its zone, or it is not the address being rejected") + }) }) diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index f3f71a321acd..ab24acc73e83 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -183,16 +183,24 @@ func (m *Membership) tick(ctx context.Context) { // would point every reader at an owner that no longer exists. This is the // announced form of what the sweeper does by inference, and the two must not // disagree about what "gone" removes. +// +// Instances first, then connections, which is deliberate and is the same order +// ReapStale takes. The two paths run concurrently in the ordinary case, a +// replica shutting down while a peer is sweeping it, and each locks the same +// two tables; opposite orders would let each hold the row the other is waiting +// for. PostgreSQL breaks such a cycle by aborting one side, so the cost is a +// failed shutdown rather than lost data, but an inversion that costs nothing to +// remove should not be left in. func (r *Registry) Deregister(ctx context.Context, id string) error { if err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - if err := tx.Where("owner_instance_id = ?", id).Delete(&NodeConnection{}).Error; err != nil { - return fmt.Errorf("deleting connections owned by %q: %w", id, err) - } // No RowsAffected check: deregistering a row another replica already // swept is the normal outcome of a slow shutdown, not an error. if err := tx.Where("id = ?", id).Delete(&Instance{}).Error; err != nil { return fmt.Errorf("deleting instance %q: %w", id, err) } + if err := tx.Where("owner_instance_id = ?", id).Delete(&NodeConnection{}).Error; err != nil { + return fmt.Errorf("deleting connections owned by %q: %w", id, err) + } return nil }); err != nil { return fmt.Errorf("deregistering instance %q: %w", id, err) @@ -225,6 +233,11 @@ func (r *Registry) Deregister(ctx context.Context, id string) error { // PostgreSQL only, like Live: distributed mode requires it, and the interval // arithmetic is measured on the database's clock because liveness is compared // across replicas. +// +// Instances are deleted before connections, and Deregister takes the same order +// on purpose, so the two paths cannot deadlock against each other. Here the +// order is also forced: the connection delete asks which instance rows survived, +// so it has to run second. func (r *Registry) ReapStale(ctx context.Context, self string, within time.Duration) (instances int64, connections int64, err error) { err = r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { res := tx.Where("id <> ? AND last_seen <= now() - make_interval(secs => ?)", self, within.Seconds()). diff --git a/core/services/cluster/membership_test.go b/core/services/cluster/membership_test.go index d490d529dd47..c331eafe23d2 100644 --- a/core/services/cluster/membership_test.go +++ b/core/services/cluster/membership_test.go @@ -130,6 +130,31 @@ var _ = Describe("Reaping dead replicas", func() { Expect(live).To(BeEmpty(), "a replica that shut down cleanly left its row behind for peers to dial") }) + It("takes the two tables in one order, shared with the sweeper, so the two cannot deadlock", func() { + // A replica deregistering and a peer sweeping it run concurrently by + // design, and both lock rows in instances and in node_connections. In + // opposite orders each can end up holding the row the other waits for. + // The order is asserted on the SQL because the alternative, racing two + // transactions until they actually deadlock, is exactly the flaky spec + // this one replaces. + Expect(reg.Register(ctx, "leaving", "10.0.0.2:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "leaving") + Expect(err).ToNot(HaveOccurred()) + + deregRec := newSQLRecorder() + Expect(cluster.NewRegistry(db.Session(&gorm.Session{Logger: deregRec})). + Deregister(ctx, "leaving")).To(Succeed()) + + reapRec := newSQLRecorder() + _, _, err = cluster.NewRegistry(db.Session(&gorm.Session{Logger: reapRec})). + ReapStale(ctx, "sweeper", time.Minute) + Expect(err).ToNot(HaveOccurred()) + + Expect(deregRec.deleteOrder()).To(Equal([]string{"instances", "node_connections"})) + Expect(reapRec.deleteOrder()).To(Equal(deregRec.deleteOrder()), + "the sweeper and deregistration must lock the same two tables in the same order") + }) + It("tolerates a repeated deregistration, because a sweeper may have got there first", func() { Expect(reg.Register(ctx, "gone", "10.0.0.2:8080", "v1")).To(Succeed()) Expect(reg.Deregister(ctx, "gone")).To(Succeed()) diff --git a/core/services/cluster/ownership_test.go b/core/services/cluster/ownership_test.go index 6d6fdf58bc7e..cd59f8056f6c 100644 --- a/core/services/cluster/ownership_test.go +++ b/core/services/cluster/ownership_test.go @@ -57,6 +57,29 @@ func (r *sqlRecorder) only() string { return r.statements[0] } +// deleteOrder returns the tables the recorded statements deleted from, in the +// order they were issued. It is how a spec pins a lock order: the order two +// paths take the same tables in is a property of the SQL, and asserting it on +// an outcome instead would mean racing two transactions into a real deadlock. +func (r *sqlRecorder) deleteOrder() []string { + r.mu.Lock() + defer r.mu.Unlock() + ExpectWithOffset(1, r.errs).To(BeEmpty(), "a recorded statement failed") + var tables []string + for _, stmt := range r.statements { + idx := strings.Index(strings.ToLower(stmt), "delete from ") + if idx < 0 { + continue + } + fields := strings.Fields(stmt[idx+len("delete from "):]) + if len(fields) == 0 { + continue + } + tables = append(tables, strings.Trim(fields[0], `"`)) + } + return tables +} + var _ = Describe("Connection ownership", func() { var ( db *gorm.DB @@ -71,12 +94,17 @@ var _ = Describe("Connection ownership", func() { reg = cluster.NewRegistry(db) }) - It("increments the epoch on every claim", func() { + It("hands every claim an epoch no other claim was given", func() { + // Uniqueness, not order. Claim's contract is that no two claims ever + // share an epoch; the insert path draws its sequence value before the + // row lock, so a claim that follows a Release can be handed a lower + // number than one already issued. Asserting e2 > e1 here would pin an + // ordering the fence does not need and does not promise. e1, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) e2, err := reg.Claim(ctx, "w1", "inst-b") Expect(err).ToNot(HaveOccurred()) - Expect(e2).To(BeNumerically(">", e1)) + Expect(e2).ToNot(Equal(e1)) }) It("reports the latest owner", func() { @@ -160,7 +188,7 @@ var _ = Describe("Connection ownership", func() { Expect(err).To(MatchError(cluster.ErrNoConnection)) }) - It("claims in one statement that increments in SQL and stamps on the database clock", func() { + It("claims in one statement that draws its epoch from the database sequence and stamps on the database clock", func() { rec := newSQLRecorder() recording := cluster.NewRegistry(db.Session(&gorm.Session{Logger: rec})) diff --git a/core/services/cluster/splice.go b/core/services/cluster/splice.go index 5df97e890da0..604bbfec93d4 100644 --- a/core/services/cluster/splice.go +++ b/core/services/cluster/splice.go @@ -45,10 +45,12 @@ func Splice(a, b io.ReadWriteCloser) error { // Wait for the second direction so no copy is still touching either stream // once Splice has returned. This is load-bearing: it assumes Close unblocks // a copy parked in Read or Write, and a stream where that is false hangs - // here rather than leaking a goroutine. Both callers satisfy it. net.Conn - // does, and so does go-yamux/v5, whose Close sets readErr and calls - // notifyWaiting to wake a parked Read while a parked Write returns - // ErrStreamClosed. + // here rather than leaking a goroutine. The two stream types this is built + // for satisfy it: net.Conn does, and so does go-yamux/v5, whose Close sets + // readErr and calls notifyWaiting to wake a parked Read while a parked + // Write returns ErrStreamClosed. Nothing outside this package's own specs + // calls Splice yet, so a phase 2 caller relaying over anything else has to + // check this property rather than assume it. <-errs if first != nil { @@ -80,6 +82,14 @@ func copyStream(dst io.Writer, src io.Reader) error { // the peer is gone, and yamux produces exactly that when a Write races its // session's shutdown (see isMuxSessionFailure). // +// A socket-level abort (ECONNRESET, EPIPE) is deliberately absent too, which +// makes the same underlying event, a peer aborting mid-stream, reach the caller +// as nil over a yamux tunnel and as an error over a raw socket. That asymmetry +// is intended: the yamux endings are the teardown Splice's own Close provokes, +// so this primitive is the only thing that can tell them from a fault, whereas +// whether an aborted request is routine or a failure is the relay's policy and +// only the relay knows which request was abandoned. +// // The mux checks run first, and that ordering is load-bearing: a dying yamux // session usually hands every live stream its own cause wrapped up // (session.go:330), and that cause is routinely a closed-socket error, so diff --git a/core/services/cluster/splice_test.go b/core/services/cluster/splice_test.go index 7bbe3246de05..7b423a142730 100644 --- a/core/services/cluster/splice_test.go +++ b/core/services/cluster/splice_test.go @@ -308,6 +308,9 @@ var _ = Describe("Splice", func() { Expect(err).ToNot(HaveOccurred()) var far *yamux.Stream Eventually(accepted, "10s").Should(Receive(&far)) + // Deadline so a stream that never carries the byte fails this spec + // instead of parking the suite until its own timeout. + Expect(far.SetReadDeadline(time.Now().Add(10 * time.Second))).To(Succeed()) _, err = far.Read(make([]byte, 1)) Expect(err).ToNot(HaveOccurred()) diff --git a/core/services/cluster/wsconn.go b/core/services/cluster/wsconn.go index ca932d741cc8..aa8346de6453 100644 --- a/core/services/cluster/wsconn.go +++ b/core/services/cluster/wsconn.go @@ -119,7 +119,7 @@ func (c *wsConn) SetDeadline(t time.Time) error { func (c *wsConn) SetReadDeadline(t time.Time) error { return c.ws.SetReadDeadline(t) } // SetWriteDeadline takes writeMu because gorilla stores the write deadline in a -// plain struct field (conn.go:787) and applies it when it next flushes, so +// plain struct field (conn.go:796) and applies it when it next flushes, so // setting it while a write is in flight is a data race, not merely a late bound. func (c *wsConn) SetWriteDeadline(t time.Time) error { c.writeMu.Lock() From a2751471459ad3c042ed6964bdd9a048fe6dbd10 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 06:46:02 +0000 Subject: [PATCH 040/153] feat(cluster): resolve tunnel ownership against a live owner OwnerRow is a bare row read of node_connections. A connection row outlives the replica that wrote it: a replica that dies stops heartbeating, but its rows survive until a peer's sweep removes them, which is up to InstanceLiveness plus one InstanceHeartbeat later. For that whole window the table names a process that is gone. The next component phase 2 builds is the relaying dialer, and a dialer reading OwnerRow would relay into a corpse for roughly 35 seconds after every replica death, then report the worker as unreachable when it is in fact absent, which is the distinction the phase 1 end-to-end specs pinned. Owner is the resolving read: one statement joining instances, returning ErrNoConnection when the row is missing OR its owner is not live. Both cases are one answer on purpose, since both mean no replica here holds this tunnel; they differ only in which sweep has run. It is one statement, not a row read followed by an instance lookup, because between two statements the owner can die and the caller would act on an owner the second read would have rejected. OwnerRow stays, unjoined, for readers that need the row itself, and a spec holds the two apart: with an aged-out owner, OwnerRow still names it and Owner refuses, so neither can quietly become the other. The liveness predicate is now one string, instanceIsLive, shared by Live and by Owner's join. Two spellings of one fact drift, and this drift would show as a relay to a replica one query calls dead and another calls alive. It is table-qualified so it is unambiguous inside the join, and the cutoff stays on the database clock, so replica clock skew cannot widen or narrow the window. Both mutations were run. Dropping the liveness predicate from the join fails 3 specs, the aged-owner one among them. Replacing the database clock with a Go-side time.Now() fails 1: the aged-owner specs still pass, because the two clocks agree on one host, and only the recorded-SQL spec sees the literal timestamp. That is why that spec exists. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/instance.go | 20 ++++- core/services/cluster/ownership.go | 55 ++++++++++-- core/services/cluster/ownership_test.go | 114 ++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 10 deletions(-) diff --git a/core/services/cluster/instance.go b/core/services/cluster/instance.go index 4140ce6bc796..c991d0da8d17 100644 --- a/core/services/cluster/instance.go +++ b/core/services/cluster/instance.go @@ -91,13 +91,27 @@ func (r *Registry) Heartbeat(ctx context.Context, id string) error { return nil } +// instanceIsLive is the one predicate that decides whether a replica is still +// alive, and it takes the window in seconds as its single bind parameter. Every +// reader of that fact uses this string: Live to list the survivors, Owner to +// refuse an owner that is not among them. Two spellings of one fact drift, and +// the drift would show up as a relay to a replica one query calls dead and +// another calls alive. +// +// The column is table-qualified because Owner reads it across a join, where an +// unqualified last_seen would be ambiguous. Postgres folds the unquoted name to +// the same table gorm quotes, so the qualification costs Live nothing. +// +// The cutoff is computed by the database for the same reason Register stamps +// there: liveness is compared across replicas, so a reader's own clock must not +// decide whether another replica is alive. +const instanceIsLive = `instances.last_seen > now() - make_interval(secs => ?)` + // Live returns the instances whose LastSeen is newer than now-within. func (r *Registry) Live(ctx context.Context, within time.Duration) ([]Instance, error) { var out []Instance - // The cutoff is computed by the database for the same reason Register stamps - // there: a reader's clock must not decide whether another replica is alive. if err := r.db.WithContext(ctx). - Where("last_seen > now() - make_interval(secs => ?)", within.Seconds()). + Where(instanceIsLive, within.Seconds()). Order("id"). Find(&out).Error; err != nil { return nil, fmt.Errorf("listing live instances: %w", err) diff --git a/core/services/cluster/ownership.go b/core/services/cluster/ownership.go index 131b3d0b976d..48d9f94c58c9 100644 --- a/core/services/cluster/ownership.go +++ b/core/services/cluster/ownership.go @@ -168,14 +168,12 @@ func (r *Registry) Claim(ctx context.Context, nodeID, ownerID string) (int64, er // It answers "what does the table say", NOT "who holds this tunnel". The owner // it names may be dead: a replica that dies stops heartbeating, and its rows // survive until another replica's sweep removes them, which is up to -// InstanceLiveness plus one InstanceHeartbeat later. Any caller that ACTS on -// the answer must join instances itself and treat a non-live owner as -// ErrNoConnection; relaying to the row without that check is relaying into a -// process that is gone. +// InstanceLiveness plus one InstanceHeartbeat later. // -// The name says row on purpose, so that the joining version can take the plain -// name when phase 2 introduces the first caller that needs it. Nothing in -// phase 1 reads this outside tests, which is why the join is not here yet. +// Callers that ACT on the answer want Owner, which joins instances and treats a +// non-live owner as ErrNoConnection. This one is for callers that need to see +// the row itself, such as a sweeper deciding what to clean up, or a spec +// proving the two reads differ. func (r *Registry) OwnerRow(ctx context.Context, nodeID string) (string, int64, error) { var conn NodeConnection err := r.db.WithContext(ctx).Where("node_id = ?", nodeID).First(&conn).Error @@ -188,6 +186,49 @@ func (r *Registry) OwnerRow(ctx context.Context, nodeID string) (string, int64, return conn.OwnerInstanceID, conn.Epoch, nil } +// Owner returns the replica that holds nodeID's tunnel AND is still live, with +// the epoch of that claim, or ErrNoConnection when there is no such replica. +// +// This is the read anything that ACTS on the answer must use. A connection row +// outlives its owner: a replica that dies stops heartbeating but its rows stay +// until a peer's sweep removes them, which is up to InstanceLiveness plus one +// InstanceHeartbeat later. For that whole window OwnerRow names a process that +// is gone, and a relay built on it would dial a corpse and report the worker as +// unreachable rather than as absent. +// +// A missing row and a dead owner are one answer on purpose. Both mean "no +// replica here holds this worker's tunnel", which is what a caller decides on; +// they differ only in which sweep has already run, and that is the sweeper's +// business rather than the caller's. +// +// One statement, joined, not a row read followed by an instance lookup: between +// two statements the owner can die, and the caller would act on an owner the +// second read would have rejected. The join makes the two facts one snapshot. +// +// The window is InstanceLiveness rather than a parameter, which is the window +// the membership loop sweeps with. A caller free to pick its own could keep +// relaying to a replica the sweeper has already declared dead, or give up on +// one the sweeper is still keeping. +func (r *Registry) Owner(ctx context.Context, nodeID string) (string, int64, error) { + var conn NodeConnection + err := r.db.WithContext(ctx). + Model(&NodeConnection{}). + // Only the connection's own columns are selected: the join exists to + // filter, and SELECT * across it would hand gorm the instances columns + // to scan into a NodeConnection. + Select("node_connections.*"). + Joins("JOIN instances ON instances.id = node_connections.owner_instance_id AND "+instanceIsLive, InstanceLiveness.Seconds()). + Where("node_connections.node_id = ?", nodeID). + Take(&conn).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return "", 0, fmt.Errorf("looking up live owner of node %q: %w", nodeID, ErrNoConnection) + } + if err != nil { + return "", 0, fmt.Errorf("looking up live owner of node %q: %w", nodeID, err) + } + return conn.OwnerInstanceID, conn.Epoch, nil +} + // Release drops the claim identified by ownerID and epoch. Both are in the // WHERE so a replica that has only just noticed its dead socket cannot delete // the claim a later reconnect established elsewhere: the row it is trying to diff --git a/core/services/cluster/ownership_test.go b/core/services/cluster/ownership_test.go index cd59f8056f6c..b7abbbd6a896 100644 --- a/core/services/cluster/ownership_test.go +++ b/core/services/cluster/ownership_test.go @@ -188,6 +188,120 @@ var _ = Describe("Connection ownership", func() { Expect(err).To(MatchError(cluster.ErrNoConnection)) }) + // Owner is the resolving read: it answers "who holds this tunnel and can be + // relayed to", where OwnerRow answers "what does the table say". The gap + // between the two is a whole liveness window wide, because a replica that + // dies leaves its connection rows behind until a peer's sweep removes them. + Describe("resolving the owner that can actually be relayed to", func() { + // Aged far enough past InstanceLiveness that the exact window boundary + // is not what these specs are measuring. + agedOut := 10 * time.Minute + + // age rewrites an instance's heartbeat into the past. Sleeping for a + // liveness window is forbidden in a spec, and would be measuring the + // clock rather than the query. + age := func(id string, by time.Duration) { + ExpectWithOffset(1, db.Model(&cluster.Instance{}).Where("id = ?", id). + Update("last_seen", time.Now().Add(-by)).Error).To(Succeed()) + } + + It("names an owner whose replica is live", func() { + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + claimed, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + + owner, epoch, err := reg.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("inst-a")) + Expect(epoch).To(Equal(claimed), "the resolved epoch must be the fence token the claim was handed") + }) + + It("refuses to name an owner that has no instance row at all", func() { + // What a completed sweep leaves for the moment between deleting the + // instance row and deleting the connections it orphaned, and what a + // re-registering replica's own connection rows look like meanwhile. + _, err := reg.Claim(ctx, "w1", "inst-gone") + Expect(err).ToNot(HaveOccurred()) + + _, _, err = reg.Owner(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + }) + + It("refuses to name an owner whose heartbeat has aged past the liveness window", func() { + // The window this task exists to close: the replica is dead, no peer + // has swept it yet, and the row still names it. + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + age("inst-a", agedOut) + + _, _, err = reg.Owner(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + }) + + It("names an owner again once its heartbeat comes back", func() { + // Liveness is a window, not a latch: a replica that stalls and + // recovers still owns the sockets it never dropped, so resolution + // has to follow last_seen rather than remember a verdict. + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + claimed, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + age("inst-a", agedOut) + Expect(reg.Heartbeat(ctx, "inst-a")).To(Succeed()) + + owner, epoch, err := reg.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("inst-a")) + Expect(epoch).To(Equal(claimed)) + }) + + It("still reports the dead owner through OwnerRow, which is why the two reads are separate", func() { + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + claimed, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + age("inst-a", agedOut) + + owner, epoch, err := reg.OwnerRow(ctx, "w1") + Expect(err).ToNot(HaveOccurred(), "OwnerRow reads the row and nothing else; hiding the dead owner here would leave the sweeper with no way to see what it has to clean up") + Expect(owner).To(Equal("inst-a")) + Expect(epoch).To(Equal(claimed)) + + _, _, err = reg.Owner(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection), "Owner and OwnerRow must not agree here, or one of them is redundant") + }) + + It("reports a node with no connection at all the same way", func() { + _, _, err := reg.Owner(ctx, "ghost") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + }) + + It("resolves in one joined statement measured on the database clock", func() { + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + + rec := newSQLRecorder() + recording := cluster.NewRegistry(db.Session(&gorm.Session{Logger: rec})) + _, _, err = recording.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + + sql := strings.ToLower(rec.only()) + // only() rules out the read-then-look-up shape: two statements + // leave a window in which the owner dies between them, which is the + // race the join closes. + Expect(sql).To(ContainSubstring("join")) + Expect(sql).To(ContainSubstring("instances")) + // Liveness is compared across replicas, so the cutoff has to be + // computed on the one clock they all share. A Go-side time.Now() + // would appear as a bound parameter and a plain comparison instead, + // and replica clock skew would then widen or narrow the window. + Expect(sql).To(ContainSubstring("now()")) + Expect(sql).To(ContainSubstring("make_interval")) + Expect(sql).ToNot(MatchRegexp(`last_seen\s*>\s*'`), + "the liveness cutoff must not be a literal timestamp from this process's clock") + }) + }) + It("claims in one statement that draws its epoch from the database sequence and stamps on the database clock", func() { rec := newSQLRecorder() recording := cluster.NewRegistry(db.Session(&gorm.Session{Logger: rec})) From df7cebf642f984f3f512125d46c3d25ca7bce680 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 07:09:18 +0000 Subject: [PATCH 041/153] fix(cluster): make the ownership comments say only what holds Review round 1 on the joined Owner read. The behaviour was accepted; three comments claimed more than the code delivered, one spec pinned less than its doc promised, and one pre-existing spec ranked epochs. instanceIsLive said every reader of instance liveness uses it, which was false: ReapStale spelled the complement by hand. The complement is now written as NOT (instanceIsLive), so "stale" is exactly "not live", including how each side treats a NULL last_seen, and the sentence is true. Inverting that predicate fails 3 reaper specs, so the routing is held. The Select("node_connections.*") in Owner was justified by a SELECT * hazard that cannot occur: with a join present and nothing selected, gorm expands the model's own columns table-qualified (callbacks.BuildQuerySQL), and the suite is green with the Select removed. It stays, because the projection should be a property of this query, and the comment now says that instead. Owner gained the dialect guard Claim has. now() and make_interval are PostgreSQL, so on the SQLite single-binary path it failed with "no such function: now", which reads as a missing migration; that regression already shipped once in phase 1. The refusal is deliberately not ErrNoConnection: a deployment with no cluster has no answer about ownership, and reporting absence would let a caller conclude the worker is not connected. A spec in the non-PostgreSQL block holds all three properties. The new specs aged rows by ten minutes, which any window between zero and ten minutes satisfies, so nothing tied Owner's window to the one the sweeper uses. They now age to just past InstanceLiveness, and a sibling ages to half of it and must still resolve. Widening the window tenfold fails 2 specs, narrowing it tenfold fails 1; before this both were silent. The concurrent-claim spec asserted the stored epoch was the highest handed out, and justified it with claims drawing their epoch after the row lock, which contradicts Claim's own doc: the insert path draws nextval while the tuple is built. It now asserts the stored epoch is one of the epochs handed out, and ranks nothing. OwnerRow's doc justified the function with a sweeper that does not call it. ReapStale deletes orphans with a set difference; the callers are this package's specs and one e2e assertion. It says that, and states plainly that a caller needing to know who owns a node in order to dial it wants Owner. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/instance.go | 8 ++-- core/services/cluster/membership.go | 6 ++- core/services/cluster/ownership.go | 31 ++++++++++---- core/services/cluster/ownership_test.go | 56 ++++++++++++++++++------- 4 files changed, 74 insertions(+), 27 deletions(-) diff --git a/core/services/cluster/instance.go b/core/services/cluster/instance.go index c991d0da8d17..606337a381ff 100644 --- a/core/services/cluster/instance.go +++ b/core/services/cluster/instance.go @@ -93,10 +93,10 @@ func (r *Registry) Heartbeat(ctx context.Context, id string) error { // instanceIsLive is the one predicate that decides whether a replica is still // alive, and it takes the window in seconds as its single bind parameter. Every -// reader of that fact uses this string: Live to list the survivors, Owner to -// refuse an owner that is not among them. Two spellings of one fact drift, and -// the drift would show up as a relay to a replica one query calls dead and -// another calls alive. +// reader of that fact is written in terms of it: Live lists the rows it selects, +// Owner refuses an owner it rejects, and ReapStale deletes its negation. Two +// spellings of one fact drift, and the drift would show up as a relay to a +// replica one query calls dead and another calls alive. // // The column is table-qualified because Owner reads it across a join, where an // unqualified last_seen would be ambiguous. Postgres folds the unquoted name to diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index ab24acc73e83..825dcf6e1cdb 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -240,7 +240,11 @@ func (r *Registry) Deregister(ctx context.Context, id string) error { // so it has to run second. func (r *Registry) ReapStale(ctx context.Context, self string, within time.Duration) (instances int64, connections int64, err error) { err = r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - res := tx.Where("id <> ? AND last_seen <= now() - make_interval(secs => ?)", self, within.Seconds()). + // Negated rather than spelled as its own comparison: "stale" has to be + // exactly "not live", including how each treats a row whose last_seen + // is NULL, and a hand-written complement is a second definition that + // only looks like the first. + res := tx.Where("id <> ? AND NOT ("+instanceIsLive+")", self, within.Seconds()). Delete(&Instance{}) if res.Error != nil { return fmt.Errorf("deleting stale instances: %w", res.Error) diff --git a/core/services/cluster/ownership.go b/core/services/cluster/ownership.go index 48d9f94c58c9..8a80adede74b 100644 --- a/core/services/cluster/ownership.go +++ b/core/services/cluster/ownership.go @@ -170,10 +170,16 @@ func (r *Registry) Claim(ctx context.Context, nodeID, ownerID string) (int64, er // survive until another replica's sweep removes them, which is up to // InstanceLiveness plus one InstanceHeartbeat later. // -// Callers that ACT on the answer want Owner, which joins instances and treats a -// non-live owner as ErrNoConnection. This one is for callers that need to see -// the row itself, such as a sweeper deciding what to clean up, or a spec -// proving the two reads differ. +// Anything that needs to know WHO owns a node in order to act on it, a dialer +// above all, wants Owner: it joins instances and treats a non-live owner as +// ErrNoConnection. Dialing what this function returns is dialing a process that +// may be gone. +// +// What is left for this one is observing the table as such, independently of +// liveness. Its callers today are this package's specs, including the one that +// holds the two reads apart, and the e2e cluster spec that watches ownership +// move between replicas. No production caller reads it, and the sweeper is not +// one: ReapStale deletes orphans with a set difference in SQL. func (r *Registry) OwnerRow(ctx context.Context, nodeID string) (string, int64, error) { var conn NodeConnection err := r.db.WithContext(ctx).Where("node_id = ?", nodeID).First(&conn).Error @@ -210,12 +216,23 @@ func (r *Registry) OwnerRow(ctx context.Context, nodeID string) (string, int64, // relaying to a replica the sweeper has already declared dead, or give up on // one the sweeper is still keeping. func (r *Registry) Owner(ctx context.Context, nodeID string) (string, int64, error) { + // Refused rather than attempted, for the reason Claim refuses: now() and + // make_interval are PostgreSQL, so on the single-binary SQLite path this + // would fail with "no such function: now", which reads as a missing + // migration. It is deliberately not ErrNoConnection. A deployment with no + // cluster has no answer to give about who owns a tunnel, and reporting + // absence would let a caller conclude the worker is not connected. + if !isPostgres(r.db) { + return "", 0, fmt.Errorf("looking up live owner of node %q: connection ownership requires PostgreSQL, this deployment runs on %q", nodeID, r.db.Dialector.Name()) + } var conn NodeConnection err := r.db.WithContext(ctx). Model(&NodeConnection{}). - // Only the connection's own columns are selected: the join exists to - // filter, and SELECT * across it would hand gorm the instances columns - // to scan into a NodeConnection. + // Not load-bearing: gorm already expands this model's own columns, + // table-qualified, when a join is present and nothing was selected + // (callbacks.BuildQuerySQL). Written out so the projection is a + // property of this query rather than of that behaviour, since the join + // is here to filter and the row scanned back must stay this table's. Select("node_connections.*"). Joins("JOIN instances ON instances.id = node_connections.owner_instance_id AND "+instanceIsLive, InstanceLiveness.Seconds()). Where("node_connections.node_id = ?", nodeID). diff --git a/core/services/cluster/ownership_test.go b/core/services/cluster/ownership_test.go index b7abbbd6a896..cd422199c638 100644 --- a/core/services/cluster/ownership_test.go +++ b/core/services/cluster/ownership_test.go @@ -193,9 +193,16 @@ var _ = Describe("Connection ownership", func() { // between the two is a whole liveness window wide, because a replica that // dies leaves its connection rows behind until a peer's sweep removes them. Describe("resolving the owner that can actually be relayed to", func() { - // Aged far enough past InstanceLiveness that the exact window boundary - // is not what these specs are measuring. - agedOut := 10 * time.Minute + // Just past the window the membership loop sweeps with, not an arbitrary + // large age: a row aged ten minutes is rejected by any window between + // zero and ten minutes, so it would pin "filtered by SOME window" while + // letting Owner and the sweeper drift apart. The two seconds keep the + // spec off the exact boundary without loosening what it holds. + agedOut := cluster.InstanceLiveness + 2*time.Second + // Old enough that a narrowed window would reject it, still inside the + // one Owner must use. It is the other half of the same pin: agedOut + // fails a widened window, this fails a narrowed one. + agedButLive := cluster.InstanceLiveness / 2 // age rewrites an instance's heartbeat into the past. Sleeping for a // liveness window is forbidden in a spec, and would be measuring the @@ -216,6 +223,18 @@ var _ = Describe("Connection ownership", func() { Expect(epoch).To(Equal(claimed), "the resolved epoch must be the fence token the claim was handed") }) + It("still names an owner whose heartbeat is old but inside the window", func() { + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + claimed, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + age("inst-a", agedButLive) + + owner, epoch, err := reg.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred(), "a replica within the sweeper's window is alive, and its workers are still reachable through it") + Expect(owner).To(Equal("inst-a")) + Expect(epoch).To(Equal(claimed)) + }) + It("refuses to name an owner that has no instance row at all", func() { // What a completed sweep leaves for the moment between deleting the // instance row and deleting the connections it orphaned, and what a @@ -350,21 +369,16 @@ var _ = Describe("Connection ownership", func() { } Expect(seen).To(HaveLen(claimants)) - // Exactly one row, and its epoch is the highest handed out. That last - // part holds here because no Release intervenes: every claim after the - // first blocks on the row lock and draws its epoch after taking it, in - // commit order. It is not a general guarantee about epochs, which are - // unique but unordered. + // Exactly one row, holding one of the epochs that was handed out: a + // winner, not a value nobody was given. Which of the eight wins is not + // asserted, and neither is any ordering among them. Epochs are unique + // and unordered, and a spec that ranked them here would teach the + // opposite of what Claim documents, whatever the sequence happens to do + // on this path. var rows []cluster.NodeConnection Expect(db.Where("node_id = ?", "w-race").Find(&rows).Error).To(Succeed()) Expect(rows).To(HaveLen(1)) - var max int64 - for e := range seen { - if e > max { - max = e - } - } - Expect(rows[0].Epoch).To(Equal(max)) + Expect(seen).To(HaveKey(rows[0].Epoch), "the stored epoch was never handed to any claimant") }) }) @@ -387,6 +401,18 @@ var _ = Describe("Connection ownership on a non-PostgreSQL dialect", func() { Expect(cluster.Migrate(ctx, db)).To(Succeed()) }) + It("refuses to resolve an owner, rather than failing as a missing function", func() { + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + + _, _, err := cluster.NewRegistry(db).Owner(ctx, "w1") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("requires PostgreSQL")) + Expect(err.Error()).ToNot(ContainSubstring("no such function"), + "a dialect that cannot answer must say so, not surface as a missing migration") + Expect(err).ToNot(MatchError(cluster.ErrNoConnection), + "a deployment with no cluster has no answer about ownership; reporting absence would let a caller conclude the worker is not connected") + }) + It("refuses to claim, rather than pretending to fence", func() { Expect(cluster.Migrate(ctx, db)).To(Succeed()) From 032725c81bb2e3138ef2156e1d2143f60766df22 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 07:39:49 +0000 Subject: [PATCH 042/153] feat(cluster): hold worker tunnels, and re-claim them after a sweep Phase 1 left the connection fence with a table and no sockets behind it. This adds the registry that holds them: Attach claims the node and then stores the session, Open hands out a stream over the tunnel this replica holds, Detach releases the claim it was handed, and Held names what this process is carrying. The claim is written before the session is stored. A claimant that installs itself and only then finds it cannot claim has, for that window, published a tunnel no row records, so Held names it while a peer asking Owner is told the worker is connected nowhere. ErrNotOwner is produced at one place, the map miss. It is a routing fact: some other replica may hold that worker perfectly well. A broken socket under a held entry is returned as itself, because answering "not held here" would send a dialer looking elsewhere for a worker this replica is holding. Epochs are compared for equality and never ordered. Claim guarantees an epoch is unique and never reissued; it does not guarantee the later claim draws the larger number, because the sequence value on the insert path is drawn before the row lock. The membership loop now re-claims on re-register, which closes the hole phase 1 named in ReapStale. A replica that stalls long enough is swept by a peer, losing its instance row and, in the same transaction, every connection it owned; Register rebuilds the instance row and nothing else, so without this it serves workers that every other replica reports as connected nowhere. Re-claiming draws a fresh epoch, so an attachment carries two: the token Attach handed back, which is what Detach matches and which never moves, and the epoch of the row currently held, which is what Release is given. Collapsing them would leave the re-claimed row outliving the socket with no caller able to remove it. A tunnel whose session is already closed is skipped rather than claimed back, because claiming is an upsert and would take the row from whoever holds the worker now. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/membership.go | 68 ++++- core/services/cluster/tunnel.go | 278 ++++++++++++++++++ core/services/cluster/tunnel_test.go | 414 +++++++++++++++++++++++++++ 3 files changed, 749 insertions(+), 11 deletions(-) create mode 100644 core/services/cluster/tunnel.go create mode 100644 core/services/cluster/tunnel_test.go diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index 825dcf6e1cdb..ff511cdd9ffb 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -50,9 +50,11 @@ type Membership struct { done chan struct{} stopOnce sync.Once - // mu guards started, which tells Stop whether there is a loop to join. + // mu guards started, which tells Stop whether there is a loop to join, and + // tunnels, which SetTunnels may write while the loop is already reading it. mu sync.Mutex started bool + tunnels *TunnelRegistry } // NewMembership returns the membership loop for one replica. The address is @@ -75,6 +77,20 @@ func NewMembership(reg *Registry, id, addr, version string) *Membership { // registration is synchronous and its failure is returned: a replica whose // address never reaches the table is invisible to its peers, and starting // anyway would hide that behind a background log line. +// SetTunnels gives the loop the registry holding this replica's worker tunnels, +// so it can re-claim them after its rows have been swept. A Membership without +// one still heartbeats and sweeps; it simply has nothing to re-claim, which is +// the single-binary case and the case of a replica that accepts no tunnels. +// +// It is a setter rather than a constructor argument because the tunnel registry +// is what the tunnel endpoint is built on, and that is wired after membership +// is already running. +func (m *Membership) SetTunnels(t *TunnelRegistry) { + m.mu.Lock() + defer m.mu.Unlock() + m.tunnels = t +} + func (m *Membership) Start(ctx context.Context) error { if err := m.reg.Register(ctx, m.id, m.addr, m.version); err != nil { return err @@ -155,13 +171,20 @@ func (m *Membership) tick(ctx context.Context) { // enough to look dead. Re-register rather than heartbeat: a heartbeat // carries no address, so the row has to be rebuilt from scratch. // - // This rebuilds the instance row ONLY. The sweep that removed it also - // removed the connections this replica owned, and re-claiming those - // needs the tunnel registry phase 2 introduces; see ReapStale. + // Register rebuilds the instance row ONLY. The sweep that removed it + // removed the connections this replica owned in the same transaction, + // so the tunnels still held here have to be claimed again or this + // replica serves workers that, as far as every other replica can see, + // are connected nowhere. xlog.Warn("Cluster instance row was reaped, re-registering", "id", m.id) if err := m.reg.Register(ctx, m.id, m.addr, m.version); err != nil { xlog.Error("Re-registering cluster instance failed", "id", m.id, "error", err) + // Nothing to re-claim onto: a claim naming an instance row that + // does not exist is deleted by the next sweep that runs, this + // replica's own included, and the sweep is what has just happened. + return } + m.reclaimTunnels(ctx) } else if err != nil { xlog.Warn("Cluster instance heartbeat failed", "id", m.id, "error", err) } @@ -176,6 +199,30 @@ func (m *Membership) tick(ctx context.Context) { } } +// reclaimTunnels re-writes a claim for every worker tunnel this replica still +// holds, after the sweep that deleted them. It is separate from tick only so +// the lock around the registry reference is not held across the database work. +func (m *Membership) reclaimTunnels(ctx context.Context) { + m.mu.Lock() + tunnels := m.tunnels + m.mu.Unlock() + if tunnels == nil { + return + } + + reclaimed, err := tunnels.Reclaim(ctx) + if err != nil { + // Logged rather than returned, and the loop keeps running: the next + // heartbeat fails the same way if the row is still missing, so the + // re-claim is retried. A worker whose claim never lands is reachable + // only through the replica it is connected to, which is this one. + xlog.Error("Re-claiming worker tunnels after this replica was reaped failed", "id", m.id, "error", err) + } + if reclaimed > 0 { + xlog.Info("Re-claimed worker tunnels after this replica was reaped", "id", m.id, "tunnels", reclaimed) + } +} + // Deregister removes one replica and the connections it owned. // // It deletes both, in one transaction, for the same reason ReapStale does: a @@ -222,13 +269,12 @@ func (r *Registry) Deregister(ctx context.Context, id string) error { // row would then delete the connections of workers that are, at that moment, // connected to it. // -// That protection is one-sided, and only the instance row recovers on its own. -// A replica that stalls long enough is reaped BY ANOTHER replica, taking its -// connection rows with it, and the re-register in tick rebuilds the instance -// row and nothing else: the sockets are still held here while the table says -// nobody holds them. Phase 2 closes this by re-claiming, on re-register, every -// connection this replica still holds locally, which needs the tunnel registry -// that owns those sockets. +// That protection is one-sided. A replica that stalls long enough is reaped BY +// ANOTHER replica, taking its connection rows with it, and Register rebuilds +// the instance row and nothing else. What restores the rest is the re-claim in +// tick, which writes a fresh claim for every tunnel the tunnel registry still +// holds; until it runs, this replica holds sockets the table records nobody +// holding. // // PostgreSQL only, like Live: distributed mode requires it, and the interval // arithmetic is measured on the database's clock because liveness is compared diff --git a/core/services/cluster/tunnel.go b/core/services/cluster/tunnel.go new file mode 100644 index 000000000000..7ca928a5e7b9 --- /dev/null +++ b/core/services/cluster/tunnel.go @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: MIT + +package cluster + +import ( + "context" + "errors" + "fmt" + "net" + "sort" + "sync" + "time" + + "github.com/libp2p/go-yamux/v5" + "github.com/mudler/xlog" +) + +// ErrNotOwner reports that this replica does not hold the tunnel for a node. +// +// It is a ROUTING fact and nothing else: some other replica may hold that +// worker perfectly well, and a caller that sees it relays through the owner the +// database names. It must therefore never be produced by anything that merely +// failed. A database error, a broken socket, a session that shut down under a +// held entry: each of those is reported as itself, because reporting them as +// "not held here" tells a dialer to look elsewhere for a worker that is right +// here, and the design forbids absence standing in for unreachable. This is the +// same rule Claim and Owner follow when they refuse a dialect rather than +// answering ErrNoConnection. +var ErrNotOwner = errors.New("cluster: this replica does not hold the tunnel for that node") + +// tunnelReleaseTimeout bounds the release Detach performs. Detach is called +// from the goroutine that has just watched a worker's session die, and that +// goroutine must not be parked on a database that went away with it. +const tunnelReleaseTimeout = 5 * time.Second + +// TunnelRegistry holds the worker tunnels this replica has accepted, and keeps +// the node_connections table agreeing with what it holds. +// +// It is the local half of the connection fence: the table says which replica +// owns a worker, and this says which socket that ownership actually resolves +// to. The two are written in one order, always, by Attach. +type TunnelRegistry struct { + reg *Registry + selfID string + + mu sync.Mutex + tunnels map[string]*heldTunnel +} + +// heldTunnel is one accepted worker tunnel. +// +// The two epochs are the same number until this replica is swept and re-claims, +// and they are separate fields because they answer different questions. +// +// token is what Attach handed back, and it is the only value Detach matches. +// It identifies one local attachment for that attachment's whole life, which is +// what lets a superseded holder's Detach be recognised as stale: epochs are +// never reissued, so a token from an earlier attachment cannot collide with a +// later one's. +// +// claim is the epoch of the row this replica currently holds for the node, and +// it is what Release must be given, because that is the row the fence matches +// on. A re-claim draws a fresh epoch and moves this one; leaving Release to use +// the token instead would match nothing, and the row would outlive the socket +// with no caller able to tell. +// +// Neither is ever ordered against the other, or against anything else. Claim +// guarantees uniqueness, not monotonicity. +type heldTunnel struct { + sess *yamux.Session + token int64 + claim int64 +} + +// NewTunnelRegistry returns a registry that claims tunnels as selfID. The ID +// must be the same one this replica registers in the instances table, since +// that is what Owner joins a claim against to decide the owner is alive. +func NewTunnelRegistry(reg *Registry, selfID string) *TunnelRegistry { + return &TunnelRegistry{ + reg: reg, + selfID: selfID, + tunnels: map[string]*heldTunnel{}, + } +} + +// Attach records this replica as the owner of nodeID's tunnel and stores the +// session, returning the epoch the caller must later hand to Detach. +// +// The claim is written BEFORE the session is stored, and the order is the +// point. A claimant that installs itself first and only then finds it cannot +// claim has, for that window, published a tunnel no row records: Held names it, +// and a peer asking Owner is told the worker is connected nowhere. Claiming +// first means a failed claim leaves this replica exactly as it was. +// +// A worker that re-dials onto this same replica supersedes its own earlier +// attachment, and the superseded session is closed here. Nothing else can close +// it: whoever accepted it is parked in AcceptStream on a session that is not +// broken, only replaced, and would wait there until the far side noticed. The +// caller keeps ownership of the session it passed in; only a session this +// registry evicted is closed by this registry. +func (t *TunnelRegistry) Attach(ctx context.Context, nodeID string, sess *yamux.Session) (int64, error) { + if sess == nil { + // Claiming would publish a tunnel that cannot carry anything, and the + // fence would then have to be unwound by a Detach nobody will call. + return 0, fmt.Errorf("attaching tunnel for node %q: no session", nodeID) + } + + epoch, err := t.reg.Claim(ctx, nodeID, t.selfID) + if err != nil { + return 0, err + } + + t.mu.Lock() + previous := t.tunnels[nodeID] + t.tunnels[nodeID] = &heldTunnel{sess: sess, token: epoch, claim: epoch} + t.mu.Unlock() + + if previous != nil && previous.sess != sess { + xlog.Debug("worker re-dialled this replica, dropping its previous tunnel", "node", nodeID) + _ = previous.sess.Close() + } + return epoch, nil +} + +// Detach drops the attachment epoch identifies and releases its claim. An epoch +// that is not the one Attach handed the current holder is a no-op, which is how +// a superseded holder noticing its dead socket is stopped from evicting the +// attachment that replaced it. +// +// Matched by EQUALITY, never by order. An epoch is unique and never reissued, +// but a claim inserted after a Release can draw a lower number than one already +// issued, so a stale token may compare either way against the live one. +// +// Releasing a claim this replica no longer holds is ordinary rather than +// exceptional: it is what a worker having re-homed to another replica looks +// like from here, so it is logged and not returned. Detach has no error to +// return to, being the last thing a dying tunnel's goroutine does. +func (t *TunnelRegistry) Detach(nodeID string, epoch int64) { + t.mu.Lock() + held, ok := t.tunnels[nodeID] + if !ok || held.token != epoch { + t.mu.Unlock() + return + } + delete(t.tunnels, nodeID) + claim := held.claim + t.mu.Unlock() + + // Not the caller's context, and not the one Attach was given: both belong + // to the request or the process that set the tunnel up, and by the time a + // tunnel is being torn down either may already be cancelled, which would + // leave the row behind on every ordinary disconnect. + ctx, cancel := context.WithTimeout(context.Background(), tunnelReleaseTimeout) + defer cancel() + // The claim, not the token: the row carries whatever epoch was last claimed + // for this attachment, and Release matches the row exactly. + if err := t.reg.Release(ctx, nodeID, t.selfID, claim); err != nil { + if errors.Is(err, ErrNoConnection) { + xlog.Debug("worker tunnel claim was already superseded", "node", nodeID, "epoch", claim) + return + } + xlog.Warn("Releasing a worker tunnel claim failed; peers will drop it when this replica's heartbeat ages out", + "node", nodeID, "epoch", claim, "error", err) + } +} + +// Open returns a stream to the worker over the tunnel this replica holds. +// +// ErrNotOwner means only that no tunnel for nodeID is held here. Every other +// failure is returned as itself, wrapped: a session that died under a held +// entry is a transport condition, and answering ErrNotOwner for it would send a +// dialer looking elsewhere for a worker this replica is holding. +// +// A failed open does not evict the entry. Whether a tunnel is held here is +// decided by Attach and Detach, and letting one bad open unhold it would race +// the goroutine that owns the session and is about to detach it properly. +func (t *TunnelRegistry) Open(ctx context.Context, nodeID string) (net.Conn, error) { + t.mu.Lock() + held, ok := t.tunnels[nodeID] + t.mu.Unlock() + if !ok { + return nil, fmt.Errorf("opening a stream to node %q: %w", nodeID, ErrNotOwner) + } + + stream, err := held.sess.OpenStream(ctx) + if err != nil { + return nil, fmt.Errorf("opening a stream to node %q over the tunnel held here: %w", nodeID, err) + } + return stream, nil +} + +// Held returns the nodes whose tunnels this replica holds, sorted. +// +// It answers what this process holds, which is not the same question as who the +// table says owns a node; Owner answers that one. The membership loop uses this +// to know what to re-claim after its rows have been swept. +func (t *TunnelRegistry) Held() []string { + t.mu.Lock() + defer t.mu.Unlock() + out := make([]string, 0, len(t.tunnels)) + for nodeID := range t.tunnels { + out = append(out, nodeID) + } + sort.Strings(out) + return out +} + +// Reclaim writes a fresh claim for every tunnel still held here, and returns +// how many it wrote. +// +// It exists for one case: this replica stalled long enough for a peer to sweep +// it, which deleted its instance row AND every connection row it owned, and it +// has just re-registered. Re-registration rebuilds the instance row only, so +// without this the sockets are still held here while the table records nobody +// holding them, and every other replica answers "not connected" for workers +// that are connected. +// +// A closed session is skipped rather than claimed. Claiming is an upsert, so it +// takes the row from whoever holds it now, and a worker whose socket here is +// closed has already reconnected somewhere: claiming it back would point every +// dialer at a replica that cannot carry a byte to it. The check narrows that +// window rather than closing it, since a socket can be dead without this side +// having noticed; yamux's keepalive bounds how long that lasts, and the +// worker's next reconnect supersedes the claim in any case. +// +// The entry of a skipped tunnel is left alone. Whoever attached it owns its +// lifetime and will detach it; Reclaim is not an eviction path, and evicting +// here would race that goroutine. +// +// A single node's failure does not abort the rest: the tunnels are independent, +// and a claim that failed is retried on the next sweep this replica survives. +func (t *TunnelRegistry) Reclaim(ctx context.Context) (int, error) { + t.mu.Lock() + held := make(map[string]*heldTunnel, len(t.tunnels)) + for nodeID, tunnel := range t.tunnels { + held[nodeID] = tunnel + } + t.mu.Unlock() + + var reclaimed int + var errs []error + for nodeID, tunnel := range held { + if tunnel.sess.IsClosed() { + xlog.Debug("skipping re-claim of a worker tunnel whose session is closed", "node", nodeID) + continue + } + epoch, err := t.reg.Claim(ctx, nodeID, t.selfID) + if err != nil { + errs = append(errs, err) + continue + } + t.mu.Lock() + // Re-read under the lock: the tunnel may have been detached, or + // superseded by a reconnect, while the claim was in flight. Writing the + // new claim onto whatever is there now would give a different + // attachment an epoch it never drew, and its Release would then match + // nothing. + // + // The epoch just drawn is then held by no attachment. If it also + // happened to be the last write to the row, the row outlives the + // attachment that no longer matches it: it still names this replica, + // truthfully, but no Detach will remove it. It is cleared by the + // worker's next claim anywhere, since Claim upserts, and otherwise by + // the sweep that eventually removes this replica. Serialising Attach + // and Reclaim per node would close it; the window is one database + // round trip during the tick that follows a sweep of this replica, and + // the machinery costs more than it removes. + if current, ok := t.tunnels[nodeID]; ok && current == tunnel { + current.claim = epoch + reclaimed++ + } + t.mu.Unlock() + } + if len(errs) > 0 { + return reclaimed, fmt.Errorf("re-claiming worker tunnels: %w", errors.Join(errs...)) + } + return reclaimed, nil +} diff --git a/core/services/cluster/tunnel_test.go b/core/services/cluster/tunnel_test.go new file mode 100644 index 000000000000..33f4c2d858c4 --- /dev/null +++ b/core/services/cluster/tunnel_test.go @@ -0,0 +1,414 @@ +package cluster_test + +import ( + "context" + "fmt" + "path/filepath" + "sync" + "time" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" + + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// workerTunnel returns the two halves of a worker's tunnel: the frontend holds +// the server half, because the worker is the side that dials. yamuxPair already +// builds exactly that pairing for peer links; this names the halves the way the +// worker path uses them so a spec cannot silently attach the wrong end. +func workerTunnel() (frontend *yamux.Session, worker *yamux.Session) { + GinkgoHelper() + worker, frontend = yamuxPair() + return frontend, worker +} + +// echoOnce accepts one stream on the worker's half and echoes what it reads. +// It is how a spec proves Open produced a stream that carries bytes, rather +// than a handle that merely exists. +func echoOnce(worker *yamux.Session) { + go func() { + defer GinkgoRecover() + stream, err := worker.AcceptStream() + if err != nil { + return + } + defer func() { _ = stream.Close() }() + buf := make([]byte, 4) + if _, err := stream.Read(buf); err != nil { + return + } + _, _ = stream.Write(buf) + }() +} + +// drain accepts and discards every stream on the worker's half, so a spec that +// opens streams without reading them does not park on the accept backlog. +func drain(worker *yamux.Session) { + go func() { + defer GinkgoRecover() + for { + stream, err := worker.AcceptStream() + if err != nil { + return + } + _ = stream.Close() + } + }() +} + +var _ = Describe("The worker tunnel registry", func() { + var ( + db *gorm.DB + reg *cluster.Registry + tun *cluster.TunnelRegistry + ctx context.Context + ) + + BeforeEach(func() { + db = testutil.SetupTestDB() + ctx = context.Background() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg = cluster.NewRegistry(db) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + tun = cluster.NewTunnelRegistry(reg, "me") + }) + + It("claims the node in the database before it stores the session", func() { + // A claimant that installs itself and only then tries to claim has, + // for that window, made the registry disagree with the table: Held + // names a tunnel no row records, and a peer asking Owner is told the + // worker is connected nowhere. The failure is injected through the + // production refusal path, a dialect with no epoch sequence, so the + // spec exercises the real error return rather than a fake. + sqliteDB, err := gorm.Open(sqlite.Open(filepath.Join(GinkgoT().TempDir(), "cluster.db")), &gorm.Config{}) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.Migrate(ctx, sqliteDB)).To(Succeed()) + unclaimable := cluster.NewTunnelRegistry(cluster.NewRegistry(sqliteDB), "me") + + frontend, _ := workerTunnel() + _, err = unclaimable.Attach(ctx, "w1", frontend) + Expect(err).To(HaveOccurred()) + + Expect(unclaimable.Held()).To(BeEmpty(), + "a claimant whose claim failed installed itself anyway, so the registry now disagrees with the table") + _, err = unclaimable.Open(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNotOwner)) + }) + + It("records the claim, so another replica can find the owner", func() { + frontend, _ := workerTunnel() + epoch, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + + Expect(tun.Held()).To(ConsistOf("w1")) + owner, stored, err := reg.OwnerRow(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("me")) + Expect(stored).To(Equal(epoch), "Attach handed back an epoch that is not the one it wrote") + }) + + It("opens a stream that carries bytes to the worker", func() { + frontend, worker := workerTunnel() + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + echoOnce(worker) + + conn, err := tun.Open(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(conn).ToNot(BeNil()) + DeferCleanup(func() { _ = conn.Close() }) + + Expect(conn.SetDeadline(time.Now().Add(10 * time.Second))).To(Succeed()) + _, err = conn.Write([]byte("ping")) + Expect(err).ToNot(HaveOccurred()) + buf := make([]byte, 4) + _, err = conn.Read(buf) + Expect(err).ToNot(HaveOccurred()) + Expect(string(buf)).To(Equal("ping")) + }) + + It("reports ErrNotOwner for a node whose tunnel it does not hold", func() { + // This is a routing fact and nothing more: some other replica may hold + // the worker perfectly well. It must never be produced by anything that + // merely failed. + _, err := tun.Open(ctx, "nobody") + Expect(err).To(MatchError(cluster.ErrNotOwner)) + }) + + It("supersedes an earlier attachment, and the superseded holder's Detach is a no-op", func() { + first, _ := workerTunnel() + firstEpoch, err := tun.Attach(ctx, "w1", first) + Expect(err).ToNot(HaveOccurred()) + + second, secondWorker := workerTunnel() + secondEpoch, err := tun.Attach(ctx, "w1", second) + Expect(err).ToNot(HaveOccurred()) + // Compared for difference, never for order. Claim guarantees an epoch + // is unique and never reissued; it does NOT guarantee the later claim + // draws the larger number, because the sequence value on the insert + // path is drawn before the row lock. + Expect(secondEpoch).ToNot(Equal(firstEpoch)) + + Expect(first.IsClosed()).To(BeTrue(), + "the superseded session was left open, so whoever is accepting on it never learns it was replaced") + + tun.Detach("w1", firstEpoch) + + Expect(tun.Held()).To(ConsistOf("w1"), "a stale Detach evicted the live session") + owner, stored, err := reg.OwnerRow(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("me")) + Expect(stored).To(Equal(secondEpoch), "a stale Detach released the live claim") + + echoOnce(secondWorker) + conn, err := tun.Open(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + Expect(conn.SetDeadline(time.Now().Add(10 * time.Second))).To(Succeed()) + _, err = conn.Write([]byte("ping")) + Expect(err).ToNot(HaveOccurred()) + buf := make([]byte, 4) + _, err = conn.Read(buf) + Expect(err).ToNot(HaveOccurred()) + Expect(string(buf)).To(Equal("ping")) + + tun.Detach("w1", secondEpoch) + Expect(tun.Held()).To(BeEmpty()) + _, _, err = reg.OwnerRow(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + }) + + It("ignores a Detach naming an epoch it was never handed", func() { + frontend, _ := workerTunnel() + epoch, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + + // Not an ordering probe: both directions are tried because an epoch is + // unique but unordered, so a stale token can compare either way against + // the live one and neither may be allowed to evict it. + tun.Detach("w1", epoch+1) + tun.Detach("w1", epoch-1) + + Expect(tun.Held()).To(ConsistOf("w1")) + _, stored, err := reg.OwnerRow(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(stored).To(Equal(epoch)) + }) + + It("does not report a held tunnel whose session has died as ErrNotOwner", func() { + // Absence and unreachability are different answers and callers act + // differently on them: a dialer told the worker is not here relays + // elsewhere or reports it gone, where the truth is that this replica + // holds the tunnel and the socket underneath it broke. + frontend, worker := workerTunnel() + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + + Expect(worker.Close()).To(Succeed()) + Eventually(frontend.IsClosed, "10s").Should(BeTrue()) + + _, err = tun.Open(ctx, "w1") + Expect(err).To(HaveOccurred()) + Expect(err).ToNot(MatchError(cluster.ErrNotOwner), + "a broken socket was reported as this replica not holding the tunnel") + Expect(tun.Held()).To(ConsistOf("w1"), + "holding the tunnel is a routing fact, and a failed Open is not what un-holds it") + }) + + It("serves Attach, Open, Held and Detach from independent goroutines", func() { + // Run under -race. The point is contention on one node's entry, not a + // tidy per-goroutine partition: a registry whose map is only ever + // touched by one goroutine at a time proves nothing about the one that + // is not. + const workers = 8 + start := make(chan struct{}) + var wg sync.WaitGroup + + attached := make(chan int64, workers) + for i := 0; i < workers; i++ { + wg.Add(1) + go func(i int) { + defer GinkgoRecover() + defer wg.Done() + frontend, worker := workerTunnel() + drain(worker) + <-start + epoch, err := tun.Attach(ctx, fmt.Sprintf("w%d", i%2), frontend) + Expect(err).ToNot(HaveOccurred()) + attached <- epoch + if conn, err := tun.Open(ctx, fmt.Sprintf("w%d", i%2)); err == nil { + _ = conn.Close() + } + }(i) + } + readers := make(chan struct{}) + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer GinkgoRecover() + defer wg.Done() + <-start + for { + select { + case <-readers: + return + default: + tun.Held() + } + } + }() + } + + close(start) + epochs := make([]int64, 0, workers) + for i := 0; i < workers; i++ { + epochs = append(epochs, <-attached) + } + close(readers) + wg.Wait() + + // Exactly one attachment per node survived, whichever won, and the + // table agrees with the map about which. + Expect(tun.Held()).To(ConsistOf("w0", "w1")) + for _, node := range tun.Held() { + _, stored, err := reg.OwnerRow(ctx, node) + Expect(err).ToNot(HaveOccurred()) + Expect(epochs).To(ContainElement(stored), + "the table records an epoch no Attach ever handed out") + } + + // Every loser's Detach is a no-op; only the two winners empty the map. + for _, epoch := range epochs { + tun.Detach("w0", epoch) + tun.Detach("w1", epoch) + } + Expect(tun.Held()).To(BeEmpty()) + }) +}) + +var _ = Describe("Re-claiming tunnels after this replica's rows were reaped", func() { + var ( + db *gorm.DB + reg *cluster.Registry + tun *cluster.TunnelRegistry + ctx context.Context + ) + + BeforeEach(func() { + db = testutil.SetupTestDB() + ctx = context.Background() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg = cluster.NewRegistry(db) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + tun = cluster.NewTunnelRegistry(reg, "me") + }) + + // reapSelf performs the two deletes a peer's sweep performs on this + // replica: the instance row and, in the same transaction, every connection + // it owned. Deregister is that transaction; ReapStale reaches it by aging + // last_seen, which cannot be done deterministically against a heartbeat + // loop that is refreshing the same column. That ReapStale deletes both is + // pinned separately, with no loop running, in the reaping specs. + reapSelf := func() { + GinkgoHelper() + Expect(reg.Deregister(ctx, "me")).To(Succeed()) + } + + It("re-claims every held tunnel when the loop finds its instance row gone", func() { + // Without this a replica that stalled long enough to be swept sits + // holding live worker sockets that no row records. Every other replica + // then answers "not connected" for workers that are connected, which is + // the absence-versus-unreachable failure the design forbids. + frontend, _ := workerTunnel() + epoch, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + membership.SetTunnels(tun) + Expect(membership.Start(ctx)).To(Succeed()) + DeferCleanup(membership.Stop) + + reapSelf() + _, _, err = reg.OwnerRow(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + + owner := func() (string, error) { + owner, _, err := reg.OwnerRow(ctx, "w1") + return owner, err + } + Eventually(owner, 3*cluster.InstanceHeartbeat, time.Second).Should(Equal("me")) + + _, reclaimed, err := reg.OwnerRow(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(reclaimed).ToNot(Equal(epoch), "the re-claim reused the epoch of a claim the sweep deleted") + + // The holder still carries the epoch Attach handed it, and is the only + // thing that will ever release this row. If Detach matched only the + // epoch the re-claim drew, the row would outlive the socket and no + // caller could tell. + tun.Detach("w1", epoch) + Expect(tun.Held()).To(BeEmpty()) + _, _, err = reg.OwnerRow(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + }) + + It("does not re-claim a tunnel whose session has already died", func() { + // Re-claiming is an upsert, so it takes the row from whoever holds it + // now. A worker whose socket here is dead has already reconnected + // somewhere, and claiming it back would point every dialer at a replica + // that cannot carry a byte to it. + live, _ := workerTunnel() + liveEpoch, err := tun.Attach(ctx, "live", live) + Expect(err).ToNot(HaveOccurred()) + dead, deadWorker := workerTunnel() + _, err = tun.Attach(ctx, "dead", dead) + Expect(err).ToNot(HaveOccurred()) + Expect(deadWorker.Close()).To(Succeed()) + Eventually(dead.IsClosed, "10s").Should(BeTrue()) + + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + membership.SetTunnels(tun) + Expect(membership.Start(ctx)).To(Succeed()) + DeferCleanup(membership.Stop) + + reapSelf() + owner := func() (string, error) { + owner, _, err := reg.OwnerRow(ctx, "live") + return owner, err + } + Eventually(owner, 3*cluster.InstanceHeartbeat, time.Second).Should(Equal("me")) + + _, reclaimedLive, err := reg.OwnerRow(ctx, "live") + Expect(err).ToNot(HaveOccurred()) + Expect(reclaimedLive).ToNot(Equal(liveEpoch)) + + _, _, err = reg.OwnerRow(ctx, "dead") + Expect(err).To(MatchError(cluster.ErrNoConnection), + "a tunnel whose session is closed was claimed back from whoever holds the worker now") + }) + + It("keeps re-claiming out of the ordinary heartbeat, which has nothing to rebuild", func() { + // A claim per tick would draw a fresh epoch every five seconds for + // every worker on this replica, and every one of those writes is a + // chance to take a row a reconnect has just moved elsewhere. + frontend, _ := workerTunnel() + epoch, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + membership.SetTunnels(tun) + Expect(membership.Start(ctx)).To(Succeed()) + DeferCleanup(membership.Stop) + + stored := func() (int64, error) { + _, stored, err := reg.OwnerRow(ctx, "w1") + return stored, err + } + Consistently(stored, 2*cluster.InstanceHeartbeat, time.Second).Should(Equal(epoch)) + }) +}) From 2466e42906c2c20b61ca65836cc5d10d3a4cdfc4 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 08:32:51 +0000 Subject: [PATCH 043/153] fix(cluster): make a claim and its record indivisible per node Two Attach calls for one node both claim, and PostgreSQL serialises the two upserts, but nothing ordered the two map writes against the two commits. The entry left installed could be the one whose claim lost the row, and its Detach then released an epoch the row does not carry, so the release matched nothing and the row survived the socket. Nothing swept that row. This replica is alive and heartbeating, so ReapStale leaves its rows alone, and no reconnect is coming for a worker that has gone. Owner kept naming this replica as the live owner of a tunnel it no longer held, and every dialer sent here was answered ErrNotOwner, which is the relay into a replica that cannot serve the request that this phase exists to prevent. Claims for one node now pass through a gate, so claim and record are indivisible. It is per node rather than one lock over the registry, the way PeerPool locks per peer: the claim is a database round trip, and a slow one for a single worker must not hold up Open for every other. Detach is not gated, because it takes no context and must never park behind an in-flight database call, and it changes no epoch. Reclaim takes the same gate, which makes its claim the newest one for that node, so it records the epoch on whatever attachment is installed rather than only on the one it listed. Refusing to record onto an attachment that replaced the listed one would leave that row with nothing able to release it. The interleave the gate does not cover is Detach, and a claim whose attachment detached while it was in flight is now released again rather than left behind. Also: restore Start's doc comment, which SetTunnels had swallowed; keep reaping other replicas when this one fails to rebuild its own row, rather than skipping the sweep along with the re-claim; scope the comment about an unnoticed dead socket to the keepalive of the session whoever accepted the tunnel built, since the worker session config does not exist yet; and pin the sortedness of Held, the nil-session refusal, and both re-claim interleaves with specs. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/membership.go | 22 +-- core/services/cluster/tunnel.go | 199 ++++++++++++++++++---- core/services/cluster/tunnel_test.go | 239 +++++++++++++++++++++++++++ 3 files changed, 414 insertions(+), 46 deletions(-) diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index ff511cdd9ffb..4a5f3e19ccab 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -73,10 +73,6 @@ func NewMembership(reg *Registry, id, addr, version string) *Membership { } } -// Start registers this replica and begins heartbeating and sweeping. The first -// registration is synchronous and its failure is returned: a replica whose -// address never reaches the table is invisible to its peers, and starting -// anyway would hide that behind a background log line. // SetTunnels gives the loop the registry holding this replica's worker tunnels, // so it can re-claim them after its rows have been swept. A Membership without // one still heartbeats and sweeps; it simply has nothing to re-claim, which is @@ -91,6 +87,10 @@ func (m *Membership) SetTunnels(t *TunnelRegistry) { m.tunnels = t } +// Start registers this replica and begins heartbeating and sweeping. The first +// registration is synchronous and its failure is returned: a replica whose +// address never reaches the table is invisible to its peers, and starting +// anyway would hide that behind a background log line. func (m *Membership) Start(ctx context.Context) error { if err := m.reg.Register(ctx, m.id, m.addr, m.version); err != nil { return err @@ -177,14 +177,16 @@ func (m *Membership) tick(ctx context.Context) { // replica serves workers that, as far as every other replica can see, // are connected nowhere. xlog.Warn("Cluster instance row was reaped, re-registering", "id", m.id) - if err := m.reg.Register(ctx, m.id, m.addr, m.version); err != nil { + if err := m.reg.Register(ctx, m.id, m.addr, m.version); err == nil { + m.reclaimTunnels(ctx) + } else { + // Re-claiming is skipped and only re-claiming: a claim written now + // would name an instance row that does not exist, and the very next + // sweep deletes it as an orphan. The sweep below still runs, because + // what it removes is other replicas, and this replica failing to + // rebuild its own row is no reason to stop reaping theirs. xlog.Error("Re-registering cluster instance failed", "id", m.id, "error", err) - // Nothing to re-claim onto: a claim naming an instance row that - // does not exist is deleted by the next sweep that runs, this - // replica's own included, and the sweep is what has just happened. - return } - m.reclaimTunnels(ctx) } else if err != nil { xlog.Warn("Cluster instance heartbeat failed", "id", m.id, "error", err) } diff --git a/core/services/cluster/tunnel.go b/core/services/cluster/tunnel.go index 7ca928a5e7b9..891bf77b8238 100644 --- a/core/services/cluster/tunnel.go +++ b/core/services/cluster/tunnel.go @@ -45,6 +45,10 @@ type TunnelRegistry struct { mu sync.Mutex tunnels map[string]*heldTunnel + // claiming holds one gate per node that a claim is in flight for. It is + // what makes "claim, then record the epoch" indivisible per node; see + // enterClaim. + claiming map[string]chan struct{} } // heldTunnel is one accepted worker tunnel. @@ -77,12 +81,66 @@ type heldTunnel struct { // that is what Owner joins a claim against to decide the owner is alive. func NewTunnelRegistry(reg *Registry, selfID string) *TunnelRegistry { return &TunnelRegistry{ - reg: reg, - selfID: selfID, - tunnels: map[string]*heldTunnel{}, + reg: reg, + selfID: selfID, + tunnels: map[string]*heldTunnel{}, + claiming: map[string]chan struct{}{}, } } +// enterClaim takes the gate for nodeID, so that no two claims for one node are +// ever in flight at the same time. leaveClaim releases it. +// +// It exists because a claim and the record of that claim are two steps, and +// between them the database has already moved. Two Attach calls for one node +// both claim, and PostgreSQL serialises the two upserts, but nothing orders the +// two map writes against the two commits: the entry that ends up installed can +// carry the epoch of the claim that did NOT win the row. Its Detach then +// releases an epoch the row does not hold, the release matches nothing, and the +// row survives the socket. Nothing sweeps that, because the replica named on it +// is alive and heartbeating, so Owner keeps naming this replica as the owner of +// a tunnel it no longer holds and every dialer routed here gets ErrNotOwner. +// +// The gate is per node rather than one lock over the whole registry so that a +// slow claim for one worker does not hold up Open for any other, the same +// reason PeerPool locks per peer. Detach is deliberately NOT gated: it takes no +// context and must never park behind an in-flight database call. It does not +// need to be, because it changes no epoch; what it can interleave with is +// covered where that matters, in Reclaim. +// +// The entry is deleted rather than kept, so the map holds only the claims +// actually in flight and cannot grow with the number of workers ever seen. +func (t *TunnelRegistry) enterClaim(ctx context.Context, nodeID string) error { + for { + t.mu.Lock() + gate, busy := t.claiming[nodeID] + if !busy { + t.claiming[nodeID] = make(chan struct{}) + t.mu.Unlock() + return nil + } + t.mu.Unlock() + + // Re-checked in the loop rather than taken on waking: several waiters + // are released by one close, and only one of them may proceed. + select { + case <-gate: + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// leaveClaim releases the gate enterClaim took. The channel is closed rather +// than sent on, so every waiter wakes rather than one. +func (t *TunnelRegistry) leaveClaim(nodeID string) { + t.mu.Lock() + gate := t.claiming[nodeID] + delete(t.claiming, nodeID) + t.mu.Unlock() + close(gate) +} + // Attach records this replica as the owner of nodeID's tunnel and stores the // session, returning the epoch the caller must later hand to Detach. // @@ -98,6 +156,10 @@ func NewTunnelRegistry(reg *Registry, selfID string) *TunnelRegistry { // broken, only replaced, and would wait there until the far side noticed. The // caller keeps ownership of the session it passed in; only a session this // registry evicted is closed by this registry. +// +// Two Attach calls for one node are serialised, claim and record together, so +// the entry that survives is always the one whose claim the row carries. See +// enterClaim for what an unserialised pair leaves behind. func (t *TunnelRegistry) Attach(ctx context.Context, nodeID string, sess *yamux.Session) (int64, error) { if sess == nil { // Claiming would publish a tunnel that cannot carry anything, and the @@ -105,6 +167,11 @@ func (t *TunnelRegistry) Attach(ctx context.Context, nodeID string, sess *yamux. return 0, fmt.Errorf("attaching tunnel for node %q: no session", nodeID) } + if err := t.enterClaim(ctx, nodeID); err != nil { + return 0, fmt.Errorf("attaching tunnel for node %q: %w", nodeID, err) + } + defer t.leaveClaim(nodeID) + epoch, err := t.reg.Claim(ctx, nodeID, t.selfID) if err != nil { return 0, err @@ -220,8 +287,9 @@ func (t *TunnelRegistry) Held() []string { // closed has already reconnected somewhere: claiming it back would point every // dialer at a replica that cannot carry a byte to it. The check narrows that // window rather than closing it, since a socket can be dead without this side -// having noticed; yamux's keepalive bounds how long that lasts, and the -// worker's next reconnect supersedes the claim in any case. +// having noticed, and how long that lasts is decided by the keepalive on the +// session whoever accepted the tunnel built. The worker's next reconnect +// supersedes the claim in any case. // // The entry of a skipped tunnel is left alone. Whoever attached it owns its // lifetime and will detach it; Reclaim is not an eviction path, and evicting @@ -231,48 +299,107 @@ func (t *TunnelRegistry) Held() []string { // and a claim that failed is retried on the next sweep this replica survives. func (t *TunnelRegistry) Reclaim(ctx context.Context) (int, error) { t.mu.Lock() - held := make(map[string]*heldTunnel, len(t.tunnels)) - for nodeID, tunnel := range t.tunnels { - held[nodeID] = tunnel + held := make([]string, 0, len(t.tunnels)) + for nodeID := range t.tunnels { + held = append(held, nodeID) } t.mu.Unlock() var reclaimed int var errs []error - for nodeID, tunnel := range held { - if tunnel.sess.IsClosed() { - xlog.Debug("skipping re-claim of a worker tunnel whose session is closed", "node", nodeID) - continue - } - epoch, err := t.reg.Claim(ctx, nodeID, t.selfID) - if err != nil { + for _, nodeID := range held { + if err := t.reclaimOne(ctx, nodeID); err != nil { + if errors.Is(err, errTunnelNotReclaimed) { + continue + } errs = append(errs, err) continue } - t.mu.Lock() - // Re-read under the lock: the tunnel may have been detached, or - // superseded by a reconnect, while the claim was in flight. Writing the - // new claim onto whatever is there now would give a different - // attachment an epoch it never drew, and its Release would then match - // nothing. - // - // The epoch just drawn is then held by no attachment. If it also - // happened to be the last write to the row, the row outlives the - // attachment that no longer matches it: it still names this replica, - // truthfully, but no Detach will remove it. It is cleared by the - // worker's next claim anywhere, since Claim upserts, and otherwise by - // the sweep that eventually removes this replica. Serialising Attach - // and Reclaim per node would close it; the window is one database - // round trip during the tick that follows a sweep of this replica, and - // the machinery costs more than it removes. - if current, ok := t.tunnels[nodeID]; ok && current == tunnel { - current.claim = epoch - reclaimed++ - } - t.mu.Unlock() + reclaimed++ } if len(errs) > 0 { return reclaimed, fmt.Errorf("re-claiming worker tunnels: %w", errors.Join(errs...)) } return reclaimed, nil } + +// errTunnelNotReclaimed reports that a node was passed over rather than failed: +// its session is closed, or the attachment went away while the claim was in +// flight. It never leaves this file. It exists so Reclaim's count stays honest +// without "skipped" having to look like an error to its caller. +var errTunnelNotReclaimed = errors.New("cluster: tunnel not re-claimed") + +// reclaimOne writes a fresh claim for one node and records it on whatever +// attachment is installed for that node. +// +// Whatever is installed when the gate is taken, not whatever Reclaim listed a +// moment earlier. A worker that re-dialled in between has an entry carrying the +// epoch of ITS claim, and the gate makes that claim strictly older than this +// one, so the row now holds this epoch and only this entry can release it. +// Refusing to record onto an attachment because it is not the one listed would +// leave that row with no attachment able to release it, which is the leak this +// whole function exists to prevent. +// +// If nothing is installed, the attachment detached while the claim was in +// flight. Detach is not gated, so this is reachable, and it is the one case +// where a claim is drawn that no attachment will ever release: the row would +// name this replica for a tunnel it does not hold, and Owner would send every +// dialer here to be told ErrNotOwner. The claim is therefore released again. +// Releasing it cannot take anyone else's row, because Release matches the epoch +// exactly and no epoch is ever reissued. +func (t *TunnelRegistry) reclaimOne(ctx context.Context, nodeID string) error { + // The gate is taken before the entry is even read, so that everything this + // function decides is decided about the attachment its claim will land on. + // Reading first and gating after would leave a window in which a re-dial + // replaces the entry, and the liveness this checked would be a property of + // a session it is no longer claiming for. + if err := t.enterClaim(ctx, nodeID); err != nil { + return fmt.Errorf("re-claiming node %q: %w", nodeID, err) + } + + t.mu.Lock() + tunnel, ok := t.tunnels[nodeID] + t.mu.Unlock() + if !ok { + // Detached between Reclaim listing the nodes and this gate. Nothing was + // claimed, so there is nothing to undo. + t.leaveClaim(nodeID) + return errTunnelNotReclaimed + } + if tunnel.sess.IsClosed() { + xlog.Debug("skipping re-claim of a worker tunnel whose session is closed", "node", nodeID) + t.leaveClaim(nodeID) + return errTunnelNotReclaimed + } + + epoch, err := t.reg.Claim(ctx, nodeID, t.selfID) + if err != nil { + t.leaveClaim(nodeID) + return err + } + + t.mu.Lock() + current, installed := t.tunnels[nodeID] + if installed { + // current is necessarily the entry read above: the gate is still held, + // and Attach and reclaimOne are the only writers that install one. The + // identity is therefore not re-checked; the case that IS reachable is + // the entry being gone, because Detach is not gated. + current.claim = epoch + } + t.mu.Unlock() + t.leaveClaim(nodeID) + + if installed { + return nil + } + + // Released outside the gate: it is a second round trip, and holding the + // gate across it would park a worker re-dialling this node behind a + // cleanup. A re-dial that claims first simply makes this release match + // nothing, which is the same no-op it would have been. + if err := t.reg.Release(ctx, nodeID, t.selfID, epoch); err != nil && !errors.Is(err, ErrNoConnection) { + return fmt.Errorf("releasing a re-claim for detached node %q: %w", nodeID, err) + } + return errTunnelNotReclaimed +} diff --git a/core/services/cluster/tunnel_test.go b/core/services/cluster/tunnel_test.go index 33f4c2d858c4..ac7ff4ad8c37 100644 --- a/core/services/cluster/tunnel_test.go +++ b/core/services/cluster/tunnel_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "strings" "sync" "time" @@ -15,8 +16,81 @@ import ( . "github.com/onsi/gomega" "gorm.io/driver/sqlite" "gorm.io/gorm" + gormlogger "gorm.io/gorm/logger" ) +// claimHook runs an action once, from inside the database call that issued a +// matching statement, on that call's own goroutine. +// +// It is how a spec pins an interleaving instead of racing for one. gorm calls +// its logger's Trace after the statement has executed and before the Create or +// Delete that issued it returns (gorm@v1.31.1/callbacks.go:139-145), with the +// bind values interpolated into the SQL, so an action installed here runs at +// the one instant a claim has been written and not yet recorded. Racing two +// goroutines and hoping to land in that window is the flaky spec this replaces. +// +// It fires at most once: the action itself issues statements through the same +// session, and an unguarded hook would recurse. +type claimHook struct { + gormlogger.Interface + mu sync.Mutex + fired bool + match func(sql string) bool + action func(sql string) +} + +func newClaimHook(match func(sql string) bool) *claimHook { + return &claimHook{Interface: gormlogger.Default.LogMode(gormlogger.Silent), match: match} +} + +func (h *claimHook) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) { + sql, rows := fc() + h.mu.Lock() + fire := !h.fired && h.action != nil && h.match(sql) + if fire { + h.fired = true + } + h.mu.Unlock() + if fire { + h.action(sql) + } + h.Interface.Trace(ctx, begin, func() (string, int64) { return sql, rows }, err) +} + +// isClaimOf matches the upsert Claim issues for one node. +func isClaimOf(nodeIDs ...string) func(string) bool { + return func(sql string) bool { + if !strings.Contains(sql, "INSERT INTO \"node_connections\"") { + return false + } + for _, nodeID := range nodeIDs { + if strings.Contains(sql, "'"+nodeID+"'") { + return true + } + } + return false + } +} + +// claimedNode reports which of the named nodes a claim statement was for. +func claimedNode(sql string, nodeIDs ...string) string { + GinkgoHelper() + for _, nodeID := range nodeIDs { + if strings.Contains(sql, "'"+nodeID+"'") { + return nodeID + } + } + Fail("the claim statement named none of " + strings.Join(nodeIDs, ", ")) + return "" +} + +// serializationProbe is how long a spec watches for something that must not +// happen. It bounds an assertion about an ABSENT event, which is the only kind +// of wait a spec cannot replace with a channel: there is no event to receive. +// The thing it watches for takes one database round trip when the serialisation +// it guards is missing, so this is orders of magnitude longer than it needs. +const serializationProbe = 500 * time.Millisecond + // workerTunnel returns the two halves of a worker's tunnel: the frontend holds // the server half, because the worker is the side that dials. yamuxPair already // builds exactly that pairing for peer links; this names the halves the way the @@ -220,6 +294,85 @@ var _ = Describe("The worker tunnel registry", func() { "holding the tunnel is a routing fact, and a failed Open is not what un-holds it") }) + It("refuses a nil session rather than claiming a tunnel that cannot carry anything", func() { + // A claim written for a session that does not exist publishes a tunnel + // to every replica in the deployment, and the fence would then have to + // be unwound by a Detach nobody is going to call. + _, err := tun.Attach(ctx, "w1", nil) + Expect(err).To(HaveOccurred()) + + Expect(tun.Held()).To(BeEmpty()) + _, _, err = reg.OwnerRow(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection), + "a node with no session was published to the deployment as connected here") + }) + + It("returns the nodes it holds in sorted order", func() { + // Attached out of order on purpose: sorted output is what makes a log + // line and a re-claim pass comparable between two runs, and a map + // range would only look sorted until it did not. + for _, nodeID := range []string{"w3", "w1", "w2"} { + frontend, _ := workerTunnel() + _, err := tun.Attach(ctx, nodeID, frontend) + Expect(err).ToNot(HaveOccurred()) + } + Expect(tun.Held()).To(Equal([]string{"w1", "w2", "w3"})) + }) + + It("serialises two Attach calls for one node, so the surviving entry holds the row's epoch", func() { + // Two claims for one node are serialised by PostgreSQL, but nothing + // orders the two map writes against the two commits. Unserialised, the + // entry left installed can carry the epoch of the claim that lost the + // row: its Detach then releases an epoch the row does not hold, the + // release matches nothing, and the row outlives the socket. Nothing + // sweeps that, because this replica is alive and heartbeating, so Owner + // keeps sending dialers here to be told ErrNotOwner. + hook := newClaimHook(isClaimOf("w1")) + hooked := cluster.NewTunnelRegistry( + cluster.NewRegistry(db.Session(&gorm.Session{Logger: hook})), "me") + + secondSession, _ := workerTunnel() + secondStarted := make(chan struct{}) + secondEpochs := make(chan int64, 1) + hook.action = func(string) { + // Launched from inside the first claim, so the second Attach is + // provably reaching for the same node while the first is between + // its claim and its store. Starting it before the call would leave + // which one claims first to the scheduler. + go func() { + defer GinkgoRecover() + close(secondStarted) + epoch, err := hooked.Attach(ctx, "w1", secondSession) + Expect(err).ToNot(HaveOccurred()) + secondEpochs <- epoch + }() + <-secondStarted + Consistently(secondEpochs, serializationProbe, 10*time.Millisecond).ShouldNot(Receive(), + "a second Attach for this node claimed AND recorded its epoch while the first was between its own claim and store") + } + + firstSession, _ := workerTunnel() + firstEpoch, err := hooked.Attach(ctx, "w1", firstSession) + Expect(err).ToNot(HaveOccurred()) + var secondEpoch int64 + Eventually(secondEpochs, "10s").Should(Receive(&secondEpoch)) + Expect(secondEpoch).ToNot(Equal(firstEpoch)) + + _, stored, err := reg.OwnerRow(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect([]int64{firstEpoch, secondEpoch}).To(ContainElement(stored)) + + // Whichever attachment survived, one of these two Detach calls is the + // live one and must take the row with it. If neither does, the row is + // carrying an epoch no attachment holds. + hooked.Detach("w1", firstEpoch) + hooked.Detach("w1", secondEpoch) + Expect(hooked.Held()).To(BeEmpty()) + _, _, err = reg.OwnerRow(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection), + "the surviving attachment could not release its row, so the row outlived the socket") + }) + It("serves Attach, Open, Held and Detach from independent goroutines", func() { // Run under -race. The point is contention on one node's entry, not a // tidy per-goroutine partition: a registry whose map is only ever @@ -288,6 +441,11 @@ var _ = Describe("The worker tunnel registry", func() { tun.Detach("w1", epoch) } Expect(tun.Held()).To(BeEmpty()) + for _, node := range []string{"w0", "w1"} { + _, _, err := reg.OwnerRow(ctx, node) + Expect(err).To(MatchError(cluster.ErrNoConnection), + "node %s kept a row no attachment could release", node) + } }) }) @@ -392,6 +550,87 @@ var _ = Describe("Re-claiming tunnels after this replica's rows were reaped", fu "a tunnel whose session is closed was claimed back from whoever holds the worker now") }) + It("records the re-claim on the attachment installed now, not the one it listed", func() { + // Reclaim lists the nodes it holds, then claims them one at a time. A + // worker that re-dials in between leaves an entry the list never saw. + // The claim is drawn under that node's gate, so it is the NEWEST claim + // for the node and the row carries it: recording it on the entry that + // is installed is the only thing that lets that entry release the row. + // Refusing to record it because the entry is not the one listed would + // leave the row behind when the socket dies. + hook := newClaimHook(isClaimOf("w1", "w2")) + hooked := cluster.NewTunnelRegistry( + cluster.NewRegistry(db.Session(&gorm.Session{Logger: hook})), "me") + + for _, nodeID := range []string{"w1", "w2"} { + frontend, _ := workerTunnel() + _, err := hooked.Attach(ctx, nodeID, frontend) + Expect(err).ToNot(HaveOccurred()) + } + + // The re-dial lands on whichever node this pass has not reached yet, + // so the spec does not depend on which one Reclaim takes first. + type redial struct { + node string + epoch int64 + } + redialled := make(chan redial, 1) + hook.action = func(sql string) { + node := "w2" + if claimedNode(sql, "w1", "w2") == "w2" { + node = "w1" + } + frontend, _ := workerTunnel() + epoch, err := hooked.Attach(ctx, node, frontend) + Expect(err).ToNot(HaveOccurred()) + redialled <- redial{node: node, epoch: epoch} + } + + count, err := hooked.Reclaim(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(2)) + + var latest redial + Expect(redialled).To(Receive(&latest)) + _, stored, err := reg.OwnerRow(ctx, latest.node) + Expect(err).ToNot(HaveOccurred()) + Expect(stored).ToNot(Equal(latest.epoch), "the re-claim never reached the node that re-dialled") + + // The attachment that re-dialled is the one holding the socket, so its + // Detach has to be the one that removes the row. + hooked.Detach(latest.node, latest.epoch) + _, _, err = reg.OwnerRow(ctx, latest.node) + Expect(err).To(MatchError(cluster.ErrNoConnection), + "the re-claim was recorded on nothing, so the attachment that holds the socket cannot release its row") + }) + + It("releases a re-claim whose attachment detached while the claim was in flight", func() { + // Detach is not gated against a re-claim, so this interleave is real: + // the claim commits, then the socket dies and Detach releases the epoch + // it was given, which the claim has already replaced. Left alone, the + // row names this replica for a tunnel it no longer holds, nothing + // sweeps it because this replica is alive, and Owner sends every dialer + // here to be told ErrNotOwner. + hook := newClaimHook(isClaimOf("w1")) + hooked := cluster.NewTunnelRegistry( + cluster.NewRegistry(db.Session(&gorm.Session{Logger: hook})), "me") + + frontend, _ := workerTunnel() + epoch, err := hooked.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + + hook.action = func(string) { hooked.Detach("w1", epoch) } + + count, err := hooked.Reclaim(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(BeZero(), "a node that detached mid-claim was counted as re-claimed") + + Expect(hooked.Held()).To(BeEmpty()) + _, _, err = reg.OwnerRow(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection), + "the re-claim left a row behind that no attachment holds and no sweep will remove") + }) + It("keeps re-claiming out of the ordinary heartbeat, which has nothing to rebuild", func() { // A claim per tick would draw a fresh epoch every five seconds for // every worker on this replica, and every one of those writes is a From 9e4df1904596d2ed4a91b4c8322afffcb7df2528 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 09:06:42 +0000 Subject: [PATCH 044/153] fix(cluster): keep a session close out of the per-node claim gate The gate is justified by being held for one claim round trip, and Attach held it across the close of the session it superseded. Closing a yamux session closes the underlying conn and then waits for both its send and recv loops to exit, and the send loop can be inside a write bounded only by ConnectionWriteTimeout, so that is a wait on other goroutines. It must not stand between a worker re-dialling this node and its claim. The gate is now released after the store and before the close, which also makes Attach match reclaimOne, where it has always been released explicitly on every path. This is safe because a superseded session is no longer reachable from the map by the time it is closed: the next re-dial replaces an entry that already names the new session. Pin the re-claim half of the gate too. A worker that re-dials between a re-claim's commit and its record leaves the row carrying the re-dial's epoch while the entry carries the re-claim's, so the attachment holding the socket releases an epoch the row does not have and the row outlives it, with nothing to sweep it while this replica is alive. Only Attach's half of the serialisation was asserted; keying the two apart left every spec green. Also take the test hook's action under the lock that guards whether it has fired. It was written from the spec's goroutine and read from whichever goroutine issued the statement, which is a race in the harness that pins the serialisation specs. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/tunnel.go | 15 +++++- core/services/cluster/tunnel_test.go | 79 +++++++++++++++++++++++++--- 2 files changed, 86 insertions(+), 8 deletions(-) diff --git a/core/services/cluster/tunnel.go b/core/services/cluster/tunnel.go index 891bf77b8238..36f7c91d3d4b 100644 --- a/core/services/cluster/tunnel.go +++ b/core/services/cluster/tunnel.go @@ -170,10 +170,10 @@ func (t *TunnelRegistry) Attach(ctx context.Context, nodeID string, sess *yamux. if err := t.enterClaim(ctx, nodeID); err != nil { return 0, fmt.Errorf("attaching tunnel for node %q: %w", nodeID, err) } - defer t.leaveClaim(nodeID) epoch, err := t.reg.Claim(ctx, nodeID, t.selfID) if err != nil { + t.leaveClaim(nodeID) return 0, err } @@ -181,7 +181,20 @@ func (t *TunnelRegistry) Attach(ctx context.Context, nodeID string, sess *yamux. previous := t.tunnels[nodeID] t.tunnels[nodeID] = &heldTunnel{sess: sess, token: epoch, claim: epoch} t.mu.Unlock() + t.leaveClaim(nodeID) + // Closed after the gate is released, not under it. The gate is justified by + // being held for one claim round trip, and closing a session is not that: + // yamux closes the underlying conn and then waits for both its send and + // recv loops to exit (go-yamux/v5@v5.1.0/session.go:330-332), and the send + // loop can be inside a write bounded only by ConnectionWriteTimeout. That + // is a wait on other goroutines, and it must not stand between a worker + // re-dialling this node and its claim. + // + // Releasing first is safe because the superseded session is no longer + // reachable from the map: whoever re-dials next replaces an entry that + // already names the new session, and this close can only ever affect the + // one it just displaced. if previous != nil && previous.sess != sess { xlog.Debug("worker re-dialled this replica, dropping its previous tunnel", "node", nodeID) _ = previous.sess.Close() diff --git a/core/services/cluster/tunnel_test.go b/core/services/cluster/tunnel_test.go index ac7ff4ad8c37..67993130ee54 100644 --- a/core/services/cluster/tunnel_test.go +++ b/core/services/cluster/tunnel_test.go @@ -43,16 +43,26 @@ func newClaimHook(match func(sql string) bool) *claimHook { return &claimHook{Interface: gormlogger.Default.LogMode(gormlogger.Silent), match: match} } +// setAction installs what the hook runs, under the same lock that guards fired. +// The action is written from the spec's goroutine and read from whichever +// goroutine issues the statement, which need not be the same one. +func (h *claimHook) setAction(action func(sql string)) { + h.mu.Lock() + defer h.mu.Unlock() + h.action = action +} + func (h *claimHook) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) { sql, rows := fc() h.mu.Lock() - fire := !h.fired && h.action != nil && h.match(sql) + action := h.action + fire := !h.fired && action != nil && h.match(sql) if fire { h.fired = true } h.mu.Unlock() if fire { - h.action(sql) + action(sql) } h.Interface.Trace(ctx, begin, func() (string, int64) { return sql, rows }, err) } @@ -334,7 +344,7 @@ var _ = Describe("The worker tunnel registry", func() { secondSession, _ := workerTunnel() secondStarted := make(chan struct{}) secondEpochs := make(chan int64, 1) - hook.action = func(string) { + hook.setAction(func(string) { // Launched from inside the first claim, so the second Attach is // provably reaching for the same node while the first is between // its claim and its store. Starting it before the call would leave @@ -349,7 +359,7 @@ var _ = Describe("The worker tunnel registry", func() { <-secondStarted Consistently(secondEpochs, serializationProbe, 10*time.Millisecond).ShouldNot(Receive(), "a second Attach for this node claimed AND recorded its epoch while the first was between its own claim and store") - } + }) firstSession, _ := workerTunnel() firstEpoch, err := hooked.Attach(ctx, "w1", firstSession) @@ -575,7 +585,7 @@ var _ = Describe("Re-claiming tunnels after this replica's rows were reaped", fu epoch int64 } redialled := make(chan redial, 1) - hook.action = func(sql string) { + hook.setAction(func(sql string) { node := "w2" if claimedNode(sql, "w1", "w2") == "w2" { node = "w1" @@ -584,7 +594,7 @@ var _ = Describe("Re-claiming tunnels after this replica's rows were reaped", fu epoch, err := hooked.Attach(ctx, node, frontend) Expect(err).ToNot(HaveOccurred()) redialled <- redial{node: node, epoch: epoch} - } + }) count, err := hooked.Reclaim(ctx) Expect(err).ToNot(HaveOccurred()) @@ -604,6 +614,61 @@ var _ = Describe("Re-claiming tunnels after this replica's rows were reaped", fu "the re-claim was recorded on nothing, so the attachment that holds the socket cannot release its row") }) + It("serialises a re-claim against an Attach for the same node", func() { + // The re-claim takes the same gate Attach does, and for the same + // reason. Unserialised, a worker re-dialling between the re-claim's + // commit and its record leaves the row carrying the re-dial's epoch + // while the entry carries the re-claim's, so the attachment holding the + // socket releases an epoch the row does not have. Nothing sweeps the + // row that is left, because this replica is alive: Owner keeps naming + // it as the owner of a tunnel it does not hold. + hook := newClaimHook(isClaimOf("w1")) + hooked := cluster.NewTunnelRegistry( + cluster.NewRegistry(db.Session(&gorm.Session{Logger: hook})), "me") + + first, _ := workerTunnel() + attachEpoch, err := hooked.Attach(ctx, "w1", first) + Expect(err).ToNot(HaveOccurred()) + + redialSession, _ := workerTunnel() + redialStarted := make(chan struct{}) + redialEpochs := make(chan int64, 1) + hook.setAction(func(string) { + // Launched from inside the re-claim's own claim, so the re-dial is + // provably reaching for this node while the re-claim is between its + // commit and its record. + go func() { + defer GinkgoRecover() + close(redialStarted) + epoch, err := hooked.Attach(ctx, "w1", redialSession) + Expect(err).ToNot(HaveOccurred()) + redialEpochs <- epoch + }() + <-redialStarted + Consistently(redialEpochs, serializationProbe, 10*time.Millisecond).ShouldNot(Receive(), + "a worker re-dialled and recorded its claim while a re-claim for the same node was between its own claim and record") + }) + + count, err := hooked.Reclaim(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(1)) + + var redialEpoch int64 + Eventually(redialEpochs, "10s").Should(Receive(&redialEpoch)) + Expect(redialEpoch).ToNot(Equal(attachEpoch)) + + // The re-dial claimed after the re-claim, so the row carries its epoch + // and its attachment is the one that has to be able to release it. The + // superseded token must still be a no-op. + hooked.Detach("w1", attachEpoch) + Expect(hooked.Held()).To(ConsistOf("w1")) + hooked.Detach("w1", redialEpoch) + Expect(hooked.Held()).To(BeEmpty()) + _, _, err = reg.OwnerRow(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection), + "the attachment that re-dialled could not release its row, so the row outlived the socket") + }) + It("releases a re-claim whose attachment detached while the claim was in flight", func() { // Detach is not gated against a re-claim, so this interleave is real: // the claim commits, then the socket dies and Detach releases the epoch @@ -619,7 +684,7 @@ var _ = Describe("Re-claiming tunnels after this replica's rows were reaped", fu epoch, err := hooked.Attach(ctx, "w1", frontend) Expect(err).ToNot(HaveOccurred()) - hook.action = func(string) { hooked.Detach("w1", epoch) } + hook.setAction(func(string) { hooked.Detach("w1", epoch) }) count, err := hooked.Reclaim(ctx) Expect(err).ToNot(HaveOccurred()) From 9702e01adc3c62ffb961fd11345a19cfddaa3e01 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 09:42:26 +0000 Subject: [PATCH 045/153] feat(cluster): open the door a worker dials its tunnel through A worker needs no inbound port: it dials GET /api/cluster/connect, the connection becomes one multiplexed yamux session, and the frontend opens a stream on it per request. This adds the endpoint that accepts that dial and attaches it to the tunnel registry. The dial is authenticated against the NODE's own stored token hash rather than the deployment's registration token. That is the mechanism, not yet the isolation, since a worker still registers by presenting the shared token; what it rules out is the shortcut of comparing against the configured value, which would have to be unpicked the day workers get their own secrets. Every refusal happens BEFORE the WebSocket upgrade, so a dialer reads an HTTP status rather than a handshake error. The route is registered in every deployment, single-binary ones included, which is what puts it in front of the route-coverage test that holds that rule in place; with no node registry it refuses every dial, and tells a credentialed one the frontend has no cluster rather than that its token is wrong. A lookup that FAILED is answered as a failure. Reporting a database that could not be read as "unauthorized" would send a worker re-registering, throwing away the identity its tunnel and loaded models are keyed by. Wires the tunnel registry in core/application/distributed.go and hands it to the membership loop. Without that call the re-claim after a replica is reaped had no production caller and could never run. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 26 ++ core/http/app.go | 12 + core/http/endpoints/cluster/connect.go | 191 +++++++++++++ core/http/endpoints/cluster/connect_test.go | 285 ++++++++++++++++++++ core/http/routes/cluster.go | 20 ++ core/services/cluster/tunnel.go | 11 + docs/content/features/distributed-mode.md | 18 ++ 7 files changed, 563 insertions(+) create mode 100644 core/http/endpoints/cluster/connect.go create mode 100644 core/http/endpoints/cluster/connect_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index 0021907de612..b8f2b8e9afcc 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -57,6 +57,11 @@ type DistributedServices struct { Membership *cluster.Membership // PeerSessions owns the peer links other replicas dialled into this one. PeerSessions *cluster.SessionStore + // Tunnels holds the worker tunnels this replica has accepted and keeps the + // node_connections table agreeing with them. It is handed to the membership + // loop, which re-claims what it holds after this replica has been reaped, + // and to the route that accepts a worker's dial. + Tunnels *cluster.TunnelRegistry shutdownOnce sync.Once } @@ -220,6 +225,26 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } } + // The worker tunnels this replica accepts. It claims as the SAME instance + // ID membership registers under, because that is the ID a peer's Owner + // lookup joins a claim against to decide the owner is alive; two IDs here + // would make every claim this replica writes look like it belongs to a + // replica that does not exist. + tunnels := cluster.NewTunnelRegistry(clusterRegistry, cfg.Distributed.InstanceID) + // Without this the re-claim in the heartbeat loop is dead code: a replica + // stalled long enough to be swept loses the connection rows it owned, and + // nothing would ever write them back, so every other replica would answer + // "not connected" for workers that are connected right here. + // + // Nil when no peer-reachable address could be determined above. There is no + // heartbeat loop to hand it to in that case, and no other replica can reach + // this one anyway; the registry is still built, because it is what the + // tunnel endpoint attaches to and what this replica opens its own streams + // through. + if membership != nil { + membership.SetTunnels(tunnels) + } + // Let scheduling rules be keyed by a model alias. The registry resolves a // rule's name through the config loader to find the model it governs, so an // operator can pin placement to a stable name like "production" and have it @@ -528,6 +553,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade Cluster: clusterRegistry, Membership: membership, PeerSessions: peerSessions, + Tunnels: tunnels, }, nil } diff --git a/core/http/app.go b/core/http/app.go index 1efc7da41a3f..28a9bd19d9eb 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -567,6 +567,18 @@ func API(application *application.Application) (*echo.Echo, error) { routes.RegisterClusterRoutes(e, distCfg.RegistrationToken, d.PeerSessions.Accept) } + // The worker tunnel, registered unconditionally. Both arguments are nil + // outside distributed mode and the handler refuses every dial then, which + // is what makes registering it always safe; what it buys is the + // route-coverage test walking the route in a plain single-binary + // application, and that test is what holds the rule that an unauthenticated + // dial is refused BEFORE the WebSocket upgrade. + var tunnels *clustersvc.TunnelRegistry + if d := application.Distributed(); d != nil { + tunnels = d.Tunnels + } + routes.RegisterWorkerTunnelRoute(e, registry, tunnels) + // Distributed SSE routes (job progress + agent events via NATS) if d := application.Distributed(); d != nil { if d.Dispatcher != nil { diff --git a/core/http/endpoints/cluster/connect.go b/core/http/endpoints/cluster/connect.go new file mode 100644 index 000000000000..85c48b05277c --- /dev/null +++ b/core/http/endpoints/cluster/connect.go @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: MIT + +package cluster + +import ( + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "errors" + "net/http" + "strings" + + "github.com/gorilla/websocket" + "github.com/labstack/echo/v4" + "github.com/libp2p/go-yamux/v5" + clustersvc "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/xlog" + "gorm.io/gorm" +) + +// ConnectHandler serves the door a worker knocks on: it authenticates the dial +// against the node's OWN stored token, upgrades it to a WebSocket, wraps that as +// a yamux server session and attaches it to the tunnel registry. +// +// The worker dials out and never listens, which is the whole point of the +// tunnel: a worker behind NAT, in another cluster, or on a laptop needs no +// inbound port. It is therefore the yamux CLIENT and this side the SERVER, so +// this side owns the even stream IDs and is the side that opens streams. +// Nothing here accepts streams: in this design the frontend asks and the worker +// answers, so a worker that opened a stream into this session would park on the +// accept backlog rather than be served. +// +// The route is registered in every deployment, including single-binary ones, so +// that the route-coverage test under build tag `auth` walks it. A nil registry +// or a nil tunnel registry therefore has to be a real answer rather than a +// panic; see the 503 below. +// +// It is deliberately absent from auth.RouteFeatureRegistry. That registry gates +// a route on the FEATURES OF AN AUTHENTICATED USER, and there is no user here: +// the dialer is a worker process holding a machine credential, and the global +// auth middleware never runs on this path because it sits under +// auth.ClusterPathPrefix. +func ConnectHandler(registry *nodes.NodeRegistry, tunnels *clustersvc.TunnelRegistry) echo.HandlerFunc { + // gorilla's default CheckOrigin restricts a browser to same-origin and lets + // a header-less client (which every worker is) through, so the zero value + // is what this link wants. The same choice PeerHandler makes. + upgrader := websocket.Upgrader{} + + return func(c echo.Context) error { + // Everything below happens BEFORE the upgrade, and the order inside it + // is load-bearing. The credential is read first because a dial with no + // Authorization header at all is the anonymous case, and the + // route-coverage test issues exactly that, with no query string: it + // must see 401 rather than a 400 about a missing node id. + token, ok := bearerToken(c.Request()) + if !ok { + return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") + } + + // Not 401. A frontend with no cluster cannot authenticate anybody, and + // answering "unauthorized" would send the operator hunting a token + // problem that does not exist. It is checked after the header so that + // an anonymous dial still gets the 401 the coverage test requires. + if registry == nil || tunnels == nil { + return echo.NewHTTPError(http.StatusServiceUnavailable, "distributed mode not enabled") + } + + nodeID := c.QueryParam("id") + if nodeID == "" { + return echo.NewHTTPError(http.StatusBadRequest, "missing node id") + } + + node, err := registry.Get(c.Request().Context(), nodeID) + switch { + case errors.Is(err, gorm.ErrRecordNotFound): + // A node this frontend has never seen. Reported as 401 rather than + // 404 so a caller cannot enumerate node IDs by status code. + xlog.Debug("worker tunnel dial named an unknown node", "node", nodeID) + return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") + case err != nil: + // A query that FAILED is neither a rejection nor an absence. This + // is the phase's standing rule in its HTTP form: telling a worker + // its credentials are wrong when the database merely could not be + // read sends it re-registering instead of retrying, and a worker + // that re-registers has thrown away the identity its tunnel and its + // loaded models are keyed by. + xlog.Error("Looking up a worker for its tunnel dial failed", "node", nodeID, "error", err) + return echo.NewHTTPError(http.StatusInternalServerError, "node lookup failed") + } + + if !authorizedWorker(token, node.TokenHash) { + xlog.Debug("worker tunnel dial presented the wrong token", "node", nodeID) + return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") + } + + ws, err := upgrader.Upgrade(c.Response(), c.Request(), nil) + if err != nil { + // Upgrade has already written its own failure to the client. + xlog.Debug("worker tunnel upgrade failed", "node", nodeID, "error", err) + return nil + } + + sess, err := yamux.Server(clustersvc.WebsocketConn(ws), nil, nil) + if err != nil { + xlog.Error("Worker tunnel session setup failed", "node", nodeID, "error", err) + _ = ws.Close() + return nil + } + + // From here the connection is hijacked, so no status can reach the + // worker any more: a failure is a closed socket, which is what its + // reconnect loop reads. + epoch, err := tunnels.Attach(c.Request().Context(), nodeID, sess) + if err != nil { + xlog.Error("Attaching a worker tunnel failed", "node", nodeID, "error", err) + _ = sess.Close() + return nil + } + + xlog.Info("Worker tunnel established", "node", nodeID, "remote", ws.RemoteAddr().String()) + // The session outlives this handler, so something other than the + // request goroutine has to notice it die. yamux closes shutdownCh from + // its receive loop the moment the underlying conn fails + // (go-yamux/v5@v5.1.0/session.go:691-695 calling close at + // session.go:297-311), and its default config keepalives every 30s + // (mux.go:73-74), so a worker that vanishes without a FIN is noticed + // too rather than held forever. + go func() { + <-sess.CloseChan() + // The token Attach returned, never a fresh or zero one. Detach + // matches it by EQUALITY: it identifies THIS attachment, so a + // worker that has already re-dialled onto this replica is not + // evicted by its predecessor's teardown. + tunnels.Detach(nodeID, epoch) + xlog.Debug("worker tunnel closed", "node", nodeID) + }() + return nil + } +} + +// bearerToken returns the token from an Authorization: Bearer header, and +// whether one was present at all. +// +// The presence of a credential and its correctness are separate answers on +// purpose: "no credential" is what decides the pre-upgrade 401, and it has to be +// decidable before anything about the node is known. +func bearerToken(r *http.Request) (string, bool) { + // RFC 7235 makes the scheme case-insensitive; the token after it is not. + const prefix = "Bearer " + header := r.Header.Get("Authorization") + if len(header) < len(prefix) || !strings.EqualFold(header[:len(prefix)], prefix) { + return "", false + } + token := header[len(prefix):] + if token == "" { + return "", false + } + return token, true +} + +// authorizedWorker compares a presented token against the hash stored on the +// node's own row, in constant time. +// +// Against the NODE's stored hash, not the configured registration token, which +// is a deliberate strengthening over how /api/node/ authenticates. A tunnel is a +// durable, multiplexed pipe into a worker; a credential that authorizes every +// worker at once would mean one leak lets an attacker impersonate any worker +// whose ID it can read, and take over that worker's traffic by claiming its +// tunnel. +// +// What that buys TODAY is the mechanism, not yet the isolation, and the +// difference should not be overstated: a worker registers by presenting the +// deployment's registration token, so the hash on its row is that token's hash +// and a leaked registration token still opens a tunnel for a node whose ID the +// attacker knows. What this does rule out is the shortcut of comparing against +// the configured token itself, which is what would have to be unpicked later; +// the day a worker is issued its own secret at registration, this check starts +// isolating workers from each other with no change here. +// +// An empty stored hash authorizes nobody. Such a row exists whenever a worker +// registered against a frontend with no registration token configured, and the +// alternative, letting any token through for those, would make the check depend +// on a setting the dialer can neither see nor be blamed for. +func authorizedWorker(token, storedHash string) bool { + if storedHash == "" { + return false + } + sum := sha256.Sum256([]byte(token)) + return subtle.ConstantTimeCompare([]byte(hex.EncodeToString(sum[:])), []byte(storedHash)) == 1 +} diff --git a/core/http/endpoints/cluster/connect_test.go b/core/http/endpoints/cluster/connect_test.go new file mode 100644 index 000000000000..7236199171f0 --- /dev/null +++ b/core/http/endpoints/cluster/connect_test.go @@ -0,0 +1,285 @@ +package cluster_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "net/url" + "strings" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/auth" + "github.com/mudler/LocalAI/core/http/routes" + clustersvc "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/testutil" + + "github.com/gorilla/websocket" + "github.com/labstack/echo/v4" + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +// workerToken is the secret one worker holds. It is stored on that worker's row +// as a hash, never in a config value, which is the whole point of the check the +// specs below pin: a second worker's token, and the deployment-wide +// registration token, are both wrong for this node. +const workerToken = "worker-1-secret" + +func tokenHash(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +// bearer builds the header a worker dials with. +func bearer(token string) http.Header { + h := http.Header{} + h.Set("Authorization", "Bearer "+token) + return h +} + +// wsConnectURL is the worker tunnel route on a test server, named as nodeID. +func wsConnectURL(s *httptest.Server, nodeID string) string { + return "ws" + strings.TrimPrefix(s.URL, "http") + clustersvc.ConnectPath + + "?id=" + url.QueryEscape(nodeID) +} + +var _ = Describe("Worker tunnel handler", func() { + var ( + srv *httptest.Server + db *gorm.DB + reg *clustersvc.Registry + tun *clustersvc.TunnelRegistry + nodeID string + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + db = testutil.SetupTestDB() + + nodeReg, err := nodes.NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + + node := &nodes.BackendNode{ + Name: "worker-1", + Address: "10.0.0.9:50051", + TokenHash: tokenHash(workerToken), + } + Expect(nodeReg.Register(ctx, node, true)).To(Succeed()) + nodeID = node.ID + Expect(nodeID).ToNot(BeEmpty()) + + reg = clustersvc.NewRegistry(db) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + tun = clustersvc.NewTunnelRegistry(reg, "me") + + e := echo.New() + routes.RegisterWorkerTunnelRoute(e, nodeReg, tun) + srv = httptest.NewServer(e) + DeferCleanup(srv.Close) + }) + + It("refuses an anonymous dial before upgrading", func() { + // A plain GET, not a WebSocket dial: this is exactly what the + // route-coverage test issues, and a handler that upgrades first answers + // it with gorilla's own 400 handshake failure instead of a 401. + req, err := http.NewRequestWithContext(ctx, http.MethodGet, srv.URL+clustersvc.ConnectPath+"?id="+nodeID, nil) + Expect(err).ToNot(HaveOccurred()) + resp, err := http.DefaultClient.Do(req) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = resp.Body.Close() }() + + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + Expect(resp.Header.Get("Upgrade")).To(BeEmpty(), + "the handler upgraded an unauthenticated dial") + Expect(tun.Held()).To(BeEmpty()) + }) + + It("refuses a dial that carries no credentials at all", func() { + _, resp, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), nil) + Expect(err).To(HaveOccurred()) + Expect(resp).ToNot(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + Expect(tun.Held()).To(BeEmpty()) + }) + + It("refuses a token that is not the node's own", func() { + // The registration token is the credential every worker in the + // deployment holds. Accepting it here would mean one leaked shared + // secret impersonates any worker whose ID an attacker can read. + _, resp, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), bearer("deployment-registration-token")) + Expect(err).To(HaveOccurred()) + Expect(resp).ToNot(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + Expect(tun.Held()).To(BeEmpty()) + }) + + It("refuses a dial that names a node it has never seen", func() { + _, resp, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, "no-such-node"), bearer(workerToken)) + Expect(err).To(HaveOccurred()) + Expect(resp).ToNot(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + Expect(tun.Held()).To(BeEmpty()) + }) + + It("refuses an authenticated dial that names no node", func() { + _, resp, err := websocket.DefaultDialer.Dial( + "ws"+strings.TrimPrefix(srv.URL, "http")+clustersvc.ConnectPath, bearer(workerToken)) + Expect(err).To(HaveOccurred()) + Expect(resp).ToNot(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + }) + + It("attaches an authenticated worker and carries bytes to it", func() { + conn, _, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), bearer(workerToken)) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + + // The worker is the side that dials, so its half of the mux is the + // yamux CLIENT and the frontend's is the server. + workerSess, err := yamux.Client(clustersvc.WebsocketConn(conn), nil, nil) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = workerSess.Close() }) + + Eventually(tun.Held, "10s").Should(ConsistOf(nodeID)) + + owner, _, err := reg.OwnerRow(ctx, nodeID) + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("me"), + "the tunnel was stored without the claim that tells other replicas where it is") + + go func() { + defer GinkgoRecover() + stream, aerr := workerSess.AcceptStream() + if aerr != nil { + return + } + defer func() { _ = stream.Close() }() + buf := make([]byte, 4) + if _, rerr := stream.Read(buf); rerr != nil { + return + } + _, _ = stream.Write(buf) + }() + + stream, err := tun.Open(ctx, nodeID) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = stream.Close() }) + _, err = stream.Write([]byte("ping")) + Expect(err).ToNot(HaveOccurred()) + echoed := make([]byte, 4) + _, err = stream.Read(echoed) + Expect(err).ToNot(HaveOccurred()) + Expect(string(echoed)).To(Equal("ping")) + }) + + It("detaches the tunnel and drops its claim when the worker goes away", func() { + conn, _, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), bearer(workerToken)) + Expect(err).ToNot(HaveOccurred()) + Eventually(tun.Held, "10s").Should(ConsistOf(nodeID)) + + Expect(conn.Close()).To(Succeed()) + + Eventually(tun.Held, "10s").Should(BeEmpty(), + "a dead tunnel is still held here, so every dialer routed to this replica gets a socket that carries nothing") + Eventually(func() error { + _, _, err := reg.OwnerRow(ctx, nodeID) + return err + }, "10s").Should(MatchError(clustersvc.ErrNoConnection), + "the claim outlived the socket, so this replica keeps being named the owner of a worker it no longer holds") + }) + + It("reports a lookup failure as a failure, not as a refusal", func() { + // ErrNotOwner, 401 and 404 are all ANSWERS. A database that cannot be + // read is none of them: telling a worker its credentials are wrong when + // the frontend simply could not look them up sends it re-registering + // instead of retrying. + Expect(db.Exec(`DROP TABLE backend_nodes CASCADE`).Error).To(Succeed()) + + _, resp, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), bearer(workerToken)) + Expect(err).To(HaveOccurred()) + Expect(resp).ToNot(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError)) + }) +}) + +var _ = Describe("Worker tunnel handler without distributed mode", func() { + // The route is registered in every deployment so that the route-coverage + // test sees it, which is what pins the reject-before-upgrade rule. With no + // node registry there is nothing to authenticate against, so it must refuse + // every dial rather than publish an unauthenticated multiplexer. + var srv *httptest.Server + + BeforeEach(func() { + e := echo.New() + routes.RegisterWorkerTunnelRoute(e, nil, nil) + srv = httptest.NewServer(e) + DeferCleanup(srv.Close) + }) + + It("refuses an anonymous dial with 401", func() { + req, err := http.NewRequestWithContext(GinkgoT().Context(), http.MethodGet, srv.URL+clustersvc.ConnectPath, nil) + Expect(err).ToNot(HaveOccurred()) + resp, err := http.DefaultClient.Do(req) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = resp.Body.Close() }() + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + + It("tells a credentialed worker the frontend has no cluster, rather than rejecting it", func() { + // A single-binary frontend cannot authenticate anybody, and saying + // "unauthorized" would send an operator hunting a token problem that + // does not exist. + _, resp, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, "w1"), bearer(workerToken)) + Expect(err).To(HaveOccurred()) + Expect(resp).ToNot(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusServiceUnavailable)) + }) +}) + +var _ = Describe("Worker tunnel auth coverage", func() { + // The same argument the peer link's coverage specs make: the tunnel route + // authenticates a worker against its own stored token, not a session, so it + // only works while its path sits under the prefix the global auth + // middleware exempts. + var srv *httptest.Server + + BeforeEach(func() { + e := echo.New() + // A nil DB with one legacy API key is the cheapest configuration that + // turns the middleware ON without a database. + e.Use(auth.Middleware(nil, &config.ApplicationConfig{ApiKeys: []string{"an-api-key"}})) + routes.RegisterWorkerTunnelRoute(e, nil, nil) + e.GET("/api/nodes", func(c echo.Context) error { return c.NoContent(http.StatusOK) }) + srv = httptest.NewServer(e) + DeferCleanup(srv.Close) + }) + + It("refuses an uncredentialed request to a route outside the cluster prefix", func() { + resp, err := http.Get(srv.URL + "/api/nodes") + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = resp.Body.Close() }() + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized), + "the global auth middleware is not actually guarding this server, so the assertion below would prove nothing") + }) + + It("lets a worker dial reach the handler, which is the only thing that can authenticate it", func() { + // The worker's token is not one of the API keys the middleware knows, + // so a 503 from the handler's own no-cluster check can only mean the + // request was let through by the middleware. + req, err := http.NewRequestWithContext(GinkgoT().Context(), http.MethodGet, srv.URL+clustersvc.ConnectPath+"?id=w1", nil) + Expect(err).ToNot(HaveOccurred()) + req.Header.Set("Authorization", "Bearer "+workerToken) + + resp, err := http.DefaultClient.Do(req) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = resp.Body.Close() }() + Expect(resp.StatusCode).To(Equal(http.StatusServiceUnavailable), + "a worker dial must reach the handler; 401 here means the tunnel route left the auth-exempt prefix %q", auth.ClusterPathPrefix) + }) +}) diff --git a/core/http/routes/cluster.go b/core/http/routes/cluster.go index 3e462db1cd75..458122d036e1 100644 --- a/core/http/routes/cluster.go +++ b/core/http/routes/cluster.go @@ -3,6 +3,7 @@ package routes import ( clusterep "github.com/mudler/LocalAI/core/http/endpoints/cluster" clustersvc "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/nodes" "github.com/labstack/echo/v4" "github.com/libp2p/go-yamux/v5" @@ -23,3 +24,22 @@ import ( func RegisterClusterRoutes(e *echo.Echo, token string, onPeer func(string, *yamux.Session)) { e.GET(clustersvc.PeerPath, clusterep.PeerHandler(token, onPeer)) } + +// RegisterWorkerTunnelRoute registers the endpoint a worker dials to open its +// tunnel. registry authenticates the dial against the node's own stored token; +// tunnels is what the resulting session is attached to. +// +// Unlike the peer link this is registered in EVERY deployment, single-binary +// ones included, and both arguments may be nil there. Two reasons. The handler +// fails closed without a registry, since a token can only be checked against a +// node row and there are none; and being registered unconditionally is what +// puts the route in front of the route-coverage test under build tag `auth`, +// which is the thing that holds the reject-before-upgrade rule in place. A +// route registered only in distributed mode is invisible to that test. +// +// Like the peer link, it carries no auth middleware and derives its path from +// core/services/cluster's own constant, so the handler and the worker's dialler +// cannot end up on different paths. +func RegisterWorkerTunnelRoute(e *echo.Echo, registry *nodes.NodeRegistry, tunnels *clustersvc.TunnelRegistry) { + e.GET(clustersvc.ConnectPath, clusterep.ConnectHandler(registry, tunnels)) +} diff --git a/core/services/cluster/tunnel.go b/core/services/cluster/tunnel.go index 36f7c91d3d4b..66baa475fdce 100644 --- a/core/services/cluster/tunnel.go +++ b/core/services/cluster/tunnel.go @@ -15,6 +15,17 @@ import ( "github.com/mudler/xlog" ) +// ConnectPath is the route a worker dials to open its tunnel, and the route the +// HTTP layer registers the handler on. It lives here, beside the registry that +// holds what the dial produces, for the reason PeerPath does: the HTTP +// endpoints package imports this one, never the other way round. +// +// The literal is spelled out rather than derived from auth.ClusterPathPrefix +// because importing core/http/auth is exactly the dependency this package must +// not have. A spec in the endpoints package, which can see both, holds the two +// from drifting apart. +const ConnectPath = "/api/cluster/connect" + // ErrNotOwner reports that this replica does not hold the tunnel for a node. // // It is a ROUTING fact and nothing else: some other replica may hold that diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index bede4de44380..eb511d3a752c 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -102,6 +102,22 @@ environment: The peer link is served at `/api/cluster/peer` and authenticates with `LOCALAI_REGISTRATION_TOKEN`, the same shared secret workers register with. Replicas that disagree about it cannot link. A replica that stops heartbeating for 30 seconds is dropped from the table by the others, along with the worker-connection rows it owned. +### Worker tunnels + +A worker can open one long-lived, multiplexed tunnel to the frontend instead of listening on a port of its own. It dials `GET /api/cluster/connect?id=`, the connection is upgraded to a WebSocket, and every subsequent request the frontend makes to that worker travels as a stream inside it. Nothing dials *into* the worker, so a worker behind NAT, in another Kubernetes cluster or on a laptop needs no inbound port and no reachable address. + +The dial is authenticated against **that node's own stored token**, not the deployment-wide registration token: the frontend hashes the presented bearer token and compares it with the hash recorded on the node's row at registration. A worker that presents a token belonging to no node, or names a node ID the frontend has never seen, is refused with `401` before the WebSocket upgrade happens. A frontend that cannot read its node table answers `500` rather than `401`, so a worker retries instead of re-registering under a new identity. + +One honest caveat about that today: a worker currently registers by presenting the deployment's registration token, so the hash stored on its row *is* that token's hash, and a leaked registration token plus a known node ID still gets a tunnel. What the tunnel endpoint does not do is trust the configured token directly, so when workers are issued their own secrets at registration the isolation becomes real with no change to this route. + +The tunnel lands on exactly one frontend replica, and that replica records itself as the owner of the worker's connection in the `node_connections` table. When the socket dies the claim is dropped with it. If the replica stalls long enough for its peers to reap it, it re-claims every tunnel it still holds as soon as it re-registers. + +| Method | Path | Description | +|--------|------|-------------| +| `GET` | `/api/cluster/connect?id=` | Worker opens its multiplexed tunnel (`Authorization: Bearer `) | + +The route is exempt from the normal session/API-key authentication (it authenticates itself, like `/api/cluster/peer`) and is registered in every deployment. Outside distributed mode there is no node table to check a token against, so it answers `503`. + ### The model load deadline scales with the checkpoint The `LoadModel` deadline starts *after* the backend is installed and the model files are staged, so it covers only the worker backend's own checkpoint read and pipeline init. That work is proportional to the bytes on disk, which makes any fixed deadline a model-size cliff rather than a timeout: a 70 GB video checkpoint on a Jetson Thor worker failed reproducibly against the old fixed 5m default (`rpc error: code = DeadlineExceeded` after 953.5s of wall clock, roughly 11m of which was backend install and staging), and simply raising the constant would only move the cliff to the next larger model while making a genuinely wedged *small* model hang for the whole inflated duration. @@ -533,6 +549,8 @@ Used by workers themselves (registration, heartbeat, etc.). Authenticated via th | `GET` | `/api/node/:id/models` | Query own loaded models | | `DELETE` | `/api/node/:id` | Deregister self | +The worker tunnel at `GET /api/cluster/connect` is also worker-facing but is authenticated differently: against the node's own stored token rather than the shared registration token. See [Worker tunnels](#worker-tunnels). + ### `/api/nodes/` - Admin management Used by the WebUI and admin API consumers. Requires admin authentication. From 7deda4a2115e0c965831a16f53cf7ffe196f1be5 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 10:36:10 +0000 Subject: [PATCH 046/153] fix(cluster): harden the worker tunnel, and stop starting a database per spec Review follow-up. Twelve findings, none blocking, grouped here by what they protect. Panics. The handler now recovers between the WebSocket upgrade and the hand-off, the way the peer link next door already did: net/http recovers the panic but leaves the hijacked socket open, so without this a worker keeps a session this replica has no entry for and will never detach. The claim gate in Attach and reclaimOne is now released with defer, so a panic under Claim cannot wedge one node's gate for the life of the process. SetTunnels gained the nil-receiver guard its sibling Stop has. Operability. A deployment with no registration token stores an empty token_hash on every worker, so every tunnel dial 401s forever on a frontend that looks correctly configured. That now warns at startup, logs its own line rather than sharing the "wrong token" one, and is stated in the docs together with the fact that setting the token later needs the workers to register again. Authorization. A node still awaiting admin approval is refused with 403. The rest of /api/node/ gates on nothing, but the two places that hand a node something durable, its API key and its NATS credential, both refuse a pending one, and a tunnel is that kind of grant. Draining and unhealthy nodes keep their tunnels on purpose. Comments that claimed more than the code. The global auth middleware does run on this path and then declines to reject; the future per-node secret only lands without a change here if it lands in TokenHash; the empty-hash guard is defensive rather than deciding; ClusterPathPrefix is no longer only replica-to-replica; the docs no longer say a reaped replica re-claims unconditionally. And the test harness. SetupTestDB started a PostgreSQL container per BeforeEach with a readiness deadline it asserted on, which is one chance per spec to fail one spec inside its setup, anywhere, never twice in the same place: the shape of the flake seen twice here and never reproduced. It now starts one container per process and creates a database per call, which is the pattern tests/e2e already proved. Isolation is unchanged and is now asserted for the first time. All 69 call sites are untouched; the eleven consumer packages run 1404 specs green, and jobs went from 34.3s to 3.3s, agents from 13.8s to 1.9s, cluster from 97.4s to 37.5s. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/app.go | 13 ++ core/http/auth/public_routes.go | 11 +- core/http/endpoints/cluster/connect.go | 87 +++++++++- core/http/endpoints/cluster/connect_test.go | 89 ++++++++++ core/services/cluster/membership.go | 8 + core/services/cluster/tunnel.go | 99 +++++++---- core/services/cluster/tunnel_test.go | 75 ++++++++ core/services/testutil/testdb.go | 183 ++++++++++++++++++-- core/services/testutil/testdb_test.go | 52 ++++++ docs/content/features/distributed-mode.md | 10 +- 10 files changed, 559 insertions(+), 68 deletions(-) create mode 100644 core/services/testutil/testdb_test.go diff --git a/core/http/app.go b/core/http/app.go index 28a9bd19d9eb..21592883f942 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -576,6 +576,19 @@ func API(application *application.Application) (*echo.Echo, error) { var tunnels *clustersvc.TunnelRegistry if d := application.Distributed(); d != nil { tunnels = d.Tunnels + if distCfg.RegistrationToken == "" { + // A separate warning from the peer link's, for the same missing + // knob, because the broken thing is different and an operator has + // to be told both. A worker registering against a frontend with no + // registration token sends no token, so RegisterNodeEndpoint stores + // an empty token_hash, and the tunnel authenticates against exactly + // that column: every dial 401s, forever, on a deployment that looks + // correctly configured. Fixing it needs the token set AND the + // workers re-registered, which is why it is worth saying at boot + // rather than leaving in the 401s. + xlog.Warn("Worker tunnels will refuse every dial: no registration token is configured, so no worker has a stored token to authenticate against", + "route", clustersvc.ConnectPath, "knob", "LOCALAI_REGISTRATION_TOKEN") + } } routes.RegisterWorkerTunnelRoute(e, registry, tunnels) diff --git a/core/http/auth/public_routes.go b/core/http/auth/public_routes.go index 98ef48672f8b..8d85697ccdab 100644 --- a/core/http/auth/public_routes.go +++ b/core/http/auth/public_routes.go @@ -80,9 +80,14 @@ func isPublicRoute(method, path string) bool { return false } -// ClusterPathPrefix is the replica-to-replica namespace. Its handlers check the -// cluster token in the Authorization header themselves, so the check below lets -// them through the global session middleware. +// ClusterPathPrefix is the machine-to-machine cluster namespace. It carries two +// different trust relationships, on two different credentials: the +// replica-to-replica peer link, which checks the shared cluster token, and the +// worker-to-frontend tunnel, which checks the dialing node's own stored token +// hash. What they have in common is the only thing this prefix asserts, that +// each handler checks its own Authorization header, so the check below lets them +// through the global session middleware rather than rejecting a caller that has +// no session and no user. // // The cluster routes do NOT derive their paths from this constant: they are // registered from core/services/cluster's own literal, because that package diff --git a/core/http/endpoints/cluster/connect.go b/core/http/endpoints/cluster/connect.go index 85c48b05277c..e76450692fd9 100644 --- a/core/http/endpoints/cluster/connect.go +++ b/core/http/endpoints/cluster/connect.go @@ -37,10 +37,16 @@ import ( // panic; see the 503 below. // // It is deliberately absent from auth.RouteFeatureRegistry. That registry gates -// a route on the FEATURES OF AN AUTHENTICATED USER, and there is no user here: -// the dialer is a worker process holding a machine credential, and the global -// auth middleware never runs on this path because it sits under -// auth.ClusterPathPrefix. +// a route on the FEATURES OF AN AUTHENTICATED USER, resolved from auth.GetUser, +// and there is no user here: the dialer is a worker process holding a machine +// credential. +// +// The global auth middleware does RUN on this path; what it does not do is +// reject. It attempts session, bearer and legacy-key authentication first, so it +// may even have set auth_user from a worker token that happens to match an API +// key, and then core/http/auth/middleware.go:90 lets the request through +// because usesAlternativeAuthentication reports the path as one whose +// credentials its own route checks. Nothing here reads what it set. func ConnectHandler(registry *nodes.NodeRegistry, tunnels *clustersvc.TunnelRegistry) echo.HandlerFunc { // gorilla's default CheckOrigin restricts a browser to same-origin and lets // a header-less client (which every worker is) through, so the zero value @@ -62,6 +68,14 @@ func ConnectHandler(registry *nodes.NodeRegistry, tunnels *clustersvc.TunnelRegi // answering "unauthorized" would send the operator hunting a token // problem that does not exist. It is checked after the header so that // an anonymous dial still gets the 401 the coverage test requires. + // + // Only the registry half is covered by a spec. The two are read from one + // application.Distributed() in core/http/app.go and initDistributed + // returns an error rather than a partial struct, so a non-nil registry + // beside a nil tunnel registry is unreachable and no spec constructs it; + // the second half is defence against a future wiring that splits them, + // where the cost would be a nil dereference in Attach after the + // connection is already hijacked. if registry == nil || tunnels == nil { return echo.NewHTTPError(http.StatusServiceUnavailable, "distributed mode not enabled") } @@ -89,11 +103,42 @@ func ConnectHandler(registry *nodes.NodeRegistry, tunnels *clustersvc.TunnelRegi return echo.NewHTTPError(http.StatusInternalServerError, "node lookup failed") } + // Split from the mismatch below because they are different operator + // problems with different fixes. An empty stored hash means the node + // registered against a frontend with no LOCALAI_REGISTRATION_TOKEN, so + // no worker on that deployment can ever tunnel until one is set and the + // workers re-register; a mismatch means one worker holds the wrong + // secret. One log line for both leaves an operator reading "wrong token" + // while every worker fails identically. + if node.TokenHash == "" { + xlog.Warn("Refusing a worker tunnel: this node has no stored token, which means it registered with no registration token configured", + "node", nodeID, "knob", "LOCALAI_REGISTRATION_TOKEN") + return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") + } if !authorizedWorker(token, node.TokenHash) { xlog.Debug("worker tunnel dial presented the wrong token", "node", nodeID) return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") } + // Authenticated but not authorised, so 403 rather than 401: the fix is an + // admin approving the node, not a different credential, and answering + // 401 would send an operator looking at tokens. + // + // Only StatusPending is refused. The rest of /api/node/ self-service + // gates on nothing at all, but the two places that hand a node something + // DURABLE both refuse a pending one: the agent worker's API key + // (core/http/endpoints/localai/nodes.go:224) and its NATS credential + // (nodes.go:293). A tunnel is that kind of grant, not a heartbeat: it is + // a standing pipe into the worker recorded in node_connections and + // relayed to by every other replica. Draining and unhealthy nodes keep + // their tunnels on purpose; draining means finish what you have, and a + // node marked unhealthy for missed heartbeats needs the pipe to recover + // through. + if node.Status == nodes.StatusPending { + xlog.Warn("Refusing a worker tunnel: this node is awaiting admin approval", "node", nodeID) + return echo.NewHTTPError(http.StatusForbidden, "node is pending approval") + } + ws, err := upgrader.Upgrade(c.Response(), c.Request(), nil) if err != nil { // Upgrade has already written its own failure to the client. @@ -108,6 +153,23 @@ func ConnectHandler(registry *nodes.NodeRegistry, tunnels *clustersvc.TunnelRegi return nil } + // The same guard PeerHandler carries, for the same reason and one more. + // net/http recovers a panic from this goroutine but does not close the + // hijacked connection, and middleware.Recover does not either, so a + // panic below would leave the worker holding a live session this replica + // has no entry for and will never detach. The extra reason here is that + // Attach does database work: a panic inside it, with the session left + // open, is a tunnel nothing can reach and nothing will clean up. + // + // It re-panics rather than swallowing. Whatever it caught is a bug, and + // the recovery middleware above is what should report it. + defer func() { + if r := recover(); r != nil { + _ = sess.Close() + panic(r) + } + }() + // From here the connection is hijacked, so no status can reach the // worker any more: a failure is a closed socket, which is what its // reconnect loop reads. @@ -176,12 +238,19 @@ func bearerToken(r *http.Request) (string, bool) { // attacker knows. What this does rule out is the shortcut of comparing against // the configured token itself, which is what would have to be unpicked later; // the day a worker is issued its own secret at registration, this check starts -// isolating workers from each other with no change here. +// isolating workers from each other with no change here, PROVIDED the secret +// lands in TokenHash. The one per-node secret LocalAI mints today, the agent +// worker's api_token, does not: provisionAgentWorkerKey writes an auth.User and +// an auth.APIKey referenced by node.AuthUserID / node.APIKeyID and never touches +// this column. If the next task follows that precedent instead, this comparison +// is what has to change. // -// An empty stored hash authorizes nobody. Such a row exists whenever a worker -// registered against a frontend with no registration token configured, and the -// alternative, letting any token through for those, would make the check depend -// on a setting the dialer can neither see nor be blamed for. +// The empty-hash guard is defensive rather than deciding: a stored hash is +// hex-encoded SHA-256, so 64 bytes or nothing, and ConstantTimeCompare already +// returns 0 on a length mismatch (crypto/internal/fips140/subtle/constant_time.go:17-20 +// returns 0 outright when the lengths differ). It is kept because a reader should not have to +// derive "an unregistered node authorizes nobody" from a length rule, and +// because the caller logs that case separately. func authorizedWorker(token, storedHash string) bool { if storedHash == "" { return false diff --git a/core/http/endpoints/cluster/connect_test.go b/core/http/endpoints/cluster/connect_test.go index 7236199171f0..072f69d0226c 100644 --- a/core/http/endpoints/cluster/connect_test.go +++ b/core/http/endpoints/cluster/connect_test.go @@ -194,6 +194,45 @@ var _ = Describe("Worker tunnel handler", func() { "the claim outlived the socket, so this replica keeps being named the owner of a worker it no longer holds") }) + It("refuses a node whose row carries no stored token", func() { + // A deployment with no registration token configured produces exactly + // these rows: the worker sends no token, so registration stores none, + // and there is nothing here to authenticate against. + Expect(db.Exec(`UPDATE backend_nodes SET token_hash = '' WHERE id = ?`, nodeID).Error).To(Succeed()) + + _, resp, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), bearer(workerToken)) + Expect(err).To(HaveOccurred()) + Expect(resp).ToNot(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + Expect(tun.Held()).To(BeEmpty()) + }) + + It("refuses a node that is still awaiting admin approval", func() { + // Approval is what gates a node's participation, and a tunnel is a + // standing pipe recorded in node_connections, not a heartbeat. 403 and + // not 401: the credential is right, the authorisation is missing, and + // the fix is an admin rather than a different token. + Expect(db.Exec(`UPDATE backend_nodes SET status = ? WHERE id = ?`, nodes.StatusPending, nodeID).Error).To(Succeed()) + + _, resp, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), bearer(workerToken)) + Expect(err).To(HaveOccurred()) + Expect(resp).ToNot(BeNil()) + Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) + Expect(tun.Held()).To(BeEmpty()) + }) + + It("still admits a draining node, which has work to finish", func() { + // Only pending is refused. Draining means "start nothing new", not + // "lose the pipe your in-flight requests travel on", and a node marked + // unhealthy for missed heartbeats needs the tunnel to recover through. + Expect(db.Exec(`UPDATE backend_nodes SET status = ? WHERE id = ?`, nodes.StatusDraining, nodeID).Error).To(Succeed()) + + conn, _, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), bearer(workerToken)) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + Eventually(tun.Held, "10s").Should(ConsistOf(nodeID)) + }) + It("reports a lookup failure as a failure, not as a refusal", func() { // ErrNotOwner, 401 and 404 are all ANSWERS. A database that cannot be // read is none of them: telling a worker its credentials are wrong when @@ -208,6 +247,56 @@ var _ = Describe("Worker tunnel handler", func() { }) }) +var _ = Describe("Worker tunnel handler when the attach panics", func() { + // net/http recovers a panic from the request goroutine but does NOT close a + // hijacked connection, so without the handler's own recover the worker keeps + // a live session this replica has no entry for and will never detach: its + // opens fill yamux's 256-deep backlog and then hang with no error. The panic + // is injected through a real path rather than a fake one, a registry built + // over no database at all, which is what Attach's first database call + // dereferences. + var ( + srv *httptest.Server + db *gorm.DB + nodeID string + ) + + BeforeEach(func() { + ctx := context.Background() + db = testutil.SetupTestDB() + nodeReg, err := nodes.NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + node := &nodes.BackendNode{Name: "worker-1", Address: "10.0.0.9:50051", TokenHash: tokenHash(workerToken)} + Expect(nodeReg.Register(ctx, node, true)).To(Succeed()) + nodeID = node.ID + + e := echo.New() + routes.RegisterWorkerTunnelRoute(e, nodeReg, clustersvc.NewTunnelRegistry(nil, "me")) + srv = httptest.NewServer(e) + DeferCleanup(func() { + // A hijacked connection the handler never closed would park Close + // forever, turning the assertion below into a suite hang. + srv.CloseClientConnections() + srv.Close() + }) + }) + + It("closes the worker's session instead of stranding it", func() { + conn, _, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), bearer(workerToken)) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + + workerSess, err := yamux.Client(clustersvc.WebsocketConn(conn), nil, nil) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = workerSess.Close() }) + + // Asserting on OpenStream would hang rather than fail: yamux only + // acknowledges a stream once the peer accepts it, and the leak this + // pins is precisely that nobody ever will. + Eventually(workerSess.IsClosed, "10s").Should(BeTrue()) + }) +}) + var _ = Describe("Worker tunnel handler without distributed mode", func() { // The route is registered in every deployment so that the route-coverage // test sees it, which is what pins the reject-before-upgrade rule. With no diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index 4a5f3e19ccab..6a8650927b15 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -81,7 +81,15 @@ func NewMembership(reg *Registry, id, addr, version string) *Membership { // It is a setter rather than a constructor argument because the tunnel registry // is what the tunnel endpoint is built on, and that is wired after membership // is already running. +// +// Safe on a nil receiver, like Stop. This package deliberately produces a nil +// *Membership (core/application/distributed.go leaves it nil when no +// peer-reachable address can be derived), so a setter that panicked on one +// would be a trap for the next caller rather than an impossibility. func (m *Membership) SetTunnels(t *TunnelRegistry) { + if m == nil { + return + } m.mu.Lock() defer m.mu.Unlock() m.tunnels = t diff --git a/core/services/cluster/tunnel.go b/core/services/cluster/tunnel.go index 66baa475fdce..65adc4aad4ca 100644 --- a/core/services/cluster/tunnel.go +++ b/core/services/cluster/tunnel.go @@ -182,18 +182,33 @@ func (t *TunnelRegistry) Attach(ctx context.Context, nodeID string, sess *yamux. return 0, fmt.Errorf("attaching tunnel for node %q: %w", nodeID, err) } - epoch, err := t.reg.Claim(ctx, nodeID, t.selfID) + // The gated part is a closure so its release can be DEFERRED while the + // session close below still happens outside the gate. Releasing on each + // return path instead leaves one way out uncovered: a panic. Claim does + // database work, and a panic anywhere under it would leave this node's gate + // closed for the life of the process, so every later Attach or Reclaim for + // that worker would block in enterClaim until its own context expired. The + // caller's recover would report the panic and the worker would look + // permanently unable to reconnect, with nothing linking the two. + var previous *heldTunnel + epoch, err := func() (int64, error) { + defer t.leaveClaim(nodeID) + + epoch, err := t.reg.Claim(ctx, nodeID, t.selfID) + if err != nil { + return 0, err + } + + t.mu.Lock() + previous = t.tunnels[nodeID] + t.tunnels[nodeID] = &heldTunnel{sess: sess, token: epoch, claim: epoch} + t.mu.Unlock() + return epoch, nil + }() if err != nil { - t.leaveClaim(nodeID) return 0, err } - t.mu.Lock() - previous := t.tunnels[nodeID] - t.tunnels[nodeID] = &heldTunnel{sess: sess, token: epoch, claim: epoch} - t.mu.Unlock() - t.leaveClaim(nodeID) - // Closed after the gate is released, not under it. The gate is justified by // being held for one claim round trip, and closing a session is not that: // yamux closes the underlying conn and then waits for both its send and @@ -381,38 +396,48 @@ func (t *TunnelRegistry) reclaimOne(ctx context.Context, nodeID string) error { return fmt.Errorf("re-claiming node %q: %w", nodeID, err) } - t.mu.Lock() - tunnel, ok := t.tunnels[nodeID] - t.mu.Unlock() - if !ok { - // Detached between Reclaim listing the nodes and this gate. Nothing was - // claimed, so there is nothing to undo. - t.leaveClaim(nodeID) - return errTunnelNotReclaimed - } - if tunnel.sess.IsClosed() { - xlog.Debug("skipping re-claim of a worker tunnel whose session is closed", "node", nodeID) - t.leaveClaim(nodeID) - return errTunnelNotReclaimed - } + // Gated part in a closure so the release is DEFERRED, for the reason Attach + // gives: a panic under Claim would otherwise wedge this node's gate for the + // life of the process. The trailing Release still runs outside the gate. + var epoch int64 + var installed bool + if err := func() error { + defer t.leaveClaim(nodeID) - epoch, err := t.reg.Claim(ctx, nodeID, t.selfID) - if err != nil { - t.leaveClaim(nodeID) - return err - } + t.mu.Lock() + tunnel, ok := t.tunnels[nodeID] + t.mu.Unlock() + if !ok { + // Detached between Reclaim listing the nodes and this gate. Nothing + // was claimed, so there is nothing to undo. + return errTunnelNotReclaimed + } + if tunnel.sess.IsClosed() { + xlog.Debug("skipping re-claim of a worker tunnel whose session is closed", "node", nodeID) + return errTunnelNotReclaimed + } - t.mu.Lock() - current, installed := t.tunnels[nodeID] - if installed { - // current is necessarily the entry read above: the gate is still held, - // and Attach and reclaimOne are the only writers that install one. The - // identity is therefore not re-checked; the case that IS reachable is - // the entry being gone, because Detach is not gated. - current.claim = epoch + var err error + epoch, err = t.reg.Claim(ctx, nodeID, t.selfID) + if err != nil { + return err + } + + t.mu.Lock() + current, present := t.tunnels[nodeID] + installed = present + if installed { + // current is necessarily the entry read above: the gate is still + // held, and Attach and reclaimOne are the only writers that install + // one. The identity is therefore not re-checked; the case that IS + // reachable is the entry being gone, because Detach is not gated. + current.claim = epoch + } + t.mu.Unlock() + return nil + }(); err != nil { + return err } - t.mu.Unlock() - t.leaveClaim(nodeID) if installed { return nil diff --git a/core/services/cluster/tunnel_test.go b/core/services/cluster/tunnel_test.go index 67993130ee54..acc81d81813f 100644 --- a/core/services/cluster/tunnel_test.go +++ b/core/services/cluster/tunnel_test.go @@ -716,3 +716,78 @@ var _ = Describe("Re-claiming tunnels after this replica's rows were reaped", fu Consistently(stored, 2*cluster.InstanceHeartbeat, time.Second).Should(Equal(epoch)) }) }) + +var _ = Describe("The worker tunnel registry's claim gate", func() { + // These specs need no database. They pin what happens when the database + // call under the gate does not return normally, which is the one exit a + // release-on-every-return-path cannot cover. + // + // The panic is produced by the production code itself: a registry with no + // *Registry behind it dereferences nothing at the gate and then panics + // inside Claim, which is exactly where a real one does its work. + var tun *cluster.TunnelRegistry + + BeforeEach(func() { + tun = cluster.NewTunnelRegistry(nil, "me") + }) + + // attachPanics runs one Attach that is expected to panic, swallowing the + // panic so the spec can go on to ask what state it left behind. + attachPanics := func(ctx context.Context, nodeID string) { + defer GinkgoRecover() + defer func() { _ = recover() }() + frontend, _ := workerTunnel() + _, _ = tun.Attach(ctx, nodeID, frontend) + Fail("Attach was expected to panic inside Claim, so this spec is no longer testing what it claims") + } + + It("frees the node's gate when the claim panics", func() { + attachPanics(context.Background(), "w1") + + // A wedged gate is indistinguishable from a slow one except by waiting, + // so the second attempt is given a deadline. Reaching Claim means + // panicking again; returning a context error means it never got past + // enterClaim and this worker could never reconnect to this replica. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + reached := make(chan any, 1) + go func() { + defer GinkgoRecover() + defer func() { reached <- recover() }() + frontend, _ := workerTunnel() + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).To(MatchError(context.DeadlineExceeded), + "the gate for this node was never released, so every later dial from it blocks until its own context expires") + }() + Eventually(reached, "5s").Should(Receive(Not(BeNil())), + "the second Attach did not reach Claim, so the panicking one left the gate closed") + }) + + It("leaves another node's gate alone", func() { + // The gate is per node so that one wedged worker cannot stop the rest; + // this holds that property against the panic path too. + attachPanics(context.Background(), "w1") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + reached := make(chan any, 1) + go func() { + defer GinkgoRecover() + defer func() { reached <- recover() }() + frontend, _ := workerTunnel() + _, _ = tun.Attach(ctx, "w2", frontend) + }() + Eventually(reached, "5s").Should(Receive(Not(BeNil()))) + }) +}) + +var _ = Describe("Membership.SetTunnels", func() { + It("is safe on a nil receiver, like Stop", func() { + // A nil *Membership is a value this codebase deliberately produces when + // no peer-reachable address can be derived, so the asymmetry with Stop + // would be a trap for the next caller. + var m *cluster.Membership + Expect(func() { m.SetTunnels(cluster.NewTunnelRegistry(nil, "me")) }).ToNot(Panic()) + }) +}) diff --git a/core/services/testutil/testdb.go b/core/services/testutil/testdb.go index 80e511201b7d..755589cd438c 100644 --- a/core/services/testutil/testdb.go +++ b/core/services/testutil/testdb.go @@ -2,7 +2,11 @@ package testutil import ( "context" + "fmt" + "net/url" "runtime" + "sync" + "sync/atomic" "time" "github.com/testcontainers/testcontainers-go" @@ -16,27 +20,176 @@ import ( . "github.com/onsi/gomega" ) -// SetupTestDB creates a fresh PostgreSQL 16 container and returns a gorm.DB. -// The container is cleaned up via DeferCleanup when the test completes. +// One PostgreSQL container per test PROCESS, not per spec, with a database per +// SetupTestDB call. +// +// Starting a container per spec was both slow and flaky. Slow because a +// postgres:16 start is seconds and the packages behind this helper hold several +// hundred specs; flaky because every start was a fresh chance to miss the +// readiness deadline, and a miss lands in the caller's BeforeEach as a failure +// of whichever spec happened to be running. That is the exact shape of the +// intermittent single-spec failure seen twice in this package and never +// reproduced: one spec of many, no pattern, never twice in the same place. +// Starting the container once per process leaves one chance to miss it instead +// of one per spec, and moves that chance onto a deadline that only has to be met +// while nothing else is competing for the machine. +// +// Isolation is unchanged and is what callers actually depend on: each call still +// hands back an empty database that no other spec can see. The database is +// dropped when the spec that asked for it ends. Advisory locks, sequences and +// extensions are all per-database in PostgreSQL, so nothing the packages behind +// this helper rely on leaks between specs. +// +// This mirrors the pattern already proven in tests/e2e/distributed +// (testhelpers_test.go), which is where the argument and the measurements come +// from. +// +// One container per process rather than one shared across `ginkgo -p` workers is +// deliberate: parallel Ginkgo processes are separate OS processes, each gets its +// own container, and nothing has to coordinate database names across them. +var ( + sharedOnce sync.Once + sharedPG *tcpostgres.PostgresContainer + sharedDSN string + sharedErr error + + // dbCounter makes each database name unique within this process. The + // container is not shared across processes, so a process-local counter is + // enough. + dbCounter atomic.Int64 +) + +// The container outlives every spec, so its teardown belongs to the suite. This +// registers one AfterSuite in every suite that imports this package, which is +// every suite that could have started a container; it is a no-op in the ones +// that never call SetupTestDB. +// +// Package-level rather than something callers have to remember: a helper whose +// cleanup depends on 56 test files each declaring a hook is a helper that leaks +// containers the first time someone forgets. Registration happens during package +// initialisation, which is before RunSpecs, so Ginkgo is still building its tree. +var _ = AfterSuite(func() { + if sharedPG == nil { + return + } + // Best-effort: a failed terminate must not fail a suite whose specs all + // passed. Testcontainers' reaper removes it in that case. + _ = sharedPG.Terminate(context.Background()) +}) + +// sharedPostgres returns the DSN of this process's PostgreSQL container, +// starting it on first use. +// +// The error is remembered rather than only asserted inside the sync.Once: an +// assertion there fails the one spec that happened to be first, and every later +// spec would then find a nil container and fail for some unrelated-looking +// reason. Re-asserting the stored error makes every affected spec say the same +// true thing. +func sharedPostgres() string { + GinkgoHelper() + + sharedOnce.Do(func() { + ctx := context.Background() + sharedPG, sharedErr = tcpostgres.Run(ctx, "postgres:16", + tcpostgres.WithDatabase("testdb"), + tcpostgres.WithUsername("test"), + tcpostgres.WithPassword("test"), + // The deadline is per process now, not per spec, so it is generous + // on purpose: it is paid once, and the cost of missing it is a + // whole suite rather than one spec. + testcontainers.WithWaitStrategyAndDeadline(120*time.Second, + wait.ForLog("database system is ready to accept connections").WithOccurrence(2)), + ) + if sharedErr != nil { + return + } + sharedDSN, sharedErr = sharedPG.ConnectionString(ctx, "sslmode=disable") + }) + + Expect(sharedErr).ToNot(HaveOccurred(), "the suite's PostgreSQL container could not be started") + return sharedDSN +} + +// SetupTestDB returns a gorm.DB on a PostgreSQL database created for the calling +// spec. The database is dropped, and its connection pool closed, when the spec +// ends. func SetupTestDB() *gorm.DB { + GinkgoHelper() if runtime.GOOS == "darwin" { Skip("testcontainers requires Docker, not available on macOS CI") } - ctx := context.Background() - pgC, err := tcpostgres.Run(ctx, "postgres:16", - tcpostgres.WithDatabase("testdb"), - tcpostgres.WithUsername("test"), - tcpostgres.WithPassword("test"), - testcontainers.WithWaitStrategyAndDeadline(60*time.Second, - wait.ForLog("database system is ready to accept connections").WithOccurrence(2)), - ) - Expect(err).ToNot(HaveOccurred()) - DeferCleanup(func() { pgC.Terminate(context.Background()) }) - connStr, err := pgC.ConnectionString(ctx, "sslmode=disable") - Expect(err).ToNot(HaveOccurred()) - db, err := gorm.Open(postgres.Open(connStr), &gorm.Config{ + + dsn := sharedPostgres() + name := fmt.Sprintf("testdb_%d", dbCounter.Add(1)) + + // Scoped so a failed CREATE cannot leak the pool: the assertion panics out + // of this function, and a leaked pool per failing spec exhausts the + // server's connection limit for every spec after it. + // + // CREATE and DROP DATABASE cannot run against the target database itself, + // so both go through a short-lived connection to the container's own + // maintenance database. + func() { + admin := openPool(dsn) + defer closePool(admin) + Expect(admin.Exec(fmt.Sprintf("CREATE DATABASE %q", name)).Error).To(Succeed()) + }() + + db, err := gorm.Open(postgres.Open(replaceDBName(dsn, name)), &gorm.Config{ Logger: logger.Default.LogMode(logger.Silent), }) Expect(err).ToNot(HaveOccurred()) + + DeferCleanup(func() { + // The caller's own DeferCleanups were registered later and so run + // first, which is what lets a spec keep using this handle in its + // teardown. + closePool(db) + + drop, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Discard}) + if err != nil { + // Reported, never asserted. A cleanup that fails the spec turns one + // database hiccup into a failure that buries whatever the spec was + // actually about. + AddReportEntry("drop test database skipped", fmt.Sprintf("%s: %v", name, err)) + return + } + defer closePool(drop) + // FORCE terminates whatever connections the spec left open, including + // any a background goroutine is still holding (PostgreSQL 13+). + if err := drop.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %q WITH (FORCE)", name)).Error; err != nil { + AddReportEntry("drop test database failed", fmt.Sprintf("%s: %v", name, err)) + } + }) + + return db +} + +// openPool connects to dsn with logging off. Used for the short-lived +// maintenance connections only; the database a spec is handed keeps gorm's +// silent logger so a caller can still swap it. +func openPool(dsn string) *gorm.DB { + GinkgoHelper() + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Discard}) + Expect(err).ToNot(HaveOccurred()) return db } + +func closePool(db *gorm.DB) { + if db == nil { + return + } + if sqlDB, err := db.DB(); err == nil { + _ = sqlDB.Close() + } +} + +// replaceDBName swaps the database component of a DSN, preserving credentials, +// host, port and query parameters. +func replaceDBName(dsn, name string) string { + GinkgoHelper() + u, err := url.Parse(dsn) + Expect(err).ToNot(HaveOccurred()) + u.Path = "/" + name + return u.String() +} diff --git a/core/services/testutil/testdb_test.go b/core/services/testutil/testdb_test.go new file mode 100644 index 000000000000..d9dd173eb20f --- /dev/null +++ b/core/services/testutil/testdb_test.go @@ -0,0 +1,52 @@ +package testutil_test + +import ( + "testing" + + "github.com/mudler/LocalAI/core/services/testutil" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestTestutil(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Test Utilities Suite") +} + +// The container is shared per process now, so the isolation callers depend on +// comes from a database per call rather than from a server per call. That is +// the property 69 call sites across eleven packages assume without saying so, +// and nothing else in the tree asserts it. +var _ = Describe("SetupTestDB", func() { + type row struct { + ID int + } + + It("hands back a database no other caller can see into", func() { + first := testutil.SetupTestDB() + second := testutil.SetupTestDB() + + Expect(first.Exec(`CREATE TABLE isolation_probe (id int)`).Error).To(Succeed()) + Expect(first.Exec(`INSERT INTO isolation_probe VALUES (1)`).Error).To(Succeed()) + + var found []row + err := second.Raw(`SELECT id FROM isolation_probe`).Scan(&found).Error + Expect(err).To(HaveOccurred(), + "two SetupTestDB calls landed on the same database, so every spec can now see every other spec's rows") + + // The second database must also be usable, not merely different: an + // isolation check that passed because the second handle was broken + // would prove nothing. + Expect(second.Exec(`CREATE TABLE isolation_probe (id int)`).Error).To(Succeed()) + }) + + It("hands back an empty database", func() { + db := testutil.SetupTestDB() + var tables int64 + Expect(db.Raw( + `SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public'`, + ).Scan(&tables).Error).To(Succeed()) + Expect(tables).To(BeZero(), "a spec was handed a database another spec had already migrated") + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index eb511d3a752c..59cf9ebfc48c 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -106,15 +106,17 @@ The peer link is served at `/api/cluster/peer` and authenticates with `LOCALAI_R A worker can open one long-lived, multiplexed tunnel to the frontend instead of listening on a port of its own. It dials `GET /api/cluster/connect?id=`, the connection is upgraded to a WebSocket, and every subsequent request the frontend makes to that worker travels as a stream inside it. Nothing dials *into* the worker, so a worker behind NAT, in another Kubernetes cluster or on a laptop needs no inbound port and no reachable address. -The dial is authenticated against **that node's own stored token**, not the deployment-wide registration token: the frontend hashes the presented bearer token and compares it with the hash recorded on the node's row at registration. A worker that presents a token belonging to no node, or names a node ID the frontend has never seen, is refused with `401` before the WebSocket upgrade happens. A frontend that cannot read its node table answers `500` rather than `401`, so a worker retries instead of re-registering under a new identity. +The dial is authenticated against **the hash stored on that node's row, which today is still the registration token's hash**: the frontend hashes the presented bearer token and compares it with what registration recorded. A worker that presents a token belonging to no node, or names a node ID the frontend has never seen, is refused with `401` before the WebSocket upgrade happens. A node still awaiting admin approval is refused with `403`. A frontend that cannot read its node table answers `500` rather than `401`, so a worker retries instead of re-registering under a new identity. -One honest caveat about that today: a worker currently registers by presenting the deployment's registration token, so the hash stored on its row *is* that token's hash, and a leaked registration token plus a known node ID still gets a tunnel. What the tunnel endpoint does not do is trust the configured token directly, so when workers are issued their own secrets at registration the isolation becomes real with no change to this route. +So the isolation is not there yet, and the caveat is worth stating plainly: because a worker registers by presenting the deployment's registration token, a leaked registration token plus a known node ID still gets a tunnel. What the tunnel endpoint does not do is trust the configured token directly, so when workers are issued their own per-node secrets and those secrets are what registration stores, the isolation becomes real with no change to this route. -The tunnel lands on exactly one frontend replica, and that replica records itself as the owner of the worker's connection in the `node_connections` table. When the socket dies the claim is dropped with it. If the replica stalls long enough for its peers to reap it, it re-claims every tunnel it still holds as soon as it re-registers. +**Worker tunnels need `LOCALAI_REGISTRATION_TOKEN` set.** A worker registering against a frontend with no registration token configured sends no token, so nothing is stored on its row, and the tunnel has nothing to authenticate it against: every dial is refused with `401`. LocalAI warns about this at startup. Setting the token later is not enough on its own; the workers have to register again for the hash to be written. + +The tunnel lands on exactly one frontend replica, and that replica records itself as the owner of the worker's connection in the `node_connections` table. When the socket dies the claim is dropped with it. If the replica stalls long enough for its peers to reap it, it re-claims the tunnels it still holds on a live session as soon as it re-registers, skipping any whose socket has already gone. That re-claim needs the replica to have an advertised address: without one it never had an instance row to begin with, and its tunnels are usable only by the replica holding them. | Method | Path | Description | |--------|------|-------------| -| `GET` | `/api/cluster/connect?id=` | Worker opens its multiplexed tunnel (`Authorization: Bearer `) | +| `GET` | `/api/cluster/connect?id=` | Worker opens its multiplexed tunnel (`Authorization: Bearer `) | The route is exempt from the normal session/API-key authentication (it authenticates itself, like `/api/cluster/peer`) and is registered in every deployment. Outside distributed mode there is no node table to check a token against, so it answers `503`. From 0762fc09fc6bf79314ded5e43fcf29b00047381d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 11:17:48 +0000 Subject: [PATCH 047/153] fix(testutil): stop the shared-database change from disarming two regressions Two advisory-lock specs named their database by literal, ALTER DATABASE testdb. Once the test helper started handing every spec its own database on a shared server, that statement landed on the maintenance database and did nothing to the one the spec was holding, so both specs went green having never reproduced the condition they exist for. They regress a model-load advisory-lock wedge that has already shipped to production once, so the previous commit's de-flaking silently disarmed a regression test for a real deployed bug. Both sites now read the name back with current_database() and, more importantly, assert the override actually landed before relying on it. A literal name can go stale again; an assertion that the setting is in force cannot pass while it is not. Removing either production override now fails the matching spec with the real 55P03 and 57014 again. That literal also meant every CREATE DATABASE and every DROP ... WITH (FORCE) ran under the 300ms bound it set on the maintenance database, which is a new load-dependent single-spec flake inside the change that was meant to remove one. The helper's maintenance connections now pin one connection and clear both timeouts on it, so no setting a spec makes can bound them, and a white-box spec imposes the leak deliberately and proves it does not reach them. Also pins the reclaimOne gate deferral the previous commit added without a test, by panicking inside the re-claim's own claim statement, and drops the per-dial empty-token log line to debug now that the boot warning says it once. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/cluster/connect.go | 7 ++- .../advisorylock/advisorylock_test.go | 61 ++++++++++++++---- core/services/cluster/tunnel_test.go | 42 +++++++++++++ core/services/testutil/testdb.go | 58 +++++++++++++++-- .../services/testutil/testdb_internal_test.go | 63 +++++++++++++++++++ 5 files changed, 214 insertions(+), 17 deletions(-) create mode 100644 core/services/testutil/testdb_internal_test.go diff --git a/core/http/endpoints/cluster/connect.go b/core/http/endpoints/cluster/connect.go index e76450692fd9..ca94ead48126 100644 --- a/core/http/endpoints/cluster/connect.go +++ b/core/http/endpoints/cluster/connect.go @@ -111,7 +111,12 @@ func ConnectHandler(registry *nodes.NodeRegistry, tunnels *clustersvc.TunnelRegi // secret. One log line for both leaves an operator reading "wrong token" // while every worker fails identically. if node.TokenHash == "" { - xlog.Warn("Refusing a worker tunnel: this node has no stored token, which means it registered with no registration token configured", + // Debug, not Warn. Every worker in such a deployment fails this way + // on every reconnect, so warning per dial buries the log; the fact + // is stated once, at boot, where core/http/app.go warns that no + // registration token is configured. What this line adds is which + // node, for an operator who has already read that warning. + xlog.Debug("refusing a worker tunnel: this node has no stored token, so it registered with no registration token configured", "node", nodeID, "knob", "LOCALAI_REGISTRATION_TOKEN") return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") } diff --git a/core/services/advisorylock/advisorylock_test.go b/core/services/advisorylock/advisorylock_test.go index f1bd3e75ed5c..536666f6cb5e 100644 --- a/core/services/advisorylock/advisorylock_test.go +++ b/core/services/advisorylock/advisorylock_test.go @@ -2,7 +2,9 @@ package advisorylock import ( "context" + "fmt" "runtime" + "strings" "sync" "sync/atomic" "time" @@ -14,6 +16,51 @@ import ( "gorm.io/gorm" ) +// alterThisDatabase applies a server-side setting to the database this handle is +// actually connected to, and proves it landed. +// +// The name is read back from the connection rather than written as a literal. +// The test helper hands each spec its own database on a shared server, so a +// hard-coded name ALTERs a database this handle never touches: the statement +// succeeds, the override does nothing, and the two specs below go green having +// exercised none of the condition they exist for. They regress a model-load +// advisory-lock wedge that has already shipped to production once, so a green +// spec that proves nothing is the worst outcome available here. +// +// The read-back is the guard. Idle connections are dropped first so the next one +// is opened fresh and inherits the new database-level default; SHOW then reports +// what a waiter's own connection would inherit. If that ever stops matching, the +// spec fails here rather than passing for the wrong reason. +func alterThisDatabase(db *gorm.DB, setting, value string) { + GinkgoHelper() + + var name string + Expect(db.Raw("SELECT current_database()").Scan(&name).Error).ToNot(HaveOccurred()) + Expect(name).ToNot(BeEmpty()) + + Expect(db.Exec(fmt.Sprintf("ALTER DATABASE %q SET %s = %s", name, setting, quoteLiteral(value))).Error). + ToNot(HaveOccurred()) + + sqlDB, err := db.DB() + Expect(err).ToNot(HaveOccurred()) + // database/sql retains no idle connections at 0, closing the ones it is + // already holding, so every connection after this point is opened fresh and + // inherits the new database-level default. + sqlDB.SetMaxIdleConns(0) + + var applied string + Expect(db.Raw("SHOW " + setting).Scan(&applied).Error).ToNot(HaveOccurred()) + Expect(applied).To(Equal(value), + "the %s override did not reach the database this spec is holding (%s), so the spec below would pass without ever reproducing the condition it regresses", + setting, name) +} + +// quoteLiteral wraps a settings value as a SQL string literal. The values here +// are spec constants, so this only has to be correct, not hostile-input-proof. +func quoteLiteral(v string) string { + return "'" + strings.ReplaceAll(v, "'", "''") + "'" +} + var _ = Describe("AdvisoryLock", func() { Context("PostgreSQL advisory locks", func() { var db *gorm.DB @@ -166,12 +213,7 @@ var _ = Describe("AdvisoryLock", func() { // blocked on pg_advisory_lock() is aborted by the server after this // window and surfaces SQLSTATE 55P03 ("canceling statement due to // lock timeout") to the caller instead of waiting for its turn. - Expect(db.Exec("ALTER DATABASE testdb SET lock_timeout = '300ms'").Error).ToNot(HaveOccurred()) - sqlDB, err := db.DB() - Expect(err).ToNot(HaveOccurred()) - // Drop pooled connections so subsequent ones reconnect and inherit - // the new database-level lock_timeout default. - sqlDB.SetMaxIdleConns(0) + alterThisDatabase(db, "lock_timeout", "300ms") holding := make(chan struct{}) released := make(chan struct{}) @@ -214,12 +256,7 @@ var _ = Describe("AdvisoryLock", func() { // statement_timeout=60s; a cold model load holds the lock far longer, // so every concurrent caller died with SQLSTATE 57014 ("canceling // statement due to statement timeout") rather than waiting its turn. - Expect(db.Exec("ALTER DATABASE testdb SET statement_timeout = '300ms'").Error).ToNot(HaveOccurred()) - sqlDB, err := db.DB() - Expect(err).ToNot(HaveOccurred()) - // Drop pooled connections so subsequent ones reconnect and inherit - // the new database-level statement_timeout default. - sqlDB.SetMaxIdleConns(0) + alterThisDatabase(db, "statement_timeout", "300ms") holding := make(chan struct{}) released := make(chan struct{}) diff --git a/core/services/cluster/tunnel_test.go b/core/services/cluster/tunnel_test.go index acc81d81813f..7d0136b47e67 100644 --- a/core/services/cluster/tunnel_test.go +++ b/core/services/cluster/tunnel_test.go @@ -715,6 +715,48 @@ var _ = Describe("Re-claiming tunnels after this replica's rows were reaped", fu } Consistently(stored, 2*cluster.InstanceHeartbeat, time.Second).Should(Equal(epoch)) }) + + It("frees the node's gate when a re-claim panics", func() { + // The same property Attach's gate specs pin, on the other function that + // takes the gate. Reclaim runs from the heartbeat loop, so a wedged gate + // here is worse than one wedged by a dial: nothing retries it, and the + // worker can never re-attach to this replica because its Attach blocks + // in enterClaim until its own context expires. + // + // The panic is thrown from inside the re-claim's own Claim statement, + // on that statement's goroutine, using the same gorm Trace hook the + // interleaving specs above use. That is the window a real panic under + // Claim would land in: the row is written and the gate is held. + hook := newClaimHook(isClaimOf("w1")) + hooked := cluster.NewTunnelRegistry( + cluster.NewRegistry(db.Session(&gorm.Session{Logger: hook})), "me") + + frontend, _ := workerTunnel() + _, err := hooked.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + + // Installed after the attach so the hook fires on the RE-claim, not on + // the claim that set the tunnel up. + hook.setAction(func(string) { panic("claim exploded") }) + + panicked := func() (p bool) { + defer func() { p = recover() != nil }() + _, _ = hooked.Reclaim(ctx) + return + }() + Expect(panicked).To(BeTrue(), + "the hook did not fire inside the re-claim, so this spec is no longer testing what it claims") + + // A wedged gate is indistinguishable from a slow one except by waiting. + // The hook has already fired once and will not fire again, so this + // Attach either completes or never reaches Claim at all. + bounded, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + next, _ := workerTunnel() + _, err = hooked.Attach(bounded, "w1", next) + Expect(err).ToNot(HaveOccurred(), + "the panicking re-claim left this node's gate closed, so the worker can never attach to this replica again") + }) }) var _ = Describe("The worker tunnel registry's claim gate", func() { diff --git a/core/services/testutil/testdb.go b/core/services/testutil/testdb.go index 755589cd438c..7e2c061dd8a4 100644 --- a/core/services/testutil/testdb.go +++ b/core/services/testutil/testdb.go @@ -146,7 +146,7 @@ func SetupTestDB() *gorm.DB { // teardown. closePool(db) - drop, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Discard}) + drop, err := openTolerantPool(dsn) if err != nil { // Reported, never asserted. A cleanup that fails the spec turns one // database hiccup into a failure that buries whatever the spec was @@ -165,16 +165,66 @@ func SetupTestDB() *gorm.DB { return db } -// openPool connects to dsn with logging off. Used for the short-lived -// maintenance connections only; the database a spec is handed keeps gorm's -// silent logger so a caller can still swap it. +// openPool connects to dsn with logging off and with every server-side timeout +// disabled on the session. Used for the short-lived maintenance connections +// only; the database a spec is handed keeps gorm's silent logger and the +// server's defaults, because setting timeouts on it is a thing specs do on +// purpose. +// +// The timeouts are cleared because CREATE DATABASE and DROP DATABASE must not +// be bounded by anything a spec configured. A spec that sets a short +// statement_timeout on ITS database cannot reach this one, but a spec that +// names the maintenance database by mistake can, and that is not hypothetical: +// two advisory-lock specs did exactly that until this round. The consequence +// there is a load-dependent failure in another spec's setup or a silently +// swallowed DROP, which is the same invisible single-spec flake this helper was +// rewritten to remove. +// +// MaxOpenConns(1) is what makes the SET reach the statement that follows it: a +// session setting lives on one connection, and with a single connection in the +// pool there is no other one for CREATE or DROP to land on. func openPool(dsn string) *gorm.DB { GinkgoHelper() db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Discard}) Expect(err).ToNot(HaveOccurred()) + + sqlDB, err := db.DB() + Expect(err).ToNot(HaveOccurred()) + sqlDB.SetMaxOpenConns(1) + + Expect(db.Exec("SET statement_timeout = 0").Error).To(Succeed()) + Expect(db.Exec("SET lock_timeout = 0").Error).To(Succeed()) return db } +// openTolerantPool is openPool for the cleanup path, which must report a +// failure rather than assert one: an assertion here would fail a spec that had +// already passed, and bury whatever the next real failure was. +func openTolerantPool(dsn string) (*gorm.DB, error) { + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Discard}) + if err != nil { + return nil, err + } + sqlDB, err := db.DB() + if err != nil { + closePool(db) + return nil, err + } + sqlDB.SetMaxOpenConns(1) + // The DROP below is the statement most likely to be slow, since it waits on + // FORCE terminating other sessions, so it is the one a leaked timeout would + // abort. Measured at up to 169ms under load, against a 300ms bound. + if err := db.Exec("SET statement_timeout = 0").Error; err != nil { + closePool(db) + return nil, err + } + if err := db.Exec("SET lock_timeout = 0").Error; err != nil { + closePool(db) + return nil, err + } + return db, nil +} + func closePool(db *gorm.DB) { if db == nil { return diff --git a/core/services/testutil/testdb_internal_test.go b/core/services/testutil/testdb_internal_test.go new file mode 100644 index 000000000000..457d6f7d5f5f --- /dev/null +++ b/core/services/testutil/testdb_internal_test.go @@ -0,0 +1,63 @@ +package testutil + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// These are white-box on purpose: the property is about the connection this +// package makes for itself, which no caller can reach. +var _ = Describe("the maintenance connection", func() { + It("cannot be bounded by a timeout set on the maintenance database", func() { + // The leak this pins is not hypothetical. Two advisory-lock specs named + // a database by literal, and once the helper started handing out + // per-spec databases those ALTERs landed on the maintenance database + // instead, so every CREATE DATABASE and every DROP ... WITH (FORCE) ran + // under a 300ms bound. A CREATE that trips it fails another spec's + // setup; a DROP that trips it is swallowed and leaks a database. Both + // are load-dependent single-spec failures, which is the exact shape + // this helper was rewritten to remove. + dsn := sharedPostgres() + + var maintenance string + func() { + probe := openPool(dsn) + defer closePool(probe) + Expect(probe.Raw("SELECT current_database()").Scan(&maintenance).Error).To(Succeed()) + }() + Expect(maintenance).ToNot(BeEmpty()) + + // Impose the leak, then assert a fresh maintenance connection is + // unaffected. Reset first so a failure below cannot leave the bound in + // place for the rest of the suite. + DeferCleanup(func() { + reset := openPool(dsn) + defer closePool(reset) + Expect(reset.Exec(fmt.Sprintf("ALTER DATABASE %q RESET statement_timeout", maintenance)).Error).To(Succeed()) + Expect(reset.Exec(fmt.Sprintf("ALTER DATABASE %q RESET lock_timeout", maintenance)).Error).To(Succeed()) + }) + func() { + impose := openPool(dsn) + defer closePool(impose) + Expect(impose.Exec(fmt.Sprintf("ALTER DATABASE %q SET statement_timeout = '1ms'", maintenance)).Error).To(Succeed()) + Expect(impose.Exec(fmt.Sprintf("ALTER DATABASE %q SET lock_timeout = '1ms'", maintenance)).Error).To(Succeed()) + }() + + fresh := openPool(dsn) + defer closePool(fresh) + var statementTimeout, lockTimeout string + Expect(fresh.Raw("SHOW statement_timeout").Scan(&statementTimeout).Error).To(Succeed()) + Expect(fresh.Raw("SHOW lock_timeout").Scan(&lockTimeout).Error).To(Succeed()) + Expect(statementTimeout).To(Equal("0"), + "a statement_timeout on the maintenance database reached the helper's own connection, so CREATE and DROP DATABASE are bounded by whatever a spec configured") + Expect(lockTimeout).To(Equal("0"), + "a lock_timeout on the maintenance database reached the helper's own connection") + + // And the thing the timeouts would actually abort still works while the + // bound is in force. A 1ms statement_timeout is far below the 14-26ms a + // CREATE DATABASE takes here, so this could not pass by being fast. + Expect(SetupTestDB()).ToNot(BeNil()) + }) +}) From 7746bcfc51720c3c98a27f81d0c92cffa04a8898 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 11:43:23 +0000 Subject: [PATCH 048/153] fix(testutil): clear the maintenance timeouts at connection startup The guard added last commit was circular. It cleared the maintenance database's timeouts by executing SET statement_timeout = 0 on a connection that had already inherited that database's bound, so the statement clearing the bound ran under the bound it was clearing. Under the white-box spec's deliberate 1ms that gave it 1ms, and it failed roughly once in fifty at 8-way concurrency with SQLSTATE 57014. The guard against invisible load-dependent flakes had become one. The clearing is now delivered as a connection startup option, options=-c statement_timeout=0 -c lock_timeout=0 on the maintenance DSN, so there is no statement left to abort. Raising the imposed bound would only have bought headroom and left the circularity in place. pgx puts every URL query parameter into settings, options is absent from notRuntimeParams so it becomes a runtime parameter, and runtime parameters are copied into the startup message (pgconn/config.go:340-378, 606-617; pgconn/pgconn.go:382-388). The spec now discriminates on pg_settings.reset_val, the value in force when the connection started: 0 for a startup option, 1ms for a session SET. A first attempt using a deliberately slow first statement did NOT discriminate, because under the circular design the SET is itself the first statement, so by the time a spec runs anything the session is already unbounded. Reinstating the circular clearing now reddens the spec deterministically rather than intermittently. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/testutil/testdb.go | 87 ++++++++++--------- .../services/testutil/testdb_internal_test.go | 53 ++++++++++- 2 files changed, 98 insertions(+), 42 deletions(-) diff --git a/core/services/testutil/testdb.go b/core/services/testutil/testdb.go index 7e2c061dd8a4..4f36d52c3d31 100644 --- a/core/services/testutil/testdb.go +++ b/core/services/testutil/testdb.go @@ -165,61 +165,70 @@ func SetupTestDB() *gorm.DB { return db } -// openPool connects to dsn with logging off and with every server-side timeout -// disabled on the session. Used for the short-lived maintenance connections -// only; the database a spec is handed keeps gorm's silent logger and the -// server's defaults, because setting timeouts on it is a thing specs do on -// purpose. +// maintenanceDSN is dsn with every server-side timeout disabled as a CONNECTION +// STARTUP OPTION rather than as a statement. // -// The timeouts are cleared because CREATE DATABASE and DROP DATABASE must not -// be bounded by anything a spec configured. A spec that sets a short -// statement_timeout on ITS database cannot reach this one, but a spec that -// names the maintenance database by mistake can, and that is not hypothetical: -// two advisory-lock specs did exactly that until this round. The consequence -// there is a load-dependent failure in another spec's setup or a silently -// swallowed DROP, which is the same invisible single-spec flake this helper was -// rewritten to remove. +// The timeouts have to go because CREATE DATABASE and DROP DATABASE must not be +// bounded by anything a spec configured. A spec that sets a short +// statement_timeout on ITS own database cannot reach this connection, but a spec +// that names the maintenance database by mistake can, and that is not +// hypothetical: two advisory-lock specs did exactly that. // -// MaxOpenConns(1) is what makes the SET reach the statement that follows it: a -// session setting lives on one connection, and with a single connection in the -// pool there is no other one for CREATE or DROP to land on. +// Clearing it with `SET statement_timeout = 0` on an already-open connection is +// circular and was a real defect here: that connection has already inherited the +// database's bound, so the statement that clears the bound runs under it and can +// be aborted by it with SQLSTATE 57014. It failed roughly once in fifty at +// 8-way concurrency, which is the same invisible load-dependent single-spec +// flake this helper exists to remove. A startup option removes the circularity +// instead of buying headroom against it: the value is delivered in the startup +// packet, so the connection is already unbounded before it can run anything. +// +// The route is verified in the driver rather than assumed. pgx puts every URL +// query parameter into settings (pgconn/config.go:614), `options` is absent from +// notRuntimeParams (pgconn/config.go:340-362) so it becomes a runtime parameter +// (pgconn/config.go:374-378), and runtime parameters are copied into the startup +// message (pgconn/pgconn.go:382-388). PostgreSQL treats `options` as backend +// command-line switches, so `-c statement_timeout=0` is applied before the +// session accepts a query. +func maintenanceDSN(dsn string) (string, error) { + u, err := url.Parse(dsn) + if err != nil { + return "", err + } + q := u.Query() + // Percent-encoded by Encode, and pgx decodes query values before they reach + // settings, so the server receives the switches with their spaces intact. + q.Set("options", "-c statement_timeout=0 -c lock_timeout=0") + u.RawQuery = q.Encode() + return u.String(), nil +} + +// openPool connects to the maintenance database with logging off and no +// server-side timeouts. Used for the short-lived maintenance connections only; +// the database a spec is handed keeps gorm's silent logger and the server's +// defaults, because setting timeouts on it is a thing specs do on purpose. func openPool(dsn string) *gorm.DB { GinkgoHelper() - db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Discard}) - Expect(err).ToNot(HaveOccurred()) - - sqlDB, err := db.DB() + db, err := openTolerantPool(dsn) Expect(err).ToNot(HaveOccurred()) - sqlDB.SetMaxOpenConns(1) - - Expect(db.Exec("SET statement_timeout = 0").Error).To(Succeed()) - Expect(db.Exec("SET lock_timeout = 0").Error).To(Succeed()) return db } // openTolerantPool is openPool for the cleanup path, which must report a // failure rather than assert one: an assertion here would fail a spec that had // already passed, and bury whatever the next real failure was. +// +// It carries the same startup options, and the DROP is the statement that most +// needs them: FORCE waits on terminating other sessions, measured at up to 169ms +// against the 300ms bound that used to leak here, and a DROP aborted mid-way is +// swallowed and leaks a database. func openTolerantPool(dsn string) (*gorm.DB, error) { - db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Discard}) + maintenance, err := maintenanceDSN(dsn) if err != nil { return nil, err } - sqlDB, err := db.DB() + db, err := gorm.Open(postgres.Open(maintenance), &gorm.Config{Logger: logger.Discard}) if err != nil { - closePool(db) - return nil, err - } - sqlDB.SetMaxOpenConns(1) - // The DROP below is the statement most likely to be slow, since it waits on - // FORCE terminating other sessions, so it is the one a leaked timeout would - // abort. Measured at up to 169ms under load, against a 300ms bound. - if err := db.Exec("SET statement_timeout = 0").Error; err != nil { - closePool(db) - return nil, err - } - if err := db.Exec("SET lock_timeout = 0").Error; err != nil { - closePool(db) return nil, err } return db, nil diff --git a/core/services/testutil/testdb_internal_test.go b/core/services/testutil/testdb_internal_test.go index 457d6f7d5f5f..b3003c0d0016 100644 --- a/core/services/testutil/testdb_internal_test.go +++ b/core/services/testutil/testdb_internal_test.go @@ -5,6 +5,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gorm.io/driver/postgres" + "gorm.io/gorm" + gormlogger "gorm.io/gorm/logger" ) // These are white-box on purpose: the property is about the connection this @@ -45,6 +48,11 @@ var _ = Describe("the maintenance connection", func() { Expect(impose.Exec(fmt.Sprintf("ALTER DATABASE %q SET lock_timeout = '1ms'", maintenance)).Error).To(Succeed()) }() + // The bound is delivered before the first statement, so the check + // below is also the connection's first statement. That ordering is the + // point: clearing the bound with a SET would be circular, because the + // clearing statement inherits the bound it is clearing and can be + // aborted by it with 57014. There is no such bootstrap statement now. fresh := openPool(dsn) defer closePool(fresh) var statementTimeout, lockTimeout string @@ -55,9 +63,48 @@ var _ = Describe("the maintenance connection", func() { Expect(lockTimeout).To(Equal("0"), "a lock_timeout on the maintenance database reached the helper's own connection") - // And the thing the timeouts would actually abort still works while the - // bound is in force. A 1ms statement_timeout is far below the 14-26ms a - // CREATE DATABASE takes here, so this could not pass by being fast. + // A control, and the reason this spec is not a race. Clearing the bound + // with a statement is circular: the clearing statement runs on a + // connection that has already inherited the bound. Whether that + // particular statement exceeds 1ms is a matter of load, which makes the + // defect an intermittent one; whether the FIRST statement on a plain + // connection is bounded at all is not. So the control asks the + // deterministic question, with a first statement that certainly exceeds + // the bound. + func() { + plain, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: gormlogger.Discard}) + Expect(err).ToNot(HaveOccurred()) + defer closePool(plain) + err = plain.Exec("SELECT pg_sleep(0.05)").Error + Expect(err).To(HaveOccurred(), + "the imposed bound does not reach a fresh connection's first statement, so this spec's subject is not actually under test") + Expect(err.Error()).To(ContainSubstring("57014"), + "expected the imposed statement_timeout to abort this, got something else") + }() + + // The same first statement on a maintenance connection is unbounded. + Expect(fresh.Exec("SELECT pg_sleep(0.05)").Error).To(Succeed()) + + // And this is the assertion that says WHY, which is the part a + // statement-based clearing cannot satisfy. reset_val is the value the + // session would fall back to, that is, the value that was in force when + // the connection started, before it could run anything. Clearing the + // bound with `SET statement_timeout = 0` leaves reset_val at the + // database's 1ms: the session is unbounded only because a statement + // said so, and that statement ran under the 1ms bound and can be + // aborted by it. Delivering it as a startup option makes the connection + // unbounded with no statement in between, which is the difference + // between a fix and a wider margin. + var resetVal string + Expect(fresh.Raw( + "SELECT reset_val FROM pg_settings WHERE name = 'statement_timeout'", + ).Scan(&resetVal).Error).To(Succeed()) + Expect(resetVal).To(Equal("0"), + "the maintenance connection started under a %s bound and cleared it with a statement, so the clearing statement itself runs under the bound it is clearing", resetVal) + + // And the operation the bound would abort still works while it is in + // force. 1ms is far below the 14-26ms a CREATE DATABASE takes here, so + // this cannot pass by being fast. Expect(SetupTestDB()).ToNot(BeNil()) }) }) From 148fff1ca99a150f7c67295d995c12cd04b811ac Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 12:28:00 +0000 Subject: [PATCH 049/153] feat(worker): dial, hold and serve the tunnel, on a credential of its own The worker end of the tunnel. It dials wss:///api/cluster/connect, holds one yamux session as the CLIENT, and serves every stream the frontend opens on it. Nothing dials into the worker, which is the point: no inbound port, no reachable address. Each stream opens with a length-prefixed frame naming a tag and a target, and the worker answers before either side speaks the tunnelled protocol. The reply is sent on every stream, not only on refusal, because the protocols carried here are client-speaks-first and a reply sent only sometimes would arrive interleaved with a response body. Two tags today: grpc reaches a backend process, and only on 127.0.0.1 within this worker's own backend port range, because a tunnel terminates inside the worker and letting the frontend name a host would make every worker a proxy into its own LAN; http reaches the worker's file-transfer server, whose address the frontend is not asked about. An unknown tag, an unreachable local service and an unparseable request are three refusals and stay three on the wire. A frontend gives up on the first and retries the second. Each is answered AND the stream is ended: a worker that says why and leaves the stream open has parked the caller on a request nobody will answer, and a deadline on the far side cannot tell that from a slow worker. The specs assert the stream ends rather than that an error occurred, which is what phase 1 shipped in three places and held in none. Reconnects double from 500ms to a 30s ceiling, each wait drawn between half the interval and all of it, and the interval returns to its floor only after a session that LASTED. Resetting on connect is how a rolling restart, where every dial succeeds and dies moments later, becomes a retry storm against the first replica back up. Nothing is assumed to survive a reconnect: the credential is read at dial time, never captured. And the credential is now real. The tunnel endpoint advertised authenticating a worker against its own secret, but registration stored the hash of the shared registration token, so a leak plus a known node ID still opened a tunnel. Registration now mints a per-node secret, returns the plaintext once as tunnel_token, and stores only its SHA-256 in a new column; the endpoint compares against that and does not fall back to the old one. Rotating on every registration follows from storing only the hash, since a re-registering worker cannot be told the secret it already holds; its live tunnel is unaffected, because the credential is checked when a tunnel is dialled and never again. Unlike the agent API key and the NATS JWT next to it, the credential IS issued to a node awaiting approval: the tunnel route re-reads the node's status on every dial and refuses a pending one, so it is inert until an admin acts, and withholding it would strand every worker that registers exactly once. A node that has not registered since this change cannot tunnel, and the column cannot be back-filled because the plaintext only ever existed in the response that minted it. The boot warning that said tunnels need LOCALAI_REGISTRATION_TOKEN is replaced: it was true while the tunnel authenticated against that token's hash, and says the wrong thing now. What is still true, and is what it warns about instead, is that without one, registration itself is unauthenticated. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/cli/workerregistry/client.go | 37 +- core/cli/workerregistry/credentials.go | 21 + core/http/app.go | 25 +- core/http/endpoints/cluster/connect.go | 62 +- core/http/endpoints/cluster/connect_test.go | 53 +- core/http/endpoints/localai/nodes.go | 39 ++ core/http/endpoints/localai/nodes_test.go | 88 +++ core/services/cluster/tunnelproto.go | 239 ++++++++ core/services/nodes/registry.go | 48 +- core/services/worker/config.go | 7 +- core/services/worker/tunnel.go | 617 ++++++++++++++++++++ core/services/worker/tunnel_test.go | 509 ++++++++++++++++ core/services/worker/worker.go | 58 +- docs/content/features/distributed-mode.md | 32 +- 14 files changed, 1747 insertions(+), 88 deletions(-) create mode 100644 core/services/cluster/tunnelproto.go create mode 100644 core/services/worker/tunnel.go create mode 100644 core/services/worker/tunnel_test.go diff --git a/core/cli/workerregistry/client.go b/core/cli/workerregistry/client.go index cf46455c95c0..f8728166fef9 100644 --- a/core/cli/workerregistry/client.go +++ b/core/cli/workerregistry/client.go @@ -58,9 +58,15 @@ func (c *RegistrationClient) setAuth(req *http.Request) { // RegisterResponse is the JSON body returned by /api/node/register. type RegisterResponse struct { - ID string `json:"id"` - Status string `json:"status,omitempty"` // "pending" until an admin approves the node - APIToken string `json:"api_token,omitempty"` + ID string `json:"id"` + Status string `json:"status,omitempty"` // "pending" until an admin approves the node + APIToken string `json:"api_token,omitempty"` + // TunnelToken is this node's own credential for GET /api/cluster/connect. + // The frontend mints a fresh one on every registration and keeps only its + // hash, so this is the ONLY time the plaintext exists anywhere but in this + // worker's memory: a worker that discards it cannot get it back without + // registering again. + TunnelToken string `json:"tunnel_token,omitempty"` NatsJWT string `json:"nats_jwt,omitempty"` NatsUserSeed string `json:"nats_user_seed,omitempty"` } @@ -108,27 +114,42 @@ func (c *RegistrationClient) Register(ctx context.Context, body map[string]any) } // RegisterWithRetry retries registration with exponential backoff. +// +// It drops every field of the response it does not name, the tunnel credential +// among them. Callers that need one use RegisterFullWithRetry. func (c *RegistrationClient) RegisterWithRetry(ctx context.Context, body map[string]any, maxRetries int) (nodeID, apiToken, natsJWT, natsSeed string, err error) { + res, err := c.RegisterFullWithRetry(ctx, body, maxRetries) + if err != nil { + return "", "", "", "", err + } + return res.ID, res.APIToken, res.NatsJWT, res.NatsUserSeed, nil +} + +// RegisterFullWithRetry retries registration with exponential backoff and +// returns the whole response. +func (c *RegistrationClient) RegisterFullWithRetry(ctx context.Context, body map[string]any, maxRetries int) (*RegisterResponse, error) { backoff := 2 * time.Second maxBackoff := 30 * time.Second + var err error for attempt := 1; attempt <= maxRetries; attempt++ { - nodeID, apiToken, natsJWT, natsSeed, err = c.Register(ctx, body) + var res *RegisterResponse + res, err = c.RegisterFull(ctx, body) if err == nil { - return nodeID, apiToken, natsJWT, natsSeed, nil + return res, nil } if attempt == maxRetries { - return "", "", "", "", fmt.Errorf("failed after %d attempts: %w", maxRetries, err) + return nil, fmt.Errorf("failed after %d attempts: %w", maxRetries, err) } xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", backoff, "error", err) select { case <-ctx.Done(): - return "", "", "", "", ctx.Err() + return nil, ctx.Err() case <-time.After(backoff): } backoff = min(backoff*2, maxBackoff) } - return nodeID, apiToken, natsJWT, natsSeed, err + return nil, err } // Heartbeat sends a single heartbeat POST with the given body. diff --git a/core/cli/workerregistry/credentials.go b/core/cli/workerregistry/credentials.go index 24dd6f3c8ed7..f9d8c2231ddc 100644 --- a/core/cli/workerregistry/credentials.go +++ b/core/cli/workerregistry/credentials.go @@ -50,6 +50,11 @@ type NATSCredentialManager struct { jwt string seed string nodeID string + // tunnelToken is the node's own tunnel credential from the most recent + // registration. It is kept here because every re-registration this manager + // performs ROTATES it, so the tunnel client has to read the current value + // at dial time rather than be handed one at startup. + tunnelToken string } // NewNATSCredentialManager builds a manager over register. When requireCreds is @@ -87,6 +92,22 @@ func (m *NATSCredentialManager) store(res *RegisterResponse) { if res.NatsJWT != "" && res.NatsUserSeed != "" { m.jwt, m.seed = res.NatsJWT, res.NatsUserSeed } + // Guarded the same way the NATS pair is: a response that carries no tunnel + // token (a frontend that predates them, or one whose minting failed) must + // not wipe a working credential this worker already holds. Overwriting with + // "" would lock the tunnel out until the next registration that did carry + // one, which is the opposite of what an empty field means. + if res.TunnelToken != "" { + m.tunnelToken = res.TunnelToken + } +} + +// TunnelToken returns the node's current tunnel credential, empty until one has +// been issued. It is the callback the tunnel client reads on every dial. +func (m *NATSCredentialManager) TunnelToken() string { + m.mu.RLock() + defer m.mu.RUnlock() + return m.tunnelToken } // Current returns the latest NATS credentials (both empty until acquired). diff --git a/core/http/app.go b/core/http/app.go index 21592883f942..6bfb3ac8d0bb 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -577,16 +577,21 @@ func API(application *application.Application) (*echo.Echo, error) { if d := application.Distributed(); d != nil { tunnels = d.Tunnels if distCfg.RegistrationToken == "" { - // A separate warning from the peer link's, for the same missing - // knob, because the broken thing is different and an operator has - // to be told both. A worker registering against a frontend with no - // registration token sends no token, so RegisterNodeEndpoint stores - // an empty token_hash, and the tunnel authenticates against exactly - // that column: every dial 401s, forever, on a deployment that looks - // correctly configured. Fixing it needs the token set AND the - // workers re-registered, which is why it is worth saying at boot - // rather than leaving in the 401s. - xlog.Warn("Worker tunnels will refuse every dial: no registration token is configured, so no worker has a stored token to authenticate against", + // A different warning from the peer link's, for the same missing + // knob, because what breaks is different. Tunnels themselves work + // without a registration token: each node is minted its own tunnel + // credential at registration whether or not one is configured. What + // is missing is the gate in FRONT of that. With no registration + // token, RegisterNodeEndpoint validates nothing, so anyone who can + // reach this frontend can register a node and be handed a working + // tunnel credential for it. + // + // This warning replaced one that said the opposite, that tunnels + // would refuse every dial without this token. That was true while + // the tunnel authenticated against the registration token's own + // hash, and stopped being true when nodes got credentials of their + // own. + xlog.Warn("Node registration is unauthenticated, so any caller that can reach this frontend can register a worker and be issued a tunnel credential", "route", clustersvc.ConnectPath, "knob", "LOCALAI_REGISTRATION_TOKEN") } } diff --git a/core/http/endpoints/cluster/connect.go b/core/http/endpoints/cluster/connect.go index ca94ead48126..5e6ace97f7b6 100644 --- a/core/http/endpoints/cluster/connect.go +++ b/core/http/endpoints/cluster/connect.go @@ -104,23 +104,21 @@ func ConnectHandler(registry *nodes.NodeRegistry, tunnels *clustersvc.TunnelRegi } // Split from the mismatch below because they are different operator - // problems with different fixes. An empty stored hash means the node - // registered against a frontend with no LOCALAI_REGISTRATION_TOKEN, so - // no worker on that deployment can ever tunnel until one is set and the - // workers re-register; a mismatch means one worker holds the wrong - // secret. One log line for both leaves an operator reading "wrong token" - // while every worker fails identically. - if node.TokenHash == "" { - // Debug, not Warn. Every worker in such a deployment fails this way - // on every reconnect, so warning per dial buries the log; the fact - // is stated once, at boot, where core/http/app.go warns that no - // registration token is configured. What this line adds is which - // node, for an operator who has already read that warning. - xlog.Debug("refusing a worker tunnel: this node has no stored token, so it registered with no registration token configured", - "node", nodeID, "knob", "LOCALAI_REGISTRATION_TOKEN") + // problems with different fixes. An empty stored hash means this node + // last registered against a LocalAI that predates per-node tunnel + // credentials, so it holds no secret this route can check and must + // register again; a mismatch means the worker is presenting the wrong + // one, usually a stale credential from before a rotation. One log line + // for both leaves an operator reading "wrong token" while a whole fleet + // of not-yet-restarted workers fails identically. + if node.TunnelTokenHash == "" { + // Debug, not Warn. Every worker in that state fails this way on + // every reconnect, so warning per dial buries the log. + xlog.Debug("refusing a worker tunnel: this node has no tunnel credential, so it has not registered since they were introduced", + "node", nodeID) return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") } - if !authorizedWorker(token, node.TokenHash) { + if !authorizedWorker(token, node.TunnelTokenHash) { xlog.Debug("worker tunnel dial presented the wrong token", "node", nodeID) return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") } @@ -229,32 +227,26 @@ func bearerToken(r *http.Request) (string, bool) { // authorizedWorker compares a presented token against the hash stored on the // node's own row, in constant time. // -// Against the NODE's stored hash, not the configured registration token, which -// is a deliberate strengthening over how /api/node/ authenticates. A tunnel is a -// durable, multiplexed pipe into a worker; a credential that authorizes every -// worker at once would mean one leak lets an attacker impersonate any worker -// whose ID it can read, and take over that worker's traffic by claiming its -// tunnel. +// Against the NODE's OWN tunnel credential, not the deployment's registration +// token. A tunnel is a durable, multiplexed pipe into a worker, and a +// credential that authorizes every worker at once would mean one leak lets an +// attacker impersonate any worker whose ID it can read and take over that +// worker's traffic by claiming its tunnel. The secret compared here is minted +// per node at registration (attachTunnelToken in +// core/http/endpoints/localai/nodes.go), returned to that worker once, and +// stored only as this hash, so knowing the registration token no longer gets +// anyone a tunnel. // -// What that buys TODAY is the mechanism, not yet the isolation, and the -// difference should not be overstated: a worker registers by presenting the -// deployment's registration token, so the hash on its row is that token's hash -// and a leaked registration token still opens a tunnel for a node whose ID the -// attacker knows. What this does rule out is the shortcut of comparing against -// the configured token itself, which is what would have to be unpicked later; -// the day a worker is issued its own secret at registration, this check starts -// isolating workers from each other with no change here, PROVIDED the secret -// lands in TokenHash. The one per-node secret LocalAI mints today, the agent -// worker's api_token, does not: provisionAgentWorkerKey writes an auth.User and -// an auth.APIKey referenced by node.AuthUserID / node.APIKeyID and never touches -// this column. If the next task follows that precedent instead, this comparison -// is what has to change. +// Note which column: BackendNode.TunnelTokenHash, not TokenHash. TokenHash is +// still the hash of whatever token the worker registered WITH, which on most +// deployments is the shared registration token, and comparing against it is +// exactly the weakness this replaced. // // The empty-hash guard is defensive rather than deciding: a stored hash is // hex-encoded SHA-256, so 64 bytes or nothing, and ConstantTimeCompare already // returns 0 on a length mismatch (crypto/internal/fips140/subtle/constant_time.go:17-20 // returns 0 outright when the lengths differ). It is kept because a reader should not have to -// derive "an unregistered node authorizes nobody" from a length rule, and +// derive "a node with no credential authorizes nobody" from a length rule, and // because the caller logs that case separately. func authorizedWorker(token, storedHash string) bool { if storedHash == "" { diff --git a/core/http/endpoints/cluster/connect_test.go b/core/http/endpoints/cluster/connect_test.go index 072f69d0226c..15206d50444c 100644 --- a/core/http/endpoints/cluster/connect_test.go +++ b/core/http/endpoints/cluster/connect_test.go @@ -24,12 +24,17 @@ import ( "gorm.io/gorm" ) -// workerToken is the secret one worker holds. It is stored on that worker's row -// as a hash, never in a config value, which is the whole point of the check the -// specs below pin: a second worker's token, and the deployment-wide +// workerToken is the tunnel credential one worker holds. Registration mints it +// per node and keeps only its hash, which is the whole point of the check the +// specs below pin: a second worker's credential, and the deployment-wide // registration token, are both wrong for this node. const workerToken = "worker-1-secret" +// registrationToken stands in for the shared secret every worker in a +// deployment registers with. It is stored on the row too, in a DIFFERENT +// column, and a spec below pins that presenting it does not open a tunnel. +const registrationToken = "deployment-registration-token" + func tokenHash(token string) string { sum := sha256.Sum256([]byte(token)) return hex.EncodeToString(sum[:]) @@ -66,9 +71,13 @@ var _ = Describe("Worker tunnel handler", func() { Expect(err).ToNot(HaveOccurred()) node := &nodes.BackendNode{ - Name: "worker-1", - Address: "10.0.0.9:50051", - TokenHash: tokenHash(workerToken), + Name: "worker-1", + Address: "10.0.0.9:50051", + // Both hashes are set, and they differ. That is what a real + // registration produces: TokenHash is the shared token the worker + // registered WITH, TunnelTokenHash is the secret minted FOR it. + TokenHash: tokenHash(registrationToken), + TunnelTokenHash: tokenHash(workerToken), } Expect(nodeReg.Register(ctx, node, true)).To(Succeed()) nodeID = node.ID @@ -108,11 +117,13 @@ var _ = Describe("Worker tunnel handler", func() { Expect(tun.Held()).To(BeEmpty()) }) - It("refuses a token that is not the node's own", func() { + It("refuses the deployment's registration token, which this node also stores", func() { // The registration token is the credential every worker in the - // deployment holds. Accepting it here would mean one leaked shared - // secret impersonates any worker whose ID an attacker can read. - _, resp, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), bearer("deployment-registration-token")) + // deployment holds, and it IS on this node's row, in TokenHash. + // Accepting it would mean one leaked shared secret impersonates any + // worker whose ID an attacker can read; the node's own tunnel + // credential is the only thing this route accepts. + _, resp, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), bearer(registrationToken)) Expect(err).To(HaveOccurred()) Expect(resp).ToNot(BeNil()) Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) @@ -194,13 +205,19 @@ var _ = Describe("Worker tunnel handler", func() { "the claim outlived the socket, so this replica keeps being named the owner of a worker it no longer holds") }) - It("refuses a node whose row carries no stored token", func() { - // A deployment with no registration token configured produces exactly - // these rows: the worker sends no token, so registration stores none, - // and there is nothing here to authenticate against. - Expect(db.Exec(`UPDATE backend_nodes SET token_hash = '' WHERE id = ?`, nodeID).Error).To(Succeed()) - - _, resp, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), bearer(workerToken)) + It("refuses a node with no tunnel credential, without falling back to its registration token", func() { + // A node registered by a LocalAI predating per-node tunnel credentials + // produces exactly this row: a registration-token hash in token_hash + // and nothing in tunnel_token_hash. It cannot be back-filled, because + // the plaintext only ever existed in the response that minted it, so + // such a node must register again. + // + // The dial presents the REGISTRATION token, which is still on the row. + // A handler that fell back to token_hash when the tunnel hash is empty + // would let it in, which is the weakness this whole change removed. + Expect(db.Exec(`UPDATE backend_nodes SET tunnel_token_hash = '' WHERE id = ?`, nodeID).Error).To(Succeed()) + + _, resp, err := websocket.DefaultDialer.Dial(wsConnectURL(srv, nodeID), bearer(registrationToken)) Expect(err).To(HaveOccurred()) Expect(resp).ToNot(BeNil()) Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) @@ -266,7 +283,7 @@ var _ = Describe("Worker tunnel handler when the attach panics", func() { db = testutil.SetupTestDB() nodeReg, err := nodes.NewNodeRegistry(db) Expect(err).ToNot(HaveOccurred()) - node := &nodes.BackendNode{Name: "worker-1", Address: "10.0.0.9:50051", TokenHash: tokenHash(workerToken)} + node := &nodes.BackendNode{Name: "worker-1", Address: "10.0.0.9:50051", TunnelTokenHash: tokenHash(workerToken)} Expect(nodeReg.Register(ctx, node, true)).To(Succeed()) nodeID = node.ID diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index 331c7203b9a9..0b42b8314365 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -2,6 +2,7 @@ package localai import ( "context" + "crypto/rand" "crypto/sha256" "crypto/subtle" "encoding/hex" @@ -250,6 +251,7 @@ func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, au } } + attachTunnelToken(ctx, response, registry, node) attachNatsJWT(response, node, natsCfg) return c.JSON(http.StatusCreated, response) @@ -294,6 +296,43 @@ func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecr } } +// attachTunnelToken mints this node a fresh tunnel credential, stores only its +// hash, and puts the plaintext in the registration response. +// +// It is minted for EVERY node that registers, pending ones included, which is a +// deliberate divergence from the two other per-node credentials in this file: +// the agent worker's API key (provisionAgentWorkerKey) and its NATS JWT +// (attachNatsJWT) are both withheld from a node awaiting approval. Those two +// are bearer grants that WORK the moment they are issued, so issuing one to an +// unapproved node would route around the admin. A tunnel credential is not: +// core/http/endpoints/cluster/connect.go re-reads the node's status on every +// dial and refuses a pending node with 403, so the credential is inert until an +// admin approves and stays inert if approval is revoked. Withholding it would +// instead strand every worker that registers exactly once (the static-NATS +// path in core/services/worker/worker.go does), because approval alone does not +// prompt a re-registration and nothing else can hand it the secret. +// +// A failure to mint or to store is logged and the response goes out without the +// token. Registration is what gets a worker into the cluster at all, and +// failing it over a credential the worker does not need until it tunnels would +// turn a tunnel problem into a node that cannot join. The worker sees no +// tunnel_token, reports that it has no credential, and retries at its next +// registration. +func attachTunnelToken(ctx context.Context, response map[string]any, registry *nodes.NodeRegistry, node *nodes.BackendNode) { + if node == nil { + return + } + // crypto/rand.Text: at least 128 bits of randomness, no error to handle and + // no length constant to get wrong. + plaintext := rand.Text() + sum := sha256.Sum256([]byte(plaintext)) + if err := registry.SetTunnelTokenHash(ctx, node.ID, hex.EncodeToString(sum[:])); err != nil { + xlog.Error("Failed to store a tunnel credential for node", "node", node.Name, "error", err) + return + } + response["tunnel_token"] = plaintext +} + // attachNatsJWT adds a per-node NATS user JWT to a register/approve response when minting is enabled. func attachNatsJWT(response map[string]any, node *nodes.BackendNode, natsCfg natsauth.Config) { if !natsCfg.CanMintWorkers() || node == nil || node.Status == nodes.StatusPending { diff --git a/core/http/endpoints/localai/nodes_test.go b/core/http/endpoints/localai/nodes_test.go index 8390f1a48221..dc1a77f465a7 100644 --- a/core/http/endpoints/localai/nodes_test.go +++ b/core/http/endpoints/localai/nodes_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "crypto/subtle" + "encoding/hex" "encoding/json" "net/http" "net/http/httptest" @@ -20,6 +21,12 @@ import ( . "github.com/onsi/gomega" ) +// hashOf is how the node registry stores a secret: hex-encoded SHA-256. +func hashOf(secret string) string { + sum := sha256.Sum256([]byte(secret)) + return hex.EncodeToString(sum[:]) +} + var _ = DescribeTable("token validation", func(expectedToken, providedToken string, wantMatch bool) { if expectedToken == "" { @@ -106,6 +113,87 @@ var _ = Describe("Node HTTP handlers", func() { Expect(resp["status"]).To(Equal(nodes.StatusHealthy)) }) + // register posts one registration and returns the decoded response. + register := func(body string, expectedToken string, autoApprove bool) map[string]any { + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + + handler := RegisterNodeEndpoint(registry, expectedToken, autoApprove, nil, "", natsauth.Config{}) + ExpectWithOffset(1, handler(c)).To(Succeed()) + ExpectWithOffset(1, rec.Code).To(Equal(http.StatusCreated)) + + var resp map[string]any + ExpectWithOffset(1, json.Unmarshal(rec.Body.Bytes(), &resp)).To(Succeed()) + return resp + } + + It("mints a per-node tunnel credential and stores only its hash", func() { + resp := register(`{"name":"worker-tunnel","address":"10.0.0.3:50051","token":"shared-registration-token"}`, + "shared-registration-token", true) + + plaintext, _ := resp["tunnel_token"].(string) + Expect(plaintext).ToNot(BeEmpty()) + // Not the registration token. That is the whole point: a leaked + // registration token plus a known node ID used to open a tunnel, + // because the tunnel authenticated against the hash of exactly the + // value every worker in the deployment holds. + Expect(plaintext).ToNot(Equal("shared-registration-token")) + + node, err := registry.Get(context.Background(), resp["id"].(string)) + Expect(err).ToNot(HaveOccurred()) + // Stored as a hash, never as the secret. + Expect(node.TunnelTokenHash).To(Equal(hashOf(plaintext))) + Expect(node.TunnelTokenHash).ToNot(Equal(plaintext)) + // And it is a DIFFERENT column from the registration token's hash, + // which is what the tunnel used to compare against. + Expect(node.TunnelTokenHash).ToNot(Equal(node.TokenHash)) + Expect(node.TokenHash).To(Equal(hashOf("shared-registration-token"))) + + // The security property, which none of the above actually pins: a + // second node registering with the SAME shared token gets a + // DIFFERENT credential. Everything above is satisfied by a secret + // derived deterministically from the registration token, which + // would isolate nothing; a mutation that did exactly that passed + // every assertion before this one. + other := register(`{"name":"worker-tunnel-2","address":"10.0.0.3:50052","token":"shared-registration-token"}`, + "shared-registration-token", true) + Expect(other["tunnel_token"]).ToNot(Equal(plaintext)) + }) + + It("rotates the tunnel credential on every re-registration", func() { + body := `{"name":"worker-rotate","address":"10.0.0.4:50051"}` + first := register(body, "", true) + second := register(body, "", true) + + Expect(second["id"]).To(Equal(first["id"]), "re-registration must keep the node identity") + firstToken := first["tunnel_token"].(string) + secondToken := second["tunnel_token"].(string) + // Only the hash is stored, so a re-registering worker cannot be told + // the secret it already holds; the alternative to rotating would be + // storing the plaintext. + Expect(secondToken).ToNot(Equal(firstToken)) + + node, err := registry.Get(context.Background(), first["id"].(string)) + Expect(err).ToNot(HaveOccurred()) + Expect(node.TunnelTokenHash).To(Equal(hashOf(secondToken))) + Expect(node.TunnelTokenHash).ToNot(Equal(hashOf(firstToken))) + }) + + It("issues a tunnel credential to a node still awaiting approval", func() { + resp := register(`{"name":"worker-pending","address":"10.0.0.5:50051"}`, "", false) + Expect(resp["status"]).To(Equal(nodes.StatusPending)) + // Deliberately unlike the agent API key and the NATS JWT, which are + // both withheld from a pending node. Those work the moment they are + // issued; this one does not, because the tunnel endpoint re-reads + // the node's status on every dial and refuses a pending node. A + // worker that registers exactly once would otherwise never receive + // one, since approval alone prompts no re-registration. + Expect(resp["tunnel_token"]).ToNot(BeEmpty()) + }) + It("returns nats_jwt when account seed is configured", func() { akp, err := nkeys.CreateAccount() Expect(err).ToNot(HaveOccurred()) diff --git a/core/services/cluster/tunnelproto.go b/core/services/cluster/tunnelproto.go new file mode 100644 index 000000000000..39f5014b50b4 --- /dev/null +++ b/core/services/cluster/tunnelproto.go @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: MIT + +package cluster + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "strings" +) + +// The framing every stream on a worker tunnel opens with. +// +// A yamux stream on its own carries no destination: the frontend opens one and +// the worker has to be told what it is for. So the first thing on every stream +// is a request frame naming a TAG (which local service) and a TARGET (which +// instance of it), and the worker answers with a reply frame before either side +// speaks the tunnelled protocol. +// +// The reply is not optional and is not sent only on failure, which is the part +// that is easy to get wrong. The protocols carried here are client-speaks-first +// (gRPC sends an HTTP/2 preface, HTTP sends a request line), so a reply sent +// only when the worker refuses would arrive interleaved with a response body on +// the streams that succeeded, and the frontend would have no safe moment to +// look for it. Always sending one costs a round trip per stream, which is paid +// once per pooled connection rather than once per request. +// +// Both frames are length-prefixed rather than newline-delimited so a reader +// consumes exactly the header and not one byte of what follows: the stream is +// handed to gRPC or net/http afterwards, and a buffered reader that over-read +// would eat the beginning of their conversation. + +const ( + // StreamTagGRPC routes a stream to a backend process on the worker. Its + // target is the address that backend listens on, which the worker resolves + // itself; see the worker's tunnel services for what it will accept. + StreamTagGRPC = "grpc" + + // StreamTagHTTP routes a stream to the worker's own HTTP server, the one + // that serves file staging and backend logs. Its target is ignored: there + // is exactly one such server per worker and only the worker knows where it + // bound. + StreamTagHTTP = "http" +) + +// maxTunnelFrame bounds a header frame. It is a defence against a peer that +// declares a huge length and never sends it, not a size the protocol needs: +// the longest real frame is a tag plus a host:port, well under a hundred +// bytes. A reader that refuses early cannot be made to allocate on demand. +const maxTunnelFrame = 1024 + +// The reply codes. They travel on the wire, so they are strings rather than +// integers: a frontend reading a code from a worker it does not recognise can +// at least log something an operator can search for. +const ( + replyAccepted = "ok" + replyCodeUnknownTag = "unknown-tag" + replyCodeUnavailable = "unavailable" + replyCodeBadRequest = "bad-request" + replyPrefixRefused = "err " + streamRequestSeparator = " " +) + +// The three refusals a worker can send, kept apart on purpose. +// +// This is the phase's standing rule in its wire form. An unknown tag is a fact +// about what this worker SERVES and will not change until the worker is +// upgraded; an unavailable target is an infrastructure failure that may well +// succeed on the next attempt; a bad request is this frontend's own bug. A +// caller retries the second, gives up on the first, and reports the third. +// Collapsing them into one error would make a frontend retry a stream that can +// never work, or abandon a backend that was merely restarting. +// +// None of them wraps a node-absence error, and none must ever be built over +// one: a refusal is proof the worker is CONNECTED and answered. +var ( + ErrStreamTagUnknown = errors.New("cluster: the worker does not serve that stream tag") + ErrStreamTargetUnavailable = errors.New("cluster: the worker could not reach the local service for that stream") + ErrStreamRequestInvalid = errors.New("cluster: the worker rejected the stream request as malformed") +) + +// WriteStreamRequest sends the opening frame naming what the stream is for. +// +// An empty tag is refused here rather than on the wire, because the worker +// would answer it with ErrStreamRequestInvalid and the caller would learn a +// round trip later what it could have been told at once. +func WriteStreamRequest(w io.Writer, tag, target string) error { + if tag == "" { + return fmt.Errorf("writing a tunnel stream request: empty tag") + } + if strings.Contains(tag, streamRequestSeparator) { + // The separator is a single space and the split is on the FIRST one, so + // a tag containing a space would silently move part of itself into the + // target. + return fmt.Errorf("writing a tunnel stream request: tag %q contains a space", tag) + } + return writeFrame(w, tag+streamRequestSeparator+target) +} + +// ReadStreamRequest reads the opening frame. The target is empty when the tag +// carries no argument. +// +// A malformed frame is returned as an ordinary error, NOT as +// ErrStreamRequestInvalid: that sentinel is what a worker SENDS to describe a +// refusal, and a reader that produced it here would leave a caller unable to +// tell "the peer refused my request" from "I could not read the peer's". +func ReadStreamRequest(r io.Reader) (tag, target string, err error) { + payload, err := readFrame(r) + if err != nil { + return "", "", fmt.Errorf("reading a tunnel stream request: %w", err) + } + tag, target, _ = strings.Cut(payload, streamRequestSeparator) + if tag == "" { + return "", "", fmt.Errorf("reading a tunnel stream request: empty tag") + } + return tag, target, nil +} + +// WriteStreamAccepted tells the frontend the stream is now carrying the +// tunnelled protocol. Everything after this frame belongs to that protocol. +func WriteStreamAccepted(w io.Writer) error { + return writeFrame(w, replyAccepted) +} + +// WriteStreamRefusal reports why a stream will not be served. The caller closes +// the stream afterwards; this only says why. +// +// An unrecognised reason is sent as bad-request with its text attached rather +// than being dropped, because a refusal a frontend cannot read is +// indistinguishable from a worker that hung up, and those are different +// problems. +func WriteStreamRefusal(w io.Writer, reason error) error { + code := replyCodeBadRequest + switch { + case errors.Is(reason, ErrStreamTagUnknown): + code = replyCodeUnknownTag + case errors.Is(reason, ErrStreamTargetUnavailable): + code = replyCodeUnavailable + case errors.Is(reason, ErrStreamRequestInvalid): + code = replyCodeBadRequest + } + + text := "" + if reason != nil { + text = strings.Map(func(r rune) rune { + // The frame is length-prefixed so a newline would not corrupt it, + // but this text reaches a log line on the far side and a cause + // spanning lines is what makes one unsearchable. + if r == '\n' || r == '\r' { + return ' ' + } + return r + }, reason.Error()) + } + frame := replyPrefixRefused + code + streamRequestSeparator + text + if len(frame) > maxTunnelFrame { + frame = frame[:maxTunnelFrame] + } + return writeFrame(w, frame) +} + +// ReadStreamReply reads the worker's answer. nil means the stream is now +// carrying the tunnelled protocol. +// +// A failure to READ the reply is returned as itself, never as one of the +// refusal sentinels. The distinction is the point of this function: a refusal +// means the worker is connected and said no, while a read failure means the +// tunnel broke, and a caller that treated the second as the first would report +// a dead link as a policy decision. +func ReadStreamReply(r io.Reader) error { + payload, err := readFrame(r) + if err != nil { + return fmt.Errorf("reading a tunnel stream reply: %w", err) + } + if payload == replyAccepted { + return nil + } + rest, ok := strings.CutPrefix(payload, replyPrefixRefused) + if !ok { + return fmt.Errorf("reading a tunnel stream reply: unrecognised reply %q", payload) + } + code, text, _ := strings.Cut(rest, streamRequestSeparator) + switch code { + case replyCodeUnknownTag: + return fmt.Errorf("%w: %s", ErrStreamTagUnknown, text) + case replyCodeUnavailable: + return fmt.Errorf("%w: %s", ErrStreamTargetUnavailable, text) + case replyCodeBadRequest: + return fmt.Errorf("%w: %s", ErrStreamRequestInvalid, text) + default: + // A code from a newer worker. Reported as an error carrying the code + // rather than mapped onto the nearest known one, so a frontend does not + // retry forever against a refusal that means something else entirely. + return fmt.Errorf("tunnel stream refused with unrecognised code %q: %s", code, text) + } +} + +// writeFrame writes one length-prefixed frame in a single Write. +// +// One Write, not two: the underlying stream is a yamux stream whose writes +// become discrete data frames, and splitting the length from the payload would +// put the reader one frame away from a header for no reason. It also keeps the +// adapter in wsconn.go to one WebSocket message per frame. +func writeFrame(w io.Writer, payload string) error { + if len(payload) > maxTunnelFrame { + return fmt.Errorf("tunnel frame is %d bytes, over the %d-byte limit", len(payload), maxTunnelFrame) + } + buf := make([]byte, 2+len(payload)) + binary.BigEndian.PutUint16(buf[:2], uint16(len(payload))) + copy(buf[2:], payload) + _, err := w.Write(buf) + return err +} + +// readFrame reads one length-prefixed frame. +// +// io.ReadFull rather than Read: a yamux stream returns whatever has arrived, +// and a header split across two data frames is ordinary rather than +// exceptional. It also converts a truncated frame into io.ErrUnexpectedEOF, +// which is what a peer that hung up mid-header should look like. +func readFrame(r io.Reader) (string, error) { + var size [2]byte + if _, err := io.ReadFull(r, size[:]); err != nil { + return "", err + } + n := binary.BigEndian.Uint16(size[:]) + if int(n) > maxTunnelFrame { + return "", fmt.Errorf("tunnel frame declares %d bytes, over the %d-byte limit", n, maxTunnelFrame) + } + if n == 0 { + return "", nil + } + payload := make([]byte, n) + if _, err := io.ReadFull(r, payload); err != nil { + return "", err + } + return string(payload), nil +} diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index 0e2195cdaecd..606bcf2af6f8 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -23,15 +23,27 @@ import ( // Workers are generic — they don't have a fixed backend type. // The SmartRouter dynamically installs backends via NATS backend.install events. type BackendNode struct { - ID string `gorm:"primaryKey;size:36" json:"id"` - Name string `gorm:"uniqueIndex;size:255" json:"name"` - NodeType string `gorm:"size:32;default:backend" json:"node_type"` // backend, agent - Address string `gorm:"size:255" json:"address"` // host:port for gRPC - HTTPAddress string `gorm:"size:255" json:"http_address"` // host:port for HTTP file transfer - Status string `gorm:"size:32;default:registering" json:"status"` // registering, healthy, unhealthy, draining, pending - TokenHash string `gorm:"size:64" json:"-"` // SHA-256 of registration token - TotalVRAM uint64 `gorm:"column:total_vram" json:"total_vram"` // Total GPU VRAM in bytes - AvailableVRAM uint64 `gorm:"column:available_vram" json:"available_vram"` // Available GPU VRAM in bytes + ID string `gorm:"primaryKey;size:36" json:"id"` + Name string `gorm:"uniqueIndex;size:255" json:"name"` + NodeType string `gorm:"size:32;default:backend" json:"node_type"` // backend, agent + Address string `gorm:"size:255" json:"address"` // host:port for gRPC + HTTPAddress string `gorm:"size:255" json:"http_address"` // host:port for HTTP file transfer + Status string `gorm:"size:32;default:registering" json:"status"` // registering, healthy, unhealthy, draining, pending + TokenHash string `gorm:"size:64" json:"-"` // SHA-256 of registration token + // TunnelTokenHash is the SHA-256 of this node's OWN tunnel credential, the + // one it presents at GET /api/cluster/connect. It is not the registration + // token: registration mints a fresh random secret per node, returns the + // plaintext once, and stores only this hash, so a leaked registration token + // no longer opens a tunnel for every node whose ID an attacker can read. + // + // Empty means no tunnel credential has been minted for this node yet, which + // is what a node registered by an older LocalAI looks like. Such a node + // cannot tunnel until it registers again. That is deliberate: the column + // cannot be back-filled, because the plaintext exists only in the response + // that minted it. + TunnelTokenHash string `gorm:"size:64" json:"-"` + TotalVRAM uint64 `gorm:"column:total_vram" json:"total_vram"` // Total GPU VRAM in bytes + AvailableVRAM uint64 `gorm:"column:available_vram" json:"available_vram"` // Available GPU VRAM in bytes // ReservedVRAM is a soft, in-tick reservation deducted by the scheduler when // it picks this node to load a model. Workers reset it back to 0 on each // heartbeat (the worker is the source of truth for actual free VRAM); the @@ -731,6 +743,24 @@ func (r *NodeRegistry) UpdateAuthRefs(ctx context.Context, nodeID, authUserID, a }).Error } +// SetTunnelTokenHash records the hash of a freshly minted tunnel credential for +// a node, replacing whatever was there. +// +// Replacing is the whole design and not an accident of the implementation. Only +// the hash is stored, so a re-registering worker cannot be told the secret it +// already has, and the alternative to rotating would be storing the plaintext. +// The live tunnel of a worker that re-registers is unaffected, because the +// credential is checked when a tunnel is DIALLED and never again; what changes +// is which secret its next reconnect must present, and the worker learns that +// in the same response that rotated it. +func (r *NodeRegistry) SetTunnelTokenHash(ctx context.Context, nodeID, hash string) error { + // Not Updates(struct): a struct update zero-skips, so this could never + // clear the column, and a caller that means to clear it would be silently + // ignored. + return r.db.WithContext(ctx).Model(&BackendNode{}).Where("id = ?", nodeID). + Update("tunnel_token_hash", hash).Error +} + // ApproveNode sets a pending node's status to healthy. func (r *NodeRegistry) ApproveNode(ctx context.Context, nodeID string) error { result := r.db.WithContext(ctx).Model(&BackendNode{}). diff --git a/core/services/worker/config.go b/core/services/worker/config.go index 8a63ad8e0567..5c29cd4990f1 100644 --- a/core/services/worker/config.go +++ b/core/services/worker/config.go @@ -60,7 +60,12 @@ type Config struct { RegistrationRequireAuth bool `env:"LOCALAI_REGISTRATION_REQUIRE_AUTH" default:"false" help:"Refuse to start the HTTP file-transfer server when no registration token is set (otherwise it fails open and serves read/write to models/staging/data unauthenticated)" group:"registration"` DistributedRequireAuth bool `env:"LOCALAI_DISTRIBUTED_REQUIRE_AUTH" default:"false" help:"Umbrella switch implying both --nats-require-auth and --registration-require-auth" group:"distributed"` HeartbeatInterval string `env:"LOCALAI_HEARTBEAT_INTERVAL" default:"10s" help:"Interval between heartbeats" group:"registration"` - NodeLabels string `env:"LOCALAI_NODE_LABELS" help:"Comma-separated key=value labels for this node (e.g. tier=fast,gpu=a100)" group:"registration"` + // WorkerTunnel holds one outbound multiplexed connection to the frontend + // and serves the frontend's requests over it, so the worker needs no + // inbound port. Turning it off leaves the worker reachable only at the + // addresses it advertises, which is the pre-tunnel behaviour. + WorkerTunnel bool `env:"LOCALAI_WORKER_TUNNEL" default:"true" help:"Hold one outbound multiplexed tunnel to the frontend and serve its requests over it, so this worker needs no inbound port." group:"distributed"` + NodeLabels string `env:"LOCALAI_NODE_LABELS" help:"Comma-separated key=value labels for this node (e.g. tier=fast,gpu=a100)" group:"registration"` // MaxReplicasPerModel caps how many replicas of any one model can run on // this worker concurrently. Default 1 = historical single-replica // behavior. Set higher when a node has enough VRAM to host multiple diff --git a/core/services/worker/tunnel.go b/core/services/worker/tunnel.go new file mode 100644 index 000000000000..5ab812d8c23e --- /dev/null +++ b/core/services/worker/tunnel.go @@ -0,0 +1,617 @@ +package worker + +import ( + "cmp" + "context" + "errors" + "fmt" + "math/rand/v2" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "github.com/gorilla/websocket" + "github.com/libp2p/go-yamux/v5" + "github.com/mudler/xlog" + + "github.com/mudler/LocalAI/core/services/cluster" +) + +// The worker end of the tunnel. +// +// The worker DIALS OUT and never listens. It holds one WebSocket to the +// frontend load balancer, multiplexed with yamux, and every request the +// frontend makes of this worker arrives as a stream inside it. That is the +// whole point: a worker behind NAT, in another cluster or on a laptop needs no +// inbound port and no reachable address. +// +// This side is the yamux CLIENT and it only ACCEPTS streams; the frontend is +// the server and it only opens them. The frontend asks, the worker answers. +// Nothing here opens a stream, and a stream this side opened would park on the +// frontend's accept backlog, which accepts none. + +const ( + // tunnelBackoffBase is the shortest wait between reconnects, before jitter. + tunnelBackoffBase = 500 * time.Millisecond + + // tunnelBackoffMax is the ceiling on that wait. + // + // The ceiling is the interesting half. Without one, a worker that sits + // through a long frontend outage backs off into hours and does not come + // back for a long time after the frontend does; with one, the worst case + // for rejoining is bounded by this. The floor and the jitter are what stop + // a fleet of workers from turning a rolling restart into a retry storm + // against the first replica back up. + tunnelBackoffMax = 30 * time.Second + + // tunnelHealthyAfter is how long a session must last before the backoff is + // allowed back to its floor. + // + // Resetting on CONNECT rather than on a session that lasted is the classic + // way to build a reconnect storm that looks like a backoff: during a + // rolling restart a replica accepts the dial and dies moments later, so + // every attempt "succeeds" and every wait is the floor. This is set to the + // yamux keepalive interval, which is the shortest interval over which a + // session that is merely up can be told from one that is working. + tunnelHealthyAfter = 30 * time.Second + + // tunnelHandshakeTimeout bounds the WebSocket upgrade, matching the peer + // link's. + tunnelHandshakeTimeout = 10 * time.Second + + // tunnelHeaderTimeout bounds how long a stream may go without sending the + // request frame that says what it is for. It is generous because it bounds + // only the frontend's own framing, which it writes immediately after + // opening the stream; it is present because without it a stream that sends + // nothing holds a goroutine and one of the session's stream slots for as + // long as the tunnel lives. + tunnelHeaderTimeout = 15 * time.Second +) + +// LocalService opens a connection to one service running on this worker. +// +// target is the tag-specific argument from the stream's request frame, and the +// service decides what it will accept: the frontend naming an address does not +// oblige the worker to dial it. See loopbackService, which is what the worker +// actually installs. +type LocalService func(ctx context.Context, target string) (net.Conn, error) + +// TunnelConfig configures the tunnel a worker holds to the frontend. +type TunnelConfig struct { + // FrontendURL is the same value the worker registers against + // (LOCALAI_REGISTER_TO). Its scheme is mapped to ws/wss here. + FrontendURL string + + // NodeID is the identity registration assigned this worker. + NodeID string + + // Token supplies the node's own tunnel credential. + // + // A function and not a string, and that is load-bearing rather than + // stylistic. The credential is re-minted on every registration, so a client + // that captured one at startup would keep presenting a value the frontend + // stopped accepting the moment anything re-registered this worker, and + // would lock itself out with no way back. It is called once per DIAL. + Token func() string + + // Services routes an accepted stream by the tag in its request frame. A tag + // with no entry here is refused; see Tunnel.accept. + Services map[string]LocalService + + // Seams the specs replace. They are unexported so they are not part of the + // package's API: a caller cannot reach them, and the internal test file can. + sleep func(ctx context.Context, d time.Duration) error + now func() time.Time + headerTimeout time.Duration +} + +// Tunnel is a running worker tunnel: one goroutine holding one session at a +// time, reconnecting when it dies, until Close. +type Tunnel struct { + endpoint string + nodeID string + token func() string + services map[string]LocalService + dialer *websocket.Dialer + + headerTimeout time.Duration + sleep func(ctx context.Context, d time.Duration) error + // now measures how long a session lasted, and nothing else. Deadlines are + // taken from time.Now directly: a spec that fakes this clock to exercise + // the backoff must not thereby move every I/O deadline in the package. + now func() time.Time + + cancel context.CancelFunc + done chan struct{} + closeOnce sync.Once +} + +// StartTunnel dials the frontend and holds the tunnel until ctx is cancelled or +// Close is called. +// +// The returned error is about this CONFIGURATION, never about the frontend. A +// frontend that is down, that has not been upgraded, or that refuses the +// credential is not a reason for a worker to fail to start: it retries, with +// backoff, in the background. Failing to start on a dial would make a frontend +// restart into a fleet-wide worker outage. +func StartTunnel(ctx context.Context, cfg TunnelConfig) (*Tunnel, error) { + if cfg.NodeID == "" { + return nil, errors.New("starting the worker tunnel: no node id") + } + if cfg.Token == nil { + return nil, errors.New("starting the worker tunnel: no credential source") + } + endpoint, err := tunnelEndpoint(cfg.FrontendURL, cfg.NodeID) + if err != nil { + return nil, err + } + + // Copied so the tunnel's routing table cannot change under the accept loop + // after it has started. + services := make(map[string]LocalService, len(cfg.Services)) + for tag, svc := range cfg.Services { + services[tag] = svc + } + + t := &Tunnel{ + endpoint: endpoint, + nodeID: cfg.NodeID, + token: cfg.Token, + services: services, + dialer: &websocket.Dialer{ + HandshakeTimeout: tunnelHandshakeTimeout, + // A worker reaches its frontend over the public internet in the + // deployments this exists for, so unlike the replica-to-replica + // peer link this DOES honour the environment's proxy settings. + Proxy: http.ProxyFromEnvironment, + }, + headerTimeout: cmp.Or(cfg.headerTimeout, tunnelHeaderTimeout), + sleep: cfg.sleep, + now: cfg.now, + done: make(chan struct{}), + } + if t.sleep == nil { + t.sleep = tunnelSleep + } + if t.now == nil { + t.now = time.Now + } + + loopCtx, cancel := context.WithCancel(ctx) + t.cancel = cancel + go func() { + defer close(t.done) + t.run(loopCtx) + }() + return t, nil +} + +// Close stops the tunnel and waits for its loop to finish. It is idempotent. +func (t *Tunnel) Close() error { + t.closeOnce.Do(func() { + t.cancel() + <-t.done + }) + return nil +} + +// run holds one session at a time, reconnecting with bounded backoff. +func (t *Tunnel) run(ctx context.Context) { + attempt := 0 + for { + if ctx.Err() != nil { + return + } + + start := t.now() + err := t.connectAndServe(ctx) + if ctx.Err() != nil { + return + } + + // A session that LASTED is the only evidence the frontend is healthy. + // See tunnelHealthyAfter for why "we connected" is not. + if t.now().Sub(start) >= tunnelHealthyAfter { + attempt = 0 + } + attempt++ + + delay := tunnelBackoffDelay(attempt) + t.logSessionEnded(err, attempt, delay) + if err := t.sleep(ctx, delay); err != nil { + return + } + } +} + +// connectAndServe dials, serves streams until the session ends, and leaves +// nothing running behind it. +func (t *Tunnel) connectAndServe(ctx context.Context) error { + ws, err := t.dial(ctx) + if err != nil { + return err + } + + sess, err := yamux.Client(cluster.WebsocketConn(ws), nil, nil) + if err != nil { + _ = ws.Close() + return fmt.Errorf("starting the worker tunnel session: %w", err) + } + xlog.Info("Worker tunnel established", "node", t.nodeID, "frontend", t.endpoint) + + // Streams are served under a context of the SESSION's, not the loop's. A + // stream goroutine parked in a local dial would otherwise outlive the + // session it belongs to and hold the reconnect below behind it. + sessCtx, endSession := context.WithCancel(ctx) + + // AcceptStream takes no context, so something else has to break it when the + // worker is shutting down; closing the session is that something. + watchdogDone := make(chan struct{}) + go func() { + defer close(watchdogDone) + select { + case <-sessCtx.Done(): + _ = sess.Close() + case <-sess.CloseChan(): + } + }() + + var streams sync.WaitGroup + serveErr := t.serve(sessCtx, sess, &streams) + + endSession() + _ = sess.Close() + <-watchdogDone + // Closing the session unblocks every stream goroutine: Session.close walks + // its stream table calling forceClose on each (session.go:334-338), and + // forceClose puts both directions in halfReset and calls notifyWaiting + // (stream.go:371-388), which wakes a parked Read and fails a parked Write. + // Waiting here is what keeps a reconnect from overlapping the streams of + // the session it replaced. + streams.Wait() + return serveErr +} + +// serve accepts streams until the session ends. +// +// One goroutine per stream, and an error from a stream never reaches this loop. +// A single malformed or unroutable request must not cost this worker every +// other request in flight on the same session. +func (t *Tunnel) serve(ctx context.Context, sess *yamux.Session, streams *sync.WaitGroup) error { + for { + stream, err := sess.AcceptStream() + if err != nil { + return err + } + streams.Add(1) + go func() { + defer streams.Done() + t.handleStream(ctx, stream) + }() + } +} + +// handleStream reads one stream's request frame and either splices it to a +// local service or refuses it. +func (t *Tunnel) handleStream(ctx context.Context, stream net.Conn) { + // A panic under one stream must not take the worker down, and here that is + // not a figure of speech: nothing supervises this goroutine, so an + // unrecovered panic ends the PROCESS, which ends the session and every + // other stream on it. Unlike the frontend's handler next door this does not + // re-panic, because there is no recovery middleware above it to report the + // panic; re-panicking would only be the crash. + // + // It covers what runs ON THIS goroutine: reading the request frame, the + // route lookup, and the local service's dial, which is the one of the three + // that runs caller-supplied code. It does NOT cover a panic inside Splice's + // own copy goroutines, which no recover here can reach. + defer func() { + if r := recover(); r != nil { + xlog.Error("Panic while serving a worker tunnel stream", "node", t.nodeID, "panic", r) + _ = stream.Close() + } + }() + + local, ok := t.accept(ctx, stream) + if !ok { + // accept has already answered and closed the stream. + return + } + + // Splice owns closing both ends from here. + if err := cluster.Splice(stream, local); err != nil { + xlog.Debug("worker tunnel stream ended with an error", "node", t.nodeID, "error", err) + } +} + +// accept reads the request frame and resolves it to a local connection. The +// second result is false when the stream was refused, in which case the refusal +// has been sent and the stream closed. +func (t *Tunnel) accept(ctx context.Context, stream net.Conn) (net.Conn, bool) { + // Deliberately time.Now and not t.now: this is an I/O deadline, and the + // clock seam exists only to measure how long a session lasted. + if err := stream.SetReadDeadline(time.Now().Add(t.headerTimeout)); err != nil { + // Nothing is readable on a stream whose deadline cannot be set, so this + // is reported as an infrastructure failure rather than pushed past. + t.refuse(stream, fmt.Errorf("%w: arming the request deadline: %v", cluster.ErrStreamTargetUnavailable, err)) + return nil, false + } + + tag, target, err := cluster.ReadStreamRequest(stream) + if err != nil { + // Includes the deadline above expiring. Both are "this stream never + // told me what it wanted", which is the frontend's problem to fix, not + // something a retry against this worker resolves. + t.refuse(stream, fmt.Errorf("%w: %v", cluster.ErrStreamRequestInvalid, err)) + return nil, false + } + + svc, known := t.services[tag] + if !known { + // A ROUTING fact about this worker, and it is reported as itself. A + // frontend that reads this knows a retry is pointless until the worker + // is upgraded, which is not what it should conclude from the + // unavailable below. + t.refuse(stream, fmt.Errorf("%w: %q", cluster.ErrStreamTagUnknown, tag)) + return nil, false + } + + // Cleared before the local dial rather than after the reply: everything + // past the request frame belongs to the tunnelled protocol, which brings + // its own deadlines, and one left armed here would abort a long inference + // stream in the middle. + if err := stream.SetReadDeadline(time.Time{}); err != nil { + t.refuse(stream, fmt.Errorf("%w: clearing the request deadline: %v", cluster.ErrStreamTargetUnavailable, err)) + return nil, false + } + + local, err := svc(ctx, target) + if err != nil { + // An INFRASTRUCTURE failure, which a frontend may retry, and which must + // never be reported as the unknown tag above. + t.refuse(stream, fmt.Errorf("%w: %v", cluster.ErrStreamTargetUnavailable, err)) + return nil, false + } + + if err := cluster.WriteStreamAccepted(stream); err != nil { + // The frontend never learns the stream was accepted, so it cannot be + // used; closing the local connection here is what stops an accepted + // backend connection leaking per failed reply. + xlog.Debug("worker tunnel could not accept a stream", "node", t.nodeID, "error", err) + _ = local.Close() + _ = stream.Close() + return nil, false + } + return local, true +} + +// refuse reports why a stream will not be served and then ENDS it. +// +// The close is the part that matters and it is not optional. A worker that says +// why and leaves the stream open has parked the frontend on a request that will +// never be answered, which reads as a slow worker rather than a refused +// request, and a deadline on the far side cannot tell those apart. The reply is +// what makes the refusal legible; the close is what makes it prompt. +// +// The reply is therefore best-effort and the close is not: a reply that could +// not be written still gets the stream closed. +func (t *Tunnel) refuse(stream net.Conn, reason error) { + if err := cluster.WriteStreamRefusal(stream, reason); err != nil { + xlog.Debug("worker tunnel could not report why it refused a stream", "node", t.nodeID, "error", err) + } + _ = stream.Close() + xlog.Debug("worker tunnel refused a stream", "node", t.nodeID, "reason", reason) +} + +// dial opens the WebSocket and returns it. +func (t *Tunnel) dial(ctx context.Context) (*websocket.Conn, error) { + // Read HERE, once per dial. See TunnelConfig.Token. + token := t.token() + if token == "" { + // Not a dial that fails with "unauthorized": this worker has no + // credential yet, which is a different condition from the frontend + // rejecting one, and an operator reading "unauthorized" would go + // looking for a token mismatch that does not exist. + return nil, errors.New("dialling the worker tunnel: this node has no tunnel credential yet") + } + header := http.Header{} + header.Set("Authorization", "Bearer "+token) + + ws, resp, err := t.dialer.DialContext(ctx, t.endpoint, header) + if err != nil { + if resp != nil { + // gorilla reports every non-101 as the same ErrBadHandshake, so + // without the status a 401, a 403 and a 503 are one log line. + defer func() { _ = resp.Body.Close() }() + return nil, &tunnelDialError{status: resp.StatusCode, cause: err} + } + return nil, fmt.Errorf("dialling the worker tunnel: %w", err) + } + return ws, nil +} + +// tunnelDialError carries the HTTP status a refused dial came back with, so the +// four refusals the frontend can give are not logged as one. +type tunnelDialError struct { + status int + cause error +} + +func (e *tunnelDialError) Error() string { + return fmt.Sprintf("dialling the worker tunnel: frontend answered %d: %v", e.status, e.cause) +} + +func (e *tunnelDialError) Unwrap() error { return e.cause } + +// logSessionEnded says why the tunnel is reconnecting, at a level that matches +// what the operator can do about it. +// +// The distinctions are the point rather than decoration. "Awaiting approval" +// and "your token is wrong" and "this frontend does not do tunnels" send an +// operator to three different places, and a worker retries all three the same +// way: none of them is a reason to stop, because a re-registration or an admin +// action fixes each without restarting the worker. +func (t *Tunnel) logSessionEnded(err error, attempt int, delay time.Duration) { + if err == nil { + xlog.Info("Worker tunnel closed, reconnecting", "node", t.nodeID, "attempt", attempt, "retry_in", delay) + return + } + + var dialErr *tunnelDialError + if errors.As(err, &dialErr) { + switch dialErr.status { + case http.StatusUnauthorized: + xlog.Warn("Frontend rejected this worker's tunnel credential; re-registering will mint a fresh one", + "node", t.nodeID, "retry_in", delay) + case http.StatusForbidden: + xlog.Info("Worker tunnel refused: this node is awaiting admin approval", + "node", t.nodeID, "retry_in", delay) + case http.StatusNotFound: + xlog.Debug("frontend does not serve worker tunnels, so it predates them", + "node", t.nodeID, "retry_in", delay) + case http.StatusServiceUnavailable: + xlog.Debug("frontend is not running in distributed mode, so it holds no worker tunnels", + "node", t.nodeID, "retry_in", delay) + default: + xlog.Warn("Worker tunnel dial refused", "node", t.nodeID, "status", dialErr.status, + "attempt", attempt, "retry_in", delay, "error", err) + } + return + } + xlog.Warn("Worker tunnel ended, reconnecting", "node", t.nodeID, "attempt", attempt, "retry_in", delay, "error", err) +} + +// tunnelBackoffDelay returns how long to wait before reconnect attempt n. +// +// Equal jitter: half the delay is fixed and half is drawn. Full jitter, which +// draws over the whole interval, can produce a near-zero wait, and a worker +// that can draw a near-zero wait can spin; keeping a floor means no single +// worker ever does, while the drawn half is what stops a fleet that all lost +// the same replica from resynchronising onto the same instant. +func tunnelBackoffDelay(attempt int) time.Duration { + if attempt < 1 { + attempt = 1 + } + d := tunnelBackoffMax + // The shift is guarded twice. The bound on attempt keeps the shift itself + // defined, and the positivity check catches the overflow that would + // otherwise turn a long outage into a NEGATIVE delay, which is a tight loop + // wearing a backoff's costume. + if attempt <= 40 { + if scaled := tunnelBackoffBase << (attempt - 1); scaled > 0 && scaled < tunnelBackoffMax { + d = scaled + } + } + return d/2 + time.Duration(rand.Int64N(int64(d/2)+1)) +} + +// tunnelSleep waits for d, or returns early when ctx is cancelled. +func tunnelSleep(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// tunnelEndpoint turns the frontend URL a worker registers against into the +// WebSocket URL it dials its tunnel on. +func tunnelEndpoint(frontendURL, nodeID string) (string, error) { + if frontendURL == "" { + return "", errors.New("starting the worker tunnel: no frontend URL") + } + u, err := url.Parse(frontendURL) + if err != nil { + return "", fmt.Errorf("starting the worker tunnel: parsing frontend URL %q: %w", frontendURL, err) + } + switch u.Scheme { + case "http", "ws": + u.Scheme = "ws" + case "https", "wss": + u.Scheme = "wss" + default: + return "", fmt.Errorf("starting the worker tunnel: frontend URL %q has scheme %q, want http or https", frontendURL, u.Scheme) + } + if u.Host == "" { + return "", fmt.Errorf("starting the worker tunnel: frontend URL %q has no host", frontendURL) + } + // Appended rather than assigned, so a frontend served under a path prefix + // keeps it. Registration builds its URLs the same way. + u.Path = strings.TrimRight(u.Path, "/") + cluster.ConnectPath + u.RawQuery = url.Values{"id": []string{nodeID}}.Encode() + return u.String(), nil +} + +// loopbackService routes a tagged stream to a process listening on this +// worker's own loopback interface. +// +// The HOST the frontend names is discarded and only the port is used, which is +// deliberate and is the security property this function exists for. A tunnel +// terminates inside the worker process, so a stream arriving on it can reach +// anything the worker can reach; without this, whoever holds the frontend end +// could make every worker in the fleet dial arbitrary hosts on its private +// network, turning the tunnel into a proxy into the worker's LAN. Discarding +// the host reduces the reachable set to this machine. +// +// It also happens to be what makes the tunnel work BEFORE the workers stop +// advertising themselves: today the frontend names the address the worker +// registered, which is a routable one, and after that change it will name a +// loopback one. Both resolve to the same place here. +// +// The port range is the one the worker's own port allocator hands to backend +// processes, so a stream cannot be pointed at some unrelated service that +// happens to be listening on this host. +func loopbackService(minPort, maxPort int) LocalService { + return func(ctx context.Context, target string) (net.Conn, error) { + _, portStr, err := net.SplitHostPort(target) + if err != nil { + return nil, fmt.Errorf("routing a tunnel stream: %q is not a host:port: %w", target, err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + return nil, fmt.Errorf("routing a tunnel stream: %q has no numeric port: %w", target, err) + } + if port < minPort || port > maxPort { + return nil, fmt.Errorf("routing a tunnel stream: port %d is outside this worker's backend range [%d, %d]", port, minPort, maxPort) + } + var d net.Dialer + return d.DialContext(ctx, "tcp", net.JoinHostPort("127.0.0.1", portStr)) + } +} + +// fixedService routes a tagged stream to one address on this worker, ignoring +// whatever the frontend named. +// +// There is exactly one HTTP server per worker and only the worker knows where +// it bound, so the frontend has nothing useful to say about the target and is +// not given the chance to say it. +func fixedService(addr string) LocalService { + return func(ctx context.Context, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "tcp", addr) + } +} + +// loopbackAddr rewrites a bind address into one that reaches the same listener +// from inside this process. +// +// A server bound to 0.0.0.0 is reachable on loopback, but dialling 0.0.0.0 is +// only accidentally equivalent to dialling localhost and is not on every +// platform, so the wildcard is replaced rather than dialled. +func loopbackAddr(bindAddr string) string { + host, port, err := net.SplitHostPort(bindAddr) + if err != nil { + return bindAddr + } + if host == "" || host == "0.0.0.0" || host == "::" || host == "[::]" { + return net.JoinHostPort("127.0.0.1", port) + } + return bindAddr +} diff --git a/core/services/worker/tunnel_test.go b/core/services/worker/tunnel_test.go new file mode 100644 index 000000000000..09285c401591 --- /dev/null +++ b/core/services/worker/tunnel_test.go @@ -0,0 +1,509 @@ +package worker + +import ( + "context" + "encoding/binary" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "time" + + "github.com/gorilla/websocket" + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/cluster" +) + +// awaitErr runs fn on its own goroutine and reports its result on a channel. +// +// Every blocking read in this file goes through it, and that is the single most +// load-bearing decision in the whole suite. The obvious way to assert "the +// worker refused this stream promptly" is to arm a read deadline and expect an +// error, and phase 1 shipped exactly that in three places: it held in none, +// because a stream the worker never answers AT ALL satisfies a deadline +// assertion just as well as one it refused. Reading with NO deadline, on +// another goroutine, and asserting the channel delivers, inverts that: a parked +// stream delivers nothing and the Eventually fails. +func awaitErr(fn func() error) <-chan error { + ch := make(chan error, 1) + go func() { ch <- fn() }() + return ch +} + +// tunnelDial is what the fake frontend saw on one incoming dial. +type tunnelDial struct { + token string + nodeID string +} + +// fakeFrontend is the far side of the tunnel: it speaks the real WebSocket +// upgrade and the real yamux server handshake, so these specs exercise the +// wire, not a mock of it. It is deliberately NOT core/http's handler; that one +// needs a database, and what is under test here is the client. +type fakeFrontend struct { + srv *httptest.Server + sessions chan *yamux.Session + dials chan tunnelDial + + // closeAtOnce makes every accepted session die immediately, which is what a + // frontend replica going down during a rolling restart looks like from the + // worker. + closeAtOnce bool +} + +func newFakeFrontend(closeAtOnce bool) *fakeFrontend { + f := &fakeFrontend{ + sessions: make(chan *yamux.Session, 64), + dials: make(chan tunnelDial, 256), + closeAtOnce: closeAtOnce, + } + upgrader := websocket.Upgrader{} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != cluster.ConnectPath { + w.WriteHeader(http.StatusNotFound) + return + } + select { + case f.dials <- tunnelDial{ + token: strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "), + nodeID: r.URL.Query().Get("id"), + }: + default: + } + + ws, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + sess, err := yamux.Server(cluster.WebsocketConn(ws), nil, nil) + if err != nil { + _ = ws.Close() + return + } + if f.closeAtOnce { + _ = sess.Close() + return + } + select { + case f.sessions <- sess: + default: + _ = sess.Close() + } + })) + return f +} + +func (f *fakeFrontend) close() { + for { + select { + case sess := <-f.sessions: + _ = sess.Close() + default: + f.srv.Close() + return + } + } +} + +// echoListener is a stand-in for a backend gRPC process on the worker: a local +// TCP listener that reads and writes back. +func echoListener() net.Listener { + ln, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).ToNot(HaveOccurred()) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + defer func() { _ = conn.Close() }() + _, _ = io.Copy(conn, conn) + }() + } + }() + return ln +} + +// dialLocalTCP is the simplest possible LocalService: connect to whatever the +// frontend named. +func dialLocalTCP(ctx context.Context, target string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "tcp", target) +} + +var _ = Describe("Worker tunnel client", func() { + var ( + ctx context.Context + cancel context.CancelFunc + frontend *fakeFrontend + tunnel *Tunnel + ) + + BeforeEach(func() { + ctx, cancel = context.WithCancel(context.Background()) + }) + + AfterEach(func() { + if tunnel != nil { + Expect(tunnel.Close()).To(Succeed()) + tunnel = nil + } + cancel() + if frontend != nil { + frontend.close() + frontend = nil + } + }) + + // start brings up the client against the fake frontend already created. + start := func(mutate func(*TunnelConfig)) { + cfg := TunnelConfig{ + FrontendURL: frontend.srv.URL, + NodeID: "node-1", + Token: func() string { return "tunnel-secret" }, + Services: map[string]LocalService{}, + } + if mutate != nil { + mutate(&cfg) + } + var err error + tunnel, err = StartTunnel(ctx, cfg) + Expect(err).ToNot(HaveOccurred()) + } + + // session waits for the frontend to have accepted the worker's dial. + session := func() *yamux.Session { + var sess *yamux.Session + EventuallyWithOffset(1, frontend.sessions, "10s").Should(Receive(&sess)) + return sess + } + + Describe("carrying a tagged stream to a local service", func() { + It("routes a stream tagged for gRPC to the local address it names", func() { + ln := echoListener() + DeferCleanup(func() { _ = ln.Close() }) + + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services[cluster.StreamTagGRPC] = dialLocalTCP + }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagGRPC, ln.Addr().String())).To(Succeed()) + + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + Eventually(reply, "10s").Should(Receive(BeNil())) + + _, err = stream.Write([]byte("ping")) + Expect(err).ToNot(HaveOccurred()) + + buf := make([]byte, 4) + read := awaitErr(func() error { + _, err := io.ReadFull(stream, buf) + return err + }) + Eventually(read, "10s").Should(Receive(BeNil())) + Expect(string(buf)).To(Equal("ping")) + }) + }) + + Describe("refusing a stream it cannot serve", func() { + // The refusal specs all read with NO deadline, on another goroutine. + // See awaitErr: a deadline would be satisfied by a stream that was + // merely parked, which is the exact defect this phase inherited. + + It("refuses an unknown tag promptly, and the stream ENDS rather than parking", func() { + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services[cluster.StreamTagGRPC] = dialLocalTCP + }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(stream, "no-such-tag", "")).To(Succeed()) + + // Two facts, in order, on one goroutine: the worker SAID why, and + // then the stream ended. A worker that only says why and leaves the + // stream open never sends on this channel, so the Eventually below + // fails rather than passing on a deadline. + type outcome struct{ reply, end error } + done := make(chan outcome, 1) + go func() { + var got outcome + got.reply = cluster.ReadStreamReply(stream) + _, got.end = stream.Read(make([]byte, 1)) + done <- got + }() + + var got outcome + Eventually(done, "10s").Should(Receive(&got)) + Expect(got.reply).To(MatchError(cluster.ErrStreamTagUnknown)) + Expect(got.end).To(MatchError(io.EOF)) + }) + + It("reports a local service it could not reach as unavailable, not as an unknown tag", func() { + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services[cluster.StreamTagGRPC] = func(context.Context, string) (net.Conn, error) { + return nil, fmt.Errorf("connection refused") + } + }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagGRPC, "127.0.0.1:1")).To(Succeed()) + + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + var got error + Eventually(reply, "10s").Should(Receive(&got)) + // Distinct conditions must not be reported as each other: a caller + // gives up on an unknown tag and retries an unavailable target. + Expect(got).To(MatchError(cluster.ErrStreamTargetUnavailable)) + Expect(got).ToNot(MatchError(cluster.ErrStreamTagUnknown)) + }) + + It("ends a stream whose request never arrives instead of holding it open", func() { + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services[cluster.StreamTagGRPC] = dialLocalTCP + c.headerTimeout = 50 * time.Millisecond + }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + // Nothing is written. A worker that waits forever for a request it + // will never get holds a goroutine and a stream slot per dial. + ended := awaitErr(func() error { + _, err := io.Copy(io.Discard, stream) + return err + }) + Eventually(ended, "10s").Should(Receive(BeNil())) + }) + }) + + Describe("surviving a bad stream", func() { + It("keeps serving the session after one stream it could not read", func() { + ln := echoListener() + DeferCleanup(func() { _ = ln.Close() }) + + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services[cluster.StreamTagGRPC] = dialLocalTCP + }) + sess := session() + + // A frame that declares far more than it sends, then hangs up. The + // worker cannot parse it and must not take the session down with it. + bad, err := sess.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + var hdr [2]byte + binary.BigEndian.PutUint16(hdr[:], 900) + _, err = bad.Write(append(hdr[:], []byte("gr")...)) + Expect(err).ToNot(HaveOccurred()) + Expect(bad.CloseWrite()).To(Succeed()) + badEnded := awaitErr(func() error { + _, err := io.Copy(io.Discard, bad) + return err + }) + Eventually(badEnded, "10s").Should(Receive(BeNil())) + + // Same session, a stream the worker can serve. + good, err := sess.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(good, cluster.StreamTagGRPC, ln.Addr().String())).To(Succeed()) + reply := awaitErr(func() error { return cluster.ReadStreamReply(good) }) + Eventually(reply, "10s").Should(Receive(BeNil())) + + _, err = good.Write([]byte("still here")) + Expect(err).ToNot(HaveOccurred()) + buf := make([]byte, len("still here")) + read := awaitErr(func() error { + _, err := io.ReadFull(good, buf) + return err + }) + Eventually(read, "10s").Should(Receive(BeNil())) + Expect(string(buf)).To(Equal("still here")) + }) + It("serves streams concurrently, so one live stream does not block the next", func() { + ln := echoListener() + DeferCleanup(func() { _ = ln.Close() }) + + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services[cluster.StreamTagGRPC] = dialLocalTCP + }) + sess := session() + + // The first stream is accepted and then left open with nothing + // flowing, which is what an idle inference stream or a paused file + // transfer looks like. Serving streams from the accept loop rather + // than a goroutine each would park every later request behind it. + first, err := sess.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(first, cluster.StreamTagGRPC, ln.Addr().String())).To(Succeed()) + firstReply := awaitErr(func() error { return cluster.ReadStreamReply(first) }) + Eventually(firstReply, "10s").Should(Receive(BeNil())) + + second, err := sess.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(second, cluster.StreamTagGRPC, ln.Addr().String())).To(Succeed()) + secondReply := awaitErr(func() error { return cluster.ReadStreamReply(second) }) + Eventually(secondReply, "10s").Should(Receive(BeNil())) + }) + + It("keeps the session after a local service panics", func() { + ln := echoListener() + DeferCleanup(func() { _ = ln.Close() }) + + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services["explodes"] = func(context.Context, string) (net.Conn, error) { + panic("a local service blew up") + } + c.Services[cluster.StreamTagGRPC] = dialLocalTCP + }) + sess := session() + + // Nothing supervises a stream goroutine, so an unrecovered panic + // here ends the process, which is the loudest possible way to kill + // the session. + boom, err := sess.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(boom, "explodes", "")).To(Succeed()) + boomEnded := awaitErr(func() error { + _, err := io.Copy(io.Discard, boom) + return err + }) + Eventually(boomEnded, "10s").Should(Receive(BeNil())) + + good, err := sess.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(good, cluster.StreamTagGRPC, ln.Addr().String())).To(Succeed()) + reply := awaitErr(func() error { return cluster.ReadStreamReply(good) }) + Eventually(reply, "10s").Should(Receive(BeNil())) + }) + }) + + Describe("reconnecting", func() { + It("backs off exponentially between reconnects, bounded and never tight", func() { + frontend = newFakeFrontend(true) // every session dies at once + + delays := make(chan time.Duration, 64) + start(func(c *TunnelConfig) { + c.sleep = func(ctx context.Context, d time.Duration) error { + select { + case delays <- d: + default: + } + return ctx.Err() + } + }) + + observed := make([]time.Duration, 0, 10) + for i := 0; i < 10; i++ { + var d time.Duration + Eventually(delays, "20s").Should(Receive(&d), "expected reconnect attempt %d", i+1) + observed = append(observed, d) + } + + for i, d := range observed { + // Never a tight loop: a worker that reconnect-storms a frontend + // during a rolling restart is a denial of service against the + // control plane. + Expect(d).To(BeNumerically(">", 0), "delay %d was not positive", i+1) + // Bounded: without a ceiling a worker that misses a rolling + // restart backs off into hours and never comes back. + Expect(d).To(BeNumerically("<=", tunnelBackoffMax), "delay %d exceeded the ceiling", i+1) + } + // And it actually grows. The jitter has a floor of half the + // unjittered delay, so the fourth attempt is at least 4x the base + // however the dice fall. + Expect(observed[3]).To(BeNumerically(">=", 4*tunnelBackoffBase)) + }) + + It("keeps backing off after a session that died at once", func() { + frontend = newFakeFrontend(true) + + delays := make(chan time.Duration, 64) + start(func(c *TunnelConfig) { + c.sleep = func(ctx context.Context, d time.Duration) error { + select { + case delays <- d: + default: + } + return ctx.Err() + } + }) + + var last time.Duration + for i := 0; i < 6; i++ { + Eventually(delays, "20s").Should(Receive(&last)) + } + // A session that came up and died immediately is not evidence the + // frontend is healthy, so the delay must NOT be back at the floor. + Expect(last).To(BeNumerically(">", tunnelBackoffBase)) + }) + + It("returns to its shortest delay after a session that lasted", func() { + frontend = newFakeFrontend(true) + + // A clock that jumps a minute on every reading. The loop reads it + // once when a session comes up and once when it ends, so every + // session looks like it lasted a minute, which is longer than the + // threshold below which a session is not counted as healthy. + var ticks atomic.Int64 + base := time.Now() + + delays := make(chan time.Duration, 64) + start(func(c *TunnelConfig) { + c.now = func() time.Time { + return base.Add(time.Duration(ticks.Add(1)) * time.Minute) + } + c.sleep = func(ctx context.Context, d time.Duration) error { + select { + case delays <- d: + default: + } + return ctx.Err() + } + }) + + for i := 0; i < 6; i++ { + var d time.Duration + Eventually(delays, "20s").Should(Receive(&d), "expected reconnect attempt %d", i+1) + Expect(d).To(BeNumerically("<=", tunnelBackoffBase), + "delay %d did not return to the floor after a session that lasted", i+1) + } + }) + + It("presents the credential current at DIAL time, not the one it started with", func() { + frontend = newFakeFrontend(true) + + var issued atomic.Int64 + start(func(c *TunnelConfig) { + c.Token = func() string { return fmt.Sprintf("token-%d", issued.Add(1)) } + c.sleep = func(ctx context.Context, _ time.Duration) error { return ctx.Err() } + }) + + // Nothing survives a reconnect: the new owner replica has no record + // of the old session, and the worker's own credential may have been + // rotated by a re-registration in between. A client that captured + // its token once locks itself out on the first rotation. + var first, second tunnelDial + Eventually(frontend.dials, "20s").Should(Receive(&first)) + Eventually(frontend.dials, "20s").Should(Receive(&second)) + Expect(first.token).To(Equal("token-1")) + Expect(second.token).To(Equal("token-2")) + Expect(first.nodeID).To(Equal("node-1")) + Expect(second.nodeID).To(Equal("node-1")) + }) + }) +}) diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index b409804626c2..842485cfbc0f 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -17,6 +17,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" grpc "github.com/mudler/LocalAI/pkg/grpc" @@ -96,13 +97,25 @@ func Run(ctx *cliContext.Context, cfg *Config) error { var ( nodeID string connectNats func() (*messaging.Client, error) + // tunnelToken reads the node's CURRENT tunnel credential. It is a + // function because the frontend rotates the credential on every + // registration, so the value a reconnect must present is not + // necessarily the one this worker started with. + tunnelToken func() string ) if cfg.NatsJWT != "" || cfg.NatsUserSeed != "" { - nid, _, _, _, regErr := regClient.RegisterWithRetry(shutdownCtx, registrationBody, 10) + res, regErr := regClient.RegisterFullWithRetry(shutdownCtx, registrationBody, 10) if regErr != nil { return fmt.Errorf("failed to register with frontend: %w", regErr) } - nodeID = nid + nodeID = res.ID + // This path registers exactly once and never again, so the credential + // it holds cannot go stale by rotation from its own side. It can still + // be superseded from OUTSIDE, by a second worker registering under the + // same node name, and there is nothing this worker can do about that + // but log the 401 and keep retrying. + staticTunnelToken := res.TunnelToken + tunnelToken = func() string { return staticTunnelToken } connectNats = func() (*messaging.Client, error) { return connectNATS(cfg.NatsURL, cfg.NatsJWT, cfg.NatsUserSeed, "", "", cfg.NatsAuthRequired(), natsTLS) } @@ -118,6 +131,10 @@ func Run(ctx *cliContext.Context, cfg *Config) error { return fmt.Errorf("failed to register with frontend: %w", regErr) } nodeID = res.ID + // The manager re-registers to refresh NATS credentials, and every + // registration rotates the tunnel credential too, so this reads the + // manager rather than capturing a value. + tunnelToken = credMgr.TunnelToken connectNats = func() (*messaging.Client, error) { var opts []messaging.Option if credMgr.HasCredentials() { @@ -181,6 +198,41 @@ func Run(ctx *cliContext.Context, cfg *Config) error { return fmt.Errorf("starting HTTP file transfer server: %w", err) } + // The tunnel is started here, after the HTTP server it fronts is listening + // and before any backend process exists. Both orders are deliberate: a + // stream tagged for HTTP that arrived before the server bound would be + // refused as unavailable, while a stream tagged for gRPC resolves its + // backend at dial time, so nothing has to exist yet for the tunnel to be + // useful. + // + // A failure to START it is fatal, unlike a failure to CONNECT: it means the + // frontend URL or this node's identity is unusable, and a worker that + // silently ran without its tunnel would look healthy while being + // unreachable to everything that dials through it. + tunnelBasePort := cfg.effectiveBasePort() + if cfg.WorkerTunnel { + tunnel, terr := StartTunnel(shutdownCtx, TunnelConfig{ + FrontendURL: cfg.RegisterTo, + NodeID: nodeID, + Token: tunnelToken, + Services: map[string]LocalService{ + // The frontend names a backend process by its port; the worker + // decides that only its own loopback, and only within its own + // backend port range, is reachable through it. + cluster.StreamTagGRPC: loopbackService(tunnelBasePort, cfg.effectiveMaxPort(tunnelBasePort)), + cluster.StreamTagHTTP: fixedService(loopbackAddr(httpAddr)), + }, + }) + if terr != nil { + nodes.ShutdownFileTransferServer(httpServer) + return fmt.Errorf("starting the worker tunnel: %w", terr) + } + defer func() { + if err := tunnel.Close(); err != nil { + xlog.Warn("Closing the worker tunnel failed", "error", err) + } + }() + } // Connect to NATS xlog.Info("Connecting to NATS", "url", sanitize.URL(cfg.NatsURL)) natsClient, err := connectNats() @@ -212,7 +264,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error { }() // Process supervisor — manages multiple backend gRPC processes on different ports - basePort := cfg.effectiveBasePort() + basePort := tunnelBasePort // Buffered so NATS stop handler can send without blocking sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 59cf9ebfc48c..1a56516429e4 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -106,20 +106,43 @@ The peer link is served at `/api/cluster/peer` and authenticates with `LOCALAI_R A worker can open one long-lived, multiplexed tunnel to the frontend instead of listening on a port of its own. It dials `GET /api/cluster/connect?id=`, the connection is upgraded to a WebSocket, and every subsequent request the frontend makes to that worker travels as a stream inside it. Nothing dials *into* the worker, so a worker behind NAT, in another Kubernetes cluster or on a laptop needs no inbound port and no reachable address. -The dial is authenticated against **the hash stored on that node's row, which today is still the registration token's hash**: the frontend hashes the presented bearer token and compares it with what registration recorded. A worker that presents a token belonging to no node, or names a node ID the frontend has never seen, is refused with `401` before the WebSocket upgrade happens. A node still awaiting admin approval is refused with `403`. A frontend that cannot read its node table answers `500` rather than `401`, so a worker retries instead of re-registering under a new identity. +#### Each worker has its own tunnel credential -So the isolation is not there yet, and the caveat is worth stating plainly: because a worker registers by presenting the deployment's registration token, a leaked registration token plus a known node ID still gets a tunnel. What the tunnel endpoint does not do is trust the configured token directly, so when workers are issued their own per-node secrets and those secrets are what registration stores, the isolation becomes real with no change to this route. +The dial is authenticated against **that node's own tunnel credential**, which is not the registration token. Registration mints a fresh random secret per node, returns the plaintext once in the registration response as `tunnel_token`, and stores only its SHA-256. So a leaked registration token no longer opens a tunnel: an attacker who has it, and who knows a node ID, still cannot authenticate as that worker. -**Worker tunnels need `LOCALAI_REGISTRATION_TOKEN` set.** A worker registering against a frontend with no registration token configured sends no token, so nothing is stored on its row, and the tunnel has nothing to authenticate it against: every dial is refused with `401`. LocalAI warns about this at startup. Setting the token later is not enough on its own; the workers have to register again for the hash to be written. +A worker that presents a credential belonging to no node, or names a node ID the frontend has never seen, is refused with `401` before the WebSocket upgrade happens. A node still awaiting admin approval is refused with `403`. A frontend that cannot read its node table answers `500` rather than `401`, so a worker retries instead of re-registering under a new identity. + +**The credential is rotated on every registration.** That follows from storing only the hash: a re-registering worker cannot be told the secret it already holds, so it is given a new one. The worker's live tunnel is unaffected, because the credential is checked when a tunnel is *dialled* and never again; what changes is which secret the next reconnect presents, and the worker learns it in the same response that rotated it. + +**A node that has not registered since upgrading cannot tunnel.** Its row has no tunnel credential and the column cannot be back-filled, because the plaintext only ever existed in the response that minted it. Such a node is refused with `401` until it registers again, which a worker restart does. The frontend does *not* fall back to the registration token for these nodes. + +Unlike the agent worker's API key and its NATS credential, a tunnel credential **is** issued to a node still awaiting approval. It is inert until then: the tunnel route re-reads the node's status on every dial and refuses a pending one. Withholding it would instead strand workers that register exactly once, since approval on its own prompts no re-registration. + +A tunnel credential does not replace `LOCALAI_REGISTRATION_TOKEN`. Without one, node registration itself is unauthenticated, so anyone who can reach the frontend can register a worker and be issued a working tunnel credential for it. LocalAI warns about that at startup. The tunnel lands on exactly one frontend replica, and that replica records itself as the owner of the worker's connection in the `node_connections` table. When the socket dies the claim is dropped with it. If the replica stalls long enough for its peers to reap it, it re-claims the tunnels it still holds on a live session as soon as it re-registers, skipping any whose socket has already gone. That re-claim needs the replica to have an advertised address: without one it never had an instance row to begin with, and its tunnels are usable only by the replica holding them. | Method | Path | Description | |--------|------|-------------| -| `GET` | `/api/cluster/connect?id=` | Worker opens its multiplexed tunnel (`Authorization: Bearer `) | +| `GET` | `/api/cluster/connect?id=` | Worker opens its multiplexed tunnel (`Authorization: Bearer `) | The route is exempt from the normal session/API-key authentication (it authenticates itself, like `/api/cluster/peer`) and is registered in every deployment. Outside distributed mode there is no node table to check a token against, so it answers `503`. +#### What the worker does with the tunnel + +The worker holds the tunnel with one goroutine: it dials, serves the frontend's streams until the session dies, and dials again. Every stream opens with a small frame naming which local service it is for, and the worker answers before either side speaks the tunnelled protocol: + +| Tag | Goes to | Target | +|-----|---------|--------| +| `grpc` | a backend process on this worker | the port; the host is discarded and only `127.0.0.1` is dialled, within the worker's own backend port range | +| `http` | the worker's own file-transfer and backend-log server | ignored; there is one such server and only the worker knows where it bound | + +A stream naming a tag the worker does not serve, or a local service it could not reach, is refused with a reason and the stream is **ended** rather than left open. Those two refusals are distinct on the wire on purpose: a frontend gives up on the first and may retry the second. One bad stream never affects the others or the session. + +Reconnects use exponential backoff with jitter: the interval doubles from 500ms up to a ceiling of 30 seconds, and each wait is drawn between half of that interval and all of it, so no worker ever spins and a fleet that lost the same replica does not come back in lockstep. The interval returns to its floor only after a session that lasted at least 30 seconds. That last part is what stops a rolling frontend restart, where every dial succeeds and then dies moments later, from turning a fleet of workers into a retry storm against the first replica back up. A worker that is refused (`401`, `403`) keeps retrying on the same schedule rather than exiting: a re-registration or an admin approval fixes both without restarting it. + +Set `LOCALAI_WORKER_TUNNEL=false` on a worker to turn the tunnel off and go back to the frontend dialling the worker's advertised addresses. + ### The model load deadline scales with the checkpoint The `LoadModel` deadline starts *after* the backend is installed and the model files are staged, so it covers only the worker backend's own checkpoint read and pipeline init. That work is proportional to the bytes on disk, which makes any fixed deadline a model-size cliff rather than a timeout: a 70 GB video checkpoint on a Jetson Thor worker failed reproducibly against the old fixed 5m default (`rpc error: code = DeadlineExceeded` after 953.5s of wall clock, roughly 11m of which was backend install and staging), and simply raising the constant would only move the cliff to the next larger model while making a genuinely wedged *small* model hang for the whole inflated duration. @@ -345,6 +368,7 @@ local-ai worker \ | `--registration-require-auth` | `LOCALAI_REGISTRATION_REQUIRE_AUTH` | `false` | Refuse to start the HTTP file-transfer server when no registration token is set (it would otherwise fail open) | | `--distributed-require-auth` | `LOCALAI_DISTRIBUTED_REQUIRE_AUTH` | `false` | Umbrella switch implying both `--registration-require-auth` and `--nats-require-auth` | | `--heartbeat-interval` | `LOCALAI_HEARTBEAT_INTERVAL` | `10s` | Interval between heartbeat pings | +| `--worker-tunnel` | `LOCALAI_WORKER_TUNNEL` | `true` | Hold one outbound multiplexed tunnel to the frontend and serve its requests over it, so this worker needs no inbound port (see [Worker tunnels](#worker-tunnels)) | | `--nats-url` | `LOCALAI_NATS_URL` | *(required)* | NATS URL for backend installation and file staging | | `--nats-jwt` | `LOCALAI_NATS_JWT` | *(empty)* | Optional override for the `nats_jwt` returned at registration | | `--nats-user-seed` | `LOCALAI_NATS_USER_SEED` | *(empty)* | Optional override for `nats_user_seed` from registration | From 9bf2013428a27507626fdd25930448f91c3690ca Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 13:03:54 +0000 Subject: [PATCH 050/153] fix(worker): spec the tunnel's routing table, which was the SSRF boundary Review follow-up. One blocking finding and seven others. The blocking one first, and it is this project's recurring shape: the untested path. loopbackService is the function whose comment calls the discarded host "the security property this function exists for", and nothing tested it. The reviewer replaced its body with a dial of whatever the frontend named, no port range, and all 131 specs passed. Every spec installed the permissive test dialler, so the real routing table was exercised nowhere. It now has specs, and the property is stated as reachability rather than as a property of the code: a listener on 127.0.0.2 that only the frontend's target names must NOT be reached. Plus the port-range table, fixedService, loopbackAddr, tunnelEndpoint, and the table itself, which moved out of Run into tunnelServices so it can be built without starting a worker. One spec drives a real stream through that table over the wire, so the routing rules are exercised end to end at least once rather than only in isolation. The reviewer's mutation now reddens ten specs, and six narrower ones redden between two and four each, so no spec is riding on another. The shape changed too, not only the coverage. The dial address is built from a loopbackHost constant and strconv.Itoa of a validated int, so nothing derived from the wire reaches DialContext at all: restoring the hole takes ADDING a data flow, not deleting a check. And a taxonomy fix found while specifying it. A port outside this worker's allocator range was reported as unavailable, which tells a frontend to retry something that can never work. It is a bad request now, and a backend that is merely not listening yet stays unavailable, which is the retryable one. Agent nodes no longer get a tunnel credential. Nothing dials into an agent worker, so a tunnel replaces nothing for it and no client would open one, and the gate is at the mint site rather than in the handler: with no credential minted the hash stays empty and the existing empty-hash refusal covers it, so enforcement is structural. Two comments and one doc paragraph said an anonymous registrant gets a "working" credential. With auto-approve off the node is pending and the credential is inert, which is the distinction this same change argues three files away to justify minting for pending nodes at all. A refusal reason over the frame limit was cut on a byte boundary and could split a rune. It cuts on a rune boundary now, and the code survives truncation, which is what keeps a refusal classifiable. Also: the pending-node spec asserted only that a credential was non-empty, so a credential derived from the shared token passed it; it now pins per-node-ness the way the headline spec does. The tunnel handler's citations into nodes.go were stale before this branch landed, having been written against a file the same commit was editing, and are by function name now. The static-NATS path says plainly that an externally forced rotation locks it out until restart, and where that gets fixed. tunnelproto gained direct specs, including that a read failure is never reported as a refusal. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/app.go | 13 +- core/http/endpoints/cluster/connect.go | 9 +- core/http/endpoints/localai/nodes.go | 17 +- core/http/endpoints/localai/nodes_test.go | 40 +++- core/services/cluster/tunnelproto.go | 30 ++- core/services/cluster/tunnelproto_test.go | 186 +++++++++++++++ core/services/worker/tunnel.go | 73 +++++- core/services/worker/tunnel_test.go | 279 ++++++++++++++++++++++ core/services/worker/worker.go | 33 +-- docs/content/features/distributed-mode.md | 8 +- 10 files changed, 651 insertions(+), 37 deletions(-) create mode 100644 core/services/cluster/tunnelproto_test.go diff --git a/core/http/app.go b/core/http/app.go index 6bfb3ac8d0bb..40d14b7a4a9f 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -583,8 +583,17 @@ func API(application *application.Application) (*echo.Echo, error) { // credential at registration whether or not one is configured. What // is missing is the gate in FRONT of that. With no registration // token, RegisterNodeEndpoint validates nothing, so anyone who can - // reach this frontend can register a node and be handed a working - // tunnel credential for it. + // reach this frontend can register a node and be issued a tunnel + // credential for it. + // + // How far that gets them depends on the OTHER knob. With + // auto-approve on, the node is healthy at once and the credential + // works immediately. With it off, the node is pending, and the + // tunnel route refuses a pending node on every dial, so the + // credential is inert until an admin approves it and approval is + // the real gate. Worth stating precisely, because the same commit + // argues exactly this distinction three files away to justify + // minting for pending nodes at all. // // This warning replaced one that said the opposite, that tunnels // would refuse every dial without this token. That was true while diff --git a/core/http/endpoints/cluster/connect.go b/core/http/endpoints/cluster/connect.go index 5e6ace97f7b6..8d175704669c 100644 --- a/core/http/endpoints/cluster/connect.go +++ b/core/http/endpoints/cluster/connect.go @@ -130,8 +130,13 @@ func ConnectHandler(registry *nodes.NodeRegistry, tunnels *clustersvc.TunnelRegi // Only StatusPending is refused. The rest of /api/node/ self-service // gates on nothing at all, but the two places that hand a node something // DURABLE both refuse a pending one: the agent worker's API key - // (core/http/endpoints/localai/nodes.go:224) and its NATS credential - // (nodes.go:293). A tunnel is that kind of grant, not a heartbeat: it is + // (provisionAgentWorkerKey, guarded at its call site in + // core/http/endpoints/localai/nodes.go) and its NATS credential + // (attachNatsJWT in the same file). Cited by NAME, not by line: the + // previous version of this comment cited line numbers into a file this + // same commit was editing, and both were stale before it landed. + // + // A tunnel is that kind of grant, not a heartbeat: it is // a standing pipe into the worker recorded in node_connections and // relayed to by every other replica. Draining and unhealthy nodes keep // their tunnels on purpose; draining means finish what you have, and a diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index 0b42b8314365..5ad21d724e71 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -312,6 +312,21 @@ func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecr // path in core/services/worker/worker.go does), because approval alone does not // prompt a re-registration and nothing else can hand it the secret. // +// Only BACKEND nodes get one, and that is a decision rather than an oversight. +// An agent worker serves no gRPC backends and no file staging; nothing dials +// into it at all, so a tunnel replaces nothing for it and there is no client on +// the agent side that would ever open one. Minting anyway would hand out a +// working credential for a pipe nobody drives, which is surface without a +// feature, and it would contradict every comment in this change that says +// "backend workers, the ones that tunnel". +// +// The gate lives HERE and not in ConnectHandler, which never looks at NodeType. +// It does not need to: an agent node's tunnel credential is never minted, so +// its TunnelTokenHash stays empty and the handler's empty-hash branch refuses +// it like any other node without one. Enforcement is therefore structural. The +// day agent workers want a tunnel, relaxing this condition is the whole change, +// and it has to be a deliberate one. +// // A failure to mint or to store is logged and the response goes out without the // token. Registration is what gets a worker into the cluster at all, and // failing it over a credential the worker does not need until it tunnels would @@ -319,7 +334,7 @@ func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecr // tunnel_token, reports that it has no credential, and retries at its next // registration. func attachTunnelToken(ctx context.Context, response map[string]any, registry *nodes.NodeRegistry, node *nodes.BackendNode) { - if node == nil { + if node == nil || node.NodeType != nodes.NodeTypeBackend { return } // crypto/rand.Text: at least 128 bits of randomness, no error to handle and diff --git a/core/http/endpoints/localai/nodes_test.go b/core/http/endpoints/localai/nodes_test.go index dc1a77f465a7..ea10b0ccc17a 100644 --- a/core/http/endpoints/localai/nodes_test.go +++ b/core/http/endpoints/localai/nodes_test.go @@ -183,15 +183,49 @@ var _ = Describe("Node HTTP handlers", func() { }) It("issues a tunnel credential to a node still awaiting approval", func() { - resp := register(`{"name":"worker-pending","address":"10.0.0.5:50051"}`, "", false) - Expect(resp["status"]).To(Equal(nodes.StatusPending)) // Deliberately unlike the agent API key and the NATS JWT, which are // both withheld from a pending node. Those work the moment they are // issued; this one does not, because the tunnel endpoint re-reads // the node's status on every dial and refuses a pending node. A // worker that registers exactly once would otherwise never receive // one, since approval alone prompts no re-registration. - Expect(resp["tunnel_token"]).ToNot(BeEmpty()) + first := register(`{"name":"worker-pending","address":"10.0.0.5:50051","token":"shared"}`, "shared", false) + Expect(first["status"]).To(Equal(nodes.StatusPending)) + plaintext, _ := first["tunnel_token"].(string) + Expect(plaintext).ToNot(BeEmpty()) + + // Non-empty alone does not pin per-node-ness, and a review's + // variant of the "derived from the shared token" mutation stayed + // green on exactly that gap. A pending node's credential has to be + // as unpredictable and as per-node as an approved one's, since it + // becomes live the moment an admin approves. + Expect(plaintext).ToNot(Equal("shared")) + node, err := registry.Get(context.Background(), first["id"].(string)) + Expect(err).ToNot(HaveOccurred()) + Expect(node.TunnelTokenHash).To(Equal(hashOf(plaintext))) + Expect(node.TunnelTokenHash).ToNot(Equal(node.TokenHash)) + + second := register(`{"name":"worker-pending-2","address":"10.0.0.5:50052","token":"shared"}`, "shared", false) + Expect(second["status"]).To(Equal(nodes.StatusPending)) + Expect(second["tunnel_token"]).ToNot(Equal(plaintext)) + }) + + It("does not issue a tunnel credential to an agent node", func() { + // An agent worker serves no gRPC backends and no file staging; + // nothing dials into it, so a tunnel replaces nothing for it and no + // client on its side would open one. Minting anyway would be + // credential surface with no feature behind it. + // + // Enforcement is structural rather than a second check: with no + // credential minted, the node's hash stays empty and the tunnel + // route refuses it like any other node without one. + resp := register(`{"name":"agent-1","node_type":"agent"}`, "", true) + Expect(resp["node_type"]).To(Equal(nodes.NodeTypeAgent)) + Expect(resp).ToNot(HaveKey("tunnel_token")) + + node, err := registry.Get(context.Background(), resp["id"].(string)) + Expect(err).ToNot(HaveOccurred()) + Expect(node.TunnelTokenHash).To(BeEmpty()) }) It("returns nats_jwt when account seed is configured", func() { diff --git a/core/services/cluster/tunnelproto.go b/core/services/cluster/tunnelproto.go index 39f5014b50b4..c6433459f78c 100644 --- a/core/services/cluster/tunnelproto.go +++ b/core/services/cluster/tunnelproto.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "strings" + "unicode/utf8" ) // The framing every stream on a worker tunnel opens with. @@ -154,10 +155,7 @@ func WriteStreamRefusal(w io.Writer, reason error) error { }, reason.Error()) } frame := replyPrefixRefused + code + streamRequestSeparator + text - if len(frame) > maxTunnelFrame { - frame = frame[:maxTunnelFrame] - } - return writeFrame(w, frame) + return writeFrame(w, truncateRunes(frame, maxTunnelFrame)) } // ReadStreamReply reads the worker's answer. nil means the stream is now @@ -196,6 +194,30 @@ func ReadStreamReply(r io.Reader) error { } } +// truncateRunes cuts s to at most limit BYTES, on a rune boundary. +// +// A plain slice would cut mid-rune and put a lone continuation byte on the +// wire. Nothing breaks: the frame is length-prefixed so the framing survives, +// and the reader's string() tolerates invalid UTF-8. What it costs is the +// far side's log line ending in a replacement character, and a refusal reason +// exists to be read by a person, so it should not arrive damaged. +// +// The code that reaches this is always short; only a cause from a local service +// can be long enough to matter. +func truncateRunes(s string, limit int) string { + if len(s) <= limit { + return s + } + cut := limit + // utf8.RuneStart finds the first byte of a rune. Walking back from the + // limit lands on the start of the rune that would have been split, and at + // most 3 steps are needed since a UTF-8 rune is at most 4 bytes. + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + return s[:cut] +} + // writeFrame writes one length-prefixed frame in a single Write. // // One Write, not two: the underlying stream is a yamux stream whose writes diff --git a/core/services/cluster/tunnelproto_test.go b/core/services/cluster/tunnelproto_test.go new file mode 100644 index 000000000000..752fb9e9b84b --- /dev/null +++ b/core/services/cluster/tunnelproto_test.go @@ -0,0 +1,186 @@ +package cluster_test + +import ( + "bytes" + "encoding/binary" + "io" + "strings" + "unicode/utf8" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/cluster" +) + +var _ = Describe("Worker tunnel stream framing", func() { + Describe("the request frame", func() { + DescribeTable("round-trips a tag and a target", + func(tag, target string) { + var buf bytes.Buffer + Expect(cluster.WriteStreamRequest(&buf, tag, target)).To(Succeed()) + gotTag, gotTarget, err := cluster.ReadStreamRequest(&buf) + Expect(err).ToNot(HaveOccurred()) + Expect(gotTag).To(Equal(tag)) + Expect(gotTarget).To(Equal(target)) + }, + Entry("a tag and an address", cluster.StreamTagGRPC, "127.0.0.1:50051"), + Entry("a tag with no target", cluster.StreamTagHTTP, ""), + // The split is on the FIRST separator, so a target containing one + // must survive intact. + Entry("a target containing a space", cluster.StreamTagGRPC, "a b c"), + ) + + It("consumes exactly the frame and not one byte of what follows", func() { + // Load-bearing: the stream is handed to gRPC or net/http right + // after this, and a reader that over-read would eat the start of + // their conversation. + var buf bytes.Buffer + Expect(cluster.WriteStreamRequest(&buf, cluster.StreamTagGRPC, "127.0.0.1:1")).To(Succeed()) + buf.WriteString("PRI * HTTP/2.0") + + _, _, err := cluster.ReadStreamRequest(&buf) + Expect(err).ToNot(HaveOccurred()) + rest, err := io.ReadAll(&buf) + Expect(err).ToNot(HaveOccurred()) + Expect(string(rest)).To(Equal("PRI * HTTP/2.0")) + }) + + DescribeTable("refuses a tag it could not encode unambiguously", + func(tag string) { + var buf bytes.Buffer + Expect(cluster.WriteStreamRequest(&buf, tag, "x")).ToNot(Succeed()) + Expect(buf.Len()).To(BeZero(), "a refused request must not put a partial frame on the wire") + }, + Entry("empty", ""), + // A tag with a space would silently move part of itself into the + // target, so it is refused at the writer rather than a round trip + // later. + Entry("containing a space", "grpc stream"), + ) + + It("refuses a frame that declares more than the limit without allocating it", func() { + var hdr [2]byte + binary.BigEndian.PutUint16(hdr[:], 65535) + _, _, err := cluster.ReadStreamRequest(bytes.NewReader(hdr[:])) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("over the")) + }) + + It("reports a truncated frame as a truncated read, not as a refusal", func() { + // ReadStreamRequest must never produce ErrStreamRequestInvalid: + // that sentinel is what a worker SENDS, and a reader producing it + // would leave a caller unable to tell "the peer refused me" from + // "I could not read the peer". + var hdr [2]byte + binary.BigEndian.PutUint16(hdr[:], 10) + _, _, err := cluster.ReadStreamRequest(bytes.NewReader(append(hdr[:], 'a'))) + Expect(err).To(MatchError(io.ErrUnexpectedEOF)) + Expect(err).ToNot(MatchError(cluster.ErrStreamRequestInvalid)) + }) + }) + + Describe("the reply frame", func() { + It("reads an acceptance as nil", func() { + var buf bytes.Buffer + Expect(cluster.WriteStreamAccepted(&buf)).To(Succeed()) + Expect(cluster.ReadStreamReply(&buf)).To(Succeed()) + }) + + DescribeTable("keeps the three refusals apart", + func(sent error, others []error) { + var buf bytes.Buffer + Expect(cluster.WriteStreamRefusal(&buf, sent)).To(Succeed()) + got := cluster.ReadStreamReply(&buf) + Expect(got).To(MatchError(sent)) + // The whole point. A caller gives up on an unknown tag, retries + // an unavailable target, and reports a bad request as its own + // bug; collapsing any pair makes one of those wrong. + for _, other := range others { + Expect(got).ToNot(MatchError(other)) + } + }, + Entry("unknown tag", cluster.ErrStreamTagUnknown, + []error{cluster.ErrStreamTargetUnavailable, cluster.ErrStreamRequestInvalid}), + Entry("unavailable target", cluster.ErrStreamTargetUnavailable, + []error{cluster.ErrStreamTagUnknown, cluster.ErrStreamRequestInvalid}), + Entry("invalid request", cluster.ErrStreamRequestInvalid, + []error{cluster.ErrStreamTagUnknown, cluster.ErrStreamTargetUnavailable}), + ) + + It("carries the reason text to the far side", func() { + var buf bytes.Buffer + Expect(cluster.WriteStreamRefusal(&buf, wrapReason(cluster.ErrStreamTagUnknown, "no-such-tag"))).To(Succeed()) + Expect(cluster.ReadStreamReply(&buf).Error()).To(ContainSubstring("no-such-tag")) + }) + + It("reports an unrecognised code as itself, not as the nearest known one", func() { + // A code from a newer worker. Mapping it onto a known sentinel + // would make a frontend retry forever against a refusal that means + // something else entirely. + var buf bytes.Buffer + writeRawFrame(&buf, "err teapot short and stout") + got := cluster.ReadStreamReply(&buf) + Expect(got).To(HaveOccurred()) + Expect(got.Error()).To(ContainSubstring("teapot")) + Expect(got).ToNot(MatchError(cluster.ErrStreamTagUnknown)) + Expect(got).ToNot(MatchError(cluster.ErrStreamTargetUnavailable)) + Expect(got).ToNot(MatchError(cluster.ErrStreamRequestInvalid)) + }) + + It("reports a failure to READ the reply as itself, never as a refusal", func() { + // A refusal proves the worker is connected and said no. A read + // failure means the tunnel broke. A caller that treated the second + // as the first would report a dead link as a policy decision. + got := cluster.ReadStreamReply(bytes.NewReader(nil)) + Expect(got).To(MatchError(io.EOF)) + Expect(got).ToNot(MatchError(cluster.ErrStreamTagUnknown)) + Expect(got).ToNot(MatchError(cluster.ErrStreamTargetUnavailable)) + Expect(got).ToNot(MatchError(cluster.ErrStreamRequestInvalid)) + }) + + It("truncates an over-long reason on a rune boundary, keeping it decodable", func() { + // Two-byte runes so a byte-boundary cut lands mid-rune for half of + // all lengths; the padding tunes the frame to land exactly there. + reason := wrapReason(cluster.ErrStreamTargetUnavailable, strings.Repeat("é", 2000)) + var buf bytes.Buffer + Expect(cluster.WriteStreamRefusal(&buf, reason)).To(Succeed()) + + got := cluster.ReadStreamReply(&buf) + Expect(got).To(MatchError(cluster.ErrStreamTargetUnavailable)) + Expect(utf8.ValidString(got.Error())).To(BeTrue(), + "the truncated reason reached the far side with a split rune in it") + }) + + It("still reports the code when the reason is truncated away", func() { + // The code must survive truncation: a refusal a frontend cannot + // classify is indistinguishable from a worker that hung up. + reason := wrapReason(cluster.ErrStreamTagUnknown, strings.Repeat("x", 4000)) + var buf bytes.Buffer + Expect(cluster.WriteStreamRefusal(&buf, reason)).To(Succeed()) + Expect(cluster.ReadStreamReply(&buf)).To(MatchError(cluster.ErrStreamTagUnknown)) + }) + }) +}) + +// wrapReason builds the shape the worker sends: a sentinel with a cause. +func wrapReason(sentinel error, text string) error { + return &reasonErr{sentinel: sentinel, text: text} +} + +type reasonErr struct { + sentinel error + text string +} + +func (e *reasonErr) Error() string { return e.sentinel.Error() + ": " + e.text } +func (e *reasonErr) Unwrap() error { return e.sentinel } + +// writeRawFrame puts a payload on the wire without going through the encoder, +// so a spec can present a frame the encoder would never produce. +func writeRawFrame(buf *bytes.Buffer, payload string) { + var hdr [2]byte + binary.BigEndian.PutUint16(hdr[:], uint16(len(payload))) + buf.Write(hdr[:]) + buf.WriteString(payload) +} diff --git a/core/services/worker/tunnel.go b/core/services/worker/tunnel.go index 5ab812d8c23e..469bd47c86a3 100644 --- a/core/services/worker/tunnel.go +++ b/core/services/worker/tunnel.go @@ -371,9 +371,7 @@ func (t *Tunnel) accept(ctx context.Context, stream net.Conn) (net.Conn, bool) { local, err := svc(ctx, target) if err != nil { - // An INFRASTRUCTURE failure, which a frontend may retry, and which must - // never be reported as the unknown tag above. - t.refuse(stream, fmt.Errorf("%w: %v", cluster.ErrStreamTargetUnavailable, err)) + t.refuse(stream, classifyServiceFailure(err)) return nil, false } @@ -389,6 +387,26 @@ func (t *Tunnel) accept(ctx context.Context, stream net.Conn) (net.Conn, bool) { return local, true } +// classifyServiceFailure decides which refusal a local service's error is. +// +// A service that has ALREADY classified its own failure keeps that +// classification. loopbackService does, and the distinction is not cosmetic: a +// target outside this worker's backend port range is a request this worker will +// never serve, while a backend that is not listening yet is a condition that +// clears on its own. Reporting the first as the second tells a frontend to +// retry something that can never work; reporting the second as the first makes +// it give up on a backend that is merely starting. +// +// Anything unclassified is infrastructure, because that is what an unadorned +// dial failure is, and it must never become the unknown-tag refusal: a tag this +// worker serves does not stop being served because one dial failed. +func classifyServiceFailure(err error) error { + if errors.Is(err, cluster.ErrStreamRequestInvalid) { + return err + } + return fmt.Errorf("%w: %v", cluster.ErrStreamTargetUnavailable, err) +} + // refuse reports why a stream will not be served and then ENDS it. // // The close is the part that matters and it is not optional. A worker that says @@ -567,22 +585,59 @@ func tunnelEndpoint(frontendURL, nodeID string) (string, error) { // // The port range is the one the worker's own port allocator hands to backend // processes, so a stream cannot be pointed at some unrelated service that -// happens to be listening on this host. +// happens to be listening on this host. It is only as tight as the allocator's +// range, which by default runs to 65535; a deployment that wants it narrow sets +// LOCALAI_GRPC_MAX_PORT, which narrows both at once. +// +// Note the SHAPE, not only the checks. Nothing derived from the wire reaches +// the dialler: the address is built from the loopbackHost constant and from +// strconv.Itoa of an int this function validated, so `target` itself has no +// path to DialContext at all. Relaxing this into an arbitrary-host dialler +// therefore takes ADDING a data flow rather than deleting a check, which is the +// difference between a guard and a property. It has specs either way; the shape +// is what stops a plausible refactor from quietly restoring the hole. func loopbackService(minPort, maxPort int) LocalService { return func(ctx context.Context, target string) (net.Conn, error) { _, portStr, err := net.SplitHostPort(target) if err != nil { - return nil, fmt.Errorf("routing a tunnel stream: %q is not a host:port: %w", target, err) + return nil, fmt.Errorf("%w: routing a tunnel stream: %q is not a host:port: %v", + cluster.ErrStreamRequestInvalid, target, err) } port, err := strconv.Atoi(portStr) if err != nil { - return nil, fmt.Errorf("routing a tunnel stream: %q has no numeric port: %w", target, err) + return nil, fmt.Errorf("%w: routing a tunnel stream: %q has no numeric port: %v", + cluster.ErrStreamRequestInvalid, target, err) } if port < minPort || port > maxPort { - return nil, fmt.Errorf("routing a tunnel stream: port %d is outside this worker's backend range [%d, %d]", port, minPort, maxPort) + // Invalid rather than unavailable: no retry can bring a port + // outside this worker's own allocator range into it. + return nil, fmt.Errorf("%w: routing a tunnel stream: port %d is outside this worker's backend range [%d, %d]", + cluster.ErrStreamRequestInvalid, port, minPort, maxPort) } var d net.Dialer - return d.DialContext(ctx, "tcp", net.JoinHostPort("127.0.0.1", portStr)) + return d.DialContext(ctx, "tcp", net.JoinHostPort(loopbackHost, strconv.Itoa(port))) + } +} + +// loopbackHost is the only host any tunnel stream is ever dialled on. It is a +// constant so that "the worker dials itself and nothing else" is a fact about +// the code rather than a claim about its inputs. +const loopbackHost = "127.0.0.1" + +// tunnelServices builds the routing table the worker installs on its tunnel. +// +// It exists as its own function so the table can be specced. The table is the +// security boundary of this whole feature, and building it inline in Run left +// it reachable only by starting a worker, which meant it was covered by nothing +// and an arbitrary-host regression passed the entire suite. +func tunnelServices(cfg *Config, httpBindAddr string) map[string]LocalService { + basePort := cfg.effectiveBasePort() + return map[string]LocalService{ + // The frontend names a backend process by its port; the worker decides + // that only its own loopback, and only within its own backend port + // range, is reachable through it. + cluster.StreamTagGRPC: loopbackService(basePort, cfg.effectiveMaxPort(basePort)), + cluster.StreamTagHTTP: fixedService(loopbackAddr(httpBindAddr)), } } @@ -611,7 +666,7 @@ func loopbackAddr(bindAddr string) string { return bindAddr } if host == "" || host == "0.0.0.0" || host == "::" || host == "[::]" { - return net.JoinHostPort("127.0.0.1", port) + return net.JoinHostPort(loopbackHost, port) } return bindAddr } diff --git a/core/services/worker/tunnel_test.go b/core/services/worker/tunnel_test.go index 09285c401591..b5724a542459 100644 --- a/core/services/worker/tunnel_test.go +++ b/core/services/worker/tunnel_test.go @@ -8,6 +8,7 @@ import ( "net" "net/http" "net/http/httptest" + "strconv" "strings" "sync/atomic" "time" @@ -213,6 +214,66 @@ var _ = Describe("Worker tunnel client", func() { Eventually(read, "10s").Should(Receive(BeNil())) Expect(string(buf)).To(Equal("ping")) }) + + It("routes through the worker's OWN table, ignoring the host the frontend names", func() { + // Every other spec in this file installs dialLocalTCP, which dials + // whatever it is handed. This one installs tunnelServices, the + // table Run installs, so the wire path is exercised against the + // real routing rules at least once. + backend := echoListenerOn("127.0.0.1:0") + DeferCleanup(func() { _ = backend.Close() }) + port := portOf(backend) + + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services = tunnelServices(&Config{ + ServeAddr: fmt.Sprintf("0.0.0.0:%d", port), + GRPCMaxPort: port, + }, "0.0.0.0:1") + }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + // A host that is not this machine, and a port that is. + Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagGRPC, + fmt.Sprintf("attacker.invalid:%d", port))).To(Succeed()) + + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + Eventually(reply, "10s").Should(Receive(BeNil())) + + _, err = stream.Write([]byte("loopback")) + Expect(err).ToNot(HaveOccurred()) + buf := make([]byte, len("loopback")) + read := awaitErr(func() error { + _, err := io.ReadFull(stream, buf) + return err + }) + Eventually(read, "10s").Should(Receive(BeNil())) + Expect(string(buf)).To(Equal("loopback")) + }) + + It("refuses a port outside its range as a bad request, over the wire", func() { + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services = tunnelServices(&Config{ + ServeAddr: "0.0.0.0:50051", + GRPCMaxPort: 50051, + }, "0.0.0.0:50050") + }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagGRPC, "127.0.0.1:22")).To(Succeed()) + + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + var got error + Eventually(reply, "10s").Should(Receive(&got)) + // Three refusals, three meanings. A frontend retries unavailable + // and gives up on this one. + Expect(got).To(MatchError(cluster.ErrStreamRequestInvalid)) + Expect(got).ToNot(MatchError(cluster.ErrStreamTargetUnavailable)) + Expect(got).ToNot(MatchError(cluster.ErrStreamTagUnknown)) + }) }) Describe("refusing a stream it cannot serve", func() { @@ -507,3 +568,221 @@ var _ = Describe("Worker tunnel client", func() { }) }) }) + +// echoListenerOn is echoListener bound to a specific address, so a spec can put +// a listener somewhere the worker must NOT reach. +func echoListenerOn(addr string) net.Listener { + ln, err := net.Listen("tcp", addr) + Expect(err).ToNot(HaveOccurred()) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + defer func() { _ = conn.Close() }() + _, _ = io.Copy(conn, conn) + }() + } + }() + return ln +} + +// portOf returns the port a listener bound to. +func portOf(ln net.Listener) int { + _, portStr, err := net.SplitHostPort(ln.Addr().String()) + Expect(err).ToNot(HaveOccurred()) + port, err := strconv.Atoi(portStr) + Expect(err).ToNot(HaveOccurred()) + return port +} + +// The routing table is the security boundary of the whole tunnel, and until now +// nothing exercised it: every spec above installs dialLocalTCP, which is exactly +// the permissive dialler loopbackService exists to prevent. A review turned +// loopbackService into an arbitrary-host dialler and all 131 specs passed. +var _ = Describe("Worker tunnel local services", func() { + var ctx context.Context + + BeforeEach(func() { ctx = context.Background() }) + + Describe("loopbackService", func() { + It("reaches a loopback listener whose port is in range", func() { + ln := echoListenerOn("127.0.0.1:0") + DeferCleanup(func() { _ = ln.Close() }) + port := portOf(ln) + + conn, err := loopbackService(port, port)(ctx, ln.Addr().String()) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + + _, err = conn.Write([]byte("hi")) + Expect(err).ToNot(HaveOccurred()) + buf := make([]byte, 2) + _, err = io.ReadFull(conn, buf) + Expect(err).ToNot(HaveOccurred()) + Expect(string(buf)).To(Equal("hi")) + }) + + It("ignores the host the frontend names and dials loopback anyway", func() { + ln := echoListenerOn("127.0.0.1:0") + DeferCleanup(func() { _ = ln.Close() }) + port := portOf(ln) + + // A host that is emphatically not this machine. If it were honoured + // the dial would fail or, far worse, succeed against something else. + conn, err := loopbackService(port, port)(ctx, fmt.Sprintf("attacker.invalid:%d", port)) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + Expect(conn.RemoteAddr().String()).To(Equal(ln.Addr().String())) + }) + + It("does not reach a listener on another local address the frontend names", func() { + // The SSRF proof, stated as a reachability fact rather than as a + // property of the code. The only listener is on 127.0.0.2; nothing + // is on 127.0.0.1 at that port. A service that honoured the named + // host would connect; one that dials loopback cannot. + victim, err := net.Listen("tcp", "127.0.0.2:0") + if err != nil { + Skip("this host cannot bind a second loopback address: " + err.Error()) + } + DeferCleanup(func() { _ = victim.Close() }) + port := portOf(victim) + + conn, err := loopbackService(port, port)(ctx, victim.Addr().String()) + if err == nil { + _ = conn.Close() + Fail("the worker reached a host the frontend named, so a stream can steer it off loopback") + } + Expect(err).To(HaveOccurred()) + }) + + DescribeTable("refuses a target it will not route", + func(target string, minPort, maxPort int) { + _, err := loopbackService(minPort, maxPort)(ctx, target) + Expect(err).To(HaveOccurred()) + // Invalid, not unavailable. No retry brings a port outside this + // worker's own allocator range into it, and telling a frontend + // to retry forever is how a refusal becomes a hang. + Expect(err).To(MatchError(cluster.ErrStreamRequestInvalid)) + Expect(err).ToNot(MatchError(cluster.ErrStreamTargetUnavailable)) + }, + Entry("a port below the range", "127.0.0.1:50050", 50051, 50060), + Entry("a port above the range", "127.0.0.1:50061", 50051, 50060), + Entry("a non-numeric port", "127.0.0.1:http", 50051, 50060), + Entry("no port at all", "127.0.0.1", 50051, 50060), + Entry("an empty target", "", 50051, 50060), + ) + + It("reports a backend that is not listening as unavailable, which a frontend may retry", func() { + // The other half of the taxonomy: a port IN range with nothing on + // it is a backend that has not started yet, not a bad request. + ln := echoListenerOn("127.0.0.1:0") + port := portOf(ln) + Expect(ln.Close()).To(Succeed()) + + _, err := loopbackService(port, port)(ctx, ln.Addr().String()) + Expect(err).To(HaveOccurred()) + Expect(classifyServiceFailure(err)).To(MatchError(cluster.ErrStreamTargetUnavailable)) + Expect(classifyServiceFailure(err)).ToNot(MatchError(cluster.ErrStreamRequestInvalid)) + }) + }) + + Describe("fixedService", func() { + It("reaches its own address whatever the frontend names", func() { + ln := echoListenerOn("127.0.0.1:0") + DeferCleanup(func() { _ = ln.Close() }) + + conn, err := fixedService(ln.Addr().String())(ctx, "attacker.invalid:9") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + Expect(conn.RemoteAddr().String()).To(Equal(ln.Addr().String())) + }) + }) + + DescribeTable("loopbackAddr rewrites a bind address into a dialable one", + func(bind, want string) { + Expect(loopbackAddr(bind)).To(Equal(want)) + }, + // Dialling 0.0.0.0 only accidentally reaches localhost, and not on + // every platform, so the wildcard is replaced rather than dialled. + Entry("IPv4 wildcard", "0.0.0.0:8080", "127.0.0.1:8080"), + Entry("IPv6 wildcard", "[::]:8080", "127.0.0.1:8080"), + Entry("no host", ":8080", "127.0.0.1:8080"), + Entry("an explicit host is left alone", "10.0.0.9:8080", "10.0.0.9:8080"), + Entry("an explicit loopback is left alone", "127.0.0.1:8080", "127.0.0.1:8080"), + Entry("something that is not host:port passes through", "not-an-address", "not-an-address"), + ) + + Describe("tunnelServices", func() { + // The table Run installs. Built by its own function precisely so this + // can be asserted without starting a worker. + It("serves exactly the two tags the frontend may name", func() { + cfg := &Config{ServeAddr: "0.0.0.0:50051"} + Expect(tunnelServices(cfg, "0.0.0.0:50050")).To(HaveLen(2)) + Expect(tunnelServices(cfg, "0.0.0.0:50050")).To(HaveKey(cluster.StreamTagGRPC)) + Expect(tunnelServices(cfg, "0.0.0.0:50050")).To(HaveKey(cluster.StreamTagHTTP)) + }) + + It("bounds the gRPC service by THIS worker's configured port range", func() { + cfg := &Config{ServeAddr: "0.0.0.0:50051", GRPCMaxPort: 50052} + svc := tunnelServices(cfg, "0.0.0.0:50050")[cluster.StreamTagGRPC] + + // The HTTP server's own port sits one below the base port, so a + // gRPC-tagged stream cannot be steered onto it. + _, err := svc(ctx, "127.0.0.1:50050") + Expect(err).To(MatchError(cluster.ErrStreamRequestInvalid)) + _, err = svc(ctx, "127.0.0.1:50053") + Expect(err).To(MatchError(cluster.ErrStreamRequestInvalid)) + }) + + // Pins that the HTTP service reaches the address Run configures. It + // does NOT pin the wildcard rewrite: on Linux dialling 0.0.0.0 reaches + // loopback anyway, so this spec stays green with loopbackAddr disabled. + // The loopbackAddr table above is what holds that, and it exists + // because the accident is not portable. + It("points the HTTP service at the worker's own server", func() { + ln := echoListenerOn("127.0.0.1:0") + DeferCleanup(func() { _ = ln.Close() }) + + cfg := &Config{ServeAddr: "0.0.0.0:50051"} + svc := tunnelServices(cfg, fmt.Sprintf("0.0.0.0:%d", portOf(ln)))[cluster.StreamTagHTTP] + + conn, err := svc(ctx, "ignored:1") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = conn.Close() }) + Expect(conn.RemoteAddr().String()).To(Equal(ln.Addr().String())) + }) + }) + + DescribeTable("tunnelEndpoint builds the URL the worker dials", + func(frontendURL, nodeID, want string) { + got, err := tunnelEndpoint(frontendURL, nodeID) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(want)) + }, + Entry("http becomes ws", "http://frontend:8080", "n1", "ws://frontend:8080/api/cluster/connect?id=n1"), + Entry("https becomes wss", "https://frontend", "n1", "wss://frontend/api/cluster/connect?id=n1"), + Entry("ws passes through", "ws://frontend:8080", "n1", "ws://frontend:8080/api/cluster/connect?id=n1"), + Entry("wss passes through", "wss://frontend", "n1", "wss://frontend/api/cluster/connect?id=n1"), + // A frontend behind a path prefix keeps it: the path is appended, not + // assigned, exactly as the registration client builds its URLs. + Entry("a path prefix is kept", "https://host/localai", "n1", "wss://host/localai/api/cluster/connect?id=n1"), + Entry("a trailing slash is not doubled", "https://host/localai/", "n1", "wss://host/localai/api/cluster/connect?id=n1"), + Entry("the node id is escaped", "http://h", "a b&c", "ws://h/api/cluster/connect?id=a+b%26c"), + ) + + DescribeTable("tunnelEndpoint refuses a frontend URL it cannot dial", + func(frontendURL string) { + _, err := tunnelEndpoint(frontendURL, "n1") + Expect(err).To(HaveOccurred()) + }, + Entry("empty", ""), + // Refused rather than coerced: a worker silently dialling a scheme + // nobody configured is worse than one that says it cannot start. + Entry("a scheme that is not HTTP", "ftp://frontend"), + Entry("a bare host with no scheme", "frontend:8080/x"), + Entry("no host", "http://"), + ) +}) diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index 842485cfbc0f..9ce198c9dc74 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -17,7 +17,6 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" grpc "github.com/mudler/LocalAI/pkg/grpc" @@ -110,10 +109,20 @@ func Run(ctx *cliContext.Context, cfg *Config) error { } nodeID = res.ID // This path registers exactly once and never again, so the credential - // it holds cannot go stale by rotation from its own side. It can still - // be superseded from OUTSIDE, by a second worker registering under the - // same node name, and there is nothing this worker can do about that - // but log the 401 and keep retrying. + // it holds cannot go stale by rotation from its own side. + // + // It CAN be superseded from outside: Register upserts by NAME, so a + // second worker registering under this node's name rotates the row's + // credential, and this worker then fails every tunnel dial with 401 for + // the life of the process. It logs that once per backoff and never + // recovers on its own; a restart fixes it, because startup re-registers + // unconditionally. + // + // Deliberately NOT fixed here. Re-registering after repeated tunnel + // 401s is a decision about the worker's lifecycle, and it belongs with + // the change that removes this worker's inbound listeners, when a + // worker that cannot tunnel is a worker that cannot be reached at all. + // Today it can still be reached at the addresses it advertises. staticTunnelToken := res.TunnelToken tunnelToken = func() string { return staticTunnelToken } connectNats = func() (*messaging.Client, error) { @@ -209,19 +218,15 @@ func Run(ctx *cliContext.Context, cfg *Config) error { // frontend URL or this node's identity is unusable, and a worker that // silently ran without its tunnel would look healthy while being // unreachable to everything that dials through it. - tunnelBasePort := cfg.effectiveBasePort() if cfg.WorkerTunnel { tunnel, terr := StartTunnel(shutdownCtx, TunnelConfig{ FrontendURL: cfg.RegisterTo, NodeID: nodeID, Token: tunnelToken, - Services: map[string]LocalService{ - // The frontend names a backend process by its port; the worker - // decides that only its own loopback, and only within its own - // backend port range, is reachable through it. - cluster.StreamTagGRPC: loopbackService(tunnelBasePort, cfg.effectiveMaxPort(tunnelBasePort)), - cluster.StreamTagHTTP: fixedService(loopbackAddr(httpAddr)), - }, + // Built by tunnelServices rather than inline, so the routing + // table, which is this feature's security boundary, is reachable + // from a spec without starting a worker. + Services: tunnelServices(cfg, httpAddr), }) if terr != nil { nodes.ShutdownFileTransferServer(httpServer) @@ -264,7 +269,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error { }() // Process supervisor — manages multiple backend gRPC processes on different ports - basePort := tunnelBasePort + basePort := cfg.effectiveBasePort() // Buffered so NATS stop handler can send without blocking sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 1a56516429e4..56c0b906fc0f 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -118,7 +118,9 @@ A worker that presents a credential belonging to no node, or names a node ID the Unlike the agent worker's API key and its NATS credential, a tunnel credential **is** issued to a node still awaiting approval. It is inert until then: the tunnel route re-reads the node's status on every dial and refuses a pending one. Withholding it would instead strand workers that register exactly once, since approval on its own prompts no re-registration. -A tunnel credential does not replace `LOCALAI_REGISTRATION_TOKEN`. Without one, node registration itself is unauthenticated, so anyone who can reach the frontend can register a worker and be issued a working tunnel credential for it. LocalAI warns about that at startup. +A tunnel credential does not replace `LOCALAI_REGISTRATION_TOKEN`. Without one, node registration itself is unauthenticated, so anyone who can reach the frontend can register a worker and be issued a tunnel credential for it. How far that gets them depends on auto-approve: with auto-approve on the node is healthy at once and the credential works immediately; with it off the node is pending and the credential is inert until an admin approves, so approval is the real gate. LocalAI warns about the missing token at startup. + +Only **backend** nodes are issued one. An agent worker has no inbound surface for the tunnel to replace and no client for it, so minting one would widen the credential surface for nothing; its row keeps an empty tunnel credential and the tunnel route refuses it like any other node without one. The tunnel lands on exactly one frontend replica, and that replica records itself as the owner of the worker's connection in the `node_connections` table. When the socket dies the claim is dropped with it. If the replica stalls long enough for its peers to reap it, it re-claims the tunnels it still holds on a live session as soon as it re-registers, skipping any whose socket has already gone. That re-claim needs the replica to have an advertised address: without one it never had an instance row to begin with, and its tunnels are usable only by the replica holding them. @@ -137,7 +139,9 @@ The worker holds the tunnel with one goroutine: it dials, serves the frontend's | `grpc` | a backend process on this worker | the port; the host is discarded and only `127.0.0.1` is dialled, within the worker's own backend port range | | `http` | the worker's own file-transfer and backend-log server | ignored; there is one such server and only the worker knows where it bound | -A stream naming a tag the worker does not serve, or a local service it could not reach, is refused with a reason and the stream is **ended** rather than left open. Those two refusals are distinct on the wire on purpose: a frontend gives up on the first and may retry the second. One bad stream never affects the others or the session. +The `grpc` row is the security boundary of the tunnel, and it is worth being explicit about it. A tunnel terminates inside the worker process, so a stream arriving on it can reach anything the worker can reach; if the frontend could name the host, whoever holds the frontend end could make every worker in the fleet dial arbitrary addresses on its private network. The worker therefore builds the dial address from a constant `127.0.0.1` and a port it has validated, and the string from the wire never reaches the dialler at all. The port range is the one the worker's own allocator hands to backend processes, which by default runs to 65535; setting `LOCALAI_GRPC_MAX_PORT` narrows the allocator and this range together, and a worker with a known backend count should set it. + +A stream naming a tag the worker does not serve, a target outside that port range, or a local service it could not reach, is refused with a reason and the stream is **ended** rather than left open. Those refusals are distinct on the wire on purpose: an unknown tag and an out-of-range target are requests this worker will never serve, while an unreachable local service is a backend that has not started yet. A frontend gives up on the first two and may retry the third. One bad stream never affects the others or the session. Reconnects use exponential backoff with jitter: the interval doubles from 500ms up to a ceiling of 30 seconds, and each wait is drawn between half of that interval and all of it, so no worker ever spins and a fleet that lost the same replica does not come back in lockstep. The interval returns to its floor only after a session that lasted at least 30 seconds. That last part is what stops a rolling frontend restart, where every dial succeeds and then dies moments later, from turning a fleet of workers into a retry storm against the first replica back up. A worker that is refused (`401`, `403`) keeps retrying on the same schedule rather than exiting: a re-registration or an admin approval fixes both without restarting it. From 9095480f6f916cc1ca981e70f7de4d49a02ae25e Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 13:33:07 +0000 Subject: [PATCH 051/153] fix(worker): make the tunnel credential's node-type gate actually structural Re-review follow-up, three items. Two are the overclaiming-comment class again, and the first is that class with a real defect underneath it. attachTunnelToken said "enforcement is therefore structural": an ineligible node never gets a credential, so its hash stays empty and the tunnel route's empty-hash branch does the refusing. That was true for a node that had always been an agent and false for one that had not. Register upserts by NAME, so a backend node re-registering as an agent keeps its ID, and Register's struct Updates zero-skips the credential column while writing the new node_type. The early return left the credential the node earned as a backend sitting on a row that is now an agent, and ConnectHandler never looks at node_type. Fixed by making the claim true rather than by softening it, because the mint-site gate was chosen precisely on the grounds that it was structural: an ineligible node now has its column CLEARED, unconditionally, so the invariant does not depend on what the row happened to contain. A spec pins it and was red before the change. Same shape as the Register-upserts-by-name hazard already carried forward: a name is not an identity. Second, loopbackHost claimed to be the only host any tunnel stream is ever dialled on. It is not: fixedService dials whatever Run built it from, which is this worker's own LOCALAI_HTTP_ADDR, and loopbackAddr rewrites only a wildcard bind, so an operator who binds the file-transfer server to a routable address gets a routable dial. The property that matters is narrower and is what the comment says now: the frontend cannot STEER the dial. The grpc tag builds its address from a constant and a validated port with nothing from the wire reaching the dialler, and the http tag ignores its target entirely. Worth stating exactly rather than summarising, because the argument about what a stream can reach rests on knowing which hosts are reachable, and an overstatement at that site is what would let someone conclude the constant alone is doing the work. Third, a spec named "without allocating it" measured no allocation. It now asserts the mechanism the defence actually rests on, that the reader consumes the two length bytes and not one byte of the body, through a counting reader. The input carries a body on purpose: against input that ends after the header the assertion would pass with the limit check deleted. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/localai/nodes.go | 32 ++++++++++++++++++----- core/http/endpoints/localai/nodes_test.go | 28 ++++++++++++++++++++ core/services/cluster/tunnelproto_test.go | 30 +++++++++++++++++++-- core/services/worker/tunnel.go | 25 +++++++++++++++--- 4 files changed, 104 insertions(+), 11 deletions(-) diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index 5ad21d724e71..9d6886b81b05 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -321,11 +321,21 @@ func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecr // "backend workers, the ones that tunnel". // // The gate lives HERE and not in ConnectHandler, which never looks at NodeType. -// It does not need to: an agent node's tunnel credential is never minted, so -// its TunnelTokenHash stays empty and the handler's empty-hash branch refuses -// it like any other node without one. Enforcement is therefore structural. The -// day agent workers want a tunnel, relaxing this condition is the whole change, -// and it has to be a deliberate one. +// It does not need to, PROVIDED an ineligible node ends up with no credential +// rather than merely being handed no new one, because the handler's empty-hash +// branch is what does the refusing. So this CLEARS the column instead of +// returning early, and the difference is not theoretical: Register upserts by +// NAME, so a backend node re-registering as an agent keeps its ID, and +// Register's struct Updates zero-skips TunnelTokenHash while writing the new +// node_type. An early return left a live credential on a row that had become an +// agent. Clearing is what makes "enforcement is structural" true. +// +// It clears unconditionally rather than only when something is there, so the +// invariant holds without depending on what the row happened to contain. The +// cost is one UPDATE per agent registration. +// +// The day agent workers want a tunnel, relaxing the eligibility condition is +// the whole change, and it has to be a deliberate one. // // A failure to mint or to store is logged and the response goes out without the // token. Registration is what gets a worker into the cluster at all, and @@ -334,7 +344,17 @@ func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecr // tunnel_token, reports that it has no credential, and retries at its next // registration. func attachTunnelToken(ctx context.Context, response map[string]any, registry *nodes.NodeRegistry, node *nodes.BackendNode) { - if node == nil || node.NodeType != nodes.NodeTypeBackend { + if node == nil { + return + } + if node.NodeType != nodes.NodeTypeBackend { + // Cleared, not skipped. SetTunnelTokenHash writes the single column + // directly rather than through a struct update, so unlike Register it + // can write an empty value; see its doc. + if err := registry.SetTunnelTokenHash(ctx, node.ID, ""); err != nil { + xlog.Error("Failed to clear the tunnel credential of a node that is not a backend worker", + "node", node.Name, "type", node.NodeType, "error", err) + } return } // crypto/rand.Text: at least 128 bits of randomness, no error to handle and diff --git a/core/http/endpoints/localai/nodes_test.go b/core/http/endpoints/localai/nodes_test.go index ea10b0ccc17a..adce16826869 100644 --- a/core/http/endpoints/localai/nodes_test.go +++ b/core/http/endpoints/localai/nodes_test.go @@ -228,6 +228,34 @@ var _ = Describe("Node HTTP handlers", func() { Expect(node.TunnelTokenHash).To(BeEmpty()) }) + It("clears a tunnel credential when a node stops being a backend node", func() { + // Register upserts BY NAME, so a node can change node_type in place. + // Skipping the mint on the way through leaves the credential the + // node earned as a backend sitting on a row that is now an agent: + // Register's struct Updates zero-skips the column while writing the + // new node_type, so nothing else clears it. ConnectHandler never + // looks at node_type, so that stale hash is a usable tunnel + // credential for a node type that is not supposed to hold one. + // + // This is the same shape as the Register-upserts-by-name hazard + // already carried forward: a name is not an identity. + backend := register(`{"name":"shifty","address":"10.0.0.7:50051"}`, "", true) + Expect(backend["tunnel_token"]).ToNot(BeEmpty()) + + agent := register(`{"name":"shifty","node_type":"agent"}`, "", true) + Expect(agent["id"]).To(Equal(backend["id"]), "re-registration must keep the node identity") + Expect(agent["node_type"]).To(Equal(nodes.NodeTypeAgent)) + Expect(agent).ToNot(HaveKey("tunnel_token")) + + node, err := registry.Get(context.Background(), backend["id"].(string)) + Expect(err).ToNot(HaveOccurred()) + // The claim the gate makes is that an ineligible node HAS no + // credential, not merely that it was not handed a new one. Only + // then is the empty-hash refusal in ConnectHandler the enforcement. + Expect(node.TunnelTokenHash).To(BeEmpty(), + "the node kept the credential it earned as a backend, so the mint-site gate is not structural") + }) + It("returns nats_jwt when account seed is configured", func() { akp, err := nkeys.CreateAccount() Expect(err).ToNot(HaveOccurred()) diff --git a/core/services/cluster/tunnelproto_test.go b/core/services/cluster/tunnelproto_test.go index 752fb9e9b84b..2664d9d509f5 100644 --- a/core/services/cluster/tunnelproto_test.go +++ b/core/services/cluster/tunnelproto_test.go @@ -59,12 +59,25 @@ var _ = Describe("Worker tunnel stream framing", func() { Entry("containing a space", "grpc stream"), ) - It("refuses a frame that declares more than the limit without allocating it", func() { + It("refuses an over-long declared length after reading only the header", func() { + // The name used to say "without allocating it" and the spec + // measured nothing of the sort. What is actually checkable, and is + // the mechanism the defence rests on, is that the reader STOPS: it + // consumes the two length bytes and not one byte of the body, so a + // peer cannot make it allocate or read on demand. + // + // The body is present in the input on purpose. With an input that + // ends after the header, a reader that went on to read the body + // would still consume nothing more, and this assertion would pass + // with the limit check deleted. var hdr [2]byte binary.BigEndian.PutUint16(hdr[:], 65535) - _, _, err := cluster.ReadStreamRequest(bytes.NewReader(hdr[:])) + src := &countingReader{r: bytes.NewReader(append(hdr[:], make([]byte, 4096)...))} + + _, _, err := cluster.ReadStreamRequest(src) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("over the")) + Expect(src.n).To(Equal(2), "the reader consumed part of a frame it had already refused") }) It("reports a truncated frame as a truncated read, not as a refusal", func() { @@ -176,6 +189,19 @@ type reasonErr struct { func (e *reasonErr) Error() string { return e.sentinel.Error() + ": " + e.text } func (e *reasonErr) Unwrap() error { return e.sentinel } +// countingReader records how many bytes were actually consumed, so a spec can +// assert where a reader stopped rather than only what it returned. +type countingReader struct { + r io.Reader + n int +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += n + return n, err +} + // writeRawFrame puts a payload on the wire without going through the encoder, // so a spec can present a frame the encoder would never produce. func writeRawFrame(buf *bytes.Buffer, payload string) { diff --git a/core/services/worker/tunnel.go b/core/services/worker/tunnel.go index 469bd47c86a3..0eabacb0f9e3 100644 --- a/core/services/worker/tunnel.go +++ b/core/services/worker/tunnel.go @@ -619,9 +619,28 @@ func loopbackService(minPort, maxPort int) LocalService { } } -// loopbackHost is the only host any tunnel stream is ever dialled on. It is a -// constant so that "the worker dials itself and nothing else" is a fact about -// the code rather than a claim about its inputs. +// loopbackHost is the host every stream the FRONTEND CAN STEER is dialled on. +// +// It is a constant so that "a stream cannot choose where the worker dials" is a +// fact about the code rather than a claim about its inputs: the grpc tag builds +// its address from this and a port it validated, and nothing derived from the +// wire reaches the dialler. +// +// It is NOT the only host this file ever dials, and the difference is worth +// stating exactly rather than summarising, because the whole argument about +// what a stream can reach rests on knowing which hosts are reachable, and an +// overstatement here is what would let a future reader conclude the constant +// alone is doing the work. +// +// fixedService dials whatever address it was constructed with. Run constructs +// it from this worker's own LOCALAI_HTTP_ADDR, which an operator may set to a +// routable address; loopbackAddr only rewrites a WILDCARD bind, and leaves an +// explicit host alone on purpose, because a server bound to one address is not +// reachable on another. So the http tag can dial a non-loopback host. That host +// is one the OPERATOR configured for this worker's own server, never one a +// stream names: fixedService ignores its target entirely. The property the +// design needs is that the frontend cannot steer the dial, and that holds for +// both tags. const loopbackHost = "127.0.0.1" // tunnelServices builds the routing table the worker installs on its tunnel. From 08aa37b89fa65c7077a98bf81df94f9cd2d8db82 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 14:12:17 +0000 Subject: [PATCH 052/153] feat(cluster): relay a peer's stream onto a worker tunnel held here A worker holds ONE tunnel and it lands on ONE frontend replica, so with N replicas behind a load balancer roughly (N-1)/N of requests arrive somewhere that cannot reach the worker directly. This is the piece that carries them: the SessionStore stream handler reads which worker a peer's stream is for, opens a stream on the tunnel this replica holds, and splices the two. Splice has had no production caller since phase 1. It has one now, and being the first caller it settles the two endings phase 1 deliberately left open, both of which read as normal termination until now: - a peer-initiated *StreamError{Remote: true}, which yamux builds only from an RST frame the far side sent (stream.go:432-449); a reset this side asks for carries Remote: false, and Splice never resets anything, its own Close sending a FIN; - a graceful ErrRemoteGoAway, which handleGoAway returns for code goAwayNormal (session.go:829-833) and close hands unwrapped to every live stream (session.go:328-337). Both truncate whatever was in flight. Reporting them as normal termination is how a half-finished inference comes to look like a short one that completed, so both are now reported; the local forms stay silent, because those are the teardown Splice provokes itself. The decision cannot live in a caller reading Splice's result, since a result already mapped to nil carries nothing left to reclassify, so it lives at the classifier with the reasoning beside it. The relay logs it at debug: a client cancelling a relayed request produces one per cancellation, and the truncation is separately visible to the frontend's own gRPC or HTTP client. The relay hop gets its own request and reply frames. They have to be distinct from the worker tunnel's, because a relayed stream carries both hops' frames back to back, and a vocabulary shared between them would let a reader applied to the wrong hop hand back a plausible sentinel belonging to the other. Its three refusals stay apart for the reason the worker's three do: ErrNotOwner is a routing fact and the caller should resolve the owner again; unavailable is infrastructure at this replica and a retry is worth something; bad-request is the caller's bug. None of them is, or may be built over, an absence error. One hop, always. A stream naming a worker this replica does not hold is refused, never resolved and relayed onward, so a stale ownership row cannot become a loop between two replicas each certain the other holds the worker. PeerPool is constructed and closed alongside SessionStore, so both halves of the peer mesh now have an owner and a shutdown. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 33 +- core/services/cluster/peerlink.go | 6 +- core/services/cluster/relay.go | 333 +++++++++++++++++++ core/services/cluster/relay_internal_test.go | 63 ++++ core/services/cluster/relay_test.go | 291 ++++++++++++++++ core/services/cluster/sessions.go | 4 + core/services/cluster/splice.go | 118 +++++-- core/services/cluster/splice_test.go | 16 +- 8 files changed, 819 insertions(+), 45 deletions(-) create mode 100644 core/services/cluster/relay.go create mode 100644 core/services/cluster/relay_internal_test.go create mode 100644 core/services/cluster/relay_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index b8f2b8e9afcc..f97d67cf08b1 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -55,8 +55,14 @@ type DistributedServices struct { // peer-reachable address could be determined, which leaves this replica // invisible to its peers but otherwise fully functional. Membership *cluster.Membership - // PeerSessions owns the peer links other replicas dialled into this one. + // PeerSessions owns the peer links other replicas dialled into this one, + // and relays the streams that arrive on them onto the worker tunnels this + // replica holds. PeerSessions *cluster.SessionStore + // Peers owns the peer links this replica dialled OUT, the mirror of + // PeerSessions. It is what the relaying dialer opens a stream on when a + // request arrives here for a worker another replica holds. + Peers *cluster.PeerPool // Tunnels holds the worker tunnels this replica has accepted and keeps the // node_connections table agreeing with them. It is handed to the membership // loop, which re-claims what it holds after this replica has been reaped, @@ -82,6 +88,13 @@ func (ds *DistributedServices) Shutdown() { if ds.PeerSessions != nil { ds.PeerSessions.CloseAll() } + // Both halves of the peer mesh go down together. A pool left open + // holds a WebSocket and two yamux loop goroutines per peer for as long + // as the process lives, and an Open after this reports ErrPoolClosed, + // which is a fact about this process and never node absence. + if ds.Peers != nil { + ds.Peers.Close() + } if ds.Health != nil { ds.Health.Stop() } @@ -205,10 +218,6 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // Replica membership. NewNodeRegistry has just migrated the tables this // reads, so it has to come after it. clusterRegistry := cluster.NewRegistry(authDB) - // Accepted peer links are held with no stream handler: this replica has - // somewhere to put a link a peer dials, and refuses the streams on it, - // because nothing relays worker traffic yet. - peerSessions := cluster.NewSessionStore(nil) var membership *cluster.Membership if advertised, err := advertisedPeerAddr(cfg); err != nil { // Not fatal. A replica that cannot publish an address still serves @@ -245,6 +254,19 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade membership.SetTunnels(tunnels) } + // The links peers dial IN, with the relay installed on them. This is what + // makes more than one replica work: a worker holds one tunnel, it lands on + // one replica, and every request that arrives anywhere else reaches the + // worker through this handler. Passing nil here would leave every such + // request refused, promptly and only at debug level, which presents as a + // worker that is connected and unusable from most of the deployment. + peerSessions := cluster.NewSessionStore(cluster.NewRelay(tunnels).Stream) + // The links this replica dials OUT, the other half of the same mesh. It + // authenticates with the registration token because that is the token the + // peer route checks (see RegisterClusterRoutes); two different tokens here + // would make every peer dial 401 with nothing naming the mismatch. + peers := cluster.NewPeerPool(cfg.Distributed.InstanceID, cfg.Distributed.RegistrationToken, clusterRegistry) + // Let scheduling rules be keyed by a model alias. The registry resolves a // rule's name through the config loader to find the model it governs, so an // operator can pin placement to a stable name like "production" and have it @@ -553,6 +575,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade Cluster: clusterRegistry, Membership: membership, PeerSessions: peerSessions, + Peers: peers, Tunnels: tunnels, }, nil } diff --git a/core/services/cluster/peerlink.go b/core/services/cluster/peerlink.go index 961c1a3fa3d9..047fcbc5472a 100644 --- a/core/services/cluster/peerlink.go +++ b/core/services/cluster/peerlink.go @@ -233,8 +233,10 @@ func (p *PeerPool) Open(ctx context.Context, peerID string) (net.Conn, error) { // has left the deployment but is still listening keeps a live WebSocket and the // two yamux loop goroutines behind it for as long as this process runs; a peer // that is genuinely gone is reclaimed by the 30s keepalive default, so the real -// exposure is narrow. There is no Forget because nothing yet knows which peers -// have left; Task 5's ownership work is where that knowledge appears. +// exposure is narrow. There is no Forget: the membership sweep does know which +// replicas have left, but nothing plumbs that knowledge to this pool, so an +// entry for a departed peer outlives it and only the keepalive reclaims what it +// holds. func (p *PeerPool) link(peerID string) (*peerLink, error) { p.mu.Lock() defer p.mu.Unlock() diff --git a/core/services/cluster/relay.go b/core/services/cluster/relay.go new file mode 100644 index 000000000000..99164c6fbd74 --- /dev/null +++ b/core/services/cluster/relay.go @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: MIT + +package cluster + +import ( + "cmp" + "context" + "errors" + "fmt" + "io" + "net" + "strings" + "time" + + "github.com/mudler/xlog" +) + +// The framing every stream on a PEER link opens with. +// +// A worker holds one tunnel and it lands on one frontend replica, so every +// other replica reaches that worker by relaying through the one that holds it. +// The peer link carries traffic for every worker its far side owns, so a stream +// on it means nothing until it says which worker it is for; that is this frame. +// +// A relayed stream therefore carries TWO request frames back to back: this one, +// which the owning replica consumes, and the worker tunnel's own (tunnelproto) +// frame, which crosses untouched and is answered by the worker. A dialler reads +// one reply from each, in that order. +// +// The two vocabularies are deliberately disjoint. "relay-ok" is not "ok", and +// none of the three refusal codes below is spelled like a tunnel code, so a +// reader applied to the wrong hop fails with "unrecognised reply" instead of +// handing back a plausible sentinel that belongs to the other hop. Getting that +// wrong would report a worker's refusal as the owning replica's, and a caller +// would retry against the wrong end of the path. +const ( + relayReplyAccepted = "relay-ok" + relayCodeNotOwner = "relay-not-owner" + relayCodeUnavailable = "relay-unavailable" + relayCodeBadRequest = "relay-bad-request" +) + +// The refusals the relay hop can send, beyond ErrNotOwner which it shares with +// the local path. +// +// Three conditions, kept apart, for the reason the worker's three are kept +// apart. ErrNotOwner is a ROUTING fact: the worker may be perfectly healthy on +// another replica, and the caller should resolve the owner again. This one is +// INFRASTRUCTURE at the owning replica: the tunnel is held right here and its +// session will not carry a stream, so a retry is worth something and looking +// elsewhere is not. ErrRelayRequestInvalid is the CALLER's bug and no retry +// helps. +// +// None of them is, or may ever be built over, an absence error. A refusal is +// proof that a replica answered, and reporting absence would tell a scheduler +// that a worker which is connected has gone away. +var ( + ErrRelayUnavailable = errors.New("cluster: the owning replica could not open a stream to that worker") + ErrRelayRequestInvalid = errors.New("cluster: the owning replica rejected the relay request as malformed") +) + +// WriteRelayRequest names the worker a peer stream is for. +// +// An empty node id is refused here rather than on the wire, so a caller with a +// bug learns at once instead of a round trip later. +func WriteRelayRequest(w io.Writer, nodeID string) error { + if nodeID == "" { + return fmt.Errorf("writing a relay request: empty node id") + } + return writeFrame(w, nodeID) +} + +// ReadRelayRequest reads the opening frame of a peer stream. +// +// A malformed frame is an ordinary error, NOT ErrRelayRequestInvalid: that +// sentinel is what a relay SENDS to describe a refusal, and producing it here +// would leave a caller unable to tell "the peer refused my request" from "I +// could not read the peer's". +func ReadRelayRequest(r io.Reader) (string, error) { + payload, err := readFrame(r) + if err != nil { + return "", fmt.Errorf("reading a relay request: %w", err) + } + if payload == "" { + // An empty payload is a well-formed frame naming no worker. Treating + // it as a node called "" would send the caller a routing refusal for a + // request no replica can ever serve, so it stays the caller's bug. + return "", fmt.Errorf("reading a relay request: empty node id") + } + return payload, nil +} + +// WriteRelayAccepted tells the peer the stream now carries the worker tunnel's +// own conversation. Everything after this frame belongs to that hop. +func WriteRelayAccepted(w io.Writer) error { return writeFrame(w, relayReplyAccepted) } + +// WriteRelayRefusal reports why a peer's stream will not be relayed. The caller +// closes the stream afterwards; this only says why. +// +// An unclassified reason is sent as bad-request with its text attached, rather +// than dropped: a refusal a peer cannot read is indistinguishable from a +// replica that hung up, and those are different problems. +func WriteRelayRefusal(w io.Writer, reason error) error { + code := relayCodeBadRequest + switch { + case errors.Is(reason, ErrNotOwner): + code = relayCodeNotOwner + case errors.Is(reason, ErrRelayUnavailable): + code = relayCodeUnavailable + } + + text := "" + if reason != nil { + text = strings.Map(func(r rune) rune { + // The frame is length-prefixed so a newline would not corrupt it, + // but this text lands in a log line on the far side, and a cause + // spanning lines is what makes one unsearchable. + if r == '\n' || r == '\r' { + return ' ' + } + return r + }, reason.Error()) + } + frame := replyPrefixRefused + code + streamRequestSeparator + text + return writeFrame(w, truncateRunes(frame, maxTunnelFrame)) +} + +// ReadRelayReply reads the owning replica's answer to a relay request. nil +// means the stream is now the worker tunnel's. +// +// A failure to READ the reply is returned as itself and never as one of the +// refusal sentinels: a refusal means a replica answered, a read failure means +// the peer link broke, and reporting the second as the first would present a +// dead link as a policy decision. +func ReadRelayReply(r io.Reader) error { + payload, err := readFrame(r) + if err != nil { + return fmt.Errorf("reading a relay reply: %w", err) + } + if payload == relayReplyAccepted { + return nil + } + rest, ok := strings.CutPrefix(payload, replyPrefixRefused) + if !ok { + return fmt.Errorf("reading a relay reply: unrecognised reply %q", payload) + } + code, text, _ := strings.Cut(rest, streamRequestSeparator) + switch code { + case relayCodeNotOwner: + // ErrNotOwner and nothing else. It is a routing fact, and the sentinels + // it must never be confused with are ErrNoConnection (the worker is + // connected nowhere) and ErrPeerUnreachable (a replica will not + // answer): a caller acts on those by giving up on the worker or by + // retrying the peer, and on this one by resolving the owner again. + return fmt.Errorf("%w: %s", ErrNotOwner, text) + case relayCodeUnavailable: + return fmt.Errorf("%w: %s", ErrRelayUnavailable, text) + case relayCodeBadRequest: + return fmt.Errorf("%w: %s", ErrRelayRequestInvalid, text) + default: + // A code from a newer replica. Carried out as-is rather than mapped + // onto the nearest known one, so a caller does not retry forever + // against a refusal that means something else entirely. + return fmt.Errorf("relay stream refused with unrecognised code %q: %s", code, text) + } +} + +const ( + // relayHeaderTimeout bounds how long a peer stream may go without naming + // the worker it is for. Without it, a dialler killed between OpenStream and + // its first write holds a relay goroutine and a stream slot until the whole + // peer link dies, which is minutes on the default keepalive. + relayHeaderTimeout = 15 * time.Second + + // relayOpenTimeout bounds opening the worker-side stream. yamux blocks an + // Open once AcceptBacklog SYNs are in flight, waiting on synCh rather than + // failing (go-yamux/v5@v5.1.0/session.go:205-212); it honours the context, + // which is the only reason there is one here. Without the bound, a worker + // that has stopped accepting would turn a refusable condition into a parked + // peer, which is the one outcome this path exists to avoid. + relayOpenTimeout = 15 * time.Second +) + +// Relay splices a stream a peer opened onto a worker tunnel this replica holds. +// +// It is the piece that makes more than one frontend replica work at all: a +// worker holds ONE tunnel, it lands on ONE replica, and with N replicas behind +// a load balancer roughly (N-1)/N of requests arrive somewhere else. Those +// requests reach the worker through here. +// +// One hop, always. A stream naming a worker this replica does not hold is +// refused, never resolved and relayed onward. A second hop would turn a stale +// ownership row into a loop between two replicas, each certain the other holds +// the worker, and the loop would carry the caller's request around it; the +// dialling replica re-resolving the owner is both cheaper and terminating. +type Relay struct { + tunnels *TunnelRegistry + + // Timeouts are fields rather than constants read directly so a spec can + // exercise the deadline without waiting out a production value. They are + // not operator knobs and are not plumbed to configuration. + headerTimeout time.Duration + openTimeout time.Duration +} + +// NewRelay returns the relay for the tunnels this replica holds. Its Stream +// method is the SessionStore stream handler. +func NewRelay(tunnels *TunnelRegistry) *Relay { return newRelay(tunnels, 0, 0) } + +func newRelay(tunnels *TunnelRegistry, headerTimeout, openTimeout time.Duration) *Relay { + return &Relay{ + tunnels: tunnels, + headerTimeout: cmp.Or(headerTimeout, relayHeaderTimeout), + openTimeout: cmp.Or(openTimeout, relayOpenTimeout), + } +} + +// Stream relays one peer stream. It owns closing that stream on every path. +func (r *Relay) Stream(peerID string, stream net.Conn) { + // SessionStore runs this on a bare goroutine, so an unrecovered panic here + // ends the PROCESS, taking down every other replica's traffic through this + // one. It covers what runs on this goroutine: the frame read, the registry + // lookup and the open. It cannot cover a panic inside Splice's own copy + // goroutines, and it deliberately does not re-panic, because there is no + // recovery middleware above a goroutine the HTTP layer has already + // returned from. + defer func() { + if p := recover(); p != nil { + xlog.Error("Panic while relaying a peer stream", "peer", peerID, "panic", p) + _ = stream.Close() + } + }() + + local, ok := r.accept(peerID, stream) + if !ok { + // accept has already answered and closed the stream. + return + } + + // Splice owns closing both ends from here. + // + // The error is logged at DEBUG and nowhere else. Every relayed request that + // a client abandons mid-stream produces one, so a warning here would be one + // line per cancelled inference; and the failures that are not cancellations + // are already visible to the frontend, whose gRPC or HTTP client sees a + // response that ended without its trailers or its final chunk. What this + // line adds is the only view from the middle of the path: which node, on + // which peer link, and what yamux actually said. + if err := Splice(stream, local); err != nil { + xlog.Debug("relayed peer stream ended with an error", "peer", peerID, "error", err) + } +} + +// accept reads which worker the stream is for and opens the worker-side stream. +// The second result is false when the stream was refused, in which case the +// refusal has been sent and the stream closed. +func (r *Relay) accept(peerID string, stream net.Conn) (net.Conn, bool) { + if err := stream.SetReadDeadline(time.Now().Add(r.headerTimeout)); err != nil { + // Nothing is readable on a stream whose deadline cannot be set, so this + // is reported as infrastructure rather than pushed past. + r.refuse(peerID, stream, fmt.Errorf("%w: arming the request deadline: %v", ErrRelayUnavailable, err)) + return nil, false + } + + nodeID, err := ReadRelayRequest(stream) + if err != nil { + // Includes the deadline above expiring. Both are "this stream never + // said which worker it wanted", which is the dialling replica's bug + // and not something a retry against this one resolves. + r.refuse(peerID, stream, fmt.Errorf("%w: %v", ErrRelayRequestInvalid, err)) + return nil, false + } + + // Cleared before the open rather than after the reply: everything past this + // frame belongs to the worker tunnel's conversation, which brings its own + // deadlines, and one left armed here would abort a long inference stream in + // the middle. + if err := stream.SetReadDeadline(time.Time{}); err != nil { + r.refuse(peerID, stream, fmt.Errorf("%w: clearing the request deadline: %v", ErrRelayUnavailable, err)) + return nil, false + } + + // Not the peer's deadline, because there is none to inherit: a yamux stream + // carries no context. This bounds only the open, so a request that gets + // past it is never cut short by it. + ctx, cancel := context.WithTimeout(context.Background(), r.openTimeout) + defer cancel() + + local, err := r.tunnels.Open(ctx, nodeID) + if err != nil { + if errors.Is(err, ErrNotOwner) { + // Passed through as itself. The worker is very likely connected and + // healthy somewhere else, and this is the one answer that tells the + // caller to look for it there. + r.refuse(peerID, stream, err) + return nil, false + } + // Everything else is this replica failing, and it must NOT become + // ErrNotOwner. The tunnel is held right here, so sending the caller + // looking elsewhere would send it back to this same replica; and it + // must not become absence either, because the worker is attached and a + // scheduler told otherwise would reclaim what it is running. + r.refuse(peerID, stream, fmt.Errorf("%w: %v", ErrRelayUnavailable, err)) + return nil, false + } + + if err := WriteRelayAccepted(stream); err != nil { + // The peer never learns the stream was accepted, so it cannot be used. + // Closing the worker-side stream here is what stops one leaking per + // failed reply. + xlog.Debug("could not accept a peer stream for relaying", "peer", peerID, "node", nodeID, "error", err) + _ = local.Close() + _ = stream.Close() + return nil, false + } + return local, true +} + +// refuse reports why a stream will not be relayed and then ENDS it. +// +// The close is the part that matters and it is not optional. A replica that +// says why and leaves the stream open has parked the peer on a request that +// will never be served, which reads as a slow replica rather than a refused +// request, and no deadline on the far side can tell those apart. The reply is +// what makes the refusal legible; the close is what makes it prompt. +// +// The reply is therefore best-effort and the close is not. +func (r *Relay) refuse(peerID string, stream net.Conn, reason error) { + if err := WriteRelayRefusal(stream, reason); err != nil { + xlog.Debug("could not tell a peer why its stream was refused", "peer", peerID, "reason", reason, "error", err) + } + _ = stream.Close() +} diff --git a/core/services/cluster/relay_internal_test.go b/core/services/cluster/relay_internal_test.go new file mode 100644 index 000000000000..855ac58653bf --- /dev/null +++ b/core/services/cluster/relay_internal_test.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: MIT + +package cluster + +import ( + "net" + "time" + + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// This spec is in-package because the header deadline it exercises is a +// production constant measured in seconds, and a spec that waited it out would +// be the slowest in the suite. The seam is unexported for the same reason the +// worker tunnel's is: it is a test knob, not an operator knob. +var _ = Describe("A peer stream that never says what it wants", func() { + It("is refused rather than left holding a relay goroutine", func() { + // Without a deadline on the opening frame, a peer that opens a stream + // and then goes quiet parks a goroutine and a stream slot until the + // whole session dies. A peer need not be malicious to do it: a dialler + // killed between OpenStream and its first write leaves exactly this. + relay := newRelay(NewTunnelRegistry(nil, "me"), 50*time.Millisecond, 0) + store := NewSessionStore(relay.Stream) + DeferCleanup(store.CloseAll) + + a, b := net.Pipe() + accepted, err := yamux.Server(a, nil, nil) + Expect(err).ToNot(HaveOccurred()) + peer, err := yamux.Client(b, nil, nil) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { + _ = peer.Close() + _ = accepted.Close() + }) + store.Accept("peer-1", accepted) + + stream, err := peer.OpenStream(GinkgoT().Context()) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = stream.Close() }) + + // Read with no deadline of our own: what is being asserted is that the + // RELAY answered, and a deadline here would be satisfied by a stream + // left parked just as well. + replies := make(chan error, 1) + go func() { + defer GinkgoRecover() + replies <- ReadRelayReply(stream) + }() + var reply error + Eventually(replies, "10s").Should(Receive(&reply)) + Expect(reply).To(MatchError(ErrRelayRequestInvalid)) + + ends := make(chan error, 1) + go func() { + defer GinkgoRecover() + _, err := stream.Read(make([]byte, 1)) + ends <- err + }() + Eventually(ends, "10s").Should(Receive(HaveOccurred())) + }) +}) diff --git a/core/services/cluster/relay_test.go b/core/services/cluster/relay_test.go new file mode 100644 index 000000000000..389700b7d0e3 --- /dev/null +++ b/core/services/cluster/relay_test.go @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: MIT + +package cluster_test + +import ( + "bytes" + "context" + "io" + "net" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" + + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +// blockingRead runs one Read on its own goroutine with NO deadline set. +// +// The absence of the deadline is the point. A refusal and a stream left parked +// are indistinguishable to an assertion that waits for a deadline to expire: +// both produce an error at the same moment. Reading with no deadline at all +// means the channel only ever receives because the far side ANSWERED or ENDED +// the stream, so Eventually(...).Should(Receive()) is an assertion about the +// relay rather than about the clock. +func blockingRead(conn net.Conn) chan error { + done := make(chan error, 1) + go func() { + defer GinkgoRecover() + _, err := conn.Read(make([]byte, 1)) + done <- err + }() + return done +} + +// relayReply reads the relay's answer, on its own goroutine and with no +// deadline, for the reason blockingRead gives. +func relayReply(conn net.Conn) chan error { + done := make(chan error, 1) + go func() { + defer GinkgoRecover() + done <- cluster.ReadRelayReply(conn) + }() + return done +} + +// readInto reads exactly len(buf) bytes on its own goroutine, with no deadline. +func readInto(conn net.Conn, buf []byte) chan error { + done := make(chan error, 1) + go func() { + defer GinkgoRecover() + _, err := io.ReadFull(conn, buf) + done <- err + }() + return done +} + +// acceptOne hands back the next stream accepted on a session. +func acceptOne(sess *yamux.Session) chan net.Conn { + accepted := make(chan net.Conn, 1) + go func() { + defer GinkgoRecover() + stream, err := sess.AcceptStream() + if err != nil { + return + } + accepted <- stream + }() + return accepted +} + +var _ = Describe("The inter-replica relay", func() { + var ( + db *gorm.DB + reg *cluster.Registry + tun *cluster.TunnelRegistry + ctx context.Context + + // peer is the dialling replica's half of the peer link, the side a + // relayed request arrives from. + peer *yamux.Session + ) + + // openRelayStream opens a peer stream and names the node it is for. + openRelayStream := func(nodeID string) net.Conn { + GinkgoHelper() + stream, err := peer.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = stream.Close() }) + Expect(cluster.WriteRelayRequest(stream, nodeID)).To(Succeed()) + return stream + } + + BeforeEach(func() { + db = testutil.SetupTestDB() + ctx = context.Background() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg = cluster.NewRegistry(db) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + tun = cluster.NewTunnelRegistry(reg, "me") + + store := cluster.NewSessionStore(cluster.NewRelay(tun).Stream) + DeferCleanup(store.CloseAll) + var accepted *yamux.Session + peer, accepted = yamuxPair() + store.Accept("peer-1", accepted) + }) + + It("splices a peer's stream onto a worker tunnel it holds, in both directions", func() { + // This is the whole point of the relay: with one tunnel per worker + // landing on ONE replica, every other replica reaches that worker only + // by relaying through this path, so with N replicas it carries roughly + // (N-1)/N of production traffic. + frontend, worker := workerTunnel() + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + echoOnce(worker) + + stream := openRelayStream("w1") + Eventually(relayReply(stream), "10s").Should(Receive(BeNil())) + + _, err = stream.Write([]byte("ping")) + Expect(err).ToNot(HaveOccurred()) + echoed := make([]byte, 4) + Eventually(readInto(stream, echoed), "10s").Should(Receive(BeNil())) + Expect(string(echoed)).To(Equal("ping")) + }) + + It("does not forward the frame it consumed, so the worker sees only the tunnelled protocol", func() { + // The relay request names the node for THIS hop and stops here. The + // worker's own request frame is written by the dialling replica and + // crosses untouched, so a relay that forwarded its own header would + // make every relayed stream unparseable at the worker while every + // locally-held one worked. + frontend, worker := workerTunnel() + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + accepted := acceptOne(worker) + + stream := openRelayStream("w1") + Eventually(relayReply(stream), "10s").Should(Receive(BeNil())) + _, err = stream.Write([]byte("first")) + Expect(err).ToNot(HaveOccurred()) + + var workerSide net.Conn + Eventually(accepted, "10s").Should(Receive(&workerSide)) + first := make([]byte, 5) + Eventually(readInto(workerSide, first), "10s").Should(Receive(BeNil())) + Expect(string(first)).To(Equal("first")) + }) + + It("tears down the worker's side when the peer's side closes", func() { + frontend, worker := workerTunnel() + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + accepted := acceptOne(worker) + + stream := openRelayStream("w1") + Eventually(relayReply(stream), "10s").Should(Receive(BeNil())) + var workerSide net.Conn + Eventually(accepted, "10s").Should(Receive(&workerSide)) + + Expect(stream.Close()).To(Succeed()) + // A relay that copies but does not tear down leaves a backend + // connection per abandoned request, and a worker runs out of them. + Eventually(blockingRead(workerSide), "10s").Should(Receive(HaveOccurred())) + }) + + It("tears down the peer's side when the worker's side closes", func() { + frontend, worker := workerTunnel() + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + accepted := acceptOne(worker) + + stream := openRelayStream("w1") + Eventually(relayReply(stream), "10s").Should(Receive(BeNil())) + var workerSide net.Conn + Eventually(accepted, "10s").Should(Receive(&workerSide)) + + Expect(workerSide.Close()).To(Succeed()) + Eventually(blockingRead(stream), "10s").Should(Receive(MatchError(io.EOF))) + }) + + It("refuses a node it does not hold with the routing fact, and ENDS the stream", func() { + stream := openRelayStream("not-here") + + var reply error + Eventually(relayReply(stream), "10s").Should(Receive(&reply)) + Expect(reply).To(MatchError(cluster.ErrNotOwner)) + // Four conditions this phase forbids collapsing. ErrNotOwner says + // "ask the owner"; absence says "this worker is gone" and a scheduler + // acts on that; unreachability says "retry". + Expect(reply).ToNot(MatchError(cluster.ErrNoConnection)) + Expect(reply).ToNot(MatchError(cluster.ErrPeerUnreachable)) + Expect(reply).ToNot(MatchError(cluster.ErrInstanceNotFound)) + Expect(reply).ToNot(MatchError(cluster.ErrRelayUnavailable)) + + // Answering is not enough. A relay that says why and leaves the stream + // open has parked the peer on a request that will never be served, + // which reads as a slow replica rather than a refused request. + Eventually(blockingRead(stream), "10s").Should(Receive(MatchError(io.EOF))) + }) + + It("refuses rather than chasing a node another live replica owns", func() { + // A relay that resolved the owner and relayed onward would make a + // stale row into a loop between two replicas, each certain the other + // holds the worker. One hop, always: the dialling replica re-resolves. + Expect(reg.Register(ctx, "other", "10.0.0.2:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "other") + Expect(err).ToNot(HaveOccurred()) + + stream := openRelayStream("w1") + var reply error + Eventually(relayReply(stream), "10s").Should(Receive(&reply)) + Expect(reply).To(MatchError(cluster.ErrNotOwner)) + Eventually(blockingRead(stream), "10s").Should(Receive(MatchError(io.EOF))) + }) + + It("reports a tunnel that will not carry a stream as infrastructure, never as not-owner", func() { + // The tunnel IS held here; its session died. Answering ErrNotOwner + // would send the dialling replica looking elsewhere for a worker that + // is attached right here, and it would find this replica again. + frontend, worker := workerTunnel() + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + Expect(worker.Close()).To(Succeed()) + Eventually(frontend.IsClosed, "10s").Should(BeTrue()) + + stream := openRelayStream("w1") + var reply error + Eventually(relayReply(stream), "10s").Should(Receive(&reply)) + Expect(reply).To(MatchError(cluster.ErrRelayUnavailable)) + Expect(reply).ToNot(MatchError(cluster.ErrNotOwner)) + Expect(reply).ToNot(MatchError(cluster.ErrNoConnection)) + Expect(reply).ToNot(MatchError(cluster.ErrInstanceNotFound)) + Eventually(blockingRead(stream), "10s").Should(Receive(MatchError(io.EOF))) + }) + + It("refuses a malformed opening frame as the caller's bug", func() { + stream, err := peer.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = stream.Close() }) + // A well-formed frame carrying no node id. A relay that read this as a + // node named "" would go looking for it and answer ErrNotOwner, which + // tells the caller to retry elsewhere for a request no replica can + // ever serve. + _, err = stream.Write([]byte{0x00, 0x00}) + Expect(err).ToNot(HaveOccurred()) + + var reply error + Eventually(relayReply(stream), "10s").Should(Receive(&reply)) + Expect(reply).To(MatchError(cluster.ErrRelayRequestInvalid)) + Expect(reply).ToNot(MatchError(cluster.ErrNotOwner)) + Expect(reply).ToNot(MatchError(cluster.ErrRelayUnavailable)) + Eventually(blockingRead(stream), "10s").Should(Receive(MatchError(io.EOF))) + }) +}) + +var _ = Describe("The relay wire framing", func() { + // The relay hop and the worker tunnel hop travel back to back on one + // stream, and a dialler reads a reply from each in order. Giving them + // disjoint vocabularies means a reader applied to the wrong hop fails + // loudly rather than returning a plausible sentinel for the other hop, + // which would report a worker's refusal as the owning replica's and send a + // retry to the wrong place. + It("does not read a relay reply as a worker tunnel reply", func() { + frame := &bytes.Buffer{} + Expect(cluster.WriteRelayAccepted(frame)).To(Succeed()) + Expect(cluster.ReadStreamReply(frame)).To(HaveOccurred()) + }) + + It("does not read a worker tunnel reply as a relay reply", func() { + frame := &bytes.Buffer{} + Expect(cluster.WriteStreamAccepted(frame)).To(Succeed()) + Expect(cluster.ReadRelayReply(frame)).To(HaveOccurred()) + }) + + It("round-trips a node id", func() { + frame := &bytes.Buffer{} + Expect(cluster.WriteRelayRequest(frame, "node-7")).To(Succeed()) + nodeID, err := cluster.ReadRelayRequest(frame) + Expect(err).ToNot(HaveOccurred()) + Expect(nodeID).To(Equal("node-7")) + }) + + It("refuses to write an empty node id, rather than spending a round trip on it", func() { + Expect(cluster.WriteRelayRequest(&bytes.Buffer{}, "")).To(HaveOccurred()) + }) +}) diff --git a/core/services/cluster/sessions.go b/core/services/cluster/sessions.go index 7ee5fa3327af..afcdd1b97384 100644 --- a/core/services/cluster/sessions.go +++ b/core/services/cluster/sessions.go @@ -23,6 +23,10 @@ type SessionStore struct { // onStream handles one accepted stream and owns closing it. A nil handler // closes the stream immediately, which is what a replica with no relay // installed should do: refuse promptly rather than leave a peer parked. + // + // In distributed mode this is Relay.Stream, which splices the stream onto + // a worker tunnel this replica holds. Nil is reached only from specs, and + // from a caller that wants a store with no relay. onStream func(peerID string, stream net.Conn) mu sync.Mutex diff --git a/core/services/cluster/splice.go b/core/services/cluster/splice.go index 604bbfec93d4..691586c254ca 100644 --- a/core/services/cluster/splice.go +++ b/core/services/cluster/splice.go @@ -80,29 +80,27 @@ func copyStream(dst io.Writer, src io.Reader) error { // *net.TCPConn takes a WriteTo/ReadFrom path that would hand one back. So a // bare io.EOF arriving here came from a failing Write or Close, where it means // the peer is gone, and yamux produces exactly that when a Write races its -// session's shutdown (see isMuxSessionFailure). +// session's shutdown (see isMuxFailure). // // A socket-level abort (ECONNRESET, EPIPE) is deliberately absent too, which // makes the same underlying event, a peer aborting mid-stream, reach the caller -// as nil over a yamux tunnel and as an error over a raw socket. That asymmetry -// is intended: the yamux endings are the teardown Splice's own Close provokes, -// so this primitive is the only thing that can tell them from a fault, whereas -// whether an aborted request is routine or a failure is the relay's policy and -// only the relay knows which request was abandoned. +// as nil over a raw socket where it would be an error. That asymmetry is +// narrower than it was, since a yamux abort is now reported (see isMuxFailure), +// and what remains of it is that a raw socket cannot say who aborted. // // The mux checks run first, and that ordering is load-bearing: a dying yamux // session usually hands every live stream its own cause wrapped up -// (session.go:330), and that cause is routinely a closed-socket error, so -// consulting the generic endings first would report a peer that vanished -// mid-request as a clean completion. +// (go-yamux/v5@v5.1.0/session.go:328-337), and that cause is routinely a +// closed-socket error, so consulting the generic endings first would report a +// peer that vanished mid-request as a clean completion. func normalizeStreamErr(err error) error { if err == nil { return nil } - if isMuxSessionFailure(err) { + if isMuxFailure(err) { return err } - if isMuxStreamTeardown(err) { + if isMuxLocalTeardown(err) { return nil } if errors.Is(err, net.ErrClosed) || @@ -116,23 +114,66 @@ func normalizeStreamErr(err error) error { // declared with it because the constant itself is unexported. var normalGoAwayCode = yamux.ErrRemoteGoAway.ErrorCode -// isMuxSessionFailure reports whether err is the yamux session underneath a -// stream dying, as opposed to a single stream being torn down. The distinction -// matters because Splice must stay quiet about the teardown it provokes itself -// while still reporting a dead peer: a keepalive timeout, a broken TCP -// connection or a protocol error under a relayed request has to reach the -// caller, or a failed inference looks like a finished one. -func isMuxSessionFailure(err error) bool { - // A go-away ends the whole session. Only the "no error" code is a normal - // ending; a protocol or internal error go-away is a real failure. +// isMuxFailure reports whether err is yamux saying the conversation was CUT, +// as opposed to ended by this side. +// +// The distinction is what keeps Splice quiet about the teardown it provokes +// itself while still reporting a request that died: a keepalive timeout, a +// broken connection, a peer that reset the stream or a peer that went away +// under a relayed request has to reach the caller, or a failed inference looks +// like a finished one. +// +// TWO OF THESE ARE THE POLICY PHASE 1 LEFT OPEN, and this is where they are +// settled, by the relay in core/services/cluster/relay.go, which is Splice's +// first production caller. Both used to be reported as normal termination. +// Neither can be settled by a caller reading Splice's result, because a result +// mapped to nil carries nothing left to reclassify, so the decision has to live +// at the classifier; the two callers there are the relay and the worker tunnel, +// and both are splicing an in-flight request, so both want the same answer. +// +// 1. A peer-initiated stream reset, *StreamError{Remote: true}. yamux builds +// it in processFlags when an RST frame arrives on the stream +// (stream.go:432-449); a reset this side asked for carries Remote: false +// instead (stream.go:283-291), and Splice never resets anything anyway, its +// own Close sending a FIN (stream.go:303-331, 365-368). So Remote: true is +// unambiguously "the far side aborted this stream", which for a relayed +// request means the response was truncated. REPORTED. The caller decides +// how loud that is: a client cancelling produces one per cancellation, so +// the relay logs it at debug rather than treating it as a fault. +// +// 2. A graceful go-away from the peer, ErrRemoteGoAway. handleGoAway returns +// it for code goAwayNormal (session.go:829-833), recv closes the session +// with it, and close hands it UNWRAPPED to every live stream, because it +// already is a *GoAwayError and so escapes the ErrStreamReset wrapping +// (session.go:328-337, stream.go:371-387). Graceful describes the SESSION, +// not the requests on it: every one of those streams was mid-request. +// REPORTED, for the same reason as above. +// +// The locally-initiated forms of both stay silent, and keying on Remote is what +// separates them: ErrSessionShutdown is a *GoAwayError with Remote: false +// (const.go:96) and is exactly what this process closing its own session +// produces (session.go:284). +// +// What this does NOT do is make the far side see a failure. Splice ends both +// streams with Close, which is a FIN, and a reset after that is a no-op because +// Close has already moved the stream to streamFinished (stream.go:266-272, +// 303-331, 336-361). Propagating a truncation as an RST would mean reshaping +// Splice's teardown, and it buys little: the protocols relayed here are gRPC +// and HTTP, both of which detect a body that ended without its trailers or its +// final chunk. Reporting is what the caller needs and this is where it comes +// from. +func isMuxFailure(err error) bool { + // A go-away ends the whole session. Only a normal-code go-away this side + // sent is a normal ending. var goAway *yamux.GoAwayError if errors.As(err, &goAway) { - return goAway.ErrorCode != normalGoAwayCode + return goAway.Remote || goAway.ErrorCode != normalGoAwayCode } - // A stream error is scoped to one stream, whatever killed it. + // A stream error is scoped to one stream. Only a reset this side asked for + // is a normal ending. var streamErr *yamux.StreamError if errors.As(err, &streamErr) { - return false + return streamErr.Remote } // Session.close gives every stream it kills ErrStreamReset wrapped around // the cause, so the bare sentinel means this stream was reset and a @@ -142,32 +183,37 @@ func isMuxSessionFailure(err error) bool { // Wrapped is not the only way a dead session shows up, though. close() // publishes shutdownErr and closes shutdownCh before it force-closes the // streams, so a Write or Close landing in that window gets the raw cause - // back instead (session.go:507-510, 528-533). That form is unrecognisable + // back instead (session.go:305-308, 528-533). That form is unrecognisable // as yamux at all, which is why normalizeStreamErr no longer forgives a // bare io.EOF: for a peer that vanished, the raw cause is precisely io.EOF. return errors.Is(err, yamux.ErrStreamReset) && err != yamux.ErrStreamReset } -// isMuxStreamTeardown reports whether err is yamux ending one stream, which -// Splice mostly provokes itself: closing a stream whose session has already -// shut down normally returns ErrSessionShutdown from the FIN write, and a copy -// parked on a stream that gets closed comes back with ErrStreamClosed or a -// reset. A reset does not only mean that, though; the same sentinel heads the -// error a dying session hands its streams, which is why isMuxSessionFailure -// runs first. None of yamux's error types match net.ErrClosed, so all of this -// has to be recognised by shape. -func isMuxStreamTeardown(err error) bool { +// isMuxLocalTeardown reports whether err is yamux ending one stream at this +// side's request, which Splice mostly provokes itself: closing a stream whose +// session has already shut down normally returns ErrSessionShutdown from the +// FIN write, and a copy parked on a stream that gets closed comes back with +// ErrStreamClosed from a Write (stream.go:157-159) or the bare ErrStreamReset +// from a Read, which is what CloseRead installs (stream.go:348-349). A reset +// does not only mean that, though; the same sentinel heads the error a dying +// session hands its streams, and the remote forms belong to isMuxFailure, which +// is why that runs first. None of yamux's error types match net.ErrClosed, so +// all of this has to be recognised by shape. +func isMuxLocalTeardown(err error) bool { // Sentinels by identity, never errors.Is: the wrapped forms belong to a // dead session and are reported instead. ErrSessionShutdown is absent on // purpose rather than by oversight, being a *GoAwayError carrying the - // normal code, which the last check below covers. + // normal code with Remote false, which the last check below covers. if err == yamux.ErrStreamClosed || err == yamux.ErrStreamReset { return true } + // Remote is re-checked rather than assumed from the ordering, so that this + // predicate is true to its own name if it is ever called from anywhere + // else. var streamErr *yamux.StreamError if errors.As(err, &streamErr) { - return true + return !streamErr.Remote } var goAway *yamux.GoAwayError - return errors.As(err, &goAway) && goAway.ErrorCode == normalGoAwayCode + return errors.As(err, &goAway) && !goAway.Remote && goAway.ErrorCode == normalGoAwayCode } diff --git a/core/services/cluster/splice_test.go b/core/services/cluster/splice_test.go index 7b423a142730..7b1001cd8601 100644 --- a/core/services/cluster/splice_test.go +++ b/core/services/cluster/splice_test.go @@ -172,8 +172,12 @@ var _ = Describe("Splice", func() { Entry("a closed yamux stream", yamux.ErrStreamClosed), Entry("a reset yamux stream", yamux.ErrStreamReset), Entry("a shut-down yamux session", yamux.ErrSessionShutdown), - Entry("a stream reset by the remote", &yamux.StreamError{ErrorCode: 1, Remote: true}), - Entry("a go-away from the remote", yamux.ErrRemoteGoAway), + // The LOCAL forms of the two endings the relay settled below. They + // stay normal because they are the teardown this side asked for, + // and Remote is the only thing separating them from the endings a + // peer inflicts. + Entry("a stream this side reset", &yamux.StreamError{ErrorCode: 0, Remote: false}), + Entry("a go-away this side sent", &yamux.GoAwayError{ErrorCode: 0, Remote: false}), ) // sessionDeath is the exact shape Session.close hands every live @@ -205,6 +209,14 @@ var _ = Describe("Splice", func() { Entry("a peer that vanished", sessionDeath(io.EOF)), Entry("a protocol-error go-away", &yamux.GoAwayError{Remote: true, ErrorCode: 1}), Entry("an internal-error go-away", &yamux.GoAwayError{Remote: true, ErrorCode: 2}), + // The two endings phase 1 left open and the relay, Splice's first + // production caller, settled as failures. Both truncate whatever + // was in flight, and reporting them as normal termination is how a + // half-finished inference comes to look like a short one that + // completed. See isMuxFailure for why the decision could not be + // left to a caller reading Splice's result. + Entry("a stream the peer reset", &yamux.StreamError{ErrorCode: 1, Remote: true}), + Entry("a graceful go-away from the peer", yamux.ErrRemoteGoAway), ) // A bare io.EOF can only reach Splice from a failing Write. io.Copy From 1f3dc70f924ec68f88ba91dc09519befdac6792e Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 14:39:24 +0000 Subject: [PATCH 053/153] fix(cluster): express the splice policy once, and pin the relay's budgets Review round 1 on task 5. Eight non-blocking items, all addressed. The classifier read Remote in two predicates with a report-by-default fallthrough behind them, so reverting either read left the whole suite green: the error reached the same answer down the other path. A correctness argument that rests on mutation evidence cannot afford a shape that cannot be mutated in pieces, so the two predicates collapse into one muxVerdict deciding each error type once. Falsifying either Remote read now reddens exactly one spec. Three claims the comments made loudly and nothing tested: - clearing the header read deadline before the splice. Deleting the clear left all 49 focused specs green, while in production it is the difference between a relayed response that streams for an hour and one that dies after fifteen seconds of quiet; - the open budget bounding the open and nothing after it; - closing the worker-side stream when the acceptance reply cannot be written, which leaks one stream on the worker per failure. All three are pinned now. The first two share a spec that sets both budgets to 50ms and then watches the conversation outlive them by ten times, which is an assertion about an event that must not happen and so is the one wait a channel cannot replace. The third drives the relay with a peer stream that delivers a request and then fails every write, because no pair of live yamux sessions can be made to fail that write on cue. The disjoint-vocabulary argument was specced for the accepted frame only. Both refusal directions are covered now, and asserted as "not one of the other hop's sentinels" rather than merely "an error", since reading a relay refusal with the tunnel's reader always errors and the question is whether it errors as the wrong thing. The open budget stays non-configurable, and says so: the number that matters is how long the original client will wait, which is not known on this side and is not something a deployment-wide constant can stand in for. The honest fix is the caller's remaining budget travelling in the request frame, which belongs to the dialler that has the budget. Two comments corrected: nothing deadlines the peer stream after the clear, so the tunnelled protocol's own deadlines cannot be what justifies clearing it; and the membership sweep deletes departed replicas but reports only how many, so identifying them is work that would have to be done, not knowledge waiting to be plumbed. Recorded at muxVerdict: a remote RST that does not ride a typeWindowUpdate frame yields the bare sentinel and is still silenced, which is unreachable between two go-yamux peers but keeps the new rule from reading as unconditional. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/peerlink.go | 8 +- core/services/cluster/relay.go | 21 +- core/services/cluster/relay_internal_test.go | 245 +++++++++++++++++++ core/services/cluster/relay_test.go | 35 +++ core/services/cluster/splice.go | 112 +++++---- 5 files changed, 359 insertions(+), 62 deletions(-) diff --git a/core/services/cluster/peerlink.go b/core/services/cluster/peerlink.go index 047fcbc5472a..5f5e3186445b 100644 --- a/core/services/cluster/peerlink.go +++ b/core/services/cluster/peerlink.go @@ -233,10 +233,10 @@ func (p *PeerPool) Open(ctx context.Context, peerID string) (net.Conn, error) { // has left the deployment but is still listening keeps a live WebSocket and the // two yamux loop goroutines behind it for as long as this process runs; a peer // that is genuinely gone is reclaimed by the 30s keepalive default, so the real -// exposure is narrow. There is no Forget: the membership sweep does know which -// replicas have left, but nothing plumbs that knowledge to this pool, so an -// entry for a departed peer outlives it and only the keepalive reclaims what it -// holds. +// exposure is narrow. There is no Forget: the membership sweep DELETES departed +// replicas but reports only how many, so which ones they were would have to be +// surfaced before anything could be plumbed here. Until it is, an entry for a +// departed peer outlives it and only the keepalive reclaims what it holds. func (p *PeerPool) link(peerID string) (*peerLink, error) { p.mu.Lock() defer p.mu.Unlock() diff --git a/core/services/cluster/relay.go b/core/services/cluster/relay.go index 99164c6fbd74..9fa03807d465 100644 --- a/core/services/cluster/relay.go +++ b/core/services/cluster/relay.go @@ -178,6 +178,15 @@ const ( // which is the only reason there is one here. Without the bound, a worker // that has stopped accepting would turn a refusable condition into a parked // peer, which is the one outcome this path exists to avoid. + // + // It is deliberately NOT configurable. No operator has the information to + // set it: the number that matters is how long the ORIGINAL client is + // willing to wait, which is not known on this side of the link and is not + // something a deployment-wide constant can stand in for. The honest fix is + // the caller's remaining budget travelling in the relay request frame, and + // that belongs to the dialler that has the budget. Until then this is a + // backstop against parking, generous on purpose, because refusing healthy + // traffic costs more than waiting. relayOpenTimeout = 15 * time.Second ) @@ -271,10 +280,14 @@ func (r *Relay) accept(peerID string, stream net.Conn) (net.Conn, bool) { return nil, false } - // Cleared before the open rather than after the reply: everything past this - // frame belongs to the worker tunnel's conversation, which brings its own - // deadlines, and one left armed here would abort a long inference stream in - // the middle. + // Cleared before the open rather than after the reply, and NOTHING arms + // another deadline on this stream afterwards. That is the intent rather + // than an omission: what follows is a relayed request whose length is the + // caller's business, and a header deadline left armed here would abort a + // long inference stream after any quiet moment in the middle of it. What + // still bounds the conversation is the peer link's own keepalive, which + // kills the session under it when the far side stops answering, and + // whatever deadline the original client is holding. if err := stream.SetReadDeadline(time.Time{}); err != nil { r.refuse(peerID, stream, fmt.Errorf("%w: clearing the request deadline: %v", ErrRelayUnavailable, err)) return nil, false diff --git a/core/services/cluster/relay_internal_test.go b/core/services/cluster/relay_internal_test.go index 855ac58653bf..e23f6c55c888 100644 --- a/core/services/cluster/relay_internal_test.go +++ b/core/services/cluster/relay_internal_test.go @@ -3,9 +3,16 @@ package cluster import ( + "bytes" + "context" + "errors" + "io" "net" + "sync" "time" + "github.com/mudler/LocalAI/core/services/testutil" + "github.com/libp2p/go-yamux/v5" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -61,3 +68,241 @@ var _ = Describe("A peer stream that never says what it wants", func() { Eventually(ends, "10s").Should(Receive(HaveOccurred())) }) }) + +// backloggedPair returns a peer/relay session pair whose SYN backlog is one +// stream deep, so a single un-accepted open fills it and the next one parks. +// yamux's default is 256 (mux.go, DefaultConfig), and filling that from a spec +// would mean 256 real opens to prove one property. +func backloggedPair(backlog int) (dialled, accepted *yamux.Session) { + GinkgoHelper() + cfg := yamux.DefaultConfig() + cfg.AcceptBacklog = backlog + a, b := net.Pipe() + var err error + accepted, err = yamux.Server(a, cfg, nil) + Expect(err).ToNot(HaveOccurred()) + dialled, err = yamux.Client(b, cfg, nil) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { + _ = dialled.Close() + _ = accepted.Close() + }) + return dialled, accepted +} + +// unwritableStream is a peer stream that delivers one relay request and then +// fails every write. It stands in for a peer that vanished between opening the +// stream and hearing the answer, which is the only way the acceptance reply +// fails, and which no pair of live yamux sessions can be made to do on cue. +type unwritableStream struct { + net.Conn + request []byte + read int + closed chan struct{} + closeOne sync.Once +} + +func newUnwritableStream(nodeID string) *unwritableStream { + GinkgoHelper() + frame := &bytes.Buffer{} + Expect(WriteRelayRequest(frame, nodeID)).To(Succeed()) + return &unwritableStream{request: frame.Bytes(), closed: make(chan struct{})} +} + +func (s *unwritableStream) Read(p []byte) (int, error) { + if s.read >= len(s.request) { + // Never EOF: an EOF here would end the relay for a reason other than + // the failed write, and the spec would pass without exercising it. + <-s.closed + return 0, io.EOF + } + n := copy(p, s.request[s.read:]) + s.read += n + return n, nil +} + +func (s *unwritableStream) Write([]byte) (int, error) { return 0, errors.New("peer went away") } + +func (s *unwritableStream) Close() error { + s.closeOne.Do(func() { close(s.closed) }) + return nil +} + +func (s *unwritableStream) SetReadDeadline(time.Time) error { return nil } + +var _ = Describe("The relay's own budgets", func() { + var ( + reg *Registry + tun *TunnelRegistry + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + db := testutil.SetupTestDB() + Expect(Migrate(ctx, db)).To(Succeed()) + reg = NewRegistry(db) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + tun = NewTunnelRegistry(reg, "me") + }) + + It("stops bounding the stream once the relay hands it over", func() { + // Both budgets are set to 50ms here and both are deliberately shorter + // than the window this spec then watches. A header deadline left armed + // past acceptance, or an open budget applied to the stream it produced, + // would abort a relayed inference after 50ms of quiet, which in + // production is the difference between a response that streams for an + // hour and one that dies mid-token. + relay := newRelay(tun, 50*time.Millisecond, 50*time.Millisecond) + store := NewSessionStore(relay.Stream) + DeferCleanup(store.CloseAll) + peer, accepted := backloggedPair(256) + store.Accept("peer-1", accepted) + + worker, frontend := backloggedPair(256) + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + + workerSide := make(chan net.Conn, 1) + go func() { + defer GinkgoRecover() + stream, err := worker.AcceptStream() + if err != nil { + return + } + workerSide <- stream + }() + + stream, err := peer.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = stream.Close() }) + Expect(WriteRelayRequest(stream, "w1")).To(Succeed()) + + replies := make(chan error, 1) + go func() { + defer GinkgoRecover() + replies <- ReadRelayReply(stream) + }() + Eventually(replies, "10s").Should(Receive(BeNil())) + + var served net.Conn + Eventually(workerSide, "10s").Should(Receive(&served)) + + // One reader for both questions, so that watching for a teardown does + // not eat the bytes the second half of the spec is waiting for. + data := make(chan []byte, 4) + ended := make(chan error, 1) + go func() { + defer GinkgoRecover() + buf := make([]byte, 64) + for { + n, err := served.Read(buf) + if n > 0 { + data <- append([]byte(nil), buf[:n]...) + } + if err != nil { + ended <- err + return + } + } + }() + + // An assertion about an event that must NOT happen, which is the one + // kind a channel cannot replace: a torn-down splice ends the worker's + // side, and there is no event for "still alive". The window is ten + // times the budgets it is watching. + Consistently(ended, "500ms", "50ms").ShouldNot(Receive(), + "the relay tore the stream down on a budget that should have stopped applying at acceptance") + + // And it is not merely un-torn-down: it still carries bytes, long after + // both budgets would have expired. + _, err = stream.Write([]byte("late")) + Expect(err).ToNot(HaveOccurred()) + Eventually(data, "10s").Should(Receive(Equal([]byte("late")))) + }) + + It("refuses rather than parking when the worker's tunnel will not take another stream", func() { + // The open budget exists because yamux BLOCKS an open once the accept + // backlog is full rather than failing it, so without a bound an + // overloaded worker turns a refusable condition into a parked peer. + relay := newRelay(tun, 0, 50*time.Millisecond) + store := NewSessionStore(relay.Stream) + DeferCleanup(store.CloseAll) + peer, accepted := backloggedPair(256) + store.Accept("peer-1", accepted) + + _, frontend := backloggedPair(1) + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + // One un-accepted open fills the one-deep backlog; the relay's own open + // is the one that has to wait. + filler, err := frontend.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = filler.Close() }) + + stream, err := peer.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = stream.Close() }) + Expect(WriteRelayRequest(stream, "w1")).To(Succeed()) + + replies := make(chan error, 1) + go func() { + defer GinkgoRecover() + replies <- ReadRelayReply(stream) + }() + var reply error + Eventually(replies, "10s").Should(Receive(&reply)) + Expect(reply).To(MatchError(ErrRelayUnavailable)) + // The tunnel IS held here, so this must not read as a routing fact. + Expect(reply).ToNot(MatchError(ErrNotOwner)) + + ends := make(chan error, 1) + go func() { + defer GinkgoRecover() + _, err := stream.Read(make([]byte, 1)) + ends <- err + }() + Eventually(ends, "10s").Should(Receive(HaveOccurred())) + }) + + It("closes the worker's stream when it cannot tell the peer the stream was accepted", func() { + // The reply is the last thing that can fail after a worker stream has + // been opened. A relay that gave up without closing it would leak one + // stream on the worker per failed acceptance, and the worker cannot + // tell those from live ones. + relay := newRelay(tun, 0, 0) + worker, frontend := backloggedPair(256) + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + + workerSide := make(chan net.Conn, 1) + go func() { + defer GinkgoRecover() + stream, err := worker.AcceptStream() + if err != nil { + return + } + workerSide <- stream + }() + + peerStream := newUnwritableStream("w1") + done := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(done) + relay.Stream("peer-1", peerStream) + }() + Eventually(done, "10s").Should(BeClosed()) + + var served net.Conn + Eventually(workerSide, "10s").Should(Receive(&served)) + ended := make(chan error, 1) + go func() { + defer GinkgoRecover() + _, err := served.Read(make([]byte, 1)) + ended <- err + }() + Eventually(ended, "10s").Should(Receive(HaveOccurred()), + "the worker-side stream outlived the relay that opened it") + }) +}) diff --git a/core/services/cluster/relay_test.go b/core/services/cluster/relay_test.go index 389700b7d0e3..e0c7bd0499c2 100644 --- a/core/services/cluster/relay_test.go +++ b/core/services/cluster/relay_test.go @@ -288,4 +288,39 @@ var _ = Describe("The relay wire framing", func() { It("refuses to write an empty node id, rather than spending a round trip on it", func() { Expect(cluster.WriteRelayRequest(&bytes.Buffer{}, "")).To(HaveOccurred()) }) + + // Acceptance is not the whole surface. A refusal read by the wrong hop's + // reader must not come back as one of that hop's own sentinels: "the + // owning replica does not hold this worker" arriving as "the worker does + // not serve that tag" would send a retry to the wrong end of the path, and + // it would look like a perfectly ordinary answer on the way. + DescribeTable("does not read a relay refusal as one of the worker tunnel's", + func(reason error) { + frame := &bytes.Buffer{} + Expect(cluster.WriteRelayRefusal(frame, reason)).To(Succeed()) + err := cluster.ReadStreamReply(frame) + Expect(err).To(HaveOccurred()) + Expect(err).ToNot(MatchError(cluster.ErrStreamTagUnknown)) + Expect(err).ToNot(MatchError(cluster.ErrStreamTargetUnavailable)) + Expect(err).ToNot(MatchError(cluster.ErrStreamRequestInvalid)) + }, + Entry("not the owner", cluster.ErrNotOwner), + Entry("the tunnel will not carry a stream", cluster.ErrRelayUnavailable), + Entry("a malformed relay request", cluster.ErrRelayRequestInvalid), + ) + + DescribeTable("does not read a worker tunnel refusal as one of the relay's", + func(reason error) { + frame := &bytes.Buffer{} + Expect(cluster.WriteStreamRefusal(frame, reason)).To(Succeed()) + err := cluster.ReadRelayReply(frame) + Expect(err).To(HaveOccurred()) + Expect(err).ToNot(MatchError(cluster.ErrNotOwner)) + Expect(err).ToNot(MatchError(cluster.ErrRelayUnavailable)) + Expect(err).ToNot(MatchError(cluster.ErrRelayRequestInvalid)) + }, + Entry("an unknown stream tag", cluster.ErrStreamTagUnknown), + Entry("a local service that will not answer", cluster.ErrStreamTargetUnavailable), + Entry("a malformed stream request", cluster.ErrStreamRequestInvalid), + ) }) diff --git a/core/services/cluster/splice.go b/core/services/cluster/splice.go index 691586c254ca..32fe4358f3d7 100644 --- a/core/services/cluster/splice.go +++ b/core/services/cluster/splice.go @@ -80,27 +80,27 @@ func copyStream(dst io.Writer, src io.Reader) error { // *net.TCPConn takes a WriteTo/ReadFrom path that would hand one back. So a // bare io.EOF arriving here came from a failing Write or Close, where it means // the peer is gone, and yamux produces exactly that when a Write races its -// session's shutdown (see isMuxFailure). +// session's shutdown (see muxVerdict). // // A socket-level abort (ECONNRESET, EPIPE) is deliberately absent too, which // makes the same underlying event, a peer aborting mid-stream, reach the caller // as nil over a raw socket where it would be an error. That asymmetry is -// narrower than it was, since a yamux abort is now reported (see isMuxFailure), +// narrower than it was, since a yamux abort is now reported (see muxVerdict), // and what remains of it is that a raw socket cannot say who aborted. // -// The mux checks run first, and that ordering is load-bearing: a dying yamux -// session usually hands every live stream its own cause wrapped up -// (go-yamux/v5@v5.1.0/session.go:328-337), and that cause is routinely a +// The mux verdict is consulted first, and that ordering is load-bearing: a +// dying yamux session usually hands every live stream its own cause wrapped up +// (go-yamux/v5@v5.1.0 session.go, Session.close), and that cause is routinely a // closed-socket error, so consulting the generic endings first would report a // peer that vanished mid-request as a clean completion. func normalizeStreamErr(err error) error { if err == nil { return nil } - if isMuxFailure(err) { - return err - } - if isMuxLocalTeardown(err) { + if recognised, report := muxVerdict(err); recognised { + if report { + return err + } return nil } if errors.Is(err, net.ErrClosed) || @@ -114,14 +114,24 @@ func normalizeStreamErr(err error) error { // declared with it because the constant itself is unexported. var normalGoAwayCode = yamux.ErrRemoteGoAway.ErrorCode -// isMuxFailure reports whether err is yamux saying the conversation was CUT, -// as opposed to ended by this side. +// muxVerdict classifies a yamux ending. recognised says the error came from the +// multiplexer at all; report says the ending was INFLICTED on this stream +// rather than asked for by this side. // -// The distinction is what keeps Splice quiet about the teardown it provokes -// itself while still reporting a request that died: a keepalive timeout, a -// broken connection, a peer that reset the stream or a peer that went away -// under a relayed request has to reach the caller, or a failed inference looks -// like a finished one. +// It is ONE function, and that is the point rather than a matter of taste. The +// policy below turns on a single bit, Remote, and an earlier shape read that +// bit in two predicates with a report-by-default fallthrough behind them. +// Reverting either read left the whole suite green, because the error reached +// the same answer down the other path: the classifier could not be +// mutation-tested in pieces, which in code whose correctness argument IS its +// mutation evidence is worse than the duplication it bought. Here each type is +// decided once, so falsifying either read reddens a spec. +// +// The distinction it draws is what keeps Splice quiet about the teardown it +// provokes itself while still reporting a request that died: a keepalive +// timeout, a broken connection, a peer that reset the stream or a peer that +// went away under a relayed request has to reach the caller, or a failed +// inference looks like a finished one. // // TWO OF THESE ARE THE POLICY PHASE 1 LEFT OPEN, and this is where they are // settled, by the relay in core/services/cluster/relay.go, which is Splice's @@ -154,6 +164,14 @@ var normalGoAwayCode = yamux.ErrRemoteGoAway.ErrorCode // (const.go:96) and is exactly what this process closing its own session // produces (session.go:284). // +// The rule is "a remote reset is reported" and not "every remote reset is +// reported". yamux only builds a *StreamError when the RST rides a +// typeWindowUpdate frame (stream.go:436); an RST on any other frame type +// yields the BARE ErrStreamReset sentinel, which is claimed below as this +// side's own teardown and silenced. Every reset go-yamux itself sends uses +// typeWindowUpdate, so the gap is unreachable between two LocalAI processes and +// only a foreign multiplexer implementation could reach it. +// // What this does NOT do is make the far side see a failure. Splice ends both // streams with Close, which is a FIN, and a reset after that is a no-op because // Close has already moved the stream to streamFinished (stream.go:266-272, @@ -162,58 +180,44 @@ var normalGoAwayCode = yamux.ErrRemoteGoAway.ErrorCode // and HTTP, both of which detect a body that ended without its trailers or its // final chunk. Reporting is what the caller needs and this is where it comes // from. -func isMuxFailure(err error) bool { +func muxVerdict(err error) (recognised, report bool) { // A go-away ends the whole session. Only a normal-code go-away this side // sent is a normal ending. var goAway *yamux.GoAwayError if errors.As(err, &goAway) { - return goAway.Remote || goAway.ErrorCode != normalGoAwayCode + return true, goAway.Remote || goAway.ErrorCode != normalGoAwayCode } // A stream error is scoped to one stream. Only a reset this side asked for // is a normal ending. var streamErr *yamux.StreamError if errors.As(err, &streamErr) { - return streamErr.Remote + return true, streamErr.Remote } - // Session.close gives every stream it kills ErrStreamReset wrapped around - // the cause, so the bare sentinel means this stream was reset and a - // wrapped one means the session died under it. Identity is what separates - // them; errors.Is cannot. + // Sentinels by identity, never errors.Is, and before the wrapped check + // below: these are the endings Splice provokes itself. Closing a stream + // whose session has already shut down normally returns ErrSessionShutdown + // from the FIN write, which the go-away branch above has already claimed; + // a copy parked on a stream that gets closed comes back with + // ErrStreamClosed from a Write (stream.go:157-159) or the bare + // ErrStreamReset from a Read, which is what CloseRead installs + // (stream.go:348-349). + if err == yamux.ErrStreamClosed || err == yamux.ErrStreamReset { + return true, false + } + // The same sentinel WRAPPED means something else entirely: Session.close + // gives every stream it kills ErrStreamReset wrapped around the cause, so + // this is the session dying under a live stream. Identity above is what + // separates the two; errors.Is cannot. // // Wrapped is not the only way a dead session shows up, though. close() // publishes shutdownErr and closes shutdownCh before it force-closes the // streams, so a Write or Close landing in that window gets the raw cause // back instead (session.go:305-308, 528-533). That form is unrecognisable - // as yamux at all, which is why normalizeStreamErr no longer forgives a - // bare io.EOF: for a peer that vanished, the raw cause is precisely io.EOF. - return errors.Is(err, yamux.ErrStreamReset) && err != yamux.ErrStreamReset -} - -// isMuxLocalTeardown reports whether err is yamux ending one stream at this -// side's request, which Splice mostly provokes itself: closing a stream whose -// session has already shut down normally returns ErrSessionShutdown from the -// FIN write, and a copy parked on a stream that gets closed comes back with -// ErrStreamClosed from a Write (stream.go:157-159) or the bare ErrStreamReset -// from a Read, which is what CloseRead installs (stream.go:348-349). A reset -// does not only mean that, though; the same sentinel heads the error a dying -// session hands its streams, and the remote forms belong to isMuxFailure, which -// is why that runs first. None of yamux's error types match net.ErrClosed, so -// all of this has to be recognised by shape. -func isMuxLocalTeardown(err error) bool { - // Sentinels by identity, never errors.Is: the wrapped forms belong to a - // dead session and are reported instead. ErrSessionShutdown is absent on - // purpose rather than by oversight, being a *GoAwayError carrying the - // normal code with Remote false, which the last check below covers. - if err == yamux.ErrStreamClosed || err == yamux.ErrStreamReset { - return true - } - // Remote is re-checked rather than assumed from the ordering, so that this - // predicate is true to its own name if it is ever called from anywhere - // else. - var streamErr *yamux.StreamError - if errors.As(err, &streamErr) { - return !streamErr.Remote + // as yamux at all, and is why it is left unrecognised here rather than + // guessed at: normalizeStreamErr no longer forgives a bare io.EOF, because + // for a peer that vanished the raw cause is precisely io.EOF. + if errors.Is(err, yamux.ErrStreamReset) { + return true, true } - var goAway *yamux.GoAwayError - return errors.As(err, &goAway) && !goAway.Remote && goAway.ErrorCode == normalGoAwayCode + return false, false } From b1dc6c4668142993e9f20913b6721616538ea1ac Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 15:27:10 +0000 Subject: [PATCH 054/153] feat(cluster): reach every worker through its tunnel, never its address The tunnel, the fence, the registry and the relay were all built and none of them carried a byte: every dial from the frontend still went to the address a worker registered. This is where that stops. One WorkerDialer resolves where a worker's tunnel is held, opens a stream on it locally or relays through the owning replica, and hands back a conn past both handshakes; gRPC, the file stager's HTTP client and the log-streaming WebSocket are all pointed at it. A worker's address stops being somewhere to connect to and becomes the name of which backend process a stream is for. It still appears in URLs, logs and errors, because that is what identifies the process; what it no longer decides is where the bytes go. Nothing falls back to dialling it. BackendClientFactory now has exactly one method, NewClientForNode, and returns an error where there is no way to reach the worker. The direct-dial constructor was removed rather than kept beside it, because leaving one on the interface keeps the bypass one word away from every call site that holds an address, which is all of them. The second construction path is closed too. DistributedModelStore built remote models with a nil client, and pkg/model.Model.GRPC then dialled the raw address lazily on first use - reached in production by ShutdownModel's Free and by the backend monitor's Status. Those models now carry the tunnel-backed client, and a model that cannot be given one is logged and not listed. Four conditions stay unmixable, and one path produces absence: the dialer answers ErrNoConnection only where Owner's liveness join did. A peer that will not answer, a stale ownership row, a worker's own refusal and a missing relay path are each reported as themselves. This matters because nodes ACTS on absence, and the collapse would have it reclaim the models of a worker that is connected and busy. That is not hypothetical. Writing the mutation for it exposed the bug in this change's own first draft: probeHealth returned bare false when it could not build a client, and tryWarmPath deletes the replica row on a false probe. A frontend whose dialer broke would have emptied node_models for the whole deployment while every model kept running. probeHealth now returns alive and probed separately, the reconciler gets a ProbeUnknown outcome that neither advances nor clears a failure streak, and the health monitor skips rather than counting a miss. Task 5 left the relay's open timeout at a fixed 15s and said so: no operator has the information to set it, because the number that matters is the original client's remaining budget, which is invisible on the relay side. The dialer has that budget, so it now states it in the relay request frame and the owner takes the smaller of the two. It can only shorten - a patient client must not be able to park a relay goroutine and a stream slot on a worker that stopped accepting. Zero is written as no budget at all, since on the far side the number zero is a caller with nothing left and would refuse healthy traffic. Seven mutations, each reddening a named spec: peer-unreachable as absence; the local-failure guard dropped; max instead of min on the budget; the nil-client model restored; ProbeUnknown falling through to the reaper; OwnerRow instead of Owner; probed collapsed into alive. The first budget spec passed for the wrong reason - a handshake deadline, not the relay - and was replaced by three that each assert one link, including one where the spec plays the owning replica and reads the budget out of the frame instead of inferring it from a clock. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 73 ++- core/application/startup.go | 6 + core/http/app.go | 13 +- core/http/endpoints/localai/nodes.go | 53 +- core/http/routes/nodes.go | 14 +- core/services/cluster/dialer.go | 250 ++++++++ core/services/cluster/dialer_test.go | 563 ++++++++++++++++++ core/services/cluster/relay.go | 110 +++- core/services/cluster/relay_internal_test.go | 79 ++- core/services/cluster/relay_test.go | 8 +- .../nodes/backend_client_factory_test.go | 101 ++++ core/services/nodes/distributed_store.go | 48 +- core/services/nodes/distributed_store_test.go | 69 ++- core/services/nodes/file_stager_http.go | 141 +++-- .../nodes/file_stager_verify_deadline_test.go | 2 +- .../nodes/file_transfer_server_test.go | 64 +- core/services/nodes/health.go | 19 +- core/services/nodes/health_mock_test.go | 23 + core/services/nodes/health_test.go | 40 ++ core/services/nodes/interfaces.go | 95 ++- .../nodes/local_stub_invalidator_test.go | 6 +- core/services/nodes/reconciler.go | 66 +- .../nodes/reconciler_busy_probe_test.go | 42 ++ core/services/nodes/reconciler_test.go | 2 +- .../nodes/revision_eligibility_test.go | 2 +- core/services/nodes/router.go | 80 ++- .../services/nodes/router_load_budget_test.go | 4 + .../nodes/router_load_timeout_test.go | 4 + core/services/nodes/router_reap_load_test.go | 4 + core/services/nodes/router_test.go | 4 + .../nodes/router_unreachable_worker_test.go | 70 +++ docs/content/features/distributed-mode.md | 31 + pkg/grpc/backend.go | 26 +- pkg/grpc/client.go | 14 + tests/e2e/distributed/backend_logs_test.go | 19 +- .../distributed/distributed_full_flow_test.go | 12 +- .../e2e/distributed/distributed_store_test.go | 22 +- tests/e2e/distributed/file_staging_test.go | 2 +- .../distributed/prefix_cache_routing_test.go | 4 + 39 files changed, 2007 insertions(+), 178 deletions(-) create mode 100644 core/services/cluster/dialer.go create mode 100644 core/services/cluster/dialer_test.go create mode 100644 core/services/nodes/backend_client_factory_test.go create mode 100644 core/services/nodes/router_unreachable_worker_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index f97d67cf08b1..eab8cfca5aac 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -68,6 +68,14 @@ type DistributedServices struct { // loop, which re-claims what it holds after this replica has been reaped, // and to the route that accepts a worker's dial. Tunnels *cluster.TunnelRegistry + // WorkerDialer is how anything in this process reaches a worker: locally + // when this replica holds the tunnel, and through the owning replica when + // it does not. The HTTP layer takes its WebSocket log proxy from here. + WorkerDialer *cluster.WorkerDialer + // BackendClients builds the gRPC clients for worker backend processes, over + // WorkerDialer. Exposed so the model store built in startup.go reaches + // remote models the same way every other caller does. + BackendClients nodes.BackendClientFactory shutdownOnce sync.Once } @@ -266,6 +274,22 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // peer route checks (see RegisterClusterRoutes); two different tokens here // would make every peer dial 401 with nothing naming the mismatch. peers := cluster.NewPeerPool(cfg.Distributed.InstanceID, cfg.Distributed.RegistrationToken, clusterRegistry) + // The one door to every worker. Nothing in the frontend may dial a worker's + // advertised address any more: a worker holds ONE tunnel, it lands on ONE + // replica, and this resolves which replica that is and relays through it + // when it is not this one. The three transports the frontend speaks to a + // worker (gRPC to backend processes, HTTP for file staging and logs, a + // WebSocket for live log streaming) are all pointed at it below. + workerDialer := cluster.NewWorkerDialer(tunnels, peers) + backendClients, err := nodes.NewTunnelClientFactory(cfg.Distributed.RegistrationToken, workerDialer.GRPCDialerFor) + if err != nil { + return nil, fmt.Errorf("wiring the worker backend client factory: %w", err) + } + // Bound to the http tag: the worker ignores the target for it and routes to + // its own file-transfer and log server, wherever that bound. + workerHTTPDialer := nodes.WorkerNetDialerFor(func(nodeID string) func(ctx context.Context, network, addr string) (net.Conn, error) { + return workerDialer.DialerFor(nodeID, cluster.StreamTagHTTP) + }) // Let scheduling rules be keyed by a model alias. The registry resolves a // rule's name through the config loader to find the model it governs, so an @@ -307,6 +331,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade cfg.Distributed.StaleNodeThresholdOrDefault(), routerAuthToken, !cfg.Distributed.DisablePerModelHealthCheck, + backendClients, ) // Initialize job store @@ -366,7 +391,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade return "", fmt.Errorf("node %s has no HTTP address for file transfer", nodeID) } return node.HTTPAddress, nil - }, cfg.Distributed.RegistrationToken) + }, cfg.Distributed.RegistrationToken, workerHTTPDialer) xlog.Info("File stager initialized (HTTP direct transfer)") } // Create RemoteUnloaderAdapter — needed by SmartRouter and startup.go @@ -483,6 +508,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade FileStager: fileStager, GalleriesJSON: routerGalleriesJSON, AuthToken: routerAuthToken, + ClientFactory: backendClients, DB: authDB, ConflictResolver: conflictResolver, PinnedResolver: pinnedResolver, @@ -542,6 +568,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade Unloader: remoteUnloader, Adapter: remoteUnloader, RegistrationToken: cfg.Distributed.RegistrationToken, + ClientFactory: backendClients, DB: authDB, Interval: 30 * time.Second, ScaleDownDelay: 5 * time.Minute, @@ -556,27 +583,29 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade success = true return &DistributedServices{ - Nats: natsClient, - Store: store, - Registry: registry, - Router: router, - Health: healthMon, - Reconciler: reconciler, - JobStore: jobStore, - Dispatcher: dispatcher, - AgentStore: agentStore, - AgentBridge: agentBridge, - DistStores: distStores, - FileMgr: fileMgr, - FileStager: fileStager, - ModelAdapter: modelAdapter, - Unloader: remoteUnloader, - ModelCleanup: modelCleanup, - Cluster: clusterRegistry, - Membership: membership, - PeerSessions: peerSessions, - Peers: peers, - Tunnels: tunnels, + Nats: natsClient, + Store: store, + Registry: registry, + Router: router, + Health: healthMon, + Reconciler: reconciler, + JobStore: jobStore, + Dispatcher: dispatcher, + AgentStore: agentStore, + AgentBridge: agentBridge, + DistStores: distStores, + FileMgr: fileMgr, + FileStager: fileStager, + ModelAdapter: modelAdapter, + Unloader: remoteUnloader, + ModelCleanup: modelCleanup, + Cluster: clusterRegistry, + Membership: membership, + PeerSessions: peerSessions, + Peers: peers, + Tunnels: tunnels, + WorkerDialer: workerDialer, + BackendClients: backendClients, }, nil } diff --git a/core/application/startup.go b/core/application/startup.go index abc2f4a17571..c790a1403617 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -283,9 +283,15 @@ func New(opts ...config.AppOption) (*Application, error) { // Wire ModelRouter so grpcModel() delegates to SmartRouter in distributed mode application.modelLoader.SetModelRouter(distSvc.ModelAdapter.AsModelRouter()) // Wire DistributedModelStore so shutdown/list/watchdog can find remote models + // The client factory is not optional here. Without it the store builds + // remote models with no client, and pkg/model.Model.GRPC then dials the + // worker's raw address with gRPC's own dialer, which is the direct dial + // the tunnel replaces; ShutdownModel's Free and the backend monitor's + // Status both reach it. distStore := nodes.NewDistributedModelStore( model.NewInMemoryModelStore(), distSvc.Registry, + distSvc.BackendClients, ) application.modelLoader.SetModelStore(distStore) // Drop the local stub when a model's last replica leaves the registry. diff --git a/core/http/app.go b/core/http/app.go index 40d14b7a4a9f..d1fa8ced93a2 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -1,12 +1,14 @@ package http import ( + "context" "embed" "errors" "fmt" "io/fs" "math" "mime" + "net" "net/http" "os" "path/filepath" @@ -541,15 +543,24 @@ func API(application *application.Application) (*echo.Echo, error) { distCfg := application.ApplicationConfig().Distributed var registry *nodes.NodeRegistry var remoteUnloader nodes.NodeCommandSender + // How the admin log-proxy routes reach a worker's own HTTP server. Left nil + // outside distributed mode, where there are no workers and no tunnels; the + // routes then refuse rather than dialling an address directly. + var workerHTTPDialFor nodes.WorkerNetDialerFor if d := application.Distributed(); d != nil { registry = d.Registry if d.Router != nil { remoteUnloader = d.Router.Unloader() } + if d.WorkerDialer != nil { + workerHTTPDialFor = func(nodeID string) func(ctx context.Context, network, addr string) (net.Conn, error) { + return d.WorkerDialer.DialerFor(nodeID, clustersvc.StreamTagHTTP) + } + } } natsCfg := distCfg.NatsAuthConfig() routes.RegisterNodeSelfServiceRoutes(e, registry, distCfg.RegistrationToken, distCfg.AutoApproveNodes, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret, natsCfg) - routes.RegisterNodeAdminRoutes(e, registry, remoteUnloader, application.GalleryService(), opcache, application.ApplicationConfig(), adminMiddleware, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret, application.ApplicationConfig().Distributed.RegistrationToken, natsCfg) + routes.RegisterNodeAdminRoutes(e, registry, remoteUnloader, application.GalleryService(), opcache, application.ApplicationConfig(), adminMiddleware, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret, application.ApplicationConfig().Distributed.RegistrationToken, natsCfg, workerHTTPDialFor) // Replica-to-replica peer link. Registered only in distributed mode: in // single-node mode there are no peers, and the route authenticates with the diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index 9d6886b81b05..5a2ae3573d1b 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "net/url" "sync" @@ -810,7 +811,7 @@ func DeleteModelOnNodeEndpoint(unloader nodes.NodeCommandSender, registry *nodes // NodeBackendLogsListEndpoint proxies a request to a worker node's /v1/backend-logs // endpoint to list model IDs that have backend logs. -func NodeBackendLogsListEndpoint(registry *nodes.NodeRegistry, registrationToken string) echo.HandlerFunc { +func NodeBackendLogsListEndpoint(registry *nodes.NodeRegistry, registrationToken string, dialFor nodes.WorkerNetDialerFor) echo.HandlerFunc { return func(c echo.Context) error { ctx := c.Request().Context() nodeID := c.Param("id") @@ -823,7 +824,7 @@ func NodeBackendLogsListEndpoint(registry *nodes.NodeRegistry, registrationToken return c.JSON(http.StatusBadGateway, nodeError(http.StatusBadGateway, "node has no HTTP address")) } - resp, err := proxyHTTPToWorker(node.HTTPAddress, "/v1/backend-logs", registrationToken) + resp, err := proxyHTTPToWorker(ctx, dialFor, nodeID, node.HTTPAddress, "/v1/backend-logs", registrationToken) if err != nil { return c.JSON(http.StatusBadGateway, nodeError(http.StatusBadGateway, fmt.Sprintf("failed to reach worker: %v", err))) } @@ -838,7 +839,7 @@ func NodeBackendLogsListEndpoint(registry *nodes.NodeRegistry, registrationToken // NodeBackendLogsLinesEndpoint proxies a request to a worker node's // /v1/backend-logs/{modelId} endpoint to get buffered log lines. -func NodeBackendLogsLinesEndpoint(registry *nodes.NodeRegistry, registrationToken string) echo.HandlerFunc { +func NodeBackendLogsLinesEndpoint(registry *nodes.NodeRegistry, registrationToken string, dialFor nodes.WorkerNetDialerFor) echo.HandlerFunc { return func(c echo.Context) error { ctx := c.Request().Context() nodeID := c.Param("id") @@ -854,7 +855,7 @@ func NodeBackendLogsLinesEndpoint(registry *nodes.NodeRegistry, registrationToke } path := "/v1/backend-logs/" + url.PathEscape(modelID) - resp, err := proxyHTTPToWorker(node.HTTPAddress, path, registrationToken) + resp, err := proxyHTTPToWorker(ctx, dialFor, nodeID, node.HTTPAddress, path, registrationToken) if err != nil { return c.JSON(http.StatusBadGateway, nodeError(http.StatusBadGateway, fmt.Sprintf("failed to reach worker: %v", err))) } @@ -869,7 +870,7 @@ func NodeBackendLogsLinesEndpoint(registry *nodes.NodeRegistry, registrationToke // NodeBackendLogsWSEndpoint proxies a WebSocket connection to a worker node's // /v1/backend-logs/{modelId}/ws endpoint for real-time log streaming. -func NodeBackendLogsWSEndpoint(registry *nodes.NodeRegistry, registrationToken string) echo.HandlerFunc { +func NodeBackendLogsWSEndpoint(registry *nodes.NodeRegistry, registrationToken string, dialFor nodes.WorkerNetDialerFor) echo.HandlerFunc { browserUpgrader := websocket.Upgrader{ CheckOrigin: func(r *http.Request) bool { origin := r.Header.Get("Origin") @@ -901,15 +902,28 @@ func NodeBackendLogsWSEndpoint(registry *nodes.NodeRegistry, registrationToken s return err } - // Dial the worker WebSocket + // Dial the worker WebSocket over that worker's tunnel. The URL still + // names the worker's registered address, for the Host header; the + // NetDialContext below is what decides where the connection goes. A + // missing dialer is a failure, not a direct dial: see + // nodes.ErrNoWorkerDialer. workerURL := fmt.Sprintf("ws://%s/v1/backend-logs/%s/ws", node.HTTPAddress, url.PathEscape(modelID)) workerHeaders := http.Header{} if registrationToken != "" { workerHeaders.Set("Authorization", "Bearer "+registrationToken) } - workerDialer := websocket.Dialer{HandshakeTimeout: 10 * time.Second} - workerWS, _, err := workerDialer.Dial(workerURL, workerHeaders) + var workerDial func(ctx context.Context, network, addr string) (net.Conn, error) + if dialFor != nil { + workerDial = dialFor(nodeID) + } + if workerDial == nil { + return c.JSON(http.StatusBadGateway, nodeError(http.StatusBadGateway, + fmt.Sprintf("cannot reach node %s: %v", nodeID, nodes.ErrNoWorkerDialer))) + } + + workerDialer := websocket.Dialer{HandshakeTimeout: 10 * time.Second, NetDialContext: workerDial} + workerWS, _, err := workerDialer.DialContext(ctx, workerURL, workerHeaders) if err != nil { browserWS.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseInternalServerErr, "failed to connect to worker")) @@ -1378,10 +1392,25 @@ func DeleteSchedulingEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc { } } -// proxyHTTPToWorker makes a GET request to a worker's HTTP server with bearer token auth. -func proxyHTTPToWorker(httpAddress, path, token string) (*http.Response, error) { +// proxyHTTPToWorker makes a GET request to a worker's HTTP server with bearer +// token auth, over that worker's tunnel. +// +// The URL still names the worker's registered HTTP address, because that is +// what the Host header and every error message should say; what it no longer +// decides is where the bytes go. dialFor supplies the transport, and a nil one +// is an error rather than a fall back to connecting to httpAddress: a worker +// behind NAT has no address to connect to, and a direct dial is the bypass this +// whole change removes. +func proxyHTTPToWorker(ctx context.Context, dialFor nodes.WorkerNetDialerFor, nodeID, httpAddress, path, token string) (*http.Response, error) { + if dialFor == nil { + return nil, fmt.Errorf("reaching node %s: %w", nodeID, nodes.ErrNoWorkerDialer) + } + dial := dialFor(nodeID) + if dial == nil { + return nil, fmt.Errorf("reaching node %s: %w", nodeID, nodes.ErrNoWorkerDialer) + } reqURL := fmt.Sprintf("http://%s%s", httpAddress, path) - req, err := http.NewRequest("GET", reqURL, nil) + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { return nil, err } @@ -1389,6 +1418,6 @@ func proxyHTTPToWorker(httpAddress, path, token string) (*http.Response, error) req.Header.Set("Authorization", "Bearer "+token) } - client := httpclient.NewWithTimeout(15 * time.Second) + client := httpclient.NewWithTimeout(15*time.Second, httpclient.WithTransport(&http.Transport{DialContext: dial})) return client.Do(req) } diff --git a/core/http/routes/nodes.go b/core/http/routes/nodes.go index 053d6c19cf30..a511c6b7c1fc 100644 --- a/core/http/routes/nodes.go +++ b/core/http/routes/nodes.go @@ -61,7 +61,13 @@ func RegisterNodeSelfServiceRoutes(e *echo.Echo, registry *nodes.NodeRegistry, r // backend install path (POST /:id/backends/install). That handler enqueues a // ManagementOp on the gallery channel rather than blocking on a NATS reply, so // the browser gets HTTP 202 + jobID immediately instead of waiting up to 3 minutes. -func RegisterNodeAdminRoutes(e *echo.Echo, registry *nodes.NodeRegistry, unloader nodes.NodeCommandSender, galleryService *galleryop.GalleryService, opcache *galleryop.OpCache, appConfig *config.ApplicationConfig, adminMw echo.MiddlewareFunc, authDB *gorm.DB, hmacSecret string, registrationToken string, natsCfg natsauth.Config) { +// +// workerDialFor is how the log-proxy routes reach a worker's own HTTP server: +// over the tunnel that worker holds, never by connecting to the address it +// registered. It is nil outside distributed mode, and those two routes then +// answer 502 rather than dialling, because a worker with no tunnel has nothing +// for them to proxy to. +func RegisterNodeAdminRoutes(e *echo.Echo, registry *nodes.NodeRegistry, unloader nodes.NodeCommandSender, galleryService *galleryop.GalleryService, opcache *galleryop.OpCache, appConfig *config.ApplicationConfig, adminMw echo.MiddlewareFunc, authDB *gorm.DB, hmacSecret string, registrationToken string, natsCfg natsauth.Config, workerDialFor nodes.WorkerNetDialerFor) { if registry == nil { return } @@ -101,8 +107,8 @@ func RegisterNodeAdminRoutes(e *echo.Echo, registry *nodes.NodeRegistry, unloade admin.POST("/:id/models/delete", localai.DeleteModelOnNodeEndpoint(unloader, registry)) // Backend log streaming (proxied from worker HTTP server) - admin.GET("/:id/backend-logs", localai.NodeBackendLogsListEndpoint(registry, registrationToken)) - admin.GET("/:id/backend-logs/:modelId", localai.NodeBackendLogsLinesEndpoint(registry, registrationToken)) + admin.GET("/:id/backend-logs", localai.NodeBackendLogsListEndpoint(registry, registrationToken, workerDialFor)) + admin.GET("/:id/backend-logs/:modelId", localai.NodeBackendLogsLinesEndpoint(registry, registrationToken, workerDialFor)) // Label management admin.GET("/:id/labels", localai.GetNodeLabelsEndpoint(registry)) @@ -123,7 +129,7 @@ func RegisterNodeAdminRoutes(e *echo.Echo, registry *nodes.NodeRegistry, unloade admin.DELETE("/:id/vram-budget", localai.ResetVRAMBudgetEndpoint(registry)) // WebSocket proxy for real-time log streaming from workers - e.GET("/ws/nodes/:id/backend-logs/:modelId", localai.NodeBackendLogsWSEndpoint(registry, registrationToken), readyMw, adminMw) + e.GET("/ws/nodes/:id/backend-logs/:modelId", localai.NodeBackendLogsWSEndpoint(registry, registrationToken, workerDialFor), readyMw, adminMw) } // nodeTokenAuth validates the registration token for node self-service endpoints. diff --git a/core/services/cluster/dialer.go b/core/services/cluster/dialer.go new file mode 100644 index 000000000000..b4a04551a973 --- /dev/null +++ b/core/services/cluster/dialer.go @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: MIT + +package cluster + +import ( + "context" + "errors" + "fmt" + "net" + "time" + + "github.com/mudler/xlog" +) + +// PeerOpener opens a stream to another frontend replica. *PeerPool is the +// production implementation. +// +// It is an interface so a spec can decide what a peer does without standing up +// a second frontend. Note that a TYPED nil (a (*PeerPool)(nil) stored here) +// would not compare equal to nil and would be called; the dialer's nil check is +// for the untyped nil a deployment with no peer mesh passes. +type PeerOpener interface { + Open(ctx context.Context, peerID string) (net.Conn, error) +} + +// ErrNoRelayPath reports that a worker's tunnel is held by ANOTHER replica and +// this one has no way to reach that replica. +// +// It is its own condition, kept out of the four this phase refuses to collapse. +// Not ErrNotOwner, because that tells a caller to resolve the owner again and +// the answer would not change. Not ErrPeerUnreachable, because no peer was +// dialled and none refused; saying otherwise would blame a replica that is +// probably fine. And above all not ErrNoConnection: the worker IS connected, +// and core/services/nodes reclaims the models of a worker it believes absent. +var ErrNoRelayPath = errors.New("cluster: this replica cannot relay to the owner of that worker") + +// dialHandshakeTimeout bounds the request/reply exchange that opens every +// stream, when the caller stated no deadline of its own. +// +// It is a backstop and not a budget. A worker or a relay that accepted a stream +// and then said nothing would otherwise park the caller until the session's +// keepalive killed it, which is 30 seconds on the yamux default and longer if a +// deployment ever raises it. Where the caller DOES carry a deadline, that +// deadline is used instead whenever it is the shorter of the two, for the same +// reason the relay takes the smaller of its ceiling and the stated budget. +const dialHandshakeTimeout = 15 * time.Second + +// WorkerDialer opens connections to a worker through its tunnel, wherever in +// the deployment that tunnel happens to be held. +// +// It is the single door: a worker holds ONE tunnel, it lands on ONE frontend +// replica, and nothing else in the frontend may dial a worker's advertised +// address. Every protocol the frontend speaks to a worker (gRPC to a backend +// process, HTTP for file staging and logs, a WebSocket for log streaming) goes +// through the functions below, because a worker behind NAT has no address to +// dial and a worker that has one must not be reached that way either: a direct +// dial works in a single-replica test and fails in production. +type WorkerDialer struct { + // Both are read off the tunnel registry rather than passed separately, so + // the identity this dialer compares owners against is by construction the + // identity the registry CLAIMS as. Two ids here would make this replica + // relay to itself for every worker it holds. + tunnels *TunnelRegistry + peers PeerOpener +} + +// NewWorkerDialer returns the dialer for the tunnels this replica holds and the +// peer links it can relay over. A nil peers means this replica cannot relay, +// which is reported as ErrNoRelayPath rather than as a worker being absent. +func NewWorkerDialer(tunnels *TunnelRegistry, peers PeerOpener) *WorkerDialer { + return &WorkerDialer{tunnels: tunnels, peers: peers} +} + +// Dial opens one stream to a local service on a worker: tag says which service +// (see StreamTagGRPC and StreamTagHTTP) and target which instance of it. +// +// The returned conn is past both handshakes and carries the tunnelled protocol +// and nothing else, with no deadline armed on it: what follows may be an +// inference that is quiet for minutes, and a deadline left over from the +// handshake would abort it. +// +// The errors are kept apart on purpose and a caller may act on them +// differently. ErrNoConnection means no live replica holds this worker's +// tunnel, which is the one answer that means the worker is absent. ErrNotOwner +// means the routing was stale and re-resolving may find it. ErrPeerUnreachable +// means a replica would not answer, ErrNoRelayPath that none could be dialled, +// and the tunnelproto sentinels that the worker itself refused. Nothing here +// ever converts one of the others into absence. +func (d *WorkerDialer) Dial(ctx context.Context, nodeID, tag, target string) (net.Conn, error) { + stream, err := d.tunnels.Open(ctx, nodeID) + if err == nil { + return d.handshake(ctx, stream, nodeID, tag, target) + } + if !errors.Is(err, ErrNotOwner) { + // The tunnel is held HERE and its session would not carry a stream. + // Reported as itself: answering ErrNotOwner would send the caller to + // resolve an owner that is this same replica, and answering absence + // would tell a scheduler to reclaim a worker that is attached. + return nil, err + } + return d.relay(ctx, nodeID, tag, target) +} + +// DialerFor returns a net.Dialer-shaped function bound to one worker and one +// local service on it. +// +// This is the shape http.Transport.DialContext and websocket.Dialer's +// NetDialContext want. The NETWORK is ignored and the ADDRESS becomes the +// stream's target, which is what makes an http.Client or a WebSocket dialler +// built on it reach the worker without either of them knowing a tunnel exists: +// the URL still names the worker's registered address, and that address travels +// as the target rather than to a socket. What the worker does with it is the +// worker's decision (the grpc tag takes the port and dials its own loopback, +// the http tag ignores it entirely), which is the property that stops a +// frontend from steering a worker's dial. +func (d *WorkerDialer) DialerFor(nodeID, tag string) func(ctx context.Context, network, addr string) (net.Conn, error) { + return func(ctx context.Context, _, addr string) (net.Conn, error) { + return d.Dial(ctx, nodeID, tag, addr) + } +} + +// GRPCDialerFor returns a grpc.WithContextDialer-shaped function bound to one +// worker's backend processes. gRPC's dialer takes no network argument, which is +// why this is not DialerFor's shape. +func (d *WorkerDialer) GRPCDialerFor(nodeID string) func(ctx context.Context, addr string) (net.Conn, error) { + return func(ctx context.Context, addr string) (net.Conn, error) { + return d.Dial(ctx, nodeID, StreamTagGRPC, addr) + } +} + +// relay opens the stream through the replica that holds the worker's tunnel. +func (d *WorkerDialer) relay(ctx context.Context, nodeID, tag, target string) (net.Conn, error) { + // Owner, never OwnerRow: the row outlives its owner by up to a liveness + // window plus a heartbeat, and dialling what the unjoined read returns + // means dialling a process that is gone and reporting the worker as + // unreachable rather than as absent. The join is what makes a dead owner + // come back as ErrNoConnection here. + owner, _, err := d.tunnels.reg.Owner(ctx, nodeID) + if err != nil { + // ErrNoConnection and database failures both pass through as + // themselves. This is the ONLY path by which this function can produce + // an absence error, and it produces it only when Owner did. + return nil, err + } + if owner == d.tunnels.selfID { + // The table names this replica and the registry above said the tunnel + // is not held here, so the attachment went away between the claim and + // now. Relaying would send the request into this same process, which + // would resolve the same owner and relay again. Reported as the routing + // fact so the caller re-resolves, which terminates: the row is either + // re-claimed by whoever holds the worker now, or swept. + return nil, fmt.Errorf("opening a stream to node %q: the connection row names this replica, which no longer holds the tunnel: %w", nodeID, ErrNotOwner) + } + if d.peers == nil { + return nil, fmt.Errorf("opening a stream to node %q held by replica %q: %w", nodeID, owner, ErrNoRelayPath) + } + + stream, err := d.peers.Open(ctx, owner) + if err != nil { + // Whatever the pool said, unchanged in its unwrap chain: + // ErrPeerUnreachable, ErrInstanceNotFound for an owner swept since the + // lookup above, or ErrPoolClosed while this process shuts down. None of + // them is a statement about the WORKER, and none is converted into one. + return nil, fmt.Errorf("opening a stream to node %q through replica %q: %w", nodeID, owner, err) + } + + // The caller's remaining time, stated so the owning replica can bound its + // own open by it. Only this side knows it; see relayOpenTimeout for what + // the owner falls back to without it. + if err := WriteRelayRequest(stream, nodeID, remainingBudget(ctx)); err != nil { + _ = stream.Close() + return nil, fmt.Errorf("naming node %q on a stream to replica %q: %w", nodeID, owner, err) + } + if err := ReadRelayReply(stream); err != nil { + // ReadRelayReply already separates a refusal (ErrNotOwner, + // ErrRelayUnavailable, ErrRelayRequestInvalid) from a failure to read + // one, and neither kind is ever an absence error. + _ = stream.Close() + return nil, fmt.Errorf("relaying to node %q through replica %q: %w", nodeID, owner, err) + } + return d.handshake(ctx, stream, nodeID, tag, target) +} + +// handshake names the worker-side service on a stream and waits for the +// worker's answer, leaving the stream ready for the tunnelled protocol. +// +// It owns closing the stream on every failure. A stream left open after a +// failed handshake holds a yamux slot on the tunnel for the life of the +// session, and a frontend that retries would exhaust the worker's stream +// budget rather than the worker's patience. +func (d *WorkerDialer) handshake(ctx context.Context, stream net.Conn, nodeID, tag, target string) (net.Conn, error) { + if deadline, ok := handshakeDeadline(ctx); ok { + if err := stream.SetDeadline(deadline); err != nil { + _ = stream.Close() + return nil, fmt.Errorf("arming the handshake deadline for node %q: %w", nodeID, err) + } + } + + if err := WriteStreamRequest(stream, tag, target); err != nil { + _ = stream.Close() + return nil, fmt.Errorf("asking node %q for %q on %q: %w", nodeID, tag, target, err) + } + if err := ReadStreamReply(stream); err != nil { + _ = stream.Close() + return nil, fmt.Errorf("opening %q on node %q: %w", tag, nodeID, err) + } + + // Cleared unconditionally rather than only when one was armed, so that this + // stays true of the stream whatever the caller's context carried. What + // follows is the caller's protocol, and its length is the caller's + // business: a request may sit quiet for minutes between tokens, and the + // handshake's deadline would end it. The session's keepalive is what still + // bounds a peer that has stopped answering. + if err := stream.SetDeadline(time.Time{}); err != nil { + _ = stream.Close() + return nil, fmt.Errorf("clearing the handshake deadline for node %q: %w", nodeID, err) + } + xlog.Debug("opened a tunnelled stream to a worker", "node", nodeID, "tag", tag, "target", target) + return stream, nil +} + +// handshakeDeadline is when the handshake must be done by: the caller's own +// deadline when it has one and it is the sooner, and the backstop otherwise. +func handshakeDeadline(ctx context.Context) (time.Time, bool) { + backstop := time.Now().Add(dialHandshakeTimeout) + deadline, ok := ctx.Deadline() + if !ok || deadline.After(backstop) { + return backstop, true + } + return deadline, true +} + +// remainingBudget is how long the caller is still willing to wait, or zero when +// it did not say. +// +// Zero rather than a negative number for an expired context: the frame writer +// treats zero as "not stated", and a caller that has already run out is about +// to fail on its own context anyway. Stating a negative budget would instead +// make the owning replica refuse, which is the same outcome by a longer route. +func remainingBudget(ctx context.Context) time.Duration { + deadline, ok := ctx.Deadline() + if !ok { + return 0 + } + remaining := time.Until(deadline) + if remaining <= 0 { + return 0 + } + return remaining +} diff --git a/core/services/cluster/dialer_test.go b/core/services/cluster/dialer_test.go new file mode 100644 index 000000000000..ffcfe3775e13 --- /dev/null +++ b/core/services/cluster/dialer_test.go @@ -0,0 +1,563 @@ +// SPDX-License-Identifier: MIT + +package cluster_test + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "time" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" + + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +// stubPeers is a PeerOpener that hands back streams on one session, or one +// error. It stands in for the replica-to-replica link so a spec can decide what +// a peer does without standing up a second frontend. +type stubPeers struct { + sess *yamux.Session + err error +} + +func (s *stubPeers) Open(ctx context.Context, _ string) (net.Conn, error) { + if s.err != nil { + return nil, s.err + } + return s.sess.OpenStream(ctx) +} + +// dialResult carries what a Dial produced, so a spec can wait on a channel +// rather than on a clock. +type dialResult struct { + conn net.Conn + err error +} + +// dialAsync runs one Dial on its own goroutine. Dial talks to a worker that a +// spec drives by hand, so the spec has to be free to answer while the dial is +// still in flight. +func dialAsync(d *cluster.WorkerDialer, ctx context.Context, nodeID, tag, target string) chan dialResult { + done := make(chan dialResult, 1) + go func() { + defer GinkgoRecover() + conn, err := d.Dial(ctx, nodeID, tag, target) + done <- dialResult{conn: conn, err: err} + }() + return done +} + +// relayRequest is what an owning replica saw in the frame that opened a +// relayed stream. +type relayRequest struct { + nodeID string + budget time.Duration + err error +} + +// servedRequest is what a worker saw on a stream opened through the tunnel. +type servedRequest struct { + tag string + target string + stream net.Conn + err error +} + +// serveOneStream accepts one stream on the worker's half, reads the tunnel +// request frame and accepts it, then echoes the four bytes it is sent. It is +// how a spec proves the dial produced a stream that carries the tunnelled +// protocol, and that the frame the worker sees is the one the dialer wrote. +func serveOneStream(worker *yamux.Session) chan servedRequest { + seen := make(chan servedRequest, 1) + go func() { + defer GinkgoRecover() + stream, err := worker.AcceptStream() + if err != nil { + seen <- servedRequest{err: err} + return + } + tag, target, err := cluster.ReadStreamRequest(stream) + if err != nil { + seen <- servedRequest{err: err} + return + } + if err := cluster.WriteStreamAccepted(stream); err != nil { + seen <- servedRequest{err: err} + return + } + seen <- servedRequest{tag: tag, target: target, stream: stream} + buf := make([]byte, 4) + if _, err := io.ReadFull(stream, buf); err != nil { + return + } + _, _ = stream.Write(buf) + }() + return seen +} + +// refuseOneStream accepts one stream and refuses it with reason. +func refuseOneStream(worker *yamux.Session, reason error) { + go func() { + defer GinkgoRecover() + stream, err := worker.AcceptStream() + if err != nil { + return + } + defer func() { _ = stream.Close() }() + if _, _, err := cluster.ReadStreamRequest(stream); err != nil { + return + } + _ = cluster.WriteStreamRefusal(stream, reason) + }() +} + +// expectNotAbsence asserts an error is none of the sentinels a caller is +// entitled to act on as "this worker has gone away". +// +// It is the assertion this whole phase turns on. core/services/nodes reclaims a +// worker's models when it concludes the worker is absent, so an unreachable +// peer or a refusing worker arriving as absence would evict healthy work. +func expectNotAbsence(err error) { + GinkgoHelper() + Expect(err).To(HaveOccurred()) + Expect(err).ToNot(MatchError(cluster.ErrNoConnection)) + Expect(err).ToNot(MatchError(cluster.ErrInstanceNotFound)) +} + +var _ = Describe("The worker dialer", func() { + var ( + db *gorm.DB + reg *cluster.Registry + mine *cluster.TunnelRegistry + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + db = testutil.SetupTestDB() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg = cluster.NewRegistry(db) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + mine = cluster.NewTunnelRegistry(reg, "me") + }) + + // ownerRelay stands up a SECOND replica that holds w1's tunnel and relays + // for it, and returns the peer opener this replica reaches it through plus + // the worker's own half of the tunnel. + ownerRelay := func(nodeID string) (*stubPeers, *yamux.Session) { + GinkgoHelper() + Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1")).To(Succeed()) + ownerTunnels := cluster.NewTunnelRegistry(reg, "owner") + frontend, worker := workerTunnel() + _, err := ownerTunnels.Attach(ctx, nodeID, frontend) + Expect(err).ToNot(HaveOccurred()) + + store := cluster.NewSessionStore(cluster.NewRelay(ownerTunnels).Stream) + DeferCleanup(store.CloseAll) + dialling, accepted := yamuxPair() + store.Accept("me", accepted) + return &stubPeers{sess: dialling}, worker + } + + Describe("when this replica holds the tunnel", func() { + It("opens a stream straight down it, naming the service the caller asked for", func() { + frontend, worker := workerTunnel() + _, err := mine.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + seen := serveOneStream(worker) + + d := cluster.NewWorkerDialer(mine, nil) + result := dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000") + + var req servedRequest + Eventually(seen, "10s").Should(Receive(&req)) + Expect(req.err).ToNot(HaveOccurred()) + Expect(req.tag).To(Equal(cluster.StreamTagGRPC)) + // The TARGET is what tells the worker which backend process the + // stream is for. A dialer that dropped it would send every request + // to whichever port the worker guessed. + Expect(req.target).To(Equal("127.0.0.1:41000")) + + var out dialResult + Eventually(result, "10s").Should(Receive(&out)) + Expect(out.err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = out.conn.Close() }) + + // Bytes, not a handle. A dial that returns a stream the tunnelled + // protocol cannot use is worse than one that fails. + _, err = out.conn.Write([]byte("ping")) + Expect(err).ToNot(HaveOccurred()) + echoed := make([]byte, 4) + Eventually(readInto(out.conn, echoed), "10s").Should(Receive(BeNil())) + Expect(string(echoed)).To(Equal("ping")) + }) + + It("leaves no read deadline armed on the stream it hands back", func() { + // The handshake is bounded; the request that follows it is the + // caller's business and may be a generation that is quiet for + // minutes. A deadline left armed here would abort it. + frontend, worker := workerTunnel() + _, err := mine.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + seen := serveOneStream(worker) + + deadlined, cancel := context.WithTimeout(ctx, 300*time.Millisecond) + defer cancel() + d := cluster.NewWorkerDialer(mine, nil) + result := dialAsync(d, deadlined, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000") + Eventually(seen, "10s").Should(Receive()) + + var out dialResult + Eventually(result, "10s").Should(Receive(&out)) + Expect(out.err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = out.conn.Close() }) + + // The dial context's deadline has now passed. A stream still + // carrying it would fail this write and this read. + Eventually(func() error { + _, err := out.conn.Write([]byte("ping")) + return err + }, "10s").Should(Succeed()) + echoed := make([]byte, 4) + Eventually(readInto(out.conn, echoed), "10s").Should(Receive(BeNil())) + Expect(string(echoed)).To(Equal("ping")) + }) + + It("reports a worker's refusal as the worker's refusal, never as absence", func() { + frontend, worker := workerTunnel() + _, err := mine.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + refuseOneStream(worker, cluster.ErrStreamTagUnknown) + + d := cluster.NewWorkerDialer(mine, nil) + var out dialResult + Eventually(dialAsync(d, ctx, "w1", "nonsense", ""), "10s").Should(Receive(&out)) + Expect(out.err).To(MatchError(cluster.ErrStreamTagUnknown)) + // A refusal is PROOF the worker is connected and answered. + expectNotAbsence(out.err) + }) + + It("reports a broken tunnel held here as itself, not as a routing fact", func() { + // ErrNotOwner tells a caller to look for the worker elsewhere. For + // a tunnel held right here that sends it back to this replica, and + // the loop is only broken by the request failing anyway. + frontend, worker := workerTunnel() + _, err := mine.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + Expect(worker.Close()).To(Succeed()) + + d := cluster.NewWorkerDialer(mine, nil) + var out dialResult + Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) + Expect(out.err).To(HaveOccurred()) + Expect(out.err).ToNot(MatchError(cluster.ErrNotOwner)) + expectNotAbsence(out.err) + }) + }) + + Describe("when another replica holds the tunnel", func() { + It("relays through the owner and carries bytes to the worker", func() { + peers, worker := ownerRelay("w1") + seen := serveOneStream(worker) + + d := cluster.NewWorkerDialer(mine, peers) + result := dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000") + + var req servedRequest + Eventually(seen, "10s").Should(Receive(&req)) + Expect(req.err).ToNot(HaveOccurred()) + // The relay consumed its own frame and forwarded nothing of it, so + // the worker sees only the tunnel's request. + Expect(req.tag).To(Equal(cluster.StreamTagGRPC)) + Expect(req.target).To(Equal("127.0.0.1:41000")) + + var out dialResult + Eventually(result, "10s").Should(Receive(&out)) + Expect(out.err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = out.conn.Close() }) + + _, err := out.conn.Write([]byte("ping")) + Expect(err).ToNot(HaveOccurred()) + echoed := make([]byte, 4) + Eventually(readInto(out.conn, echoed), "10s").Should(Receive(BeNil())) + Expect(string(echoed)).To(Equal("ping")) + }) + + It("states the caller's remaining budget in the relay request", func() { + // The owning replica's own open bound is a backstop nobody can set + // correctly: the number that matters is how long the ORIGINAL + // client will wait, and this replica is the only one that holds it. + // This spec plays the owner by hand so it can read the frame rather + // than infer it from a timing. + Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "owner") + Expect(err).ToNot(HaveOccurred()) + dialling, ownerSide := yamuxPair() + DeferCleanup(func() { _ = ownerSide.Close() }) + + requests := make(chan relayRequest, 1) + go func() { + defer GinkgoRecover() + stream, err := ownerSide.AcceptStream() + if err != nil { + return + } + defer func() { _ = stream.Close() }() + nodeID, budget, err := cluster.ReadRelayRequest(stream) + requests <- relayRequest{nodeID: nodeID, budget: budget, err: err} + }() + + budgeted, cancel := context.WithTimeout(ctx, 4*time.Second) + defer cancel() + d := cluster.NewWorkerDialer(mine, &stubPeers{sess: dialling}) + dialAsync(d, budgeted, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000") + + var req relayRequest + Eventually(requests, "10s").Should(Receive(&req)) + Expect(req.err).ToNot(HaveOccurred()) + Expect(req.nodeID).To(Equal("w1")) + // Whatever is left of the four seconds, and nothing invented: a + // dialer that stated its own constant would satisfy neither bound. + Expect(req.budget).To(BeNumerically(">", 2*time.Second)) + Expect(req.budget).To(BeNumerically("<=", 4*time.Second)) + }) + + It("states no budget at all for a caller that set no deadline", func() { + // Zero on the wire would be read by the owner as a caller with + // nothing left, and it would refuse traffic that is perfectly + // healthy. "Not stated" has to stay distinguishable from "expired". + Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "owner") + Expect(err).ToNot(HaveOccurred()) + dialling, ownerSide := yamuxPair() + DeferCleanup(func() { _ = ownerSide.Close() }) + + requests := make(chan relayRequest, 1) + go func() { + defer GinkgoRecover() + stream, err := ownerSide.AcceptStream() + if err != nil { + return + } + defer func() { _ = stream.Close() }() + nodeID, budget, err := cluster.ReadRelayRequest(stream) + requests <- relayRequest{nodeID: nodeID, budget: budget, err: err} + }() + + d := cluster.NewWorkerDialer(mine, &stubPeers{sess: dialling}) + dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000") + + var req relayRequest + Eventually(requests, "10s").Should(Receive(&req)) + Expect(req.err).ToNot(HaveOccurred()) + Expect(req.budget).To(BeZero()) + }) + + It("reports an unreachable peer as unreachable, NEVER as absence", func() { + // The catastrophe this phase exists to prevent. A scheduler ACTS on + // absence: told a connected worker is gone, it reclaims every model + // the worker is running. + Expect(reg.Register(ctx, "owner", "127.0.0.1:1", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "owner") + Expect(err).ToNot(HaveOccurred()) + + pool := cluster.NewPeerPool("me", "tok", reg) + DeferCleanup(pool.Close) + d := cluster.NewWorkerDialer(mine, pool) + + var out dialResult + Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "20s").Should(Receive(&out)) + Expect(out.err).To(MatchError(cluster.ErrPeerUnreachable)) + expectNotAbsence(out.err) + }) + + It("passes a stale ownership refusal back as the routing fact", func() { + // The owner's table row survives a tunnel that has gone. The relay + // answers ErrNotOwner, and only that answer tells this replica to + // resolve the owner again rather than give up on the worker. + Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "owner") + Expect(err).ToNot(HaveOccurred()) + ownerTunnels := cluster.NewTunnelRegistry(reg, "owner") + store := cluster.NewSessionStore(cluster.NewRelay(ownerTunnels).Stream) + DeferCleanup(store.CloseAll) + dialling, accepted := yamuxPair() + store.Accept("me", accepted) + + d := cluster.NewWorkerDialer(mine, &stubPeers{sess: dialling}) + var out dialResult + Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) + Expect(out.err).To(MatchError(cluster.ErrNotOwner)) + expectNotAbsence(out.err) + }) + + It("refuses rather than relaying to itself when the table names this replica", func() { + // The row says this replica owns the tunnel and the registry says + // it does not hold it. Relaying would send the request to this same + // process, which would resolve the same owner and relay again. + _, err := reg.Claim(ctx, "w1", "me") + Expect(err).ToNot(HaveOccurred()) + + d := cluster.NewWorkerDialer(mine, &stubPeers{err: errors.New("no peer should be dialled")}) + var out dialResult + Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) + Expect(out.err).To(MatchError(cluster.ErrNotOwner)) + expectNotAbsence(out.err) + }) + + It("reports having no way to relay as its own condition", func() { + Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "owner") + Expect(err).ToNot(HaveOccurred()) + + d := cluster.NewWorkerDialer(mine, nil) + var out dialResult + Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) + Expect(out.err).To(MatchError(cluster.ErrNoRelayPath)) + Expect(out.err).ToNot(MatchError(cluster.ErrNotOwner)) + Expect(out.err).ToNot(MatchError(cluster.ErrPeerUnreachable)) + expectNotAbsence(out.err) + }) + }) + + Describe("when no live replica holds the tunnel", func() { + It("reports absence when the worker has no connection row at all", func() { + d := cluster.NewWorkerDialer(mine, &stubPeers{err: errors.New("no peer should be dialled")}) + var out dialResult + Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) + Expect(out.err).To(MatchError(cluster.ErrNoConnection)) + }) + + It("reports absence when the row's owner has stopped heartbeating", func() { + // End to end over the join Owner does: the row is there, the owner + // is not. A dialer built on the unjoined read would dial a corpse + // and report the worker as unreachable rather than as absent. + Expect(reg.Register(ctx, "ghost", "10.0.0.9:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "ghost") + Expect(err).ToNot(HaveOccurred()) + Expect(db.Exec( + `UPDATE instances SET last_seen = now() - make_interval(secs => ?) WHERE id = ?`, + cluster.InstanceLiveness.Seconds()*4, "ghost").Error).To(Succeed()) + + owner, _, err := reg.OwnerRow(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("ghost")) + + d := cluster.NewWorkerDialer(mine, &stubPeers{err: errors.New("no peer should be dialled")}) + var out dialResult + Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) + Expect(out.err).To(MatchError(cluster.ErrNoConnection)) + }) + }) + + Describe("the dialer functions it hands to the transports", func() { + It("binds one node and one tag, and passes the address through as the target", func() { + frontend, worker := workerTunnel() + _, err := mine.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + seen := serveOneStream(worker) + + d := cluster.NewWorkerDialer(mine, nil) + dial := d.DialerFor("w1", cluster.StreamTagHTTP) + done := make(chan dialResult, 1) + go func() { + defer GinkgoRecover() + conn, err := dial(ctx, "tcp", "10.0.0.3:9090") + done <- dialResult{conn: conn, err: err} + }() + + var req servedRequest + Eventually(seen, "10s").Should(Receive(&req)) + Expect(req.err).ToNot(HaveOccurred()) + Expect(req.tag).To(Equal(cluster.StreamTagHTTP)) + Expect(req.target).To(Equal("10.0.0.3:9090")) + + var out dialResult + Eventually(done, "10s").Should(Receive(&out)) + Expect(out.err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = out.conn.Close() }) + }) + + It("gives gRPC a dialer fixed on the grpc tag", func() { + frontend, worker := workerTunnel() + _, err := mine.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + seen := serveOneStream(worker) + + d := cluster.NewWorkerDialer(mine, nil) + dial := d.GRPCDialerFor("w1") + done := make(chan dialResult, 1) + go func() { + defer GinkgoRecover() + conn, err := dial(ctx, "127.0.0.1:41000") + done <- dialResult{conn: conn, err: err} + }() + + var req servedRequest + Eventually(seen, "10s").Should(Receive(&req)) + Expect(req.err).ToNot(HaveOccurred()) + Expect(req.tag).To(Equal(cluster.StreamTagGRPC)) + Expect(req.target).To(Equal("127.0.0.1:41000")) + + var out dialResult + Eventually(done, "10s").Should(Receive(&out)) + Expect(out.err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = out.conn.Close() }) + }) + }) +}) + +var _ = Describe("The relay request frame", func() { + It("carries a stated budget and reads it back", func() { + frame := &bytes.Buffer{} + Expect(cluster.WriteRelayRequest(frame, "node-7", 2500*time.Millisecond)).To(Succeed()) + nodeID, budget, err := cluster.ReadRelayRequest(frame) + Expect(err).ToNot(HaveOccurred()) + Expect(nodeID).To(Equal("node-7")) + Expect(budget).To(Equal(2500 * time.Millisecond)) + }) + + It("writes no budget at all when none is stated", func() { + // Zero must not reach the wire as the number zero: on the far side that + // is a caller with no time left, and the relay would refuse healthy + // traffic instead of falling back to its ceiling. + frame := &bytes.Buffer{} + Expect(cluster.WriteRelayRequest(frame, "node-7", 0)).To(Succeed()) + Expect(frame.Len()).To(Equal(2 + len("node-7"))) + nodeID, budget, err := cluster.ReadRelayRequest(frame) + Expect(err).ToNot(HaveOccurred()) + Expect(nodeID).To(Equal("node-7")) + Expect(budget).To(BeZero()) + }) + + It("refuses a node id that would split across the separator", func() { + Expect(cluster.WriteRelayRequest(&bytes.Buffer{}, "node 7", time.Second)).ToNot(Succeed()) + Expect(cluster.WriteRelayRequest(&bytes.Buffer{}, "", time.Second)).ToNot(Succeed()) + }) + + It("rejects a budget that is not a number of milliseconds", func() { + // The writer cannot produce this; a mismatched peer can, and reading it + // as "not stated" would silently restore the ceiling this frame exists + // to replace. + var raw bytes.Buffer + writeRawFrame(&raw, "node-7 soon") + _, _, err := cluster.ReadRelayRequest(&raw) + Expect(err).To(HaveOccurred()) + }) + + It("treats an expired stated budget as an error rather than as silence", func() { + var raw bytes.Buffer + writeRawFrame(&raw, "node-7 0") + _, _, err := cluster.ReadRelayRequest(&raw) + Expect(err).To(HaveOccurred()) + Expect(fmt.Sprint(err)).To(ContainSubstring("expired")) + }) +}) diff --git a/core/services/cluster/relay.go b/core/services/cluster/relay.go index 9fa03807d465..9392834f4e14 100644 --- a/core/services/cluster/relay.go +++ b/core/services/cluster/relay.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net" + "strconv" "strings" "time" @@ -59,35 +60,78 @@ var ( ErrRelayRequestInvalid = errors.New("cluster: the owning replica rejected the relay request as malformed") ) -// WriteRelayRequest names the worker a peer stream is for. +// WriteRelayRequest names the worker a peer stream is for, and how much time +// the ORIGINAL client still has. +// +// The budget is what makes the relay's own open bound honest. Everything on the +// far side of this frame is work done on behalf of a caller the relay cannot +// see, so without it the relay can only fall back to a deployment-wide constant +// that no operator has the information to set (see relayOpenTimeout). The +// dialler does have the information, because it holds the caller's context, so +// it is the one that states it. +// +// A zero budget means "not stated" and is written as no budget at all, which is +// also what an older replica sends. It is NEVER written as the number zero: on +// the far side that would be indistinguishable from a caller with no time left, +// and the relay would refuse traffic that is perfectly healthy. // // An empty node id is refused here rather than on the wire, so a caller with a -// bug learns at once instead of a round trip later. -func WriteRelayRequest(w io.Writer, nodeID string) error { +// bug learns at once instead of a round trip later. So is a node id containing +// the separator, because the split below takes the FIRST one and a node id with +// a space in it would silently move part of itself into the budget. +func WriteRelayRequest(w io.Writer, nodeID string, budget time.Duration) error { if nodeID == "" { return fmt.Errorf("writing a relay request: empty node id") } - return writeFrame(w, nodeID) + if strings.Contains(nodeID, streamRequestSeparator) { + return fmt.Errorf("writing a relay request: node id %q contains a space", nodeID) + } + if budget <= 0 { + return writeFrame(w, nodeID) + } + // A plain count of milliseconds rather than a duration string: an integer + // has one spelling, so two replicas cannot disagree about it the way they + // could about a units vocabulary that grew between their versions. Rounded + // UP so a sub-millisecond budget stays positive and keeps meaning "almost + // none" rather than collapsing into "not stated". + millis := (budget + time.Millisecond - 1) / time.Millisecond + return writeFrame(w, nodeID+streamRequestSeparator+strconv.FormatInt(int64(millis), 10)) } -// ReadRelayRequest reads the opening frame of a peer stream. +// ReadRelayRequest reads the opening frame of a peer stream. The budget is zero +// when the dialling replica stated none, which is also what a replica too old +// to state one sends. // // A malformed frame is an ordinary error, NOT ErrRelayRequestInvalid: that // sentinel is what a relay SENDS to describe a refusal, and producing it here // would leave a caller unable to tell "the peer refused my request" from "I // could not read the peer's". -func ReadRelayRequest(r io.Reader) (string, error) { +func ReadRelayRequest(r io.Reader) (string, time.Duration, error) { payload, err := readFrame(r) if err != nil { - return "", fmt.Errorf("reading a relay request: %w", err) + return "", 0, fmt.Errorf("reading a relay request: %w", err) } - if payload == "" { + nodeID, budgetText, stated := strings.Cut(payload, streamRequestSeparator) + if nodeID == "" { // An empty payload is a well-formed frame naming no worker. Treating // it as a node called "" would send the caller a routing refusal for a // request no replica can ever serve, so it stays the caller's bug. - return "", fmt.Errorf("reading a relay request: empty node id") + return "", 0, fmt.Errorf("reading a relay request: empty node id") + } + if !stated { + return nodeID, 0, nil } - return payload, nil + millis, err := strconv.ParseInt(budgetText, 10, 64) + if err != nil { + return "", 0, fmt.Errorf("reading a relay request for node %q: budget %q is not a number of milliseconds: %w", nodeID, budgetText, err) + } + if millis <= 0 { + // A caller with nothing left to spend. Reported as such rather than + // folded into "not stated", so the relay refuses at once instead of + // waiting out a backstop on behalf of a client that has already gone. + return nodeID, 0, fmt.Errorf("reading a relay request for node %q: budget %d ms has already expired", nodeID, millis) + } + return nodeID, time.Duration(millis) * time.Millisecond, nil } // WriteRelayAccepted tells the peer the stream now carries the worker tunnel's @@ -172,21 +216,25 @@ const ( // peer link dies, which is minutes on the default keepalive. relayHeaderTimeout = 15 * time.Second - // relayOpenTimeout bounds opening the worker-side stream. yamux blocks an - // Open once AcceptBacklog SYNs are in flight, waiting on synCh rather than - // failing (go-yamux/v5@v5.1.0/session.go:205-212); it honours the context, - // which is the only reason there is one here. Without the bound, a worker - // that has stopped accepting would turn a refusable condition into a parked - // peer, which is the one outcome this path exists to avoid. + // relayOpenTimeout is the CEILING on opening the worker-side stream. yamux + // blocks an Open once AcceptBacklog SYNs are in flight, waiting on synCh + // rather than failing (go-yamux/v5@v5.1.0/session.go:205-212); it honours + // the context, which is the only reason there is one here. Without the + // bound, a worker that has stopped accepting would turn a refusable + // condition into a parked peer, which is the one outcome this path exists + // to avoid. // - // It is deliberately NOT configurable. No operator has the information to - // set it: the number that matters is how long the ORIGINAL client is - // willing to wait, which is not known on this side of the link and is not - // something a deployment-wide constant can stand in for. The honest fix is - // the caller's remaining budget travelling in the relay request frame, and - // that belongs to the dialler that has the budget. Until then this is a - // backstop against parking, generous on purpose, because refusing healthy - // traffic costs more than waiting. + // It is deliberately NOT configurable, and it is no longer the whole + // answer. The number that actually matters is how long the ORIGINAL client + // is willing to wait, which no deployment-wide constant can stand in for; + // the dialling replica now states it in the request frame and accept takes + // the SMALLER of the two. This remains the backstop for a caller that + // stated nothing, generous on purpose, because refusing healthy traffic + // costs more than waiting. + // + // The stated budget only ever SHORTENS the wait. A caller willing to wait + // an hour must not be able to park this replica's relay goroutine and a + // yamux stream slot for an hour on a worker that has stopped accepting. relayOpenTimeout = 15 * time.Second ) @@ -271,7 +319,7 @@ func (r *Relay) accept(peerID string, stream net.Conn) (net.Conn, bool) { return nil, false } - nodeID, err := ReadRelayRequest(stream) + nodeID, budget, err := ReadRelayRequest(stream) if err != nil { // Includes the deadline above expiring. Both are "this stream never // said which worker it wanted", which is the dialling replica's bug @@ -296,7 +344,17 @@ func (r *Relay) accept(peerID string, stream net.Conn) (net.Conn, bool) { // Not the peer's deadline, because there is none to inherit: a yamux stream // carries no context. This bounds only the open, so a request that gets // past it is never cut short by it. - ctx, cancel := context.WithTimeout(context.Background(), r.openTimeout) + // + // The SMALLER of the ceiling and what the caller said it still has. Taking + // the caller's number when it is larger would let one patient client park a + // relay goroutine on a worker that has stopped accepting for as long as it + // liked; taking the ceiling when the caller's is smaller would keep waiting + // on behalf of a client that has already given up. + open := r.openTimeout + if budget > 0 && budget < open { + open = budget + } + ctx, cancel := context.WithTimeout(context.Background(), open) defer cancel() local, err := r.tunnels.Open(ctx, nodeID) diff --git a/core/services/cluster/relay_internal_test.go b/core/services/cluster/relay_internal_test.go index e23f6c55c888..2a5e631b0ccd 100644 --- a/core/services/cluster/relay_internal_test.go +++ b/core/services/cluster/relay_internal_test.go @@ -105,7 +105,7 @@ type unwritableStream struct { func newUnwritableStream(nodeID string) *unwritableStream { GinkgoHelper() frame := &bytes.Buffer{} - Expect(WriteRelayRequest(frame, nodeID)).To(Succeed()) + Expect(WriteRelayRequest(frame, nodeID, 0)).To(Succeed()) return &unwritableStream{request: frame.Bytes(), closed: make(chan struct{})} } @@ -176,7 +176,7 @@ var _ = Describe("The relay's own budgets", func() { stream, err := peer.OpenStream(ctx) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = stream.Close() }) - Expect(WriteRelayRequest(stream, "w1")).To(Succeed()) + Expect(WriteRelayRequest(stream, "w1", 0)).To(Succeed()) replies := make(chan error, 1) go func() { @@ -243,7 +243,7 @@ var _ = Describe("The relay's own budgets", func() { stream, err := peer.OpenStream(ctx) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = stream.Close() }) - Expect(WriteRelayRequest(stream, "w1")).To(Succeed()) + Expect(WriteRelayRequest(stream, "w1", 0)).To(Succeed()) replies := make(chan error, 1) go func() { @@ -265,6 +265,79 @@ var _ = Describe("The relay's own budgets", func() { Eventually(ends, "10s").Should(Receive(HaveOccurred())) }) + It("bounds its open by the caller's stated budget when that is the shorter", func() { + // The ceiling here is 10s and the caller says it has 50ms. Without the + // stated budget this replica would hold a relay goroutine and a yamux + // stream slot for the full ceiling on behalf of a client that gave up + // almost immediately. + relay := newRelay(tun, 0, 10*time.Second) + store := NewSessionStore(relay.Stream) + DeferCleanup(store.CloseAll) + peer, accepted := backloggedPair(256) + store.Accept("peer-1", accepted) + + _, frontend := backloggedPair(1) + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + // One un-accepted open fills the one-deep backlog, so the relay's own + // open is the one that has to wait out a budget. + filler, err := frontend.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = filler.Close() }) + + stream, err := peer.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = stream.Close() }) + Expect(WriteRelayRequest(stream, "w1", 50*time.Millisecond)).To(Succeed()) + + replies := make(chan error, 1) + go func() { + defer GinkgoRecover() + replies <- ReadRelayReply(stream) + }() + var reply error + // Two seconds is twenty times the stated budget and a fifth of the + // ceiling, so only a relay that honoured the budget answers inside it. + Eventually(replies, "2s").Should(Receive(&reply)) + Expect(reply).To(MatchError(ErrRelayUnavailable)) + // The tunnel IS held here. A budget running out must not turn into a + // routing fact, and it must never become absence. + Expect(reply).ToNot(MatchError(ErrNotOwner)) + Expect(reply).ToNot(MatchError(ErrNoConnection)) + }) + + It("does not let a stated budget stretch its own ceiling", func() { + // A patient client must not be able to park this replica. The ceiling + // is 50ms and the caller says it will wait ten seconds; the refusal + // still has to arrive on the ceiling. + relay := newRelay(tun, 0, 50*time.Millisecond) + store := NewSessionStore(relay.Stream) + DeferCleanup(store.CloseAll) + peer, accepted := backloggedPair(256) + store.Accept("peer-1", accepted) + + _, frontend := backloggedPair(1) + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + filler, err := frontend.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = filler.Close() }) + + stream, err := peer.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = stream.Close() }) + Expect(WriteRelayRequest(stream, "w1", 10*time.Second)).To(Succeed()) + + replies := make(chan error, 1) + go func() { + defer GinkgoRecover() + replies <- ReadRelayReply(stream) + }() + var reply error + Eventually(replies, "2s").Should(Receive(&reply)) + Expect(reply).To(MatchError(ErrRelayUnavailable)) + }) + It("closes the worker's stream when it cannot tell the peer the stream was accepted", func() { // The reply is the last thing that can fail after a worker stream has // been opened. A relay that gave up without closing it would leak one diff --git a/core/services/cluster/relay_test.go b/core/services/cluster/relay_test.go index e0c7bd0499c2..04fa46f20285 100644 --- a/core/services/cluster/relay_test.go +++ b/core/services/cluster/relay_test.go @@ -89,7 +89,7 @@ var _ = Describe("The inter-replica relay", func() { stream, err := peer.OpenStream(ctx) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = stream.Close() }) - Expect(cluster.WriteRelayRequest(stream, nodeID)).To(Succeed()) + Expect(cluster.WriteRelayRequest(stream, nodeID, 0)).To(Succeed()) return stream } @@ -279,14 +279,14 @@ var _ = Describe("The relay wire framing", func() { It("round-trips a node id", func() { frame := &bytes.Buffer{} - Expect(cluster.WriteRelayRequest(frame, "node-7")).To(Succeed()) - nodeID, err := cluster.ReadRelayRequest(frame) + Expect(cluster.WriteRelayRequest(frame, "node-7", 0)).To(Succeed()) + nodeID, _, err := cluster.ReadRelayRequest(frame) Expect(err).ToNot(HaveOccurred()) Expect(nodeID).To(Equal("node-7")) }) It("refuses to write an empty node id, rather than spending a round trip on it", func() { - Expect(cluster.WriteRelayRequest(&bytes.Buffer{}, "")).To(HaveOccurred()) + Expect(cluster.WriteRelayRequest(&bytes.Buffer{}, "", 0)).To(HaveOccurred()) }) // Acceptance is not the whole surface. A refusal read by the wrong hop's diff --git a/core/services/nodes/backend_client_factory_test.go b/core/services/nodes/backend_client_factory_test.go new file mode 100644 index 000000000000..be20f5e347b0 --- /dev/null +++ b/core/services/nodes/backend_client_factory_test.go @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: MIT + +package nodes + +import ( + "context" + "net" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + grpcpkg "github.com/mudler/LocalAI/pkg/grpc" +) + +var _ = Describe("The backend client factory", func() { + Describe("without a worker tunnel dialer", func() { + It("refuses to build a client for a node rather than dialling its address", func() { + // The whole point. A factory that answered here with a client + // pointed at the raw address would work on a single-host developer + // setup and fail against every worker that has no inbound port, + // which is the worst way for this to behave. + f := &tokenClientFactory{token: "tok"} + _, err := f.NewClientForNode("node-1", "10.0.0.1:41000", false) + Expect(err).To(MatchError(ErrNoWorkerDialer)) + }) + + It("offers no direct-dial constructor for anything to reach for", func() { + // Structural, not documented. A NewClient alongside NewClientForNode + // would be reachable from every call site that holds an address, + // which is all of them, and reintroducing the bypass would then be + // a one-word edit that compiles and passes every other spec. + var factory any = &tunnelClientFactory{} + _, hasDirectDial := factory.(interface { + NewClient(address string, parallel bool) grpcpkg.Backend + }) + Expect(hasDirectDial).To(BeFalse()) + }) + + It("refuses to be constructed at all", func() { + _, err := NewTunnelClientFactory("tok", nil) + Expect(err).To(MatchError(ErrNoWorkerDialer)) + }) + }) + + Describe("with a worker tunnel dialer", func() { + It("builds a client that reaches the backend through the node's dialer", func() { + // The proof is that the client's transport is the one this factory + // was given: it carries bytes from a listener that the address in + // the request never names. + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = listener.Close() }) + + asked := make(chan string, 4) + f, err := NewTunnelClientFactory("", func(nodeID string) func(ctx context.Context, addr string) (net.Conn, error) { + return func(ctx context.Context, addr string) (net.Conn, error) { + asked <- nodeID + "|" + addr + var d net.Dialer + return d.DialContext(ctx, "tcp", listener.Addr().String()) + } + }) + Expect(err).ToNot(HaveOccurred()) + + client, err := f.NewClientForNode("node-1", "10.255.255.1:41000", false) + Expect(err).ToNot(HaveOccurred()) + + // The address is unroutable on purpose: only a client that used the + // dialer can reach anything at all. The health check itself fails, + // because nothing on the far side speaks gRPC; what it proves is + // which transport was asked. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + defer GinkgoRecover() + _, _ = client.HealthCheck(ctx) + }() + Eventually(asked, "10s").Should(Receive(Equal("node-1|10.255.255.1:41000"))) + }) + + It("refuses a request with no node id", func() { + f, err := NewTunnelClientFactory("", func(string) func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + return func(ctx context.Context, addr string) (net.Conn, error) { + return d.DialContext(ctx, "tcp", addr) + } + }) + Expect(err).ToNot(HaveOccurred()) + _, err = f.NewClientForNode("", "10.0.0.1:41000", false) + Expect(err).To(MatchError(ErrNoWorkerDialer)) + }) + + It("refuses when the dialer has none for that node", func() { + f, err := NewTunnelClientFactory("", func(string) func(ctx context.Context, addr string) (net.Conn, error) { + return nil + }) + Expect(err).ToNot(HaveOccurred()) + _, err = f.NewClientForNode("node-1", "10.0.0.1:41000", false) + Expect(err).To(MatchError(ErrNoWorkerDialer)) + }) + }) +}) diff --git a/core/services/nodes/distributed_store.go b/core/services/nodes/distributed_store.go index ba1379367413..542aec69e693 100644 --- a/core/services/nodes/distributed_store.go +++ b/core/services/nodes/distributed_store.go @@ -2,7 +2,9 @@ package nodes import ( "context" + "fmt" + grpc "github.com/mudler/LocalAI/pkg/grpc" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/xlog" ) @@ -14,10 +16,26 @@ import ( type DistributedModelStore struct { local model.ModelStore registry ModelLookup + // clients builds the gRPC client for a model that lives on a worker. + // + // It is not optional in a real deployment, and the reason is the second + // construction path this store used to be: a *model.Model built with a nil + // client makes pkg/model.Model.GRPC dial its raw address with gRPC's own + // dialer the first time anything touches it, which is exactly the direct + // dial to a worker's advertised address the tunnel replaces. That path is + // reached in production, by ShutdownModel's Free and by the backend + // monitor's Status, so it is not theoretical. + clients BackendClientFactory } -func NewDistributedModelStore(local model.ModelStore, registry ModelLookup) *DistributedModelStore { - return &DistributedModelStore{local: local, registry: registry} +// NewDistributedModelStore returns the store, which reaches a remote model's +// backend through clients. +// +// A nil clients is a programming error and is treated as one: Range refuses to +// synthesise a model it cannot give a working client to, rather than handing +// back one that silently dials the worker's address. See the field comment. +func NewDistributedModelStore(local model.ModelStore, registry ModelLookup, clients BackendClientFactory) *DistributedModelStore { + return &DistributedModelStore{local: local, registry: registry, clients: clients} } // Get checks the local cache only. In distributed mode, models must be routed @@ -80,9 +98,33 @@ func (s *DistributedModelStore) Range(fn func(string, *model.Model) bool) { continue } - m := model.NewModel(nm.ModelName, node.Address, nil) + // NewModelWithClient, never NewModel: a model built without a client + // lazily dials its address with gRPC's default dialer the first time + // anything calls GRPC() on it, which reaches a worker only while + // workers still listen on a routable address. Building the client here + // means the bypass has no path left rather than an unused one. + client, err := s.clientFor(nm.NodeID, node.Address) + if err != nil { + xlog.Error("DistributedModelStore: not listing a remote model it cannot reach", + "model", nm.ModelName, "nodeID", nm.NodeID, "error", err) + continue + } + m := model.NewModelWithClient(nm.ModelName, node.Address, client) if !fn(nm.ModelName, m) { return } } } + +// clientFor builds the backend client for a model running on a worker. +// +// It fails rather than falling back. A fallback here would be invisible: the +// listing would look complete, shutdown would appear to work, and the direct +// dial underneath it would succeed on a single-host developer setup and fail +// against every worker that has no inbound port. +func (s *DistributedModelStore) clientFor(nodeID, address string) (grpc.Backend, error) { + if s.clients == nil { + return nil, fmt.Errorf("no backend client factory is wired into the distributed model store: %w", ErrNoWorkerDialer) + } + return s.clients.NewClientForNode(nodeID, address, false) +} diff --git a/core/services/nodes/distributed_store_test.go b/core/services/nodes/distributed_store_test.go index 9b6e4ccc9464..08362fbfddca 100644 --- a/core/services/nodes/distributed_store_test.go +++ b/core/services/nodes/distributed_store_test.go @@ -2,11 +2,13 @@ package nodes import ( "context" + "errors" "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + grpc "github.com/mudler/LocalAI/pkg/grpc" "github.com/mudler/LocalAI/pkg/model" ) @@ -48,15 +50,17 @@ var _ ModelLookup = (*fakeModelLookup)(nil) var _ = Describe("DistributedModelStore", func() { var ( - local *model.InMemoryModelStore - lookup *fakeModelLookup - store *DistributedModelStore + local *model.InMemoryModelStore + lookup *fakeModelLookup + clients *fakeBackendClientFactory + store *DistributedModelStore ) BeforeEach(func() { local = model.NewInMemoryModelStore() lookup = newFakeModelLookup() - store = NewDistributedModelStore(local, lookup) + clients = newFakeBackendClientFactory() + store = NewDistributedModelStore(local, lookup, clients) }) Describe("Get", func() { @@ -113,6 +117,63 @@ var _ = Describe("DistributedModelStore", func() { Expect(visited).To(HaveLen(2)) }) + It("gives every remote model a client that reaches the worker through its node", func() { + // The second construction path, closed. A model built with a nil + // client makes pkg/model.Model.GRPC dial its raw address with + // gRPC's own dialer the first time anything touches it, which + // bypasses the worker's tunnel completely. It is reached in + // production: ShutdownModel calls Free on it and the backend + // monitor calls Status. + dbNode := &BackendNode{ID: "node-2", Address: "10.0.0.3:50051"} + lookup.nodes["node-2"] = dbNode + lookup.allModels = []NodeModel{{NodeID: "node-2", ModelName: "remote-model"}} + + var got *model.Model + store.Range(func(id string, m *model.Model) bool { + if id == "remote-model" { + got = m + } + return true + }) + Expect(got).ToNot(BeNil()) + // The client is the factory's, so GRPC() returns it rather than + // building one by dialling. Asked for by NODE, not by address. + Expect(got.GRPC(false, nil)).To(BeIdenticalTo(grpc.Backend(clients.defaultClient))) + Expect(clients.nodesSeen()).To(ContainElement("node-2")) + }) + + It("refuses to list a remote model it has no way to reach", func() { + // Loudly, not by falling back. A model handed back here with a + // direct-dialling client works on a single-host developer setup and + // fails against every worker with no inbound port, which is the + // worst way for this defect to behave. + clients.refuseForNode = errors.New("no tunnel for you") + dbNode := &BackendNode{ID: "node-2", Address: "10.0.0.3:50051"} + lookup.nodes["node-2"] = dbNode + lookup.allModels = []NodeModel{{NodeID: "node-2", ModelName: "remote-model"}} + + visited := map[string]bool{} + store.Range(func(id string, _ *model.Model) bool { + visited[id] = true + return true + }) + Expect(visited).ToNot(HaveKey("remote-model")) + }) + + It("refuses when no client factory was wired at all", func() { + bare := NewDistributedModelStore(local, lookup, nil) + dbNode := &BackendNode{ID: "node-2", Address: "10.0.0.3:50051"} + lookup.nodes["node-2"] = dbNode + lookup.allModels = []NodeModel{{NodeID: "node-2", ModelName: "remote-model"}} + + visited := map[string]bool{} + bare.Range(func(id string, _ *model.Model) bool { + visited[id] = true + return true + }) + Expect(visited).ToNot(HaveKey("remote-model")) + }) + It("handles DB list error gracefully", func() { localModel := model.NewModel("model-x", "10.0.0.1:50051", nil) local.Set("model-x", localModel) diff --git a/core/services/nodes/file_stager_http.go b/core/services/nodes/file_stager_http.go index 0a64db5abb2a..2c3146946443 100644 --- a/core/services/nodes/file_stager_http.go +++ b/core/services/nodes/file_stager_http.go @@ -30,9 +30,21 @@ import ( // Files are transferred between the frontend and backend nodes over a small // HTTP server running alongside the gRPC backend process. type HTTPFileStager struct { - httpAddrFor func(nodeID string) (string, error) - token string - client *http.Client + httpAddrFor func(nodeID string) (string, error) + token string + // dialFor supplies the transport for one worker. It is per node because a + // worker is reached over ITS OWN tunnel, and an http.Transport carries one + // DialContext: one shared transport could only ever reach one worker. + // + // nil means no tunnel dialer is wired, and every request is then refused + // rather than sent to the worker's advertised address; see + // ErrNoWorkerDialer for why that is not a fallback. + dialFor WorkerNetDialerFor + // clients caches one *http.Client per node. Caching is what keeps the + // connection pool: a client built per request would open a fresh tunnel + // stream for every chunk of a multi-gigabyte upload. + clientsMu sync.Mutex + clients map[string]*http.Client responseTimeout time.Duration // timeout waiting for server response after upload maxRetries int // number of retry attempts for transient failures } @@ -40,7 +52,8 @@ type HTTPFileStager struct { // NewHTTPFileStager creates a new HTTP file stager. // httpAddrFor should return the HTTP address (host:port) for the given node ID. // token is the registration token used for authentication. -func NewHTTPFileStager(httpAddrFor func(nodeID string) (string, error), token string) *HTTPFileStager { +// dialFor supplies the per-node transport; see the dialFor field. +func NewHTTPFileStager(httpAddrFor func(nodeID string) (string, error), token string, dialFor WorkerNetDialerFor) *HTTPFileStager { responseTimeout := 30 * time.Minute if v := os.Getenv("LOCALAI_FILE_TRANSFER_TIMEOUT"); v != "" { if d, err := time.ParseDuration(v); err == nil { @@ -55,11 +68,49 @@ func NewHTTPFileStager(httpAddrFor func(nodeID string) (string, error), token st } } + return &HTTPFileStager{ + httpAddrFor: httpAddrFor, + token: token, + dialFor: dialFor, + clients: map[string]*http.Client{}, + responseTimeout: responseTimeout, + maxRetries: maxRetries, + } +} + +// clientFor returns the HTTP client that reaches one worker, building it on +// first use. +// +// Every setting below is carried over unchanged from the single shared client +// this replaced, except DialContext, which now opens a stream on that worker's +// tunnel instead of connecting to its advertised address. HTTP/2 stays off for +// the reason it always was: its flow control stalls large uploads. +// +// What the tunnel dial does NOT carry over is the net.Dialer's own 30s connect +// timeout and 15s keepalive, because neither has anything left to act on: there +// is no TCP connect to time out, and liveness on the link is the yamux +// session's keepalive rather than the socket's. What still bounds a request is +// the context the caller passes. +// +// No client.Timeout is set, and that is deliberate: for large uploads +// http.Client.Timeout covers the whole request including the body, and firing +// mid-write closes the connection and shows up server-side as "connection reset +// by peer". The upload loop's own resume budget bounds the transfer instead. +func (h *HTTPFileStager) clientFor(nodeID string) (*http.Client, error) { + if h.dialFor == nil { + return nil, fmt.Errorf("staging files to node %s: %w", nodeID, ErrNoWorkerDialer) + } + h.clientsMu.Lock() + defer h.clientsMu.Unlock() + if c, ok := h.clients[nodeID]; ok { + return c, nil + } + dial := h.dialFor(nodeID) + if dial == nil { + return nil, fmt.Errorf("staging files to node %s: %w", nodeID, ErrNoWorkerDialer) + } transport := &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 15 * time.Second, // aggressive keepalive for LAN transfers - }).DialContext, + DialContext: dial, ForceAttemptHTTP2: false, // HTTP/2 flow control can stall large uploads MaxIdleConns: 10, IdleConnTimeout: 90 * time.Second, @@ -68,19 +119,9 @@ func NewHTTPFileStager(httpAddrFor func(nodeID string) (string, error), token st WriteBufferSize: 256 << 10, // 256 KB ReadBufferSize: 256 << 10, // 256 KB } - - return &HTTPFileStager{ - httpAddrFor: httpAddrFor, - token: token, - // No Timeout set — for large uploads, http.Client.Timeout covers the - // entire request lifecycle including the body upload. If it fires - // mid-write, Go closes the connection causing "connection reset by peer" - // on the server. Instead we use ResponseHeaderTimeout on the transport - // to cover only the wait-for-server-response phase. - client: httpclient.New(httpclient.WithTransport(transport)), - responseTimeout: responseTimeout, - maxRetries: maxRetries, - } + c := httpclient.New(httpclient.WithTransport(transport)) + h.clients[nodeID] = c + return c, nil } // ReleaseRemote removes one exact ephemeral key from a backend node. @@ -92,6 +133,10 @@ func (h *HTTPFileStager) ReleaseRemote(ctx context.Context, nodeID, key string) if err != nil { return fmt.Errorf("resolving HTTP address for node %s: %w", nodeID, err) } + client, err := h.clientFor(nodeID) + if err != nil { + return err + } releaseURL := (&url.URL{Scheme: "http", Host: addr, Path: "/v1/files/" + key}).String() req, err := http.NewRequestWithContext(ctx, http.MethodDelete, releaseURL, nil) if err != nil { @@ -100,7 +145,7 @@ func (h *HTTPFileStager) ReleaseRemote(ctx context.Context, nodeID, key string) if h.token != "" { req.Header.Set("Authorization", "Bearer "+h.token) } - resp, err := h.client.Do(req) + resp, err := client.Do(req) if err != nil { return fmt.Errorf("releasing %q from node %s: %w", key, nodeID, err) } @@ -123,6 +168,10 @@ func (h *HTTPFileStager) ReleaseRemoteRequest(ctx context.Context, nodeID, reque if err != nil { return fmt.Errorf("resolving HTTP address for node %s: %w", nodeID, err) } + client, err := h.clientFor(nodeID) + if err != nil { + return err + } payload, err := json.Marshal(struct { RequestID string `json:"request_id"` }{RequestID: requestID}) @@ -138,7 +187,7 @@ func (h *HTTPFileStager) ReleaseRemoteRequest(ctx context.Context, nodeID, reque if h.token != "" { req.Header.Set("Authorization", "Bearer "+h.token) } - resp, err := h.client.Do(req) + resp, err := client.Do(req) if err != nil { return fmt.Errorf("releasing request inputs from node %s: %w", nodeID, err) } @@ -170,9 +219,13 @@ func (h *HTTPFileStager) EnsureRemote(ctx context.Context, nodeID, localPath, ke if err != nil { return "", fmt.Errorf("resolving HTTP address for node %s: %w", nodeID, err) } + client, err := h.clientFor(nodeID) + if err != nil { + return "", err + } // Probe: check if the remote already has the file with matching content hash. - if remotePath, ok, probeErr := h.probeExisting(ctx, addr, localPath, key); probeErr != nil { + if remotePath, ok, probeErr := h.probeExisting(ctx, client, addr, localPath, key); probeErr != nil { return "", fmt.Errorf("claiming existing file on node %s: %w", nodeID, probeErr) } else if ok { xlog.Info("Upload skipped (file already exists with matching hash)", "node", nodeID, "key", key, "remotePath", remotePath) @@ -232,9 +285,9 @@ func (h *HTTPFileStager) EnsureRemote(ctx context.Context, nodeID, localPath, ke // matching ours unlocks resume from the reported size; any other // outcome (missing file, hash mismatch, partial-of-different-file) // resets to 0 and uploads the entire file. - startOffset := h.resumeOffset(resumeCtx, addr, key, localHash, fileSize) + startOffset := h.resumeOffset(resumeCtx, client, addr, key, localHash, fileSize) - result, err := h.doUpload(ctx, resumeCtx, addr, nodeID, localPath, key, url, fileSize, startOffset, localHash) + result, err := h.doUpload(ctx, resumeCtx, client, addr, nodeID, localPath, key, url, fileSize, startOffset, localHash) if err == nil { if attempt > 1 { xlog.Info("File upload succeeded after retry", "node", nodeID, "file", filepath.Base(localPath), "attempt", attempt) @@ -321,7 +374,7 @@ func nextBackoff(attempt int) time.Duration { // different target hash). It returns the server-reported size when the // server's X-Target-SHA256 matches our expected final hash AND the size is // strictly less than the local file size. -func (h *HTTPFileStager) resumeOffset(ctx context.Context, addr, key, localHash string, fileSize int64) int64 { +func (h *HTTPFileStager) resumeOffset(ctx context.Context, client *http.Client, addr, key, localHash string, fileSize int64) int64 { if localHash == "" || fileSize <= 0 { return 0 } @@ -333,7 +386,7 @@ func (h *HTTPFileStager) resumeOffset(ctx context.Context, addr, key, localHash if h.token != "" { req.Header.Set("Authorization", "Bearer "+h.token) } - resp, err := h.client.Do(req) + resp, err := client.Do(req) if err != nil { return 0 } @@ -366,7 +419,7 @@ func (h *HTTPFileStager) resumeOffset(ctx context.Context, addr, key, localHash // the bytes from startOffset to fileSize-1. The outerCtx is the long-lived // resume budget; reqCtx is what's bound to the request (currently the same as // the parent ctx, since http.Client doesn't expose a per-request timeout). -func (h *HTTPFileStager) doUpload(ctx, outerCtx context.Context, addr, nodeID, localPath, key, url string, fileSize, startOffset int64, expectedHash string) (string, error) { +func (h *HTTPFileStager) doUpload(ctx, outerCtx context.Context, client *http.Client, addr, nodeID, localPath, key, url string, fileSize, startOffset int64, expectedHash string) (string, error) { if startOffset < 0 || startOffset > fileSize { startOffset = 0 } @@ -421,7 +474,7 @@ func (h *HTTPFileStager) doUpload(ctx, outerCtx context.Context, addr, nodeID, l req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", startOffset, fileSize-1, fileSize)) } - resp, err := h.client.Do(req) + resp, err := client.Do(req) if err != nil { xlog.Error("File upload failed", "node", nodeID, "file", filepath.Base(localPath), "size", humanFileSize(fileSize), "offset", startOffset, "error", err) @@ -526,7 +579,7 @@ func isTransientError(err error) bool { // upload can be skipped. HEAD and hash errors fall through to a normal PUT. // Matching ephemeral files are claimed first; a 404 or 405 claim response // identifies an older worker and also falls through to PUT. -func (h *HTTPFileStager) probeExisting(ctx context.Context, addr, localPath, key string) (string, bool, error) { +func (h *HTTPFileStager) probeExisting(ctx context.Context, client *http.Client, addr, localPath, key string) (string, bool, error) { url := fmt.Sprintf("http://%s/v1/files/%s", addr, key) req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) @@ -537,7 +590,7 @@ func (h *HTTPFileStager) probeExisting(ctx context.Context, addr, localPath, key req.Header.Set("Authorization", "Bearer "+h.token) } - resp, err := h.client.Do(req) + resp, err := client.Do(req) if err != nil { return "", false, nil } @@ -568,7 +621,7 @@ func (h *HTTPFileStager) probeExisting(ctx context.Context, addr, localPath, key } if strings.HasPrefix(key, "ephemeral/") { - claimed, err := h.claimExisting(ctx, addr, key) + claimed, err := h.claimExisting(ctx, client, addr, key) if err != nil { return "", false, err } @@ -580,7 +633,7 @@ func (h *HTTPFileStager) probeExisting(ctx context.Context, addr, localPath, key return remotePath, true, nil } -func (h *HTTPFileStager) claimExisting(ctx context.Context, addr, key string) (bool, error) { +func (h *HTTPFileStager) claimExisting(ctx context.Context, client *http.Client, addr, key string) (bool, error) { claimURL := (&url.URL{ Scheme: "http", Host: addr, @@ -594,7 +647,7 @@ func (h *HTTPFileStager) claimExisting(ctx context.Context, addr, key string) (b if h.token != "" { req.Header.Set("Authorization", "Bearer "+h.token) } - resp, err := h.client.Do(req) + resp, err := client.Do(req) if err != nil { return false, fmt.Errorf("claiming %q: %w", key, err) } @@ -788,6 +841,10 @@ func (h *HTTPFileStager) FetchRemoteByKey(ctx context.Context, nodeID, key, loca if err != nil { return fmt.Errorf("resolving HTTP address for node %s: %w", nodeID, err) } + client, err := h.clientFor(nodeID) + if err != nil { + return err + } if err := os.MkdirAll(filepath.Dir(localDst), 0750); err != nil { return fmt.Errorf("creating directory for %s: %w", localDst, err) @@ -804,7 +861,7 @@ func (h *HTTPFileStager) FetchRemoteByKey(ctx context.Context, nodeID, key, loca req.Header.Set("Authorization", "Bearer "+h.token) } - resp, err := h.client.Do(req) + resp, err := client.Do(req) if err != nil { return fmt.Errorf("downloading from node %s: %w", nodeID, err) } @@ -850,6 +907,10 @@ func (h *HTTPFileStager) AllocRemoteTemp(ctx context.Context, nodeID string) (st if err != nil { return "", fmt.Errorf("resolving HTTP address for node %s: %w", nodeID, err) } + client, err := h.clientFor(nodeID) + if err != nil { + return "", err + } url := fmt.Sprintf("http://%s/v1/files/temp", addr) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) @@ -860,7 +921,7 @@ func (h *HTTPFileStager) AllocRemoteTemp(ctx context.Context, nodeID string) (st req.Header.Set("Authorization", "Bearer "+h.token) } - resp, err := h.client.Do(req) + resp, err := client.Do(req) if err != nil { return "", fmt.Errorf("allocating temp file on node %s: %w", nodeID, err) } @@ -891,6 +952,10 @@ func (h *HTTPFileStager) ListRemoteDir(ctx context.Context, nodeID, keyPrefix st if err != nil { return nil, fmt.Errorf("resolving HTTP address for node %s: %w", nodeID, err) } + client, err := h.clientFor(nodeID) + if err != nil { + return nil, err + } url := fmt.Sprintf("http://%s/v1/files-list/%s", addr, keyPrefix) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -901,7 +966,7 @@ func (h *HTTPFileStager) ListRemoteDir(ctx context.Context, nodeID, keyPrefix st req.Header.Set("Authorization", "Bearer "+h.token) } - resp, err := h.client.Do(req) + resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("listing dir on node %s: %w", nodeID, err) } diff --git a/core/services/nodes/file_stager_verify_deadline_test.go b/core/services/nodes/file_stager_verify_deadline_test.go index 0827bbecdc56..0239b832e7f8 100644 --- a/core/services/nodes/file_stager_verify_deadline_test.go +++ b/core/services/nodes/file_stager_verify_deadline_test.go @@ -65,7 +65,7 @@ var _ = Describe("staging verify phase and the cold-load stall window", func() { return "", err } return u.Host, nil - }, "") + }, "", directNetDialerFor) } It("survives a run of verified-and-skipped shards that upload no bytes at all", func() { diff --git a/core/services/nodes/file_transfer_server_test.go b/core/services/nodes/file_transfer_server_test.go index 918379c5c210..24a5cd8ff1c2 100644 --- a/core/services/nodes/file_transfer_server_test.go +++ b/core/services/nodes/file_transfer_server_test.go @@ -111,6 +111,50 @@ func (g *recordingEphemeralCapacity) CapacityWriter(_ string, destination io.Wri return nopWriteCloser{Writer: destination}, nil } +// directNetDialerFor is the dial function these specs give the stager. +// +// The stager exists to reach a worker over that worker's TUNNEL, and it refuses +// to reach one at all without a dialer. These specs are about the HTTP protocol +// between the stager and the file-transfer server, and they run that server on +// loopback, so a plain TCP dial is what stands in for the tunnel here. Nothing +// in production supplies this: see the wiring in core/application. +func directNetDialerFor(_ string) func(ctx context.Context, network, addr string) (net.Conn, error) { + var d net.Dialer + return d.DialContext +} + +var _ = Describe("The HTTP file stager without a worker dialer", func() { + // Every request refused, none sent. Staging reaches a worker over that + // worker's tunnel, and a stager that fell back to connecting to the + // registered address would move gigabytes over a path that exists only + // while workers still listen on one. + newBare := func() *HTTPFileStager { + return NewHTTPFileStager(func(string) (string, error) { return "127.0.0.1:1", nil }, "tok", nil) + } + + It("refuses to upload", func() { + local := filepath.Join(GinkgoT().TempDir(), "f.bin") + Expect(os.WriteFile(local, []byte("payload"), 0o600)).To(Succeed()) + _, err := newBare().EnsureRemote(context.Background(), "node-1", local, "f.bin") + Expect(err).To(MatchError(ErrNoWorkerDialer)) + }) + + It("refuses to download", func() { + dst := filepath.Join(GinkgoT().TempDir(), "out.bin") + Expect(newBare().FetchRemoteByKey(context.Background(), "node-1", "f.bin", dst)).To(MatchError(ErrNoWorkerDialer)) + }) + + It("refuses to allocate a remote temp file", func() { + _, err := newBare().AllocRemoteTemp(context.Background(), "node-1") + Expect(err).To(MatchError(ErrNoWorkerDialer)) + }) + + It("refuses to list a remote directory", func() { + _, err := newBare().ListRemoteDir(context.Background(), "node-1", "models/") + Expect(err).To(MatchError(ErrNoWorkerDialer)) + }) +}) + var _ = Describe("FileTransferServer", func() { setupTestServer := func(token string, maxUploadSize int64) (*httptest.Server, string, string, string) { stagingDir := GinkgoT().TempDir() @@ -645,7 +689,7 @@ var _ = Describe("FileTransferServer", func() { DeferCleanup(ts.Close) stager := NewHTTPFileStager(func(string) (string, error) { return strings.TrimPrefix(ts.URL, "http://"), nil - }, "") + }, "", directNetDialerFor) for range 2 { path, err := stager.EnsureRemote(context.Background(), "node-1", localPath, key) @@ -675,7 +719,7 @@ var _ = Describe("FileTransferServer", func() { DeferCleanup(ts.Close) stager := NewHTTPFileStager(func(string) (string, error) { return strings.TrimPrefix(ts.URL, "http://"), nil - }, "") + }, "", directNetDialerFor) path, err := stager.EnsureRemote(context.Background(), "node-1", localPath, "ephemeral/audio/request/input.wav") @@ -714,7 +758,7 @@ var _ = Describe("FileTransferServer", func() { DeferCleanup(ts.Close) stager := NewHTTPFileStager(func(string) (string, error) { return strings.TrimPrefix(ts.URL, "http://"), nil - }, "") + }, "", directNetDialerFor) backend := &lifecycleBackend{} client := NewFileStagingClient(backend, stager, "node-1") @@ -750,7 +794,7 @@ var _ = Describe("FileTransferServer", func() { DeferCleanup(ts.Close) stager := NewHTTPFileStager(func(string) (string, error) { return strings.TrimPrefix(ts.URL, "http://"), nil - }, "") + }, "", directNetDialerFor) path, err := stager.EnsureRemote(context.Background(), "node-1", localPath, "models/tracking/model.bin") @@ -780,7 +824,7 @@ var _ = Describe("FileTransferServer", func() { addr := strings.TrimPrefix(ts.URL, "http://") stager := NewHTTPFileStager(func(nodeID string) (string, error) { return addr, nil - }, "tok") + }, "tok", directNetDialerFor) remotePath, err := stager.EnsureRemote(context.Background(), "node-1", localPath, "present.bin") Expect(err).ToNot(HaveOccurred()) @@ -809,7 +853,7 @@ var _ = Describe("FileTransferServer", func() { addr := strings.TrimPrefix(ts.URL, "http://") stager := NewHTTPFileStager(func(nodeID string) (string, error) { return addr, nil - }, "tok") + }, "tok", directNetDialerFor) remotePath, err := stager.EnsureRemote(context.Background(), "node-1", localPath, "changed.bin") Expect(err).ToNot(HaveOccurred()) @@ -838,7 +882,7 @@ var _ = Describe("FileTransferServer", func() { addr := strings.TrimPrefix(ts.URL, "http://") stager := NewHTTPFileStager(func(nodeID string) (string, error) { return addr, nil - }, "tok") + }, "tok", directNetDialerFor) remotePath, err := stager.EnsureRemote(context.Background(), "node-1", localPath, "new.bin") Expect(err).ToNot(HaveOccurred()) @@ -874,7 +918,7 @@ var _ = Describe("FileTransferServer", func() { addr := strings.TrimPrefix(ts.URL, "http://") stager := NewHTTPFileStager(func(nodeID string) (string, error) { return addr, nil - }, "") + }, "", directNetDialerFor) remotePath, err := stager.EnsureRemote(context.Background(), "node-1", localPath, "compat.bin") Expect(err).ToNot(HaveOccurred()) @@ -1091,7 +1135,7 @@ var _ = Describe("FileTransferServer", func() { addr := strings.TrimPrefix(ts.URL, "http://") stager := NewHTTPFileStager(func(nodeID string) (string, error) { return addr, nil - }, "tok") + }, "tok", directNetDialerFor) remotePath, err := stager.EnsureRemote(context.Background(), "node-1", localPath, "resume.bin") Expect(err).ToNot(HaveOccurred()) @@ -1189,7 +1233,7 @@ var _ = Describe("FileTransferServer", func() { addr := strings.TrimPrefix(ts.URL, "http://") stager := NewHTTPFileStager(func(nodeID string) (string, error) { return addr, nil - }, "tok") + }, "tok", directNetDialerFor) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() diff --git a/core/services/nodes/health.go b/core/services/nodes/health.go index b82e57f91201..57c87a82ed32 100644 --- a/core/services/nodes/health.go +++ b/core/services/nodes/health.go @@ -49,7 +49,11 @@ type HealthMonitor struct { // NewHealthMonitor creates a new HealthMonitor. // If db is non-nil (PostgreSQL), an advisory lock is used so that only one // frontend instance runs health checks at a time in distributed mode. -// If clientFactory is nil, a default factory using the given authToken is used. +// clientFactory is what reaches a worker's backends, over that worker's tunnel. +// Omitting it (or passing nil) leaves the monitor with a factory that refuses +// every request, so per-model probes are skipped and logged rather than counted +// as misses; authToken is then only the credential a working factory would have +// carried. Production always passes one. func NewHealthMonitor(registry NodeHealthStore, db *gorm.DB, checkInterval, staleThreshold time.Duration, authToken string, perModelHealthCheck bool, clientFactory ...BackendClientFactory) *HealthMonitor { checkInterval = cmp.Or(checkInterval, 15*time.Second) // Heartbeat checkpointing lets last_heartbeat sit up to one checkpoint @@ -189,7 +193,18 @@ func (hm *HealthMonitor) doCheckAll(ctx context.Context) { if m.Address == "" || m.Address == node.Address { continue } - mClient := hm.clientFactory.NewClient(m.Address, false) + // Through the node's tunnel, never a direct dial to m.Address: + // that address is a port inside the worker. A worker this + // replica cannot reach is not evidence that its backend died, + // so the miss counter is left alone and the row survives; + // counting it as a miss would reap live models across the whole + // fleet the moment the tunnel wiring was wrong. + mClient, err := hm.clientFactory.NewClientForNode(node.ID, m.Address, false) + if err != nil { + xlog.Error("Skipping model health probe: no way to reach the worker", + "node", node.ID, "model", m.ModelName, "replica", m.ReplicaIndex, "error", err) + continue + } mCheckCtx, mCancel := context.WithTimeout(ctx, 5*time.Second) ok, _ := mClient.HealthCheck(mCheckCtx) mCancel() diff --git a/core/services/nodes/health_mock_test.go b/core/services/nodes/health_mock_test.go index d8e74004ad14..0d05991d99d3 100644 --- a/core/services/nodes/health_mock_test.go +++ b/core/services/nodes/health_mock_test.go @@ -304,6 +304,11 @@ type fakeBackendClientFactory struct { clients map[string]*fakeBackendClient // default client returned when address not in clients map defaultClient *fakeBackendClient + // forNode records every node id NewClientForNode was asked for. + forNode []string + // refuseForNode makes NewClientForNode fail, standing in for a deployment + // with no way to reach the worker. Set before the code under test runs. + refuseForNode error } func newFakeBackendClientFactory() *fakeBackendClientFactory { @@ -328,6 +333,24 @@ func (f *fakeBackendClientFactory) NewClient(address string, _ bool) grpc.Backen return f.defaultClient } +// nodesSeen records the node ids the code under test asked for, so a spec can +// assert a caller reached a worker through its NODE rather than by address. +func (f *fakeBackendClientFactory) nodesSeen() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.forNode...) +} + +func (f *fakeBackendClientFactory) NewClientForNode(nodeID, address string, parallel bool) (grpc.Backend, error) { + if f.refuseForNode != nil { + return nil, f.refuseForNode + } + f.mu.Lock() + f.forNode = append(f.forNode, nodeID) + f.mu.Unlock() + return f.NewClient(address, parallel), nil +} + // helper to make a BackendNode with given properties func makeTestNode(id, name, address string, status string, lastHeartbeat time.Time) *BackendNode { return &BackendNode{ diff --git a/core/services/nodes/health_test.go b/core/services/nodes/health_test.go index c78ccfffe0d4..6f624b0cacb5 100644 --- a/core/services/nodes/health_test.go +++ b/core/services/nodes/health_test.go @@ -285,6 +285,46 @@ var _ = Describe("HealthMonitor (mock-based)", func() { Expect(store.getCalls()).NotTo(ContainElement(ContainSubstring("MarkUnhealthy"))) }) + It("probes a model through its NODE, never by dialling the stored address", func() { + // The address on a NodeModel row is a port inside the worker. This + // frontend reaches it over the worker's tunnel, so the node has to + // be part of every probe; a probe built from the address alone is + // the direct dial the tunnel replaces. + store := newFakeNodeHealthStore() + factory := newFakeBackendClientFactory() + hm := newTestHealthMonitor(store, factory, true, staleThreshold) + hm.perModelHealthCheck = true + + node := makeTestNode("node-tun", "tun-worker", "10.0.0.20:50051", StatusHealthy, freshTime()) + store.addNode(node) + store.addNodeModel("node-tun", NodeModel{NodeID: "node-tun", ModelName: "m", Address: "10.0.0.20:50053"}) + + hm.doCheckAll(context.Background()) + Expect(factory.nodesSeen()).To(ContainElement("node-tun")) + }) + + It("leaves a model row alone when it cannot reach the worker at all", func() { + // Not a miss. A frontend with no way to reach a worker has learned + // nothing about that worker's backends, and counting it as a failed + // probe would reap every model in the fleet the moment the tunnel + // wiring broke. + store := newFakeNodeHealthStore() + factory := newFakeBackendClientFactory() + factory.refuseForNode = fmt.Errorf("no tunnel for you") + hm := newTestHealthMonitor(store, factory, true, staleThreshold) + hm.perModelHealthCheck = true + + node := makeTestNode("node-cut", "cut-worker", "10.0.0.21:50051", StatusHealthy, freshTime()) + store.addNode(node) + store.addNodeModel("node-cut", NodeModel{NodeID: "node-cut", ModelName: "m", Address: "10.0.0.21:50053"}) + + for i := 0; i < perModelMissThreshold+1; i++ { + hm.doCheckAll(context.Background()) + } + Expect(store.getCalls()).NotTo(ContainElement(ContainSubstring("RemoveNodeModel"))) + Expect(store.getNode("node-cut").Status).To(Equal(StatusHealthy)) + }) + It("preserves model row when an intermittent failure is followed by a success", func() { store := newFakeNodeHealthStore() factory := newFakeBackendClientFactory() diff --git a/core/services/nodes/interfaces.go b/core/services/nodes/interfaces.go index aafa0e47f4d2..2baea85c4484 100644 --- a/core/services/nodes/interfaces.go +++ b/core/services/nodes/interfaces.go @@ -2,6 +2,9 @@ package nodes import ( "context" + "errors" + "fmt" + "net" "time" "github.com/mudler/LocalAI/core/services/messaging" @@ -146,20 +149,96 @@ type NodeManager interface { RemoveAllNodeModelReplicas(ctx context.Context, nodeID, modelName string) error } -// BackendClientFactory creates gRPC backend clients. +// WorkerDialerFor hands back the dial function for one worker's backend +// processes: the shape grpc.WithContextDialer wants, bound to a node. +// +// It is declared here rather than taken as a core/services/cluster type so that +// this package keeps no dependency on that one. cluster is a leaf and imports +// neither this package nor core/http; the wiring in core/application supplies +// (*cluster.WorkerDialer).GRPCDialerFor, which has exactly this shape. +type WorkerDialerFor func(nodeID string) func(ctx context.Context, addr string) (net.Conn, error) + +// WorkerNetDialerFor hands back the dial function for one worker's own HTTP +// server, in the shape http.Transport.DialContext and websocket.Dialer's +// NetDialContext want. (*cluster.WorkerDialer).DialerFor bound to the http tag +// has this shape. +type WorkerNetDialerFor func(nodeID string) func(ctx context.Context, network, addr string) (net.Conn, error) + +// ErrNoWorkerDialer reports that something tried to reach a worker without a +// way to reach it through the worker's tunnel. +// +// It is deliberately an ERROR and not a fallback to dialling the worker's +// advertised address. A worker that holds a tunnel need not listen on anything +// and may be behind NAT with no address to dial, so the fallback would work +// only where the tunnel was not needed: on a single-host developer setup, and +// nowhere the feature exists for. It is also not an absence error: nothing +// about it says the worker is gone. +var ErrNoWorkerDialer = errors.New("nodes: no worker tunnel dialer is configured, so this worker cannot be reached") + +// BackendClientFactory creates the gRPC clients this frontend uses to reach +// model backends running on worker nodes. +// +// There is ONE method, and that is the design rather than an omission. A +// direct-dial constructor alongside it would be reachable from every call site +// that has an address, which is all of them, and the whole of this change is +// that having an address is no longer enough to reach a backend. Callers that +// genuinely want a raw address call pkg/grpc directly and are visible as such. type BackendClientFactory interface { - NewClient(address string, parallel bool) grpc.Backend + // NewClientForNode reaches a backend process running on a WORKER, through + // that worker's tunnel. address names WHICH process on the worker; it is + // not somewhere this process connects to. + // + // It returns an error rather than a client that falls back to a direct + // dial, so that a deployment with no tunnel dialer fails where the mistake + // is instead of quietly reopening the bypass. + NewClientForNode(nodeID, address string, parallel bool) (grpc.Backend, error) } -// tokenClientFactory is the default BackendClientFactory that creates gRPC -// clients with an optional bearer token for distributed auth. +// tokenClientFactory is the BackendClientFactory for a deployment with no +// worker tunnel dialer, which is a misconfiguration rather than a mode. It +// refuses every request, loudly, and reaches no worker. +// +// It exists so that the components that take a factory have something to hold +// when none was wired, instead of a nil they would have to guard at every use. type tokenClientFactory struct { token string } -func (f *tokenClientFactory) NewClient(address string, parallel bool) grpc.Backend { - if f.token != "" { - return grpc.NewClientWithToken(address, parallel, nil, false, f.token) +// NewClientForNode refuses. See ErrNoWorkerDialer for why this is not a direct +// dial to address. The token this factory carries is the one a working dialer +// would have used, kept only so the misconfiguration is repairable by wiring a +// dialer rather than by also re-plumbing credentials. +func (f *tokenClientFactory) NewClientForNode(nodeID, address string, _ bool) (grpc.Backend, error) { + return nil, fmt.Errorf("reaching backend %q on node %q: %w", address, nodeID, ErrNoWorkerDialer) +} + +// tunnelClientFactory reaches a worker's backend processes through the worker's +// tunnel, and is what every distributed deployment uses. +type tunnelClientFactory struct { + token string + dialFor WorkerDialerFor +} + +// NewTunnelClientFactory returns the factory that reaches worker backends +// through dialFor. A nil dialFor is refused rather than degraded: this +// constructor exists to close the direct-dial bypass, and one that silently +// handed back a direct-dialling factory would reopen it for the whole process. +func NewTunnelClientFactory(token string, dialFor WorkerDialerFor) (BackendClientFactory, error) { + if dialFor == nil { + return nil, fmt.Errorf("building the worker backend client factory: %w", ErrNoWorkerDialer) + } + return &tunnelClientFactory{token: token, dialFor: dialFor}, nil +} + +func (f *tunnelClientFactory) NewClientForNode(nodeID, address string, parallel bool) (grpc.Backend, error) { + if nodeID == "" { + // Without a node there is no tunnel to pick, and the only thing left to + // do with the address would be to dial it. + return nil, fmt.Errorf("reaching backend %q: no node id: %w", address, ErrNoWorkerDialer) + } + dial := f.dialFor(nodeID) + if dial == nil { + return nil, fmt.Errorf("reaching backend %q on node %q: %w", address, nodeID, ErrNoWorkerDialer) } - return grpc.NewClient(address, parallel, nil, false) + return grpc.NewClientWithDialer(address, parallel, nil, false, f.token, dial), nil } diff --git a/core/services/nodes/local_stub_invalidator_test.go b/core/services/nodes/local_stub_invalidator_test.go index 00ed820dc6c2..444fefb6a9b2 100644 --- a/core/services/nodes/local_stub_invalidator_test.go +++ b/core/services/nodes/local_stub_invalidator_test.go @@ -44,7 +44,7 @@ var _ = Describe("LocalStubInvalidator", func() { }) It("drops the local stub once the last replica of the model is gone", func() { - store := NewDistributedModelStore(local, registry) + store := NewDistributedModelStore(local, registry, newFakeBackendClientFactory()) Expect(registry.SetNodeModel(context.Background(), nodeA.ID, "ghost-model", 0, "loaded", "10.0.0.1:12345", 0)).To(Succeed()) local.Set("ghost-model", model.NewModel("ghost-model", "10.0.0.1:12345", nil)) @@ -64,7 +64,7 @@ var _ = Describe("LocalStubInvalidator", func() { }) It("keeps the local stub while another replica still serves the model", func() { - store := NewDistributedModelStore(local, registry) + store := NewDistributedModelStore(local, registry, newFakeBackendClientFactory()) Expect(registry.SetNodeModel(context.Background(), nodeA.ID, "shared-model", 0, "loaded", "10.0.0.1:12345", 0)).To(Succeed()) Expect(registry.SetNodeModel(context.Background(), nodeB.ID, "shared-model", 0, "loaded", "10.0.0.2:12345", 0)).To(Succeed()) local.Set("shared-model", model.NewModel("shared-model", "10.0.0.1:12345", nil)) @@ -92,7 +92,7 @@ var _ = Describe("LocalStubInvalidator", func() { }) It("drops the local stub when a whole node's replicas are removed", func() { - store := NewDistributedModelStore(local, registry) + store := NewDistributedModelStore(local, registry, newFakeBackendClientFactory()) Expect(registry.SetNodeModel(context.Background(), nodeA.ID, "node-model", 0, "loaded", "10.0.0.1:12345", 0)).To(Succeed()) local.Set("node-model", model.NewModel("node-model", "10.0.0.1:12345", nil)) diff --git a/core/services/nodes/reconciler.go b/core/services/nodes/reconciler.go index a14a3fa70d5a..32feb5da54b8 100644 --- a/core/services/nodes/reconciler.go +++ b/core/services/nodes/reconciler.go @@ -12,7 +12,6 @@ import ( "github.com/mudler/LocalAI/core/services/advisorylock" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes/prefixcache" - grpcclient "github.com/mudler/LocalAI/pkg/grpc" "github.com/mudler/xlog" "github.com/nats-io/nats.go" "google.golang.org/grpc/codes" @@ -35,13 +34,28 @@ const ( // ProbeUnreachable: nothing is listening (connection refused), or the // backend answered and affirmatively reported itself unhealthy. ProbeUnreachable + // ProbeUnknown: the probe was never made, because this frontend has no way + // to reach the worker at all (no tunnel dialer wired, or none for this + // node). It is NOT ProbeUnreachable and must never be folded into it: + // unreachable is an observation about a backend and the reaper deletes rows + // on it, while this is a statement about THIS process and says nothing + // about the worker, which may be running the model perfectly well. + // + // It is appended rather than made the zero value on purpose. ProbeAlive is + // the zero value already, and renumbering the set would silently change the + // meaning of every stored or hard-coded outcome. + ProbeUnknown ) // ModelProber checks the state of a model's backend process. // Defaulted to a gRPC health probe but overridable for tests so we don't // need to stand up a real server. type ModelProber interface { - Probe(ctx context.Context, address string) ProbeOutcome + // Probe checks the backend at address on node nodeID. The node is needed + // as well as the address because address is a port INSIDE the worker, + // reached over the tunnel that worker holds, and there is no route to it + // that does not name the node. + Probe(ctx context.Context, nodeID, address string) ProbeOutcome } // NodeProcessLister asks a worker which model backend processes it currently @@ -61,11 +75,20 @@ type NodeProcessLister interface { // as death. const probeTimeout = 1 * time.Second -// grpcModelProber does a short HealthCheck on the model's stored gRPC address. -type grpcModelProber struct{ token string } +// grpcModelProber does a short HealthCheck on the model's stored gRPC address, +// through the tunnel of the node that address belongs to. +type grpcModelProber struct{ clients BackendClientFactory } -func (g grpcModelProber) Probe(ctx context.Context, address string) ProbeOutcome { - client := grpcclient.NewClientWithToken(address, false, nil, false, g.token) +func (g grpcModelProber) Probe(ctx context.Context, nodeID, address string) ProbeOutcome { + client, err := g.clients.NewClientForNode(nodeID, address, false) + if err != nil { + // Never ProbeUnreachable: the reaper deletes a row on that answer, and + // this frontend not being able to reach a worker is no evidence that + // the worker stopped running the model. + xlog.Error("Cannot probe a model: no way to reach the worker", + "node", nodeID, "address", address, "error", err) + return ProbeUnknown + } probeCtx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() ok, err := client.HealthCheck(probeCtx) @@ -177,11 +200,17 @@ type ReplicaReconcilerOptions struct { // Adapter is the NATS sender used to retry pending backend ops. When nil, // the state-reconciler pending-drain pass is a no-op (single-node mode). Adapter *RemoteUnloaderAdapter - // RegistrationToken is used by the default gRPC prober when probing model - // addresses. Matches the worker's token so HealthCheck auth succeeds. + // RegistrationToken is the bearer token the default gRPC prober presents to + // a worker's backends. It matters only when ClientFactory is unset, since + // the factory carries its own; a prober built from the token alone can + // reach no worker at all and reports ProbeUnknown for every model. RegistrationToken string // Prober overrides the default gRPC health probe (used by tests). Prober ModelProber + // ClientFactory builds the gRPC clients the default prober uses. It is what + // carries the worker tunnel dialer; without it the default prober can reach + // no worker and says so on every probe. + ClientFactory BackendClientFactory // ProcessLister overrides the default worker process query. When nil and // no Adapter is set, the worker-authoritative pass is skipped entirely and // only the port probe runs. @@ -220,7 +249,14 @@ func NewReplicaReconciler(opts ReplicaReconcilerOptions) *ReplicaReconciler { } prober := opts.Prober if prober == nil { - prober = grpcModelProber{token: opts.RegistrationToken} + clients := opts.ClientFactory + if clients == nil { + // No tunnel dialer was wired. The prober then refuses every probe + // with ProbeUnknown rather than dialling addresses directly, which + // is loud in the log and leaves every row alone. + clients = &tokenClientFactory{token: opts.RegistrationToken} + } + prober = grpcModelProber{clients: clients} } pressureThreshold := opts.PressureThreshold if pressureThreshold == 0 { @@ -480,7 +516,15 @@ func (rc *ReplicaReconciler) probeLoadedModels(ctx context.Context) { return } seen[m.ID] = struct{}{} - switch rc.prober.Probe(ctx, m.Address) { + switch rc.prober.Probe(ctx, m.NodeID, m.Address) { + case ProbeUnknown: + // This frontend could not reach the worker to ask. The streak is + // left exactly as it was: neither cleared, which would forgive a + // backend that really is dead, nor advanced, which would reap every + // model in the fleet the moment the tunnel wiring broke. + xlog.Warn("Reconciler: could not probe a model, leaving its row alone", + "node", m.NodeID, "model", m.ModelName, "replica", m.ReplicaIndex, "address", m.Address) + continue case ProbeAlive: rc.clearProbeFailures(m.ID) // Bump updated_at so we don't probe this row again immediately. @@ -563,7 +607,7 @@ func (rc *ReplicaReconciler) sweepLeakedInFlight(ctx context.Context) { return } seen[m.ID] = struct{}{} - if rc.prober.Probe(ctx, m.Address) != ProbeAlive { + if rc.prober.Probe(ctx, m.NodeID, m.Address) != ProbeAlive { // Busy or unreachable. Busy means the counter may well be real; // unreachable is the reaper's business, not the sweeper's. rc.clearInFlightIdle(m.ID) diff --git a/core/services/nodes/reconciler_busy_probe_test.go b/core/services/nodes/reconciler_busy_probe_test.go index 9cc7b07f9923..c2a6b076836c 100644 --- a/core/services/nodes/reconciler_busy_probe_test.go +++ b/core/services/nodes/reconciler_busy_probe_test.go @@ -89,6 +89,48 @@ var _ = Describe("ReplicaReconciler — probe reaper vs busy backends", func() { "a backend that accepted the connection but was mid-request must never be reaped") }) + It("never reaps a replica it could not probe at all", func() { + // ProbeUnknown is this FRONTEND saying it has no way to reach the + // worker, which is nothing at all about the backend. Folding it into + // ProbeUnreachable would empty the whole node_models table the moment + // the tunnel wiring was wrong, and the models would still be running. + seed("unknown-1", 0) + prober := &fakeProber{outcomes: map[string]ProbeOutcome{addr: ProbeUnknown}} + rc := newReconciler(prober) + + for range probeFailuresBeforeReap * 3 { + rc.probeLoadedModels(context.Background()) + makeStale("unknown-1") + } + + var after NodeModel + Expect(db.First(&after, "id = ?", "unknown-1").Error).To(Succeed(), + "a replica this frontend could not reach must never be reaped") + }) + + It("does not let an unprobeable pass forgive a real failure streak", func() { + // The other half of the same rule. Clearing the streak on an outcome + // that observed nothing would let a flapping tunnel keep a genuinely + // dead backend in the table forever. + seed("mixed-1", 0) + prober := &fakeProber{outcomes: map[string]ProbeOutcome{addr: ProbeUnreachable}} + rc := newReconciler(prober) + + for i := 1; i < probeFailuresBeforeReap; i++ { + rc.probeLoadedModels(context.Background()) + makeStale("mixed-1") + } + prober.outcomes[addr] = ProbeUnknown + rc.probeLoadedModels(context.Background()) + makeStale("mixed-1") + + prober.outcomes[addr] = ProbeUnreachable + rc.probeLoadedModels(context.Background()) + var after NodeModel + Expect(db.First(&after, "id = ?", "mixed-1").Error).To(MatchError(gorm.ErrRecordNotFound), + "an unprobeable pass must leave the streak untouched, not reset it") + }) + It("reaps an unreachable replica even when in_flight leaked high", func() { // in_flight has no decrement guarantee: a frontend that dies mid-request // leaves the increment behind forever. Gating the reaper on it would diff --git a/core/services/nodes/reconciler_test.go b/core/services/nodes/reconciler_test.go index 049fb94418de..1ed0b47d8ad7 100644 --- a/core/services/nodes/reconciler_test.go +++ b/core/services/nodes/reconciler_test.go @@ -740,7 +740,7 @@ type fakeProber struct { calls int } -func (f *fakeProber) Probe(_ context.Context, address string) ProbeOutcome { +func (f *fakeProber) Probe(_ context.Context, _, address string) ProbeOutcome { f.calls++ if f.outcomes == nil { return ProbeUnreachable diff --git a/core/services/nodes/revision_eligibility_test.go b/core/services/nodes/revision_eligibility_test.go index 01f3bfd37475..0cc3faf6f623 100644 --- a/core/services/nodes/revision_eligibility_test.go +++ b/core/services/nodes/revision_eligibility_test.go @@ -264,7 +264,7 @@ var _ = Describe("revision eligibility consumers", func() { type recordingEligibilityProber struct{ addresses []string } -func (p *recordingEligibilityProber) Probe(_ context.Context, address string) ProbeOutcome { +func (p *recordingEligibilityProber) Probe(_ context.Context, _, address string) ProbeOutcome { p.addresses = append(p.addresses, address) return ProbeAlive } diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index d024b562101e..88effc259765 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -403,7 +403,10 @@ func (r *SmartRouter) scheduleAndLoad(ctx context.Context, backendType, tracking } } - client := r.buildClientForAddr(node, backendAddr, parallel) + client, err := r.buildClientForAddr(node, backendAddr, parallel) + if err != nil { + return nil, fmt.Errorf("building a client for model %q on node %q: %w", modelName, node.ID, err) + } // Load the model on the remote node if loadOpts != nil { @@ -731,7 +734,20 @@ func (r *SmartRouter) tryWarmPath(ctx context.Context, att *routeAttempt) *Route replicaIdx := nm.ReplicaIndex // Verify the backend process is still alive via gRPC health check - if !r.probeHealth(ctx, node, modelAddr) { + alive, probed := r.probeHealth(ctx, node, modelAddr) + if !probed { + // Nothing was asked, so nothing was learned. The row is left exactly + // as it was: removing it would reclaim a model that is loaded and + // healthy on a worker this frontend merely cannot reach right now. The + // reservation is released, and the cold path below reports the wiring + // fault with the detail a caller needs. + if err := r.registry.DecrementInFlight(ctx, node.ID, att.trackingKey, replicaIdx); err != nil { + xlog.Warn("Failed to release a reservation for an unreachable worker", + "node", node.ID, "model", att.trackingKey, "replica", replicaIdx, "error", err) + } + return nil + } + if !alive { // Stale — roll back the increment, remove the specific replica row, fall through if err := r.registry.DecrementInFlight(ctx, node.ID, att.trackingKey, replicaIdx); err != nil { xlog.Warn("Failed to release stale routing reservation", @@ -763,7 +779,20 @@ func (r *SmartRouter) tryWarmPath(ctx context.Context, att *routeAttempt) *Route // call finishes, so in-flight returns to 0 when idle. r.registry.TouchNodeModel(ctx, node.ID, att.trackingKey, replicaIdx) r.observePrefix(att.trackingKey, att.observeChain, prefixcache.ReplicaKey{NodeID: node.ID, Replica: replicaIdx}) - grpcClient := r.buildClientForAddr(node, modelAddr, att.parallel) + grpcClient, err := r.buildClientForAddr(node, modelAddr, att.parallel) + if err != nil { + // The probe above builds a client for the same node and would have + // reported !probed, so reaching here means the dialer stopped being + // able to serve this node between the two. Handled the same way and for + // the same reason: release the reservation, leave the row alone. + if relErr := r.registry.DecrementInFlight(ctx, node.ID, att.trackingKey, replicaIdx); relErr != nil { + xlog.Warn("Failed to release a reservation for an unreachable worker", + "node", node.ID, "model", att.trackingKey, "replica", replicaIdx, "error", relErr) + } + xlog.Error("Cannot build a client for a loaded model: no way to reach the worker", + "node", node.ID, "model", att.trackingKey, "replica", replicaIdx, "error", err) + return nil + } tracked := NewInFlightTrackingClient(grpcClient, r.registry, node.ID, att.trackingKey, replicaIdx) return r.newRouteResult(node, att.trackingKey, replicaIdx, grpcClient, tracked) } @@ -1383,14 +1412,25 @@ func (r *SmartRouter) installBackendOnNode(ctx context.Context, node *BackendNod } } -func (r *SmartRouter) buildClientForAddr(node *BackendNode, addr string, parallel bool) grpc.Backend { - client := r.clientFactory.NewClient(addr, parallel) +// buildClientForAddr builds the gRPC client for a backend process running on a +// worker node. +// +// addr is a port INSIDE the worker, reached over the tunnel that worker holds; +// connecting to it from here would only work for a worker that still listens on +// a routable address. The factory offers no way to do that, and an error is +// returned rather than a direct-dialling client for the reason +// ErrNoWorkerDialer gives. +func (r *SmartRouter) buildClientForAddr(node *BackendNode, addr string, parallel bool) (grpc.Backend, error) { + client, err := r.clientFactory.NewClientForNode(node.ID, addr, parallel) + if err != nil { + return nil, err + } // Wrap with file staging if configured if r.fileStager != nil { - return NewFileStagingClient(client, r.fileStager, node.ID) + return NewFileStagingClient(client, r.fileStager, node.ID), nil } - return client + return client, nil } // stageModelFiles uploads model files to the backend node via the FileStager. @@ -1957,6 +1997,13 @@ func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir // via a gRPC health check with a 2-second timeout. The client is closed after // the check. // +// TWO results, not one. alive is what the backend said; probed is whether it +// was asked at all. They are separate because the caller REAPS on a dead probe, +// and a frontend that cannot reach a worker has observed nothing about that +// worker's backends: folding the two would delete every replica row in the +// deployment the moment the tunnel wiring was wrong, while the models carried +// on running. +// // The result is memoized in r.probeCache for probeCacheTTL. With per-request // routing every inference call lands here, and unbounded re-probing can stall // behind a busy backend that serializes HealthCheck against active Predict. @@ -1964,16 +2011,27 @@ func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir // burst of N requests for a cold cache costs at most one round-trip, not N. // Failed probes invalidate the cache so the staleness recovery path // (DecrementInFlight + RemoveNodeModel) still triggers on the next request. -func (r *SmartRouter) probeHealth(ctx context.Context, node *BackendNode, addr string) bool { +// +// The client is built OUTSIDE the memoized closure, which is what keeps an +// unreachable worker out of the cache entirely: DoOrCached only ever sees a +// real answer. Building it costs a struct and no I/O, since the gRPC client +// dials lazily on its first call. +func (r *SmartRouter) probeHealth(ctx context.Context, node *BackendNode, addr string) (alive, probed bool) { + client, err := r.buildClientForAddr(node, addr, false) + if err != nil { + xlog.Error("Cannot probe a model backend: no way to reach the worker", + "node", node.ID, "address", addr, "error", err) + return false, false + } + defer closeClient(client) + key := node.ID + "|" + addr return r.probeCache.DoOrCached(key, func() bool { - client := r.buildClientForAddr(node, addr, false) - defer closeClient(client) checkCtx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() ok, _ := client.HealthCheck(checkCtx) return ok - }) + }), true } // closeClient closes a gRPC backend client if it implements io.Closer. diff --git a/core/services/nodes/router_load_budget_test.go b/core/services/nodes/router_load_budget_test.go index 921b19905d4a..f0f41f78c235 100644 --- a/core/services/nodes/router_load_budget_test.go +++ b/core/services/nodes/router_load_budget_test.go @@ -86,6 +86,10 @@ type holdClientFactory struct{ client *holdBackend } func (f *holdClientFactory) NewClient(_ string, _ bool) grpc.Backend { return f.client } +func (f *holdClientFactory) NewClientForNode(_, address string, parallel bool) (grpc.Backend, error) { + return f.NewClient(address, parallel), nil +} + var _ = Describe("size-derived remote LoadModel budget", func() { // Production, on an NVIDIA Jetson Thor worker: a 70 GB video checkpoint // (longcat-video-avatar-1.5) failed reproducibly after 953.5s with diff --git a/core/services/nodes/router_load_timeout_test.go b/core/services/nodes/router_load_timeout_test.go index 295dc35d81fd..7ee68ac123e6 100644 --- a/core/services/nodes/router_load_timeout_test.go +++ b/core/services/nodes/router_load_timeout_test.go @@ -53,6 +53,10 @@ type deadlineClientFactory struct{ client *deadlineBackend } func (f *deadlineClientFactory) NewClient(_ string, _ bool) grpc.Backend { return f.client } +func (f *deadlineClientFactory) NewClientForNode(_, address string, parallel bool) (grpc.Backend, error) { + return f.NewClient(address, parallel), nil +} + var _ = Describe("remote LoadModel deadline", func() { var ( reg *fakeModelRouter diff --git a/core/services/nodes/router_reap_load_test.go b/core/services/nodes/router_reap_load_test.go index 67376c06f535..b2619dc97f0a 100644 --- a/core/services/nodes/router_reap_load_test.go +++ b/core/services/nodes/router_reap_load_test.go @@ -45,6 +45,10 @@ type failingClientFactory struct{ client *failingLoadBackend } func (f *failingClientFactory) NewClient(_ string, _ bool) grpc.Backend { return f.client } +func (f *failingClientFactory) NewClientForNode(_, address string, parallel bool) (grpc.Backend, error) { + return f.NewClient(address, parallel), nil +} + // replicaSlotRouter pins the replica slot scheduleAndLoad allocates so a spec // can assert the reaped process key carries the real index, not a hardcoded 0. type replicaSlotRouter struct { diff --git a/core/services/nodes/router_test.go b/core/services/nodes/router_test.go index 3735ec5e4532..dbc5bd102e28 100644 --- a/core/services/nodes/router_test.go +++ b/core/services/nodes/router_test.go @@ -480,6 +480,10 @@ func (f *stubClientFactory) NewClient(_ string, _ bool) grpc.Backend { return f.client } +func (f *stubClientFactory) NewClientForNode(_, address string, parallel bool) (grpc.Backend, error) { + return f.NewClient(address, parallel), nil +} + // --------------------------------------------------------------------------- // Fake NodeCommandSender (unloader) // --------------------------------------------------------------------------- diff --git a/core/services/nodes/router_unreachable_worker_test.go b/core/services/nodes/router_unreachable_worker_test.go new file mode 100644 index 000000000000..943741396e11 --- /dev/null +++ b/core/services/nodes/router_unreachable_worker_test.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: MIT + +package nodes + +import ( + "context" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + grpc "github.com/mudler/LocalAI/pkg/grpc" +) + +// unreachableClientFactory cannot build a client for any node, standing in for +// a frontend whose worker tunnel dialer is missing or broken. +type unreachableClientFactory struct{} + +func (unreachableClientFactory) NewClientForNode(_, _ string, _ bool) (grpc.Backend, error) { + return nil, errors.New("no way to reach that worker") +} + +var _ = Describe("routing when the worker cannot be reached at all", func() { + // The catastrophe this phase exists to prevent, at the router. A frontend + // that cannot reach a worker has learned NOTHING about that worker's + // backends. Treating it as a failed health probe would reap the replica row + // for every model in the deployment while those models carried on running, + // and the reap is silent: the row is simply deleted and the model + // cold-loaded somewhere else. + loadedReg := func() *fakeModelRouter { + node := &BackendNode{ID: "X", Name: "node-x", Address: "10.0.0.1:50051"} + nm := &NodeModel{NodeID: "X", ModelName: "m", Address: "10.0.0.1:9001"} + return &fakeModelRouter{ + findAndLockNode: node, + findAndLockNM: nm, + loadedReplicaStatsByName: map[string][]ReplicaCandidate{"m": {{NodeID: "X", InFlight: 0}}}, + } + } + + It("never removes the replica row of a worker it merely cannot reach", func() { + reg := loadedReg() + router := NewSmartRouter(reg, SmartRouterOptions{ + Unloader: &fakeUnloader{}, + ClientFactory: unreachableClientFactory{}, + }) + + _, err := router.Route(context.Background(), "m", "models/m.gguf", "llama-cpp", "", nil, false) + // The request cannot be served, which is right and loud. + Expect(err).To(HaveOccurred()) + // What must NOT have happened is the replica being reclaimed. + Expect(reg.removeCalls).To(BeEmpty(), + "a worker this frontend cannot reach must never have its loaded models reaped") + }) + + It("releases the routing reservation it took before giving up", func() { + // FindAndLockNodeWithModel increments in_flight as a reservation. A + // path that returns without releasing it leaves the replica looking + // permanently busy, which is how a warm replica stops being picked at + // all. + reg := loadedReg() + router := NewSmartRouter(reg, SmartRouterOptions{ + Unloader: &fakeUnloader{}, + ClientFactory: unreachableClientFactory{}, + }) + + _, err := router.Route(context.Background(), "m", "models/m.gguf", "llama-cpp", "", nil, false) + Expect(err).To(HaveOccurred()) + Expect(reg.decrementCalls).To(ContainElement("X:m")) + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 56c0b906fc0f..288091f8047d 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -145,6 +145,37 @@ A stream naming a tag the worker does not serve, a target outside that port rang Reconnects use exponential backoff with jitter: the interval doubles from 500ms up to a ceiling of 30 seconds, and each wait is drawn between half of that interval and all of it, so no worker ever spins and a fleet that lost the same replica does not come back in lockstep. The interval returns to its floor only after a session that lasted at least 30 seconds. That last part is what stops a rolling frontend restart, where every dial succeeds and then dies moments later, from turning a fleet of workers into a retry storm against the first replica back up. A worker that is refused (`401`, `403`) keeps retrying on the same schedule rather than exiting: a re-registration or an admin approval fixes both without restarting it. +#### What the frontend sends through it + +Every connection the frontend makes to a worker now goes through that worker's tunnel. There are three, and all three are the same path underneath: + +| What | Protocol | Stream tag | +|------|----------|-----------| +| Inference, model load, health checks | gRPC to a backend process | `grpc` | +| Model file staging, backend-log listing | HTTP to the worker's own server | `http` | +| Live backend-log streaming | WebSocket to the same server | `http` | + +The address the frontend holds for a backend (the per-replica port a worker reports after an install) is still what identifies it, and it is still what appears in logs and errors. What it no longer is, is somewhere the frontend connects to: it travels inside the tunnel as the stream's target, and the worker decides what to do with it. + +A frontend with no way to reach a worker says so and fails. It does **not** fall back to connecting to the worker's advertised address. That fallback is what the tunnel exists to remove, and it is the kind of defect that works on a one-replica developer box and fails in production, so it is an error everywhere. The consequences are deliberately narrow: a model whose worker cannot be reached is not reaped, and its row is left alone, because a frontend that cannot reach a worker has learned nothing about whether that worker is still running the model. + +#### Reaching a worker another replica holds + +A worker's tunnel lands on exactly one replica, so with N replicas behind a load balancer roughly (N-1)/N of requests arrive somewhere else. Those requests are relayed: the replica that received the request looks up the owner in `node_connections`, **joined against the live `instances` rows**, opens a stream on its peer link to that owner, and the owner splices it onto the worker's tunnel. One hop, never two; a stale ownership row is answered with a routing refusal and the dialling replica resolves the owner again rather than being sent round a loop. + +The dialling replica states how much time its own client has left in the frame that opens the relayed stream, and the owner bounds its work by the smaller of that and its own 15s ceiling. Neither number can lengthen the other: a patient client cannot park the owning replica, and an impatient one cannot be kept waiting on a budget it did not ask for. + +Four outcomes are kept apart on purpose, because they call for different actions: + +| Outcome | What it means | What acts on it | +|---|---|---| +| No live owner | No replica holds this worker's tunnel | The worker is treated as absent; its models can be rescheduled | +| Not the owner | The routing was stale | Resolve the owner again | +| Peer unreachable | A replica exists and will not answer | Retry | +| The worker refused | The worker answered and said no | Report; the worker is connected | + +Only the first is absence. The others are never reported as it, and that is not a stylistic preference: a scheduler told that a connected worker has gone away reclaims every model it is running. + Set `LOCALAI_WORKER_TUNNEL=false` on a worker to turn the tunnel off and go back to the frontend dialling the worker's advertised addresses. ### The model load deadline scales with the checkpoint diff --git a/pkg/grpc/backend.go b/pkg/grpc/backend.go index ac5b95f28efe..866721d2cf8a 100644 --- a/pkg/grpc/backend.go +++ b/pkg/grpc/backend.go @@ -2,6 +2,7 @@ package grpc import ( "context" + "net" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "google.golang.org/grpc" @@ -29,7 +30,30 @@ func NewClientWithToken(address string, parallel bool, wd WatchDog, enableWatchD return buildClient(address, parallel, wd, enableWatchDog, token) } -func buildClient(address string, parallel bool, wd WatchDog, enableWatchDog bool, token string) Backend { +// NewClientWithDialer creates a gRPC client that reaches its backend through +// dialer rather than by connecting to address. +// +// It is what distributed mode uses to reach a backend process on a worker: the +// worker holds one multiplexed tunnel to a frontend replica and listens on +// nothing, so address names which backend process the stream is for and the +// dialer decides how the stream gets there. A nil dialer is a programming +// error on this path rather than a fallback, because falling back to a direct +// dial would work in a single-replica test and fail in production; callers with +// no dialer call NewClientWithToken and mean it. +func NewClientWithDialer(address string, parallel bool, wd WatchDog, enableWatchDog bool, token string, dialer func(ctx context.Context, addr string) (net.Conn, error)) Backend { + if bc, ok := embeds[address]; ok { + return bc + } + // Assigned on the concrete type rather than through a checked assertion: + // an assertion that failed would silently hand back a client that dials + // the address directly, which is the exact bypass this constructor exists + // to close. + c := buildClient(address, parallel, wd, enableWatchDog, token) + c.dialer = dialer + return c +} + +func buildClient(address string, parallel bool, wd WatchDog, enableWatchDog bool, token string) *Client { if !enableWatchDog { wd = nil } diff --git a/pkg/grpc/client.go b/pkg/grpc/client.go index f46b9dcf03a7..ba6fd0e503db 100644 --- a/pkg/grpc/client.go +++ b/pkg/grpc/client.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "net" "sync" "time" @@ -32,6 +33,13 @@ type Client struct { inFlight int parallel bool token string + // dialer replaces the transport gRPC would otherwise use to reach address. + // In distributed mode it is a stream on the worker's tunnel, so address + // stops being a socket to connect to and becomes the name of a backend + // process inside the worker; see core/services/cluster.WorkerDialer. nil + // keeps gRPC's own TCP dial, which is what every non-distributed caller + // wants. + dialer func(ctx context.Context, addr string) (net.Conn, error) sync.Mutex opMutex sync.Mutex wd WatchDog @@ -80,6 +88,12 @@ func (c *Client) dial() (*grpc.ClientConn, error) { if c.token != "" { opts = append(opts, grpc.WithPerRPCCredentials(bearerToken{token: c.token})) } + if c.dialer != nil { + // The address is still passed to grpc.NewClient because it is what + // names the target in every error message and in the authority header; + // what it no longer decides is where the bytes go. + opts = append(opts, grpc.WithContextDialer(c.dialer)) + } return grpc.NewClient(c.address, opts...) } diff --git a/tests/e2e/distributed/backend_logs_test.go b/tests/e2e/distributed/backend_logs_test.go index 82e8ac156401..a858629ef464 100644 --- a/tests/e2e/distributed/backend_logs_test.go +++ b/tests/e2e/distributed/backend_logs_test.go @@ -50,6 +50,17 @@ func waitForSingleLogSubscriber(logStore *model.BackendLogStore, modelID string) Should(Equal(1), "the WebSocket handler never subscribed to %q exactly once", modelID) } +// directWorkerDialerFor stands in for the worker tunnel in these specs. +// +// The log-proxy endpoints reach a worker over the tunnel that worker holds, and +// refuse to reach one without a dialer. These specs run the worker's HTTP +// server on loopback, so a plain TCP dial is the stand-in; production supplies +// the real one from core/application. +func directWorkerDialerFor(_ string) func(ctx context.Context, network, addr string) (net.Conn, error) { + var d net.Dialer + return d.DialContext +} + var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func() { Context("Worker HTTP log endpoints", func() { @@ -370,7 +381,7 @@ var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func // Create an Echo test server with the proxy endpoint e := echo.New() - e.GET("/api/nodes/:id/backend-logs", localai.NodeBackendLogsListEndpoint(registry, token)) + e.GET("/api/nodes/:id/backend-logs", localai.NodeBackendLogsListEndpoint(registry, token, directWorkerDialerFor)) req := httptest.NewRequest("GET", fmt.Sprintf("/api/nodes/%s/backend-logs", node.ID), nil) rec := httptest.NewRecorder() @@ -392,7 +403,7 @@ var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func Expect(registry.Register(context.Background(), node, true)).To(Succeed()) e := echo.New() - e.GET("/api/nodes/:id/backend-logs/:modelId", localai.NodeBackendLogsLinesEndpoint(registry, token)) + e.GET("/api/nodes/:id/backend-logs/:modelId", localai.NodeBackendLogsLinesEndpoint(registry, token, directWorkerDialerFor)) req := httptest.NewRequest("GET", fmt.Sprintf("/api/nodes/%s/backend-logs/remote-model", node.ID), nil) rec := httptest.NewRecorder() @@ -409,7 +420,7 @@ var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func It("should return 404 for unknown node ID", func() { e := echo.New() - e.GET("/api/nodes/:id/backend-logs", localai.NodeBackendLogsListEndpoint(registry, token)) + e.GET("/api/nodes/:id/backend-logs", localai.NodeBackendLogsListEndpoint(registry, token, directWorkerDialerFor)) req := httptest.NewRequest("GET", "/api/nodes/nonexistent-id/backend-logs", nil) rec := httptest.NewRecorder() @@ -453,7 +464,7 @@ var _ = Describe("Distributed Backend Log Streaming", Label("Distributed"), func // Start Echo server with the WebSocket proxy route e := echo.New() - e.GET("/ws/nodes/:id/backend-logs/:modelId", localai.NodeBackendLogsWSEndpoint(registry, token)) + e.GET("/ws/nodes/:id/backend-logs/:modelId", localai.NodeBackendLogsWSEndpoint(registry, token, directWorkerDialerFor)) lis, err := net.Listen("tcp", "127.0.0.1:0") Expect(err).ToNot(HaveOccurred()) diff --git a/tests/e2e/distributed/distributed_full_flow_test.go b/tests/e2e/distributed/distributed_full_flow_test.go index ad7f2669aaf0..de26573790df 100644 --- a/tests/e2e/distributed/distributed_full_flow_test.go +++ b/tests/e2e/distributed/distributed_full_flow_test.go @@ -489,7 +489,7 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() return "", err } return n.HTTPAddress, nil - }, "") + }, "", directWorkerDialerFor) // Create SmartRouter with the HTTPFileStager router := newTestSmartRouter(registry, nodes.SmartRouterOptions{FileStager: stager}) @@ -558,7 +558,7 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() return "", err } return n.HTTPAddress, nil - }, "") + }, "", directWorkerDialerFor) // Create SmartRouter with FileStager router := newTestSmartRouter(registry, nodes.SmartRouterOptions{FileStager: stager}) @@ -616,7 +616,7 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() return "", err } return n.HTTPAddress, nil - }, "") + }, "", directWorkerDialerFor) // Test AllocRemoteTemp + FetchRemote directly (the output retrieval path) remoteTmpPath, err := stager.AllocRemoteTemp(ctx, node.ID) @@ -662,7 +662,7 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() return "", err } return n.HTTPAddress, nil - }, "") + }, "", directWorkerDialerFor) router := newTestSmartRouter(registry, nodes.SmartRouterOptions{FileStager: stager}) @@ -881,7 +881,7 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() return "", err } return n.HTTPAddress, nil - }, "") + }, "", directWorkerDialerFor) // Create model files on the "frontend" frontendModelsDir := GinkgoT().TempDir() @@ -965,7 +965,7 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() return "", err } return n.HTTPAddress, nil - }, "") + }, "", directWorkerDialerFor) // Create model files: .onnx and .onnx.json in a temp "models" dir frontendModelsDir := GinkgoT().TempDir() diff --git a/tests/e2e/distributed/distributed_store_test.go b/tests/e2e/distributed/distributed_store_test.go index 679a75a02b28..d35a7e0439a8 100644 --- a/tests/e2e/distributed/distributed_store_test.go +++ b/tests/e2e/distributed/distributed_store_test.go @@ -2,6 +2,7 @@ package distributed_test import ( "context" + "net" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/pkg/model" @@ -14,6 +15,25 @@ import ( "gorm.io/gorm/logger" ) +// directBackendClients stands in for the worker tunnel in these specs. +// +// The store refuses to build a client for a remote model without a way to reach +// the worker, which is the point: a model built with no client dials its raw +// address on first use. These specs have no worker tunnel and no worker, so the +// dial is a plain TCP one; production supplies the real dialer from +// core/application. +func directBackendClients() nodes.BackendClientFactory { + GinkgoHelper() + clients, err := nodes.NewTunnelClientFactory("", func(string) func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + return func(ctx context.Context, addr string) (net.Conn, error) { + return d.DialContext(ctx, "tcp", addr) + } + }) + Expect(err).ToNot(HaveOccurred()) + return clients +} + var _ = Describe("DistributedModelStore", Label("Distributed"), func() { var ( infra *TestInfra @@ -36,7 +56,7 @@ var _ = Describe("DistributedModelStore", Label("Distributed"), func() { Expect(err).ToNot(HaveOccurred()) localStore = model.NewInMemoryModelStore() - dStore = nodes.NewDistributedModelStore(localStore, registry) + dStore = nodes.NewDistributedModelStore(localStore, registry, directBackendClients()) }) Context("Get", func() { diff --git a/tests/e2e/distributed/file_staging_test.go b/tests/e2e/distributed/file_staging_test.go index 55bd5663c6b4..e765d87c3b87 100644 --- a/tests/e2e/distributed/file_staging_test.go +++ b/tests/e2e/distributed/file_staging_test.go @@ -62,7 +62,7 @@ var _ = Describe("File Staging", Label("Distributed"), func() { It("should create HTTPFileStager with httpAddrFor function", func() { stager := nodes.NewHTTPFileStager(func(nodeID string) (string, error) { return "", fmt.Errorf("no such node: %s", nodeID) - }, "") + }, "", directWorkerDialerFor) Expect(stager).ToNot(BeNil()) // Should fail gracefully when node resolution fails diff --git a/tests/e2e/distributed/prefix_cache_routing_test.go b/tests/e2e/distributed/prefix_cache_routing_test.go index 9b1e3c117718..e899160ea2fd 100644 --- a/tests/e2e/distributed/prefix_cache_routing_test.go +++ b/tests/e2e/distributed/prefix_cache_routing_test.go @@ -51,6 +51,10 @@ func (f *prefixStubClientFactory) NewClient(_ string, _ bool) grpcPkg.Backend { return f.client } +func (f *prefixStubClientFactory) NewClientForNode(_, _ string, _ bool) (grpcPkg.Backend, error) { + return f.client, nil +} + var _ = Describe("Prefix-cache aware routing", Label("Distributed"), func() { const model = "model" From d26d8b915b4626dc07ba0e807e5f7c1320f2aa3d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 16:09:26 +0000 Subject: [PATCH 055/153] fix(cluster): make "no route" a condition of its own, and let it out of the package Review round 1 on task 6. Five blocking findings, all with the same root: the conditions the dialer kept apart were erased one layer out, because every one of them arrived at core/services/nodes as a gRPC codes.Unavailable, which is also what a backend process that died produces. Four call sites acted on that by deleting a replica row, one of them after a single failed probe. The fifth condition is ErrNoRoute: this replica could not get a request to a worker's backend, and no claim at all about the worker. A worker's presence is its HEARTBEAT, which nodes owns; a route is a separate fact that cluster owns, and the two now differ. They differ in normal operation, not exotically: a worker that has not dialled its tunnel yet after a frontend-first upgrade is unroutable on every request while it heartbeats and serves. Two properties, both mutation-tested. Every failure to resolve or open a route carries ErrNoRoute, so a consumer has one check to make. No failure carries an absence sentinel: routeFailure is the single place that rule lives, and it keeps ErrNoConnection and ErrInstanceNotFound in the message and out of the unwrap chain, the guarantee unreachableError already made for peers. Everything else stays matchable, so ErrNotOwner and ErrPeerUnreachable are unchanged for anyone who can act on them. A worker's own refusal carries no umbrella, because a worker that answers has demonstrated it is there and that is the only real evidence on the path. Crossing the boundary needed a value, not a code. NewClientWithDialer wraps the dialer and records each outcome; LastDialError hands it back behind a narrow interface, and nodes.unroutable turns it into ErrWorkerUnroutable with the cluster sentinels still in the chain. A spec asserts a dial failing with ErrNoRoute plus ErrPeerUnreachable arrives matching all three and matching neither absence sentinel. The sweep found a fourth site the review had not named: pkg/model checkIsLoaded evicts a remote model on a connection error, and a tunnel dial failure is one. Four other reap sites were cleared with reasons - inflight and the worker authoritative pass reap only on semantic answers, scale-down is driven by last_used, abandoned loads decide on the node's heartbeat. Every fixed site also grew the opposite spec, so the new check cannot pass by never reaping. probeCache carries the reason through singleflight rather than a closed-over variable. A variable is only written by the goroutine that runs the probe, so the leader would correctly decline to reap while every joiner reaped on the leader's own observation; a mutation reproduces exactly that. The docs sentence promising LOCALAI_WORKER_TUNNEL=false restores direct dialling is gone. There is no such path, so it said the operator could take a worker dark and call it a rollback. Replaced with the upgrade order that is actually safe. The deadline spec the reviewer found vacuous now waits on the dial context's own Done channel before touching the stream, so the armed deadline has really expired; the mutation that survived for the reviewer reddens it. Nine mutations, each reddening a named spec, including both halves of isAbsenceClaim independently. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 10 +- core/http/endpoints/localai/nodes.go | 35 ++-- core/services/cluster/dialer.go | 192 ++++++++++++++---- core/services/cluster/dialer_test.go | 129 +++++++++--- core/services/cluster/relay.go | 16 ++ .../nodes/backend_client_factory_test.go | 32 +++ core/services/nodes/file_stager_http.go | 9 + core/services/nodes/health.go | 14 ++ core/services/nodes/health_mock_test.go | 10 + core/services/nodes/health_test.go | 49 +++++ core/services/nodes/interfaces.go | 100 ++++++++- core/services/nodes/probe_cache.go | 29 ++- core/services/nodes/reconciler.go | 19 +- core/services/nodes/reconciler_prober_test.go | 108 ++++++++++ core/services/nodes/router.go | 31 ++- .../nodes/router_unreachable_worker_test.go | 120 ++++++++++- docs/content/features/distributed-mode.md | 13 +- pkg/grpc/backend.go | 21 +- pkg/grpc/client.go | 45 ++++ pkg/model/loader.go | 32 +++ pkg/model/remote_unroutable_internal_test.go | 61 ++++++ 21 files changed, 969 insertions(+), 106 deletions(-) create mode 100644 core/services/nodes/reconciler_prober_test.go create mode 100644 pkg/model/remote_unroutable_internal_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index eab8cfca5aac..fb608a203b67 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -387,10 +387,12 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade if err != nil { return "", err } - if node.HTTPAddress == "" { - return "", fmt.Errorf("node %s has no HTTP address for file transfer", nodeID) - } - return node.HTTPAddress, nil + // An empty HTTPAddress is no longer a refusal. A tunnel-only worker + // reports none and does not need one: the http stream tag ignores + // the target and the worker routes to its own server. The host is + // only ever the URL's host component here, and WorkerHTTPHost + // supplies one that resolves nowhere so it cannot become a dial. + return nodes.WorkerHTTPHost(nodeID, node.HTTPAddress), nil }, cfg.Distributed.RegistrationToken, workerHTTPDialer) xlog.Info("File stager initialized (HTTP direct transfer)") } diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index 5a2ae3573d1b..56b2b8743a99 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -820,11 +820,11 @@ func NodeBackendLogsListEndpoint(registry *nodes.NodeRegistry, registrationToken return c.JSON(http.StatusNotFound, nodeError(http.StatusNotFound, "node not found")) } - if node.HTTPAddress == "" { - return c.JSON(http.StatusBadGateway, nodeError(http.StatusBadGateway, "node has no HTTP address")) - } - - resp, err := proxyHTTPToWorker(ctx, dialFor, nodeID, node.HTTPAddress, "/v1/backend-logs", registrationToken) + // No HTTPAddress guard: a tunnel-only worker reports none, and the + // http stream tag ignores the target anyway. WorkerHTTPHost fills the + // URL's host with something that identifies the node and resolves + // nowhere; the tunnel decides where the bytes go. + resp, err := proxyHTTPToWorker(ctx, dialFor, nodeID, nodes.WorkerHTTPHost(nodeID, node.HTTPAddress), "/v1/backend-logs", registrationToken) if err != nil { return c.JSON(http.StatusBadGateway, nodeError(http.StatusBadGateway, fmt.Sprintf("failed to reach worker: %v", err))) } @@ -850,12 +850,8 @@ func NodeBackendLogsLinesEndpoint(registry *nodes.NodeRegistry, registrationToke return c.JSON(http.StatusNotFound, nodeError(http.StatusNotFound, "node not found")) } - if node.HTTPAddress == "" { - return c.JSON(http.StatusBadGateway, nodeError(http.StatusBadGateway, "node has no HTTP address")) - } - path := "/v1/backend-logs/" + url.PathEscape(modelID) - resp, err := proxyHTTPToWorker(ctx, dialFor, nodeID, node.HTTPAddress, path, registrationToken) + resp, err := proxyHTTPToWorker(ctx, dialFor, nodeID, nodes.WorkerHTTPHost(nodeID, node.HTTPAddress), path, registrationToken) if err != nil { return c.JSON(http.StatusBadGateway, nodeError(http.StatusBadGateway, fmt.Sprintf("failed to reach worker: %v", err))) } @@ -907,7 +903,7 @@ func NodeBackendLogsWSEndpoint(registry *nodes.NodeRegistry, registrationToken s // NetDialContext below is what decides where the connection goes. A // missing dialer is a failure, not a direct dial: see // nodes.ErrNoWorkerDialer. - workerURL := fmt.Sprintf("ws://%s/v1/backend-logs/%s/ws", node.HTTPAddress, url.PathEscape(modelID)) + workerURL := fmt.Sprintf("ws://%s/v1/backend-logs/%s/ws", nodes.WorkerHTTPHost(nodeID, node.HTTPAddress), url.PathEscape(modelID)) workerHeaders := http.Header{} if registrationToken != "" { workerHeaders.Set("Authorization", "Bearer "+registrationToken) @@ -918,8 +914,21 @@ func NodeBackendLogsWSEndpoint(registry *nodes.NodeRegistry, registrationToken s workerDial = dialFor(nodeID) } if workerDial == nil { - return c.JSON(http.StatusBadGateway, nodeError(http.StatusBadGateway, - fmt.Sprintf("cannot reach node %s: %v", nodeID, nodes.ErrNoWorkerDialer))) + // A JSON body cannot be written here: the response writer was + // hijacked by the upgrade above, so the status line is long gone + // and the write lands nowhere. The browser has to be told the same + // way every other failure past the upgrade tells it, with a close + // frame, and the socket has to be closed or it leaks for the life + // of the process. + // Best-effort: the browser may already have gone, and there is + // nothing left to report the failure to either way. The CLOSE is + // what matters and it is unconditional. + _ = browserWS.WriteMessage(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseInternalServerErr, "no route to worker")) + _ = browserWS.Close() + xlog.Error("Cannot stream backend logs: no way to reach the worker", + "node", nodeID, "error", nodes.ErrNoWorkerDialer) + return nil } workerDialer := websocket.Dialer{HandshakeTimeout: 10 * time.Second, NetDialContext: workerDial} diff --git a/core/services/cluster/dialer.go b/core/services/cluster/dialer.go index b4a04551a973..fbb95d583acc 100644 --- a/core/services/cluster/dialer.go +++ b/core/services/cluster/dialer.go @@ -34,6 +34,102 @@ type PeerOpener interface { // and core/services/nodes reclaims the models of a worker it believes absent. var ErrNoRelayPath = errors.New("cluster: this replica cannot relay to the owner of that worker") +// ErrNoRoute reports that this replica could not get a request to a worker's +// backend, and is the FIFTH condition this phase keeps apart. +// +// It says nothing about whether the worker exists or is running. A worker's +// PRESENCE is its heartbeat, which lives in core/services/nodes and which this +// package cannot see; what this package can see is whether a route exists right +// now, and those are different questions with different answers. A worker that +// is registered, heartbeating and serving models can be unroutable from here +// for a whole list of ordinary reasons: it has not dialled its tunnel yet after +// a frontend-first upgrade, the replica holding its tunnel is restarting, the +// ownership row is a moment stale, this replica has no peer mesh. +// +// Every failure to RESOLVE OR OPEN a route carries it, so a consumer that must +// not act on absence has exactly one check to make. The specific condition +// stays in the unwrap chain underneath for anyone that can act on it, with one +// deliberate exception: see routeFailure. +// +// It is not carried by a REFUSAL from the worker itself. A worker that answers +// is present by demonstration, and folding its answer into "no route" would +// throw away the one thing on this path that is real evidence. +var ErrNoRoute = errors.New("cluster: no route from this replica to that worker") + +// noRouteError reports a worker this replica cannot route to, keeping the cause +// in its message and OUT of its unwrap chain. +// +// Withholding the cause is the entire point, and it is the same guarantee +// unreachableError makes for peers. The causes this is built over are absence +// claims: ErrNoConnection ("no live replica holds this worker's tunnel") and +// ErrInstanceNotFound ("no such frontend replica"). Both are true statements +// about the CLUSTER and neither is a statement about the worker, but a consumer +// matching on them would read them as one, and the consequence is the +// catastrophe this phase is built around: a scheduler concludes a worker that +// is heartbeating and serving has gone away, and reclaims its models. +// +// The guarantee therefore belongs to the type. There is no path by which an +// absence sentinel gets out, so no call site can leak one. +type noRouteError struct { + nodeID string + cause error +} + +func (e *noRouteError) Error() string { + return fmt.Sprintf("cluster: no route from this replica to node %q: %v", e.nodeID, e.cause) +} + +// Unwrap reports only ErrNoRoute. The cause reaches a human through Error() and +// reaches no error-matching caller at all. +func (e *noRouteError) Unwrap() error { return ErrNoRoute } + +// routeFailure is the ONE place a Dial failure is turned into an error, and the +// one place the absence rule is expressed. +// +// The rule: an absence claim never reaches a caller, and everything else stays +// matchable. It is a single predicate in a single function on purpose. An +// earlier shape in this phase encoded one policy in two predicates, and +// reverting either left the suite green because the error reached the same +// answer down the other path; a rule whose correctness argument IS its mutation +// evidence cannot afford to be un-mutatable in pieces. Falsifying either half +// of isAbsenceClaim now reddens a named spec. +func routeFailure(nodeID string, cause error) error { + if isAbsenceClaim(cause) { + return &noRouteError{nodeID: nodeID, cause: cause} + } + return fmt.Errorf("reaching node %q: %w: %w", nodeID, ErrNoRoute, cause) +} + +// isAbsenceClaim reports whether an error asserts that something does not +// exist. Those are the errors routeFailure keeps out of the chain. +// +// Both are about the CLUSTER rather than about the worker. ErrNoConnection says +// no live replica holds the worker's tunnel; ErrInstanceNotFound says a peer +// replica is not in the deployment. Neither can be answered by this package +// with "and therefore the worker is gone", because this package does not know +// what a worker is beyond an id in a connection row. +func isAbsenceClaim(err error) bool { + return errors.Is(err, ErrNoConnection) || errors.Is(err, ErrInstanceNotFound) +} + +// isWorkerAnswer reports whether an error is the WORKER's own refusal, read off +// the reply it sent. +// +// Those three sentinels are the only ones ReadStreamReply produces from a frame +// the worker actually wrote. Everything else it returns is a failure to read +// one, which is the tunnel breaking rather than the worker speaking. +// +// A reply carrying a code this frontend does not recognise is deliberately NOT +// counted here, even though a worker did send it. Classifying it as an answer +// would let a newer worker's vocabulary be read by an older frontend as +// evidence about a backend, and the consequence of guessing wrong in that +// direction is a reaped replica; guessing wrong the other way costs a retry. +func isWorkerAnswer(err error) bool { + return errors.Is(err, ErrStreamTagUnknown) || + errors.Is(err, ErrStreamTargetUnavailable) || + errors.Is(err, ErrStreamRequestInvalid) +} + // dialHandshakeTimeout bounds the request/reply exchange that opens every // stream, when the caller stated no deadline of its own. // @@ -79,13 +175,18 @@ func NewWorkerDialer(tunnels *TunnelRegistry, peers PeerOpener) *WorkerDialer { // inference that is quiet for minutes, and a deadline left over from the // handshake would abort it. // -// The errors are kept apart on purpose and a caller may act on them -// differently. ErrNoConnection means no live replica holds this worker's -// tunnel, which is the one answer that means the worker is absent. ErrNotOwner -// means the routing was stale and re-resolving may find it. ErrPeerUnreachable -// means a replica would not answer, ErrNoRelayPath that none could be dialled, -// and the tunnelproto sentinels that the worker itself refused. Nothing here -// ever converts one of the others into absence. +// EVERY failure to resolve or open a route carries ErrNoRoute, and NO failure +// carries an absence sentinel. That pair is the contract, and it is what makes +// this safe to consume from a package that reclaims a worker's models when it +// decides the worker has gone: there is one check to make, and there is nothing +// to mistake for absence even if the caller makes none. +// +// Underneath the umbrella the conditions stay apart and a caller may act on +// them differently. ErrNotOwner means the routing was stale and re-resolving +// may find it; ErrPeerUnreachable means a replica would not answer; +// ErrNoRelayPath means none could be dialled. A refusal from the WORKER carries +// its own tunnelproto sentinel and no umbrella at all, because a worker that +// answers has demonstrated it is there. func (d *WorkerDialer) Dial(ctx context.Context, nodeID, tag, target string) (net.Conn, error) { stream, err := d.tunnels.Open(ctx, nodeID) if err == nil { @@ -93,10 +194,9 @@ func (d *WorkerDialer) Dial(ctx context.Context, nodeID, tag, target string) (ne } if !errors.Is(err, ErrNotOwner) { // The tunnel is held HERE and its session would not carry a stream. - // Reported as itself: answering ErrNotOwner would send the caller to - // resolve an owner that is this same replica, and answering absence - // would tell a scheduler to reclaim a worker that is attached. - return nil, err + // ErrNotOwner stays out of it: that answer would send the caller to + // resolve an owner which is this same replica. + return nil, routeFailure(nodeID, err) } return d.relay(ctx, nodeID, tag, target) } @@ -137,10 +237,12 @@ func (d *WorkerDialer) relay(ctx context.Context, nodeID, tag, target string) (n // come back as ErrNoConnection here. owner, _, err := d.tunnels.reg.Owner(ctx, nodeID) if err != nil { - // ErrNoConnection and database failures both pass through as - // themselves. This is the ONLY path by which this function can produce - // an absence error, and it produces it only when Owner did. - return nil, err + // ErrNoConnection is the ordinary answer here, and it is precisely the + // one that must not get out: it means no live replica holds this + // worker's tunnel, which a worker that has not dialled in yet produces + // on every single request while it sits there heartbeating and serving. + // routeFailure keeps it in the message and out of the chain. + return nil, routeFailure(nodeID, err) } if owner == d.tunnels.selfID { // The table names this replica and the registry above said the tunnel @@ -149,19 +251,20 @@ func (d *WorkerDialer) relay(ctx context.Context, nodeID, tag, target string) (n // would resolve the same owner and relay again. Reported as the routing // fact so the caller re-resolves, which terminates: the row is either // re-claimed by whoever holds the worker now, or swept. - return nil, fmt.Errorf("opening a stream to node %q: the connection row names this replica, which no longer holds the tunnel: %w", nodeID, ErrNotOwner) + return nil, routeFailure(nodeID, fmt.Errorf("the connection row names this replica, which no longer holds the tunnel: %w", ErrNotOwner)) } if d.peers == nil { - return nil, fmt.Errorf("opening a stream to node %q held by replica %q: %w", nodeID, owner, ErrNoRelayPath) + return nil, routeFailure(nodeID, fmt.Errorf("the tunnel is held by replica %q: %w", owner, ErrNoRelayPath)) } stream, err := d.peers.Open(ctx, owner) if err != nil { - // Whatever the pool said, unchanged in its unwrap chain: - // ErrPeerUnreachable, ErrInstanceNotFound for an owner swept since the - // lookup above, or ErrPoolClosed while this process shuts down. None of - // them is a statement about the WORKER, and none is converted into one. - return nil, fmt.Errorf("opening a stream to node %q through replica %q: %w", nodeID, owner, err) + // ErrPeerUnreachable and ErrPoolClosed keep their identity; the one + // case the pool can also produce, ErrInstanceNotFound for an owner + // swept between the lookup above and this dial, is an absence claim + // about the REPLICA and routeFailure withholds it. Either way nothing + // here is a statement about the worker. + return nil, routeFailure(nodeID, fmt.Errorf("through replica %q: %w", owner, err)) } // The caller's remaining time, stated so the owning replica can bound its @@ -169,14 +272,15 @@ func (d *WorkerDialer) relay(ctx context.Context, nodeID, tag, target string) (n // the owner falls back to without it. if err := WriteRelayRequest(stream, nodeID, remainingBudget(ctx)); err != nil { _ = stream.Close() - return nil, fmt.Errorf("naming node %q on a stream to replica %q: %w", nodeID, owner, err) + return nil, routeFailure(nodeID, fmt.Errorf("naming the node on a stream to replica %q: %w", owner, err)) } if err := ReadRelayReply(stream); err != nil { - // ReadRelayReply already separates a refusal (ErrNotOwner, - // ErrRelayUnavailable, ErrRelayRequestInvalid) from a failure to read - // one, and neither kind is ever an absence error. + // A refusal from the OWNING REPLICA, not from the worker. It says the + // owner would not relay, which is a route that does not exist, so the + // umbrella is right for all of them; ErrNotOwner, ErrRelayUnavailable + // and ErrRelayRequestInvalid stay in the chain underneath. _ = stream.Close() - return nil, fmt.Errorf("relaying to node %q through replica %q: %w", nodeID, owner, err) + return nil, routeFailure(nodeID, fmt.Errorf("through replica %q: %w", owner, err)) } return d.handshake(ctx, stream, nodeID, tag, target) } @@ -189,20 +293,27 @@ func (d *WorkerDialer) relay(ctx context.Context, nodeID, tag, target string) (n // session, and a frontend that retries would exhaust the worker's stream // budget rather than the worker's patience. func (d *WorkerDialer) handshake(ctx context.Context, stream net.Conn, nodeID, tag, target string) (net.Conn, error) { - if deadline, ok := handshakeDeadline(ctx); ok { - if err := stream.SetDeadline(deadline); err != nil { - _ = stream.Close() - return nil, fmt.Errorf("arming the handshake deadline for node %q: %w", nodeID, err) - } + if err := stream.SetDeadline(handshakeDeadline(ctx)); err != nil { + _ = stream.Close() + return nil, routeFailure(nodeID, fmt.Errorf("arming the handshake deadline: %w", err)) } if err := WriteStreamRequest(stream, tag, target); err != nil { + // The stream would not carry the request, so the tunnel broke under it. + // Nothing was asked of the worker and nothing was learned about it. _ = stream.Close() - return nil, fmt.Errorf("asking node %q for %q on %q: %w", nodeID, tag, target, err) + return nil, routeFailure(nodeID, fmt.Errorf("asking for %q on %q: %w", tag, target, err)) } if err := ReadStreamReply(stream); err != nil { _ = stream.Close() - return nil, fmt.Errorf("opening %q on node %q: %w", tag, nodeID, err) + if isWorkerAnswer(err) { + // The worker wrote a refusal, so it is connected and answering. + // This is the ONE failure on the whole path that is real evidence + // about the worker, and putting the umbrella on it would throw that + // away. + return nil, fmt.Errorf("opening %q on node %q: %w", tag, nodeID, err) + } + return nil, routeFailure(nodeID, fmt.Errorf("opening %q: %w", tag, err)) } // Cleared unconditionally rather than only when one was armed, so that this @@ -213,7 +324,7 @@ func (d *WorkerDialer) handshake(ctx context.Context, stream net.Conn, nodeID, t // bounds a peer that has stopped answering. if err := stream.SetDeadline(time.Time{}); err != nil { _ = stream.Close() - return nil, fmt.Errorf("clearing the handshake deadline for node %q: %w", nodeID, err) + return nil, routeFailure(nodeID, fmt.Errorf("clearing the handshake deadline: %w", err)) } xlog.Debug("opened a tunnelled stream to a worker", "node", nodeID, "tag", tag, "target", target) return stream, nil @@ -221,13 +332,18 @@ func (d *WorkerDialer) handshake(ctx context.Context, stream net.Conn, nodeID, t // handshakeDeadline is when the handshake must be done by: the caller's own // deadline when it has one and it is the sooner, and the backstop otherwise. -func handshakeDeadline(ctx context.Context) (time.Time, bool) { +// +// There is always one, which is why this returns no "was there one" flag: a +// context with no deadline still gets the backstop, so the caller has nothing +// to branch on. It used to return a bool that was unconditionally true, and the +// branch behind it could not be taken. +func handshakeDeadline(ctx context.Context) time.Time { backstop := time.Now().Add(dialHandshakeTimeout) deadline, ok := ctx.Deadline() if !ok || deadline.After(backstop) { - return backstop, true + return backstop } - return deadline, true + return deadline } // remainingBudget is how long the caller is still willing to wait, or zero when diff --git a/core/services/cluster/dialer_test.go b/core/services/cluster/dialer_test.go index ffcfe3775e13..2e0b945d3704 100644 --- a/core/services/cluster/dialer_test.go +++ b/core/services/cluster/dialer_test.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net" + "sync" "time" "github.com/mudler/LocalAI/core/services/cluster" @@ -26,15 +27,31 @@ import ( type stubPeers struct { sess *yamux.Session err error + + // opened records the peers this pool was asked for, so a spec can assert + // that a replica was NOT dialled. That is the only way to tell a dialer + // that resolved a live owner from one that resolved a dead row and then + // found out the hard way. + mu sync.Mutex + opened []string } -func (s *stubPeers) Open(ctx context.Context, _ string) (net.Conn, error) { +func (s *stubPeers) Open(ctx context.Context, peerID string) (net.Conn, error) { + s.mu.Lock() + s.opened = append(s.opened, peerID) + s.mu.Unlock() if s.err != nil { return nil, s.err } return s.sess.OpenStream(ctx) } +func (s *stubPeers) peersDialled() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.opened...) +} + // dialResult carries what a Dial produced, so a spec can wait on a channel // rather than on a clock. type dialResult struct { @@ -124,7 +141,8 @@ func refuseOneStream(worker *yamux.Session, reason error) { // // It is the assertion this whole phase turns on. core/services/nodes reclaims a // worker's models when it concludes the worker is absent, so an unreachable -// peer or a refusing worker arriving as absence would evict healthy work. +// peer, a stale ownership row or a worker that has not dialled its tunnel yet +// arriving as absence would evict healthy work. func expectNotAbsence(err error) { GinkgoHelper() Expect(err).To(HaveOccurred()) @@ -132,6 +150,18 @@ func expectNotAbsence(err error) { Expect(err).ToNot(MatchError(cluster.ErrInstanceNotFound)) } +// expectNoRoute asserts the umbrella is present as well as absence being gone. +// +// The umbrella is what crosses the package boundary. A consumer that reclaims +// models has one check to make, and it can only make it if EVERY failure to +// resolve or open a route carries it; a single path that forgets is a path +// where a live worker gets reaped. +func expectNoRoute(err error) { + GinkgoHelper() + expectNotAbsence(err) + Expect(err).To(MatchError(cluster.ErrNoRoute)) +} + var _ = Describe("The worker dialer", func() { var ( db *gorm.DB @@ -200,16 +230,25 @@ var _ = Describe("The worker dialer", func() { Expect(string(echoed)).To(Equal("ping")) }) - It("leaves no read deadline armed on the stream it hands back", func() { + It("leaves no deadline armed on the stream it hands back", func() { // The handshake is bounded; the request that follows it is the // caller's business and may be a generation that is quiet for - // minutes. A deadline left armed here would abort it. + // minutes. A deadline left armed here would abort it, and in + // production the dial context is the model-load or request budget, + // so the stream would die tens of seconds in. + // + // The first version of this spec did not assert that. It set a + // 300ms context and then wrote immediately, so the armed deadline + // had not expired and deleting the clear left it green: it detected + // only a deadline set in the PAST. What makes it bite is waiting for + // the dial context to actually expire FIRST, on its own Done channel + // rather than a sleep, and only then using the stream. frontend, worker := workerTunnel() _, err := mine.Attach(ctx, "w1", frontend) Expect(err).ToNot(HaveOccurred()) seen := serveOneStream(worker) - deadlined, cancel := context.WithTimeout(ctx, 300*time.Millisecond) + deadlined, cancel := context.WithTimeout(ctx, 200*time.Millisecond) defer cancel() d := cluster.NewWorkerDialer(mine, nil) result := dialAsync(d, deadlined, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000") @@ -220,12 +259,16 @@ var _ = Describe("The worker dialer", func() { Expect(out.err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = out.conn.Close() }) - // The dial context's deadline has now passed. A stream still - // carrying it would fail this write and this read. - Eventually(func() error { - _, err := out.conn.Write([]byte("ping")) - return err - }, "10s").Should(Succeed()) + // The one wait this spec cannot replace with an event of its own: + // there is nothing to observe until the dial's deadline is behind + // us, and the deadline is the thing under test. + <-deadlined.Done() + Expect(deadlined.Err()).To(HaveOccurred()) + + // Both directions, because SetDeadline arms read and write and a + // clear that only covered one would still kill a live request. + _, err = out.conn.Write([]byte("ping")) + Expect(err).ToNot(HaveOccurred()) echoed := make([]byte, 4) Eventually(readInto(out.conn, echoed), "10s").Should(Receive(BeNil())) Expect(string(echoed)).To(Equal("ping")) @@ -241,8 +284,12 @@ var _ = Describe("The worker dialer", func() { var out dialResult Eventually(dialAsync(d, ctx, "w1", "nonsense", ""), "10s").Should(Receive(&out)) Expect(out.err).To(MatchError(cluster.ErrStreamTagUnknown)) - // A refusal is PROOF the worker is connected and answered. + // A refusal is PROOF the worker is connected and answered, so it is + // the ONE failure on this path that carries no umbrella: it is real + // evidence about the worker, and folding it into "no route" would + // throw that evidence away. expectNotAbsence(out.err) + Expect(out.err).ToNot(MatchError(cluster.ErrNoRoute)) }) It("reports a broken tunnel held here as itself, not as a routing fact", func() { @@ -257,9 +304,8 @@ var _ = Describe("The worker dialer", func() { d := cluster.NewWorkerDialer(mine, nil) var out dialResult Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) - Expect(out.err).To(HaveOccurred()) Expect(out.err).ToNot(MatchError(cluster.ErrNotOwner)) - expectNotAbsence(out.err) + expectNoRoute(out.err) }) }) @@ -376,7 +422,7 @@ var _ = Describe("The worker dialer", func() { var out dialResult Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "20s").Should(Receive(&out)) Expect(out.err).To(MatchError(cluster.ErrPeerUnreachable)) - expectNotAbsence(out.err) + expectNoRoute(out.err) }) It("passes a stale ownership refusal back as the routing fact", func() { @@ -396,7 +442,7 @@ var _ = Describe("The worker dialer", func() { var out dialResult Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) Expect(out.err).To(MatchError(cluster.ErrNotOwner)) - expectNotAbsence(out.err) + expectNoRoute(out.err) }) It("refuses rather than relaying to itself when the table names this replica", func() { @@ -410,7 +456,7 @@ var _ = Describe("The worker dialer", func() { var out dialResult Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) Expect(out.err).To(MatchError(cluster.ErrNotOwner)) - expectNotAbsence(out.err) + expectNoRoute(out.err) }) It("reports having no way to relay as its own condition", func() { @@ -424,22 +470,35 @@ var _ = Describe("The worker dialer", func() { Expect(out.err).To(MatchError(cluster.ErrNoRelayPath)) Expect(out.err).ToNot(MatchError(cluster.ErrNotOwner)) Expect(out.err).ToNot(MatchError(cluster.ErrPeerUnreachable)) - expectNotAbsence(out.err) + expectNoRoute(out.err) }) }) Describe("when no live replica holds the tunnel", func() { - It("reports absence when the worker has no connection row at all", func() { - d := cluster.NewWorkerDialer(mine, &stubPeers{err: errors.New("no peer should be dialled")}) + // The rolling-upgrade case, and the one this phase must not get wrong. + // + // A worker's PRESENCE is its heartbeat, which lives in + // core/services/nodes. "No live replica holds this worker's tunnel" is + // a fact about tunnels and says nothing about the worker: a worker that + // has not dialled in yet after a frontend-first upgrade produces it on + // every request while it sits there heartbeating and serving models. + // A consumer told that is absence reclaims every one of those models. + It("answers no-route, never absence, for a worker with no connection row", func() { + peers := &stubPeers{err: errors.New("no peer should be dialled")} + d := cluster.NewWorkerDialer(mine, peers) var out dialResult Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) - Expect(out.err).To(MatchError(cluster.ErrNoConnection)) + expectNoRoute(out.err) + // The cause still reaches a human. + Expect(fmt.Sprint(out.err)).To(ContainSubstring("no connection recorded")) + Expect(peers.peersDialled()).To(BeEmpty()) }) - It("reports absence when the row's owner has stopped heartbeating", func() { + It("answers no-route, never absence, when the row's owner has stopped heartbeating", func() { // End to end over the join Owner does: the row is there, the owner - // is not. A dialer built on the unjoined read would dial a corpse - // and report the worker as unreachable rather than as absent. + // is not. The join is what stops this replica dialling a process + // that is gone, which is why the spec asserts no peer was dialled + // as well as what came back. Expect(reg.Register(ctx, "ghost", "10.0.0.9:8080", "v1")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "ghost") Expect(err).ToNot(HaveOccurred()) @@ -451,10 +510,28 @@ var _ = Describe("The worker dialer", func() { Expect(err).ToNot(HaveOccurred()) Expect(owner).To(Equal("ghost")) - d := cluster.NewWorkerDialer(mine, &stubPeers{err: errors.New("no peer should be dialled")}) + peers := &stubPeers{err: errors.New("no peer should be dialled")} + d := cluster.NewWorkerDialer(mine, peers) + var out dialResult + Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) + expectNoRoute(out.err) + Expect(peers.peersDialled()).To(BeEmpty()) + }) + + It("keeps an owner swept mid-dial out of the chain as well", func() { + // The other absence sentinel. PeerPool resolves the owner's address + // through the registry, so a replica reaped between Owner and the + // dial comes back as ErrInstanceNotFound. That is absence of a + // REPLICA, and a consumer matching absence would read it as absence + // of the WORKER. + Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "owner") + Expect(err).ToNot(HaveOccurred()) + + d := cluster.NewWorkerDialer(mine, &stubPeers{err: cluster.ErrInstanceNotFound}) var out dialResult Eventually(dialAsync(d, ctx, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) - Expect(out.err).To(MatchError(cluster.ErrNoConnection)) + expectNoRoute(out.err) }) }) diff --git a/core/services/cluster/relay.go b/core/services/cluster/relay.go index 9392834f4e14..61594c7abecb 100644 --- a/core/services/cluster/relay.go +++ b/core/services/cluster/relay.go @@ -125,6 +125,16 @@ func ReadRelayRequest(r io.Reader) (string, time.Duration, error) { if err != nil { return "", 0, fmt.Errorf("reading a relay request for node %q: budget %q is not a number of milliseconds: %w", nodeID, budgetText, err) } + if millis > maxRelayBudgetMillis { + // time.Duration is nanoseconds in an int64, so multiplying by + // time.Millisecond overflows past about 2.9e11 ms. Overflow here is + // bounded in the safe direction (it can only produce a negative or a + // small value, and both shorten the declaring peer's OWN open), but a + // bound that holds by arithmetic accident is not a bound. Anything past + // the ceiling is clamped to it, because a caller claiming to wait + // longer than the relay's own backstop gets the backstop either way. + millis = maxRelayBudgetMillis + } if millis <= 0 { // A caller with nothing left to spend. Reported as such rather than // folded into "not stated", so the relay refuses at once instead of @@ -209,6 +219,12 @@ func ReadRelayReply(r io.Reader) error { } } +// maxRelayBudgetMillis is the largest budget a peer may declare, and exists so +// the conversion below cannot overflow. A day is many orders of magnitude past +// relayOpenTimeout, which is the only thing a budget is ever compared against, +// so clamping to it changes no honest caller's behaviour. +const maxRelayBudgetMillis = int64(24 * 60 * 60 * 1000) + const ( // relayHeaderTimeout bounds how long a peer stream may go without naming // the worker it is for. Without it, a dialler killed between OpenStream and diff --git a/core/services/nodes/backend_client_factory_test.go b/core/services/nodes/backend_client_factory_test.go index be20f5e347b0..15adeb4f3117 100644 --- a/core/services/nodes/backend_client_factory_test.go +++ b/core/services/nodes/backend_client_factory_test.go @@ -5,6 +5,7 @@ package nodes import ( "context" "net" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -99,3 +100,34 @@ var _ = Describe("The backend client factory", func() { }) }) }) + +var _ = Describe("the host used to address a worker's own HTTP server", func() { + It("uses the registered address when the worker reports one", func() { + Expect(WorkerHTTPHost("node-1", "10.0.0.5:8080")).To(Equal("10.0.0.5:8080")) + }) + + It("still produces a host for a tunnel-only worker that reports none", func() { + // Task 7 removes the worker's inbound listeners, at which point a + // worker has no address to report. Refusing here would refuse exactly + // the workers the tunnel exists for, and the guards that used to do + // that returned 502 "node has no HTTP address". + host := WorkerHTTPHost("node-1", "") + Expect(host).ToNot(BeEmpty()) + Expect(host).To(ContainSubstring("node-1")) + }) + + It("produces a host that cannot resolve, so it can never become a dial", func() { + // The value fills a URL's host component and nothing else. Making it + // unresolvable is what stops a later refactor connecting to it by + // accident: .invalid is reserved by RFC 2606 and resolves nowhere. + host := WorkerHTTPHost("node-1", "") + hostname, _, err := net.SplitHostPort(host) + Expect(err).ToNot(HaveOccurred()) + Expect(hostname).To(HaveSuffix(".invalid")) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err = net.DefaultResolver.LookupHost(ctx, hostname) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/core/services/nodes/file_stager_http.go b/core/services/nodes/file_stager_http.go index 2c3146946443..0ac64fd069f6 100644 --- a/core/services/nodes/file_stager_http.go +++ b/core/services/nodes/file_stager_http.go @@ -43,6 +43,15 @@ type HTTPFileStager struct { // clients caches one *http.Client per node. Caching is what keeps the // connection pool: a client built per request would open a fresh tunnel // stream for every chunk of a multi-gigabyte upload. + // + // Entries are never pruned, and that is judged acceptable rather than + // overlooked. The map is bounded by the number of distinct workers this + // frontend has ever staged to, which is bounded by the fleet; each entry is + // a transport whose idle connections the 90s IdleConnTimeout above reclaims, + // so a departed worker's entry holds a map slot and nothing else. It is the + // same shape as PeerPool.links and would need the same thing to fix + // properly: a signal that a node has left, which the deregistration path + // does not publish today. clientsMu sync.Mutex clients map[string]*http.Client responseTimeout time.Duration // timeout waiting for server response after upload diff --git a/core/services/nodes/health.go b/core/services/nodes/health.go index 57c87a82ed32..f0cbbbf514b5 100644 --- a/core/services/nodes/health.go +++ b/core/services/nodes/health.go @@ -208,9 +208,23 @@ func (hm *HealthMonitor) doCheckAll(ctx context.Context) { mCheckCtx, mCancel := context.WithTimeout(ctx, 5*time.Second) ok, _ := mClient.HealthCheck(mCheckCtx) mCancel() + // Asked BEFORE the client is closed, because closing is what + // would discard the transport's record of why it failed. + unreached := unroutable(mClient) if closer, ok := mClient.(io.Closer); ok { closer.Close() } + if unreached != nil { + // The probe never reached a backend, so it observed + // nothing. The miss streak is left exactly as it was: + // neither advanced, which after three passes would delete + // this row and every other row in the fleet the moment a + // peer link blipped, nor cleared, which would forgive a + // backend that really has died. + xlog.Warn("Could not probe a model backend: no route to the worker", + "node", node.ID, "model", m.ModelName, "replica", m.ReplicaIndex, "error", unreached) + continue + } key := modelKey{NodeID: node.ID, ModelName: m.ModelName, ReplicaIndex: m.ReplicaIndex} hm.missesMu.Lock() diff --git a/core/services/nodes/health_mock_test.go b/core/services/nodes/health_mock_test.go index 0d05991d99d3..9f6bd311fa91 100644 --- a/core/services/nodes/health_mock_test.go +++ b/core/services/nodes/health_mock_test.go @@ -133,8 +133,17 @@ func (f *fakeNodeHealthStore) RemoveNodeModel(_ context.Context, nodeID, modelNa type fakeBackendClient struct { healthy bool err error + // dialErr makes this client report that its TRANSPORT failed, which is what + // a real client whose tunnel dial failed does. It is the half of + // unroutability that a refusing factory cannot stand in for, and the likely + // one in production: the factory only fails when the wiring is absent. + dialErr error } +// LastDialError satisfies grpc.DialErrorReporter so a spec can drive the +// "reached no backend" branch without a real tunnel. +func (c *fakeBackendClient) LastDialError() error { return c.dialErr } + func (c *fakeBackendClient) IsBusy() bool { return false } func (c *fakeBackendClient) HealthCheck(_ context.Context) (bool, error) { return c.healthy, c.err @@ -395,4 +404,5 @@ func freshTime() time.Time { // Compile-time interface checks var _ NodeHealthStore = (*fakeNodeHealthStore)(nil) var _ BackendClientFactory = (*fakeBackendClientFactory)(nil) +var _ grpc.DialErrorReporter = (*fakeBackendClient)(nil) var _ grpc.Backend = (*fakeBackendClient)(nil) diff --git a/core/services/nodes/health_test.go b/core/services/nodes/health_test.go index 6f624b0cacb5..a20384737f94 100644 --- a/core/services/nodes/health_test.go +++ b/core/services/nodes/health_test.go @@ -9,6 +9,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" "gorm.io/gorm" ) @@ -325,6 +327,53 @@ var _ = Describe("HealthMonitor (mock-based)", func() { Expect(store.getNode("node-cut").Status).To(Equal(StatusHealthy)) }) + It("leaves a model row alone when the probe never reached the worker", func() { + // The sibling of the factory case above, and the likelier one. The + // client is built fine and the tunnel DIAL fails, which gRPC + // reports with the same code as a dead backend. Counted as a miss + // it would delete every model row in the fleet after three passes + // of a peer link blip, while the models kept serving. + store := newFakeNodeHealthStore() + factory := newFakeBackendClientFactory() + hm := newTestHealthMonitor(store, factory, true, staleThreshold) + hm.perModelHealthCheck = true + + node := makeTestNode("node-blip", "blip-worker", "10.0.0.22:50051", StatusHealthy, freshTime()) + store.addNode(node) + store.addNodeModel("node-blip", NodeModel{NodeID: "node-blip", ModelName: "m", Address: "10.0.0.22:50053"}) + factory.setClient("10.0.0.22:50053", &fakeBackendClient{ + healthy: false, + err: fmt.Errorf("connection error"), + dialErr: fmt.Errorf("%w: %w", cluster.ErrNoRoute, cluster.ErrPeerUnreachable), + }) + + for i := 0; i < perModelMissThreshold+2; i++ { + hm.doCheckAll(context.Background()) + } + Expect(store.getCalls()).NotTo(ContainElement(ContainSubstring("RemoveNodeModel"))) + }) + + It("still reaps a backend that died on a worker it CAN reach", func() { + // The other direction, so the new check cannot pass by never + // reaping. A dial that succeeded and an RPC that failed is a dead + // process, and its row must still go. + store := newFakeNodeHealthStore() + factory := newFakeBackendClientFactory() + hm := newTestHealthMonitor(store, factory, true, staleThreshold) + hm.perModelHealthCheck = true + + node := makeTestNode("node-dead", "dead-worker", "10.0.0.23:50051", StatusHealthy, freshTime()) + store.addNode(node) + store.addNodeModel("node-dead", NodeModel{NodeID: "node-dead", ModelName: "m", Address: "10.0.0.23:50053"}) + // No dialErr: the transport was fine. + factory.setClient("10.0.0.23:50053", &fakeBackendClient{healthy: false, err: fmt.Errorf("connection refused")}) + + for i := 0; i < perModelMissThreshold; i++ { + hm.doCheckAll(context.Background()) + } + Expect(store.getCalls()).To(ContainElement("RemoveNodeModel:node-dead:m:0")) + }) + It("preserves model row when an intermittent failure is followed by a success", func() { store := newFakeNodeHealthStore() factory := newFakeBackendClientFactory() diff --git a/core/services/nodes/interfaces.go b/core/services/nodes/interfaces.go index 2baea85c4484..bf46abd8e57d 100644 --- a/core/services/nodes/interfaces.go +++ b/core/services/nodes/interfaces.go @@ -152,10 +152,15 @@ type NodeManager interface { // WorkerDialerFor hands back the dial function for one worker's backend // processes: the shape grpc.WithContextDialer wants, bound to a node. // -// It is declared here rather than taken as a core/services/cluster type so that -// this package keeps no dependency on that one. cluster is a leaf and imports -// neither this package nor core/http; the wiring in core/application supplies -// (*cluster.WorkerDialer).GRPCDialerFor, which has exactly this shape. +// A function type rather than a *cluster.WorkerDialer, so nothing here is bound +// to that concrete type and a spec can supply a dial without building a tunnel +// registry, a peer pool and a database. It is NOT to avoid a dependency: this +// package already imports core/services/cluster (registry.go, for Migrate), and +// an earlier version of this comment claimed otherwise. The dependency that +// does matter runs the other way, and cluster is held to it by go list -deps. +// +// core/application supplies (*cluster.WorkerDialer).GRPCDialerFor, which has +// exactly this shape. type WorkerDialerFor func(nodeID string) func(ctx context.Context, addr string) (net.Conn, error) // WorkerNetDialerFor hands back the dial function for one worker's own HTTP @@ -164,6 +169,26 @@ type WorkerDialerFor func(nodeID string) func(ctx context.Context, addr string) // has this shape. type WorkerNetDialerFor func(nodeID string) func(ctx context.Context, network, addr string) (net.Conn, error) +// ErrWorkerUnroutable reports that this frontend could not get a request to a +// worker's backend, and says NOTHING about whether that worker or its backend +// is alive. +// +// It is the fifth condition, on this side of the package boundary. A worker's +// presence is its HEARTBEAT, and this package owns that; a route to it is a +// separate fact owned by core/services/cluster, and the two now differ. A +// worker can be registered, heartbeating and serving every request another +// replica sends it while being unroutable from here: it has not dialled its +// tunnel yet after a frontend-first upgrade, the replica holding its tunnel is +// restarting, the ownership row is a moment stale, this replica has no peer +// mesh. Every one of those used to be indistinguishable from "the backend +// process died", because gRPC reports both as codes.Unavailable. +// +// Everything in this package that DELETES a node_models row must consult it +// first. That is the phase's stated catastrophe in its concrete form: a row +// deleted here is a model reclaimed and reloaded elsewhere, so mistaking a peer +// link blip for a dead backend evicts healthy work across the fleet at once. +var ErrWorkerUnroutable = errors.New("nodes: this frontend has no route to that worker") + // ErrNoWorkerDialer reports that something tried to reach a worker without a // way to reach it through the worker's tunnel. // @@ -171,9 +196,41 @@ type WorkerNetDialerFor func(nodeID string) func(ctx context.Context, network, a // advertised address. A worker that holds a tunnel need not listen on anything // and may be behind NAT with no address to dial, so the fallback would work // only where the tunnel was not needed: on a single-host developer setup, and -// nowhere the feature exists for. It is also not an absence error: nothing -// about it says the worker is gone. -var ErrNoWorkerDialer = errors.New("nodes: no worker tunnel dialer is configured, so this worker cannot be reached") +// nowhere the feature exists for. +// +// It is a SPECIALISATION of ErrWorkerUnroutable rather than a sibling, so the +// single check every reaping path makes covers both. The difference between +// them is only when they happen: this one is a boot-time misconfiguration, and +// the general one is a running deployment losing a route for a moment. Neither +// is a statement about the worker. +var ErrNoWorkerDialer = fmt.Errorf("%w: no worker tunnel dialer is configured", ErrWorkerUnroutable) + +// unroutable reports why a call on client never reached the backend, or nil +// when it did reach one. +// +// This is where core/services/cluster's five conditions cross the package +// boundary. They cannot cross on the RPC error: gRPC turns any dialer failure +// into codes.Unavailable with the cause flattened into a message, and +// codes.Unavailable is ALSO what a backend process that has died produces. +// pkg/grpc records the dialer's error VALUE instead, so cluster.ErrNoRoute and +// whatever sits under it are still matchable here. +// +// A client that reports nothing (no custom dialer, or a test double) yields +// nil, which means "the call reached a backend" and preserves the behaviour +// every non-distributed caller has always had. +func unroutable(client grpc.Backend) error { + reporter, ok := client.(grpc.DialErrorReporter) + if !ok { + return nil + } + dialErr := reporter.LastDialError() + if dialErr == nil { + return nil + } + // Multi-%w: the umbrella this package acts on, and the cluster condition + // underneath it, both stay matchable. + return fmt.Errorf("%w: %w", ErrWorkerUnroutable, dialErr) +} // BackendClientFactory creates the gRPC clients this frontend uses to reach // model backends running on worker nodes. @@ -242,3 +299,32 @@ func (f *tunnelClientFactory) NewClientForNode(nodeID, address string, parallel } return grpc.NewClientWithDialer(address, parallel, nil, false, f.token, dial), nil } + +// unroutableHostSuffix is appended to a node id to build a Host for a worker +// that reports no HTTP address. +// +// .invalid is reserved by RFC 2606 and resolves nowhere, which is the point: +// the string exists ONLY to fill the host component of a URL, and a value that +// could resolve would be one a future refactor could accidentally connect to. +const unroutableHostSuffix = ".worker.invalid:80" + +// WorkerHTTPHost is the host to put in a URL addressed to a worker's own HTTP +// server. +// +// A tunnel-only worker has no inbound address to report, and after this phase +// it does not need one: the `http` stream tag ignores the target entirely and +// the worker routes the stream to its own server wherever that bound. But an +// http.Request still needs a host, so refusing an empty HTTPAddress would +// refuse exactly the workers the tunnel exists for. This returns a name that +// identifies the node for logs and for the Host header, and that nothing can +// connect to. +// +// It is NOT a dial target and never becomes one. Every caller pairs it with a +// transport whose DialContext is that node's tunnel, so the host is read and +// discarded; see cluster.WorkerDialer.DialerFor and the `http` tag. +func WorkerHTTPHost(nodeID, httpAddress string) string { + if httpAddress != "" { + return httpAddress + } + return nodeID + unroutableHostSuffix +} diff --git a/core/services/nodes/probe_cache.go b/core/services/nodes/probe_cache.go index 422e36ede4e2..fee71383e7e1 100644 --- a/core/services/nodes/probe_cache.go +++ b/core/services/nodes/probe_cache.go @@ -73,22 +73,41 @@ func (c *probeCache) Invalidate(key string) { // probes invalidate the cache, so a transient miss doesn't pin every // subsequent request to a re-probe. func (c *probeCache) DoOrCached(key string, probe func() bool) bool { + alive, _ := c.DoOrCachedResult(key, func() (bool, error) { return probe(), nil }) + return alive +} + +// DoOrCachedResult is DoOrCached with a second result: the reason the probe +// never reached the backend, or nil when it did. +// +// The second result travels through the SINGLEFLIGHT, which is the whole reason +// it is not simply a variable the caller closes over. A closed-over variable is +// only written by the goroutine that actually runs the probe; every other +// caller coalesced into that flight adopts the leader's boolean and sees its own +// unset variable, so the leader would correctly decline to reap while its +// joiners reaped on the very same observation. Carrying it in singleflight's +// error slot hands every joiner the leader's reason as well as its answer. +// +// A probe that could not reach the backend is NOT cached either way. Caching it +// as fresh would hide a genuinely dead backend behind a network blip, and +// caching it as a failure is what Invalidate already does. +func (c *probeCache) DoOrCachedResult(key string, probe func() (bool, error)) (bool, error) { if c.IsFresh(key) { - return true + return true, nil } - v, _, _ := c.flight.Do(key, func() (any, error) { + v, unreached, _ := c.flight.Do(key, func() (any, error) { // Double-check after potentially waiting: another caller in this // flight may have just populated the cache. if c.IsFresh(key) { return true, nil } - ok := probe() + ok, unreached := probe() if ok { c.markFresh(key) } else { c.Invalidate(key) } - return ok, nil + return ok, unreached }) - return v.(bool) + return v.(bool), unreached } diff --git a/core/services/nodes/reconciler.go b/core/services/nodes/reconciler.go index 32feb5da54b8..3c192d004060 100644 --- a/core/services/nodes/reconciler.go +++ b/core/services/nodes/reconciler.go @@ -92,17 +92,34 @@ func (g grpcModelProber) Probe(ctx context.Context, nodeID, address string) Prob probeCtx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() ok, err := client.HealthCheck(probeCtx) + if unreached := unroutable(client); unreached != nil { + // The RPC never reached a backend. classifyProbeOutcome cannot tell: + // gRPC hands it codes.Unavailable for a worker this frontend has no + // route to and for a backend process that has died, and the reaper + // deletes rows on the second. + xlog.Warn("Could not probe a model: no route to the worker", + "node", nodeID, "address", address, "error", unreached) + return ProbeUnknown + } return classifyProbeOutcome(ok, err) } // classifyProbeOutcome maps a HealthCheck result onto a ProbeOutcome. // +// It is only ever reached for a probe that DID reach the worker. That is a +// precondition and not an observation it can make for itself: its caller asks +// the transport first and answers ProbeUnknown when the dial failed. Without +// that step the Unavailable case below is wrong, because a worker this frontend +// cannot route to produces exactly the same code as a backend that has died, +// and only one of the two should cost a row. +// // The gRPC client is lazy, so connection failures surface on the RPC rather // than at dial time, and the status code tells the two cases apart: // // - DeadlineExceeded: the transport was fine but nothing serviced the RPC in // time. That is a backend stuck inside a long synchronous request. -// - Unavailable: nothing is listening. The process is gone. +// - Unavailable: the worker was reached and nothing is listening on that +// port. The process is gone. // // A blackholed network also yields DeadlineExceeded and is therefore treated as // busy. That is deliberate: whole-node failures are the health monitor's job diff --git a/core/services/nodes/reconciler_prober_test.go b/core/services/nodes/reconciler_prober_test.go new file mode 100644 index 000000000000..2bacef554297 --- /dev/null +++ b/core/services/nodes/reconciler_prober_test.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: MIT + +package nodes + +import ( + "context" + "errors" + "fmt" + "net" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/cluster" + grpc "github.com/mudler/LocalAI/pkg/grpc" +) + +// proberFactory hands the prober one client, and records what it was asked for. +type proberFactory struct { + client grpc.Backend + err error + asked []string +} + +func (f *proberFactory) NewClientForNode(nodeID, address string, _ bool) (grpc.Backend, error) { + f.asked = append(f.asked, nodeID+"|"+address) + if f.err != nil { + return nil, f.err + } + return f.client, nil +} + +var _ = Describe("the reconciler's gRPC model prober", func() { + // The two lines that decide whether a row survives, both previously + // untested. Everything else in the reaper is driven through fakeProber, + // which means the mapping from a real client to a ProbeOutcome had nothing + // holding it at all. + probe := func(f *proberFactory) ProbeOutcome { + GinkgoHelper() + return grpcModelProber{clients: f}.Probe(context.Background(), "node-1", "10.0.0.1:9001") + } + + It("answers ProbeUnknown when no client can be built for the node", func() { + Expect(probe(&proberFactory{err: ErrNoWorkerDialer})).To(Equal(ProbeUnknown)) + }) + + It("answers ProbeUnknown when the client was built and the tunnel dial failed", func() { + // The likelier half. ProbeUnreachable here would delete the row after + // probeFailuresBeforeReap passes of a peer link that was merely + // restarting, and the backend would still be running the model. + Expect(probe(&proberFactory{client: &fakeBackendClient{ + healthy: false, + err: fmt.Errorf("rpc error: code = Unavailable"), + dialErr: fmt.Errorf("%w: %w", cluster.ErrNoRoute, cluster.ErrPeerUnreachable), + }})).To(Equal(ProbeUnknown)) + }) + + It("asks for the client by NODE, not by address alone", func() { + f := &proberFactory{client: &fakeBackendClient{healthy: true}} + Expect(probe(f)).To(Equal(ProbeAlive)) + Expect(f.asked).To(ContainElement("node-1|10.0.0.1:9001")) + }) + + It("answers ProbeAlive for a healthy backend it reached", func() { + Expect(probe(&proberFactory{client: &fakeBackendClient{healthy: true}})).To(Equal(ProbeAlive)) + }) + + It("still answers ProbeUnreachable for a dead backend on a worker it reached", func() { + // The other direction. The new check must not turn the reaper off: a + // backend that answered "unhealthy" over a working transport is a ghost + // and its row should go. + Expect(probe(&proberFactory{client: &fakeBackendClient{healthy: false}})).To(Equal(ProbeUnreachable)) + }) + + It("does not report a transport that recovered", func() { + // LastDialError is cleared by a successful dial, so a client that + // failed once and then reconnected must not keep reading as + // unroutable; otherwise a row could never be reaped again after one + // blip on that client. + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = listener.Close() }) + + attempt := 0 + f, err := NewTunnelClientFactory("", func(string) func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + return func(ctx context.Context, _ string) (net.Conn, error) { + attempt++ + if attempt == 1 { + return nil, errors.New("first dial fails") + } + return d.DialContext(ctx, "tcp", listener.Addr().String()) + } + }) + Expect(err).ToNot(HaveOccurred()) + client, err := f.NewClientForNode("node-1", "10.0.0.1:9001", false) + Expect(err).ToNot(HaveOccurred()) + + _, _ = client.HealthCheck(context.Background()) + Expect(unroutable(client)).ToNot(BeNil()) + + // gRPC re-dials on the next call; the listener now accepts. + Eventually(func() error { + _, _ = client.HealthCheck(context.Background()) + return unroutable(client) + }, "20s").Should(BeNil()) + }) +}) diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index 88effc259765..564949255784 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -2013,11 +2013,18 @@ func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir // (DecrementInFlight + RemoveNodeModel) still triggers on the next request. // // The client is built OUTSIDE the memoized closure, which is what keeps an -// unreachable worker out of the cache entirely: DoOrCached only ever sees a -// real answer. Building it costs a struct and no I/O, since the gRPC client +// unreachable worker out of the cache entirely: DoOrCachedResult only ever sees +// a real answer. Building it costs a struct and no I/O, since the gRPC client // dials lazily on its first call. +// +// The client is the RAW factory client rather than buildClientForAddr's, on +// purpose. A health check stages no files, so the staging wrapper buys nothing +// here; and the wrapper hides the transport, because it embeds grpc.Backend and +// so does not carry LastDialError through. Wrapping would leave this function +// unable to tell a dead backend from an unreachable worker, which is the whole +// question it now answers. func (r *SmartRouter) probeHealth(ctx context.Context, node *BackendNode, addr string) (alive, probed bool) { - client, err := r.buildClientForAddr(node, addr, false) + client, err := r.clientFactory.NewClientForNode(node.ID, addr, false) if err != nil { xlog.Error("Cannot probe a model backend: no way to reach the worker", "node", node.ID, "address", addr, "error", err) @@ -2026,12 +2033,24 @@ func (r *SmartRouter) probeHealth(ctx context.Context, node *BackendNode, addr s defer closeClient(client) key := node.ID + "|" + addr - return r.probeCache.DoOrCached(key, func() bool { + alive, unreached := r.probeCache.DoOrCachedResult(key, func() (bool, error) { checkCtx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() ok, _ := client.HealthCheck(checkCtx) - return ok - }), true + if ok { + return true, nil + } + // The RPC failed. gRPC reports a dead backend and an unreachable + // worker with the same code, so the only way to tell them apart is to + // ask the transport whether it was the one that failed. + return false, unroutable(client) + }) + if unreached != nil { + xlog.Warn("Could not probe a model backend: no route to the worker", + "node", node.ID, "address", addr, "error", unreached) + return false, false + } + return alive, true } // closeClient closes a gRPC backend client if it implements io.Closer. diff --git a/core/services/nodes/router_unreachable_worker_test.go b/core/services/nodes/router_unreachable_worker_test.go index 943741396e11..206956e1194f 100644 --- a/core/services/nodes/router_unreachable_worker_test.go +++ b/core/services/nodes/router_unreachable_worker_test.go @@ -4,20 +4,38 @@ package nodes import ( "context" - "errors" + "fmt" + "net" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/mudler/LocalAI/core/services/cluster" grpc "github.com/mudler/LocalAI/pkg/grpc" ) // unreachableClientFactory cannot build a client for any node, standing in for -// a frontend whose worker tunnel dialer is missing or broken. +// a frontend whose worker tunnel dialer is missing or broken. This is the +// BOOT-TIME half of unroutability. type unreachableClientFactory struct{} func (unreachableClientFactory) NewClientForNode(_, _ string, _ bool) (grpc.Backend, error) { - return nil, errors.New("no way to reach that worker") + return nil, ErrNoWorkerDialer +} + +// deadDialFactory builds clients normally and fails the DIAL, which is the +// RUNNING half and by far the likelier one: the factory only fails when the +// wiring is absent, while the dial fails whenever the replica holding a +// worker's tunnel is momentarily unreachable, which one frontend restart +// produces for every worker that replica holds. +func deadDialFactory(cause error) BackendClientFactory { + GinkgoHelper() + f, err := NewTunnelClientFactory("", func(string) func(ctx context.Context, addr string) (net.Conn, error) { + return func(context.Context, string) (net.Conn, error) { return nil, cause } + }) + Expect(err).ToNot(HaveOccurred()) + return f } var _ = Describe("routing when the worker cannot be reached at all", func() { @@ -68,3 +86,99 @@ var _ = Describe("routing when the worker cannot be reached at all", func() { Expect(reg.decrementCalls).To(ContainElement("X:m")) }) }) + +var _ = Describe("routing when the worker's tunnel dial fails", func() { + // The reviewer's spec. It is the boundary test: the factory succeeds, the + // gRPC client is built, and the DIAL fails underneath with a + // cluster.ErrNoRoute. gRPC flattens that into codes.Unavailable, which is + // also what a dead backend produces, so without a way to carry the + // distinction past the package boundary a peer link blip is read as a dead + // process and the replica row is deleted after ONE miss. + loadedReg := func() *fakeModelRouter { + node := &BackendNode{ID: "X", Name: "node-x", Address: "10.0.0.1:50051"} + nm := &NodeModel{NodeID: "X", ModelName: "m", Address: "10.0.0.1:9001"} + return &fakeModelRouter{ + findAndLockNode: node, + findAndLockNM: nm, + loadedReplicaStatsByName: map[string][]ReplicaCandidate{"m": {{NodeID: "X", InFlight: 0}}}, + } + } + + route := func(reg *fakeModelRouter, cause error) error { + router := NewSmartRouter(reg, SmartRouterOptions{ + Unloader: &fakeUnloader{}, + ClientFactory: deadDialFactory(cause), + }) + _, err := router.Route(context.Background(), "m", "models/m.gguf", "llama-cpp", "", nil, false) + return err + } + + It("never reaps a replica whose OWNER replica is unreachable", func() { + reg := loadedReg() + Expect(route(reg, fmt.Errorf("through replica %q: %w: %w", "peer-2", cluster.ErrNoRoute, cluster.ErrPeerUnreachable))).To(HaveOccurred()) + Expect(reg.removeCalls).To(BeEmpty(), + "a worker whose OWNER replica is unreachable must never have its loaded models reaped") + }) + + It("never reaps a replica that has not dialled its tunnel yet", func() { + // The rolling-upgrade case end to end. A frontend-first upgrade puts + // every not-yet-restarted worker here at once, and every one of them is + // heartbeating and serving while it happens. + reg := loadedReg() + Expect(route(reg, fmt.Errorf("reaching node %q: %w", "X", cluster.ErrNoRoute))).To(HaveOccurred()) + Expect(reg.removeCalls).To(BeEmpty()) + }) + + It("still releases the reservation it took", func() { + reg := loadedReg() + Expect(route(reg, fmt.Errorf("%w", cluster.ErrNoRoute))).To(HaveOccurred()) + Expect(reg.decrementCalls).To(ContainElement("X:m")) + }) + + It("carries the cluster condition all the way across the package boundary", func() { + // Not just "something failed": the specific reason survives gRPC, which + // is what makes the five conditions usable on this side. If this ever + // reduces to a bare code, the consumers above are guessing again. + f := deadDialFactory(fmt.Errorf("through replica %q: %w: %w", "peer-2", cluster.ErrNoRoute, cluster.ErrPeerUnreachable)) + client, err := f.NewClientForNode("X", "10.0.0.1:9001", false) + Expect(err).ToNot(HaveOccurred()) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, _ = client.HealthCheck(ctx) + + unreached := unroutable(client) + Expect(unreached).To(MatchError(ErrWorkerUnroutable)) + Expect(unreached).To(MatchError(cluster.ErrNoRoute)) + Expect(unreached).To(MatchError(cluster.ErrPeerUnreachable)) + // And never absence, at either end of the trip. + Expect(unreached).ToNot(MatchError(cluster.ErrNoConnection)) + Expect(unreached).ToNot(MatchError(cluster.ErrInstanceNotFound)) + }) + + It("reports nothing for a client whose dial succeeded", func() { + // The other direction, so the seam cannot pass by always saying yes: a + // backend that genuinely died must still be reapable. + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = listener.Close() }) + + f, err := NewTunnelClientFactory("", func(string) func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + return func(ctx context.Context, _ string) (net.Conn, error) { + return d.DialContext(ctx, "tcp", listener.Addr().String()) + } + }) + Expect(err).ToNot(HaveOccurred()) + client, err := f.NewClientForNode("X", "10.0.0.1:9001", false) + Expect(err).ToNot(HaveOccurred()) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + // The listener accepts and speaks no gRPC, so the RPC fails while the + // DIAL succeeds. That is exactly a dead-ish backend on a reachable + // worker, and it must not read as unroutable. + _, _ = client.HealthCheck(ctx) + Expect(unroutable(client)).To(BeNil()) + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 288091f8047d..5cd4cca96004 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -176,7 +176,16 @@ Four outcomes are kept apart on purpose, because they call for different actions Only the first is absence. The others are never reported as it, and that is not a stylistic preference: a scheduler told that a connected worker has gone away reclaims every model it is running. -Set `LOCALAI_WORKER_TUNNEL=false` on a worker to turn the tunnel off and go back to the frontend dialling the worker's advertised addresses. +#### There is no frontend-side fallback, and upgrade order matters + +`LOCALAI_WORKER_TUNNEL=false` still stops a worker dialling its tunnel, but it no longer has a frontend counterpart: after this change **no frontend path dials a worker's advertised address**, so a worker with the tunnel off is a worker the frontend cannot reach. Setting it is not a rollback. The rollback is to run the previous frontend release. + +That makes upgrade order matter, in one direction only: + +- **Upgrade the workers first, then the frontends.** A worker on the new build dials its tunnel and is reachable by frontends of either version, because the old frontend still dials its advertised address and the worker still listens. +- **Upgrading the frontends first** leaves every not-yet-restarted worker unroutable until it restarts. Those workers keep running their models and keep heartbeating, and the frontend reports them as unroutable rather than as gone: their `node_models` rows are left alone, nothing is rescheduled, and requests for those models fail loudly with "no route" until the worker reconnects. It is a degraded window, not an eviction, but it is a window, and doing it the other way round has none. + +A worker that cannot reach its frontend retries with exponential backoff and never gives up, so restarting a worker is all that is needed to close the window. ### The model load deadline scales with the checkpoint @@ -403,7 +412,7 @@ local-ai worker \ | `--registration-require-auth` | `LOCALAI_REGISTRATION_REQUIRE_AUTH` | `false` | Refuse to start the HTTP file-transfer server when no registration token is set (it would otherwise fail open) | | `--distributed-require-auth` | `LOCALAI_DISTRIBUTED_REQUIRE_AUTH` | `false` | Umbrella switch implying both `--registration-require-auth` and `--nats-require-auth` | | `--heartbeat-interval` | `LOCALAI_HEARTBEAT_INTERVAL` | `10s` | Interval between heartbeat pings | -| `--worker-tunnel` | `LOCALAI_WORKER_TUNNEL` | `true` | Hold one outbound multiplexed tunnel to the frontend and serve its requests over it, so this worker needs no inbound port (see [Worker tunnels](#worker-tunnels)) | +| `--worker-tunnel` | `LOCALAI_WORKER_TUNNEL` | `true` | Hold one outbound multiplexed tunnel to the frontend and serve its requests over it, so this worker needs no inbound port (see [Worker tunnels](#worker-tunnels)). Turning it off makes the worker unreachable: the frontend has no path that dials a worker's advertised address. | | `--nats-url` | `LOCALAI_NATS_URL` | *(required)* | NATS URL for backend installation and file staging | | `--nats-jwt` | `LOCALAI_NATS_JWT` | *(empty)* | Optional override for the `nats_jwt` returned at registration | | `--nats-user-seed` | `LOCALAI_NATS_USER_SEED` | *(empty)* | Optional override for `nats_user_seed` from registration | diff --git a/pkg/grpc/backend.go b/pkg/grpc/backend.go index 866721d2cf8a..6a32ec29d12f 100644 --- a/pkg/grpc/backend.go +++ b/pkg/grpc/backend.go @@ -49,10 +49,29 @@ func NewClientWithDialer(address string, parallel bool, wd WatchDog, enableWatch // the address directly, which is the exact bypass this constructor exists // to close. c := buildClient(address, parallel, wd, enableWatchDog, token) - c.dialer = dialer + // Wrapped rather than stored bare, so every dial outcome is recorded. This + // is the seam that carries the reason a dial failed past gRPC, which + // flattens it into codes.Unavailable; see (*Client).LastDialError. + c.dialer = func(ctx context.Context, addr string) (net.Conn, error) { + conn, err := dialer(ctx, addr) + c.recordDialErr(err) + return conn, err + } return c } +// DialErrorReporter is implemented by a Backend that reaches its process +// through a custom transport and can say whether that transport, rather than +// the process, is what failed. +// +// It is a separate interface and NOT part of Backend on purpose: only the +// handful of callers that act on the difference need it, and widening Backend +// would make every wrapper and every test double implement a method they have +// no answer for. +type DialErrorReporter interface { + LastDialError() error +} + func buildClient(address string, parallel bool, wd WatchDog, enableWatchDog bool, token string) *Client { if !enableWatchDog { wd = nil diff --git a/pkg/grpc/client.go b/pkg/grpc/client.go index ba6fd0e503db..c73f813784f0 100644 --- a/pkg/grpc/client.go +++ b/pkg/grpc/client.go @@ -40,6 +40,14 @@ type Client struct { // keeps gRPC's own TCP dial, which is what every non-distributed caller // wants. dialer func(ctx context.Context, addr string) (net.Conn, error) + + // dialErrMu guards lastDialErr. Its own mutex rather than the embedded one: + // the embedded Mutex guards inFlight and is taken on every call, and a + // dialer runs underneath gRPC's own machinery where reentering it is not + // something this type can reason about. + dialErrMu sync.Mutex + lastDialErr error + sync.Mutex opMutex sync.Mutex wd WatchDog @@ -1438,3 +1446,40 @@ func (c *Client) ModelMetadata(ctx context.Context, in *pb.ModelOptions, opts .. client := pb.NewBackendClient(conn) return client.ModelMetadata(ctx, in, opts...) } + +// LastDialError returns the error from the most recent attempt by this client's +// custom dialer, or nil when the last attempt succeeded or there is no custom +// dialer. +// +// It exists because gRPC destroys the distinction its callers need. A dialer +// failure reaches an RPC as codes.Unavailable with the cause flattened into a +// message string, and codes.Unavailable is ALSO what a backend process that +// died produces. Those two call for opposite actions: a dead backend's registry +// row should be reaped, and a transport that could not reach a live backend +// must never cause one to be. Recording the error here is what lets a caller +// tell them apart, with the original error VALUE intact, so +// core/services/cluster's sentinels survive the trip. +// +// Scope, stated exactly. This is the last dial on this CLIENT, not the last +// dial for a particular RPC. A client used for one probe and closed gives exact +// attribution, which is how every reaping path in core/services/nodes uses it. +// A client shared across concurrent RPCs can attribute a dial failure to the +// wrong one; both directions of that error are safe, because a caller consults +// this only when its RPC already failed, and the outcomes are "treat a dead +// backend as unreachable-for-now" (the row survives one extra round) or "treat +// a transport failure as a backend failure" (the behaviour before this +// existed). +func (c *Client) LastDialError() error { + c.dialErrMu.Lock() + defer c.dialErrMu.Unlock() + return c.lastDialErr +} + +// recordDialErr stores the outcome of one dial. A success CLEARS the previous +// failure rather than leaving it, so a client that recovered does not keep +// reporting a dial error that no longer describes anything. +func (c *Client) recordDialErr(err error) { + c.dialErrMu.Lock() + c.lastDialErr = err + c.dialErrMu.Unlock() +} diff --git a/pkg/model/loader.go b/pkg/model/loader.go index 9a3a9fd7806c..a69831871d99 100644 --- a/pkg/model/loader.go +++ b/pkg/model/loader.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + grpc "github.com/mudler/LocalAI/pkg/grpc" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" @@ -691,6 +692,20 @@ func (ml *ModelLoader) checkIsLoaded(s string) *Model { // Remote/distributed model — no local process to check. // Only evict on definitive connection errors (node is down). // Timeouts may mean the node is busy, so keep the model cached. + // + // "The node is down" is exactly what this can no longer conclude on + // its own. In distributed mode the client reaches the backend over + // the worker's tunnel, and a failure of THAT transport (the replica + // holding the tunnel is restarting, the worker has not dialled in + // yet after a frontend-first upgrade) arrives as the same + // codes.Unavailable a dead worker produces. Evicting on it would + // unload a model that is loaded and serving. The client records + // which of the two happened; see grpc.DialErrorReporter. + if dialErr := transportFailure(client); dialErr != nil { + xlog.Warn("Remote model health check could not reach the worker, keeping cached", + "model", s, "error", dialErr) + return m + } if isConnectionError(err) { xlog.Warn("Remote model unreachable (connection error), removing from cache", "model", s, "error", err) if delErr := ml.deleteProcess(cTimeout, s, false); delErr != nil { @@ -715,3 +730,20 @@ func (ml *ModelLoader) checkIsLoaded(s string) *Model { m.MarkHealthy() return m } + +// transportFailure reports why a call never reached the backend, or nil when it +// did reach one. +// +// It is the one question that separates "this backend is gone" from "this +// process cannot currently get to it", and gRPC does not answer it: a dialer +// failure and a dead listener both surface as codes.Unavailable. A client with +// no custom transport answers nil, which is right for every locally spawned +// backend, where the address IS a socket on this machine and a failed +// connection really does mean the process died. +func transportFailure(client grpc.Backend) error { + reporter, ok := client.(grpc.DialErrorReporter) + if !ok { + return nil + } + return reporter.LastDialError() +} diff --git a/pkg/model/remote_unroutable_internal_test.go b/pkg/model/remote_unroutable_internal_test.go new file mode 100644 index 000000000000..25a440d800be --- /dev/null +++ b/pkg/model/remote_unroutable_internal_test.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: MIT + +package model + +import ( + "context" + "errors" + "net" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + grpc "github.com/mudler/LocalAI/pkg/grpc" + "github.com/mudler/LocalAI/pkg/system" +) + +var _ = Describe("the health check on a remote model whose transport failed", func() { + // The fourth site of the same shape as the reconciler, the health monitor + // and the router, found by sweeping rather than by being named. + // + // checkIsLoaded evicts a remote model on a "connection error", which used + // to mean exactly one thing: the worker's socket did not answer. In + // distributed mode the client reaches the backend over the worker's tunnel, + // and a failure of THAT transport arrives as the same codes.Unavailable. + // Evicting on it unloads a model that is loaded and serving, on a worker + // that is heartbeating. + var ml *ModelLoader + + BeforeEach(func() { + systemState, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir())) + Expect(err).ToNot(HaveOccurred()) + ml = NewModelLoader(systemState) + }) + + It("keeps the model when the tunnel dial failed", func() { + client := grpc.NewClientWithDialer("10.0.0.1:9001", false, nil, false, "", + func(context.Context, string) (net.Conn, error) { + return nil, errors.New("cluster: no route from this replica to that worker") + }) + m := NewModelWithClient("remote-model", "10.0.0.1:9001", client) + ml.store.Set("remote-model", m) + + Expect(ml.checkIsLoaded("remote-model")).To(BeIdenticalTo(m), + "a model on a worker this frontend cannot route to must stay cached, not be unloaded") + _, stillThere := ml.store.Get("remote-model") + Expect(stillThere).To(BeTrue()) + }) + + It("still evicts a remote model whose worker WAS reached and did not answer", func() { + // The other direction, so the new check cannot pass by never evicting. + // No custom dialer, so the transport reports nothing and a connection + // error means what it always meant. + client := grpc.NewClientWithToken("127.0.0.1:1", false, nil, false, "") + m := NewModelWithClient("dead-model", "127.0.0.1:1", client) + ml.store.Set("dead-model", m) + + Expect(ml.checkIsLoaded("dead-model")).To(BeNil()) + _, stillThere := ml.store.Get("dead-model") + Expect(stillThere).To(BeFalse()) + }) +}) From b0a51f57a1d72616d35733ba2ed801fad70d3df3 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 16:33:59 +0000 Subject: [PATCH 056/153] fix(grpc): let the transport answer through the wrappers, not only past gRPC Re-review round 2. One blocking defect, and it was the concern I filed myself last round and mis-scoped as a future trap. It was live, and it sat on the most destructive reaping path of the five. RouteResult.Client is an InFlightTrackingClient, over a FileStagingClient when a stager is configured. model_router puts that on the cached remote model and pkg/model's checkIsLoaded asks IT whether the transport failed. Both wrappers embed grpc.Backend, which does not declare LastDialError, so the type assertion read nil and the guard added last round fell straight through to the old eviction. That eviction sends backend.stop over NATS to every node holding the model and deletes every replica row, where the other sites delete one. The spec covering it built a bare client by hand, which is why it passed while production did not. This is the third time in this task a correct fix was disarmed one layer out, so the fix is a mechanism rather than two methods. BackendUnwrapper is one line per decorator, LastDialErrorOf walks the chain, and both consumers now call it instead of each keeping its own assertion. One implementation, no per-caller policy to get wrong. Sweeping every type that embeds or holds a grpc.Backend found a third decorator the review had not named, and it is itself a reaping consumer of the same collapsed signal. ConnectionEvictingClient is built for remote models in initializers.go and its evict callback runs ShutdownModel; it fires during INFERENCE rather than on a health check, so a tunnel blip mid-request was enough to stop a model that was loaded and serving. It consults the transport first now. A locally spawned backend has no custom transport, so that path is unchanged byte for byte. Everything else touching a Backend is a consumer rather than a decorator; there is no fourth. The probe cache joiner shape is pinned. It was the right design last round with nothing holding it: the mutation back to a closed-over variable passed all 602 specs in the package. Eight goroutines coalesced on a probe that blocks on a channel now assert every joiner gets the leader's REASON and not just its answer, which is the difference between a leader declining to reap and its seven joiners reaping on the leader's own observation. The LastDialError scope note claimed an exactness it does not have at checkIsLoaded, which reads a shared long-lived client after releasing opMutex. It now says which caller is not exact, why the imprecision is accepted there, and what making it exact would cost. The four-outcome table in the docs still said a worker with no live owner is treated as absent and rescheduled, contradicting the code and the paragraph nine lines below it. None of those outcomes is absence any more, and the table says so, names the fifth, and points at the heartbeat as the thing that does decide presence. Five mutations, each reddening named specs, including the two the reviewer found surviving. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/nodes/file_staging_client.go | 8 ++ core/services/nodes/inflight.go | 23 +++- core/services/nodes/interfaces.go | 11 +- core/services/nodes/probe_cache_test.go | 55 +++++++++ core/services/nodes/wrapper_transport_test.go | 115 ++++++++++++++++++ docs/content/features/distributed-mode.md | 9 +- pkg/grpc/backend.go | 48 ++++++++ pkg/grpc/client.go | 29 +++-- pkg/model/connection_evicting_client.go | 33 ++++- pkg/model/loader.go | 17 ++- pkg/model/remote_unroutable_internal_test.go | 47 +++++++ 11 files changed, 362 insertions(+), 33 deletions(-) create mode 100644 core/services/nodes/wrapper_transport_test.go diff --git a/core/services/nodes/file_staging_client.go b/core/services/nodes/file_staging_client.go index 7fa91d71603e..c3d89fc68333 100644 --- a/core/services/nodes/file_staging_client.go +++ b/core/services/nodes/file_staging_client.go @@ -74,6 +74,14 @@ func (f *FileStagingClient) stageTTSReferences(ctx context.Context, lifecycle *s return nil } +// Unwrap exposes the client this one decorates, so grpc.LastDialErrorOf can see +// past it. Without it a staged client answers "the transport was fine" for +// every dial, because embedding grpc.Backend inherits only what Backend +// declares and DialErrorReporter is deliberately not on Backend. +func (f *FileStagingClient) Unwrap() grpc.Backend { return f.Backend } + +var _ grpc.BackendUnwrapper = (*FileStagingClient)(nil) + // NewFileStagingClient creates a new file staging wrapper. func NewFileStagingClient(inner grpc.Backend, stager FileStager, nodeID string) *FileStagingClient { return &FileStagingClient{ diff --git a/core/services/nodes/inflight.go b/core/services/nodes/inflight.go index 08df8411a5d2..50ddc081e493 100644 --- a/core/services/nodes/inflight.go +++ b/core/services/nodes/inflight.go @@ -30,10 +30,15 @@ import ( type InFlightTrackingClient struct { grpc.ControlBackend // passthrough for control-plane / streaming-constructor methods inner grpc.InferenceBackend // tracked inference methods delegate here - registry InFlightTracker - nodeID string - modelName string - replicaIndex int + // wrapped is the SAME object as ControlBackend and inner, kept at its full + // type so Unwrap can hand it back. The two fields above are deliberately + // narrowed to the sub-interfaces, which is what gives the compile-time + // guarantee below, and neither of them can be returned as a grpc.Backend. + wrapped grpc.Backend + registry InFlightTracker + nodeID string + modelName string + replicaIndex int firstOnce sync.Once // guards onFirstComplete onFirstComplete func() // called once after the first tracked inference call completes @@ -44,11 +49,21 @@ type InFlightTrackingClient struct { // InferenceBackend method is left unwrapped. var _ grpc.Backend = (*InFlightTrackingClient)(nil) +// And it must stay transparent to grpc.LastDialErrorOf. This is the wrapper +// SmartRouter puts on every routed client, so a remote model's cached client is +// one of these; without Unwrap, the transport guard in pkg/model reads nil for +// every model the router produced and evicts on a tunnel blip. +var _ grpc.BackendUnwrapper = (*InFlightTrackingClient)(nil) + +// Unwrap exposes the client this one decorates. +func (c *InFlightTrackingClient) Unwrap() grpc.Backend { return c.wrapped } + // NewInFlightTrackingClient wraps a gRPC backend client with in-flight tracking. func NewInFlightTrackingClient(inner grpc.Backend, registry InFlightTracker, nodeID, modelName string, replicaIndex int) *InFlightTrackingClient { return &InFlightTrackingClient{ ControlBackend: inner, inner: inner, + wrapped: inner, registry: registry, nodeID: nodeID, modelName: modelName, diff --git a/core/services/nodes/interfaces.go b/core/services/nodes/interfaces.go index bf46abd8e57d..dc4adf8a161d 100644 --- a/core/services/nodes/interfaces.go +++ b/core/services/nodes/interfaces.go @@ -217,13 +217,12 @@ var ErrNoWorkerDialer = fmt.Errorf("%w: no worker tunnel dialer is configured", // // A client that reports nothing (no custom dialer, or a test double) yields // nil, which means "the call reached a backend" and preserves the behaviour -// every non-distributed caller has always had. +// every non-distributed caller has always had. Decorators are looked through; +// see grpc.BackendUnwrapper for why that is not optional. func unroutable(client grpc.Backend) error { - reporter, ok := client.(grpc.DialErrorReporter) - if !ok { - return nil - } - dialErr := reporter.LastDialError() + // LastDialErrorOf and not a type assertion: the assertion could not see + // past a decorator, and SmartRouter hands every routed client out wrapped. + dialErr := grpc.LastDialErrorOf(client) if dialErr == nil { return nil } diff --git a/core/services/nodes/probe_cache_test.go b/core/services/nodes/probe_cache_test.go index 58e6fa111cb9..d28c4c7b7ade 100644 --- a/core/services/nodes/probe_cache_test.go +++ b/core/services/nodes/probe_cache_test.go @@ -1,6 +1,7 @@ package nodes import ( + "errors" "sync" "sync/atomic" "time" @@ -104,6 +105,60 @@ var _ = Describe("probeCache", func() { } }) + It("hands every coalesced joiner the leader's REASON, not just its answer", func() { + // The hole this shape exists to close, and the one a closed-over + // variable reintroduces. The reason is written only in the goroutine + // that runs the probe; every caller coalesced into that flight would + // read its own unset variable and see nil. In production that means the + // leader correctly declines to reap a replica on an unreachable worker + // while all seven joiners reap it, on the leader's own observation. + c := newProbeCache(time.Minute) + unreached := errors.New("no route to the worker") + + // The probe blocks until every goroutine is inside flight.Do, so the + // joiners are genuinely coalesced rather than serialised. Released by a + // channel, so nothing here waits on a clock. + entered := make(chan struct{}) + release := make(chan struct{}) + var calls int32 + probe := func() (bool, error) { + atomic.AddInt32(&calls, 1) + close(entered) + <-release + return false, unreached + } + + const N = 8 + start := make(chan struct{}) + var wg sync.WaitGroup + reasons := make([]error, N) + alive := make([]bool, N) + for i := 0; i < N; i++ { + wg.Add(1) + go func(i int) { + defer GinkgoRecover() + defer wg.Done() + <-start + alive[i], reasons[i] = c.DoOrCachedResult("k", probe) + }(i) + } + close(start) + + // Only unblock the leader once at least one goroutine is inside the + // probe; the rest are then either waiting on the flight or about to be. + <-entered + close(release) + wg.Wait() + + Expect(atomic.LoadInt32(&calls)).To(Equal(int32(1)), + "singleflight must collapse %d concurrent probes into one", N) + for i := range reasons { + Expect(alive[i]).To(BeFalse(), "goroutine %d saw a different answer", i) + Expect(reasons[i]).To(MatchError(unreached), + "goroutine %d joined the flight and got the answer without the reason, which is how a joiner reaps what the leader would not", i) + } + }) + It("treats different keys independently", func() { c := newProbeCache(time.Minute) var aCalls, bCalls int32 diff --git a/core/services/nodes/wrapper_transport_test.go b/core/services/nodes/wrapper_transport_test.go new file mode 100644 index 000000000000..ff504929a60c --- /dev/null +++ b/core/services/nodes/wrapper_transport_test.go @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: MIT + +package nodes + +import ( + "context" + "errors" + "net" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/cluster" + grpc "github.com/mudler/LocalAI/pkg/grpc" +) + +// The reviewer's spec, plus the production shape it was pointing at. +// +// The guard added at the fourth reap site asks the client whether the TRANSPORT +// failed. In production that client is not the one the factory built: SmartRouter +// hands out result.Client, which is an *InFlightTrackingClient, over a +// *FileStagingClient whenever a stager is configured. Both embed grpc.Backend, +// which does not declare LastDialError, so a type assertion on the outermost +// type read nil and the guard was inert for exactly the models the router +// produces. Every spec that constructed a raw client by hand passed anyway. +// +// This is the third time in this task that a correct fix was disarmed by a +// layer further out, which is why the mechanism is now one walker rather than a +// per-caller assertion. +var _ = Describe("the transport answer through the wrappers the router builds", func() { + var ( + cause error + raw grpc.Backend + ) + + BeforeEach(func() { + cause = errors.New("cluster: no route from this replica to that worker") + f, err := NewTunnelClientFactory("", func(string) func(ctx context.Context, addr string) (net.Conn, error) { + return func(context.Context, string) (net.Conn, error) { return nil, cause } + }) + Expect(err).ToNot(HaveOccurred()) + raw, err = f.NewClientForNode("X", "10.0.0.1:9001", false) + Expect(err).ToNot(HaveOccurred()) + + // Provoke one dial so there is something to report. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, _ = raw.HealthCheck(ctx) + }) + + It("the raw factory client reports, as designed", func() { + Expect(unroutable(raw)).To(MatchError(ErrWorkerUnroutable)) + }) + + It("reports through the in-flight tracker, which is what RouteResult.Client is", func() { + tracked := NewInFlightTrackingClient(raw, &fakeModelRouter{}, "X", "m", 0) + Expect(unroutable(tracked)).To(MatchError(ErrWorkerUnroutable)) + }) + + It("reports through the file staging client, which buildClientForAddr adds", func() { + staged := NewFileStagingClient(raw, nil, "X") + Expect(unroutable(staged)).To(MatchError(ErrWorkerUnroutable)) + }) + + It("reports through BOTH, nested the way production nests them", func() { + // buildClientForAddr wraps in staging, newRouteResult wraps that in + // tracking, model_router puts the result on the cached model, and + // pkg/model's checkIsLoaded asks it. Two layers, and a walker that + // stopped at one would still be wrong here. + nested := NewInFlightTrackingClient(NewFileStagingClient(raw, nil, "X"), &fakeModelRouter{}, "X", "m", 0) + Expect(unroutable(nested)).To(MatchError(ErrWorkerUnroutable)) + }) + + It("still reports nothing through the wrappers when the dial succeeded", func() { + // The other direction, so forwarding cannot pass by always answering + // "unroutable": a backend that genuinely died must still be reapable + // through the same wrappers. + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = listener.Close() }) + + f, err := NewTunnelClientFactory("", func(string) func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + return func(ctx context.Context, _ string) (net.Conn, error) { + return d.DialContext(ctx, "tcp", listener.Addr().String()) + } + }) + Expect(err).ToNot(HaveOccurred()) + live, err := f.NewClientForNode("X", "10.0.0.1:9001", false) + Expect(err).ToNot(HaveOccurred()) + _, _ = live.HealthCheck(context.Background()) + + nested := NewInFlightTrackingClient(NewFileStagingClient(live, nil, "X"), &fakeModelRouter{}, "X", "m", 0) + Expect(unroutable(nested)).To(BeNil()) + }) + + It("keeps the cluster condition matchable through the wrappers", func() { + // Not merely "something failed". The five conditions have to survive + // the decorators as well as gRPC, or the consumers are guessing again. + routed := errors.New("x") + f, err := NewTunnelClientFactory("", func(string) func(ctx context.Context, addr string) (net.Conn, error) { + return func(context.Context, string) (net.Conn, error) { return nil, routed } + }) + Expect(err).ToNot(HaveOccurred()) + c, err := f.NewClientForNode("X", "10.0.0.1:9001", false) + Expect(err).ToNot(HaveOccurred()) + routed = cluster.ErrNoRoute + _, _ = c.HealthCheck(context.Background()) + + nested := NewInFlightTrackingClient(NewFileStagingClient(c, nil, "X"), &fakeModelRouter{}, "X", "m", 0) + got := unroutable(nested) + Expect(got).To(MatchError(cluster.ErrNoRoute)) + Expect(got).ToNot(MatchError(cluster.ErrNoConnection)) + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 5cd4cca96004..fc49074ea300 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -165,16 +165,19 @@ A worker's tunnel lands on exactly one replica, so with N replicas behind a load The dialling replica states how much time its own client has left in the frame that opens the relayed stream, and the owner bounds its work by the smaller of that and its own 15s ceiling. Neither number can lengthen the other: a patient client cannot park the owning replica, and an impatient one cannot be kept waiting on a budget it did not ask for. -Four outcomes are kept apart on purpose, because they call for different actions: +These outcomes are kept apart on purpose, because they call for different actions: | Outcome | What it means | What acts on it | |---|---|---| -| No live owner | No replica holds this worker's tunnel | The worker is treated as absent; its models can be rescheduled | +| No live owner | No replica holds this worker's tunnel | No route right now; the worker's models are **left alone** | | Not the owner | The routing was stale | Resolve the owner again | | Peer unreachable | A replica exists and will not answer | Retry | +| No relay path | This replica cannot reach the owner at all | Report; requests here fail until it can | | The worker refused | The worker answered and said no | Report; the worker is connected | -Only the first is absence. The others are never reported as it, and that is not a stylistic preference: a scheduler told that a connected worker has gone away reclaims every model it is running. +**None of them is absence.** A worker's presence is its **heartbeat**, and a route to it is a separate fact that can be false while the worker is registered, heartbeating and serving every request another replica sends it. So the frontend answers "no route", never "this worker is gone", and nothing on this list causes a model to be rescheduled or a `node_models` row to be deleted. + +That distinction is the whole point rather than a nicety. A scheduler told that a connected worker has gone away stops its backend and reclaims every model it is running, and the events that produce "no route" are ordinary ones: a frontend replica restarting, an ownership row a moment stale, a worker that has not dialled its tunnel yet. A worker is treated as absent only when its **heartbeat** goes stale, which is a separate mechanism with its own threshold (see `--stale-node-threshold`). #### There is no frontend-side fallback, and upgrade order matters diff --git a/pkg/grpc/backend.go b/pkg/grpc/backend.go index 6a32ec29d12f..63d8341c3df7 100644 --- a/pkg/grpc/backend.go +++ b/pkg/grpc/backend.go @@ -72,6 +72,54 @@ type DialErrorReporter interface { LastDialError() error } +// BackendUnwrapper is implemented by a Backend that DECORATES another one. +// +// Every wrapper in this codebase must implement it, and the reason is a defect +// that shipped: a wrapper embeds the Backend interface, so it inherits every +// declared method and NOTHING else. DialErrorReporter is deliberately not +// declared on Backend, so a wrapped client silently stopped answering "did the +// transport fail" and the guard built on that answer read nil in production +// while passing every spec that constructed a raw client by hand. +// +// Implementing this is what makes a decorator transparent to LastDialErrorOf, +// and it is one line rather than a re-implementation per wrapper, so there is +// no per-wrapper policy to get wrong. +type BackendUnwrapper interface { + Unwrap() Backend +} + +// maxBackendUnwrapDepth bounds the walk below. Three wrappers exist today and +// they nest at most two deep; the bound is a guard against a cycle a future +// wrapper could introduce, not a limit anything real approaches. +const maxBackendUnwrapDepth = 16 + +// LastDialErrorOf reports why the most recent dial under b failed, looking +// THROUGH any decorators, or nil when the dial succeeded or nothing under b has +// a custom transport. +// +// It is the single implementation of that question. Its callers +// (core/services/nodes and pkg/model) each had their own type assertion, and an +// assertion cannot see past a wrapper: in production the client handed to +// pkg/model is an *InFlightTrackingClient over a *FileStagingClient over the +// real one, so both callers were asking a wrapper that had no answer and +// reading nil as "the transport was fine". +func LastDialErrorOf(b Backend) error { + for range maxBackendUnwrapDepth { + if b == nil { + return nil + } + if reporter, ok := b.(DialErrorReporter); ok { + return reporter.LastDialError() + } + wrapper, ok := b.(BackendUnwrapper) + if !ok { + return nil + } + b = wrapper.Unwrap() + } + return nil +} + func buildClient(address string, parallel bool, wd WatchDog, enableWatchDog bool, token string) *Client { if !enableWatchDog { wd = nil diff --git a/pkg/grpc/client.go b/pkg/grpc/client.go index c73f813784f0..5af71d9d34cb 100644 --- a/pkg/grpc/client.go +++ b/pkg/grpc/client.go @@ -1460,15 +1460,26 @@ func (c *Client) ModelMetadata(ctx context.Context, in *pb.ModelOptions, opts .. // tell them apart, with the original error VALUE intact, so // core/services/cluster's sentinels survive the trip. // -// Scope, stated exactly. This is the last dial on this CLIENT, not the last -// dial for a particular RPC. A client used for one probe and closed gives exact -// attribution, which is how every reaping path in core/services/nodes uses it. -// A client shared across concurrent RPCs can attribute a dial failure to the -// wrong one; both directions of that error are safe, because a caller consults -// this only when its RPC already failed, and the outcomes are "treat a dead -// backend as unreachable-for-now" (the row survives one extra round) or "treat -// a transport failure as a backend failure" (the behaviour before this -// existed). +// Scope, stated exactly, including where it is NOT exact. +// +// This is the last dial on this CLIENT, not the last dial for a particular RPC. +// Three of the four callers build a client for one probe and close it, so +// attribution there is exact. The fourth, pkg/model's checkIsLoaded, reads the +// model's long-lived SHARED client and consults this after HealthCheck has +// released opMutex, so a concurrent RPC on the same client can record or clear +// the value inside that window. An earlier version of this comment claimed +// exactness for all four; it was wrong. +// +// The imprecision is accepted there rather than designed away, and the reason +// is which way it can go. A caller consults this only when its own RPC already +// failed, so the two outcomes are: a concurrent dial FAILURE makes a genuinely +// dead backend look unreachable-for-now, and its row survives one extra round +// until the transport recovers; or a concurrent dial SUCCESS clears the value +// and a transport failure reads as a backend failure, which is exactly the +// behaviour that existed before any of this. Neither is a new hazard, and the +// second requires a transport that recovered inside the window. Making it exact +// would mean threading a per-call handle through every Backend method, which is +// a far larger change than the failure it would prevent. func (c *Client) LastDialError() error { c.dialErrMu.Lock() defer c.dialErrMu.Unlock() diff --git a/pkg/model/connection_evicting_client.go b/pkg/model/connection_evicting_client.go index 5cf920ede6d2..a984e830aa27 100644 --- a/pkg/model/connection_evicting_client.go +++ b/pkg/model/connection_evicting_client.go @@ -22,6 +22,12 @@ type ConnectionEvictingClient struct { once sync.Once } +var _ grpc.BackendUnwrapper = (*ConnectionEvictingClient)(nil) + +// Unwrap exposes the client this one decorates, so grpc.LastDialErrorOf can see +// past it. +func (c *ConnectionEvictingClient) Unwrap() grpc.Backend { return c.Backend } + func newConnectionEvictingClient(inner grpc.Backend, modelID string, evict func()) grpc.Backend { return &ConnectionEvictingClient{ Backend: inner, @@ -31,13 +37,28 @@ func newConnectionEvictingClient(inner grpc.Backend, modelID string, evict func( } func (c *ConnectionEvictingClient) checkErr(err error) { - if err != nil && isConnectionError(err) { - c.once.Do(func() { - xlog.Warn("Connection error during inference, evicting model from cache", - "model", c.modelID, "error", err) - c.evict() - }) + if err == nil || !isConnectionError(err) { + return + } + // The fifth site of the same shape, and the one reached during INFERENCE + // rather than a health check. evict() runs ShutdownModel, which for a remote + // model sends backend.stop over NATS to every node holding it and deletes + // every replica row. In distributed mode the client underneath reaches the + // backend over the worker's tunnel, and a failure of THAT transport arrives + // as the same codes.Unavailable a dead backend produces; evicting on it + // stops a model that is loaded and serving, on a worker that is + // heartbeating. A locally spawned backend has no custom transport, so this + // reports nil and the behaviour there is exactly what it always was. + if dialErr := grpc.LastDialErrorOf(c.Backend); dialErr != nil { + xlog.Warn("Inference failed because the worker could not be reached; keeping the model", + "model", c.modelID, "error", dialErr) + return } + c.once.Do(func() { + xlog.Warn("Connection error during inference, evicting model from cache", + "model", c.modelID, "error", err) + c.evict() + }) } // --- Intercepted inference methods --- diff --git a/pkg/model/loader.go b/pkg/model/loader.go index a69831871d99..57784ae88e3f 100644 --- a/pkg/model/loader.go +++ b/pkg/model/loader.go @@ -701,6 +701,12 @@ func (ml *ModelLoader) checkIsLoaded(s string) *Model { // codes.Unavailable a dead worker produces. Evicting on it would // unload a model that is loaded and serving. The client records // which of the two happened; see grpc.DialErrorReporter. + // The client here is long-lived and shared, so this reads the last + // dial on it rather than the one this HealthCheck made; see + // (*grpc.Client).LastDialError for why that imprecision is + // accepted. Both directions of it land on behaviour that already + // existed, and the common case (a worker with no route at all) has + // no concurrent success to clear the value. if dialErr := transportFailure(client); dialErr != nil { xlog.Warn("Remote model health check could not reach the worker, keeping cached", "model", s, "error", dialErr) @@ -741,9 +747,10 @@ func (ml *ModelLoader) checkIsLoaded(s string) *Model { // backend, where the address IS a socket on this machine and a failed // connection really does mean the process died. func transportFailure(client grpc.Backend) error { - reporter, ok := client.(grpc.DialErrorReporter) - if !ok { - return nil - } - return reporter.LastDialError() + // LastDialErrorOf and not a type assertion. The client reaching this + // function for a routed remote model is an *InFlightTrackingClient, often + // over a *FileStagingClient, and an assertion on the outermost type reads + // nil for both: they embed grpc.Backend, which does not declare + // LastDialError. That is exactly how this guard shipped inert. + return grpc.LastDialErrorOf(client) } diff --git a/pkg/model/remote_unroutable_internal_test.go b/pkg/model/remote_unroutable_internal_test.go index 25a440d800be..06883176f796 100644 --- a/pkg/model/remote_unroutable_internal_test.go +++ b/pkg/model/remote_unroutable_internal_test.go @@ -11,6 +11,7 @@ import ( . "github.com/onsi/gomega" grpc "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/system" ) @@ -59,3 +60,49 @@ var _ = Describe("the health check on a remote model whose transport failed", fu Expect(stillThere).To(BeFalse()) }) }) + +var _ = Describe("the eviction wrapper on a remote model whose transport failed", func() { + // The FIFTH site of the same shape, found by sweeping the decorators rather + // than being named. initializers.go builds this wrapper for exactly the + // remote models the router produces, and its evict callback runs + // ShutdownModel, which sends backend.stop over NATS to every node holding + // the model and deletes every replica row. It fires during INFERENCE, not + // on a health check, so a tunnel blip mid-request was enough. + failingDial := func(cause error) grpc.Backend { + return grpc.NewClientWithDialer("10.0.0.1:9001", false, nil, false, "", + func(context.Context, string) (net.Conn, error) { return nil, cause }) + } + + It("does not evict when the worker could not be reached", func() { + evicted := 0 + client := newConnectionEvictingClient( + failingDial(errors.New("cluster: no route from this replica to that worker")), + "remote-model", func() { evicted++ }) + + _, err := client.Predict(context.Background(), &pb.PredictOptions{}) + Expect(err).To(HaveOccurred()) + Expect(evicted).To(BeZero(), + "a worker this frontend cannot route to must not have its backend stopped and its rows deleted") + }) + + It("still evicts when the worker WAS reached and the connection failed", func() { + // The other direction. No custom dialer, so nothing reports a transport + // failure and a connection error means what it always meant. + evicted := 0 + client := newConnectionEvictingClient( + grpc.NewClientWithToken("127.0.0.1:1", false, nil, false, ""), + "dead-model", func() { evicted++ }) + + _, err := client.Predict(context.Background(), &pb.PredictOptions{}) + Expect(err).To(HaveOccurred()) + Expect(evicted).To(Equal(1)) + }) + + It("is transparent to the transport question, so a wrapper of it still sees through", func() { + client := newConnectionEvictingClient( + failingDial(errors.New("cluster: no route from this replica to that worker")), + "remote-model", func() {}) + _, _ = client.Predict(context.Background(), &pb.PredictOptions{}) + Expect(grpc.LastDialErrorOf(client)).ToNot(BeNil()) + }) +}) From 050e585b18009b4aa79350fb3db4d3d76c467819 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 17:46:50 +0000 Subject: [PATCH 057/153] test(nodes): make the joiner spec deterministic, and gate the wrapper shape in lint Re-review round 2. One blocking item, and it was a spec I wrote: eight goroutines raced at the probe cache and nothing made them coalesce, so a straggler that missed the flight re-entered the probe and double-closed a channel. It panicked about one run in three and took the four-suite race block down. The green verification I reported was not reproducible, which means one green run was never evidence for a spec that coordinates goroutines. Its comment claimed the probe blocked until every goroutine was inside flight.Do, and that gap was exactly the panic: the comment described the design intended rather than the one written. It is deterministic now rather than tolerant. singleflight.DoChan registers its channel on an in-flight call under the group's own mutex and returns without running its function, so calling it while the leader is provably parked inside the probe joins that exact flight with no window and no dependence on the scheduler. The spec asserts the join really happened, that the joiner got the reason and not only the answer, and that the probe ran once; the entered channel is sent on rather than closed so a second probe fails an assertion instead of panicking. Twenty runs green under race against the committed code, five out of five red on the mutation back to a closed-over variable. The future-decorator gap is closed in the lint gate, but not the way the review suggested, and the reason is worth recording. HasMethod rejects inline signatures outright, its method-reference form needs a package ruleguard's own typechecker can import and that typechecker cannot import this module, and Implements tests the value method set while every Unwrap is on a pointer receiver, so it fired on all three wrappers that already had one. So the safe shape is structural instead. grpc.WrappedBackend gives the same pass-through method set plus Unwrap on a value receiver, and a decorator that embeds it is transparent by construction; forgetting stops being expressible rather than merely discouraged, which is the move loopbackService already makes in the worker. FileStagingClient and ConnectionEvictingClient embed it and their hand-written Unwrap methods are gone. The ruleguard rule then only has to catch the raw embedding, needs no type filter, and cannot misfire. It was verified to fire on a throwaway wrapper and stay silent on a correct one, and reports nothing across core and pkg with the baseline disabled. InFlightTrackingClient is the one exception and says why in a nolint: it embeds ControlBackend deliberately so that leaving an inference method unwrapped breaks the build, and WrappedBackend embeds the full interface, so adopting it would silently restore pass-through for every inference method and delete that guarantee. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .golangci.yml | 22 ++++++ core/services/nodes/file_staging_client.go | 18 ++--- core/services/nodes/inflight.go | 10 +++ core/services/nodes/probe_cache_test.go | 83 ++++++++++++++-------- go.mod | 1 + go.sum | 2 + hack/lint/backend_wrappers.go | 46 ++++++++++++ pkg/grpc/backend.go | 24 +++++++ pkg/model/connection_evicting_client.go | 12 ++-- 9 files changed, 169 insertions(+), 49 deletions(-) create mode 100644 hack/lint/backend_wrappers.go diff --git a/.golangci.yml b/.golangci.yml index d25d1ccb4789..7a68e69a75f2 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -19,7 +19,22 @@ linters: - staticcheck enable: - forbidigo + # gocritic is enabled for ONE checker: ruleguard, which runs the rules in + # hack/lint/. Every other gocritic check is off (disable-all below), so + # this adds no style noise; it is here purely as the gate that catches a + # gRPC backend wrapper written without Unwrap. See + # hack/lint/backend_wrappers.go for why that cannot be a compile-time + # assertion. + - gocritic settings: + gocritic: + disable-all: true + enabled-checks: + - ruleguard + settings: + ruleguard: + failOn: all + rules: '${base-path}/hack/lint/backend_wrappers.go' forbidigo: forbid: - pattern: '^t\.Errorf$' @@ -126,3 +141,10 @@ linters: - path: ^backend/go/whisper/sources/ text: 'http\.(DefaultClient|Get|Post|PostForm|Head)' linters: [forbidigo] + # Test doubles embed grpc.Backend to inherit the interface's method set + # over a NIL value; they decorate nothing, hold no inner client, and have + # no transport answer to forward. The rule targets production wrappers, + # which is where swallowing that answer deletes replica rows. + # gocritic here is only the backend-wrapper ruleguard rule. + - path: _test\.go$ + linters: [gocritic] diff --git a/core/services/nodes/file_staging_client.go b/core/services/nodes/file_staging_client.go index c3d89fc68333..f2599a3383cd 100644 --- a/core/services/nodes/file_staging_client.go +++ b/core/services/nodes/file_staging_client.go @@ -34,9 +34,9 @@ const stagedInputReleaseTimeout = 30 * time.Second // Methods that require no file staging are inherited from the embedded // grpc.Backend; only methods with staging logic are overridden below. type FileStagingClient struct { - grpc.Backend // embedded for pass-through of non-staging methods - stager FileStager - nodeID string + grpc.WrappedBackend // pass-through of non-staging methods, plus Unwrap + stager FileStager + nodeID string mu sync.RWMutex remoteModelPath string // set during LoadModel from staged ModelPath @@ -74,20 +74,14 @@ func (f *FileStagingClient) stageTTSReferences(ctx context.Context, lifecycle *s return nil } -// Unwrap exposes the client this one decorates, so grpc.LastDialErrorOf can see -// past it. Without it a staged client answers "the transport was fine" for -// every dial, because embedding grpc.Backend inherits only what Backend -// declares and DialErrorReporter is deliberately not on Backend. -func (f *FileStagingClient) Unwrap() grpc.Backend { return f.Backend } - var _ grpc.BackendUnwrapper = (*FileStagingClient)(nil) // NewFileStagingClient creates a new file staging wrapper. func NewFileStagingClient(inner grpc.Backend, stager FileStager, nodeID string) *FileStagingClient { return &FileStagingClient{ - Backend: inner, - stager: stager, - nodeID: nodeID, + WrappedBackend: grpc.WrappedBackend{Backend: inner}, + stager: stager, + nodeID: nodeID, } } diff --git a/core/services/nodes/inflight.go b/core/services/nodes/inflight.go index 50ddc081e493..ee598e989f9e 100644 --- a/core/services/nodes/inflight.go +++ b/core/services/nodes/inflight.go @@ -27,6 +27,16 @@ import ( // interface therefore breaks this file's build (see the var assertion below) // until it is wrapped with track() - so a new inference path can't be added // without an in-flight accounting decision. +// The ruleguard rule wants grpc.WrappedBackend here, and this is the one +// decorator that cannot use it. Embedding ControlBackend rather than Backend is +// what produces the compile-time guarantee below: leave an InferenceBackend +// method unwrapped and this type stops satisfying grpc.Backend. WrappedBackend +// embeds the FULL interface, so adopting it would silently restore +// pass-through for every inference method and delete that guarantee. The +// transparency the rule protects is provided explicitly instead, by the Unwrap +// and the grpc.BackendUnwrapper assertion above. +// +//nolint:gocritic // embeds ControlBackend on purpose; Unwrap is declared explicitly type InFlightTrackingClient struct { grpc.ControlBackend // passthrough for control-plane / streaming-constructor methods inner grpc.InferenceBackend // tracked inference methods delegate here diff --git a/core/services/nodes/probe_cache_test.go b/core/services/nodes/probe_cache_test.go index d28c4c7b7ade..8d1e059127a8 100644 --- a/core/services/nodes/probe_cache_test.go +++ b/core/services/nodes/probe_cache_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "golang.org/x/sync/singleflight" ) var _ = Describe("probeCache", func() { @@ -112,51 +113,75 @@ var _ = Describe("probeCache", func() { // read its own unset variable and see nil. In production that means the // leader correctly declines to reap a replica on an unreachable worker // while all seven joiners reap it, on the leader's own observation. + // + // The FIRST version of this spec raced eight goroutines at the cache + // and hoped they coalesced. Nothing made them: a goroutine that arrived + // after the leader's flight finished started its own, re-entered the + // probe and double-closed a channel, so the spec panicked about one run + // in three. Its comment claimed the probe blocked until every goroutine + // was inside flight.Do, which was the design intended rather than the + // one written, and that gap was exactly the panic. + // + // This version does not hope. singleflight.DoChan registers its channel + // on the in-flight call under the group's own mutex and returns WITHOUT + // running its function (x/sync@v0.22.0 singleflight.go:127-132), so + // calling it while the leader is provably parked inside the probe joins + // that exact flight, with no window and no scheduler dependency. The + // group is reachable because this spec lives in the package. c := newProbeCache(time.Minute) unreached := errors.New("no route to the worker") - // The probe blocks until every goroutine is inside flight.Do, so the - // joiners are genuinely coalesced rather than serialised. Released by a - // channel, so nothing here waits on a clock. - entered := make(chan struct{}) + // Buffered, and sent on rather than closed: a probe that somehow ran + // twice must fail an assertion, not panic and take the suite with it. + entered := make(chan struct{}, 4) release := make(chan struct{}) var calls int32 probe := func() (bool, error) { atomic.AddInt32(&calls, 1) - close(entered) + entered <- struct{}{} <-release return false, unreached } - const N = 8 - start := make(chan struct{}) - var wg sync.WaitGroup - reasons := make([]error, N) - alive := make([]bool, N) - for i := 0; i < N; i++ { - wg.Add(1) - go func(i int) { - defer GinkgoRecover() - defer wg.Done() - <-start - alive[i], reasons[i] = c.DoOrCachedResult("k", probe) - }(i) + type leaderResult struct { + alive bool + unreached error } - close(start) + leader := make(chan leaderResult, 1) + go func() { + defer GinkgoRecover() + alive, reason := c.DoOrCachedResult("k", probe) + leader <- leaderResult{alive: alive, unreached: reason} + }() + + // The leader is now inside the probe, so the group holds an entry for + // "k" and will hold it until the probe returns. + Eventually(entered, "10s").Should(Receive()) + + // Deterministically coalesced. This function must never run; if the + // join failed it would, and the assertion below on the probe count + // would catch it too. + joined := c.flight.DoChan("k", func() (any, error) { + Fail("DoChan started its own flight, so nothing was coalesced") + return false, nil + }) - // Only unblock the leader once at least one goroutine is inside the - // probe; the rest are then either waiting on the flight or about to be. - <-entered close(release) - wg.Wait() + + var got leaderResult + Eventually(leader, "10s").Should(Receive(&got)) + Expect(got.alive).To(BeFalse()) + Expect(got.unreached).To(MatchError(unreached), "the caller that RAN the probe must get the reason") + + var shared singleflight.Result + Eventually(joined, "10s").Should(Receive(&shared)) + Expect(shared.Shared).To(BeTrue(), "this caller did not actually join the leader's flight") + Expect(shared.Val).To(Equal(false)) + Expect(shared.Err).To(MatchError(unreached), + "a joiner got the answer without the reason, which is how a joiner reaps what the leader would not") Expect(atomic.LoadInt32(&calls)).To(Equal(int32(1)), - "singleflight must collapse %d concurrent probes into one", N) - for i := range reasons { - Expect(alive[i]).To(BeFalse(), "goroutine %d saw a different answer", i) - Expect(reasons[i]).To(MatchError(unreached), - "goroutine %d joined the flight and got the answer without the reason, which is how a joiner reaps what the leader would not", i) - } + "the probe must have run exactly once") }) It("treats different keys independently", func() { diff --git a/go.mod b/go.mod index a74e4200a310..4833b8e3d819 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/otiai10/openaigo v1.7.0 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/prometheus/client_golang v1.23.2 + github.com/quasilyte/go-ruleguard/dsl v0.3.23 github.com/robfig/cron/v3 v3.0.1 github.com/russross/blackfriday v1.6.0 github.com/sashabaranov/go-openai v1.41.2 diff --git a/go.sum b/go.sum index 891a60d053b2..95fe9da7ed67 100644 --- a/go.sum +++ b/go.sum @@ -1221,6 +1221,8 @@ github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEo github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/quasilyte/go-ruleguard/dsl v0.3.23 h1:lxjt5B6ZCiBeeNO8/oQsegE6fLeCzuMRoVWSkXC4uvY= +github.com/quasilyte/go-ruleguard/dsl v0.3.23/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= diff --git a/hack/lint/backend_wrappers.go b/hack/lint/backend_wrappers.go new file mode 100644 index 000000000000..c320ac9e5ca0 --- /dev/null +++ b/hack/lint/backend_wrappers.go @@ -0,0 +1,46 @@ +//go:build ruleguard + +// Package gorules holds the go-ruleguard rules gocritic runs inside +// `make lint`. It is never compiled into the binary: the build tag keeps it out +// of every normal build, and golangci-lint loads the file as data. +package gorules + +import "github.com/quasilyte/go-ruleguard/dsl" + +// backendWrapperMustBeUnwrappable fires on a struct that decorates a gRPC +// backend by embedding the raw interface. +// +// This exists because the same defect shipped twice. A wrapper that embeds +// grpc.Backend inherits exactly the methods Backend declares and nothing else. +// grpc.DialErrorReporter is deliberately NOT on Backend, so a wrapped client +// silently stops answering "did the transport fail, or did the backend die", +// and every guard built on that answer reads nil. The consequence is not +// subtle: core/services/nodes and pkg/model delete replica rows and stop +// backends on that answer, so a wrapper that swallows it turns a momentary loss +// of route into fleet-wide model reclamation. +// +// The rule is SYNTACTIC, and deliberately so. The obvious formulation, "embeds +// a backend and has no Unwrap", cannot be written: HasMethod rejects inline +// signatures outright ("inline func signatures are not supported yet"), its +// method-reference form needs a package ruleguard's own typechecker can import +// and it cannot import this module, and Implements tests the VALUE method set +// while every Unwrap here would be on a pointer receiver. So instead of +// checking for the method, this checks for the shape that CANNOT lack it: +// grpc.WrappedBackend provides the same pass-through method set plus Unwrap, +// with a value receiver, so anything embedding it is transparent by +// construction. Forgetting is then not expressible rather than merely +// discouraged, which is the same move loopbackService makes in the worker. +// +// Test doubles are excluded by path in .golangci.yml: they embed a NIL backend +// to inherit the interface's method set, decorate nothing, and have no +// transport answer to forward. +func backendWrapperMustBeUnwrappable(m dsl.Matcher) { + m.Import("github.com/mudler/LocalAI/pkg/grpc") + + m.Match( + `type $w struct { $*_; grpc.Backend; $*_ }`, + `type $w struct { $*_; grpc.ControlBackend; $*_ }`, + `type $w struct { $*_; grpc.InferenceBackend; $*_ }`, + ). + Report(`$w decorates a gRPC backend by embedding the raw interface, so grpc.LastDialErrorOf cannot see through it and every transport-failure guard behind it reads nil, which deletes replica rows for workers that are merely unroutable. Embed grpc.WrappedBackend instead: it gives the same pass-through plus Unwrap. If $w decorates nothing, silence this with //nolint:gocritic and say so.`) +} diff --git a/pkg/grpc/backend.go b/pkg/grpc/backend.go index 63d8341c3df7..f38799260b71 100644 --- a/pkg/grpc/backend.go +++ b/pkg/grpc/backend.go @@ -88,6 +88,30 @@ type BackendUnwrapper interface { Unwrap() Backend } +// WrappedBackend is what a decorator embeds INSTEAD of a Backend. +// +// It provides the pass-through method set exactly as embedding the interface +// did, and it provides Unwrap, so a decorator built on it is transparent to +// LastDialErrorOf by CONSTRUCTION rather than by remembering. That is the whole +// design: the same defect shipped twice, both times because a wrapper inherited +// only what Backend declares and DialErrorReporter is deliberately not on +// Backend, so the transport answer every reaping guard depends on silently +// became nil. +// +// Forgetting is therefore no longer possible for anything that embeds this, and +// embedding the raw interface instead is caught by the ruleguard rule in +// hack/lint/. A compile-time assertion cannot do that job: it only fires for a +// type that already declares the intent, which is precisely the type that did +// not forget. +// +// Unwrap takes a VALUE receiver, which is safe because this holds one interface +// and no lock, and is what lets the value type of any embedder satisfy +// BackendUnwrapper. +type WrappedBackend struct{ Backend } + +// Unwrap exposes the decorated client. +func (w WrappedBackend) Unwrap() Backend { return w.Backend } + // maxBackendUnwrapDepth bounds the walk below. Three wrappers exist today and // they nest at most two deep; the bound is a guard against a cycle a future // wrapper could introduce, not a limit anything real approaches. diff --git a/pkg/model/connection_evicting_client.go b/pkg/model/connection_evicting_client.go index a984e830aa27..f957051b2459 100644 --- a/pkg/model/connection_evicting_client.go +++ b/pkg/model/connection_evicting_client.go @@ -16,7 +16,7 @@ import ( // still returned to the caller — the NEXT request will trigger rescheduling // via SmartRouter. type ConnectionEvictingClient struct { - grpc.Backend + grpc.WrappedBackend modelID string evict func() once sync.Once @@ -24,15 +24,11 @@ type ConnectionEvictingClient struct { var _ grpc.BackendUnwrapper = (*ConnectionEvictingClient)(nil) -// Unwrap exposes the client this one decorates, so grpc.LastDialErrorOf can see -// past it. -func (c *ConnectionEvictingClient) Unwrap() grpc.Backend { return c.Backend } - func newConnectionEvictingClient(inner grpc.Backend, modelID string, evict func()) grpc.Backend { return &ConnectionEvictingClient{ - Backend: inner, - modelID: modelID, - evict: evict, + WrappedBackend: grpc.WrappedBackend{Backend: inner}, + modelID: modelID, + evict: evict, } } From 21f1fcc80b002677d5e961773b701897d70cecf5 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 18:03:52 +0000 Subject: [PATCH 058/153] docs(grpc): withdraw the lint-cost claim, and make the one nolint tidy-proof I reported that enabling gocritic pushed make lint past 600s. That was wrong, and it was wrong in a way worth naming: those runs happened right after I changed pkg/grpc's and core/services/nodes' interfaces, so the Go build cache was cold for essentially the whole repository including every backend, and test suites were running concurrently on the same machine. I attributed a cold-cache full-repo typecheck under load to the linter I had just enabled, and raised it as a cost without ever timing it against a baseline. A number with no control is not a measurement. Measured properly, with the golangci cache cleaned before every run and isolated GOCACHE directories for the cold ones so the shared cache was not wiped: warm, base 15s then 7s and current 8s then 7s; cold, base 87s and current 78s running base first, base 136s and current 79s running current first. The spread between the two cold base runs is larger than any gap between base and current, so gocritic with only the ruleguard checker costs nothing measurable. So the rule stays, unscoped. Scoping it to core and pkg was the fallback for a cost that does not exist, and adding that configuration would buy nothing. The one override gets the protection it needs instead. InFlightTrackingClient's nolint is exactly the kind of thing a later reader tidies away, so it now opens by saying not to, and states what breaks rather than what is intended: WrappedBackend embeds the full Backend interface, so adopting it there would promote every inference method as untracked pass-through, the build would stay GREEN, and in-flight accounting would silently stop covering whatever was added next. WrappedBackend's own doc carries the counterpart warning so a reader arriving from either side finds it. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/nodes/inflight.go | 27 ++++++++++++++++++--------- pkg/grpc/backend.go | 6 ++++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/core/services/nodes/inflight.go b/core/services/nodes/inflight.go index ee598e989f9e..b8b363a2544b 100644 --- a/core/services/nodes/inflight.go +++ b/core/services/nodes/inflight.go @@ -27,16 +27,25 @@ import ( // interface therefore breaks this file's build (see the var assertion below) // until it is wrapped with track() - so a new inference path can't be added // without an in-flight accounting decision. -// The ruleguard rule wants grpc.WrappedBackend here, and this is the one -// decorator that cannot use it. Embedding ControlBackend rather than Backend is -// what produces the compile-time guarantee below: leave an InferenceBackend -// method unwrapped and this type stops satisfying grpc.Backend. WrappedBackend -// embeds the FULL interface, so adopting it would silently restore -// pass-through for every inference method and delete that guarantee. The -// transparency the rule protects is provided explicitly instead, by the Unwrap -// and the grpc.BackendUnwrapper assertion above. +// DO NOT "fix" this by embedding grpc.WrappedBackend, and do not delete the +// nolint below. Both look like tidy-ups and both silently remove a guarantee. // -//nolint:gocritic // embeds ControlBackend on purpose; Unwrap is declared explicitly +// The ruleguard rule in hack/lint/ asks every decorator to embed +// grpc.WrappedBackend, because that makes Unwrap structural. This is the one +// decorator that must not, and the reason is the paragraph above: embedding +// ControlBackend rather than Backend is exactly what forces every +// InferenceBackend method to be declared and tracked here, on pain of a build +// failure. grpc.WrappedBackend embeds the FULL Backend interface, so adopting +// it would promote every inference method as untracked pass-through, the build +// would stay green, and in-flight accounting would silently stop covering +// whatever was added next. +// +// The transparency the rule exists to protect is still provided, explicitly: +// the wrapped field, the Unwrap method and the grpc.BackendUnwrapper assertion +// above. A spec drives it (see the wrapper transport specs), so removing them +// reddens rather than merely regressing. +// +//nolint:gocritic // embeds ControlBackend deliberately; see the paragraph above before changing this type InFlightTrackingClient struct { grpc.ControlBackend // passthrough for control-plane / streaming-constructor methods inner grpc.InferenceBackend // tracked inference methods delegate here diff --git a/pkg/grpc/backend.go b/pkg/grpc/backend.go index f38799260b71..8448afcb92ef 100644 --- a/pkg/grpc/backend.go +++ b/pkg/grpc/backend.go @@ -107,6 +107,12 @@ type BackendUnwrapper interface { // Unwrap takes a VALUE receiver, which is safe because this holds one interface // and no lock, and is what lets the value type of any embedder satisfy // BackendUnwrapper. +// +// It is NOT for every decorator. Embedding this promotes the whole Backend +// surface as pass-through, so a decorator that deliberately embeds a NARROWER +// interface to force itself to handle each method (see +// nodes.InFlightTrackingClient) must keep doing that and declare Unwrap by +// hand; adopting this there would restore pass-through silently. type WrappedBackend struct{ Backend } // Unwrap exposes the decorated client. From 256d4ee7396dd0202df9c212e27d33752a12fe17 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 18:47:37 +0000 Subject: [PATCH 059/153] feat(distributed): stop workers listening, and stop them advertising A worker now opens no listener on a routable interface and states no endpoint at registration. Backend processes and the file-transfer server bind loopback, and the frontend reaches both through the tunnel the worker dials. The bind address is built from loopbackHost, the same constant the tunnel's grpc tag dials, so "the worker binds where its tunnel dials" is one fact in one place rather than two literals that can drift. All three advertisement sites are closed, not one: the registration body, RegisterNodeRequest, and the per-backend address in the install reply. That third one was hiding a live bug. stopModelExact refuses a stop whose ExpectedAddress does not match what the worker recorded for the process. The worker recorded 127.0.0.1:port; handleBackendInstall reported advertiseHost:port; the router stored the reported one and sent it straight back. On any worker whose advertise host was not 127.0.0.1, every acknowledged model stop failed with an address mismatch. Nothing caught it because the e2e harness set LOCALAI_ADVERTISE_ADDR=127.0.0.1, which made the rewrite a no-op. Removing the rewrite makes the two strings the same by construction. The brief was wrong about two of the four functions it called dead. effectiveBasePort is the base of the backend port allocator and resolveHTTPAddr is the file server's bind address; deleting them would have deleted the port allocator and the file server. Only the two advertise* helpers were dead, and addr_test.go is rewritten rather than deleted, because the port arithmetic it pinned still needs pinning. NodeModel.Address survives with a narrowed meaning and is renamed WorkerLocalAddress, along with the install reply field that feeds it. The frontend still has to say WHICH backend process on a worker it means, and the port in this string is how it says it: it travels as a stream target and the worker dials its own loopback. The gorm column and the json key stay "address", so neither a migration nor an API break rides along. Every fall-back to the node's address is gone. installBackendOnNode now errors when a worker reports success without naming one, because substituting the now-always-empty node address would name an empty target, and the worker refuses that as an invalid stream, which is classified as the worker answering about its backend. That is the "a present worker reads as something it is not" class this phase forbids. DistributedModelStore.Range had the same shape and was already wrong: it built each remote model's client from the node's base gRPC port, never the port a backend process listens on, so Free and Status went to the wrong place. It uses the replica's address now. BackendNode.Address and HTTPAddress are kept but made provably inert: no writer, no reader that acts on them, and Register force-clears both on re-registration so an upgraded worker's stale advertisement does not outlive its own upgrade in the API and the Nodes page. Dropping the columns is a ~90-site edit across the specs, the e2e suite, the MCP dto and the UI; it is recorded as a follow-up rather than folded in here. A persistent tunnel 401 still does not trigger re-registration, and now for a reason rather than a deferral. Register CLEARS the node's replica rows, so re-registering on a 401 would delete a live worker's rows on every retry, and under the name collision that causes the 401 the two workers would take turns doing it forever: a credential failure causing model reclamation. It also cannot fix the named cause, since a collision is indistinguishable from a restart. The 401 log now names both causes and says nothing can reach this worker, which is true only now that it has no listener. The container healthcheck did not break the way the brief expected, since the listener still exists on loopback and the probe runs inside the container. It did have a real #10987 defect that this change makes the common case: it read LOCALAI_SERVE_ADDR only, while effectiveBasePort reads LOCALAI_ADDR first, so a worker on a non-default base port was probed on 50050 and reported unhealthy while working. It follows the same precedence now. Docs, the compose file and the e2e harness are updated in step: no inbound rule or published port is needed for a worker, the two advertise variables are gone, the remaining address variables are read for their port only, the firewall-the-file-transfer-port warning is narrowed to the LOCALAI_HTTP_ADDR opt-out, and the upgrade-order note no longer claims the worker still listens. The Nodes page showed node.address, which is now always blank, so it shows the node id instead. Eight mutations, all red on a named spec, including reverting the loopback bind, re-adding the address to the registration body, restoring both node-address fall-backs, dropping the force-clear, storing the endpoint's address again, and un-fixing the healthcheck. One of them caught a defect in a spec I had just written: it asserted 200 where the endpoint returns 201, which went unnoticed because core/http/endpoints/localai is not on the task's verify list. It is run here. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/localai/nodes.go | 27 ++--- core/http/endpoints/localai/nodes_test.go | 40 +++++-- .../src/components/nodes/NodePanel.jsx | 6 +- core/services/messaging/subjects.go | 15 ++- core/services/nodes/disk_headroom_test.go | 2 +- core/services/nodes/distributed_store.go | 16 ++- core/services/nodes/distributed_store_test.go | 51 +++++++-- core/services/nodes/health.go | 16 ++- core/services/nodes/health_mock_test.go | 12 ++ core/services/nodes/health_test.go | 14 +-- .../nodes/managers_distributed_test.go | 16 +-- core/services/nodes/model_router.go | 4 +- core/services/nodes/model_router_test.go | 14 ++- core/services/nodes/reconciler.go | 12 +- .../nodes/reconciler_busy_probe_test.go | 14 +-- .../nodes/reconciler_inflight_leak_test.go | 16 +-- core/services/nodes/reconciler_test.go | 36 +++--- .../nodes/reconciler_worker_processes_test.go | 14 +-- core/services/nodes/registry.go | 64 ++++++++--- core/services/nodes/registry_test.go | 36 +++++- .../nodes/revision_eligibility_test.go | 6 +- core/services/nodes/router.go | 58 +++++++--- .../nodes/router_eviction_alias_test.go | 2 +- .../nodes/router_eviction_selector_test.go | 2 +- .../services/nodes/router_load_budget_test.go | 2 +- core/services/nodes/router_load_job_test.go | 2 +- .../nodes/router_load_timeout_test.go | 2 +- core/services/nodes/router_reap_load_test.go | 2 +- .../services/nodes/router_reservation_test.go | 2 +- .../nodes/router_revision_lifecycle_test.go | 2 +- .../nodes/router_staging_context_test.go | 4 +- .../nodes/router_staging_deadline_test.go | 4 +- core/services/nodes/router_test.go | 70 +++++++++--- .../nodes/router_unreachable_worker_test.go | 4 +- core/services/nodes/unloader.go | 2 +- core/services/nodes/unloader_test.go | 6 +- core/services/worker/addr_test.go | 103 ++++++++++-------- core/services/worker/config.go | 22 ++-- core/services/worker/lifecycle.go | 26 ++--- core/services/worker/registration.go | 51 +++------ core/services/worker/supervisor.go | 37 +++++-- core/services/worker/tunnel.go | 9 +- core/services/worker/worker.go | 25 +++-- docker-compose.distributed.yaml | 23 ++-- docs/content/features/distributed-mode.md | 53 +++++---- scripts/build/healthcheck.sh | 23 +++- scripts/build/healthcheck_test.sh | 15 +++ tests/e2e/distributed/cluster/cluster.go | 6 +- .../distributed/model_config_revision_test.go | 2 +- 49 files changed, 633 insertions(+), 357 deletions(-) diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index 56b2b8743a99..6d2bd378af01 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -77,10 +77,14 @@ func GetNodeEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc { // RegisterNodeRequest is the request body for registering a new worker node. type RegisterNodeRequest struct { - Name string `json:"name"` - NodeType string `json:"node_type,omitempty"` // "backend" (default) or "agent" - Address string `json:"address"` - HTTPAddress string `json:"http_address,omitempty"` + Name string `json:"name"` + NodeType string `json:"node_type,omitempty"` // "backend" (default) or "agent" + // No address and no http_address. A worker has no inbound endpoint to + // register: it holds one outbound tunnel to a frontend replica and every + // protocol the frontend speaks to it travels on that. An older worker still + // sends both keys and they are ignored, which is the intended outcome: + // storing them would put a dialable-looking endpoint back in the API for + // something nothing dials. Token string `json:"token,omitempty"` TotalVRAM uint64 `json:"total_vram,omitempty"` AvailableVRAM uint64 `json:"available_vram,omitempty"` @@ -145,22 +149,15 @@ func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, au fmt.Sprintf("invalid node_type %q; must be %q or %q", nodeType, nodes.NodeTypeBackend, nodes.NodeTypeAgent))) } - // Backend workers require address; agent workers don't serve gRPC + // A backend worker no longer has to state an address; the tunnel it + // dials is what makes it reachable, and requiring one here would refuse + // exactly the workers this design is for. if req.Name == "" { return c.JSON(http.StatusBadRequest, nodeError(http.StatusBadRequest, "name is required")) } - if nodeType == nodes.NodeTypeBackend && req.Address == "" { - return c.JSON(http.StatusBadRequest, nodeError(http.StatusBadRequest, "address is required for backend workers")) - } if len(req.Name) > 255 { return c.JSON(http.StatusBadRequest, nodeError(http.StatusBadRequest, "name exceeds 255 characters")) } - if len(req.Address) > 512 { - return c.JSON(http.StatusBadRequest, nodeError(http.StatusBadRequest, "address exceeds 512 characters")) - } - if len(req.HTTPAddress) > 512 { - return c.JSON(http.StatusBadRequest, nodeError(http.StatusBadRequest, "http_address exceeds 512 characters")) - } // Hash the token for storage (if provided) var tokenHash string @@ -180,8 +177,6 @@ func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, au node := &nodes.BackendNode{ Name: req.Name, NodeType: nodeType, - Address: req.Address, - HTTPAddress: req.HTTPAddress, TokenHash: tokenHash, TotalVRAM: req.TotalVRAM, AvailableVRAM: req.AvailableVRAM, diff --git a/core/http/endpoints/localai/nodes_test.go b/core/http/endpoints/localai/nodes_test.go index adce16826869..63f49e7c3c7e 100644 --- a/core/http/endpoints/localai/nodes_test.go +++ b/core/http/endpoints/localai/nodes_test.go @@ -318,7 +318,11 @@ var _ = Describe("Node HTTP handlers", func() { Expect(errObj["message"]).To(ContainSubstring("exceeds 255 characters")) }) - It("returns 400 when address is missing for backend node type", func() { + It("registers a backend worker that states no address", func() { + // This used to be a 400. It is the shape every worker now + // registers with: it has no inbound endpoint, it holds one outbound + // tunnel, and refusing it here would refuse exactly the workers the + // tunnel exists for. e := echo.New() body := `{"name":"worker-no-addr"}` req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) @@ -328,13 +332,35 @@ var _ = Describe("Node HTTP handlers", func() { handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) Expect(handler(c)).To(Succeed()) - Expect(rec.Code).To(Equal(http.StatusBadRequest)) + Expect(rec.Code).To(Equal(http.StatusCreated)) - var resp map[string]any - Expect(json.Unmarshal(rec.Body.Bytes(), &resp)).To(Succeed()) - errObj, ok := resp["error"].(map[string]any) - Expect(ok).To(BeTrue()) - Expect(errObj["message"]).To(ContainSubstring("address is required")) + stored, err := registry.GetByName(context.Background(), "worker-no-addr") + Expect(err).ToNot(HaveOccurred()) + Expect(stored.NodeType).To(Equal(nodes.NodeTypeBackend)) + Expect(stored.Address).To(BeEmpty()) + Expect(stored.HTTPAddress).To(BeEmpty()) + }) + + It("stores no address even when a worker still sends one", func() { + // An older worker keeps sending both keys. Storing them would put a + // dialable-looking endpoint back into the API and the Nodes page for + // something nothing dials, and would leave a reader of either one + // unsure which workers are reached how. + e := echo.New() + body := `{"name":"worker-legacy-addr","address":"10.0.0.9:50051","http_address":"10.0.0.9:50050"}` + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + + handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + Expect(handler(c)).To(Succeed()) + Expect(rec.Code).To(Equal(http.StatusCreated)) + + stored, err := registry.GetByName(context.Background(), "worker-legacy-addr") + Expect(err).ToNot(HaveOccurred()) + Expect(stored.Address).To(BeEmpty()) + Expect(stored.HTTPAddress).To(BeEmpty()) }) It("returns 400 when node_type is invalid", func() { diff --git a/core/http/react-ui/src/components/nodes/NodePanel.jsx b/core/http/react-ui/src/components/nodes/NodePanel.jsx index 623db00936bf..b73714ab9972 100644 --- a/core/http/react-ui/src/components/nodes/NodePanel.jsx +++ b/core/http/react-ui/src/components/nodes/NodePanel.jsx @@ -19,7 +19,11 @@ export default function NodePanel({ node, models = [], onApprove, onDrain, onRes
{node.name} - {node.address} + {/* A worker has no address to show: it holds an outbound tunnel and + binds nothing routable. Its id is what identifies it in routing + logs, so that is what an operator needs here. Pre-tunnel nodes + may still carry an address until they re-register. */} + {node.address || node.id}
e.stopPropagation()}> {node.status === 'pending' && ( diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index dc3a435d9279..43366bced1c6 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -193,9 +193,18 @@ type BackendInstallRequest struct { // BackendInstallReply is the response from a backend.install NATS request. type BackendInstallReply struct { - Success bool `json:"success"` - Address string `json:"address,omitempty"` // gRPC address of the backend process (host:port) - Error string `json:"error,omitempty"` + Success bool `json:"success"` + // WorkerLocalAddress is where the backend process listens ON THE WORKER, + // which is a loopback address. It is not dialable from the frontend and + // never was meant to be read that way: the frontend takes its PORT and + // names it as the target of a stream on that worker's tunnel, and the + // worker dials its own loopback there. + // + // The json tag stays "address" so a worker and a frontend from different + // releases still understand each other. An older worker sends its + // advertised host here; only the port is read, and the port is the same. + WorkerLocalAddress string `json:"address,omitempty"` + Error string `json:"error,omitempty"` } // SubjectNodeBackendUpgrade tells a worker node to force-reinstall a backend diff --git a/core/services/nodes/disk_headroom_test.go b/core/services/nodes/disk_headroom_test.go index f14abd8292d2..536add2f3773 100644 --- a/core/services/nodes/disk_headroom_test.go +++ b/core/services/nodes/disk_headroom_test.go @@ -145,7 +145,7 @@ var _ = Describe("scheduling a model onto a cluster without disk headroom", func reg.findIdleNode = &BackendNode{ID: "n1", Name: "nvidia-thor", Address: "10.0.0.1:50051"} backend = &holdBackend{} unloader = &fakeUnloader{ - installReply: &messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:9001"}, + installReply: &messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:9001"}, } router = NewSmartRouter(reg, SmartRouterOptions{ Unloader: unloader, diff --git a/core/services/nodes/distributed_store.go b/core/services/nodes/distributed_store.go index 542aec69e693..5e1a359035fa 100644 --- a/core/services/nodes/distributed_store.go +++ b/core/services/nodes/distributed_store.go @@ -91,10 +91,14 @@ func (s *DistributedModelStore) Range(fn func(string, *model.Model) bool) { } seen[nm.ModelName] = true - // Look up the node address - node, err := s.registry.Get(ctx, nm.NodeID) - if err != nil { - xlog.Warn("DistributedModelStore: failed to get node for model", "model", nm.ModelName, "nodeID", nm.NodeID, "error", err) + // The REPLICA's address, not the node's. This used to name the node, + // which was the worker's base gRPC port and never the port the backend + // process actually listens on, so Free and Status on a model reached + // from here went to the wrong place; with workers no longer advertising + // anything it would name nothing at all. + if nm.WorkerLocalAddress == "" { + xlog.Warn("DistributedModelStore: not listing a replica whose backend process is unnamed", + "model", nm.ModelName, "nodeID", nm.NodeID, "replica", nm.ReplicaIndex) continue } @@ -103,13 +107,13 @@ func (s *DistributedModelStore) Range(fn func(string, *model.Model) bool) { // anything calls GRPC() on it, which reaches a worker only while // workers still listen on a routable address. Building the client here // means the bypass has no path left rather than an unused one. - client, err := s.clientFor(nm.NodeID, node.Address) + client, err := s.clientFor(nm.NodeID, nm.WorkerLocalAddress) if err != nil { xlog.Error("DistributedModelStore: not listing a remote model it cannot reach", "model", nm.ModelName, "nodeID", nm.NodeID, "error", err) continue } - m := model.NewModelWithClient(nm.ModelName, node.Address, client) + m := model.NewModelWithClient(nm.ModelName, nm.WorkerLocalAddress, client) if !fn(nm.ModelName, m) { return } diff --git a/core/services/nodes/distributed_store_test.go b/core/services/nodes/distributed_store_test.go index 08362fbfddca..92ed6d0a5b4b 100644 --- a/core/services/nodes/distributed_store_test.go +++ b/core/services/nodes/distributed_store_test.go @@ -99,11 +99,11 @@ var _ = Describe("DistributedModelStore", func() { local.Set("model-a", localModel) // DB model (not in local) - dbNode := &BackendNode{ID: "node-2", Address: "10.0.0.3:50051"} + dbNode := &BackendNode{ID: "node-2"} lookup.nodes["node-2"] = dbNode lookup.allModels = []NodeModel{ - {NodeID: "node-2", ModelName: "model-b"}, - {NodeID: "node-2", ModelName: "model-a"}, // duplicate — should be skipped + {NodeID: "node-2", ModelName: "model-b", WorkerLocalAddress: "127.0.0.1:50052"}, + {NodeID: "node-2", ModelName: "model-a", WorkerLocalAddress: "127.0.0.1:50053"}, // duplicate, should be skipped } visited := make(map[string]bool) @@ -124,9 +124,9 @@ var _ = Describe("DistributedModelStore", func() { // bypasses the worker's tunnel completely. It is reached in // production: ShutdownModel calls Free on it and the backend // monitor calls Status. - dbNode := &BackendNode{ID: "node-2", Address: "10.0.0.3:50051"} + dbNode := &BackendNode{ID: "node-2"} lookup.nodes["node-2"] = dbNode - lookup.allModels = []NodeModel{{NodeID: "node-2", ModelName: "remote-model"}} + lookup.allModels = []NodeModel{{NodeID: "node-2", ModelName: "remote-model", WorkerLocalAddress: "127.0.0.1:50052"}} var got *model.Model store.Range(func(id string, m *model.Model) bool { @@ -142,15 +142,48 @@ var _ = Describe("DistributedModelStore", func() { Expect(clients.nodesSeen()).To(ContainElement("node-2")) }) + It("names the replica's own backend process, not the node", func() { + // This used to pass the NODE's address, which was the worker's base + // gRPC port and never the port a backend process listens on, so + // Free and Status on a model listed here went to the wrong process. + // A node has no address at all now, so the same code would name the + // empty string and the worker would refuse the stream as invalid, a + // refusal that reads as the backend answering about itself. + lookup.nodes["node-2"] = &BackendNode{ID: "node-2"} + lookup.allModels = []NodeModel{{ + NodeID: "node-2", ModelName: "remote-model", ReplicaIndex: 1, + WorkerLocalAddress: "127.0.0.1:50057", + }} + + store.Range(func(string, *model.Model) bool { return true }) + Expect(clients.addressesSeen()).To(ConsistOf("127.0.0.1:50057")) + }) + + It("skips a replica row that names no backend process", func() { + // Nothing can be routed to it, and handing back a model whose + // client targets an empty address turns every Free and Status on it + // into an invalid-stream refusal from the worker. + lookup.nodes["node-2"] = &BackendNode{ID: "node-2"} + lookup.allModels = []NodeModel{{NodeID: "node-2", ModelName: "unnamed-model"}} + + visited := map[string]bool{} + store.Range(func(id string, _ *model.Model) bool { + visited[id] = true + return true + }) + Expect(visited).ToNot(HaveKey("unnamed-model")) + Expect(clients.addressesSeen()).To(BeEmpty()) + }) + It("refuses to list a remote model it has no way to reach", func() { // Loudly, not by falling back. A model handed back here with a // direct-dialling client works on a single-host developer setup and // fails against every worker with no inbound port, which is the // worst way for this defect to behave. clients.refuseForNode = errors.New("no tunnel for you") - dbNode := &BackendNode{ID: "node-2", Address: "10.0.0.3:50051"} + dbNode := &BackendNode{ID: "node-2"} lookup.nodes["node-2"] = dbNode - lookup.allModels = []NodeModel{{NodeID: "node-2", ModelName: "remote-model"}} + lookup.allModels = []NodeModel{{NodeID: "node-2", ModelName: "remote-model", WorkerLocalAddress: "127.0.0.1:50052"}} visited := map[string]bool{} store.Range(func(id string, _ *model.Model) bool { @@ -162,9 +195,9 @@ var _ = Describe("DistributedModelStore", func() { It("refuses when no client factory was wired at all", func() { bare := NewDistributedModelStore(local, lookup, nil) - dbNode := &BackendNode{ID: "node-2", Address: "10.0.0.3:50051"} + dbNode := &BackendNode{ID: "node-2"} lookup.nodes["node-2"] = dbNode - lookup.allModels = []NodeModel{{NodeID: "node-2", ModelName: "remote-model"}} + lookup.allModels = []NodeModel{{NodeID: "node-2", ModelName: "remote-model", WorkerLocalAddress: "127.0.0.1:50052"}} visited := map[string]bool{} bare.Range(func(id string, _ *model.Model) bool { diff --git a/core/services/nodes/health.go b/core/services/nodes/health.go index f0cbbbf514b5..cf8b7b57fbad 100644 --- a/core/services/nodes/health.go +++ b/core/services/nodes/health.go @@ -190,16 +190,22 @@ func (hm *HealthMonitor) doCheckAll(ctx context.Context) { if hm.perModelHealthCheck { models, _ := hm.registry.GetNodeModels(ctx, node.ID) for _, m := range models { - if m.Address == "" || m.Address == node.Address { + // A row with no address names no backend process, so there is + // nothing to probe. The old second arm of this test skipped a + // replica whose address equalled the NODE's; a node has no + // address any more, so that comparison could only ever be true + // for two empty strings and has been dropped rather than left + // to read as a live rule. + if m.WorkerLocalAddress == "" { continue } - // Through the node's tunnel, never a direct dial to m.Address: + // Through the node's tunnel, never a direct dial to m.WorkerLocalAddress: // that address is a port inside the worker. A worker this // replica cannot reach is not evidence that its backend died, // so the miss counter is left alone and the row survives; // counting it as a miss would reap live models across the whole // fleet the moment the tunnel wiring was wrong. - mClient, err := hm.clientFactory.NewClientForNode(node.ID, m.Address, false) + mClient, err := hm.clientFactory.NewClientForNode(node.ID, m.WorkerLocalAddress, false) if err != nil { xlog.Error("Skipping model health probe: no way to reach the worker", "node", node.ID, "model", m.ModelName, "replica", m.ReplicaIndex, "error", err) @@ -241,12 +247,12 @@ func (hm *HealthMonitor) doCheckAll(ctx context.Context) { if misses < perModelMissThreshold { xlog.Debug("Model backend probe failed, awaiting threshold before removal", "node", node.ID, "model", m.ModelName, "replica", m.ReplicaIndex, - "address", m.Address, "misses", misses, "threshold", perModelMissThreshold) + "address", m.WorkerLocalAddress, "misses", misses, "threshold", perModelMissThreshold) continue } xlog.Warn("Model backend unhealthy after consecutive misses, removing from registry", "node", node.ID, "model", m.ModelName, "replica", m.ReplicaIndex, - "address", m.Address, "misses", misses) + "address", m.WorkerLocalAddress, "misses", misses) if err := hm.registry.RemoveNodeModel(ctx, node.ID, m.ModelName, m.ReplicaIndex); err != nil { xlog.Warn("Failed to remove unhealthy model from registry", "node", node.ID, "model", m.ModelName, "replica", m.ReplicaIndex, "error", err) diff --git a/core/services/nodes/health_mock_test.go b/core/services/nodes/health_mock_test.go index 9f6bd311fa91..2685694b9959 100644 --- a/core/services/nodes/health_mock_test.go +++ b/core/services/nodes/health_mock_test.go @@ -315,6 +315,10 @@ type fakeBackendClientFactory struct { defaultClient *fakeBackendClient // forNode records every node id NewClientForNode was asked for. forNode []string + // forNodeAddr records the ADDRESS asked for alongside each node id, so a + // spec can pin which of the two addresses a caller reached for: a replica + // row's own, or the node's, the second of which is now always empty. + forNodeAddr []string // refuseForNode makes NewClientForNode fail, standing in for a deployment // with no way to reach the worker. Set before the code under test runs. refuseForNode error @@ -350,12 +354,20 @@ func (f *fakeBackendClientFactory) nodesSeen() []string { return append([]string(nil), f.forNode...) } +// addressesSeen records the addresses passed alongside those node ids. +func (f *fakeBackendClientFactory) addressesSeen() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.forNodeAddr...) +} + func (f *fakeBackendClientFactory) NewClientForNode(nodeID, address string, parallel bool) (grpc.Backend, error) { if f.refuseForNode != nil { return nil, f.refuseForNode } f.mu.Lock() f.forNode = append(f.forNode, nodeID) + f.forNodeAddr = append(f.forNodeAddr, address) f.mu.Unlock() return f.NewClient(address, parallel), nil } diff --git a/core/services/nodes/health_test.go b/core/services/nodes/health_test.go index a20384737f94..95f10e7dca20 100644 --- a/core/services/nodes/health_test.go +++ b/core/services/nodes/health_test.go @@ -245,7 +245,7 @@ var _ = Describe("HealthMonitor (mock-based)", func() { // node should remain healthy because heartbeat is fresh node := makeTestNode("node-crash", "crash-worker", "10.0.0.9:50051", StatusHealthy, freshTime()) store.addNode(node) - store.addNodeModel("node-crash", NodeModel{NodeID: "node-crash", ModelName: "piper-model", Address: "10.0.0.9:50053"}) + store.addNodeModel("node-crash", NodeModel{NodeID: "node-crash", ModelName: "piper-model", WorkerLocalAddress: "10.0.0.9:50053"}) // gRPC backend is dead — but health is heartbeat-based, not gRPC-based factory.setClient("10.0.0.9:50051", &fakeBackendClient{healthy: false, err: fmt.Errorf("connection refused")}) @@ -265,7 +265,7 @@ var _ = Describe("HealthMonitor (mock-based)", func() { node := makeTestNode("node-model", "model-worker", "10.0.0.10:50051", StatusHealthy, freshTime()) store.addNode(node) - store.addNodeModel("node-model", NodeModel{NodeID: "node-model", ModelName: "piper-model", Address: "10.0.0.10:50053"}) + store.addNodeModel("node-model", NodeModel{NodeID: "node-model", ModelName: "piper-model", WorkerLocalAddress: "10.0.0.10:50053"}) // Model backend is dead factory.setClient("10.0.0.10:50053", &fakeBackendClient{healthy: false, err: fmt.Errorf("connection refused")}) @@ -299,7 +299,7 @@ var _ = Describe("HealthMonitor (mock-based)", func() { node := makeTestNode("node-tun", "tun-worker", "10.0.0.20:50051", StatusHealthy, freshTime()) store.addNode(node) - store.addNodeModel("node-tun", NodeModel{NodeID: "node-tun", ModelName: "m", Address: "10.0.0.20:50053"}) + store.addNodeModel("node-tun", NodeModel{NodeID: "node-tun", ModelName: "m", WorkerLocalAddress: "10.0.0.20:50053"}) hm.doCheckAll(context.Background()) Expect(factory.nodesSeen()).To(ContainElement("node-tun")) @@ -318,7 +318,7 @@ var _ = Describe("HealthMonitor (mock-based)", func() { node := makeTestNode("node-cut", "cut-worker", "10.0.0.21:50051", StatusHealthy, freshTime()) store.addNode(node) - store.addNodeModel("node-cut", NodeModel{NodeID: "node-cut", ModelName: "m", Address: "10.0.0.21:50053"}) + store.addNodeModel("node-cut", NodeModel{NodeID: "node-cut", ModelName: "m", WorkerLocalAddress: "10.0.0.21:50053"}) for i := 0; i < perModelMissThreshold+1; i++ { hm.doCheckAll(context.Background()) @@ -340,7 +340,7 @@ var _ = Describe("HealthMonitor (mock-based)", func() { node := makeTestNode("node-blip", "blip-worker", "10.0.0.22:50051", StatusHealthy, freshTime()) store.addNode(node) - store.addNodeModel("node-blip", NodeModel{NodeID: "node-blip", ModelName: "m", Address: "10.0.0.22:50053"}) + store.addNodeModel("node-blip", NodeModel{NodeID: "node-blip", ModelName: "m", WorkerLocalAddress: "10.0.0.22:50053"}) factory.setClient("10.0.0.22:50053", &fakeBackendClient{ healthy: false, err: fmt.Errorf("connection error"), @@ -364,7 +364,7 @@ var _ = Describe("HealthMonitor (mock-based)", func() { node := makeTestNode("node-dead", "dead-worker", "10.0.0.23:50051", StatusHealthy, freshTime()) store.addNode(node) - store.addNodeModel("node-dead", NodeModel{NodeID: "node-dead", ModelName: "m", Address: "10.0.0.23:50053"}) + store.addNodeModel("node-dead", NodeModel{NodeID: "node-dead", ModelName: "m", WorkerLocalAddress: "10.0.0.23:50053"}) // No dialErr: the transport was fine. factory.setClient("10.0.0.23:50053", &fakeBackendClient{healthy: false, err: fmt.Errorf("connection refused")}) @@ -382,7 +382,7 @@ var _ = Describe("HealthMonitor (mock-based)", func() { node := makeTestNode("node-flap", "flap-worker", "10.0.0.11:50051", StatusHealthy, freshTime()) store.addNode(node) - store.addNodeModel("node-flap", NodeModel{NodeID: "node-flap", ModelName: "piper-model", Address: "10.0.0.11:50053"}) + store.addNodeModel("node-flap", NodeModel{NodeID: "node-flap", ModelName: "piper-model", WorkerLocalAddress: "10.0.0.11:50053"}) deadClient := &fakeBackendClient{healthy: false, err: fmt.Errorf("connection refused")} liveClient := &fakeBackendClient{healthy: true} diff --git a/core/services/nodes/managers_distributed_test.go b/core/services/nodes/managers_distributed_test.go index b83200eeb1a3..a72707ea2871 100644 --- a/core/services/nodes/managers_distributed_test.go +++ b/core/services/nodes/managers_distributed_test.go @@ -386,9 +386,9 @@ var _ = Describe("DistributedBackendManager", func() { n2 := registerHealthyBackend("worker-b", "10.0.0.2:50051") mc.scriptReply(messaging.SubjectNodeBackendInstall(n1.ID), - messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:50100"}) + messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:50100"}) mc.scriptReply(messaging.SubjectNodeBackendInstall(n2.ID), - messaging.BackendInstallReply{Success: true, Address: "10.0.0.2:50100"}) + messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.2:50100"}) Expect(mgr.InstallBackend(ctx, op("vllm-development"), nil)).To(Succeed()) }) @@ -420,7 +420,7 @@ var _ = Describe("DistributedBackendManager", func() { bad := registerHealthyBackend("worker-bad", "10.0.0.2:50051") mc.scriptReply(messaging.SubjectNodeBackendInstall(ok.ID), - messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:50100"}) + messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:50100"}) mc.scriptReply(messaging.SubjectNodeBackendInstall(bad.ID), messaging.BackendInstallReply{Success: false, Error: "out of memory"}) @@ -459,7 +459,7 @@ var _ = Describe("DistributedBackendManager", func() { other := registerHealthyBackend("worker-other", "10.0.0.2:50051") mc.scriptReply(messaging.SubjectNodeBackendInstall(target.ID), - messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:50100"}) + messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:50100"}) // No reply scripted for `other`: if InstallBackend fans out // to it, the fakeNoRespondersErr default would surface and // the test would fail. @@ -615,7 +615,7 @@ var _ = Describe("DistributedBackendManager", func() { It("invokes progressCb once per worker-published progress event", func() { node := registerHealthyBackend("worker-prog", "10.0.0.7:50051") - mc.scriptReply(messaging.SubjectNodeBackendInstall(node.ID), messaging.BackendInstallReply{Success: true, Address: "10.0.0.7:50051"}) + mc.scriptReply(messaging.SubjectNodeBackendInstall(node.ID), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.7:50051"}) mc.scheduleProgressPublish(node.ID, "op-prog-1", []messaging.BackendInstallProgressEvent{ {OpID: "op-prog-1", NodeID: node.ID, Backend: "vllm", FileName: "vllm.tar", Current: "100 MB", Total: "1 GB", Percentage: 10}, {OpID: "op-prog-1", NodeID: node.ID, Backend: "vllm", FileName: "vllm.tar", Current: "1 GB", Total: "1 GB", Percentage: 100}, @@ -659,7 +659,7 @@ var _ = Describe("DistributedBackendManager", func() { Context("InstallBackend tolerates silent (pre-Phase-2) workers", func() { It("completes successfully even when no progress events are ever published", func() { node := registerHealthyBackend("worker-silent", "10.0.0.8:50051") - mc.scriptReply(messaging.SubjectNodeBackendInstall(node.ID), messaging.BackendInstallReply{Success: true, Address: "10.0.0.8:50051"}) + mc.scriptReply(messaging.SubjectNodeBackendInstall(node.ID), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.8:50051"}) // NO scheduleProgressPublish call - silent worker. var ticks int @@ -702,7 +702,7 @@ var _ = Describe("DistributedBackendManager", func() { It("emits a success entry for each healthy node visited", func() { node := registerHealthyBackend("worker-ok", "10.0.0.9:50051") mc.scriptReply(messaging.SubjectNodeBackendInstall(node.ID), - messaging.BackendInstallReply{Success: true, Address: "10.0.0.9:50051"}) + messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.9:50051"}) opVal := op("vllm") opVal.ID = "op-node-success" @@ -929,7 +929,7 @@ var _ = Describe("DistributedBackendManager", func() { // Fallback re-fires legacy backend.install with Force=true. mc.scriptReplyMatching(messaging.SubjectNodeBackendInstall(n.ID), func(req messaging.BackendInstallRequest) bool { return req.Force }, - messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:50100"}) + messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:50100"}) Expect(mgr.UpgradeBackend(ctx, upgradeOp("vllm-development"), nil)).To(Succeed()) }) diff --git a/core/services/nodes/model_router.go b/core/services/nodes/model_router.go index 2d29fe528a92..661c6526c2c0 100644 --- a/core/services/nodes/model_router.go +++ b/core/services/nodes/model_router.go @@ -68,7 +68,7 @@ func (a *ModelRouterAdapter) Route(ctx context.Context, backend, modelID, modelN // If file staging is configured, it's already wrapped with FileStagingClient // by SmartRouter. Use NewModelWithClient so the wrapper is preserved when // the ModelLoader returns this model on subsequent requests. - m := model.NewModelWithClient(modelID, result.Node.Address, result.Client) + m := model.NewModelWithClient(modelID, result.WorkerLocalAddress, result.Client) // Publish the picked node ID into the per-request holder attached to // ctx (by middleware.ExposeNodeHeader). No-op when the holder is @@ -80,7 +80,7 @@ func (a *ModelRouterAdapter) Route(ctx context.Context, backend, modelID, modelN // concurrently to different replicas. distributedhdr.Stamp(ctx, result.Node.ID) - xlog.Info("Model routed to remote node", "model", modelName, "node", result.Node.Name, "address", result.Node.Address) + xlog.Info("Model routed to remote node", "model", modelName, "node", result.Node.Name, "address", result.WorkerLocalAddress) return m, nil } diff --git a/core/services/nodes/model_router_test.go b/core/services/nodes/model_router_test.go index fe19884070f5..42e7a5dd19f2 100644 --- a/core/services/nodes/model_router_test.go +++ b/core/services/nodes/model_router_test.go @@ -199,13 +199,15 @@ var _ = Describe("ModelRouterAdapter", func() { Describe("Route", func() { It("delegates to SmartRouter and stores release func", func() { fakeNode := &BackendNode{ - ID: "node-1", - Name: "test-node", - Address: "10.0.0.1:50051", + ID: "node-1", + Name: "test-node", } + // The replica row carries the address now; the node has none. A row + // without one is not routable and the warm path declines it. fakeNM := &NodeModel{ - NodeID: "node-1", - ModelName: "test-model", + NodeID: "node-1", + ModelName: "test-model", + WorkerLocalAddress: "127.0.0.1:50052", } fakeReg := newFakeModelRouterForSmartRouter() @@ -214,7 +216,7 @@ var _ = Describe("ModelRouterAdapter", func() { // The fake gRPC client that SmartRouter will use for health check factory := newFakeBackendClientFactory() - factory.setClient("10.0.0.1:50051", &fakeBackendClient{healthy: true}) + factory.setClient("127.0.0.1:50052", &fakeBackendClient{healthy: true}) sr := NewSmartRouter(fakeReg, SmartRouterOptions{ ClientFactory: factory, diff --git a/core/services/nodes/reconciler.go b/core/services/nodes/reconciler.go index 3c192d004060..2b235ec6dcc3 100644 --- a/core/services/nodes/reconciler.go +++ b/core/services/nodes/reconciler.go @@ -533,14 +533,14 @@ func (rc *ReplicaReconciler) probeLoadedModels(ctx context.Context) { return } seen[m.ID] = struct{}{} - switch rc.prober.Probe(ctx, m.NodeID, m.Address) { + switch rc.prober.Probe(ctx, m.NodeID, m.WorkerLocalAddress) { case ProbeUnknown: // This frontend could not reach the worker to ask. The streak is // left exactly as it was: neither cleared, which would forgive a // backend that really is dead, nor advanced, which would reap every // model in the fleet the moment the tunnel wiring broke. xlog.Warn("Reconciler: could not probe a model, leaving its row alone", - "node", m.NodeID, "model", m.ModelName, "replica", m.ReplicaIndex, "address", m.Address) + "node", m.NodeID, "model", m.ModelName, "replica", m.ReplicaIndex, "address", m.WorkerLocalAddress) continue case ProbeAlive: rc.clearProbeFailures(m.ID) @@ -552,14 +552,14 @@ func (rc *ReplicaReconciler) probeLoadedModels(ctx context.Context) { // Reachable but mid-request. Proof of life, so clear the streak. rc.clearProbeFailures(m.ID) xlog.Debug("Reconciler: model busy, skipping liveness reap", - "node", m.NodeID, "model", m.ModelName, "replica", m.ReplicaIndex, "address", m.Address) + "node", m.NodeID, "model", m.ModelName, "replica", m.ReplicaIndex, "address", m.WorkerLocalAddress) continue } failures := rc.recordProbeFailure(m.ID) if failures < probeFailuresBeforeReap { xlog.Debug("Reconciler: model unreachable, waiting for more misses before reaping", - "node", m.NodeID, "model", m.ModelName, "replica", m.ReplicaIndex, "address", m.Address, + "node", m.NodeID, "model", m.ModelName, "replica", m.ReplicaIndex, "address", m.WorkerLocalAddress, "failures", failures, "threshold", probeFailuresBeforeReap) continue } @@ -569,7 +569,7 @@ func (rc *ReplicaReconciler) probeLoadedModels(ctx context.Context) { } rc.clearProbeFailures(m.ID) xlog.Warn("Reconciler: model unreachable, removed from registry", - "node", m.NodeID, "model", m.ModelName, "replica", m.ReplicaIndex, "address", m.Address, + "node", m.NodeID, "model", m.ModelName, "replica", m.ReplicaIndex, "address", m.WorkerLocalAddress, "failures", failures) } rc.pruneProbeFailures(seen) @@ -624,7 +624,7 @@ func (rc *ReplicaReconciler) sweepLeakedInFlight(ctx context.Context) { return } seen[m.ID] = struct{}{} - if rc.prober.Probe(ctx, m.NodeID, m.Address) != ProbeAlive { + if rc.prober.Probe(ctx, m.NodeID, m.WorkerLocalAddress) != ProbeAlive { // Busy or unreachable. Busy means the counter may well be real; // unreachable is the reaper's business, not the sweeper's. rc.clearInFlightIdle(m.ID) diff --git a/core/services/nodes/reconciler_busy_probe_test.go b/core/services/nodes/reconciler_busy_probe_test.go index c2a6b076836c..7bf53ee57b0d 100644 --- a/core/services/nodes/reconciler_busy_probe_test.go +++ b/core/services/nodes/reconciler_busy_probe_test.go @@ -48,13 +48,13 @@ var _ = Describe("ReplicaReconciler — probe reaper vs busy backends", func() { // seed inserts a stale loaded row so the probe pass picks it up. seed := func(id string, inFlight int) { Expect(db.Create(&NodeModel{ - ID: id, - NodeID: node.ID, - ModelName: id, - Address: addr, - State: "loaded", - InFlight: inFlight, - UpdatedAt: time.Now().Add(-5 * time.Minute), + ID: id, + NodeID: node.ID, + ModelName: id, + WorkerLocalAddress: addr, + State: "loaded", + InFlight: inFlight, + UpdatedAt: time.Now().Add(-5 * time.Minute), }).Error).To(Succeed()) } diff --git a/core/services/nodes/reconciler_inflight_leak_test.go b/core/services/nodes/reconciler_inflight_leak_test.go index 4898d8852f38..94fe7793002a 100644 --- a/core/services/nodes/reconciler_inflight_leak_test.go +++ b/core/services/nodes/reconciler_inflight_leak_test.go @@ -46,14 +46,14 @@ var _ = Describe("ReplicaReconciler — leaked in_flight sweeper", func() { seed := func(id string, inFlight int, idleFor time.Duration) { Expect(db.Create(&NodeModel{ - ID: id, - NodeID: node.ID, - ModelName: id, - Address: addr, - State: "loaded", - InFlight: inFlight, - LastUsed: time.Now().Add(-idleFor), - UpdatedAt: time.Now(), + ID: id, + NodeID: node.ID, + ModelName: id, + WorkerLocalAddress: addr, + State: "loaded", + InFlight: inFlight, + LastUsed: time.Now().Add(-idleFor), + UpdatedAt: time.Now(), }).Error).To(Succeed()) } diff --git a/core/services/nodes/reconciler_test.go b/core/services/nodes/reconciler_test.go index 1ed0b47d8ad7..b91cacc29c26 100644 --- a/core/services/nodes/reconciler_test.go +++ b/core/services/nodes/reconciler_test.go @@ -770,20 +770,20 @@ var _ = Describe("ReplicaReconciler — state reconciliation", func() { Expect(registry.Register(context.Background(), node, true)).To(Succeed()) // Two loaded models — one stale (will probe), one fresh (skipped). stale := &NodeModel{ - ID: "stale-1", - NodeID: node.ID, - ModelName: "stale-model", - Address: "10.0.0.1:12345", - State: "loaded", - UpdatedAt: time.Now().Add(-5 * time.Minute), + ID: "stale-1", + NodeID: node.ID, + ModelName: "stale-model", + WorkerLocalAddress: "10.0.0.1:12345", + State: "loaded", + UpdatedAt: time.Now().Add(-5 * time.Minute), } fresh := &NodeModel{ - ID: "fresh-1", - NodeID: node.ID, - ModelName: "fresh-model", - Address: "10.0.0.1:54321", - State: "loaded", - UpdatedAt: time.Now(), // within probeStaleAfter + ID: "fresh-1", + NodeID: node.ID, + ModelName: "fresh-model", + WorkerLocalAddress: "10.0.0.1:54321", + State: "loaded", + UpdatedAt: time.Now(), // within probeStaleAfter } Expect(db.Create(stale).Error).To(Succeed()) Expect(db.Create(fresh).Error).To(Succeed()) @@ -815,12 +815,12 @@ var _ = Describe("ReplicaReconciler — state reconciliation", func() { node := &BackendNode{Name: "n1", NodeType: NodeTypeBackend, Address: "10.0.0.1:50051"} Expect(registry.Register(context.Background(), node, true)).To(Succeed()) stale := &NodeModel{ - ID: "stale-2", - NodeID: node.ID, - ModelName: "alive-model", - Address: "10.0.0.1:12345", - State: "loaded", - UpdatedAt: time.Now().Add(-5 * time.Minute), + ID: "stale-2", + NodeID: node.ID, + ModelName: "alive-model", + WorkerLocalAddress: "10.0.0.1:12345", + State: "loaded", + UpdatedAt: time.Now().Add(-5 * time.Minute), } Expect(db.Create(stale).Error).To(Succeed()) diff --git a/core/services/nodes/reconciler_worker_processes_test.go b/core/services/nodes/reconciler_worker_processes_test.go index 8fd848b1d2de..84459690f066 100644 --- a/core/services/nodes/reconciler_worker_processes_test.go +++ b/core/services/nodes/reconciler_worker_processes_test.go @@ -54,13 +54,13 @@ var _ = Describe("ReplicaReconciler — reconcile against worker processes", fun seed := func(id, modelName string, replica int, age time.Duration) { Expect(db.Create(&NodeModel{ - ID: id, - NodeID: node.ID, - ModelName: modelName, - ReplicaIndex: replica, - Address: "10.0.0.1:12345", - State: "loaded", - UpdatedAt: time.Now().Add(-age), + ID: id, + NodeID: node.ID, + ModelName: modelName, + ReplicaIndex: replica, + WorkerLocalAddress: "10.0.0.1:12345", + State: "loaded", + UpdatedAt: time.Now().Add(-age), }).Error).To(Succeed()) } diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index 606bcf2af6f8..cecc605555fc 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -23,11 +23,24 @@ import ( // Workers are generic — they don't have a fixed backend type. // The SmartRouter dynamically installs backends via NATS backend.install events. type BackendNode struct { - ID string `gorm:"primaryKey;size:36" json:"id"` - Name string `gorm:"uniqueIndex;size:255" json:"name"` - NodeType string `gorm:"size:32;default:backend" json:"node_type"` // backend, agent - Address string `gorm:"size:255" json:"address"` // host:port for gRPC - HTTPAddress string `gorm:"size:255" json:"http_address"` // host:port for HTTP file transfer + ID string `gorm:"primaryKey;size:36" json:"id"` + Name string `gorm:"uniqueIndex;size:255" json:"name"` + NodeType string `gorm:"size:32;default:backend" json:"node_type"` // backend, agent + // Address and HTTPAddress are what a PRE-TUNNEL worker advertised as its + // inbound gRPC and HTTP endpoints. Nothing dials them and nothing reads + // them to make a decision: a worker holds one outbound tunnel and every + // protocol the frontend speaks to it travels on that. + // + // A worker running this release sends neither, and Register force-clears + // both ON RE-REGISTRATION so an upgraded worker's stale advertisement does + // not survive its own upgrade and keep showing in the API and the UI. A + // first registration writes what it was given, which is how a spec that + // builds a node with an address still gets one. They are kept as + // columns rather than dropped only because dropping them is a wide, + // mechanical change across the fleet of specs that build a BackendNode, + // and they are inert either way. + Address string `gorm:"size:255" json:"address"` + HTTPAddress string `gorm:"size:255" json:"http_address"` Status string `gorm:"size:32;default:registering" json:"status"` // registering, healthy, unhealthy, draining, pending TokenHash string `gorm:"size:64" json:"-"` // SHA-256 of registration token // TunnelTokenHash is the SHA-256 of this node's OWN tunnel credential, the @@ -149,14 +162,28 @@ var ( // // Multiple replicas of the same model on the same node are allowed; each // replica has its own ReplicaIndex (0..MaxReplicasPerModel-1), its own -// gRPC Address (each replica is a separate worker process on its own port), -// and its own InFlight counter. +// WorkerLocalAddress (each replica is a separate worker process on its own +// port), and its own InFlight counter. type NodeModel struct { - ID string `gorm:"primaryKey;size:36" json:"id"` - NodeID string `gorm:"index;size:36" json:"node_id"` - ModelName string `gorm:"index;size:255" json:"model_name"` - ReplicaIndex int `gorm:"column:replica_index;default:0;index" json:"replica_index"` - Address string `gorm:"size:255" json:"address"` // gRPC address for this replica's backend process + ID string `gorm:"primaryKey;size:36" json:"id"` + NodeID string `gorm:"index;size:36" json:"node_id"` + ModelName string `gorm:"index;size:255" json:"model_name"` + ReplicaIndex int `gorm:"column:replica_index;default:0;index" json:"replica_index"` + // WorkerLocalAddress is where this replica's backend process listens ON + // ITS WORKER. It is a loopback address and it is not dialable from here. + // + // It survived the removal of every worker address for one reason: the + // frontend still has to say WHICH backend process on a worker it means, + // and the port in this string is how it says it. It travels as the target + // of a stream on that worker's tunnel; the worker reads the port, checks + // it against its own allocator range, and dials its own loopback. Nothing + // in the frontend may treat it as a dial target, which is why it is not + // called Address any more: the old name is what a reader had to already + // know the design to interpret correctly. + // + // The column and the json key stay "address" so no migration and no API + // break rides along with the rename. + WorkerLocalAddress string `gorm:"column:address;size:255" json:"address"` State string `gorm:"size:32;default:idle" json:"state"` // staging, loading, loaded, unloading, idle InFlight int `json:"in_flight"` // number of active requests on this replica LastUsed time.Time `json:"last_used"` @@ -648,6 +675,15 @@ func (r *NodeRegistry) Register(ctx context.Context, node *BackendNode, autoAppr return fmt.Errorf("clearing worker VRAM budget for node %s: %w", node.Name, err) } } + // Force-clear the advertised addresses. Updates(struct) zero-skips, so a + // node that registered before workers stopped advertising would keep the + // host:port it reported then for the rest of its life, and the API and + // the Nodes page would keep showing an endpoint that nothing dials and + // that may not even exist any more. + if err := r.db.WithContext(ctx).Model(&BackendNode{}).Where("id = ?", node.ID). + Updates(map[string]any{"address": node.Address, "http_address": node.HTTPAddress}).Error; err != nil { + return fmt.Errorf("clearing the advertised addresses for node %s: %w", node.Name, err) + } // Force-write the disk columns. Updates(struct) above zero-skips, and a // worker whose models filesystem is 100% full re-registers with // available_disk == 0 — the single most important reading there is. @@ -724,7 +760,7 @@ func (r *NodeRegistry) Register(ctx context.Context, node *BackendNode, autoAppr return fmt.Errorf("looking up node %s: %w", node.Name, err) } - xlog.Info("Node registered", "name", node.Name, "address", node.Address, "status", node.Status) + xlog.Info("Node registered", "name", node.Name, "id", node.ID, "status", node.Status) // Cluster capacity may have changed: a new healthy node, a returning // node, or one with different MaxReplicasPerModel. Wake any configs the // reconciler put in cooldown — the next tick will re-flag if still @@ -1715,7 +1751,7 @@ func (r *NodeRegistry) ClaimModelCleanupRetries(ctx context.Context, now, leaseU func (r *NodeRegistry) RemoveClaimedModelCleanup(ctx context.Context, replica NodeModel) (bool, error) { result := r.db.WithContext(ctx). Where("id = ? AND node_id = ? AND model_name = ? AND replica_index = ? AND state = ? AND address = ? AND config_revision = ?", - replica.ID, replica.NodeID, replica.ModelName, replica.ReplicaIndex, "unloading", replica.Address, replica.ConfigRevision). + replica.ID, replica.NodeID, replica.ModelName, replica.ReplicaIndex, "unloading", replica.WorkerLocalAddress, replica.ConfigRevision). Delete(&NodeModel{}) if result.Error != nil { return false, result.Error diff --git a/core/services/nodes/registry_test.go b/core/services/nodes/registry_test.go index 8d14323ec403..4287849ec8f2 100644 --- a/core/services/nodes/registry_test.go +++ b/core/services/nodes/registry_test.go @@ -70,6 +70,32 @@ var _ = Describe("NodeRegistry", func() { Expect(registry.Register(context.Background(), node, true)).To(Succeed()) Expect(node.Status).To(Equal(StatusHealthy)) }) + + It("clears the advertised addresses a pre-tunnel worker left behind", func() { + // The struct update zero-skips, so an upgraded worker that stops + // sending an address would otherwise keep the one it reported before + // the upgrade for the rest of the row's life, and the API and the + // Nodes page would keep offering an endpoint nothing dials and that + // may not exist any more. + ctx := context.Background() + legacy := makeNode("worker-upgraded", "10.0.0.8:50051", 8_000_000_000) + legacy.HTTPAddress = "10.0.0.8:50050" + Expect(registry.Register(ctx, legacy, true)).To(Succeed()) + + stored, err := registry.GetByName(ctx, "worker-upgraded") + Expect(err).ToNot(HaveOccurred()) + Expect(stored.Address).To(Equal("10.0.0.8:50051"), "precondition: the old row carries the advertisement") + + // Same name, no address: what the upgraded worker sends. + upgraded := makeNode("worker-upgraded", "", 8_000_000_000) + Expect(registry.Register(ctx, upgraded, true)).To(Succeed()) + + stored, err = registry.GetByName(ctx, "worker-upgraded") + Expect(err).ToNot(HaveOccurred()) + Expect(stored.ID).To(Equal(legacy.ID), "precondition: this is the same row, not a new one") + Expect(stored.Address).To(BeEmpty()) + Expect(stored.HTTPAddress).To(BeEmpty()) + }) }) Describe("Re-registration", func() { @@ -196,7 +222,7 @@ var _ = Describe("NodeRegistry", func() { Expect(err).ToNot(HaveOccurred()) Expect(nm2.ID).To(Equal(nm1.ID), "ID should remain stable across SetNodeModel calls") - Expect(nm2.Address).To(Equal("10.0.0.99:50053"), "Address should be updated") + Expect(nm2.WorkerLocalAddress).To(Equal("10.0.0.99:50053"), "Address should be updated") }) }) @@ -1068,8 +1094,8 @@ var _ = Describe("NodeRegistry", func() { for _, m := range models { byIdx[m.ReplicaIndex] = m } - Expect(byIdx[0].Address).To(Equal("127.0.0.1:50100")) - Expect(byIdx[1].Address).To(Equal("127.0.0.1:50101")) + Expect(byIdx[0].WorkerLocalAddress).To(Equal("127.0.0.1:50100")) + Expect(byIdx[1].WorkerLocalAddress).To(Equal("127.0.0.1:50101")) Expect(byIdx[0].ID).ToNot(Equal(byIdx[1].ID)) }) @@ -1086,7 +1112,7 @@ var _ = Describe("NodeRegistry", func() { survivor, err := registry.GetNodeModel(context.Background(), node.ID, "kept-model", 1) Expect(err).ToNot(HaveOccurred()) Expect(survivor).ToNot(BeNil()) - Expect(survivor.Address).To(Equal("127.0.0.1:50111")) + Expect(survivor.WorkerLocalAddress).To(Equal("127.0.0.1:50111")) // Replica 0 is gone _, err = registry.GetNodeModel(context.Background(), node.ID, "kept-model", 0) @@ -1829,7 +1855,7 @@ var _ = Describe("NodeRegistry", func() { Expect(err).ToNot(HaveOccurred()) Expect(models).To(ConsistOf(And( HaveField("ConfigRevision", "rev-new"), - HaveField("Address", "10.0.2.20:7001"), + HaveField("WorkerLocalAddress", "10.0.2.20:7001"), HaveField("State", "loaded"), ))) }) diff --git a/core/services/nodes/revision_eligibility_test.go b/core/services/nodes/revision_eligibility_test.go index 0cc3faf6f623..6d3a3481ada9 100644 --- a/core/services/nodes/revision_eligibility_test.go +++ b/core/services/nodes/revision_eligibility_test.go @@ -54,7 +54,7 @@ var _ = Describe("revision eligibility consumers", func() { } Expect(db.Create(&NodeModel{ ID: kind, NodeID: node.ID, ModelName: modelName, ReplicaIndex: i, - Address: kind, State: state, ConfigRevision: revision, LastUsed: time.Now().Add(time.Duration(i) * time.Minute), + WorkerLocalAddress: kind, State: state, ConfigRevision: revision, LastUsed: time.Now().Add(time.Duration(i) * time.Minute), UpdatedAt: time.Now().Add(-time.Hour), }).Error).To(Succeed()) } @@ -148,7 +148,7 @@ var _ = Describe("revision eligibility consumers", func() { Expect(db.Model(&NodeModel{}).Where("id = ?", "mismatch").Update("replica_index", 9).Error).To(Succeed()) Expect(db.Create(&NodeModel{ ID: "current-extra", NodeID: nodes["current"].ID, ModelName: modelName, - ReplicaIndex: 4, Address: "current-extra", State: "loaded", + ReplicaIndex: 4, WorkerLocalAddress: "current-extra", State: "loaded", ConfigRevision: "current", LastUsed: time.Now().Add(-time.Hour), }).Error).To(Succeed()) @@ -214,7 +214,7 @@ var _ = Describe("revision eligibility consumers", func() { // the minimum and the oldest eligible current row may be selected. Expect(db.Create(&NodeModel{ ID: "current-extra", NodeID: nodes["current"].ID, ModelName: modelName, - ReplicaIndex: 4, Address: "current-extra", State: "loaded", + ReplicaIndex: 4, WorkerLocalAddress: "current-extra", State: "loaded", ConfigRevision: "current", LastUsed: time.Now().Add(time.Minute), }).Error).To(Succeed()) unloader := &fakeUnloader{} diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index 564949255784..fbbeab061b4f 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -493,7 +493,7 @@ func (r *SmartRouter) cleanupStaleLoad(ctx context.Context, node *BackendNode, m } replica, err := r.registry.GetNodeModel(context.WithoutCancel(ctx), node.ID, modelName, replicaIndex) if err != nil { - replica = &NodeModel{NodeID: node.ID, ModelName: modelName, ReplicaIndex: replicaIndex, Address: address, State: "unloading", ConfigRevision: revision, EffectiveOptionsHash: hash} + replica = &NodeModel{NodeID: node.ID, ModelName: modelName, ReplicaIndex: replicaIndex, WorkerLocalAddress: address, State: "unloading", ConfigRevision: revision, EffectiveOptionsHash: hash} } r.modelCleanup.Cleanup(context.WithoutCancel(ctx), []NodeModel{*replica}, false) } @@ -602,9 +602,13 @@ func (r *SmartRouter) ScheduleAndLoadModel(ctx context.Context, modelName string // RouteResult contains the routing decision. type RouteResult struct { - Node *BackendNode - Client grpc.Backend - Release func() // Must be called when the request is done (decrements in-flight) + Node *BackendNode + Client grpc.Backend + // WorkerLocalAddress is where the routed replica's backend process listens + // on its worker. Carried so callers that record or log where a model went + // name the process rather than the node, which has no address. + WorkerLocalAddress string + Release func() // Must be called when the request is done (decrements in-flight) } // Route finds the best node for the given model and backend type. @@ -727,12 +731,25 @@ func (r *SmartRouter) tryWarmPath(ctx context.Context, att *routeAttempt) *Route if err != nil || node == nil { return nil } - modelAddr := node.Address - if nm.Address != "" { - modelAddr = nm.Address - } + modelAddr := nm.WorkerLocalAddress replicaIdx := nm.ReplicaIndex + // A replica row that does not name its backend process cannot be routed + // to. There is no node address left to stand in for it, and naming an + // empty target would make the worker refuse the stream as invalid: a + // refusal that reads as the worker answering about its backend, which is + // evidence this row is not entitled to produce. Fall through to a cold + // load, which either replaces the row or reports a real failure. + if modelAddr == "" { + if err := r.registry.DecrementInFlight(ctx, node.ID, att.trackingKey, replicaIdx); err != nil { + xlog.Warn("Failed to release a reservation for an unnamed replica", + "node", node.ID, "model", att.trackingKey, "replica", replicaIdx, "error", err) + } + xlog.Warn("Loaded replica row names no backend process; cold-loading instead", + "node", node.ID, "model", att.trackingKey, "replica", replicaIdx) + return nil + } + // Verify the backend process is still alive via gRPC health check alive, probed := r.probeHealth(ctx, node, modelAddr) if !probed { @@ -794,7 +811,7 @@ func (r *SmartRouter) tryWarmPath(ctx context.Context, att *routeAttempt) *Route return nil } tracked := NewInFlightTrackingClient(grpcClient, r.registry, node.ID, att.trackingKey, replicaIdx) - return r.newRouteResult(node, att.trackingKey, replicaIdx, grpcClient, tracked) + return r.newRouteResult(node, modelAddr, att.trackingKey, replicaIdx, grpcClient, tracked) } // coldLoad schedules the model onto a node and loads it, returning a route to @@ -811,7 +828,7 @@ func (r *SmartRouter) coldLoad(ctx context.Context, att *routeAttempt, initialIn r.observePrefix(att.trackingKey, att.observeChain, prefixcache.ReplicaKey{NodeID: result.Node.ID, Replica: result.ReplicaIndex}) tracked := NewInFlightTrackingClient(result.Client, r.registry, result.Node.ID, att.trackingKey, result.ReplicaIndex) - return r.newRouteResult(result.Node, att.trackingKey, result.ReplicaIndex, result.Client, tracked), nil + return r.newRouteResult(result.Node, result.BackendAddr, att.trackingKey, result.ReplicaIndex, result.Client, tracked), nil } // newColdLoadContext builds the detached, progress-extended context a cold load @@ -1394,12 +1411,16 @@ func (r *SmartRouter) installBackendOnNode(ctx context.Context, node *BackendNod if !reply.Success { return "", fmt.Errorf("worker replied with error: %s", reply.Error) } - // Return the backend's gRPC address (per-replica port from worker) - addr := reply.Address - if addr == "" { - addr = node.Address // fallback to node base address + // Where the backend process listens on that worker. There is no node + // address to fall back to any more, and there should not be: a worker + // that reports success without naming the port it started the process + // on has told us nothing routable, and inventing a target would surface + // later as the worker refusing an invalid stream, which reads as + // evidence about the backend rather than about this install. + if reply.WorkerLocalAddress == "" { + return "", fmt.Errorf("worker %s reported backend %q installed but named no address for the process", node.ID, backendType) } - return addr, nil + return reply.WorkerLocalAddress, nil }) select { case <-ctx.Done(): @@ -2064,7 +2085,7 @@ func (r *SmartRouter) probeHealth(ctx context.Context, node *BackendNode, addr s // disconnect, handler error, validation failure after load) previously left // in_flight pinned at 1 forever, and every eviction query requires // in_flight = 0, so that replica's VRAM could never be reclaimed. -func (r *SmartRouter) newRouteResult(node *BackendNode, trackingKey string, replicaIdx int, raw grpc.Backend, tracked *InFlightTrackingClient) *RouteResult { +func (r *SmartRouter) newRouteResult(node *BackendNode, workerLocalAddr, trackingKey string, replicaIdx int, raw grpc.Backend, tracked *InFlightTrackingClient) *RouteResult { var once sync.Once release := func() { once.Do(func() { @@ -2078,8 +2099,9 @@ func (r *SmartRouter) newRouteResult(node *BackendNode, trackingKey string, repl } tracked.OnFirstComplete(release) return &RouteResult{ - Node: node, - Client: tracked, + Node: node, + Client: tracked, + WorkerLocalAddress: workerLocalAddr, Release: func() { release() closeClient(raw) diff --git a/core/services/nodes/router_eviction_alias_test.go b/core/services/nodes/router_eviction_alias_test.go index 09a6577520f3..ebd463b97a86 100644 --- a/core/services/nodes/router_eviction_alias_test.go +++ b/core/services/nodes/router_eviction_alias_test.go @@ -52,7 +52,7 @@ var _ = Describe("Eviction against an alias-keyed replica floor", func() { rowID++ Expect(db.Create(&NodeModel{ ID: fmt.Sprintf("alias-row-%d", rowID), NodeID: node.ID, ModelName: model, - Address: node.Address, State: "loaded", InFlight: 0, + WorkerLocalAddress: node.Address, State: "loaded", InFlight: 0, LastUsed: time.Now().Add(-idleFor), UpdatedAt: time.Now(), }).Error).To(Succeed()) } diff --git a/core/services/nodes/router_eviction_selector_test.go b/core/services/nodes/router_eviction_selector_test.go index 8d0caaffbef5..1c283647daef 100644 --- a/core/services/nodes/router_eviction_selector_test.go +++ b/core/services/nodes/router_eviction_selector_test.go @@ -52,7 +52,7 @@ var _ = Describe("Eviction under a node selector", func() { rowID++ Expect(db.Create(&NodeModel{ ID: fmt.Sprintf("row-%d", rowID), NodeID: node.ID, ModelName: model, - Address: node.Address, State: "loaded", InFlight: inFlight, + WorkerLocalAddress: node.Address, State: "loaded", InFlight: inFlight, LastUsed: time.Now().Add(-idleFor), UpdatedAt: time.Now(), }).Error).To(Succeed()) } diff --git a/core/services/nodes/router_load_budget_test.go b/core/services/nodes/router_load_budget_test.go index f0f41f78c235..d004a0e3fc8d 100644 --- a/core/services/nodes/router_load_budget_test.go +++ b/core/services/nodes/router_load_budget_test.go @@ -112,7 +112,7 @@ var _ = Describe("size-derived remote LoadModel budget", func() { backend = &holdBackend{} factory = &holdClientFactory{client: backend} unloader = &fakeUnloader{ - installReply: &messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:9001"}, + installReply: &messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:9001"}, } dir = GinkgoT().TempDir() }) diff --git a/core/services/nodes/router_load_job_test.go b/core/services/nodes/router_load_job_test.go index 65d1ef939c35..0454c3b2e454 100644 --- a/core/services/nodes/router_load_job_test.go +++ b/core/services/nodes/router_load_job_test.go @@ -51,7 +51,7 @@ var _ = Describe("Route cold-load jobs", func() { backend = &stubBackend{healthResult: true, loadResult: &pb.Result{Success: true}} factory = &stubClientFactory{client: backend} unloader = &fakeUnloader{ - installReply: &messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:9001"}, + installReply: &messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:9001"}, } }) diff --git a/core/services/nodes/router_load_timeout_test.go b/core/services/nodes/router_load_timeout_test.go index 7ee68ac123e6..49a014cdc86e 100644 --- a/core/services/nodes/router_load_timeout_test.go +++ b/core/services/nodes/router_load_timeout_test.go @@ -71,7 +71,7 @@ var _ = Describe("remote LoadModel deadline", func() { backend = &deadlineBackend{} factory = &deadlineClientFactory{client: backend} unloader = &fakeUnloader{ - installReply: &messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:9001"}, + installReply: &messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:9001"}, } }) diff --git a/core/services/nodes/router_reap_load_test.go b/core/services/nodes/router_reap_load_test.go index b2619dc97f0a..10801385cd97 100644 --- a/core/services/nodes/router_reap_load_test.go +++ b/core/services/nodes/router_reap_load_test.go @@ -73,7 +73,7 @@ var _ = Describe("reaping an abandoned remote load", func() { reg = &replicaSlotRouter{fakeModelRouter: base, replica: 2} backend = &failingLoadBackend{} unloader = &fakeUnloader{ - installReply: &messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:9001"}, + installReply: &messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:9001"}, } }) diff --git a/core/services/nodes/router_reservation_test.go b/core/services/nodes/router_reservation_test.go index c9f6cf73bb6f..ad251fb6dd96 100644 --- a/core/services/nodes/router_reservation_test.go +++ b/core/services/nodes/router_reservation_test.go @@ -50,7 +50,7 @@ var _ = Describe("SmartRouter routing reservation", func() { newResult := func() *RouteResult { raw := &stubBackend{} tracked := NewInFlightTrackingClient(raw, registry, node.ID, "m", 0) - return router.newRouteResult(node, "m", 0, raw, tracked) + return router.newRouteResult(node, "127.0.0.1:50052", "m", 0, raw, tracked) } It("releases the reservation when the route is torn down without any inference", func() { diff --git a/core/services/nodes/router_revision_lifecycle_test.go b/core/services/nodes/router_revision_lifecycle_test.go index 7b1de6144b72..6670f76ad8c6 100644 --- a/core/services/nodes/router_revision_lifecycle_test.go +++ b/core/services/nodes/router_revision_lifecycle_test.go @@ -56,7 +56,7 @@ var _ = Describe("revision-bound load publication", func() { node = &BackendNode{Name: "revision-worker", NodeType: NodeTypeBackend, Address: "10.0.0.1:50051", TotalVRAM: 64_000_000_000, AvailableVRAM: 64_000_000_000} Expect(registry.Register(ctx, node, true)).To(Succeed()) backend = &stubBackend{healthResult: true, loadResult: &pb.Result{Success: true}} - unloader = &fakeUnloader{installReply: &messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:9001"}} + unloader = &fakeUnloader{installReply: &messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:9001"}} }) It("quarantines and exactly stops a load that finishes after its revision changes", func() { diff --git a/core/services/nodes/router_staging_context_test.go b/core/services/nodes/router_staging_context_test.go index f0b07a7a53e8..1a577df373c5 100644 --- a/core/services/nodes/router_staging_context_test.go +++ b/core/services/nodes/router_staging_context_test.go @@ -52,8 +52,8 @@ var _ = Describe("Route cold-load staging context", func() { backend := &stubBackend{loadResult: &pb.Result{Success: true}} factory := &stubClientFactory{client: backend} unloader := &fakeUnloader{installReply: &messaging.BackendInstallReply{ - Success: true, - Address: "10.0.0.1:9001", + Success: true, + WorkerLocalAddress: "10.0.0.1:9001", }} stager := &cancelOnStageStager{} diff --git a/core/services/nodes/router_staging_deadline_test.go b/core/services/nodes/router_staging_deadline_test.go index 35d1d2ae568c..66026987d9c2 100644 --- a/core/services/nodes/router_staging_deadline_test.go +++ b/core/services/nodes/router_staging_deadline_test.go @@ -97,8 +97,8 @@ var _ = Describe("cold-load staging deadline", func() { } factory = &stubClientFactory{client: &stubBackend{loadResult: &pb.Result{Success: true}}} unloader = &fakeUnloader{installReply: &messaging.BackendInstallReply{ - Success: true, - Address: "10.0.0.1:9001", + Success: true, + WorkerLocalAddress: "10.0.0.1:9001", }} modelDir = GinkgoT().TempDir() }) diff --git a/core/services/nodes/router_test.go b/core/services/nodes/router_test.go index dbc5bd102e28..43b4353c12ed 100644 --- a/core/services/nodes/router_test.go +++ b/core/services/nodes/router_test.go @@ -613,8 +613,8 @@ var _ = Describe("SmartRouter", func() { factory = &stubClientFactory{client: backend} unloader = &fakeUnloader{ installReply: &messaging.BackendInstallReply{ - Success: true, - Address: "10.0.0.1:9001", + Success: true, + WorkerLocalAddress: "10.0.0.1:9001", }, } }) @@ -622,7 +622,7 @@ var _ = Describe("SmartRouter", func() { Context("model already loaded on a healthy node", func() { It("returns the client and a release function", func() { node := &BackendNode{ID: "n1", Name: "node-1", Address: "10.0.0.1:50051"} - nm := &NodeModel{NodeID: "n1", ModelName: "my-model", Address: "10.0.0.1:9001"} + nm := &NodeModel{NodeID: "n1", ModelName: "my-model", WorkerLocalAddress: "10.0.0.1:9001"} reg.findAndLockNode = node reg.findAndLockNM = nm backend.healthResult = true @@ -764,8 +764,8 @@ var _ = Describe("SmartRouter", func() { factory = &stubClientFactory{client: backend} unloader = &fakeUnloader{ installReply: &messaging.BackendInstallReply{ - Success: true, - Address: "10.0.0.1:9001", + Success: true, + WorkerLocalAddress: "10.0.0.1:9001", }, } }) @@ -926,8 +926,8 @@ var _ = Describe("SmartRouter", func() { factory = &stubClientFactory{client: backend} unloader = &fakeUnloader{ installReply: &messaging.BackendInstallReply{ - Success: true, - Address: "10.0.0.1:9001", + Success: true, + WorkerLocalAddress: "10.0.0.1:9001", }, } }) @@ -1023,15 +1023,15 @@ var _ = Describe("SmartRouter", func() { factory := &stubClientFactory{client: backend} unloader := &fakeUnloader{ installReply: &messaging.BackendInstallReply{ - Success: true, - Address: "10.0.0.71:9001", + Success: true, + WorkerLocalAddress: "10.0.0.71:9001", }, } reg := &fakeModelRouter{ // Step 1: cached model found on old node findAndLockNode: cachedNode, - findAndLockNM: &NodeModel{NodeID: "n-old", ModelName: "sel-model", Address: "10.0.0.70:9001"}, + findAndLockNM: &NodeModel{NodeID: "n-old", ModelName: "sel-model", WorkerLocalAddress: "10.0.0.70:9001"}, // Scheduling config with selector that old node does NOT match getModelScheduling: &ModelSchedulingConfig{ ModelName: "sel-model", @@ -1292,7 +1292,7 @@ var _ = Describe("SmartRouter", func() { started := make(chan struct{}, 5) release := make(chan struct{}) unloader := &fakeUnloader{ - installReply: &messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:50100"}, + installReply: &messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:50100"}, } unloader.installHook = func() { started <- struct{}{} @@ -1331,7 +1331,7 @@ var _ = Describe("SmartRouter", func() { It("does NOT coalesce installs for different (modelID, replica) keys", func() { node := &BackendNode{ID: "n1", Name: "node-1", Address: "10.0.0.1:50051"} unloader := &fakeUnloader{ - installReply: &messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:50100"}, + installReply: &messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:50100"}, } router := NewSmartRouter(&fakeModelRouter{}, SmartRouterOptions{ Unloader: unloader, @@ -1346,6 +1346,44 @@ var _ = Describe("SmartRouter", func() { Expect(err3).ToNot(HaveOccurred()) Expect(unloader.installCalls).To(HaveLen(3)) }) + + It("returns the address the worker named for the backend process", func() { + node := &BackendNode{ID: "n1", Name: "node-1"} + unloader := &fakeUnloader{ + installReply: &messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "127.0.0.1:50100"}, + } + router := NewSmartRouter(&fakeModelRouter{}, SmartRouterOptions{ + Unloader: unloader, + ClientFactory: &stubClientFactory{client: &stubBackend{}}, + }) + + addr, err := router.installBackendOnNode(context.Background(), node, "llama-cpp", "model-A", 0) + Expect(err).ToNot(HaveOccurred()) + Expect(addr).To(Equal("127.0.0.1:50100")) + }) + + It("fails when the worker reports success but names no address", func() { + // There is no node address left to stand in for it. Substituting one + // used to be the behaviour here, and with workers no longer + // advertising it would substitute the empty string: the frontend + // would then open a stream naming an empty target, the worker would + // refuse it as invalid, and that refusal reads as the WORKER + // answering about its backend rather than as this install having + // produced nothing routable. + node := &BackendNode{ID: "n1", Name: "node-1"} + unloader := &fakeUnloader{ + installReply: &messaging.BackendInstallReply{Success: true}, + } + router := NewSmartRouter(&fakeModelRouter{}, SmartRouterOptions{ + Unloader: unloader, + ClientFactory: &stubClientFactory{client: &stubBackend{}}, + }) + + addr, err := router.installBackendOnNode(context.Background(), node, "llama-cpp", "model-A", 0) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("named no address")) + Expect(addr).To(BeEmpty()) + }) }) }) @@ -1405,7 +1443,7 @@ var _ = Describe("SmartRouter prefix-cache routing", func() { backend = &stubBackend{healthResult: true} factory = &stubClientFactory{client: backend} unloader = &fakeUnloader{ - installReply: &messaging.BackendInstallReply{Success: true, Address: "10.0.0.1:9001"}, + installReply: &messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:9001"}, } }) @@ -1413,7 +1451,7 @@ var _ = Describe("SmartRouter prefix-cache routing", func() { // "m" on node "X", plus matching replica stats so buildPreference can run. loadedReg := func() *fakeModelRouter { node := &BackendNode{ID: "X", Name: "node-x", Address: "10.0.0.1:50051"} - nm := &NodeModel{NodeID: "X", ModelName: "m", Address: "10.0.0.1:9001"} + nm := &NodeModel{NodeID: "X", ModelName: "m", WorkerLocalAddress: "10.0.0.1:9001"} return &fakeModelRouter{ findAndLockNode: node, findAndLockNM: nm, @@ -1504,7 +1542,7 @@ var _ = Describe("SmartRouter prefix-cache routing", func() { // node. This is the replica-granular regression this change fixes. idx := prefixcache.NewIndex(prefixcache.DefaultConfig()) node := &BackendNode{ID: "X", Name: "node-x", Address: "10.0.0.1:50051"} - nm := &NodeModel{NodeID: "X", ModelName: "m", ReplicaIndex: 0, Address: "10.0.0.1:9001"} + nm := &NodeModel{NodeID: "X", ModelName: "m", ReplicaIndex: 0, WorkerLocalAddress: "10.0.0.1:9001"} reg := &fakeModelRouter{ findAndLockNode: node, findAndLockNM: nm, @@ -1583,7 +1621,7 @@ var _ = Describe("SmartRouter prefix-cache routing", func() { // forced-disturb signal. findAndLockNode returns Y so Route succeeds. disturbReg := func() *fakeModelRouter { nodeY := &BackendNode{ID: "Y", Name: "node-y", Address: "10.0.0.2:50051"} - nm := &NodeModel{NodeID: "Y", ModelName: "m", Address: "10.0.0.2:9001"} + nm := &NodeModel{NodeID: "Y", ModelName: "m", WorkerLocalAddress: "10.0.0.2:9001"} return &fakeModelRouter{ findAndLockNode: nodeY, findAndLockNM: nm, diff --git a/core/services/nodes/router_unreachable_worker_test.go b/core/services/nodes/router_unreachable_worker_test.go index 206956e1194f..6a5a4501be60 100644 --- a/core/services/nodes/router_unreachable_worker_test.go +++ b/core/services/nodes/router_unreachable_worker_test.go @@ -47,7 +47,7 @@ var _ = Describe("routing when the worker cannot be reached at all", func() { // cold-loaded somewhere else. loadedReg := func() *fakeModelRouter { node := &BackendNode{ID: "X", Name: "node-x", Address: "10.0.0.1:50051"} - nm := &NodeModel{NodeID: "X", ModelName: "m", Address: "10.0.0.1:9001"} + nm := &NodeModel{NodeID: "X", ModelName: "m", WorkerLocalAddress: "10.0.0.1:9001"} return &fakeModelRouter{ findAndLockNode: node, findAndLockNM: nm, @@ -96,7 +96,7 @@ var _ = Describe("routing when the worker's tunnel dial fails", func() { // process and the replica row is deleted after ONE miss. loadedReg := func() *fakeModelRouter { node := &BackendNode{ID: "X", Name: "node-x", Address: "10.0.0.1:50051"} - nm := &NodeModel{NodeID: "X", ModelName: "m", Address: "10.0.0.1:9001"} + nm := &NodeModel{NodeID: "X", ModelName: "m", WorkerLocalAddress: "10.0.0.1:9001"} return &fakeModelRouter{ findAndLockNode: node, findAndLockNM: nm, diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index b95b1330bee7..a4187e94afef 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -109,7 +109,7 @@ func (a *RemoteUnloaderAdapter) StopModelReplica(ctx context.Context, nodeID str reply, err := messaging.RequestJSON[messaging.ModelStopRequest, messaging.ModelStopReply](a.nats, messaging.SubjectNodeModelStop(nodeID), messaging.ModelStopRequest{ ModelName: replica.ModelName, ProcessKey: model.BackendProcessKey(replica.ModelName, replica.ReplicaIndex), - ExpectedAddress: replica.Address, + ExpectedAddress: replica.WorkerLocalAddress, Force: force, ConfigRevision: replica.ConfigRevision, }, exactModelStopTimeout) diff --git a/core/services/nodes/unloader_test.go b/core/services/nodes/unloader_test.go index 3564f93c85cf..f801ff529558 100644 --- a/core/services/nodes/unloader_test.go +++ b/core/services/nodes/unloader_test.go @@ -326,7 +326,7 @@ var _ = Describe("RemoteUnloaderAdapter", func() { Describe("StopModelReplica", func() { It("requests an acknowledged stop for the exact process", func() { mc.requestReply, _ = json.Marshal(messaging.ModelStopReply{Matched: true, Terminated: true, ProcessKey: "llama#2"}) - replica := NodeModel{ModelName: "llama", ReplicaIndex: 2, Address: "127.0.0.1:5002", ConfigRevision: "rev-1"} + replica := NodeModel{ModelName: "llama", ReplicaIndex: 2, WorkerLocalAddress: "127.0.0.1:5002", ConfigRevision: "rev-1"} reply, err := adapter.StopModelReplica(context.Background(), "node-1", replica, true) Expect(err).NotTo(HaveOccurred()) @@ -427,7 +427,7 @@ func (f *failOnceMessagingClient) Close() {} var _ = Describe("RemoteUnloaderAdapter timeout configuration", func() { It("passes the configured install timeout to the messaging client", func() { mc := newScriptedMessagingClient() - mc.scriptReply(messaging.SubjectNodeBackendInstall("n1"), messaging.BackendInstallReply{Success: true, Address: "127.0.0.1:0"}) + mc.scriptReply(messaging.SubjectNodeBackendInstall("n1"), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "127.0.0.1:0"}) adapter := NewRemoteUnloaderAdapter(nil, mc, 7*time.Minute, 11*time.Minute) _, err := adapter.InstallBackend("n1", "llama-cpp", "", "[]", "", "", "", 0, "", nil) @@ -477,7 +477,7 @@ var _ = Describe("RemoteUnloaderAdapter NATS timeout handling", func() { var _ = Describe("RemoteUnloaderAdapter install progress streaming", func() { It("forwards BackendInstallProgressEvent values into the onProgress callback when the worker publishes them", func() { mc := newScriptedMessagingClient() - mc.scriptReply(messaging.SubjectNodeBackendInstall("n1"), messaging.BackendInstallReply{Success: true, Address: "127.0.0.1:0"}) + mc.scriptReply(messaging.SubjectNodeBackendInstall("n1"), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "127.0.0.1:0"}) mc.scheduleProgressPublish("n1", "op-abc", []messaging.BackendInstallProgressEvent{ {OpID: "op-abc", NodeID: "n1", Backend: "vllm", FileName: "vllm.tar.zst", Current: "100 MB", Total: "1 GB", Percentage: 10}, {OpID: "op-abc", NodeID: "n1", Backend: "vllm", FileName: "vllm.tar.zst", Current: "500 MB", Total: "1 GB", Percentage: 50}, diff --git a/core/services/worker/addr_test.go b/core/services/worker/addr_test.go index 4f5b1ba67f4f..633d183b292d 100644 --- a/core/services/worker/addr_test.go +++ b/core/services/worker/addr_test.go @@ -1,14 +1,19 @@ package worker import ( - "os" - "strings" + "net" + "strconv" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) var _ = Describe("Worker address resolution", func() { + // advertiseAddr and advertiseHTTPAddr used to be specced here. They are + // gone with the addresses they resolved: a worker advertises nothing. What + // they pinned that still matters is below: the port arithmetic they shared + // with the two functions that survived, and the fact that neither of those + // resolves to anything but this host. Describe("effectiveBasePort", func() { DescribeTable("returns the correct port", func(addr, serve string, want int) { @@ -25,61 +30,63 @@ var _ = Describe("Worker address resolution", func() { ) }) - Describe("advertiseAddr", func() { - It("returns AdvertiseAddr when set", func() { - cfg := &Config{ - AdvertiseAddr: "public.example.com:50051", - Addr: "10.0.0.5:60000", - } - Expect(cfg.advertiseAddr()).To(Equal("public.example.com:50051")) - }) - - It("returns Addr when set", func() { - cfg := &Config{Addr: "worker1.example.com:60000"} - Expect(cfg.advertiseAddr()).To(Equal("worker1.example.com:60000")) - }) - - It("falls back to hostname:basePort", func() { - cfg := &Config{ServeAddr: "0.0.0.0:50051"} - got := cfg.advertiseAddr() - _, port, _ := strings.Cut(got, ":") - Expect(port).To(Equal("50051")) - - hostname, _ := os.Hostname() - if hostname != "" { - host, _, _ := strings.Cut(got, ":") - Expect(host).To(Equal(hostname)) - } - }) - }) - Describe("resolveHTTPAddr", func() { DescribeTable("returns the correct address", func(httpAddr, addr, serve, want string) { cfg := &Config{HTTPAddr: httpAddr, Addr: addr, ServeAddr: serve} Expect(cfg.resolveHTTPAddr()).To(Equal(want)) }, + // An explicit HTTPAddr is bound exactly as written, wildcard + // included: an operator who asks for a routable bind gets one, and + // the tunnel still reaches it because the http tag ignores the + // target and dials whatever this returned. Entry("HTTPAddr takes priority", "0.0.0.0:8080", "", "", "0.0.0.0:8080"), - Entry("derives from Addr port minus 1", "", "worker1:60000", "0.0.0.0:50051", "0.0.0.0:59999"), - Entry("derives from ServeAddr port minus 1", "", "", "0.0.0.0:50051", "0.0.0.0:50050"), - Entry("default when nothing set", "", "", "", "0.0.0.0:50050"), + Entry("derives from Addr port minus 1", "", "worker1:60000", "0.0.0.0:50051", "127.0.0.1:59999"), + Entry("derives from ServeAddr port minus 1", "", "", "0.0.0.0:50051", "127.0.0.1:50050"), + Entry("default when nothing set", "", "", "", "127.0.0.1:50050"), ) + + It("takes only the port from Addr, never its host", func() { + // The host half of Addr names an interface nothing binds any more. + // A default bind that carried it forward would put the + // file-transfer server back on a routable address. + cfg := &Config{Addr: "0.0.0.0:60000"} + Expect(cfg.resolveHTTPAddr()).To(Equal("127.0.0.1:59999")) + }) }) - Describe("advertiseHTTPAddr", func() { - DescribeTable("returns the correct address", - func(advertiseHTTP, advertise, addr, serve, want string) { - cfg := &Config{ - AdvertiseHTTPAddr: advertiseHTTP, - AdvertiseAddr: advertise, - Addr: addr, - ServeAddr: serve, - } - Expect(cfg.advertiseHTTPAddr()).To(Equal(want)) - }, - Entry("AdvertiseHTTPAddr takes priority", "public.example.com:8080", "", "", "", "public.example.com:8080"), - Entry("derives from advertiseAddr host + basePort-1", "", "", "worker1.example.com:60000", "", "worker1.example.com:59999"), - Entry("uses AdvertiseAddr host with basePort-1", "", "public.example.com:60000", "10.0.0.5:60000", "", "public.example.com:59999"), - ) + Describe("backendListenAddr", func() { + It("binds a backend process on the host the tunnel dials", func() { + // Not a literal on either side: this asserts the bind is built from + // the same constant the grpc stream tag dials, which is what makes + // "the worker binds where its tunnel dials" true rather than + // coincidental. + Expect(backendListenAddr(50052)).To(Equal(net.JoinHostPort(loopbackHost, strconv.Itoa(50052)))) + }) + + It("binds no wildcard", func() { + // Stated separately from the equality above so a change to + // loopbackHost itself cannot make both pass while publishing every + // backend process on every interface. + host, _, err := net.SplitHostPort(backendListenAddr(50052)) + Expect(err).ToNot(HaveOccurred()) + ip := net.ParseIP(host) + Expect(ip).ToNot(BeNil(), "the backend bind address must be an IP, not a name that could resolve anywhere") + Expect(ip.IsLoopback()).To(BeTrue(), "backend processes must bind loopback only") + }) + }) + + Describe("registrationBody", func() { + It("advertises no address at all", func() { + // The registration body is one of the three places this worker used + // to state where it could be reached. A key here is not inert: the + // frontend stores it, the API returns it, and the Nodes page shows + // it as an endpoint. + cfg := &Config{NodeName: "w1", Addr: "0.0.0.0:50051", ModelsPath: GinkgoT().TempDir()} + body := cfg.registrationBody() + Expect(body).To(HaveKeyWithValue("name", "w1")) + Expect(body).ToNot(HaveKey("address")) + Expect(body).ToNot(HaveKey("http_address")) + }) }) }) diff --git a/core/services/worker/config.go b/core/services/worker/config.go index 5c29cd4990f1..29ea5fec36f5 100644 --- a/core/services/worker/config.go +++ b/core/services/worker/config.go @@ -16,11 +16,16 @@ package worker // // Model loading (LoadModel) is always via direct gRPC — no NATS needed for that. type Config struct { - // Primary address — the reachable address of this worker. - // Host is used for advertise, port is the base for gRPC backends. - // HTTP file transfer runs on port-1. - Addr string `env:"LOCALAI_ADDR" help:"Address where this worker is reachable (host:port). Port is base for gRPC backends, port-1 for HTTP." group:"server"` - ServeAddr string `env:"LOCALAI_SERVE_ADDR" default:"0.0.0.0:50051" help:"(Advanced) gRPC base port bind address" group:"server" hidden:""` + // Addr and ServeAddr are read for their PORT only. A worker binds nothing + // on a routable interface: backend processes and the file-transfer server + // both listen on loopback and are reached through this worker's outbound + // tunnel. The port still matters because it is the base of the backend + // port range (and port-1 is the HTTP server), so an operator who needs a + // different range sets it here. The host half is ignored, and is kept + // accepted rather than rejected so an upgraded worker starts on the + // environment it already had. + Addr string `env:"LOCALAI_ADDR" help:"Base port for this worker, as host:port; only the port is used. Backends take ports upward from it, the HTTP file-transfer server takes port-1. Nothing binds a routable interface." group:"server"` + ServeAddr string `env:"LOCALAI_SERVE_ADDR" default:"0.0.0.0:50051" help:"(Advanced) gRPC base port; only the port is used" group:"server" hidden:""` // GRPCMaxPort bounds the dynamic gRPC port allocator at [basePort, this]. // The width of that range is how many backend processes this worker can run @@ -46,14 +51,13 @@ type Config struct { // anyway; the master can still push the file on demand (existing behaviour). PrefetchModels []string `env:"LOCALAI_PREFETCH_MODELS,PREFETCH_MODELS" help:"Comma-separated gallery model IDs to download from LOCALAI_GALLERIES at worker boot (e.g. 'llama-3.2-1b-instruct,phi-3-mini-4k'). Skipped if already on disk and SHA matches." group:"server"` - // HTTP file transfer - HTTPAddr string `env:"LOCALAI_HTTP_ADDR" default:"" help:"HTTP file transfer server address (default: gRPC port + 1)" group:"server" hidden:""` - AdvertiseHTTPAddr string `env:"LOCALAI_ADVERTISE_HTTP_ADDR" help:"HTTP address the frontend uses to reach this node for file transfer" group:"server" hidden:""` + // HTTPAddr binds the HTTP file-transfer server. Default is loopback on + // basePort-1; an explicit value is bound exactly as given. + HTTPAddr string `env:"LOCALAI_HTTP_ADDR" default:"" help:"HTTP file transfer server bind address (default: loopback on the gRPC base port - 1)" group:"server" hidden:""` EphemeralStagingByteLimit int64 `env:"LOCALAI_EPHEMERAL_STAGING_BYTE_LIMIT" default:"0" help:"Maximum bytes used by worker request-input staging across HTTP and S3 caches. Zero or negative uses min(10 GiB, 10% of filesystem capacity)." group:"server"` EphemeralStagingMinFreeBytes int64 `env:"LOCALAI_EPHEMERAL_STAGING_MIN_FREE_BYTES" default:"0" help:"Filesystem space kept free while staging request inputs. Zero or negative uses max(1 GiB, 5% of filesystem capacity)." group:"server"` // Registration (required) - AdvertiseAddr string `env:"LOCALAI_ADVERTISE_ADDR" help:"Address the frontend uses to reach this node (defaults to hostname:port from Addr)" group:"registration" hidden:""` RegisterTo string `env:"LOCALAI_REGISTER_TO" required:"" help:"Frontend URL for registration" group:"registration"` NodeName string `env:"LOCALAI_NODE_NAME" help:"Node name for registration (defaults to hostname)" group:"registration"` RegistrationToken string `env:"LOCALAI_REGISTRATION_TOKEN" help:"Token for authenticating with the frontend" group:"registration"` diff --git a/core/services/worker/lifecycle.go b/core/services/worker/lifecycle.go index f9be39c8cf4a..b864c8e7c749 100644 --- a/core/services/worker/lifecycle.go +++ b/core/services/worker/lifecycle.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "maps" - "net" "slices" "strings" "syscall" @@ -98,20 +97,17 @@ func (s *backendSupervisor) handleBackendInstall(data []byte, reply func([]byte) return } - advertiseAddr := addr - advAddr := s.cfg.advertiseAddr() - if advAddr != addr { - _, port, err := net.SplitHostPort(addr) - if err != nil { - xlog.Error("Failed to parse backend listen address; using it unchanged", "addr", addr, "error", err) - } else if advertiseHost, _, err := net.SplitHostPort(advAddr); err != nil { - xlog.Error("Failed to parse worker advertise address; using backend listen address", "addr", advAddr, "error", err) - } else { - advertiseAddr = net.JoinHostPort(advertiseHost, port) - } - } - resp := messaging.BackendInstallReply{Success: true, Address: advertiseAddr} - replyJSON(reply, resp) + // The address goes back exactly as the process listens on it. It used + // to be rewritten onto this worker's advertise host, which made the + // reply the worker's third advertisement site; the frontend now reads + // only the port out of it and dials nothing. + // + // The rewrite was also wrong in a way nothing caught: the worker + // records the loopback address and stopModelExact refuses a stop whose + // ExpectedAddress does not match it, so on any worker whose advertise + // host was not 127.0.0.1 every acknowledged model stop failed with an + // address mismatch. + replyJSON(reply, messaging.BackendInstallReply{Success: true, WorkerLocalAddress: addr}) }() } diff --git a/core/services/worker/registration.go b/core/services/worker/registration.go index 31dd6dbf78fc..c3ac109a9359 100644 --- a/core/services/worker/registration.go +++ b/core/services/worker/registration.go @@ -1,7 +1,6 @@ package worker import ( - "cmp" "fmt" "math" "net" @@ -33,6 +32,10 @@ func clampCPUUsage(usage float64) float64 { // effectiveBasePort returns the port used as base for gRPC backend processes. // Priority: Addr port → ServeAddr port → 50051 +// +// Only the PORT of those settings is read. Their host halves name an interface +// this worker no longer binds: every backend listens on loopback and is reached +// through the tunnel. func (cfg *Config) effectiveBasePort() int { for _, addr := range []string{cfg.Addr, cfg.ServeAddr} { if addr == "" { @@ -82,45 +85,21 @@ func (cfg *Config) effectiveMaxPort(basePort int) int { return cfg.GRPCMaxPort } -// advertiseAddr returns the address the frontend should use to reach this node. -func (cfg *Config) advertiseAddr() string { - if cfg.AdvertiseAddr != "" { - return cfg.AdvertiseAddr - } - if cfg.Addr != "" { - return cfg.Addr - } - hostname, err := os.Hostname() - if err != nil { - xlog.Warn("Failed to determine worker hostname; advertising localhost", "error", err) - } - return fmt.Sprintf("%s:%d", cmp.Or(hostname, "localhost"), cfg.effectiveBasePort()) -} - // resolveHTTPAddr returns the address to bind the HTTP file transfer server to. // Uses basePort-1 so it doesn't conflict with dynamically allocated gRPC ports // which grow upward from basePort. +// +// The default is loopback for the same reason backend processes are: the +// frontend reaches this server over the tunnel, whose http tag dials whatever +// address this returns. An operator who sets HTTPAddr explicitly still gets +// exactly that bind (see loopbackAddr, which rewrites only a wildcard), so a +// deployment that has some other local reason to expose the server can, and +// nothing in the frontend depends on it. func (cfg *Config) resolveHTTPAddr() string { if cfg.HTTPAddr != "" { return cfg.HTTPAddr } - return fmt.Sprintf("0.0.0.0:%d", cfg.effectiveBasePort()-1) -} - -// advertiseHTTPAddr returns the HTTP address the frontend should use to reach -// this node for file transfer. -func (cfg *Config) advertiseHTTPAddr() string { - if cfg.AdvertiseHTTPAddr != "" { - return cfg.AdvertiseHTTPAddr - } - advertiseAddr := cfg.advertiseAddr() - advHost, _, err := net.SplitHostPort(advertiseAddr) - if err != nil { - xlog.Warn("Invalid worker advertise address; advertising file transfer on localhost", "addr", advertiseAddr, "error", err) - advHost = "localhost" - } - httpPort := cfg.effectiveBasePort() - 1 - return net.JoinHostPort(advHost, strconv.Itoa(httpPort)) + return net.JoinHostPort(loopbackHost, strconv.Itoa(cfg.effectiveBasePort()-1)) } // registrationBody builds the JSON body for node registration. @@ -163,10 +142,12 @@ func (cfg *Config) registrationBody() map[string]any { if maxReplicas < 1 { maxReplicas = 1 } + // No address and no http_address: this worker has nothing inbound to + // advertise. It holds one outbound tunnel and the frontend reaches every + // service on it through that, so an address here would be a value that + // looks dialable, is stored, is shown, and is never dialled. body := map[string]any{ "name": nodeName, - "address": cfg.advertiseAddr(), - "http_address": cfg.advertiseHTTPAddr(), "total_vram": totalVRAM, "available_vram": totalVRAM, // initially all VRAM is available "gpu_vendor": gpuVendor, diff --git a/core/services/worker/supervisor.go b/core/services/worker/supervisor.go index 60754efcff79..9dba6a7ecbdf 100644 --- a/core/services/worker/supervisor.go +++ b/core/services/worker/supervisor.go @@ -5,9 +5,11 @@ import ( "errors" "fmt" "maps" + "net" "os" "path/filepath" "slices" + "strconv" "strings" "sync" "time" @@ -22,10 +24,26 @@ import ( "github.com/mudler/xlog" ) +// backendListenAddr is where a backend process binds, which is also the only +// address anything ever reaches it on. +// +// It is built from loopbackHost, the constant the tunnel's grpc tag dials, so +// "the worker binds where its tunnel dials" is one fact in one place rather +// than two literals that can drift. A backend is reached only over this +// worker's tunnel; a wildcard bind would publish every backend process on every +// interface to serve a route nothing takes, and on a worker with a public +// interface that is an unauthenticated inference server. +func backendListenAddr(port int) string { + return net.JoinHostPort(loopbackHost, strconv.Itoa(port)) +} + // backendProcess represents a single gRPC backend process. type backendProcess struct { proc *process.Process - addr string // gRPC address (host:port) + // addr is where this process listens, and it is worker-local: see + // backendListenAddr. The frontend is told this string and reads only its + // port out of it. + addr string port int // serving marks a process that has answered a gRPC health check and is // therefore actually listening on addr. It is the opening bracket of the @@ -462,10 +480,9 @@ func (s *backendSupervisor) startBackend(backend, backendName, backendPath strin s.mu.Unlock() return "", fmt.Errorf("allocating gRPC port for backend %s: %w", backend, err) } - bindAddr := fmt.Sprintf("0.0.0.0:%d", port) - clientAddr := fmt.Sprintf("127.0.0.1:%d", port) + procAddr := backendListenAddr(port) - proc, err := s.ml.StartProcess(backendPath, backend, bindAddr, nil) + proc, err := s.ml.StartProcess(backendPath, backend, procAddr, nil) if err != nil { s.releasePortForKey(backend, port) s.mu.Unlock() @@ -486,13 +503,13 @@ func (s *backendSupervisor) startBackend(backend, backendName, backendPath strin s.processes[backend] = &backendProcess{ proc: proc, - addr: clientAddr, + addr: procAddr, port: port, backendName: backendName, backendDir: backendDir, backendDirID: dirInfo, } - xlog.Info("Backend process started", "backend", backend, "addr", clientAddr) + xlog.Info("Backend process started", "backend", backend, "addr", procAddr) // Capture reference before unlocking for race-safe health check. // Another goroutine could stopBackend and recycle the port while we poll. @@ -505,7 +522,7 @@ func (s *backendSupervisor) startBackend(backend, backendName, backendPath strin // 4s window made the worker reply Success on a not-yet-listening port, // which manifested upstream as "connect: connection refused" on the // frontend's first LoadModel dial. - client := grpc.NewClientWithToken(clientAddr, false, nil, false, s.cfg.RegistrationToken) + client := grpc.NewClientWithToken(procAddr, false, nil, false, s.cfg.RegistrationToken) const ( readinessPollInterval = 200 * time.Millisecond readinessTimeout = 30 * time.Second @@ -524,8 +541,8 @@ func (s *backendSupervisor) startBackend(backend, backendName, backendPath strin if !s.markBackendServing(backend, bp) { return "", fmt.Errorf("backend %s was stopped during startup", backend) } - xlog.Debug("Backend gRPC server is ready", "backend", backend, "addr", clientAddr) - return clientAddr, nil + xlog.Debug("Backend gRPC server is ready", "backend", backend, "addr", procAddr) + return procAddr, nil } if healthErr != nil { lastHealthErr = healthErr @@ -547,7 +564,7 @@ func (s *backendSupervisor) startBackend(backend, backendName, backendPath strin // real cause). Stop the half-started process, recycle the port, and // surface the failure to the caller with the backend's stderr tail. stderrTail := readLastLinesFromFile(proc.StderrPath(), 20) - xlog.Error("Backend gRPC server not ready before deadline; aborting install", "backend", backend, "addr", clientAddr, "timeout", readinessTimeout, "healthError", lastHealthErr, "stderr", stderrTail) + xlog.Error("Backend gRPC server not ready before deadline; aborting install", "backend", backend, "addr", procAddr, "timeout", readinessTimeout, "healthError", lastHealthErr, "stderr", stderrTail) if killErr := proc.Stop(); killErr != nil { xlog.Warn("Failed to stop unready backend process", "backend", backend, "error", killErr) } diff --git a/core/services/worker/tunnel.go b/core/services/worker/tunnel.go index 0eabacb0f9e3..0ff28ea85a6f 100644 --- a/core/services/worker/tunnel.go +++ b/core/services/worker/tunnel.go @@ -483,7 +483,14 @@ func (t *Tunnel) logSessionEnded(err error, attempt int, delay time.Duration) { if errors.As(err, &dialErr) { switch dialErr.status { case http.StatusUnauthorized: - xlog.Warn("Frontend rejected this worker's tunnel credential; re-registering will mint a fresh one", + // Named causes, because this worker cannot recover from either on + // its own and the two need different actions. It has no inbound + // listener and no advertised address, so a tunnel it cannot open is + // a worker nothing can reach: this is an outage, not a warning about + // a degraded path. + xlog.Warn("Frontend rejected this worker's tunnel credential, so nothing can reach this worker; "+ + "either another worker registered under this node name and rotated the credential (check LOCALAI_NODE_NAME is unique), "+ + "or the frontend's record of this node was replaced. Restarting this worker re-registers and mints a fresh credential", "node", t.nodeID, "retry_in", delay) case http.StatusForbidden: xlog.Info("Worker tunnel refused: this node is awaiting admin approval", diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index 9ce198c9dc74..f6c0b33c99ec 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -29,7 +29,7 @@ import ( // Run starts the distributed agent worker: registers with the frontend, // subscribes to NATS lifecycle subjects, and blocks on signals. func Run(ctx *cliContext.Context, cfg *Config) error { - xlog.Info("Starting worker", "advertise", cfg.advertiseAddr(), "basePort", cfg.effectiveBasePort()) + xlog.Info("Starting worker", "basePort", cfg.effectiveBasePort()) // Fail fast (before prefetch/registration/NATS) when enforcement is on but no // registration token is set: the worker's HTTP file-transfer server fails @@ -115,14 +115,23 @@ func Run(ctx *cliContext.Context, cfg *Config) error { // second worker registering under this node's name rotates the row's // credential, and this worker then fails every tunnel dial with 401 for // the life of the process. It logs that once per backoff and never - // recovers on its own; a restart fixes it, because startup re-registers - // unconditionally. + // recovers on its own; a restart fixes it only until the other worker + // registers again. // - // Deliberately NOT fixed here. Re-registering after repeated tunnel - // 401s is a decision about the worker's lifecycle, and it belongs with - // the change that removes this worker's inbound listeners, when a - // worker that cannot tunnel is a worker that cannot be reached at all. - // Today it can still be reached at the addresses it advertises. + // Still deliberately not auto-re-registered, and now for a concrete + // reason rather than a deferral. Register CLEARS this node's NodeModel + // rows, on the assumption that a re-registering worker restarted with + // nothing loaded, so re-registering on a 401 would delete a live + // worker's replica rows on every retry, and under the name collision + // that produces the 401 the two workers would take turns doing it + // forever. That is a credential failure causing model reclamation, + // which is the one outcome this whole design exists to prevent. + // + // The fix belongs to whichever comes first: a re-auth path that mints a + // tunnel credential WITHOUT the rest of registration's side effects, or + // a worker identity that is not the operator-chosen name, which is what + // would make a collision detectable instead of silent. Until then the + // 401 is loud, names both causes, and the operator acts on it. staticTunnelToken := res.TunnelToken tunnelToken = func() string { return staticTunnelToken } connectNats = func() (*messaging.Client, error) { diff --git a/docker-compose.distributed.yaml b/docker-compose.distributed.yaml index 014a74bf91a4..937b86534e89 100644 --- a/docker-compose.distributed.yaml +++ b/docker-compose.distributed.yaml @@ -104,17 +104,18 @@ services: - BASE_IMAGE=ubuntu:24.04 command: - worker - # No HEALTHCHECK_ENDPOINT override is needed: the image's healthcheck + # No published ports and no advertised address: the worker holds one + # outbound tunnel to the frontend and binds only loopback, so nothing has to + # reach into this container. + # + # No HEALTHCHECK_ENDPOINT override is needed either: the image's healthcheck # detects worker mode and derives the port from LOCALAI_SERVE_ADDR below - # (gRPC base port - 1 = 50050). The worker's /readyz reports 503 when its - # NATS connection is down, and also when a backend process it is already - # serving no longer answers a TCP dial on its gRPC address. So `unhealthy` - # here means the worker genuinely cannot receive work, or cannot serve the - # models it is holding. + # (gRPC base port - 1 = 50050). It runs inside the container, so a loopback + # bind is enough for it. The worker's /readyz reports 503 when its NATS + # connection is down or a backend it serves no longer answers locally. So + # `unhealthy` means the worker cannot receive work or serve a held model. environment: LOCALAI_SERVE_ADDR: "0.0.0.0:50051" - LOCALAI_ADVERTISE_ADDR: "worker-1:50051" - LOCALAI_ADVERTISE_HTTP_ADDR: "worker-1:50050" DEBUG: "true" LOCALAI_REGISTER_TO: "http://localai:8080" LOCALAI_NODE_NAME: "worker-1" @@ -177,7 +178,11 @@ services: # Copy the worker-1 service above and change: # - Service name (e.g., worker-2) # - LOCALAI_NODE_NAME (must be unique) - # - LOCALAI_ADVERTISE_ADDR (must match service name) + # + # Nothing else. A worker has no address to make unique: it binds loopback + # inside its own container and dials out to the frontend. Note that + # LOCALAI_NODE_NAME really must differ: the registry upserts by name, so two + # workers sharing one steal each other's row and each other's tunnel credential. # # Workers are generic — no backend type needed. The SmartRouter # will dynamically install the required backend via NATS when diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index fc49074ea300..3727ab1ad02d 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -181,15 +181,24 @@ That distinction is the whole point rather than a nicety. A scheduler told that #### There is no frontend-side fallback, and upgrade order matters -`LOCALAI_WORKER_TUNNEL=false` still stops a worker dialling its tunnel, but it no longer has a frontend counterpart: after this change **no frontend path dials a worker's advertised address**, so a worker with the tunnel off is a worker the frontend cannot reach. Setting it is not a rollback. The rollback is to run the previous frontend release. +`LOCALAI_WORKER_TUNNEL=false` still stops a worker dialling its tunnel, but it no longer has a frontend counterpart: **no frontend path dials a worker's advertised address**, and a worker on this release advertises none and listens on no routable interface. A worker with the tunnel off is a worker nothing can reach. Setting it is not a rollback. The rollback is to run the previous release on both sides. That makes upgrade order matter, in one direction only: -- **Upgrade the workers first, then the frontends.** A worker on the new build dials its tunnel and is reachable by frontends of either version, because the old frontend still dials its advertised address and the worker still listens. +- **Upgrade the workers first, then the frontends.** A worker on this build dials its tunnel, and a frontend of either version reaches it through that. An old frontend that would have dialled its advertised address no longer gets one, so this order is what keeps the fleet routable throughout. - **Upgrading the frontends first** leaves every not-yet-restarted worker unroutable until it restarts. Those workers keep running their models and keep heartbeating, and the frontend reports them as unroutable rather than as gone: their `node_models` rows are left alone, nothing is rescheduled, and requests for those models fail loudly with "no route" until the worker reconnects. It is a degraded window, not an eviction, but it is a window, and doing it the other way round has none. A worker that cannot reach its frontend retries with exponential backoff and never gives up, so restarting a worker is all that is needed to close the window. +#### Workers bind nothing routable + +A worker on this release opens **no inbound listener on a routable interface**. Its backend gRPC processes and its HTTP file-transfer server all bind loopback, and the frontend reaches both through the tunnel. Concretely: + +- **No inbound firewall rule, published port, Service or Ingress is needed for a worker.** A worker needs outbound access to the frontend URL (`LOCALAI_REGISTER_TO`) and to NATS (`LOCALAI_NATS_URL`), and nothing else. +- **`LOCALAI_ADVERTISE_ADDR` and `LOCALAI_ADVERTISE_HTTP_ADDR` are gone.** There is nothing to advertise. Both are ignored if still set; remove them. +- **`LOCALAI_ADDR` and `LOCALAI_SERVE_ADDR` are read for their port only.** The port is the base of the backend port range, and `port-1` is the HTTP file-transfer port. The host half names an interface nothing binds. +- The node's `address` and `http_address` fields in `GET /api/nodes` are empty, and are cleared for nodes that reported them before the upgrade. + ### The model load deadline scales with the checkpoint The `LoadModel` deadline starts *after* the backend is installed and the model files are staged, so it covers only the worker backend's own checkpoint read and pipeline init. That work is proportional to the bytes on disk, which makes any fixed deadline a model-size cliff rather than a timeout: a 70 GB video checkpoint on a Jetson Thor worker failed reproducibly against the old fixed 5m default (`rpc error: code = DeadlineExceeded` after 953.5s of wall clock, roughly 11m of which was backend install and staging), and simply raising the constant would only move the cliff to the next larger model while making a genuinely wedged *small* model hang for the whole inflated duration. @@ -352,7 +361,9 @@ during installation as well as the committed snapshot. {{% /notice %}} {{% notice warning %}} -The worker HTTP file transfer server is authenticated by `LOCALAI_REGISTRATION_TOKEN`. If the token is **empty**, the server **fails open** - anyone who can reach the port gets read/write access to the worker's models/staging/data directories (a remote model-poisoning / exfiltration vector). The worker logs a loud warning at startup in this case. Always set `LOCALAI_REGISTRATION_TOKEN` in distributed mode, and set `LOCALAI_DISTRIBUTED_REQUIRE_AUTH=true` (frontend **and** workers) to make a missing token *or* missing NATS credentials a hard startup error rather than a silent fail-open. Firewall the file-transfer port (gRPC base − 1) so only the frontend can reach it. +The worker HTTP file transfer server is authenticated by `LOCALAI_REGISTRATION_TOKEN`. If the token is **empty**, the server **fails open** - anyone who can reach the port gets read/write access to the worker's models/staging/data directories (a remote model-poisoning / exfiltration vector). The worker logs a loud warning at startup in this case. Always set `LOCALAI_REGISTRATION_TOKEN` in distributed mode, and set `LOCALAI_DISTRIBUTED_REQUIRE_AUTH=true` (frontend **and** workers) to make a missing token *or* missing NATS credentials a hard startup error rather than a silent fail-open. + +By default the server binds loopback, so "anyone who can reach the port" means a process on the worker host, and no firewall rule is required. Setting `LOCALAI_HTTP_ADDR` to a routable address opts back out of that and puts the fail-open case back on the network - if you do it, firewall the port. {{% /notice %}} ### Watching Backend Installs @@ -402,11 +413,10 @@ local-ai worker \ | Flag | Env Var | Default | Description | |------|---------|---------|-------------| -| `--addr` | `LOCALAI_SERVE_ADDR` | `0.0.0.0:50051` | gRPC listen address | +| `--addr` | `LOCALAI_ADDR` | *(unset)* | Base port for backend gRPC processes. Only the port is used; nothing binds the host | +| `--serve-addr` | `LOCALAI_SERVE_ADDR` | `0.0.0.0:50051` | Same, used when `--addr` is unset | | `--grpc-max-port` | `LOCALAI_GRPC_MAX_PORT` | `65535` | Highest port the worker may assign to a backend gRPC process. Each backend gets its own port, allocated upward from the base port, so the width of `[base port, this]` caps how many backends this worker can run at once (see [Backend gRPC port range](#backend-grpc-port-range)) | -| `--advertise-addr` | `LOCALAI_ADVERTISE_ADDR` | *(auto)* | Address the frontend uses to reach this node (see below) | -| `--http-addr` | `LOCALAI_HTTP_ADDR` | gRPC port - 1 | HTTP file transfer server bind address | -| `--advertise-http-addr` | `LOCALAI_ADVERTISE_HTTP_ADDR` | *(auto)* | HTTP address the frontend uses for file transfer | +| `--http-addr` | `LOCALAI_HTTP_ADDR` | `127.0.0.1:{gRPC port - 1}` | HTTP file transfer server bind address | | `--ephemeral-staging-byte-limit` | `LOCALAI_EPHEMERAL_STAGING_BYTE_LIMIT` | `0` (automatic) | Maximum bytes held by request-input staging across the worker's HTTP staging directory and S3 cache. Automatic mode uses the smaller of 10 GiB and 10% of filesystem capacity. | | `--ephemeral-staging-min-free-bytes` | `LOCALAI_EPHEMERAL_STAGING_MIN_FREE_BYTES` | `0` (automatic) | Free filesystem space preserved while staging request inputs. Automatic mode uses the larger of 1 GiB and 5% of filesystem capacity. | | `--register-to` | `LOCALAI_REGISTER_TO` | *(required)* | Frontend URL for self-registration | @@ -428,9 +438,9 @@ local-ai worker \ | `--vram-budget` | `LOCALAI_VRAM_BUDGET` | *(empty)* | Cap the VRAM this node advertises for model placement, as a percentage (e.g. `80%`) or an absolute amount (e.g. `12GB`). Empty uses all detected VRAM. See [Per-node VRAM budget](#per-node-vram-budget). | {{% notice tip %}} -**Advertise address:** The `--addr` flag is the local bind address for gRPC. The `--advertise-addr` is the address the frontend stores and uses to reach the worker via gRPC. If not set, the worker auto-derives it by replacing `0.0.0.0` with the OS hostname (which in Docker is the container ID, resolvable via Docker DNS). Set `--advertise-addr` explicitly when the auto-detected hostname is not routable from the frontend (e.g., in Kubernetes, use the pod's service DNS name). +**There is no advertise address.** A worker states no endpoint at registration and binds nothing routable; the frontend reaches it through the tunnel it dials. `--advertise-addr` and `--advertise-http-addr` no longer exist. `--addr` and `--http-addr` remain, and set where the worker listens **locally**: only the port of `--addr` is used, and `--http-addr` binds loopback by default. -**HTTP file transfer:** Each worker also runs a small HTTP server for file transfer (model files, configs). By default it listens on the gRPC base port - 1 (e.g., if gRPC base is 50051, HTTP is on 50050). gRPC ports grow upward from the base port as additional models are loaded. Set `--advertise-http-addr` if the auto-detected address is not routable from the frontend. +**HTTP file transfer:** Each worker also runs a small HTTP server for file transfer (model files, configs). It listens on loopback at the gRPC base port - 1 (e.g., if gRPC base is 50051, HTTP is on 50050). gRPC ports grow upward from the base port as additional models are loaded. {{% /notice %}} ### Ephemeral request-input storage @@ -441,7 +451,7 @@ Set both capacity variables to positive byte counts when a worker needs fixed li ### Worker Health Probes -The worker's HTTP server (base port - 1, default 50050) exposes two unauthenticated probes: +The worker's HTTP server (loopback, base port - 1, default 50050) exposes two unauthenticated probes. They are reachable from the worker host - which is where a container healthcheck runs - and not from the network: | Endpoint | Meaning | |----------|---------| @@ -454,34 +464,29 @@ The same applies to the data path. A worker can hold a live NATS link while the Only backends in the middle of their lifecycle are dialled. A backend that is still starting is skipped until its gRPC server has answered a health check, which can take 10 to 15 seconds on a slow node, and a backend that is stopping is skipped from the moment shutdown begins. Neither a cold start nor an ordinary shutdown makes a worker report 503, so a Kubernetes `readinessProbe` at the usual 10s period does not pull a worker out of rotation every time it loads a model. -The container image's `HEALTHCHECK` detects worker mode and probes this endpoint automatically; no `HEALTHCHECK_ENDPOINT` override is needed. Set `HEALTHCHECK_ENDPOINT` only to pin an explicit URL. +The container image's `HEALTHCHECK` detects worker mode and probes this endpoint automatically, deriving the port from `LOCALAI_HTTP_ADDR`, else `LOCALAI_ADDR`, else `LOCALAI_SERVE_ADDR`, minus one - the same order the worker itself uses. No `HEALTHCHECK_ENDPOINT` override is needed. Set `HEALTHCHECK_ENDPOINT` only when the bind address is passed as a CLI flag rather than an environment variable, or to pin an explicit URL. -### Worker Address Configuration +### Worker Port Configuration -The simplest way to configure a worker's network address is with a single variable: +A worker needs no address configuration at all. It binds only loopback and reaches the frontend outbound, so the defaults work behind NAT, in another cluster, or on a laptop: -| Variable | Description | -|----------|-------------| -| `LOCALAI_ADDR` | Reachable address of this worker (`host:port`). The port is used as the base for gRPC backend processes, and `port-1` for the HTTP file transfer server. | - -**Example:** ```yaml environment: - LOCALAI_ADDR: "192.168.1.100:50051" LOCALAI_NATS_URL: "nats://frontend:4222" LOCALAI_REGISTER_TO: "http://frontend:8080" LOCALAI_REGISTRATION_TOKEN: "my-secret" ``` -For advanced networking scenarios (NAT, load balancers, separate gRPC/HTTP ports), the following override variables are available: +Set the variables below only to move the worker's **local** port range - for example when two workers share a host, or when the default range collides with something else. Only the port of each is used; the host half names an interface nothing binds. | Variable | Description | Default | |----------|-------------|---------| -| `LOCALAI_SERVE_ADDR` | gRPC base port bind address | `0.0.0.0:50051` | +| `LOCALAI_ADDR` | Base port for backend gRPC processes, as `host:port`. `port-1` is the HTTP file-transfer port | *(unset; falls back to `LOCALAI_SERVE_ADDR`)* | +| `LOCALAI_SERVE_ADDR` | Base port, as above, when `LOCALAI_ADDR` is unset | `0.0.0.0:50051` | | `LOCALAI_GRPC_MAX_PORT` | Highest port assignable to a backend gRPC process | `65535` | -| `LOCALAI_HTTP_ADDR` | HTTP file transfer bind address | `0.0.0.0:{gRPC port - 1}` | -| `LOCALAI_ADVERTISE_ADDR` | Public gRPC address (if different from `LOCALAI_ADDR`) | Derived from `LOCALAI_ADDR` | -| `LOCALAI_ADVERTISE_HTTP_ADDR` | Public HTTP address (if different from gRPC host) | Derived from advertise host + HTTP port | +| `LOCALAI_HTTP_ADDR` | HTTP file transfer bind address. Bound exactly as given, so this is also the way to expose that server deliberately | `127.0.0.1:{base port - 1}` | + +`LOCALAI_ADVERTISE_ADDR` and `LOCALAI_ADVERTISE_HTTP_ADDR` no longer exist. They named the endpoint the frontend dialled; nothing dials a worker any more. Remove them. ### Backend gRPC port range diff --git a/scripts/build/healthcheck.sh b/scripts/build/healthcheck.sh index eff574deda33..d47d16f3b45f 100755 --- a/scripts/build/healthcheck.sh +++ b/scripts/build/healthcheck.sh @@ -19,9 +19,9 @@ # 3. The frontend endpoint, when the mode cannot be determined. # # Ports are read from environment variables only, which is how containers are -# configured in practice (compose/k8s set LOCALAI_ADDRESS, LOCALAI_SERVE_ADDR, -# ...). If you instead pass the bind address as a CLI flag, set -# HEALTHCHECK_ENDPOINT to match. +# configured in practice (compose/k8s set LOCALAI_ADDRESS, LOCALAI_ADDR, +# LOCALAI_SERVE_ADDR, ...). If you instead pass the bind address as a CLI flag, +# set HEALTHCHECK_ENDPOINT to match. set -u # Detect the arguments local-ai was started with. PID 1 is the usual case @@ -99,9 +99,24 @@ if [ -z "$endpoint" ]; then # The worker's file-transfer server (which also serves /readyz and # /healthz) binds LOCALAI_HTTP_ADDR when set, otherwise the gRPC # base port minus one. See Config.resolveHTTPAddr. + # + # The base port comes from LOCALAI_ADDR first and LOCALAI_SERVE_ADDR + # second, which is Config.effectiveBasePort's own order. Reading + # only the second one meant a worker configured with LOCALAI_ADDR + # (the documented knob; LOCALAI_SERVE_ADDR is marked hidden) was + # probed on the default 50050 while its server sat on a different + # port. That is #10987 again: a working worker reporting + # `unhealthy` forever because the probe went somewhere nothing + # binds. + # + # The worker binds loopback, which is where this probe runs: it runs + # inside the container, so no inbound port is needed for it to work. port=$(port_of "${LOCALAI_HTTP_ADDR:-}") if [ -z "$port" ]; then - base=$(port_of "${LOCALAI_SERVE_ADDR:-}") + base=$(port_of "${LOCALAI_ADDR:-}") + if [ -z "$base" ]; then + base=$(port_of "${LOCALAI_SERVE_ADDR:-}") + fi port=$(( ${base:-50051} - 1 )) fi endpoint="http://localhost:${port}/readyz" diff --git a/scripts/build/healthcheck_test.sh b/scripts/build/healthcheck_test.sh index 86d2ff098a20..9dc811b8fbf4 100644 --- a/scripts/build/healthcheck_test.sh +++ b/scripts/build/healthcheck_test.sh @@ -101,6 +101,21 @@ echo "== worker derives the port from LOCALAI_SERVE_ADDR" run_hc 0 "local-ai worker" LOCALAI_SERVE_ADDR="0.0.0.0:60000" expect_url "http://localhost:59999/readyz" +echo "== worker derives the port from LOCALAI_ADDR" +# LOCALAI_ADDR is the worker's documented base-port knob (LOCALAI_SERVE_ADDR is +# hidden), and Config.effectiveBasePort reads it FIRST. A probe that ignored it +# went to 50050 while the server sat elsewhere, which is #10987's symptom. +run_hc 0 "local-ai worker" LOCALAI_ADDR="0.0.0.0:60000" +expect_url "http://localhost:59999/readyz" + +echo "== LOCALAI_ADDR outranks LOCALAI_SERVE_ADDR, as effectiveBasePort does" +run_hc 0 "local-ai worker" LOCALAI_ADDR="0.0.0.0:60000" LOCALAI_SERVE_ADDR="0.0.0.0:50051" +expect_url "http://localhost:59999/readyz" + +echo "== an explicit LOCALAI_HTTP_ADDR still outranks both" +run_hc 0 "local-ai worker" LOCALAI_ADDR="0.0.0.0:60000" LOCALAI_HTTP_ADDR="0.0.0.0:18081" +expect_url "http://localhost:18081/readyz" + echo "== worker honours an explicit LOCALAI_HTTP_ADDR" run_hc 0 "local-ai worker" LOCALAI_HTTP_ADDR="0.0.0.0:18080" expect_url "http://localhost:18080/readyz" diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 521882efe962..0ea29ae65ecd 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -263,10 +263,12 @@ func (c *Cluster) startWorker(i int) (*Process, error) { "--backends-path", backends, ) cmd.Env = append(cmd.Environ(), + // Ports only. A worker advertises nothing, so there is no advertise + // address to set; these two exist to keep concurrently running workers + // off each other's ports, not to make anything reachable. Both binds + // are loopback whatever is set here. fmt.Sprintf("LOCALAI_SERVE_ADDR=127.0.0.1:%d", grpcPort), - fmt.Sprintf("LOCALAI_ADVERTISE_ADDR=127.0.0.1:%d", grpcPort), fmt.Sprintf("LOCALAI_HTTP_ADDR=127.0.0.1:%d", httpPort), - fmt.Sprintf("LOCALAI_ADVERTISE_HTTP_ADDR=127.0.0.1:%d", httpPort), // Workers register with frontend 0 ONLY unless the caller opts into // SpreadWorkerRegistrations, and the cross-replica session specs depend // on that default. They prove a session minted at frontend 0 resolves at diff --git a/tests/e2e/distributed/model_config_revision_test.go b/tests/e2e/distributed/model_config_revision_test.go index 548a35eb4b93..9dde7f8b801f 100644 --- a/tests/e2e/distributed/model_config_revision_test.go +++ b/tests/e2e/distributed/model_config_revision_test.go @@ -33,7 +33,7 @@ func (s *revisionCleanupStopper) StopModelReplica(_ context.Context, nodeID stri Matched: true, Terminated: true, ProcessKey: replica.ModelName, - Address: replica.Address, + Address: replica.WorkerLocalAddress, }, nil } From 4ce7efd11244236b22976a331f7ad2f4254e1d16 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 19:36:35 +0000 Subject: [PATCH 060/153] fix(distributed): refuse a worker that cannot tunnel, and say why it was refused Review round 1 on the change that stopped workers listening. One blocking item and seven notes. LOCALAI_WORKER_TUNNEL=false was the blocking one, and the ruling was to make it fatal rather than to correct the comment that still promised it fell back to the advertised address. There is no fallback left: a worker on this branch advertises nothing and binds only loopback, so turning the tunnel off leaves it reachable by nothing while it registers, heartbeats and reports healthy, and the scheduler keeps placing models on it. That is the worst available failure shape, so a new Config.validateStartup refuses it before prefetch, registration and NATS, while the worker is still invisible to the cluster. It absorbs the pre-existing empty-registration-token check, which had the same shape and no spec. The flag is kept rather than deleted so an operator who set it is told the promise is gone instead of having the setting ignored, and the guard around StartTunnel is removed, because a branch nothing can take reads as a supported no-tunnel mode that does not exist. The justification for erroring on an install that names no address was wrong, and the review is right that this is the dangerous form of overclaiming, because the conclusion holds and the mechanism does not. It said the resulting empty target would be refused as an invalid stream and that the refusal would read as the worker answering about its backend. Nothing in this repo branches on cluster.ErrNoRoute, and nodes.unroutable treats any recorded dial error as unroutable, so that refusal reaches every reap guard as ProbeUnknown and deletes nothing. The site now stands on what holds, that an install naming no port produced nothing routable and the failure belongs to the install rather than to a later probe, and records the retracted claim so nobody re-derives it. This retracts the same paragraph in the body of 1cf847f29. The reviewer deleted the whole tryWarmPath unnamed-replica guard and the suite stayed green, including the reservation release. It is specced now, and the asymmetry the review asked about is decided at the site: the row stays, unlike the sibling !alive branch which removes it. That branch has observed a backend dead; this one has observed only that the row is unreadable, which says nothing about whether a process is running, and the row is the last record that one might be, since the acknowledged stop path refuses a stop whose ExpectedAddress does not match and an empty one cannot be cleaned up through it either. The cross-version wire claim rested on two struct tags nobody asserted: renaming only the json keys survived mutation while the gorm column rename went red through raw SQL. Both keys are pinned now, marshal and unmarshal, per struct. A worker-first upgrade showed the operator a status code and not the reason. The registration client discarded the body, so "address is required for backend workers" was read off the socket and thrown away, and the ladder then spent four minutes on a verdict the frontend reached instantly. Refusals now quote the body and carry ErrRegistrationRejected, and both the ladder and the credential manager's Acquire stop on the first one. Acquire matters more than the ladder: it is the default path and its bound is 100 attempts, not 10. 408 and 429 are deliberately not refusals, since both are the frontend asking for the same request again. Also: the stale "not blocked by firewalls" troubleshooting line, which now names the real cause and the knobs that move the port range; and the inert address fields on the MCP Node DTO, which the Assistant was still being handed. The review named http_address there and I removed address too, because it is inert by the same argument and leaving one of a pair is arbitrary. Five mutations, all red. Deleting the warm-path guard reddens four specs and falsifying only its reservation release reddens one, so the two halves are pinned separately. Renaming only the json keys reddens both wire suites. Discarding the refusal body reddens two. Dropping the rejection classification does not fail the suite, it hangs it, which is the operator-visible symptom, so it is recorded red under a ginkgo timeout. The verify list is now derived from the diff rather than from the brief, which is what let the previous round ship a spec asserting 200 where the endpoint returns 201: nine ginkgo suites, the e2e vet, route auth coverage, the leaf check, build, the healthcheck shell suite and lint. The two jsx files have no harness in this worktree and are recorded as the one unverified surface. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/cli/workerregistry/client.go | 62 +++++++- core/cli/workerregistry/client_test.go | 138 ++++++++++++++++++ core/cli/workerregistry/credentials.go | 6 + core/services/messaging/subjects_wire_test.go | 46 ++++++ core/services/nodes/model_router_test.go | 14 +- core/services/nodes/registry_wire_test.go | 39 +++++ core/services/nodes/router.go | 46 ++++-- .../nodes/router_unnamed_replica_test.go | 97 ++++++++++++ core/services/worker/addr_test.go | 45 ++++++ core/services/worker/config.go | 43 +++++- core/services/worker/worker.go | 54 +++---- docs/content/features/distributed-mode.md | 4 +- pkg/mcp/localaitools/dto.go | 16 +- pkg/mcp/localaitools/dto_test.go | 2 +- pkg/mcp/localaitools/httpapi/client.go | 14 +- 15 files changed, 570 insertions(+), 56 deletions(-) create mode 100644 core/cli/workerregistry/client_test.go create mode 100644 core/services/messaging/subjects_wire_test.go create mode 100644 core/services/nodes/registry_wire_test.go create mode 100644 core/services/nodes/router_unnamed_replica_test.go diff --git a/core/cli/workerregistry/client.go b/core/cli/workerregistry/client.go index f8728166fef9..fb00fb3f1d86 100644 --- a/core/cli/workerregistry/client.go +++ b/core/cli/workerregistry/client.go @@ -8,7 +8,9 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" + "io" "net/http" "strings" "sync" @@ -93,7 +95,7 @@ func (c *RegistrationClient) RegisterFull(ctx context.Context, body map[string]a defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, fmt.Errorf("registration failed with status %d", resp.StatusCode) + return nil, registrationStatusError(resp) } var result RegisterResponse @@ -103,6 +105,58 @@ func (c *RegistrationClient) RegisterFull(ctx context.Context, body map[string]a return &result, nil } +// ErrRegistrationRejected marks a registration the frontend REFUSED, as opposed +// to one it could not answer. +// +// Retrying a refusal cannot change it: the request is wrong, or this worker is +// not allowed to make it. The one that matters in practice is a worker of this +// release registering against a frontend that predates it, which answers +// "address is required for backend workers" with 400, because a worker no +// longer has an address to send. Without this the retry ladder spends four +// minutes on a verdict the frontend reached instantly, and the operator watches +// it before being told anything. +// +// 408 and 429 are deliberately NOT rejections. Both are the frontend asking for +// the same request again later, which is exactly what a retry does. +var ErrRegistrationRejected = errors.New("the frontend refused this registration") + +// maxRegistrationErrorBody bounds how much of a refusal's body is quoted back. +// Enough for a message, not enough for an HTML error page to bury the log line +// it is meant to explain. +const maxRegistrationErrorBody = 512 + +// registrationStatusError turns a non-2xx response into an error that says WHY. +// +// The body is the point. The frontend explains its refusals there +// ("address is required for backend workers", "invalid registration token"), +// and discarding it left an operator with a bare status code: the one line that +// would tell them which of several possible mistakes they made was read off the +// socket and thrown away. +func registrationStatusError(resp *http.Response) error { + detail, err := io.ReadAll(io.LimitReader(resp.Body, maxRegistrationErrorBody)) + if err != nil { + xlog.Debug("Could not read the frontend's registration error body", "status", resp.StatusCode, "error", err) + } + msg := strings.Join(strings.Fields(string(detail)), " ") + base := fmt.Sprintf("registration failed with status %d", resp.StatusCode) + if msg != "" { + base = fmt.Sprintf("%s: %s", base, msg) + } + if isRegistrationRejection(resp.StatusCode) { + return fmt.Errorf("%s: %w", base, ErrRegistrationRejected) + } + return errors.New(base) +} + +// isRegistrationRejection reports whether a status is a verdict rather than a +// condition that may pass. +func isRegistrationRejection(status int) bool { + if status == http.StatusRequestTimeout || status == http.StatusTooManyRequests { + return false + } + return status >= 400 && status < 500 +} + // Register sends a single registration request and returns the node ID and // optional credentials (API token for agent workers, NATS JWT when configured). func (c *RegistrationClient) Register(ctx context.Context, body map[string]any) (nodeID, apiToken, natsJWT, natsSeed string, err error) { @@ -138,6 +192,12 @@ func (c *RegistrationClient) RegisterFullWithRetry(ctx context.Context, body map if err == nil { return res, nil } + if errors.Is(err, ErrRegistrationRejected) { + // A verdict, not an outage. Reported on the first attempt so the + // reason the frontend gave is the first thing in the log rather + // than the last, after the ladder. + return nil, err + } if attempt == maxRetries { return nil, fmt.Errorf("failed after %d attempts: %w", maxRetries, err) } diff --git a/core/cli/workerregistry/client_test.go b/core/cli/workerregistry/client_test.go new file mode 100644 index 000000000000..5870d2524b5c --- /dev/null +++ b/core/cli/workerregistry/client_test.go @@ -0,0 +1,138 @@ +package workerregistry + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The case these specs exist for: a worker of this release registering against +// a frontend that predates it. A worker no longer sends an address, the old +// frontend requires one, and it answers 400 with the reason in the body. Two +// things used to go wrong there at once. The reason was discarded, so the +// operator saw only "status 400" and had to guess which of several mistakes +// they had made; and the retry ladder spent four minutes on a verdict the +// frontend reached instantly. +var _ = Describe("Registration client refusals", func() { + var ( + attempts atomic.Int32 + status atomic.Int32 + body atomic.Value // string + server *httptest.Server + client *RegistrationClient + // seen carries one token per request the handler served, so a spec can + // wait for the Nth attempt instead of sleeping for however long the + // ladder's backoff happens to be. + seen chan struct{} + ) + + BeforeEach(func() { + attempts.Store(0) + status.Store(int32(http.StatusBadRequest)) + body.Store(`{"error":{"code":400,"message":"address is required for backend workers"}}`) + seen = make(chan struct{}, 64) + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + select { + case seen <- struct{}{}: + default: + } + w.WriteHeader(int(status.Load())) + _, _ = w.Write([]byte(body.Load().(string))) + })) + client = &RegistrationClient{FrontendURL: server.URL, HTTPTimeout: 2 * time.Second} + }) + + AfterEach(func() { server.Close() }) + + It("quotes what the frontend said", func() { + _, err := client.RegisterFull(context.Background(), map[string]any{"name": "w1"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("status 400")) + Expect(err.Error()).To(ContainSubstring("address is required for backend workers")) + }) + + It("marks a 4xx as a refusal", func() { + _, err := client.RegisterFull(context.Background(), map[string]any{"name": "w1"}) + Expect(err).To(MatchError(ErrRegistrationRejected)) + }) + + It("does not mark a 5xx as a refusal", func() { + // A frontend that is restarting or wedged has not judged anything, and + // retrying it is the whole reason the ladder exists. + status.Store(int32(http.StatusBadGateway)) + body.Store("bad gateway") + _, err := client.RegisterFull(context.Background(), map[string]any{"name": "w1"}) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, ErrRegistrationRejected)).To(BeFalse()) + }) + + DescribeTable("treats a status that asks for the same request again as retryable", + func(code int) { + status.Store(int32(code)) + body.Store("later") + _, err := client.RegisterFull(context.Background(), map[string]any{"name": "w1"}) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, ErrRegistrationRejected)).To(BeFalse()) + }, + Entry("408 Request Timeout", http.StatusRequestTimeout), + Entry("429 Too Many Requests", http.StatusTooManyRequests), + ) + + It("stops the retry ladder on the first refusal", func() { + // Ten attempts on a 400 is roughly four minutes of backoff before the + // operator is told anything, and the answer is the same one the + // frontend gave immediately. + _, err := client.RegisterFullWithRetry(context.Background(), map[string]any{"name": "w1"}, 10) + Expect(err).To(MatchError(ErrRegistrationRejected)) + Expect(err.Error()).To(ContainSubstring("address is required for backend workers")) + Expect(attempts.Load()).To(Equal(int32(1))) + }) + + It("still retries something that is not a refusal", func() { + // The control. Without it, a change that returned on EVERY error would + // pass the spec above and silently delete the retry behaviour a worker + // booting alongside its frontend depends on. + status.Store(int32(http.StatusServiceUnavailable)) + body.Store("starting up") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan error, 1) + go func() { + _, err := client.RegisterFullWithRetry(ctx, map[string]any{"name": "w1"}, 10) + done <- err + }() + + // Two tokens is the whole assertion: the ladder came back for a second + // attempt on a status that is not a verdict. Waiting on the handler + // rather than on a duration makes it exact instead of tolerant. + Eventually(seen).Should(Receive()) + Eventually(seen, "10s").Should(Receive()) + cancel() + + var ladderErr error + Eventually(done).Should(Receive(&ladderErr)) + Expect(ladderErr).To(HaveOccurred()) + Expect(errors.Is(ladderErr, ErrRegistrationRejected)).To(BeFalse()) + Expect(attempts.Load()).To(BeNumerically(">=", 2)) + }) + + It("stops the credential manager's acquire loop on a refusal", func() { + // The default worker path goes through Acquire, not the ladder above, + // and its bound is 100 attempts rather than 10. A refusal there is the + // same verdict and has to end the same way. + mgr := NewNATSCredentialManager(func(ctx context.Context) (*RegisterResponse, error) { + return client.RegisterFull(ctx, map[string]any{"name": "w1"}) + }, true) + _, err := mgr.Acquire(context.Background()) + Expect(err).To(MatchError(ErrRegistrationRejected)) + Expect(attempts.Load()).To(Equal(int32(1))) + }) +}) diff --git a/core/cli/workerregistry/credentials.go b/core/cli/workerregistry/credentials.go index f9d8c2231ddc..b023b9916918 100644 --- a/core/cli/workerregistry/credentials.go +++ b/core/cli/workerregistry/credentials.go @@ -2,6 +2,7 @@ package workerregistry import ( "context" + "errors" "fmt" "sync" "time" @@ -146,6 +147,11 @@ func (m *NATSCredentialManager) Acquire(ctx context.Context) (*RegisterResponse, for attempt := 1; m.maxAttempts <= 0 || attempt <= m.maxAttempts; attempt++ { res, err := m.register(ctx) switch { + case errors.Is(err, ErrRegistrationRejected): + // The frontend refused rather than failed. Waiting through the full + // attempt ladder would delay the operator's only explanation by the + // length of the ladder and change nothing about the answer. + return nil, err case err != nil: lastReason = err xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", backoff, "error", err) diff --git a/core/services/messaging/subjects_wire_test.go b/core/services/messaging/subjects_wire_test.go new file mode 100644 index 000000000000..2c90254e8ff5 --- /dev/null +++ b/core/services/messaging/subjects_wire_test.go @@ -0,0 +1,46 @@ +package messaging + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// BackendInstallReply.WorkerLocalAddress was called Address until workers +// stopped advertising. The Go field was renamed so no reader takes it for a +// dial target; the wire key was deliberately NOT renamed, because a worker and +// a frontend from different releases have to keep understanding each other +// across a rolling upgrade. +// +// That is a cross-version compatibility property resting on one struct tag, and +// a struct tag nobody asserts is a property nobody has. Renaming just the tags +// left the whole suite green when this was written. +var _ = Describe("backend.install reply wire format", func() { + It("writes the address under the key an older frontend reads", func() { + out, err := json.Marshal(BackendInstallReply{Success: true, WorkerLocalAddress: "127.0.0.1:50052"}) + Expect(err).ToNot(HaveOccurred()) + + var raw map[string]any + Expect(json.Unmarshal(out, &raw)).To(Succeed()) + Expect(raw).To(HaveKeyWithValue("address", "127.0.0.1:50052")) + Expect(raw).ToNot(HaveKey("worker_local_address"), + "renaming the wire key would make every install reply unreadable to a frontend of another release") + }) + + It("reads the address an older worker sends", func() { + // An older worker puts its ADVERTISED host here. Only the port is used, + // and the port is the same, so accepting it is both harmless and the + // thing that keeps a mixed fleet working. + var reply BackendInstallReply + Expect(json.Unmarshal([]byte(`{"success":true,"address":"worker-1:50052"}`), &reply)).To(Succeed()) + Expect(reply.Success).To(BeTrue()) + Expect(reply.WorkerLocalAddress).To(Equal("worker-1:50052")) + }) + + It("omits the address when the install failed", func() { + out, err := json.Marshal(BackendInstallReply{Success: false, Error: "boom"}) + Expect(err).ToNot(HaveOccurred()) + Expect(string(out)).ToNot(ContainSubstring("address")) + }) +}) diff --git a/core/services/nodes/model_router_test.go b/core/services/nodes/model_router_test.go index 42e7a5dd19f2..775c13fdf1b2 100644 --- a/core/services/nodes/model_router_test.go +++ b/core/services/nodes/model_router_test.go @@ -22,6 +22,7 @@ type fakeModelRouterForSmartRouter struct { nodeModel *NodeModel findErr error decrementCalled map[string]int // "nodeID:model" -> count + removed []string // "nodeID:model:replica" per RemoveNodeModel } func newFakeModelRouterForSmartRouter() *fakeModelRouterForSmartRouter { @@ -46,9 +47,20 @@ func (f *fakeModelRouterForSmartRouter) DecrementInFlight(_ context.Context, nod func (f *fakeModelRouterForSmartRouter) IncrementInFlight(_ context.Context, _, _ string, _ int) error { return nil } -func (f *fakeModelRouterForSmartRouter) RemoveNodeModel(_ context.Context, _, _ string, _ int) error { +func (f *fakeModelRouterForSmartRouter) RemoveNodeModel(_ context.Context, nodeID, modelName string, replicaIndex int) error { + f.mu.Lock() + defer f.mu.Unlock() + f.removed = append(f.removed, fmt.Sprintf("%s:%s:%d", nodeID, modelName, replicaIndex)) return nil } + +// removedModels lists the replica rows the code under test deleted, so a spec +// can assert a branch left a row alone rather than only that it returned nil. +func (f *fakeModelRouterForSmartRouter) removedModels() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.removed...) +} func (f *fakeModelRouterForSmartRouter) RemoveAllNodeModelReplicas(_ context.Context, _, _ string) error { return nil } diff --git a/core/services/nodes/registry_wire_test.go b/core/services/nodes/registry_wire_test.go new file mode 100644 index 000000000000..34eec32e44df --- /dev/null +++ b/core/services/nodes/registry_wire_test.go @@ -0,0 +1,39 @@ +package nodes + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// NodeModel.WorkerLocalAddress was called Address. The Go field was renamed so +// no reader takes it for a frontend-dialable endpoint; the json key was kept so +// no API consumer breaks, and the gorm column was kept so no migration is +// needed. +// +// The column half is enforced by the raw-SQL fragments in this package, which +// fail loudly against a renamed column. The json half had nothing enforcing it: +// renaming only the tag left this package, messaging and endpoints/localai all +// green. These specs are that half. +var _ = Describe("NodeModel wire format", func() { + It("serves the address under the key API consumers already read", func() { + out, err := json.Marshal(NodeModel{ + NodeID: "node-1", ModelName: "m", ReplicaIndex: 1, + WorkerLocalAddress: "127.0.0.1:50052", + }) + Expect(err).ToNot(HaveOccurred()) + + var raw map[string]any + Expect(json.Unmarshal(out, &raw)).To(Succeed()) + Expect(raw).To(HaveKeyWithValue("address", "127.0.0.1:50052")) + Expect(raw).ToNot(HaveKey("worker_local_address"), + "GET /api/nodes/{id}/models and /api/nodes/models both serve this struct verbatim") + }) + + It("round-trips a body written against the documented key", func() { + var nm NodeModel + Expect(json.Unmarshal([]byte(`{"node_id":"node-1","model_name":"m","address":"127.0.0.1:50052"}`), &nm)).To(Succeed()) + Expect(nm.WorkerLocalAddress).To(Equal("127.0.0.1:50052")) + }) +}) diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index fbbeab061b4f..b861122d60b1 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -734,12 +734,28 @@ func (r *SmartRouter) tryWarmPath(ctx context.Context, att *routeAttempt) *Route modelAddr := nm.WorkerLocalAddress replicaIdx := nm.ReplicaIndex - // A replica row that does not name its backend process cannot be routed - // to. There is no node address left to stand in for it, and naming an - // empty target would make the worker refuse the stream as invalid: a - // refusal that reads as the worker answering about its backend, which is - // evidence this row is not entitled to produce. Fall through to a cold - // load, which either replaces the row or reports a real failure. + // A replica row that does not name its backend process cannot be routed to. + // There is no node address left to stand in for it, and an empty target + // names no process, so the request would open a stream the worker refuses + // as an invalid request rather than one that reaches a backend. Fall + // through to a cold load, which either replaces the row or reports a real + // failure. + // + // The row is left in place, unlike the !alive branch below which removes + // it. That branch has OBSERVED a backend dead; this one has observed only + // that the row is unreadable, which says nothing about whether a process is + // running on that worker. The row is also the last record that one might + // be: the acknowledged stop path matches on ExpectedAddress and a worker + // refuses a stop whose address does not match, so an empty one cannot be + // cleaned up through it either. Keeping the row costs a lock and a + // decrement per request before the cold load and leaves something an + // operator can see; removing it would free the replica slot for a second + // copy of the model while the first one, if it exists, keeps its VRAM with + // nothing left pointing at it. + // + // Defensive rather than reachable: installBackendOnNode below refuses an + // install that names no address, so no row written by this release can look + // like this. if modelAddr == "" { if err := r.registry.DecrementInFlight(ctx, node.ID, att.trackingKey, replicaIdx); err != nil { xlog.Warn("Failed to release a reservation for an unnamed replica", @@ -1414,9 +1430,21 @@ func (r *SmartRouter) installBackendOnNode(ctx context.Context, node *BackendNod // Where the backend process listens on that worker. There is no node // address to fall back to any more, and there should not be: a worker // that reports success without naming the port it started the process - // on has told us nothing routable, and inventing a target would surface - // later as the worker refusing an invalid stream, which reads as - // evidence about the backend rather than about this install. + // on has produced nothing routable, and the failure belongs to THIS + // install rather than to whatever later step first tries to use the + // address. Substituting one would push a known-bad value into a replica + // row and defer the error to a probe, where its cause is no longer + // visible. + // + // An earlier version of this comment justified it by saying the worker + // would refuse the resulting empty target as an invalid stream and that + // the refusal would read as the worker answering about its backend. The + // first half is true (see cluster.isWorkerAnswer) and the second is + // not: nothing in this package branches on cluster.ErrNoRoute, and + // `unroutable` treats ANY recorded dial error as unroutable, so such a + // refusal reaches every reap guard as ProbeUnknown and deletes nothing. + // The decision stands on the grounds above, which do not depend on a + // classification the frontend does not currently make. if reply.WorkerLocalAddress == "" { return "", fmt.Errorf("worker %s reported backend %q installed but named no address for the process", node.ID, backendType) } diff --git a/core/services/nodes/router_unnamed_replica_test.go b/core/services/nodes/router_unnamed_replica_test.go new file mode 100644 index 000000000000..a6a01fda7526 --- /dev/null +++ b/core/services/nodes/router_unnamed_replica_test.go @@ -0,0 +1,97 @@ +package nodes + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// A replica row carries the address of the backend process it names, and that +// address is how the frontend says WHICH process on a worker it means. A row +// without one names nothing, and the warm path has to decline it rather than +// route with an empty target. +// +// This is defensive: installBackendOnNode now guarantees a non-empty address +// before any row is written, so the only rows that can look like this are ones +// an older frontend wrote. It is specced anyway because the branch touches an +// in-flight reservation, and a reservation that is taken and not returned pins +// the replica against every eviction query for the life of the row. +var _ = Describe("SmartRouter warm path with an unnamed replica", func() { + var ( + registry *fakeModelRouterForSmartRouter + clients *fakeBackendClientFactory + router *SmartRouter + node *BackendNode + ) + + BeforeEach(func() { + node = &BackendNode{ID: "node-1", Name: "node-1", Status: StatusHealthy} + registry = newFakeModelRouterForSmartRouter() + registry.node = node + clients = newFakeBackendClientFactory() + router = NewSmartRouter(registry, SmartRouterOptions{ClientFactory: clients}) + }) + + warm := func() *RouteResult { + return router.tryWarmPath(context.Background(), &routeAttempt{trackingKey: "m", modelName: "m"}) + } + + Context("when the row names no backend process", func() { + BeforeEach(func() { + registry.nodeModel = &NodeModel{NodeID: node.ID, ModelName: "m", ReplicaIndex: 0} + }) + + It("declines the warm path so the caller cold-loads", func() { + Expect(warm()).To(BeNil()) + }) + + It("never asks for a client, so the empty target reaches no dialler", func() { + // The failure this prevents: an empty target opens a stream the + // worker refuses as an invalid request. That refusal is an answer + // FROM the worker, so it is the one failure on the whole path that + // is real evidence about a backend, and this row is not entitled to + // produce evidence about anything. + Expect(warm()).To(BeNil()) + Expect(clients.nodesSeen()).To(BeEmpty()) + Expect(clients.addressesSeen()).To(BeEmpty()) + }) + + It("returns the reservation FindAndLockNodeWithModel took", func() { + // Held rather than returned, the row's in_flight never reaches 0 + // and no eviction query can ever select it, so the replica slot and + // its VRAM are pinned for the life of the row. + Expect(warm()).To(BeNil()) + registry.mu.Lock() + defer registry.mu.Unlock() + Expect(registry.decrementCalled).To(HaveKeyWithValue("node-1:m", 1)) + }) + + It("leaves the row in place", func() { + // Deliberately unlike the sibling !alive branch, which removes the + // row. A dead backend has been observed dead; this row has been + // observed to be unreadable, which says nothing about whether a + // process is running on that worker. It is also the last record + // that one may be: the acknowledged stop path matches on + // ExpectedAddress, so a stop for an empty one is refused by the + // worker, and deleting the row here would free the replica slot for + // a second copy of the same model while the first one, if it + // exists, keeps its VRAM. The cost of keeping it is one lock and + // decrement per request before the cold load, and a row an operator + // can see; the cost of removing it is an orphan nothing points at. + Expect(warm()).To(BeNil()) + Expect(registry.removedModels()).To(BeEmpty()) + }) + }) + + It("routes normally once the row names one", func() { + // The control. Without it every assertion above would also pass on a + // warm path that declined everything. + registry.nodeModel = &NodeModel{NodeID: node.ID, ModelName: "m", ReplicaIndex: 0, WorkerLocalAddress: "127.0.0.1:50052"} + Expect(warm()).ToNot(BeNil()) + Expect(clients.addressesSeen()).To(ContainElement("127.0.0.1:50052")) + registry.mu.Lock() + defer registry.mu.Unlock() + Expect(registry.decrementCalled).ToNot(HaveKey("node-1:m")) + }) +}) diff --git a/core/services/worker/addr_test.go b/core/services/worker/addr_test.go index 633d183b292d..447880653c83 100644 --- a/core/services/worker/addr_test.go +++ b/core/services/worker/addr_test.go @@ -90,3 +90,48 @@ var _ = Describe("Worker address resolution", func() { }) }) }) + +var _ = Describe("Worker startup validation", func() { + // A Config as kong would hand it over with nothing unusual set: the tunnel + // on by its default, no auth enforcement. + newConfig := func() *Config { + return &Config{WorkerTunnel: true} + } + + It("accepts the default configuration", func() { + Expect(newConfig().validateStartup()).To(Succeed()) + }) + + It("refuses to start with the tunnel turned off", func() { + // Not a warning and not a degraded mode. A worker without its tunnel + // advertises nothing, binds only loopback, and has no frontend path + // that dials it, yet it would register, heartbeat and report healthy, + // so the scheduler would keep placing models on it and every one would + // fail. Refusing at boot is the only outcome that is visible. + cfg := newConfig() + cfg.WorkerTunnel = false + err := cfg.validateStartup() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("LOCALAI_WORKER_TUNNEL")) + Expect(err.Error()).To(ContainSubstring("nothing can reach it")) + }) + + It("refuses enforcement without a registration token", func() { + cfg := newConfig() + cfg.RegistrationRequireAuth = true + Expect(cfg.validateStartup()).To(MatchError(ContainSubstring("LOCALAI_REGISTRATION_TOKEN is empty"))) + }) + + It("refuses the umbrella switch without a registration token", func() { + cfg := newConfig() + cfg.DistributedRequireAuth = true + Expect(cfg.validateStartup()).To(MatchError(ContainSubstring("LOCALAI_REGISTRATION_TOKEN is empty"))) + }) + + It("accepts enforcement once a token is set", func() { + cfg := newConfig() + cfg.DistributedRequireAuth = true + cfg.RegistrationToken = "shared" + Expect(cfg.validateStartup()).To(Succeed()) + }) +}) diff --git a/core/services/worker/config.go b/core/services/worker/config.go index 29ea5fec36f5..8402a4ce19cc 100644 --- a/core/services/worker/config.go +++ b/core/services/worker/config.go @@ -1,5 +1,7 @@ package worker +import "fmt" + // Config is the configuration for the distributed agent worker. // // Field tags are kong/kong-env metadata read by core/cli/worker.go's WorkerCMD, @@ -66,9 +68,22 @@ type Config struct { HeartbeatInterval string `env:"LOCALAI_HEARTBEAT_INTERVAL" default:"10s" help:"Interval between heartbeats" group:"registration"` // WorkerTunnel holds one outbound multiplexed connection to the frontend // and serves the frontend's requests over it, so the worker needs no - // inbound port. Turning it off leaves the worker reachable only at the - // addresses it advertises, which is the pre-tunnel behaviour. - WorkerTunnel bool `env:"LOCALAI_WORKER_TUNNEL" default:"true" help:"Hold one outbound multiplexed tunnel to the frontend and serve its requests over it, so this worker needs no inbound port." group:"distributed"` + // inbound port. + // + // Turning it off is now a fatal misconfiguration and validateStartup + // refuses to boot on it, which is a behaviour change from when this flag + // had a working "off" position. It no longer has one: this worker + // advertises no address and binds only loopback, and no frontend path + // dials a worker's address, so a worker without its tunnel is reachable by + // nothing. Left running it would be the worst available failure shape, + // because it registers, heartbeats and reports healthy, so the scheduler + // keeps placing models on it and every one of them fails. + // + // The flag is kept rather than deleted so that an operator who set it, on + // the old promise that it fell back to the advertised address, is told + // exactly that the promise is gone instead of having their setting quietly + // ignored. + WorkerTunnel bool `env:"LOCALAI_WORKER_TUNNEL" default:"true" help:"Hold one outbound multiplexed tunnel to the frontend and serve its requests over it, so this worker needs no inbound port. Setting it false is refused: a worker has no other way to be reached." group:"distributed"` NodeLabels string `env:"LOCALAI_NODE_LABELS" help:"Comma-separated key=value labels for this node (e.g. tier=fast,gpu=a100)" group:"registration"` // MaxReplicasPerModel caps how many replicas of any one model can run on // this worker concurrently. Default 1 = historical single-replica @@ -112,3 +127,25 @@ func (c Config) NatsAuthRequired() bool { func (c Config) RegistrationAuthRequired() bool { return c.RegistrationRequireAuth || c.DistributedRequireAuth } + +// validateStartup reports a configuration this worker must refuse to boot on, +// as opposed to one it can degrade under. +// +// It runs before prefetch, registration and NATS, so a refusal happens while +// the worker is still invisible to the cluster. That ordering is the point of +// checking here at all: both conditions below produce a worker that would +// register, heartbeat and be scheduled onto, so discovering them later means +// discovering them as failed inferences on a node the frontend believes is +// healthy. +func (c Config) validateStartup() error { + // The file-transfer server fails open on an empty token (see + // nodes.checkBearerToken), so enforcement plus no token is a request to + // serve the models directory unauthenticated. + if c.RegistrationAuthRequired() && c.RegistrationToken == "" { + return fmt.Errorf("registration auth is required (LOCALAI_REGISTRATION_REQUIRE_AUTH or LOCALAI_DISTRIBUTED_REQUIRE_AUTH) but LOCALAI_REGISTRATION_TOKEN is empty: refusing to start an unauthenticated file-transfer server") + } + if !c.WorkerTunnel { + return fmt.Errorf("LOCALAI_WORKER_TUNNEL is false, but this worker advertises no address and binds only loopback, and no frontend path dials a worker's address: without its tunnel nothing can reach it. Remove the setting, or run the pre-tunnel release on both the worker and the frontend") + } + return nil +} diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index f6c0b33c99ec..dd8f974eb07c 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -31,12 +31,12 @@ import ( func Run(ctx *cliContext.Context, cfg *Config) error { xlog.Info("Starting worker", "basePort", cfg.effectiveBasePort()) - // Fail fast (before prefetch/registration/NATS) when enforcement is on but no - // registration token is set: the worker's HTTP file-transfer server fails - // open on an empty token (see nodes.checkBearerToken), so refuse to start - // rather than register and then die mid-boot. - if cfg.RegistrationAuthRequired() && cfg.RegistrationToken == "" { - return fmt.Errorf("registration auth is required (LOCALAI_REGISTRATION_REQUIRE_AUTH or LOCALAI_DISTRIBUTED_REQUIRE_AUTH) but LOCALAI_REGISTRATION_TOKEN is empty — refusing to start an unauthenticated file-transfer server") + // Fail fast, before prefetch, registration and NATS, on any configuration + // that would produce a worker the cluster believes in and cannot use. See + // validateStartup for what those are and why each is fatal rather than + // degraded. + if err := cfg.validateStartup(); err != nil { + return err } systemState, err := system.GetSystemState( @@ -227,26 +227,30 @@ func Run(ctx *cliContext.Context, cfg *Config) error { // frontend URL or this node's identity is unusable, and a worker that // silently ran without its tunnel would look healthy while being // unreachable to everything that dials through it. - if cfg.WorkerTunnel { - tunnel, terr := StartTunnel(shutdownCtx, TunnelConfig{ - FrontendURL: cfg.RegisterTo, - NodeID: nodeID, - Token: tunnelToken, - // Built by tunnelServices rather than inline, so the routing - // table, which is this feature's security boundary, is reachable - // from a spec without starting a worker. - Services: tunnelServices(cfg, httpAddr), - }) - if terr != nil { - nodes.ShutdownFileTransferServer(httpServer) - return fmt.Errorf("starting the worker tunnel: %w", terr) - } - defer func() { - if err := tunnel.Close(); err != nil { - xlog.Warn("Closing the worker tunnel failed", "error", err) - } - }() + // + // Unconditional: LOCALAI_WORKER_TUNNEL=false is refused by validateStartup + // before this point, so there is no configuration that reaches here without + // one. A guard here would be a branch nothing can take, which reads as a + // supported no-tunnel mode that does not exist. + tunnel, terr := StartTunnel(shutdownCtx, TunnelConfig{ + FrontendURL: cfg.RegisterTo, + NodeID: nodeID, + Token: tunnelToken, + // Built by tunnelServices rather than inline, so the routing + // table, which is this feature's security boundary, is reachable + // from a spec without starting a worker. + Services: tunnelServices(cfg, httpAddr), + }) + if terr != nil { + nodes.ShutdownFileTransferServer(httpServer) + return fmt.Errorf("starting the worker tunnel: %w", terr) } + defer func() { + if err := tunnel.Close(); err != nil { + xlog.Warn("Closing the worker tunnel failed", "error", err) + } + }() + // Connect to NATS xlog.Info("Connecting to NATS", "url", sanitize.URL(cfg.NatsURL)) natsClient, err := connectNats() diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 3727ab1ad02d..ad2b42652093 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -1428,9 +1428,9 @@ Notes: **Port conflicts on workers:** - Each model gets its own gRPC process on an incrementing port (50051, 50052, ...) - The HTTP file transfer server runs on the base port - 1 (default: 50050) -- Ensure the port range is not blocked by firewalls or used by other services +- All of those bind loopback, so a firewall cannot be the cause. What can is another service on the same host already holding a port in the range: move the worker's range with `LOCALAI_ADDR` (see [Worker Port Configuration](#worker-port-configuration)) or bound it with `LOCALAI_GRPC_MAX_PORT` - Verify the backend gallery configuration is correct -- The worker needs network access to download backends from the gallery +- The worker needs OUTBOUND network access to the gallery, to `LOCALAI_REGISTER_TO` and to `LOCALAI_NATS_URL`. It needs no inbound access at all ## Routing pipeline diff --git a/pkg/mcp/localaitools/dto.go b/pkg/mcp/localaitools/dto.go index 24da8aa18d23..0603f951fb26 100644 --- a/pkg/mcp/localaitools/dto.go +++ b/pkg/mcp/localaitools/dto.go @@ -97,13 +97,17 @@ type SystemInfo struct { } // Node is one entry in list_nodes. +// +// It carries no address. A worker holds one outbound tunnel to a frontend +// replica and advertises no endpoint, so both `address` and `http_address` are +// empty on every node registered by a current worker. They were dropped rather +// than left empty because this struct is read by the LocalAI Assistant, and an +// always-blank field an operator can ask about invites an answer built on it. type Node struct { - ID string `json:"id"` - Address string `json:"address,omitempty"` - HTTPAddress string `json:"http_address,omitempty"` - TotalVRAM uint64 `json:"total_vram,omitempty"` - Healthy bool `json:"healthy"` - LastSeen string `json:"last_seen,omitempty"` + ID string `json:"id"` + TotalVRAM uint64 `json:"total_vram,omitempty"` + Healthy bool `json:"healthy"` + LastSeen string `json:"last_seen,omitempty"` } // SetNodeVRAMBudgetRequest is the input for set_node_vram_budget. It PUTs diff --git a/pkg/mcp/localaitools/dto_test.go b/pkg/mcp/localaitools/dto_test.go index 865d00e8c3de..2d807d0b7ccc 100644 --- a/pkg/mcp/localaitools/dto_test.go +++ b/pkg/mcp/localaitools/dto_test.go @@ -31,7 +31,7 @@ var _ = Describe("DTOs round-trip through JSON", func() { roundTripDTO(InstallBackendRequest{GalleryName: "g", BackendName: "b"}) roundTripDTO(Backend{Name: "n", Installed: true}) roundTripDTO(SystemInfo{Version: "v1", Distributed: false, ModelsPath: "/tmp", LoadedModels: []string{"a"}, InstalledBackends: []string{"x"}}) - roundTripDTO(Node{ID: "n", Address: "a", HTTPAddress: "h", TotalVRAM: 100, Healthy: true, LastSeen: "now"}) + roundTripDTO(Node{ID: "n", TotalVRAM: 100, Healthy: true, LastSeen: "now"}) roundTripDTO(VRAMEstimateRequest{ModelName: "m", ContextSize: 4096, GPULayers: -1, KVQuantBits: 8}) roundTripDTO(ImportModelURIRequest{URI: "u", BackendPreference: "llama-cpp", Overrides: map[string]any{"k": "v"}}) roundTripDTO(ImportModelURIResponse{JobID: "j", DiscoveredModelName: "m", AmbiguousBackend: true, Modality: "tts", BackendCandidates: []string{"a", "b"}, Hint: "h"}) diff --git a/pkg/mcp/localaitools/httpapi/client.go b/pkg/mcp/localaitools/httpapi/client.go index 923f35eba86c..791ddc1db859 100644 --- a/pkg/mcp/localaitools/httpapi/client.go +++ b/pkg/mcp/localaitools/httpapi/client.go @@ -458,11 +458,11 @@ func (c *Client) SystemInfo(ctx context.Context) (*localaitools.SystemInfo, erro } func (c *Client) ListNodes(ctx context.Context) ([]localaitools.Node, error) { + // address / http_address are deliberately not decoded: a worker advertises + // no endpoint, so both are empty on every current node. var raw []struct { - ID string `json:"id"` - Address string `json:"address"` - HTTPAddress string `json:"http_address"` - Status string `json:"status"` + ID string `json:"id"` + Status string `json:"status"` } if err := c.do(ctx, http.MethodGet, routeNodes, nil, &raw); err != nil { // Treat 404/disabled as "no nodes" to keep parity with single-process. @@ -474,10 +474,8 @@ func (c *Client) ListNodes(ctx context.Context) ([]localaitools.Node, error) { out := make([]localaitools.Node, 0, len(raw)) for _, n := range raw { out = append(out, localaitools.Node{ - ID: n.ID, - Address: n.Address, - HTTPAddress: n.HTTPAddress, - Healthy: n.Status == "healthy", + ID: n.ID, + Healthy: n.Status == "healthy", }) } return out, nil From d5089648a4702e37151461ccbe353732be9c9467 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 20:56:57 +0000 Subject: [PATCH 061/153] test(distributed): prove the worker tunnel end to end, under real inference Everything this phase built was proven by unit and integration specs. This is the first run of it against the real binaries: a frontend replica per process, a worker that binds nothing routable, real inference over the result. Four scenarios, each with the question "what would make this pass if the tunnel were doing nothing" answered rather than left open. A worker with no advertised address is reached through its tunnel. The roster is asserted to report it advertising nothing, so there is no address a frontend could have dialled instead, and node_connections is asserted to name the replica that serves the request. A request landing on the replica that does NOT own the worker is relayed to the one that does. With N replicas behind round robin that is (N-1)/N of production traffic, so it gets the FIRST request for its model: the backend install, the file staging on the http tag, and the gRPC load and predict all cross the relay. Which replica owns the tunnel is read from the ownership table through the production Owner query and mapped to a frontend index through the address the harness pins per replica; the non-owner is derived from that reading and asserted to be a non-owner immediately before the request, rather than assumed from the harness default. Sending the same request to the owner reddens it. Killing the owning replica re-homes the worker onto the survivor. The worker dials a balancer rather than a replica, because LOCALAI_REGISTER_TO is resolved once at boot and is the tunnel endpoint as well as the registration one: aimed at a single replica, a worker has nowhere to reconnect to when that replica dies, and the re-home cannot happen at all. Removing the kill reddens it. And the negative control for the whole suite, which is why the other three mean anything. Frontend and worker share a host here, so every backend port the frontend names in a stream target is one it could have dialled directly; if it did, the first three would pass with the tunnel inert. LOCALAI_WORKER_TUNNEL is no longer usable for this, because it is a fatal startup error and a worker that never started says nothing about a worker reachable some other way. The balancer answers the tunnel connect path itself instead, leaving a worker that registers, heartbeats, reports healthy and holds no tunnel. It is asserted to have dialled and been refused, asserted to be held by nobody, and then asserted unreachable with the refusal naming the missing route. Then the block is lifted, nothing else changes, and the same request succeeds: that is what attributes the refusal to the tunnel rather than to any of the ordinary reasons an e2e inference fails. The fifth spec measures the head-of-line blocking this phase deferred three times. 128 MiB crosses the session while a warm model is probed back to back, direct and relayed. Median latency is unchanged, the worst probe is about 3x the baseline median and about a seventeenth of the transfer window, and the transfer runs at 415-490 MB/s direct and 222-268 MB/s relayed. A session that head-of-line blocked would park a probe for the length of the window. Leave the yamux windows untuned; and note this is loopback, so it says the multiplexing does not serialise and says nothing about a link with a bandwidth-delay product. The load spec is measured against a control that the first version did not have. It passed with the bulk artifact cut to 4 KiB, because the window it read probes against was mostly cold-load overhead: it would have reported a clean bill on a session carrying no large message. The same cold load now runs twice, once empty and once bulk, and the difference between the windows is asserted to be real before any latency is read from it. Two defects on the base commit came out of this. cluster_peerlink_test.go has been red since the relay landed, deterministically, in isolation and in the suite. It asserted that an accepted peer stream is refused at once, on the premise that phase 1 installs no relay. The relay correctly waits fifteen seconds for a frame naming the worker, and the spec's budget was five. It now writes a relay request for a node no replica holds and asserts the refusal is ErrNotOwner and specifically not ErrNoConnection, which is a stronger spec than the one it replaces and the only thing in the e2e suite that exercises the relay's refusal path. The harness handed a worker's own HTTP port to a backend process. It took two ports from freeport and used one as the gRPC base and the other for the file transfer server; freeport returns adjacent ports often, and the backend allocator hands out base, base+1, base+2, so the second backend started on a worker was regularly given the HTTP server's port and died with EADDRINUSE. No spec had started two backends on one worker before, so it had never fired; the load spec starts five and it failed about one run in three. Each worker now reserves a contiguous bind-probed block laid out the way production lays it out, below the kernel's ephemeral range, with LOCALAI_GRPC_MAX_PORT bounding the allocator to it. The underlying production defect is not fixed here and is recorded in the report: allocatePort never checks that a port is free, and its default range overlaps the ephemeral range on every Linux box. Constraint 6, whether distributed mode should now refuse to start without an advertised address, is DEFERRED, and the comment and the docs that described the cost were understating it. A replica with no advertised address writes no instances row, and Owner joins a connection against a live instance, so a worker whose tunnel lands there is unroutable from every OTHER replica while being registered and healthy. Refusing to start would still be wrong, because the deployments it would break are single-host ones with no peers to be unreachable by, and telling those apart at startup is a design with its own specs. Both places now say what actually happens. Suite wall clock 592s for 15 specs, up from 502s for 10 of which 2 were red. The CI budget of 20 minutes does not move. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 20 +- docs/content/features/distributed-mode.md | 4 +- tests/e2e/distributed/cluster/cluster.go | 150 ++- .../e2e/distributed/cluster_baseline_test.go | 19 + .../e2e/distributed/cluster_peerlink_test.go | 37 +- tests/e2e/distributed/cluster_tunnel_test.go | 858 ++++++++++++++++++ 6 files changed, 1067 insertions(+), 21 deletions(-) create mode 100644 tests/e2e/distributed/cluster_tunnel_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index fb608a203b67..244f6eb680ad 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -228,11 +228,23 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade clusterRegistry := cluster.NewRegistry(authDB) var membership *cluster.Membership if advertised, err := advertisedPeerAddr(cfg); err != nil { - // Not fatal. A replica that cannot publish an address still serves - // every request that reaches it directly; what it cannot do is have - // another replica relay to it. Failing startup here would take out + // Not fatal, and the cost is worth stating exactly rather than as + // "peers cannot reach it", because it is larger than that now. + // + // Without a row in the instances table this replica is not a live + // owner as far as Registry.Owner is concerned: that read joins a + // connection against a live instance, so a worker whose tunnel lands + // HERE is answered as unroutable at every OTHER replica, for as long + // as it stays here. This replica serves that worker perfectly well + // itself; nobody else can. On N replicas behind round robin that is + // (N-1)/N of the traffic for that worker. + // + // It stays a warning all the same. Refusing to start would take out // every existing single-host deployment, whose route to a local - // database is loopback. + // database is loopback and which has no peers to be unreachable by; + // the deployments this hurts are multi-replica ones, and telling those + // two apart at startup is a change with its own design and its own + // specs rather than a line here. xlog.Warn("This replica will not be reachable by its peers: no advertised address", "error", err, "knob", "LOCALAI_DISTRIBUTED_ADVERTISE_ADDR") } else { diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index ad2b42652093..acca168eefae 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -93,7 +93,9 @@ That only holds while the database is on **another host**. If PostgreSQL runs on This replica will not be reachable by its peers: no advertised address ``` -The replica keeps serving every request that reaches it directly; what it cannot do is have another replica reach it. Set the address explicitly to fix it: +The replica keeps serving every request that reaches it directly. What it cannot do is be reached by another replica, and on a multi-replica deployment that is worse than it sounds: a **worker whose tunnel lands on this replica is unroutable from every other replica**, because the ownership lookup only accepts an owner that is registered and live. This replica serves that worker fine; the others answer requests for it with `no route from this replica to that worker`. Behind a round-robin load balancer with N replicas, that is (N-1)/N of the traffic for that worker. + +A single-replica deployment is unaffected: it has no peers, and it holds every tunnel itself. Set the address explicitly to fix a multi-replica one: ```yaml environment: diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 0ea29ae65ecd..38012204f1ad 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -9,6 +9,8 @@ package cluster import ( "fmt" + "math/rand/v2" + "net" "net/http" "os" "os/exec" @@ -51,6 +53,36 @@ type Options struct { // writing to one roster concurrently and cannot express that at all while // every worker registers through the same process. SpreadWorkerRegistrations bool + + // Models is written into every frontend's models directory before that + // replica starts, keyed by file name. It is how a spec gets a model + // configuration in front of the frontend at all: the models directory is + // scanned at startup, so a file written afterwards is not guaranteed to be + // seen, and there is no admin endpoint that creates a config. + // + // Frontends only. A worker is handed model artifacts by the frontend's file + // staging, over the tunnel, and pre-seeding the worker would hide whether + // that worked. + Models map[string]string + + // WorkerFrontendURL rewrites the URL worker i registers and holds its + // tunnel against. It is called once per worker, after every frontend is + // serving, and is given the URL the worker would otherwise have been handed + // plus every frontend's URL in index order, so a hook can put a proxy or a + // load balancer in front of one replica or of all of them. + // + // It exists for two things the fixed per-replica URL cannot express. One is + // a worker that survives its replica: LOCALAI_REGISTER_TO is resolved once + // at boot and is the tunnel endpoint as well as the registration one, so a + // worker pointed straight at a replica has nowhere to reconnect to when + // that replica dies, and the re-home this feature is built on cannot + // happen. The other is the suite's negative control, which needs a worker + // that registers, heartbeats and reports healthy exactly as usual while its + // tunnel dial never reaches a frontend; nothing else can produce that, + // because LOCALAI_WORKER_TUNNEL=false is refused at startup and a worker + // that never started proves nothing about a worker reachable some other + // way. + WorkerFrontendURL func(worker int, registrar string, frontends []string) string } // Process is one running local-ai. @@ -184,6 +216,12 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { if err := os.MkdirAll(dataPath, 0o750); err != nil { return nil, fmt.Errorf("creating %s dirs: %w", name, err) } + for file, content := range c.opts.Models { + path := filepath.Join(dir, "models", file) + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + return nil, fmt.Errorf("writing %s for %s: %w", file, name, err) + } + } cmd := exec.Command(c.opts.Binary, "run", "--address", fmt.Sprintf("127.0.0.1:%d", port), @@ -235,14 +273,14 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { } func (c *Cluster) startWorker(i int) (*Process, error) { - // Two independent free ports: the worker's file-transfer server defaults to - // basePort-1, which freeport never reserved and which is basePort of another - // worker whenever two allocations land adjacent. - ports, err := freeport.GetFreePorts(2) + // One contiguous block, laid out the way production lays it out. See + // reserveWorkerPorts. + grpcPort, err := reserveWorkerPorts() if err != nil { return nil, fmt.Errorf("allocating worker ports: %w", err) } - grpcPort, httpPort := ports[0], ports[1] + httpPort := grpcPort - 1 + maxPort := grpcPort + workerPortBlockSize - 1 name := fmt.Sprintf("worker-%d", i) dir := filepath.Join(c.baseDir, name) backends := filepath.Join(dir, "backends") @@ -264,11 +302,18 @@ func (c *Cluster) startWorker(i int) (*Process, error) { ) cmd.Env = append(cmd.Environ(), // Ports only. A worker advertises nothing, so there is no advertise - // address to set; these two exist to keep concurrently running workers - // off each other's ports, not to make anything reachable. Both binds - // are loopback whatever is set here. + // address to set; these exist to keep concurrently running workers off + // each other's ports, not to make anything reachable. Every bind is + // loopback whatever is set here. + // + // The max port is what keeps the backend allocator inside the block + // reserved for this worker. Without it the allocator walks upward to + // 65535 (core/services/worker/registration.go, effectiveMaxPort), so a + // worker running enough backends walks straight out of its block and + // into whatever else this host is using. fmt.Sprintf("LOCALAI_SERVE_ADDR=127.0.0.1:%d", grpcPort), fmt.Sprintf("LOCALAI_HTTP_ADDR=127.0.0.1:%d", httpPort), + fmt.Sprintf("LOCALAI_GRPC_MAX_PORT=%d", maxPort), // Workers register with frontend 0 ONLY unless the caller opts into // SpreadWorkerRegistrations, and the cross-replica session specs depend // on that default. They prove a session minted at frontend 0 resolves at @@ -287,7 +332,7 @@ func (c *Cluster) startWorker(i int) (*Process, error) { // rest of its life: the loop posts to the URL it was given at boot and // never re-resolves it (core/cli/workerregistry/client.go), so killing a // worker's registrar orphans that worker rather than failing it over. - "LOCALAI_REGISTER_TO="+c.FrontendURL(c.registrarFor(i)), + "LOCALAI_REGISTER_TO="+c.workerFrontendURL(i), "LOCALAI_NODE_NAME="+name, "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, "LOCALAI_NATS_URL="+c.opts.NatsURL, @@ -297,6 +342,93 @@ func (c *Cluster) startWorker(i int) (*Process, error) { return c.spawn(name, cmd, grpcPort) } +const ( + // workerPortBlockSize is how many ports one worker reserves: one for its + // HTTP file-transfer server and the rest for backend processes. A spec that + // loads more models than this on one worker exhausts the allocator, which + // fails the backend start by name (ErrNoFreePort) instead of colliding. + workerPortBlockSize = 24 + + // Workers take their ports from BELOW the ephemeral range, which on Linux + // starts at 32768 by default. That is not tidiness. Ports the kernel hands + // out for outbound connections are exactly the ports a long-lived process + // full of outbound connections is liable to be holding when a backend tries + // to bind one, and this suite's workers hold a tunnel, a NATS connection + // and a registration client each. + workerPortFloor = 20000 + workerPortCeiling = 31000 + + // workerPortAttempts bounds the search for a free block before giving up. + workerPortAttempts = 200 +) + +// reserveWorkerPorts returns the base gRPC port of a contiguous run of ports +// nothing is currently listening on, laid out the way a real worker lays them +// out: the HTTP file-transfer server at base-1, and backend processes upward +// from base. +// +// A contiguous block rather than two independent freeport allocations, and the +// difference is a defect this suite actually hit. The allocator hands backend +// processes basePort, basePort+1, basePort+2 and so on with no check that +// anything else holds them (core/services/worker/supervisor.go, allocatePort), +// so the moment freeport returned two ADJACENT ports the second backend started +// on a worker was handed the worker's own HTTP server's port and died at +// startup with "address already in use". freeport returns adjacent ports often, +// and no spec started two backends on one worker until the tunnel load +// measurement did, so it presented as a spec that failed about one run in +// three. +// +// This is not race free and cannot be: the probe closes each listener before +// the worker binds it. It removes the deterministic self-collision, keeps the +// block out of the range the kernel allocates from, and bounds the allocator to +// the block, which together is the difference between "sometimes" and "not +// observed". +func reserveWorkerPorts() (int, error) { + for attempt := 0; attempt < workerPortAttempts; attempt++ { + base := workerPortFloor + rand.IntN(workerPortCeiling-workerPortFloor) + if blockIsFree(base-1, workerPortBlockSize+1) { + return base, nil + } + } + return 0, fmt.Errorf("no free run of %d ports in [%d, %d) after %d attempts", + workerPortBlockSize+1, workerPortFloor, workerPortCeiling, workerPortAttempts) +} + +// blockIsFree reports whether count ports from first can all be bound on +// loopback right now. Every listener is held until the whole run is proven, so +// a run is not accepted on the strength of one port that a previous iteration +// of this same loop had just released. +func blockIsFree(first, count int) bool { + held := make([]net.Listener, 0, count) + defer func() { + for _, l := range held { + _ = l.Close() + } + }() + for port := first; port < first+count; port++ { + l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + return false + } + held = append(held, l) + } + return true +} + +// workerFrontendURL is the URL worker i is told to register and tunnel +// against: its registrar's, unless the caller installed a hook. +func (c *Cluster) workerFrontendURL(i int) string { + registrar := c.FrontendURL(c.registrarFor(i)) + if c.opts.WorkerFrontendURL == nil { + return registrar + } + frontends := make([]string, 0, len(c.frontends)) + for index := range c.frontends { + frontends = append(frontends, c.FrontendURL(index)) + } + return c.opts.WorkerFrontendURL(i, registrar, frontends) +} + func (c *Cluster) spawn(name string, cmd *exec.Cmd, port int) (*Process, error) { logPath := filepath.Join(c.opts.LogDir, name+".log") // Append rather than truncate: a restarted process reopens the same path, and diff --git a/tests/e2e/distributed/cluster_baseline_test.go b/tests/e2e/distributed/cluster_baseline_test.go index 63995b419156..d23ba8f6e9a9 100644 --- a/tests/e2e/distributed/cluster_baseline_test.go +++ b/tests/e2e/distributed/cluster_baseline_test.go @@ -37,6 +37,12 @@ type node struct { ID string `json:"id"` Name string `json:"name"` Status string `json:"status"` + // Address and HTTPAddress are what a PRE-TUNNEL worker advertised. A worker + // running this release sends neither, which is the fact the tunnel specs + // assert on: with nothing advertised there is no address a frontend could + // have dialled instead of the tunnel. + Address string `json:"address"` + HTTPAddress string `json:"http_address"` } // requireBinaries reports whether a missing binary must fail the spec instead of @@ -224,6 +230,19 @@ func (p *rosterProbe) idOf(name string) string { return "" } +// advertisementOf returns whatever endpoints the roster last reported a node +// advertising, joined for a failure message. Empty means the node published +// none, which is what a worker on this release does. +func (p *rosterProbe) advertisementOf(name string) string { + for _, n := range p.lastSeen { + if n.Name != name { + continue + } + return strings.TrimSpace(strings.Join([]string{n.Address, n.HTTPAddress}, " ")) + } + return "" +} + // describe is handed to Should as the failure message. Gomega calls a // func() string description lazily, so this runs only on failure and reports // whichever of the two distinct causes actually occurred. diff --git a/tests/e2e/distributed/cluster_peerlink_test.go b/tests/e2e/distributed/cluster_peerlink_test.go index d99fcf70cfd8..7c492c16b9e7 100644 --- a/tests/e2e/distributed/cluster_peerlink_test.go +++ b/tests/e2e/distributed/cluster_peerlink_test.go @@ -47,6 +47,11 @@ const ( // is short on purpose: the refusal is one frame from a replica that has // already decided, so a stream still open at this point is parked. peerRefusalTimeout = 5 * time.Second + + // unheldNodeID is a worker id no replica holds a tunnel for. It is a + // well-formed id rather than a nonsense string so the refusal it draws is + // the routing answer and not a parse failure. + unheldNodeID = "00000000-0000-0000-0000-00000000dead" ) // openClusterDB connects to the database the cluster was given, so a spec can @@ -180,16 +185,34 @@ var _ = Describe("Cluster peer link", Label("Distributed"), Label("Cluster"), fu Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = stream.Close() }) - // Phase 1 installs no relay, so the accepted stream must be refused at - // once: an ENDING (EOF from the peer's Close, or a reset), not merely - // an error. A replica that accepted the stream and then left it parked - // would fail this read too, but with yamux's ErrTimeout, and that is - // the failure a live cluster experiences as every relayed request - // hanging until its own deadline. + // Phase 2 installs the relay on this link, so an accepted stream is one + // the peer is waiting to be told which worker it is for. Name one no + // replica holds and the refusal must come back at once. + // + // This spec used to assert the opposite, that an accepted stream ended + // immediately, because phase 1 had no relay to hand it to. The relay + // made that stale rather than wrong: a stream that says nothing now + // parks for relayHeaderTimeout, which is 15 seconds, and the old + // assertion failed on a five second budget against a replica behaving + // exactly as designed. + Expect(stream.SetWriteDeadline(time.Now().Add(peerRefusalTimeout))).To(Succeed()) + Expect(clustersvc.WriteRelayRequest(stream, unheldNodeID, peerRefusalTimeout)).To(Succeed()) + + Expect(stream.SetReadDeadline(time.Now().Add(peerRefusalTimeout))).To(Succeed()) + err = clustersvc.ReadRelayReply(stream) + Expect(err).To(MatchError(clustersvc.ErrNotOwner), + "the peer did not refuse a worker it does not hold: %v", err) + Expect(err).ToNot(MatchError(clustersvc.ErrNoConnection), + "a replica that does not hold a tunnel must not report the worker as absent: that is how a scheduler evicts a healthy worker") + + // And the refusal ENDS the stream. A replica that says why and leaves + // the stream open has parked the caller on a request that will never be + // served, which reads as a slow replica rather than a refused request, + // and no deadline on the far side can tell those apart. Expect(stream.SetReadDeadline(time.Now().Add(peerRefusalTimeout))).To(Succeed()) _, err = stream.Read(make([]byte, 1)) Expect(err).To(SatisfyAny(MatchError(io.EOF), MatchError(yamux.ErrStreamReset)), - "the peer accepted the stream and then neither answered nor ended it: %v", err) + "the peer refused the stream and then left it open: %v", err) // The same dial with the wrong credentials must be refused, otherwise // the success above says nothing about authentication. diff --git a/tests/e2e/distributed/cluster_tunnel_test.go b/tests/e2e/distributed/cluster_tunnel_test.go new file mode 100644 index 000000000000..13255576c82f --- /dev/null +++ b/tests/e2e/distributed/cluster_tunnel_test.go @@ -0,0 +1,858 @@ +package distributed_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + clustersvc "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/tests/e2e/distributed/cluster" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +// End-to-end proof that a worker with no inbound port is reachable, and only +// through its tunnel. +// +// Every other spec for this feature drives the tunnel, the relay and the +// ownership fence in isolation. These run the real binaries: a frontend replica +// per process, a worker that binds nothing routable, and a real inference over +// the result. +// +// Read the fourth scenario before trusting the first three. Frontend and worker +// are the same host here, so the backend port the frontend names in a stream +// target is a port the frontend could also have dialled directly; if it did, +// the first three would pass with the tunnel doing nothing at all. The fourth +// is what rules that out, and it is why the others mean anything. +// +// A fifth spec, in its own container below, measures what one session does when +// a large message and ordinary inference share it. + +const ( + // tunnelInferenceTimeout bounds one chat completion that has to install a + // backend on a worker, stage the model file over the tunnel and load it. + // Generous because the first request to a model pays for all of that. + tunnelInferenceTimeout = 3 * time.Minute + + // tunnelOwnershipTimeout bounds the wait for a worker's tunnel to be + // claimed, or re-claimed after its owner died. A re-claim waits for the + // worker's own reconnect backoff, which is capped at tunnelBackoffMax + // (30s), plus the dial and the claim. + tunnelOwnershipTimeout = 90 * time.Second + tunnelOwnershipPoll = 500 * time.Millisecond + + // tunnelRefusalTimeout bounds a request to a worker that has no tunnel, and + // is what tells a REFUSED request from a PARKED one: resolving the route + // fails on a table read, so a request that has not come back by now is not + // slow, it is waiting on something that will never happen. + // + // It says nothing about the refusal being the RIGHT one. A 503 saying the + // model is still loading would come back inside it too; what rules that out + // is the assertion on what the body says. + tunnelRefusalTimeout = 60 * time.Second + + // mockedReply is what the mock backend answers a prompt carrying no + // directive. Asserted rather than merely "some content", so a frontend that + // answered from a cache, an error template or a local backend of its own + // cannot satisfy these specs. + mockedReply = "This is a mocked response." + + // tunnelRestoredTimeout bounds the wait for inference to work again after a + // route came back. It has to clear loadJobFailureGrace, which replays a + // failed cold load's error to every caller for 15 seconds. + tunnelRestoredTimeout = 90 * time.Second + tunnelRestoredPoll = 2 * time.Second +) + +// mockModelYAML is a model configuration served by the mock backend. The +// artifact is a real file so the frontend's file staging has something to send +// over the tunnel, which is the http-tagged half of this feature. +func mockModelYAML(name string) string { + return fmt.Sprintf("name: %s\nbackend: mock-backend\nparameters:\n model: %s.bin\n", name, name) +} + +// chatResult is one completion attempt: what the frontend answered and how long +// it took. Both halves are used, the status by the correctness specs and the +// duration by the head-of-line measurement. +type chatResult struct { + status int + body string + content string + elapsed time.Duration +} + +// chat posts one non-streaming completion and reports what came back. It never +// fails the spec itself: a refusal is the expected answer in two of these +// specs, so the caller decides what the status means. +func chat(client *http.Client, baseURL, model, prompt string) (chatResult, error) { + body, err := json.Marshal(map[string]any{ + "model": model, + "messages": []map[string]string{{"role": "user", "content": prompt}}, + }) + if err != nil { + return chatResult{}, err + } + started := time.Now() + resp, err := client.Post(baseURL+"/v1/chat/completions", "application/json", bytes.NewReader(body)) + if err != nil { + return chatResult{elapsed: time.Since(started)}, err + } + defer func() { _ = resp.Body.Close() }() + raw, err := io.ReadAll(resp.Body) + if err != nil { + return chatResult{status: resp.StatusCode, elapsed: time.Since(started)}, err + } + result := chatResult{status: resp.StatusCode, body: string(raw), elapsed: time.Since(started)} + + var parsed struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + } + if json.Unmarshal(raw, &parsed) == nil && len(parsed.Choices) > 0 { + result.content = parsed.Choices[0].Message.Content + } + return result, nil +} + +// eventuallyMockedInference retries one completion until the worker answers. +// +// It exists for the two specs that restore a route and then assert it works +// again. A failed cold load is REPLAYED to every caller for loadJobFailureGrace +// (15s, core/services/nodes/model_load_job.go) so that a failure does not turn +// into a retry storm, which means the first request after a route comes back +// gets the stale reason rather than a fresh attempt. Retrying against the real +// condition is what a client does, and it keeps the spec off a sleep. +func eventuallyMockedInference(client *http.Client, baseURL, model, why string) { + GinkgoHelper() + last := "" + Eventually(func() string { + result, err := chat(client, baseURL, model, "ping") + if err != nil { + last = err.Error() + return "" + } + last = fmt.Sprintf("status %d: %s", result.status, result.body) + if result.status != http.StatusOK { + return "" + } + return result.content + }, tunnelRestoredTimeout, tunnelRestoredPoll).Should(Equal(mockedReply), + func() string { return why + ": the last attempt answered " + last }) +} + +// inferenceClient is ONE admin session for the whole cluster, with a budget +// long enough for a cold model load across a tunnel. +// +// One per spec, never one per frontend. The auth routes share a limiter of five +// requests per minute per client IP and every request here comes from +// 127.0.0.1, so a helper that minted a session per frontend would spend that +// budget and start failing setup in specs that touch three replicas. The client +// is good at every replica anyway: sessions live in the shared Postgres, the +// harness pins one HMAC secret across replicas, and Go's cookie jar keys by +// host without the port. It is also safe to use from several goroutines, which +// the load measurement needs. +func inferenceClient(c *cluster.Cluster) *http.Client { + GinkgoHelper() + client, err := c.AdminSession(0) + Expect(err).ToNot(HaveOccurred()) + client.Timeout = tunnelInferenceTimeout + return client +} + +// expectMockedInference runs one completion and requires the worker's own +// answer. It is the assertion every positive scenario ends on. +func expectMockedInference(client *http.Client, baseURL, model, why string) chatResult { + GinkgoHelper() + result, err := chat(client, baseURL, model, "ping") + Expect(err).ToNot(HaveOccurred(), why) + Expect(result.status).To(Equal(http.StatusOK), "%s: %s", why, result.body) + Expect(result.content).To(Equal(mockedReply), + "%s: the frontend answered 200 but not with the worker's reply: %s", why, result.body) + return result +} + +// tunnelOwners reads which replica holds which worker's tunnel. +// +// It reads the node_connections table through the production Owner query, which +// joins against live instances, so a row left behind by a dead replica is not +// reported as an owner. Nothing serves this over HTTP; it is replica-to-replica +// state. +type tunnelOwners struct { + registry *clustersvc.Registry + roster *instanceRoster + ctx context.Context + + lastErr error +} + +func newTunnelOwners(db *gorm.DB) *tunnelOwners { + return &tunnelOwners{ + registry: clustersvc.NewRegistry(db), + roster: newInstanceRoster(db), + ctx: context.Background(), + } +} + +// ownerOf returns the instance ID of the live replica holding nodeID's tunnel, +// or "" when there is none. Errors are kept rather than raised so an Eventually +// can name the last one. +func (o *tunnelOwners) ownerOf(nodeID string) string { + owner, _, err := o.registry.Owner(o.ctx, nodeID) + if err != nil { + o.lastErr = err + return "" + } + o.lastErr = nil + return owner +} + +// ownerIndexOf is ownerOf expressed as a frontend index of c, or -1 when no +// live replica holds the tunnel. +// +// The mapping goes through the advertised address, which the harness pins to +// each replica's own loopback port, so it is exact rather than a guess. A +// spec asserting "this request went to the replica that does not own the +// worker" needs the index and not the opaque instance ID. +func (o *tunnelOwners) ownerIndexOf(c *cluster.Cluster, frontends int, nodeID string) int { + owner := o.ownerOf(nodeID) + if owner == "" { + return -1 + } + // Refreshes o.roster.lastSaw, which idAt reads. + o.roster.addresses() + for i := 0; i < frontends; i++ { + if o.roster.idAt(hostPortOf(c.FrontendURL(i))) == owner { + return i + } + } + return -1 +} + +func (o *tunnelOwners) describe() string { + if o.lastErr != nil { + return fmt.Sprintf("the last read of the tunnel owner failed: %v", o.lastErr) + } + return fmt.Sprintf("live replicas: %s", o.roster.describe()) +} + +// frontendBalancer stands in for the load balancer a worker dials in +// production. +// +// It exists because LOCALAI_REGISTER_TO is BOTH the registration endpoint and +// the tunnel endpoint, and the worker resolves it once at boot and never again. +// Pointed straight at a replica, a worker whose replica dies can never come +// back, so the re-home this feature is built on cannot happen; and there is no +// other way to let a worker register normally while its tunnel dial fails, +// because LOCALAI_WORKER_TUNNEL=false is refused at startup. +// +// Two behaviours, both needed: +// +// - It forwards to the FIRST target that accepts a connection, which is what +// re-homes a worker onto the survivor after its replica is killed. +// - With blockTunnel set it answers the tunnel connect path itself, with the +// status a frontend that holds no tunnels gives, while still forwarding +// registration and heartbeats. That is the suite's negative control. +// +// tunnelDials counts what it saw on that path, so a spec can assert the worker +// really tried and really was refused rather than assuming it. +type frontendBalancer struct { + server *httptest.Server + targets []*url.URL + blockTunnel atomic.Bool + tunnelDials atomic.Int64 +} + +// balancerProbeTimeout bounds the liveness probe the director makes per +// request. Every target is a local process, so a refused connection comes back +// at once and this only bounds the pathological case. +const balancerProbeTimeout = 2 * time.Second + +// newFrontendBalancer starts a balancer in front of the given frontend URLs, in +// the order it should prefer them. +func newFrontendBalancer(urls ...string) *frontendBalancer { + GinkgoHelper() + b := &frontendBalancer{} + for _, raw := range urls { + parsed, err := url.Parse(raw) + Expect(err).ToNot(HaveOccurred()) + b.targets = append(b.targets, parsed) + } + + proxy := &httputil.ReverseProxy{ + Director: func(r *http.Request) { + target := b.pick() + r.URL.Scheme = target.Scheme + r.URL.Host = target.Host + r.Host = target.Host + }, + // A dead target is the ordinary case here, not an incident: the + // director picked it and it died between the probe and the dial. + ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) { + http.Error(w, fmt.Sprintf("balancer: no frontend answered: %v", err), http.StatusBadGateway) + }, + } + + b.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, clustersvc.ConnectPath) { + b.tunnelDials.Add(1) + if b.blockTunnel.Load() { + // The status a frontend not running in distributed mode gives. + // The worker retries it with backoff and stays otherwise + // healthy, which is precisely the state the negative control + // needs: registered, heartbeating, and holding no tunnel. + http.Error(w, "balancer: worker tunnels are blocked for this spec", http.StatusServiceUnavailable) + return + } + } + proxy.ServeHTTP(w, r) + })) + DeferCleanup(b.server.Close) + return b +} + +// URL is what a worker should be given as its frontend. +func (b *frontendBalancer) URL() string { return b.server.URL } + +// pick returns the first target that accepts a connection, falling back to the +// first so a request during a total outage fails at the proxy with a status +// rather than panicking in the director. +func (b *frontendBalancer) pick() *url.URL { + for _, target := range b.targets { + conn, err := net.DialTimeout("tcp", target.Host, balancerProbeTimeout) + if err == nil { + _ = conn.Close() + return target + } + } + return b.targets[0] +} + +var _ = Describe("Worker tunnel end to end", Label("Distributed"), Label("Cluster"), func() { + // Scenario 1. A wrong implementation reaches the worker some other way, or + // cannot reach it at all. The worker binds nothing routable and advertises + // nothing, so the assertion on its empty advertisement is what says there + // is nothing else the frontend could have been given. + It("reaches a worker that advertises no address, through its tunnel", func() { + c, dsn := startClusterOnFreshDB(1, 1, withMockModel("mock-model")) + client := inferenceClient(c) + + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), probe.describe) + nodeID := probe.idOf(c.WorkerName(0)) + Expect(nodeID).ToNot(BeEmpty()) + + // The worker publishes no endpoint of any kind. Without this the + // inference below would be satisfied by a frontend that dialled an + // advertised address, which is the path this phase removed. + advertised := probe.advertisementOf(c.WorkerName(0)) + Expect(advertised).To(BeEmpty(), + "the worker advertised %q, so this spec cannot tell a tunnelled request from a direct dial", advertised) + + // And its tunnel is held HERE, so the request below is served by the + // owner rather than relayed. The relay is the next spec's subject. + owners := newTunnelOwners(openClusterDB(dsn)) + Eventually(func() int { return owners.ownerIndexOf(c, 1, nodeID) }, tunnelOwnershipTimeout, tunnelOwnershipPoll). + Should(Equal(0), owners.describe) + + expectMockedInference(client, c.FrontendURL(0), "mock-model", + "a worker with no advertised address must still serve inference over its tunnel") + }) + + // Scenario 2. With N replicas behind round robin this is (N-1)/N of + // production traffic. A wrong implementation answers it by dialling the + // worker from the replica that took the request, which works on one host + // and nowhere else, or refuses it as a worker that is not connected. + It("serves a request that landed on the replica which does not own the worker", func() { + c, dsn := startClusterOnFreshDB(2, 1, withMockModel("relayed-model")) + client := inferenceClient(c) + + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), probe.describe) + nodeID := probe.idOf(c.WorkerName(0)) + Expect(nodeID).ToNot(BeEmpty()) + + // Which replica owns the tunnel is READ, not assumed. The harness sends + // the worker to frontend 0 by default, but that is a harness default + // and a spec that assumed it would keep passing after the default + // changed while silently testing the owner path instead. + owners := newTunnelOwners(openClusterDB(dsn)) + var owner int + Eventually(func() int { + owner = owners.ownerIndexOf(c, 2, nodeID) + return owner + }, tunnelOwnershipTimeout, tunnelOwnershipPoll).Should(BeNumerically(">=", 0), owners.describe) + + // The one replica that is not the owner. With two frontends there is + // exactly one, and it is derived from the reading above rather than + // written down, so this spec exercises the relay whichever replica the + // worker landed on. + nonOwner := 1 - owner + Expect(nonOwner).ToNot(Equal(owner)) + + // This is the whole point of the spec, so it is asserted rather than + // arranged: the request is about to go to a replica the database says + // does not hold this worker's tunnel. + Expect(owners.ownerIndexOf(c, 2, nodeID)).ToNot(Equal(nonOwner), + "frontend %d owns the worker's tunnel, so a request to it would not be relayed and this spec would prove nothing", nonOwner) + + // The FIRST request for this model goes to the non-owner, so the + // backend install, the model file staging over the http tag and the + // gRPC load and predict all cross the relay. Warming the model up at + // the owner first would leave only the predict on the relayed path. + expectMockedInference(client, c.FrontendURL(nonOwner), "relayed-model", + fmt.Sprintf("frontend %d must relay to frontend %d, which owns the worker's tunnel", nonOwner, owner)) + + // The relay did not move ownership. A replica that answered by taking + // the tunnel for itself would also have returned 200 above. + Expect(owners.ownerIndexOf(c, 2, nodeID)).To(Equal(owner), + "serving a relayed request moved the tunnel to frontend %d, which is not relaying", nonOwner) + }) + + // Scenario 3. Kills the replica holding the tunnel. The worker must land on + // the survivor and serve again. A wrong implementation leaves the dead + // replica's connection row in place, so the survivor relays into a corpse, + // or lets the re-claim be fenced out by its own stale epoch. + It("re-homes a worker onto the survivor when the replica holding its tunnel dies", func() { + // The worker dials a balancer rather than a replica: in production + // LOCALAI_REGISTER_TO is the load balancer, and a worker pointed at one + // replica has nowhere to reconnect to when that replica dies. + var balancer *frontendBalancer + c, dsn := startClusterOnFreshDB(2, 1, withMockModel("failover-model"), withBalancer(&balancer)) + + client := inferenceClient(c) + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), probe.describe) + nodeID := probe.idOf(c.WorkerName(0)) + Expect(nodeID).ToNot(BeEmpty()) + + owners := newTunnelOwners(openClusterDB(dsn)) + var owner int + Eventually(func() int { + owner = owners.ownerIndexOf(c, 2, nodeID) + return owner + }, tunnelOwnershipTimeout, tunnelOwnershipPoll).Should(BeNumerically(">=", 0), owners.describe) + + survivor := 1 - owner + expectMockedInference(client, c.FrontendURL(owner), "failover-model", + "inference must work before the owner is killed, or the recovery below proves nothing") + + Expect(c.KillFrontend(owner)).To(Succeed()) + Eventually(func() bool { return c.FrontendAlive(owner) }, "20s", "500ms").Should(BeFalse()) + + // The re-home is the assertion, not the inference. A frontend that + // answered without the tunnel moving would satisfy an inference-only + // spec while the worker stayed stranded on a dead replica. + Eventually(func() int { return owners.ownerIndexOf(c, 2, nodeID) }, tunnelOwnershipTimeout, tunnelOwnershipPoll). + Should(Equal(survivor), owners.describe) + + // Read at the SURVIVOR. The probe above is bound to the replica that was + // just killed, and a roster read against a dead process reports nothing + // rather than reporting a node that went away. + atSurvivor := newRosterProbe(c, client, survivor) + Eventually(atSurvivor.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), atSurvivor.describe) + + // Same worker process, not a replacement: the node ID is the identity + // registration minted, and a worker that had restarted and re-registered + // would be a different story with the same ending. + Expect(atSurvivor.idOf(c.WorkerName(0))).To(Equal(nodeID), + "the worker re-registered rather than re-homing, so this proves nothing about the tunnel moving") + + eventuallyMockedInference(client, c.FrontendURL(survivor), "failover-model", + "the survivor must serve the re-homed worker") + }) + + // Scenario 4. THE NEGATIVE CONTROL FOR THE WHOLE SUITE. + // + // Frontend and worker share a host here, so every backend port named in a + // stream target is one the frontend could dial directly. If it did, the + // three specs above would pass with the tunnel doing nothing. This one + // takes the tunnel away and requires the worker to become unreachable, + // while leaving registration, heartbeats and the roster untouched. + // + // It cannot use LOCALAI_WORKER_TUNNEL=false: that is refused at startup + // now, and a worker that never started says nothing about a worker that is + // reachable by some other path. The balancer refuses the tunnel dial + // instead, which leaves a worker that is registered, healthy and holding + // no tunnel. + It("cannot reach a worker whose tunnel is refused, and can as soon as it is not", func() { + var balancer *frontendBalancer + c, dsn := startClusterOnFreshDB(1, 1, withMockModel("controlled-model"), + withBalancer(&balancer, func(b *frontendBalancer) { b.blockTunnel.Store(true) })) + + client := inferenceClient(c) + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), probe.describe) + nodeID := probe.idOf(c.WorkerName(0)) + Expect(nodeID).ToNot(BeEmpty()) + + // The worker is in every respect the one the specs above used: same + // binary, same environment, registered and healthy. The only difference + // is the tunnel, and both halves of that are asserted rather than + // assumed: it tried, and no replica holds it. + Eventually(balancer.tunnelDials.Load, "60s", "500ms").Should(BeNumerically(">", 0), + "the worker never dialled its tunnel, so blocking the dial is not what makes it unreachable below") + owners := newTunnelOwners(openClusterDB(dsn)) + Consistently(func() string { return owners.ownerOf(nodeID) }, "5s", "500ms").Should(BeEmpty(), + "a replica holds this worker's tunnel, so the blocker is not blocking") + + client.Timeout = tunnelRefusalTimeout + refused, err := chat(client, c.FrontendURL(0), "controlled-model", "ping") + Expect(err).ToNot(HaveOccurred(), + "the request never came back; a worker with no route must be refused, not left parked") + Expect(refused.status).ToNot(Equal(http.StatusOK), + "the frontend served an inference for a worker that holds no tunnel, so something other than the tunnel reaches it: %s", refused.body) + + // And it fails for the RIGHT reason. A frontend that was refusing for + // any other cause (a missing model, a backend it could not install, an + // unhealthy node) would satisfy the assertion above just as well, and + // would leave the three specs before this one unproven. + Expect(refused.body).To(Or( + ContainSubstring("no route"), + ContainSubstring("not connected"), + ContainSubstring("unroutable"), + ContainSubstring("tunnel"), + ), "the refusal does not name the missing route: %s", refused.body) + + // The control's own control: put the tunnel back, change nothing else, + // and the same request must now succeed. Without this the refusal above + // could be any of the ordinary reasons an e2e inference fails. + balancer.blockTunnel.Store(false) + Eventually(func() int { return owners.ownerIndexOf(c, 1, nodeID) }, tunnelOwnershipTimeout, tunnelOwnershipPoll). + Should(Equal(0), owners.describe) + + client.Timeout = tunnelInferenceTimeout + eventuallyMockedInference(client, c.FrontendURL(0), "controlled-model", + "the only thing that changed is the tunnel, so the refusal above was the missing tunnel and nothing else") + }) +}) + +// withMockModel seeds one mock-backend model configuration and its artifact +// into every frontend. +func withMockModel(name string) func(*cluster.Options) { + return func(o *cluster.Options) { + if o.Models == nil { + o.Models = map[string]string{} + } + o.Models[name+".yaml"] = mockModelYAML(name) + o.Models[name+".bin"] = tinyArtifact() + } +} + +// withBalancer sends every worker through one balancer in front of all the +// frontends, and publishes it at into so the spec can drive it. +// +// The balancer is built inside the hook rather than beside the cluster because +// it needs the frontends' ports, and those exist only once Start has brought +// them up; the hook runs per worker, after that. arm runs on the balancer the +// moment it exists, which is before the worker process is spawned, so a spec +// that needs the tunnel blocked from the very first dial can say so without +// racing the worker's first attempt. +func withBalancer(into **frontendBalancer, arm ...func(*frontendBalancer)) func(*cluster.Options) { + return func(o *cluster.Options) { + o.WorkerFrontendURL = func(_ int, _ string, frontends []string) string { + if *into == nil { + *into = newFrontendBalancer(frontends...) + for _, apply := range arm { + apply(*into) + } + } + return (*into).URL() + } + } +} + +// percentile returns the p'th percentile of durations, which must be sorted. +func percentile(sorted []time.Duration, p float64) time.Duration { + if len(sorted) == 0 { + return 0 + } + idx := int(float64(len(sorted)-1) * p) + return sorted[idx] +} + +// summarise reports the shape of a latency sample. +func summarise(label string, samples []time.Duration) string { + if len(samples) == 0 { + return label + ": no samples" + } + sorted := append([]time.Duration(nil), samples...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + var total time.Duration + for _, d := range sorted { + total += d + } + return fmt.Sprintf("%s: n=%d mean=%s p50=%s p90=%s p99=%s max=%s", + label, len(sorted), total/time.Duration(len(sorted)), + percentile(sorted, 0.50), percentile(sorted, 0.90), percentile(sorted, 0.99), sorted[len(sorted)-1]) +} + +// The deferred question of this phase: what one yamux session does when a large +// message and ordinary inference share it, with the relay adding a second hop +// for most requests. +// +// It is MEASURED here rather than asserted to be fine. Both sides' yamux +// windows are left at their defaults by a deliberate decision, pending these +// numbers, and the numbers are printed as a report entry so a future change to +// those windows has something to be compared against. +const ( + // bulkArtifactSize is the model artifact staged over the tunnel while + // probes run. It stands in for the 50MB-class message this feature has to + // carry: real model files are far larger, and if a session cannot interleave + // at this size it certainly cannot at theirs. + // + // It is sized against the CONTROL below rather than for realism alone. A + // cold load costs a few hundred milliseconds before a byte moves (backend + // install, then the load itself), so at a smaller size the transfer is a + // minority of the window being measured and a spec could report a clean + // bill from a window that was mostly not a transfer. That is not + // hypothetical: the first version of this spec used 64MiB and still passed + // with the artifact cut to 4KiB, which is the definition of measuring + // nothing. + bulkArtifactSize = 128 << 20 + + // tinyArtifactSize is the same cold load with nothing to transfer. It is + // what the bulk window is measured AGAINST, so that the part of the window + // attributable to moving bytes is a number this spec holds rather than an + // assumption about the load path. + tinyArtifactSize = 4 << 10 + + // minTransferWindow is how much longer the bulk cold load must take than + // the tiny one before the numbers below mean anything. It is THE control on + // this measurement: without it a bulk artifact that shrank, or a staging + // path that stopped transferring, would leave the spec reporting that a + // large message does not block inference having sent no large message. + minTransferWindow = 100 * time.Millisecond + + // holProbeCount is how many completions the baseline is measured over. + holProbeCount = 40 + + // minOverlappingProbes is the measurement's own negative control. A bulk + // transfer that finishes before any probe ran would report "no head-of-line + // blocking" having measured nothing at all, which is the shape of vacuous + // result this phase keeps producing. Below this the spec fails rather than + // reporting. + minOverlappingProbes = 5 + + // holStallCeiling is the coarse absolute backstop, for the case where the + // bulk transfer is itself pathologically slow and the transfer-window + // comparison would admit a probe latency no deployment would tolerate. + // + // The comparison is the assertion that bites; this is only the floor under + // it. A session that head-of-line blocks parks a probe until the transfer + // releases the session, so the stalled probe's latency is on the order of + // the transfer window's; one that interleaves answers a warm mock + // completion in the milliseconds it takes with nothing else on the wire. + // An ABSOLUTE ceiling would have to sit between those two, and on this + // hardware they are 30ms and 250ms, which is too narrow a gap to hold on a + // loaded box. The comparison holds whatever the box's speed, because both + // numbers move with it. + holStallCeiling = 5 * time.Second +) + +// bulkArtifact is the large model artifact, built once. Both bulk models share +// the string: two copies of it would be two more allocations of +// bulkArtifactSize in the test process for no gain, since what is measured is +// the transfer and not the bytes. +var bulkArtifact = sync.OnceValue(func() string { + block := strings.Repeat("localai-tunnel-payload-", 45) + "\n" // ~1KiB + return strings.Repeat(block, bulkArtifactSize/len(block)+1)[:bulkArtifactSize] +}) + +// tinyArtifact is the artifact of a model that costs a cold load and no +// transfer. +func tinyArtifact() string { + return strings.Repeat("x", tinyArtifactSize) +} + +// probeOnce runs one completion against a warm model and reports how long it +// took, failing on anything but the worker's own answer: a probe that measured +// an error response would report a latency for work that never crossed the +// tunnel. +func probeOnce(client *http.Client, baseURL, model string) (time.Duration, error) { + result, err := chat(client, baseURL, model, "ping") + if err != nil { + return 0, err + } + if result.status != http.StatusOK { + return 0, fmt.Errorf("probe answered %d: %s", result.status, result.body) + } + if result.content != mockedReply { + return 0, fmt.Errorf("probe answered 200 but not with the worker's reply: %s", result.body) + } + return result.elapsed, nil +} + +// probeN runs n completions back to back. This is the baseline: one request in +// flight at a time, nothing else on the session. +func probeN(client *http.Client, baseURL, model string, n int) ([]time.Duration, error) { + samples := make([]time.Duration, 0, n) + for i := 0; i < n; i++ { + elapsed, err := probeOnce(client, baseURL, model) + if err != nil { + return samples, err + } + samples = append(samples, elapsed) + } + return samples, nil +} + +// probeUntil runs completions back to back until stop closes. Same shape as +// probeN, so the two samples differ only in what else was on the session. +func probeUntil(client *http.Client, baseURL, model string, stop <-chan struct{}) ([]time.Duration, error) { + var samples []time.Duration + for { + select { + case <-stop: + return samples, nil + default: + } + elapsed, err := probeOnce(client, baseURL, model) + if err != nil { + return samples, err + } + samples = append(samples, elapsed) + } +} + +var _ = Describe("Worker tunnel under load", Label("Distributed"), Label("Cluster"), func() { + It("interleaves inference with a bulk transfer on one session, direct and relayed", func() { + c, dsn := startClusterOnFreshDB(2, 1, + withMockModel("hol-probe"), + withMockModel("hol-tiny-direct"), + withMockModel("hol-tiny-relayed"), + withBulkModel("hol-bulk-direct"), + withBulkModel("hol-bulk-relayed")) + + client := inferenceClient(c) + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), probe.describe) + nodeID := probe.idOf(c.WorkerName(0)) + Expect(nodeID).ToNot(BeEmpty()) + + owners := newTunnelOwners(openClusterDB(dsn)) + var owner int + Eventually(func() int { + owner = owners.ownerIndexOf(c, 2, nodeID) + return owner + }, tunnelOwnershipTimeout, tunnelOwnershipPoll).Should(BeNumerically(">=", 0), owners.describe) + nonOwner := 1 - owner + + // Warm the probe model on the worker. Everything measured below is the + // warm path, so that a probe's latency is the session's and not a cold + // load's. + expectMockedInference(client, c.FrontendURL(owner), "hol-probe", + "the probe model must load before anything is measured") + + report := []string{} + + // coldLoadUnderProbes runs one cold load of model, keeps probing the + // warm model until it finishes, and reports both. + coldLoadUnderProbes := func(at, model string) (time.Duration, []time.Duration) { + GinkgoHelper() + done := make(chan struct{}) + var result chatResult + var loadErr error + started := time.Now() + go func() { + defer close(done) + result, loadErr = chat(client, at, model, "ping") + }() + + samples, err := probeUntil(client, at, "hol-probe", done) + elapsed := time.Since(started) + Expect(err).ToNot(HaveOccurred(), "a probe failed while %s was loading", model) + Expect(loadErr).ToNot(HaveOccurred()) + Expect(result.status).To(Equal(http.StatusOK), + "loading %s failed, so nothing measured beside it is a measurement of contention: %s", model, result.body) + return elapsed, samples + } + + measure := func(label string, through int, tinyModel, bulkModel string) { + at := c.FrontendURL(through) + + baseline, err := probeN(client, at, "hol-probe", holProbeCount) + Expect(err).ToNot(HaveOccurred()) + + // The same cold load twice: once with nothing to transfer, once + // with the bulk artifact. The difference between the two windows is + // the transfer, which is what this spec is about; everything else + // about the two loads is identical. + tinyElapsed, underTiny := coldLoadUnderProbes(at, tinyModel) + bulkElapsed, underBulk := coldLoadUnderProbes(at, bulkModel) + + transferWindow := bulkElapsed - tinyElapsed + Expect(transferWindow).To(BeNumerically(">=", minTransferWindow), + "%s: the %d MiB load took %s and the empty one took %s, so at most %s of the window was spent moving bytes; nothing below would be a measurement of a large message on the session", + label, bulkArtifactSize>>20, bulkElapsed, tinyElapsed, transferWindow) + + // The second control, on the sample rather than on the window. A + // transfer nothing ran beside would report a clean bill from a + // window in which no probe was measured. + Expect(len(underBulk)).To(BeNumerically(">=", minOverlappingProbes), + "%s: only %d probes overlapped a transfer window of %s, which is too few to say anything about head-of-line blocking", + label, len(underBulk), transferWindow) + + line := fmt.Sprintf("%s\n %s\n %s\n %s\n transfer window: %s of a %s load (%d MiB), empty load %s", + label, + summarise("baseline ", baseline), + summarise("under empty load ", underTiny), + summarise("under bulk load ", underBulk), + transferWindow.Round(time.Millisecond), bulkElapsed.Round(time.Millisecond), + bulkArtifactSize>>20, tinyElapsed.Round(time.Millisecond)) + report = append(report, line) + GinkgoWriter.Println(line) + + slowest := time.Duration(0) + for _, d := range underBulk { + if d > slowest { + slowest = d + } + } + Expect(slowest).To(BeNumerically("<", transferWindow), + "%s: a probe waited %s while %s of bytes were moving, which is the shape of a session that stalled the probe until the transfer let go, not of one that interleaved them", + label, slowest, transferWindow) + Expect(slowest).To(BeNumerically("<", holStallCeiling), + "%s: a probe waited %s while the bulk transfer held the session", label, slowest) + } + + measure("direct (owner replica holds the tunnel)", owner, "hol-tiny-direct", "hol-bulk-direct") + measure("relayed (through the replica that does not own the tunnel)", nonOwner, "hol-tiny-relayed", "hol-bulk-relayed") + + AddReportEntry("head-of-line blocking on one worker tunnel", strings.Join(report, "\n")) + }) +}) + +// withBulkModel seeds a model whose artifact is large enough to keep the tunnel +// busy while probes run. +func withBulkModel(name string) func(*cluster.Options) { + return func(o *cluster.Options) { + if o.Models == nil { + o.Models = map[string]string{} + } + o.Models[name+".yaml"] = mockModelYAML(name) + o.Models[name+".bin"] = bulkArtifact() + } +} From 5f5baf9df852b9744462c0fcb774e2472fbf3552 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 1 Sep 2026 23:30:10 +0000 Subject: [PATCH 062/153] fix(cluster): stop blaming a peer for the caller's own expired deadline Review round 1 on the end-to-end proof. Zero blocking items, eleven non-blocking, and three of them turned out to be production defects rather than notes on the report. The one that matters is a misclassification the phase is built to prevent. A dial carries the caller's deadline down to the socket, so when the budget runs out the socket's timer fires and the error travels back up through the WebSocket handshake and the multiplexer. The context's cancellation is a separate timer whose func the scheduler has to run before ctx.Err() stops returning nil, and nothing orders the two. Under contention the socket's error is back in PeerPool.Open first, ctx.Err() reads nil, and a peer that is listening and healthy is reported as ErrPeerUnreachable to a caller that simply ran out of time. An unreachable peer is a fact a caller may act on and an expired deadline is not, and core/services/nodes routes around a replica it is told is unreachable. callerRanOut answers that question in one place: ctx.Err() when it is set, and otherwise the wall clock against the caller's own deadline. That is sound because it is the same instant the socket compared itself against, so if the socket's timer fired this comparison is past it too. The ambiguous instant resolves towards the caller, which is the direction that never blames a peer. The spec that caught it, peerlink_test.go's "blames the caller's deadline", was red in three of seven -race runs and had been since Task 5, which is often enough to read as noise and is why single-run verification never saw it. Rather than leave the proof to a coin flip, a second spec makes the window deterministic: Open is handed a context whose deadline has passed and whose cancellation has not been delivered, against an address nothing is listening on, so the dial fails for real. It reddens without the fix. The peer link's yamux windows were applied to one end only. A receive window is advertised by the side that RECEIVES, so configuring the dialler alone tunes exactly one direction, and the direction left on the 256 KiB default is the one that carries a relayed model artifact INTO the replica that owns the worker's tunnel. That is the largest thing the link ever moves and it is the direction the load measurement exercises: the review read it as flowing toward the dialler and it does not. PeerLinkConfig is now exported and used on both ends. Measured, same box, 128 MiB staged through the relay against the same transfer without one: the relayed path cost 1.6x to 2.0x the direct path's transfer window before, and 1.06x to 1.25x after. The SSRF reachability spec could be fooled into reporting an SSRF that did not happen. It bound the victim on 127.0.0.2 at an ephemeral port and required 127.0.0.1 at the same port to refuse, so any other spec in the run holding that number made the dial succeed; red one run in seven, green five of five in isolation. It now picks from below the kernel's ephemeral range, the same fix the harness got for the adjacent-port collision. The rest are the specs and the report saying what they mean. Scenario 1's advertisement assertion could not tell "the worker advertises nothing" from "the JSON key moved", which matters because removing the advertisement is the change it covers. It was green against a renamed key. The roster now keeps the raw key set beside the decoded fields and the spec requires both keys present before reading them as empty. Scenario 4's refusal-body check was a four-way disjunction admitting bare "tunnel", "not connected" and "unroutable". Those alternatives were inert and each would be satisfied by refusals that say nothing about routing, in the one assertion the whole negative control rests on. It is "no route" alone. The head-of-line gate bounded the worst probe by the whole transfer window, which admits about eightfold degradation and loosens as the box slows. It is now half the window, plus a scale-free ratio against the worst probe under the SAME cold load with nothing to transfer, which is the control that isolates the transfer from the load. Not tighter than that, and the reason is measured rather than cautious: under a concurrent -race suite the worst relayed probe reached a fifth of its window, so a quarter-window gate would have had 1.2x of margin, and a spec that fails one run in three is worse than no spec. The report entry printed p90 and p99 off samples of twenty, where both land on the same element and p99 often lands on the max, so one number appeared three times under three names. A quantile is now printed only when the sample can separate it. Two claims in the report were wrong and are withdrawn rather than softened. Scenario 2's race is closed by the trailing re-read of the owner, not by the pre-assertion the report credited: a move to the non-owner mid-request would serve directly and still return 200, and only the trailing read reddens on it. And "the median request is unchanged" holds on this box and not on the reviewer's, where the relayed median rises up to 82% and p99 up to 3.5x. What survives on both is structural: the worst probe is a small fraction of the window in which bytes are moving, so the session interleaves rather than serialising. Sharing a session with a bulk transfer costs latency; it does not cost service. The disk footprint note undercounted, and the reviewer lost a run to a full disk on this box, so it is worth having right: two bulk models seeded into two frontends and staged to the worker is about 768 MiB, not 512 MiB. Left alone deliberately: the worker's backend port allocator still hands out ports without checking they are free, and its default range still overlaps the kernel's ephemeral range. It is confirmed, it is out of scope here, and it is being tracked as a named follow-up rather than fixed under an e2e task. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/cluster/peer.go | 18 +- core/services/cluster/peerlink.go | 63 ++++- core/services/cluster/peerlink_test.go | 41 ++++ core/services/worker/tunnel_test.go | 54 ++++- .../e2e/distributed/cluster_baseline_test.go | 39 +++- tests/e2e/distributed/cluster_tunnel_test.go | 219 +++++++++++++----- 6 files changed, 363 insertions(+), 71 deletions(-) diff --git a/core/http/endpoints/cluster/peer.go b/core/http/endpoints/cluster/peer.go index f0fb96ebc16e..00ef12c5b59d 100644 --- a/core/http/endpoints/cluster/peer.go +++ b/core/http/endpoints/cluster/peer.go @@ -52,7 +52,23 @@ func PeerHandler(token string, onSession func(peerID string, sess *yamux.Session // Server side of the mux: the dialing peer is the client, so it owns // the odd stream IDs and this side the even ones. - sess, err := yamux.Server(clustersvc.WebsocketConn(ws), nil, nil) + // + // The SAME configuration the dialler uses, and that is load bearing + // rather than symmetry for its own sake. A yamux receive window is + // advertised by the receiving side, so a nil here left this end on the + // 256 KiB default while the dialler ran at 4 MiB, and the direction + // governed by this end is the one that carries a relayed model artifact + // INTO the replica that owns the worker's tunnel. That direction was + // measured at roughly half the throughput of the same transfer without + // a relay in it. + // + // It also puts the same ceiling on unread data at this end that + // PeerLinkConfig already documents for the dialling end, so a replica + // is now sized against that figure per link in BOTH directions. That + // is the cost of the window being useful at all: a window is a bound + // on data received and not yet read, so a receiver that will not + // buffer cannot advertise one. + sess, err := yamux.Server(clustersvc.WebsocketConn(ws), clustersvc.PeerLinkConfig(), nil) if err != nil { xlog.Error("cluster peer link session setup failed", "peer", peerID, "error", err) _ = ws.Close() diff --git a/core/services/cluster/peerlink.go b/core/services/cluster/peerlink.go index 5f5e3186445b..fec713891802 100644 --- a/core/services/cluster/peerlink.go +++ b/core/services/cluster/peerlink.go @@ -103,8 +103,20 @@ const ( peerLinkMaxWindow = 32 * 1024 * 1024 ) -// peerLinkConfig returns the yamux configuration for a replica-to-replica link. -func peerLinkConfig() *yamux.Config { +// PeerLinkConfig returns the yamux configuration for a replica-to-replica link. +// +// Exported because BOTH ENDS need it and only one of them lives here. A yamux +// receive window is advertised by the side that RECEIVES, so a link configured +// on the dialler alone is tuned in exactly one direction: bytes travelling from +// the accepting replica back to the dialler get these windows, and bytes +// travelling the other way get yamux's defaults. The other way is the one that +// carries a relayed model artifact to the replica that owns the worker's +// tunnel, which is the largest thing this link ever moves. +// +// A fresh config per call, never a shared one: yamux keeps the pointer for the +// life of the session, and two sessions sharing one struct would share whatever +// a future field on it comes to mean. +func PeerLinkConfig() *yamux.Config { cfg := yamux.DefaultConfig() cfg.InitialStreamWindowSize = peerLinkInitialWindow cfg.MaxStreamWindowSize = peerLinkMaxWindow @@ -183,9 +195,9 @@ func (p *PeerPool) Open(ctx context.Context, peerID string) (net.Conn, error) { if err == nil { return st, nil } - // A caller whose own context expired must not cost every other worker + // A caller whose own budget expired must not cost every other worker // its link: the session is fine, this request is not. - if ctxErr := ctx.Err(); ctxErr != nil { + if ctxErr := callerRanOut(ctx); ctxErr != nil { return nil, ctxErr } // A session that died between calls is the common case, not an @@ -202,10 +214,10 @@ func (p *PeerPool) Open(ctx context.Context, peerID string) (net.Conn, error) { // the peer, which may be listening and perfectly healthy. Blaming it // would let one impatient client get a good replica routed around. // - // This also swallows a genuine ErrInstanceNotFound when the context + // This also swallows a genuine ErrInstanceNotFound when the budget // happened to expire at the same moment, which is the safe direction: // a timeout must never be able to manufacture absence. - if ctxErr := ctx.Err(); ctxErr != nil { + if ctxErr := callerRanOut(ctx); ctxErr != nil { return nil, ctxErr } return nil, err @@ -216,7 +228,7 @@ func (p *PeerPool) Open(ctx context.Context, peerID string) (net.Conn, error) { // The peer answered and completed a handshake but will not carry a // stream, which is a transport condition and never absence. _ = sess.Close() - if ctxErr := ctx.Err(); ctxErr != nil { + if ctxErr := callerRanOut(ctx); ctxErr != nil { return nil, ctxErr } return nil, unreachablePeer(peerID, err) @@ -226,6 +238,41 @@ func (p *PeerPool) Open(ctx context.Context, peerID string) (net.Conn, error) { return st, nil } +// callerRanOut reports whether the CALLER's budget is what ended an attempt, +// and is the one place that question is answered. +// +// ctx.Err() alone is not that question, and the difference is a real +// misclassification rather than a nicety. A dial carries the caller's deadline +// down to the socket, so when the budget runs out the socket's own timer fires +// and the error travels back up through the WebSocket handshake and the +// multiplexer. The context's cancellation is a SEPARATE timer whose func has to +// be run by the scheduler before ctx.Err() stops returning nil, and nothing +// orders the two. Under contention the socket's error can be back here first, +// ctx.Err() reads nil, and a peer that is listening and healthy is reported as +// ErrPeerUnreachable to a caller that simply ran out of time. +// +// That is the exact confusion this package refuses everywhere else: an +// unreachable peer is a fact about the peer that a caller may act on, and an +// expired deadline is a fact about the caller that it may not. The wall clock +// settles it without waiting for a goroutine, because the deadline is the same +// instant the socket compared itself against: if the socket's timer fired, this +// comparison is past it too. +// +// A context with no deadline falls through to ctx.Err(), which is the whole +// answer for cancellation: a Canceled context has already had its error set by +// the caller of cancel, with no timer in between. +func callerRanOut(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + // Not time.Now().After: a failure at exactly the deadline is the caller's + // too, and the ambiguous instant is resolved towards never blaming a peer. + if deadline, ok := ctx.Deadline(); ok && !time.Now().Before(deadline) { + return context.DeadlineExceeded + } + return nil +} + // link returns the per-peer entry, creating it on first use. // // Entries are never pruned: a peer id opened once keeps its entry, and any @@ -300,7 +347,7 @@ func (p *PeerPool) dial(ctx context.Context, peerID string) (*yamux.Session, err // Client side of the mux: the dialling replica owns the odd stream IDs, // matching the yamux.Server the peer handler puts on its end. - sess, err := yamux.Client(WebsocketConn(ws), peerLinkConfig(), nil) + sess, err := yamux.Client(WebsocketConn(ws), PeerLinkConfig(), nil) if err != nil { _ = ws.Close() return nil, unreachablePeer(peerID, err) diff --git a/core/services/cluster/peerlink_test.go b/core/services/cluster/peerlink_test.go index 692f2ee6c22a..d4406e5e167d 100644 --- a/core/services/cluster/peerlink_test.go +++ b/core/services/cluster/peerlink_test.go @@ -30,6 +30,17 @@ func servePeerRoute(e *echo.Echo, token string, onPeer func(string, *yamux.Sessi e.GET(cluster.PeerPath, clusterep.PeerHandler(token, onPeer)) } +// deadlinePassed is a context whose deadline has elapsed and whose +// cancellation has not been delivered, which is the state a caller is in for +// the moment between the two timers that fire at its deadline. Only Deadline is +// overridden: the embedded context supplies a nil Done and a nil Err, which is +// what a context in that window reports. +type deadlinePassed struct{ context.Context } + +func (deadlinePassed) Deadline() (time.Time, bool) { + return time.Now().Add(-time.Millisecond), true +} + var _ = Describe("Peer pool", func() { var ( db *gorm.DB @@ -261,6 +272,36 @@ var _ = Describe("Peer pool", func() { Expect(err).ToNot(MatchError(cluster.ErrInstanceNotFound)) }) + It("blames the caller's deadline even when its cancellation has not landed yet", func() { + // The same rule as the spec above, at the instant that makes it hard. + // + // A dial carries the caller's deadline down to the socket, so the + // socket's timer and the context's cancellation timer fire at the same + // moment and nothing orders them. The socket's error can be back in + // Open before the scheduler has run the context's cancel func, and in + // that window ctx.Err() is nil while the caller's budget is + // unambiguously spent. Reading only ctx.Err() there reports a peer that + // is listening and healthy as unreachable. + // + // That window is real: this spec's sibling above reproduces it under + // `-race` about three runs in seven, which is exactly often enough to + // be dismissed as noise. Here it is made deterministic instead, by + // handing Open a context in precisely that state: deadline passed, + // cancellation not delivered. Nothing is faked about the dial, which + // runs for real against an address nothing is listening on. + refused, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).ToNot(HaveOccurred()) + addr := refused.Addr().String() + Expect(refused.Close()).To(Succeed()) + Expect(reg.Register(ctx, "peer-refusing", addr, "test")).To(Succeed()) + + _, err = pool.Open(deadlinePassed{ctx}, "peer-refusing") + Expect(err).To(MatchError(context.DeadlineExceeded)) + Expect(err).ToNot(MatchError(cluster.ErrPeerUnreachable), + "the caller's budget was spent before the dial was made; blaming the peer for it is how an impatient client gets a healthy replica routed around") + Expect(err).ToNot(MatchError(cluster.ErrInstanceNotFound)) + }) + It("dials once when many callers open the same peer at the same time", func() { // Without a per-peer lock held across the dial, every concurrent // caller races to dial and all but one of the resulting sessions is diff --git a/core/services/worker/tunnel_test.go b/core/services/worker/tunnel_test.go index b5724a542459..8a59f686fef2 100644 --- a/core/services/worker/tunnel_test.go +++ b/core/services/worker/tunnel_test.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "fmt" "io" + "math/rand/v2" "net" "net/http" "net/http/httptest" @@ -598,6 +599,53 @@ func portOf(ln net.Listener) int { return port } +// listenOnSecondLoopback binds 127.0.0.2 on a port that 127.0.0.1 does not +// have anything on, and will not be given anything on. +// +// The port choice is the assertion's, not an incidental. The spec it serves +// proves a reachability fact: only 127.0.0.2 is listening, so a service that +// honoured the host the frontend named would connect and one that dials +// loopback cannot. A port taken from :0 lands in the kernel's ephemeral range, +// where some unrelated socket on 127.0.0.1 can be holding the same number, and +// then the dial to 127.0.0.1 succeeds and the spec reports an SSRF that did not +// happen. That is not hypothetical: it failed about one run in seven under +// `-race` while passing every time in isolation. +// +// Choosing from BELOW the ephemeral range (32768 on Linux by default) is what +// removes it, because the kernel does not hand those out for outbound +// connections. 127.0.0.1 is probed and released rather than held: holding it +// would make the dial the spec expects to fail succeed instead. +func listenOnSecondLoopback() (net.Listener, int) { + GinkgoHelper() + const ( + floor = 20000 + ceiling = 31000 + attempts = 200 + ) + for i := 0; i < attempts; i++ { + port := floor + rand.IntN(ceiling-floor) + free, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + continue + } + if err := free.Close(); err != nil { + continue + } + victim, err := net.Listen("tcp", fmt.Sprintf("127.0.0.2:%d", port)) + if err != nil { + // A host with no second loopback address fails on every port, so + // this is the skip the spec used to make inline. + if i == 0 && strings.Contains(err.Error(), "assign requested address") { + Skip("this host cannot bind a second loopback address: " + err.Error()) + } + continue + } + return victim, port + } + Fail(fmt.Sprintf("no port in [%d, %d) was free on 127.0.0.1 and bindable on 127.0.0.2 after %d attempts", floor, ceiling, attempts)) + return nil, 0 +} + // The routing table is the security boundary of the whole tunnel, and until now // nothing exercised it: every spec above installs dialLocalTCP, which is exactly // the permissive dialler loopbackService exists to prevent. A review turned @@ -643,12 +691,8 @@ var _ = Describe("Worker tunnel local services", func() { // property of the code. The only listener is on 127.0.0.2; nothing // is on 127.0.0.1 at that port. A service that honoured the named // host would connect; one that dials loopback cannot. - victim, err := net.Listen("tcp", "127.0.0.2:0") - if err != nil { - Skip("this host cannot bind a second loopback address: " + err.Error()) - } + victim, port := listenOnSecondLoopback() DeferCleanup(func() { _ = victim.Close() }) - port := portOf(victim) conn, err := loopbackService(port, port)(ctx, victim.Addr().String()) if err == nil { diff --git a/tests/e2e/distributed/cluster_baseline_test.go b/tests/e2e/distributed/cluster_baseline_test.go index d23ba8f6e9a9..9ef29a976125 100644 --- a/tests/e2e/distributed/cluster_baseline_test.go +++ b/tests/e2e/distributed/cluster_baseline_test.go @@ -1,6 +1,7 @@ package distributed_test import ( + "encoding/json" "fmt" "net/http" "os" @@ -43,6 +44,25 @@ type node struct { // have dialled instead of the tunnel. Address string `json:"address"` HTTPAddress string `json:"http_address"` + // keys is what the payload actually carried, which a decoded struct cannot + // tell you. Both fields above are the zero value when a worker advertises + // nothing AND when the key was renamed or dropped, and the whole point of + // the change these specs cover was removing the advertisement, so a rename + // would leave "it advertises nothing" passing for a payload that no longer + // says anything either way. + keys map[string]json.RawMessage `json:"-"` +} + +// UnmarshalJSON decodes the fields above and keeps the raw key set beside them. +func (n *node) UnmarshalJSON(data []byte) error { + // A distinct type, or this method calls itself. + type decoded node + var plain decoded + if err := json.Unmarshal(data, &plain); err != nil { + return err + } + *n = node(plain) + return json.Unmarshal(data, &n.keys) } // requireBinaries reports whether a missing binary must fail the spec instead of @@ -231,16 +251,25 @@ func (p *rosterProbe) idOf(name string) string { } // advertisementOf returns whatever endpoints the roster last reported a node -// advertising, joined for a failure message. Empty means the node published -// none, which is what a worker on this release does. -func (p *rosterProbe) advertisementOf(name string) string { +// advertising, joined for a failure message, and whether the payload carried +// both advertisement keys at all. +// +// The second result is the assertion, not a detail. Removing the advertisement +// is what the change under test did, so "the node advertises nothing" and "the +// keys that would have carried it are gone from the payload" are the two +// outcomes a spec has to keep apart: the first is the feature working, the +// second is the spec having lost its subject and reporting the feature working +// for any node at all, including one that advertises plenty. +func (p *rosterProbe) advertisementOf(name string) (string, bool) { for _, n := range p.lastSeen { if n.Name != name { continue } - return strings.TrimSpace(strings.Join([]string{n.Address, n.HTTPAddress}, " ")) + _, hasAddress := n.keys["address"] + _, hasHTTP := n.keys["http_address"] + return strings.TrimSpace(strings.Join([]string{n.Address, n.HTTPAddress}, " ")), hasAddress && hasHTTP } - return "" + return "", false } // describe is handed to Should as the failure message. Gomega calls a diff --git a/tests/e2e/distributed/cluster_tunnel_test.go b/tests/e2e/distributed/cluster_tunnel_test.go index 13255576c82f..9301af5dfa69 100644 --- a/tests/e2e/distributed/cluster_tunnel_test.go +++ b/tests/e2e/distributed/cluster_tunnel_test.go @@ -362,7 +362,14 @@ var _ = Describe("Worker tunnel end to end", Label("Distributed"), Label("Cluste // The worker publishes no endpoint of any kind. Without this the // inference below would be satisfied by a frontend that dialled an // advertised address, which is the path this phase removed. - advertised := probe.advertisementOf(c.WorkerName(0)) + // + // Both halves are needed. An empty value alone would also be what a + // spec sees after the keys are renamed or dropped from the payload, + // and that would leave this reporting "advertises nothing" about a + // node it can no longer see the advertisement of at all. + advertised, carriedKeys := probe.advertisementOf(c.WorkerName(0)) + Expect(carriedKeys).To(BeTrue(), + "the roster payload no longer carries the address and http_address keys, so this spec cannot tell a worker that advertises nothing from one it cannot read") Expect(advertised).To(BeEmpty(), "the worker advertised %q, so this spec cannot tell a tunnelled request from a direct dial", advertised) @@ -408,9 +415,19 @@ var _ = Describe("Worker tunnel end to end", Label("Distributed"), Label("Cluste nonOwner := 1 - owner Expect(nonOwner).ToNot(Equal(owner)) - // This is the whole point of the spec, so it is asserted rather than - // arranged: the request is about to go to a replica the database says - // does not hold this worker's tunnel. + // The request is about to go to a replica the database says does not + // hold this worker's tunnel. Read again here rather than inferred from + // the reading above, so a spec that had derived the index some other + // way still could not send it to the owner. + // + // This does NOT close the race, and saying that it does would be the + // same overclaim this phase has had to retract twice: ownership can + // move between this read and the reply, and if it moved TO nonOwner the + // request would be served directly and still come back 200. What closes + // it is the trailing read after the request, which requires the owner to + // be unchanged; a move to nonOwner leaves that read returning nonOwner + // and reddens the spec. This one rules out only the arrangement being + // wrong from the start, which is the cheaper half. Expect(owners.ownerIndexOf(c, 2, nodeID)).ToNot(Equal(nonOwner), "frontend %d owns the worker's tunnel, so a request to it would not be relayed and this spec would prove nothing", nonOwner) @@ -421,10 +438,16 @@ var _ = Describe("Worker tunnel end to end", Label("Distributed"), Label("Cluste expectMockedInference(client, c.FrontendURL(nonOwner), "relayed-model", fmt.Sprintf("frontend %d must relay to frontend %d, which owns the worker's tunnel", nonOwner, owner)) - // The relay did not move ownership. A replica that answered by taking - // the tunnel for itself would also have returned 200 above. + // THIS is the assertion that makes the request above a relayed one. + // + // It rules out the two ways a 200 could arrive without a relay: a + // replica that answered by taking the tunnel for itself, and the + // tunnel moving to nonOwner mid-request so that it served directly. + // Both leave the owner changed, and both redden here. The only window + // left is a move away and back inside one request, which takes two + // claims, and no replica dies in this scenario to prompt either. Expect(owners.ownerIndexOf(c, 2, nodeID)).To(Equal(owner), - "serving a relayed request moved the tunnel to frontend %d, which is not relaying", nonOwner) + "the tunnel is no longer held by frontend %d, so the request to frontend %d was not necessarily relayed", owner, nonOwner) }) // Scenario 3. Kills the replica holding the tunnel. The worker must land on @@ -524,16 +547,20 @@ var _ = Describe("Worker tunnel end to end", Label("Distributed"), Label("Cluste Expect(refused.status).ToNot(Equal(http.StatusOK), "the frontend served an inference for a worker that holds no tunnel, so something other than the tunnel reaches it: %s", refused.body) - // And it fails for the RIGHT reason. A frontend that was refusing for - // any other cause (a missing model, a backend it could not install, an - // unhealthy node) would satisfy the assertion above just as well, and - // would leave the three specs before this one unproven. - Expect(refused.body).To(Or( - ContainSubstring("no route"), - ContainSubstring("not connected"), - ContainSubstring("unroutable"), - ContainSubstring("tunnel"), - ), "the refusal does not name the missing route: %s", refused.body) + // And it fails for the RIGHT reason. A frontend refusing for any other + // cause (a missing model, a backend it could not install, an unhealthy + // node) would satisfy the assertion above just as well, and would leave + // the three specs before this one unproven. + // + // One substring, not a disjunction. This is the strongest leg of the + // whole control, and a disjunction is where such a leg goes soft: the + // looser alternatives this used to carry ("tunnel", "not connected", + // "unroutable") would each be satisfied by refusals that say nothing + // about routing, and one of them is a word this deployment's messages + // are full of. "no route" is what cluster.ErrNoRoute reads as, and + // nothing else on this path produces it. + Expect(refused.body).To(ContainSubstring("no route"), + "the refusal does not name the missing route, so this spec cannot tell a worker with no tunnel from a request that failed for one of the ordinary reasons: %s", refused.body) // The control's own control: put the tunnel back, change nothing else, // and the same request must now succeed. Without this the refusal above @@ -583,13 +610,34 @@ func withBalancer(into **frontendBalancer, arm ...func(*frontendBalancer)) func( } } -// percentile returns the p'th percentile of durations, which must be sorted. -func percentile(sorted []time.Duration, p float64) time.Duration { - if len(sorted) == 0 { - return 0 +// percentileIndex is where the p'th percentile of n sorted samples falls, or +// -1 when there are none. +func percentileIndex(n int, p float64) int { + if n == 0 { + return -1 + } + return int(float64(n-1) * p) +} + +// slowestOf is the worst sample, which is the statistic a blocking question +// turns on: a session that stalls one request while a transfer holds it shows +// up in the tail and not in the middle. +func slowestOf(samples []time.Duration) time.Duration { + worst := time.Duration(0) + for _, d := range samples { + if d > worst { + worst = d + } } - idx := int(float64(len(sorted)-1) * p) - return sorted[idx] + return worst +} + +// sortedCopy returns samples in ascending order without disturbing the caller's +// slice, which the report entry reads again afterwards. +func sortedCopy(samples []time.Duration) []time.Duration { + sorted := append([]time.Duration(nil), samples...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + return sorted } // summarise reports the shape of a latency sample. @@ -597,25 +645,50 @@ func summarise(label string, samples []time.Duration) string { if len(samples) == 0 { return label + ": no samples" } - sorted := append([]time.Duration(nil), samples...) - sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + sorted := sortedCopy(samples) var total time.Duration for _, d := range sorted { total += d } - return fmt.Sprintf("%s: n=%d mean=%s p50=%s p90=%s p99=%s max=%s", - label, len(sorted), total/time.Duration(len(sorted)), - percentile(sorted, 0.50), percentile(sorted, 0.90), percentile(sorted, 0.99), sorted[len(sorted)-1]) + out := fmt.Sprintf("%s: n=%d mean=%s", label, len(sorted), total/time.Duration(len(sorted))) + // A quantile is printed only when the sample can separate it from the ones + // already printed and from the max. At the sizes here, n around 20 to 50, + // p90 and p99 routinely land on the same element and p99 often lands on the + // last one, and printing one number three times under three names invites a + // reader to compare a tail nothing measured. Omission says "this sample + // cannot answer that"; a repeated number says the opposite. + printed := map[int]bool{percentileIndex(len(sorted), 1): true} + for _, q := range []struct { + name string + p float64 + }{{"p50", 0.50}, {"p90", 0.90}, {"p99", 0.99}} { + idx := percentileIndex(len(sorted), q.p) + if printed[idx] { + continue + } + printed[idx] = true + out += fmt.Sprintf(" %s=%s", q.name, sorted[idx]) + } + return out + fmt.Sprintf(" max=%s", sorted[len(sorted)-1]) } // The deferred question of this phase: what one yamux session does when a large // message and ordinary inference share it, with the relay adding a second hop // for most requests. // -// It is MEASURED here rather than asserted to be fine. Both sides' yamux -// windows are left at their defaults by a deliberate decision, pending these -// numbers, and the numbers are printed as a report entry so a future change to -// those windows has something to be compared against. +// It is MEASURED here rather than asserted to be fine, and the numbers are +// printed as a report entry so a later change has something to be compared +// against. +// +// Be exact about which windows those numbers do and do not speak for. The +// WORKER TUNNEL's two ends both take yamux's defaults (core/services/worker, +// tunnel.go, and core/http/endpoints/cluster/connect.go), and this measurement +// is no reason to change that, but it is also no evidence that they are right: +// a default receive window is limited by the bandwidth-delay product of the +// link, and loopback has no delay to produce one. The PEER LINK's windows are raised +// on both ends. What is measured here is whether a session SERIALISES, which +// loopback answers perfectly well, and not whether a window is large enough for +// a link with latency, which it cannot answer at all. const ( // bulkArtifactSize is the model artifact staged over the tunnel while // probes run. It stands in for the 50MB-class message this feature has to @@ -630,6 +703,12 @@ const ( // hypothetical: the first version of this spec used 64MiB and still passed // with the artifact cut to 4KiB, which is the definition of measuring // nothing. + // + // What it costs, since it is the largest thing this suite puts on disk: + // two bulk models seeded into each of two frontends is 512 MiB, and each is + // then staged to the worker, which is 256 MiB more. About 768 MiB under + // TMPDIR for the length of this spec, plus one 128 MiB string resident in + // the test process. The harness removes the tree in Stop. bulkArtifactSize = 128 << 20 // tinyArtifactSize is the same cold load with nothing to transfer. It is @@ -655,19 +734,55 @@ const ( // reporting. minOverlappingProbes = 5 - // holStallCeiling is the coarse absolute backstop, for the case where the - // bulk transfer is itself pathologically slow and the transfer-window - // comparison would admit a probe latency no deployment would tolerate. + // holStallShare bounds the worst probe as a fraction of the window in which + // bytes were moving. It is the STRUCTURAL assertion: a session that + // head-of-line blocks parks a probe until the transfer lets go, so a + // stalled probe's latency is on the order of the whole window, and one that + // interleaves finishes many probes inside it. + // + // Half, not the whole window. Bounding by the window itself admits a probe + // that took nearly all of it, which is the wedge with the numbers filed + // off. // - // The comparison is the assertion that bites; this is only the floor under - // it. A session that head-of-line blocks parks a probe until the transfer - // releases the session, so the stalled probe's latency is on the order of - // the transfer window's; one that interleaves answers a warm mock - // completion in the milliseconds it takes with nothing else on the wire. - // An ABSOLUTE ceiling would have to sit between those two, and on this - // hardware they are 30ms and 250ms, which is too narrow a gap to hold on a - // loaded box. The comparison holds whatever the box's speed, because both - // numbers move with it. + // Not tighter than half, and the reason is measured rather than cautious. A + // probe's tail grows faster than the transfer window does when the box is + // busy: under a concurrent `-race` suite the worst relayed probe reached + // 20% of its window here, so a quarter would have had 1.2x of margin and a + // spec that fails one run in three is worse than no spec. Half leaves 2.4x + // on the same run and still reddens on a stall, which parks a probe for the + // window rather than a fifth of it. + holStallShare = 2 + + // holStallControlFactor bounds the worst probe against the worst probe + // under the EMPTY load in the same run, which is the second half of not + // relaxing under load: a slower box raises the control and the bound with + // it, while an absolute number would simply admit more. + // + // The empty load is the right thing to compare against and the plain + // baseline is not. Both samples then contain a cold load's contention for + // the worker, the router and the session, and the only thing that differs + // between them is 128 MiB crossing the wire. Compared against the quiet + // baseline instead, a transfer that cost nothing at all would still look + // like a regression on any box where a cold load is expensive. + // + // Eight, from both ends of the gap it has to sit in. Healthy runs measured + // 1.6x to 3.8x on this box under a concurrent `-race` suite, and about 2.5x + // to 3x on the reviewer's; a session that stalled a probe until the + // transfer let go would show the whole window over the same control, which + // is 14x to 43x on the same runs. + holStallControlFactor = 8 + + // holStallCeiling is the coarse absolute backstop under both of those, for + // a transfer so slow that a quarter of its window is a latency no + // deployment would tolerate. + // + // It is deliberately far above anything measured rather than tuned, because + // an absolute number cannot separate a wedge from a slow box. Measured + // worst probe and transfer window, for scale: 21-36ms against 261-576ms on + // the box this was written on, and 70-136ms against 590-1320ms on the + // reviewer's. An absolute ceiling that bit on the first machine's wedge + // would fail on the second machine's healthy run, which is why the two + // relative bounds above are the assertions and this is only a floor. holStallCeiling = 5 * time.Second ) @@ -825,15 +940,15 @@ var _ = Describe("Worker tunnel under load", Label("Distributed"), Label("Cluste report = append(report, line) GinkgoWriter.Println(line) - slowest := time.Duration(0) - for _, d := range underBulk { - if d > slowest { - slowest = d - } - } - Expect(slowest).To(BeNumerically("<", transferWindow), - "%s: a probe waited %s while %s of bytes were moving, which is the shape of a session that stalled the probe until the transfer let go, not of one that interleaved them", + slowest := slowestOf(underBulk) + Expect(slowest).To(BeNumerically("<", transferWindow/holStallShare), + "%s: a probe waited %s of the %s in which bytes were moving, which is the shape of a session that stalled the probe until the transfer let go, not of one that interleaved them", label, slowest, transferWindow) + control := slowestOf(underTiny) + Expect(control).To(BeNumerically(">", 0), "%s: the empty-load control produced no samples", label) + Expect(slowest).To(BeNumerically("<", holStallControlFactor*control), + "%s: the worst probe was %s while bytes were moving against %s under the same cold load with nothing to move, which is a stall rather than the contention a shared session costs", + label, slowest, control) Expect(slowest).To(BeNumerically("<", holStallCeiling), "%s: a probe waited %s while the bulk transfer held the session", label, slowest) } From 92e1927ff8d2b6e3d5f34bb34cb67be15d15a5aa Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 00:59:36 +0000 Subject: [PATCH 063/153] fix(distributed): let a worker's own refusal be evidence about its backend A worker that refuses a stream has answered, and cluster.Dial keeps the three tunnelproto sentinels out of the ErrNoRoute umbrella precisely so a consumer can act on that. No consumer did. Since workers stopped listening, a backend process that crashed on a healthy worker is no longer a dead listener's codes.Unavailable: the worker refuses the stream with ErrStreamTargetUnavailable, gRPC flattens it into Unavailable anyway, and nodes.unroutable reported the whole thing as "this frontend has no route". Every reap path then answered ProbeUnknown and left the row, so the replica slot never freed and at the default MaxReplicasPerModel=1 the only cleanup left was LRU eviction of models that were working. isWorkerAnswer is exported as cluster.IsWorkerAnswer, so the errors the dialer keeps out of the umbrella are by construction the errors the consumers treat as the worker answering. nodes.unroutable and pkg/model's transportFailure both use it; ConnectionEvictingClient, the site reached during inference, goes through transportFailure rather than asking the transport directly. A reply code this frontend does not recognise is still not an answer, so a newer worker's vocabulary costs a retry and not a replica. The reap guards keep the allow-list rather than requiring ErrNoRoute: an unrecognised dial error must mean "no route", never "the backend is gone". Also in this final pass over the branch: - Docs: recommend upgrading FRONTENDS first, with the symptom of each order. Workers-first fails now that a 4xx registration is a verdict rather than an outage, so an old frontend's "address is required for backend workers" makes each restarted worker exit and drains the fleet a node per restart. - Docs: LOCALAI_WORKER_TUNNEL=false is a fatal startup error, not a degraded mode, in both places that described it; and a frontend rollback needs every worker restarted, because re-registration force-clears the address columns. - A replica with no advertised address now says so every five minutes and names the workers only it can reach, instead of one startup warning for a cost paid for the life of the process. - callerRanOut's rule now holds at all three siblings, so an expired caller deadline stops reading as a broken tunnel; probeHealth's withdrawn reason for using the raw client is corrected; the dead DoOrCached is deleted and its coverage kept on DoOrCachedResult; sweepLeakedInFlight enumerates the outcomes that reach it. - The peer route's self-declared id is recorded as a phase-3 deferral, in the handler, in the isolation claim it narrows, and in the operator docs. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 83 ++++++++++++++++-- core/application/unadvertised_replica_test.go | 73 ++++++++++++++++ core/http/endpoints/cluster/peer.go | 24 ++++++ core/services/cluster/dialer.go | 48 ++++++++++- core/services/cluster/dialer_test.go | 33 +++++++ core/services/cluster/sessions.go | 9 ++ core/services/cluster/tunnel.go | 17 ++++ core/services/cluster/tunnel_test.go | 26 ++++++ core/services/nodes/interfaces.go | 29 +++++++ core/services/nodes/probe_cache.go | 25 +++--- core/services/nodes/probe_cache_test.go | 49 +++++++---- core/services/nodes/reconciler.go | 10 ++- core/services/nodes/reconciler_prober_test.go | 86 ++++++++++++++++++- core/services/nodes/registry.go | 8 ++ core/services/nodes/router.go | 27 +++--- docs/content/features/distributed-mode.md | 47 +++++++--- pkg/model/connection_evicting_client.go | 8 +- pkg/model/loader.go | 18 +++- pkg/model/remote_unroutable_internal_test.go | 69 +++++++++++++++ 19 files changed, 619 insertions(+), 70 deletions(-) create mode 100644 core/application/unadvertised_replica_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index 244f6eb680ad..a50ca76044e0 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -239,13 +239,20 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // itself; nobody else can. On N replicas behind round robin that is // (N-1)/N of the traffic for that worker. // - // It stays a warning all the same. Refusing to start would take out - // every existing single-host deployment, whose route to a local - // database is loopback and which has no peers to be unreachable by; - // the deployments this hurts are multi-replica ones, and telling those - // two apart at startup is a change with its own design and its own - // specs rather than a line here. - xlog.Warn("This replica will not be reachable by its peers: no advertised address", + // It does not refuse to START. Refusing would take out every existing + // single-host deployment, whose route to a local database is loopback + // and which has no peers to be unreachable by; the deployments this + // hurts are multi-replica ones, and telling those two apart at startup + // is a change with its own design and its own specs rather than a line + // here. + // + // What it does not get to do is stay quiet. One startup line scrolls + // away in seconds and the cost is paid for the whole life of the + // process, on a symptom (workers that 5xx from most of the fleet) whose + // obvious reading is "the worker is broken". So this is an ERROR, not a + // warning, and nagUnadvertisedReplica below repeats it for as long as + // the state lasts, naming the workers it is currently costing. + xlog.Error("This replica is not registered in the cluster: no advertised address. Peers cannot reach it, and any worker whose tunnel lands here will be unroutable from every other replica", "error", err, "knob", "LOCALAI_DISTRIBUTED_ADVERTISE_ADDR") } else { membership = cluster.NewMembership(clusterRegistry, cfg.Distributed.InstanceID, advertised, internal.PrintableVersion()) @@ -272,6 +279,10 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // through. if membership != nil { membership.SetTunnels(tunnels) + } else { + // The runtime symptom the startup line cannot be. See + // nagUnadvertisedReplica. + go nagUnadvertisedReplica(cfg.Context, tunnels.Held, unadvertisedNagInterval, logUnroutableWorkers) } // The links peers dial IN, with the relay installed on them. This is what @@ -623,6 +634,64 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade }, nil } +// unadvertisedNagInterval is how often a replica that could not advertise +// itself says so again. +// +// Five minutes is chosen against the log it lands in, not against the urgency: +// the condition never clears on its own, so this line is either read once and +// acted on or it is noise for the life of the process, and a noisy line gets +// filtered rather than fixed. It is still frequent enough that the state is +// visible in any window of logs an operator pulls while investigating the +// symptom it causes. +const unadvertisedNagInterval = 5 * time.Minute + +// nagUnadvertisedReplica repeats, for as long as the process runs, that this +// replica is invisible to its peers, and names what that is currently costing. +// +// It exists because the deferral it accompanies changed cost between phases and +// nothing about the deployment says so. Before workers held tunnels, a replica +// with no advertised address was merely unreachable BY peers and could still +// dial every worker directly, so a startup warning was proportionate. Now a +// worker's tunnel lands on one replica and every other replica reaches it by +// relaying to the owner, and the owner is resolved by joining the connection +// row against a LIVE INSTANCES ROW - which this replica does not have. So every +// worker that lands here is answered as unroutable everywhere else: on N +// replicas behind round robin, (N-1)/N of that worker's traffic fails, while +// this replica serves it perfectly and reports nothing. +// +// held is passed as a function rather than the registry so this can be driven +// without one, and alarm is passed rather than logged inline so a spec can +// observe the alarms instead of scraping a log. +func nagUnadvertisedReplica(ctx context.Context, held func() []string, every time.Duration, alarm func([]string)) { + ticker := time.NewTicker(every) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + alarm(held()) + } + } +} + +// logUnroutableWorkers says what the state costs RIGHT NOW. +// +// The two cases are kept apart because they call for different urgency and an +// operator can tell them apart at a glance. With no worker held this is a +// misconfiguration that has not been paid for yet; with workers held, every one +// of them is named, because "which worker is broken" is the question the +// symptom sends an operator to ask and the answer is that none of them is. +func logUnroutableWorkers(held []string) { + if len(held) == 0 { + xlog.Warn("This replica is still not registered in the cluster: no advertised address. No worker holds a tunnel here yet; the first that does will be unroutable from every other replica", + "knob", "LOCALAI_DISTRIBUTED_ADVERTISE_ADDR") + return + } + xlog.Error("This replica is not registered in the cluster and holds worker tunnels: those workers are unroutable from every OTHER replica, and requests for their models fail there with no route. The workers are healthy; this replica is invisible", + "workers", held, "worker_count", len(held), "knob", "LOCALAI_DISTRIBUTED_ADVERTISE_ADDR") +} + // advertisedPeerAddr is the host:port peers dial to reach this replica. // // The operator's value wins outright. Otherwise it is derived from the port diff --git a/core/application/unadvertised_replica_test.go b/core/application/unadvertised_replica_test.go new file mode 100644 index 000000000000..ed9eeb4d6399 --- /dev/null +++ b/core/application/unadvertised_replica_test.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: MIT + +package application + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The runtime symptom for a deferral whose cost changed between phases. +// +// Not refusing to start without an advertised address stays deferred on +// purpose: refusing would take out every single-host deployment. What is not +// deferred is telling the operator, repeatedly, that this replica is invisible +// and which workers that is costing - because the symptom it produces (a worker +// that 5xxs from most of the fleet) reads as a worker problem, and a single +// startup line has scrolled away long before anyone goes looking. +var _ = Describe("the alarm for a replica with no advertised address", func() { + It("keeps firing for as long as the state lasts, and names the workers it costs", func() { + // Repetition is the property. A one-shot alarm is the startup line + // again, which is what was already there and was not enough. + ctx, cancel := context.WithCancel(context.Background()) + DeferCleanup(cancel) + + alarms := make(chan []string, 8) + go nagUnadvertisedReplica(ctx, func() []string { return []string{"w1", "w2"} }, + time.Millisecond, func(held []string) { alarms <- held }) + + // Two, not one: the second is what a one-shot implementation fails. + var first, second []string + Eventually(alarms, "10s").Should(Receive(&first)) + Eventually(alarms, "10s").Should(Receive(&second)) + Expect(first).To(ConsistOf("w1", "w2"), + "the workers this is costing are the answer to the question the symptom provokes") + Expect(second).To(ConsistOf("w1", "w2")) + }) + + It("reads the held set on every tick rather than the one it started with", func() { + // A replica accumulates tunnels while it runs, so an alarm bound to the + // set at startup would name an empty list forever on exactly the + // deployment where the cost is real. + ctx, cancel := context.WithCancel(context.Background()) + DeferCleanup(cancel) + + workers := make(chan []string, 32) + for range 32 { + workers <- []string{"w-late"} + } + alarms := make(chan []string, 8) + go nagUnadvertisedReplica(ctx, func() []string { return <-workers }, + time.Millisecond, func(held []string) { alarms <- held }) + + var got []string + Eventually(alarms, "10s").Should(Receive(&got)) + Expect(got).To(ConsistOf("w-late")) + }) + + It("stops when the process context ends", func() { + ctx, cancel := context.WithCancel(context.Background()) + stopped := make(chan struct{}) + go func() { + defer GinkgoRecover() + nagUnadvertisedReplica(ctx, func() []string { return nil }, time.Hour, func([]string) {}) + close(stopped) + }() + + cancel() + Eventually(stopped, "10s").Should(BeClosed()) + }) +}) diff --git a/core/http/endpoints/cluster/peer.go b/core/http/endpoints/cluster/peer.go index 00ef12c5b59d..ca8ef06fd6fd 100644 --- a/core/http/endpoints/cluster/peer.go +++ b/core/http/endpoints/cluster/peer.go @@ -38,6 +38,30 @@ func PeerHandler(token string, onSession func(peerID string, sess *yamux.Session return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") } + // SELF-DECLARED, and knowingly so. Unlike the worker route next door, + // which resolves ?id= to a node row and checks that node's OWN minted + // credential, this route has only the shared cluster token to check, + // so the id is a label and not a claim anything verifies. + // + // What that costs, exactly, for anything already holding the shared + // token (every worker holds it, and it is the same token that + // authenticates registration): it can relay to every worker tunnel this + // replica owns, reaching every backend gRPC process and every worker's + // file-transfer server; and by declaring a legitimate replica's id it + // can make SessionStore.Accept evict that replica's inbound link, at + // will. Neither is a new capability in KIND - before workers stopped + // listening, a holder of that token could already dial any worker's + // advertised ports directly - but the token is now the only thing + // between an attacker and the whole fleet's tunnels. + // + // It is deferred rather than patched, because the cheap patch does not + // work: checking ?id= against the instances table stops an invented id + // and stops nothing else, since the attack declares a REAL replica's + // id, and it would buy a false sense of a closed hole. Closing it takes + // a credential per replica, minted where a replica joins the instances + // table and presented here, which is a design with its own migration + // and its own specs. Tracked as the phase-3 item named at + // nodes.BackendNode.TunnelTokenHash. peerID := c.QueryParam("id") if peerID == "" { return echo.NewHTTPError(http.StatusBadRequest, "missing peer id") diff --git a/core/services/cluster/dialer.go b/core/services/cluster/dialer.go index fbb95d583acc..3460dee3abd3 100644 --- a/core/services/cluster/dialer.go +++ b/core/services/cluster/dialer.go @@ -112,7 +112,7 @@ func isAbsenceClaim(err error) bool { return errors.Is(err, ErrNoConnection) || errors.Is(err, ErrInstanceNotFound) } -// isWorkerAnswer reports whether an error is the WORKER's own refusal, read off +// IsWorkerAnswer reports whether an error is the WORKER's own refusal, read off // the reply it sent. // // Those three sentinels are the only ones ReadStreamReply produces from a frame @@ -124,7 +124,17 @@ func isAbsenceClaim(err error) bool { // would let a newer worker's vocabulary be read by an older frontend as // evidence about a backend, and the consequence of guessing wrong in that // direction is a reaped replica; guessing wrong the other way costs a retry. -func isWorkerAnswer(err error) bool { +// +// It is EXPORTED because it is half of a contract, not an implementation +// detail. Dial keeps these three out of the ErrNoRoute umbrella so that a +// consumer can act on them; a consumer that cannot ask "was this the worker +// speaking?" has no way to use that, and for a whole phase none could, so every +// worker refusal reached the schedulers as "this frontend has no route" and +// nothing could ever be reaped. The two sides must agree on the SAME set, so +// there is one predicate and both call it: see nodes.unroutable and +// model.transportFailure, whose job is to answer "did this call reach a +// backend?" and for whom a refusal means it did. +func IsWorkerAnswer(err error) bool { return errors.Is(err, ErrStreamTagUnknown) || errors.Is(err, ErrStreamTargetUnavailable) || errors.Is(err, ErrStreamRequestInvalid) @@ -293,6 +303,32 @@ func (d *WorkerDialer) relay(ctx context.Context, nodeID, tag, target string) (n // session, and a frontend that retries would exhaust the worker's stream // budget rather than the worker's patience. func (d *WorkerDialer) handshake(ctx context.Context, stream net.Conn, nodeID, tag, target string) (net.Conn, error) { + // blameCaller attributes a handshake I/O failure to the CALLER's own spent + // budget when that is what ended it, and returns nil when it was not. + // + // The third instance of the rule peerlink.go states in full at callerRanOut: + // the handshake deadline IS the caller's deadline whenever the caller's is + // the shorter (see handshakeDeadline), so a caller that has run out makes + // the socket's own timer fire, and the resulting i/o timeout arrives here + // while ctx.Err() may still read nil because nothing orders the two timers. + // Reported plainly, that is "the tunnel would not carry the request" for a + // worker that is connected, healthy and idle. + // + // The umbrella stays on either way, so Dial's contract is unchanged; what + // changes is that context.DeadlineExceeded is matchable underneath and the + // log line names the caller instead of the worker. It deliberately runs + // BEFORE the worker-answer check below, so a refusal that arrived in the + // same instant the budget expired is reported as the caller's timeout: a + // spent deadline must never be able to manufacture evidence about a + // backend, which is the same direction peerlink.go takes for absence. + blameCaller := func(what string) error { + ctxErr := callerRanOut(ctx) + if ctxErr == nil { + return nil + } + return routeFailure(nodeID, fmt.Errorf("%s: the caller's own budget ran out: %w", what, ctxErr)) + } + if err := stream.SetDeadline(handshakeDeadline(ctx)); err != nil { _ = stream.Close() return nil, routeFailure(nodeID, fmt.Errorf("arming the handshake deadline: %w", err)) @@ -302,11 +338,17 @@ func (d *WorkerDialer) handshake(ctx context.Context, stream net.Conn, nodeID, t // The stream would not carry the request, so the tunnel broke under it. // Nothing was asked of the worker and nothing was learned about it. _ = stream.Close() + if blamed := blameCaller(fmt.Sprintf("asking for %q on %q", tag, target)); blamed != nil { + return nil, blamed + } return nil, routeFailure(nodeID, fmt.Errorf("asking for %q on %q: %w", tag, target, err)) } if err := ReadStreamReply(stream); err != nil { _ = stream.Close() - if isWorkerAnswer(err) { + if blamed := blameCaller(fmt.Sprintf("opening %q", tag)); blamed != nil { + return nil, blamed + } + if IsWorkerAnswer(err) { // The worker wrote a refusal, so it is connected and answering. // This is the ONE failure on the whole path that is real evidence // about the worker, and putting the umbrella on it would throw that diff --git a/core/services/cluster/dialer_test.go b/core/services/cluster/dialer_test.go index 2e0b945d3704..e9ce9b243d20 100644 --- a/core/services/cluster/dialer_test.go +++ b/core/services/cluster/dialer_test.go @@ -292,6 +292,39 @@ var _ = Describe("The worker dialer", func() { Expect(out.err).ToNot(MatchError(cluster.ErrNoRoute)) }) + It("blames the caller's own spent budget, not the worker, when the handshake ends on the deadline", func() { + // The third and last site of peerlink.go's callerRanOut rule. + // + // The handshake deadline IS the caller's whenever the caller's is + // shorter (handshakeDeadline), so a caller that has run out makes + // the stream's own timer fire, and the i/o timeout arrives here + // while ctx.Err() may still read nil. Without the guard this reads + // as "the tunnel would not carry the request" for a worker that is + // connected, healthy, and simply not answering yet, which is what a + // worker under load looks like. + // + // The worker accepts the stream and says nothing, so the ONLY thing + // that can end this handshake is the deadline; there is no sleep + // and no race. + frontend, worker := workerTunnel() + _, err := mine.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + go func() { + defer GinkgoRecover() + _, _ = worker.AcceptStream() + }() + + d := cluster.NewWorkerDialer(mine, nil) + var out dialResult + Eventually(dialAsync(d, deadlinePassed{ctx}, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000"), "10s").Should(Receive(&out)) + Expect(out.err).To(MatchError(context.DeadlineExceeded), + "the caller's budget was spent; the worker never got a verdict") + Expect(out.err.Error()).To(ContainSubstring("the caller's own budget ran out")) + // The umbrella is still on it, so Dial's contract is unchanged and + // no consumer reads this as the worker having gone away. + expectNoRoute(out.err) + }) + It("reports a broken tunnel held here as itself, not as a routing fact", func() { // ErrNotOwner tells a caller to look for the worker elsewhere. For // a tunnel held right here that sends it back to this replica, and diff --git a/core/services/cluster/sessions.go b/core/services/cluster/sessions.go index afcdd1b97384..aa52fa8d4842 100644 --- a/core/services/cluster/sessions.go +++ b/core/services/cluster/sessions.go @@ -76,6 +76,15 @@ func (s *SessionStore) Accept(peerID string, sess *yamux.Session) { // is false when no link from that peer is held, which a caller must not read as // the peer being absent: it may be about to dial, or dialling this replica may // simply not be its job. +// +// It has NO production caller, and that is stated rather than left to be +// discovered: nothing in the frontend routes by looking up an inbound link, +// because the relay is driven by the streams a peer opens on the session, not +// by this side going to find one. What Get exists for is the specs, which have +// no other way to observe which session this store holds, and holding exactly +// one session per peer is the property Accept's eviction is about. Deleting it +// would delete that observation with it. Anything tempted to route on it should +// read the paragraph above first: a missing entry is not an absent peer. func (s *SessionStore) Get(peerID string) (*yamux.Session, bool) { s.mu.Lock() defer s.mu.Unlock() diff --git a/core/services/cluster/tunnel.go b/core/services/cluster/tunnel.go index 65adc4aad4ca..ccd8b9cf91df 100644 --- a/core/services/cluster/tunnel.go +++ b/core/services/cluster/tunnel.go @@ -290,6 +290,23 @@ func (t *TunnelRegistry) Open(ctx context.Context, nodeID string) (net.Conn, err stream, err := held.sess.OpenStream(ctx) if err != nil { + // The same rule peerlink.go applies to a peer, applied here to a + // tunnel, because the confusion is the same one: a caller whose own + // budget ran out gets the socket's error back before the context's + // cancel func has necessarily run, so ctx.Err() can still read nil + // while the failure is entirely the caller's. Reporting it plainly + // would put "the tunnel this replica holds would not carry a stream" + // in an operator's log for a worker that is fine and a client that was + // impatient. callerRanOut settles it on the wall clock; see its + // comment for why ctx.Err() alone is not the question. + // + // The caller's error is wrapped rather than returned bare, so + // WorkerDialer's contract still holds (every failure to resolve or open + // carries ErrNoRoute) and context.DeadlineExceeded stays matchable + // underneath for anyone that wants to tell the two apart. + if ctxErr := callerRanOut(ctx); ctxErr != nil { + return nil, fmt.Errorf("opening a stream to node %q over the tunnel held here: the caller's own budget ran out: %w", nodeID, ctxErr) + } return nil, fmt.Errorf("opening a stream to node %q over the tunnel held here: %w", nodeID, err) } return stream, nil diff --git a/core/services/cluster/tunnel_test.go b/core/services/cluster/tunnel_test.go index 7d0136b47e67..ca4455541f9f 100644 --- a/core/services/cluster/tunnel_test.go +++ b/core/services/cluster/tunnel_test.go @@ -304,6 +304,32 @@ var _ = Describe("The worker tunnel registry", func() { "holding the tunnel is a routing fact, and a failed Open is not what un-holds it") }) + It("blames the caller's own spent budget, not the tunnel, when a stream cannot be opened", func() { + // The second of the three sites where peerlink.go's callerRanOut rule + // has to hold. A caller whose budget ran out gets the multiplexer's + // error back before the scheduler has necessarily run its context's + // cancel func, so ctx.Err() can still read nil while the failure is + // entirely the caller's; reported plainly it becomes "the tunnel this + // replica holds would not carry a stream" in an operator's log for a + // worker that is fine. + // + // deadlinePassed is that window made deterministic: deadline elapsed, + // cancellation not delivered. Nothing about the broken session is + // faked. + frontend, worker := workerTunnel() + _, err := tun.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + + Expect(worker.Close()).To(Succeed()) + Eventually(frontend.IsClosed, "10s").Should(BeTrue()) + + _, err = tun.Open(deadlinePassed{ctx}, "w1") + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(context.DeadlineExceeded), + "the caller's budget was spent, and only it can say so") + Expect(err.Error()).To(ContainSubstring("the caller's own budget ran out")) + }) + It("refuses a nil session rather than claiming a tunnel that cannot carry anything", func() { // A claim written for a session that does not exist publishes a tunnel // to every replica in the deployment, and the fence would then have to diff --git a/core/services/nodes/interfaces.go b/core/services/nodes/interfaces.go index dc4adf8a161d..a3ceb0b9e242 100644 --- a/core/services/nodes/interfaces.go +++ b/core/services/nodes/interfaces.go @@ -7,6 +7,7 @@ import ( "net" "time" + "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/LocalAI/core/services/messaging" grpc "github.com/mudler/LocalAI/pkg/grpc" ) @@ -219,6 +220,31 @@ var ErrNoWorkerDialer = fmt.Errorf("%w: no worker tunnel dialer is configured", // nil, which means "the call reached a backend" and preserves the behaviour // every non-distributed caller has always had. Decorators are looked through; // see grpc.BackendUnwrapper for why that is not optional. +// +// A WORKER'S OWN REFUSAL also yields nil, and that is the second half of the +// contract rather than a loophole. cluster.Dial keeps the three tunnelproto +// sentinels out of the ErrNoRoute umbrella precisely so this function can tell +// them apart, and for a whole phase nothing did: a backend process that crashed +// on a healthy worker is no longer a dead listener's codes.Unavailable, it is +// the worker refusing the stream with cluster.ErrStreamTargetUnavailable, which +// gRPC then flattens into codes.Unavailable anyway. Reporting that as +// unroutable made every reap path answer ProbeUnknown and leave the row, so the +// replica slot never freed and (at the default MaxReplicasPerModel=1) the only +// remaining cleanup was LRU eviction of HEALTHY models. A worker that answers +// has demonstrated it is there, so the answer is evidence about its backend and +// the reap guards may act on it. +// +// All three sentinels, not only the unavailable one, and the difference is +// worth stating because two of them are not observations about the process. An +// unknown tag means this worker does not serve gRPC streams at all; an invalid +// request means the stored address is not a port in this worker's range. +// Neither clears on its own, so a row that carries one is unreachable from +// EVERY replica for as long as it exists, and reaping it converges: the model +// is reloaded somewhere that works and re-registers a usable address. The +// condition the phase refuses to reap on is a TRANSIENT one, and none of these +// is transient. A reply code this frontend does not recognise is deliberately +// not in the set (see cluster.IsWorkerAnswer), so a newer worker's vocabulary +// reaches an older frontend as "no route" and costs a retry rather than a row. func unroutable(client grpc.Backend) error { // LastDialErrorOf and not a type assertion: the assertion could not see // past a decorator, and SmartRouter hands every routed client out wrapped. @@ -226,6 +252,9 @@ func unroutable(client grpc.Backend) error { if dialErr == nil { return nil } + if cluster.IsWorkerAnswer(dialErr) { + return nil + } // Multi-%w: the umbrella this package acts on, and the cluster condition // underneath it, both stay matchable. return fmt.Errorf("%w: %w", ErrWorkerUnroutable, dialErr) diff --git a/core/services/nodes/probe_cache.go b/core/services/nodes/probe_cache.go index fee71383e7e1..a5b3e2cb0573 100644 --- a/core/services/nodes/probe_cache.go +++ b/core/services/nodes/probe_cache.go @@ -33,7 +33,7 @@ type probeCache struct { } // newProbeCache returns a probeCache with the given TTL. Zero TTL disables -// caching: every call to DoOrCached invokes the probe. +// caching: every call to DoOrCachedResult invokes the probe. func newProbeCache(ttl time.Duration) *probeCache { return &probeCache{ ttl: ttl, @@ -68,17 +68,18 @@ func (c *probeCache) Invalidate(key string) { delete(c.seen, key) } -// DoOrCached returns true if key is fresh; otherwise it runs probe (coalescing -// concurrent callers via singleflight) and caches a successful result. Failed -// probes invalidate the cache, so a transient miss doesn't pin every -// subsequent request to a re-probe. -func (c *probeCache) DoOrCached(key string, probe func() bool) bool { - alive, _ := c.DoOrCachedResult(key, func() (bool, error) { return probe(), nil }) - return alive -} - -// DoOrCachedResult is DoOrCached with a second result: the reason the probe -// never reached the backend, or nil when it did. +// DoOrCachedResult returns true if key is fresh; otherwise it runs probe +// (coalescing concurrent callers via singleflight) and caches a successful +// result. Failed probes invalidate the cache, so a transient miss does not pin +// every subsequent request to a re-probe. +// +// It is the ONLY entry point. A boolean-only sibling, DoOrCached, stood beside +// it until probeHealth stopped using it, after which it was production code +// held green by nothing but its own specs; the shim that reads it as a boolean +// now lives in probe_cache_test.go, where its one caller is. +// +// The second result is the reason the probe never reached the backend, or nil +// when it did. // // The second result travels through the SINGLEFLIGHT, which is the whole reason // it is not simply a variable the caller closes over. A closed-over variable is diff --git a/core/services/nodes/probe_cache_test.go b/core/services/nodes/probe_cache_test.go index 8d1e059127a8..42eaf35c69fc 100644 --- a/core/services/nodes/probe_cache_test.go +++ b/core/services/nodes/probe_cache_test.go @@ -11,6 +11,21 @@ import ( "golang.org/x/sync/singleflight" ) +// doOrCached drives the production entry point with a boolean-only probe, +// which is what most of these specs are about. +// +// It is a spec helper and not a method, deliberately. It WAS a method, and once +// probeHealth moved to DoOrCachedResult it became production code with no +// production caller, kept green by these specs alone. Moving it here keeps the +// convenience where its only user is and stops the shim being mistaken for a +// supported way to probe. +func doOrCached(c *probeCache, key string, probe func() bool) bool { + GinkgoHelper() + alive, unreached := c.DoOrCachedResult(key, func() (bool, error) { return probe(), nil }) + Expect(unreached).To(BeNil()) + return alive +} + var _ = Describe("probeCache", func() { It("invokes the probe on a cold cache and caches success", func() { c := newProbeCache(time.Minute) @@ -20,9 +35,9 @@ var _ = Describe("probeCache", func() { return true } - Expect(c.DoOrCached("k", probe)).To(BeTrue()) - Expect(c.DoOrCached("k", probe)).To(BeTrue()) - Expect(c.DoOrCached("k", probe)).To(BeTrue()) + Expect(doOrCached(c, "k", probe)).To(BeTrue()) + Expect(doOrCached(c, "k", probe)).To(BeTrue()) + Expect(doOrCached(c, "k", probe)).To(BeTrue()) // Cached: probe ran once. Expect(atomic.LoadInt32(&calls)).To(Equal(int32(1))) @@ -38,9 +53,9 @@ var _ = Describe("probeCache", func() { return true } - Expect(c.DoOrCached("k", probe)).To(BeTrue()) + Expect(doOrCached(c, "k", probe)).To(BeTrue()) time.Sleep(5 * time.Millisecond) - Expect(c.DoOrCached("k", probe)).To(BeTrue()) + Expect(doOrCached(c, "k", probe)).To(BeTrue()) Expect(atomic.LoadInt32(&calls)).To(Equal(int32(2))) }) @@ -56,16 +71,16 @@ var _ = Describe("probeCache", func() { // First probe fails — must NOT be cached. result.Store(false) - Expect(c.DoOrCached("k", probe)).To(BeFalse()) + Expect(doOrCached(c, "k", probe)).To(BeFalse()) Expect(c.IsFresh("k")).To(BeFalse()) // Recover: second probe succeeds and is cached. result.Store(true) - Expect(c.DoOrCached("k", probe)).To(BeTrue()) + Expect(doOrCached(c, "k", probe)).To(BeTrue()) Expect(c.IsFresh("k")).To(BeTrue()) // Third call short-circuits on the fresh entry. - Expect(c.DoOrCached("k", probe)).To(BeTrue()) + Expect(doOrCached(c, "k", probe)).To(BeTrue()) Expect(atomic.LoadInt32(&calls)).To(Equal(int32(2))) }) @@ -92,7 +107,7 @@ var _ = Describe("probeCache", func() { go func(i int) { defer wg.Done() <-start - results[i] = c.DoOrCached("k", probe) + results[i] = doOrCached(c, "k", probe) }(i) } @@ -187,9 +202,9 @@ var _ = Describe("probeCache", func() { It("treats different keys independently", func() { c := newProbeCache(time.Minute) var aCalls, bCalls int32 - Expect(c.DoOrCached("a", func() bool { atomic.AddInt32(&aCalls, 1); return true })).To(BeTrue()) - Expect(c.DoOrCached("b", func() bool { atomic.AddInt32(&bCalls, 1); return true })).To(BeTrue()) - Expect(c.DoOrCached("a", func() bool { atomic.AddInt32(&aCalls, 1); return true })).To(BeTrue()) + Expect(doOrCached(c, "a", func() bool { atomic.AddInt32(&aCalls, 1); return true })).To(BeTrue()) + Expect(doOrCached(c, "b", func() bool { atomic.AddInt32(&bCalls, 1); return true })).To(BeTrue()) + Expect(doOrCached(c, "a", func() bool { atomic.AddInt32(&aCalls, 1); return true })).To(BeTrue()) Expect(atomic.LoadInt32(&aCalls)).To(Equal(int32(1))) Expect(atomic.LoadInt32(&bCalls)).To(Equal(int32(1))) @@ -203,9 +218,9 @@ var _ = Describe("probeCache", func() { return true } - Expect(c.DoOrCached("k", probe)).To(BeTrue()) - Expect(c.DoOrCached("k", probe)).To(BeTrue()) - Expect(c.DoOrCached("k", probe)).To(BeTrue()) + Expect(doOrCached(c, "k", probe)).To(BeTrue()) + Expect(doOrCached(c, "k", probe)).To(BeTrue()) + Expect(doOrCached(c, "k", probe)).To(BeTrue()) Expect(atomic.LoadInt32(&calls)).To(Equal(int32(3))) }) @@ -217,9 +232,9 @@ var _ = Describe("probeCache", func() { atomic.AddInt32(&calls, 1) return true } - Expect(c.DoOrCached("k", probe)).To(BeTrue()) + Expect(doOrCached(c, "k", probe)).To(BeTrue()) c.Invalidate("k") - Expect(c.DoOrCached("k", probe)).To(BeTrue()) + Expect(doOrCached(c, "k", probe)).To(BeTrue()) Expect(atomic.LoadInt32(&calls)).To(Equal(int32(2))) }) }) diff --git a/core/services/nodes/reconciler.go b/core/services/nodes/reconciler.go index 2b235ec6dcc3..3a506905997d 100644 --- a/core/services/nodes/reconciler.go +++ b/core/services/nodes/reconciler.go @@ -625,8 +625,14 @@ func (rc *ReplicaReconciler) sweepLeakedInFlight(ctx context.Context) { } seen[m.ID] = struct{}{} if rc.prober.Probe(ctx, m.NodeID, m.WorkerLocalAddress) != ProbeAlive { - // Busy or unreachable. Busy means the counter may well be real; - // unreachable is the reaper's business, not the sweeper's. + // Anything but alive, and the three of them agree on what this + // sweeper should do even though they disagree about everything + // else. Busy: the counter may well be real, so leave it. + // Unreachable: the row is the reaper's business, not the + // sweeper's. Unknown: this frontend has no route and therefore + // observed nothing, which is the one outcome that must never be + // read as evidence. Resetting a counter on any of the three would + // free a reservation a live request is still holding. rc.clearInFlightIdle(m.ID) continue } diff --git a/core/services/nodes/reconciler_prober_test.go b/core/services/nodes/reconciler_prober_test.go index 2bacef554297..a45a13093eba 100644 --- a/core/services/nodes/reconciler_prober_test.go +++ b/core/services/nodes/reconciler_prober_test.go @@ -3,18 +3,42 @@ package nodes import ( + "bytes" "context" "errors" "fmt" + "io" "net" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/mudler/LocalAI/core/services/cluster" grpc "github.com/mudler/LocalAI/pkg/grpc" ) +// refusalFromWorker builds the error a frontend actually holds after a worker +// refused one of its streams. +// +// It goes over the WIRE rather than being handed the sentinel directly: the +// refusal is written with the worker's own writer and read back with the +// frontend's own reader, so a reason the protocol cannot carry, or a code +// mapping that stopped round-tripping, reddens these specs instead of leaving +// them asserting against a value production never produces. The final wrap is +// the shape WorkerDialer.handshake returns for a worker answer, umbrella and +// all: there is none, which is the property under test. +func refusalFromWorker(reason error) error { + GinkgoHelper() + var frame bytes.Buffer + Expect(cluster.WriteStreamRefusal(&frame, reason)).To(Succeed()) + readBack := cluster.ReadStreamReply(&frame) + Expect(readBack).To(MatchError(reason), "the refusal must survive its own round trip") + Expect(readBack).ToNot(MatchError(cluster.ErrNoRoute)) + return fmt.Errorf("opening %q on node %q: %w", "grpc", "node-1", readBack) +} + // proberFactory hands the prober one client, and records what it was asked for. type proberFactory struct { client grpc.Backend @@ -65,13 +89,67 @@ var _ = Describe("the reconciler's gRPC model prober", func() { Expect(probe(&proberFactory{client: &fakeBackendClient{healthy: true}})).To(Equal(ProbeAlive)) }) - It("still answers ProbeUnreachable for a dead backend on a worker it reached", func() { - // The other direction. The new check must not turn the reaper off: a - // backend that answered "unhealthy" over a working transport is a ghost - // and its row should go. + It("still answers ProbeUnreachable for a backend that answered unhealthy", func() { + // One of the two shapes a dead backend takes, and the easy one: the + // process is up enough to answer and reports itself unhealthy over a + // working transport. It is a ghost and its row should go. + // + // This spec used to be named for the property the table below holds, + // which it never tested: a backend process that DIED on a tunnelled + // worker does not answer at all, and what the frontend gets back is the + // worker's refusal, not an unhealthy reply. Expect(probe(&proberFactory{client: &fakeBackendClient{healthy: false}})).To(Equal(ProbeUnreachable)) }) + DescribeTable("answers ProbeUnreachable when the WORKER ITSELF refused the stream", + // The dominant shape of a dead backend since workers stopped listening, + // and the one that produced a permanently unreapable row. The worker is + // healthy, connected and answering; what it answers is that the stream + // cannot be served. That is evidence about the backend, so the reaper + // may act on it. Reported as ProbeUnknown instead, no reap path deleted + // the row, the replica slot never freed, and at the default + // MaxReplicasPerModel=1 the only remaining cleanup was LRU eviction of + // models that were working. + func(reason error) { + Expect(probe(&proberFactory{client: &fakeBackendClient{ + healthy: false, + err: status.Error(codes.Unavailable, "connection error: transport"), + dialErr: refusalFromWorker(reason), + }})).To(Equal(ProbeUnreachable)) + }, + Entry("the worker could not reach the backend process", cluster.ErrStreamTargetUnavailable), + Entry("the worker does not serve gRPC streams at all", cluster.ErrStreamTagUnknown), + Entry("the worker rejected the stored address", cluster.ErrStreamRequestInvalid), + ) + + It("answers ProbeUnknown for a refusal code this frontend does not recognise", func() { + // The other direction, and the boundary of the exemption above. A + // newer worker's vocabulary must not be read as evidence about a + // backend: ReadStreamReply returns an unrecognised code as a plain + // error, WorkerDialer puts the no-route umbrella on it, and the row + // survives. Guessing wrong here costs a retry; guessing wrong the other + // way costs a reaped replica. + unknownCode := fmt.Errorf("reaching node %q: %w: opening %q: tunnel stream refused with unrecognised code %q: %s", + "node-1", cluster.ErrNoRoute, "grpc", "quiesced", "this worker is draining") + Expect(probe(&proberFactory{client: &fakeBackendClient{ + healthy: false, + err: status.Error(codes.Unavailable, "connection error: transport"), + dialErr: unknownCode, + }})).To(Equal(ProbeUnknown)) + }) + + It("answers ProbeUnknown when the tunnel broke while reading the worker's reply", func() { + // The condition a refusal is most easily confused with, kept apart on + // purpose: a read failure is the tunnel breaking, not the worker + // speaking, and it says nothing about the backend. + Expect(probe(&proberFactory{client: &fakeBackendClient{ + healthy: false, + err: status.Error(codes.Unavailable, "connection error: transport"), + dialErr: fmt.Errorf("reaching node %q: %w: opening %q: reading a tunnel stream reply: %w", + "node-1", cluster.ErrNoRoute, "grpc", io.ErrUnexpectedEOF), + }})).To(Equal(ProbeUnknown)) + }) + It("does not report a transport that recovered", func() { // LastDialError is cleared by a successful dial, so a client that // failed once and then reconnected must not keep reading as diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index cecc605555fc..95238d2b021c 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -49,6 +49,14 @@ type BackendNode struct { // plaintext once, and stores only this hash, so a leaked registration token // no longer opens a tunnel for every node whose ID an attacker can read. // + // Stated exactly, because the useful half of the claim is the half that is + // still true. A leaked registration token no longer lets its holder BE a + // worker; it still lets its holder REACH every worker, because + // GET /api/cluster/peer authenticates with the shared cluster token and + // takes its ?id= on trust (see core/http/endpoints/cluster/peer.go). Per + // replica-to-replica credentials are a named phase-3 item, not something + // this column already delivers. + // // Empty means no tunnel credential has been minted for this node yet, which // is what a node registered by an older LocalAI looks like. Such a node // cannot tunnel until it registers again. That is deliberate: the column diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index b861122d60b1..8cdd96782eba 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -1438,13 +1438,13 @@ func (r *SmartRouter) installBackendOnNode(ctx context.Context, node *BackendNod // // An earlier version of this comment justified it by saying the worker // would refuse the resulting empty target as an invalid stream and that - // the refusal would read as the worker answering about its backend. The - // first half is true (see cluster.isWorkerAnswer) and the second is - // not: nothing in this package branches on cluster.ErrNoRoute, and - // `unroutable` treats ANY recorded dial error as unroutable, so such a - // refusal reaches every reap guard as ProbeUnknown and deletes nothing. - // The decision stands on the grounds above, which do not depend on a - // classification the frontend does not currently make. + // the refusal would read as the worker answering about its backend. + // Both halves are true NOW (see cluster.IsWorkerAnswer and + // `unroutable`), and the decision still does not rest on either: a row + // written with an empty address would be reaped a probe cycle later + // with its cause a hop away from where it was created, and the failure + // belongs to this install. Reaping is a recovery, not a substitute for + // refusing to write the bad value. if reply.WorkerLocalAddress == "" { return "", fmt.Errorf("worker %s reported backend %q installed but named no address for the process", node.ID, backendType) } @@ -2067,11 +2067,14 @@ func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir // dials lazily on its first call. // // The client is the RAW factory client rather than buildClientForAddr's, on -// purpose. A health check stages no files, so the staging wrapper buys nothing -// here; and the wrapper hides the transport, because it embeds grpc.Backend and -// so does not carry LastDialError through. Wrapping would leave this function -// unable to tell a dead backend from an unreachable worker, which is the whole -// question it now answers. +// purpose, and the reason is narrower than it used to be. The staging wrapper +// no longer hides the transport: since it became a grpc.WrappedBackend it +// carries LastDialError through, so wrapping would not cost this function the +// answer it needs. What it buys is nothing at all, because a health check +// stages no files, and an unused wrapper on the hottest path in the router is +// an allocation and an indirection per probe. The earlier justification +// ("the wrapper does not carry LastDialError through") is no longer true and is +// recorded here so nobody re-derives the decision from it. func (r *SmartRouter) probeHealth(ctx context.Context, node *BackendNode, addr string) (alive, probed bool) { client, err := r.clientFactory.NewClientForNode(node.ID, addr, false) if err != nil { diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index acca168eefae..6c4399eb4818 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -87,14 +87,25 @@ Frontend replicas record themselves in an `instances` table and open direct link When `LOCALAI_DISTRIBUTED_ADVERTISE_ADDR` is unset, the address is derived: LocalAI asks the kernel which local address routes to PostgreSQL, and pairs it with the port it serves on. Every replica reaches the same database, so that address is on a network they demonstrably share. -That only holds while the database is on **another host**. If PostgreSQL runs on the same host or pod (compose, single-node, a sidecar), the route to it is loopback, and advertising a loopback address would send every peer to itself. LocalAI refuses to guess in that case and logs: +That only holds while the database is on **another host**. If PostgreSQL runs on the same host or pod (compose, single-node, a sidecar), the route to it is loopback, and advertising a loopback address would send every peer to itself. LocalAI refuses to guess in that case. It starts anyway - refusing would break every single-host deployment, which has no peers to be unreachable by - and logs an error at startup: ``` -This replica will not be reachable by its peers: no advertised address +ERROR This replica is not registered in the cluster: no advertised address. Peers cannot reach it, + and any worker whose tunnel lands here will be unroutable from every other replica ``` The replica keeps serving every request that reaches it directly. What it cannot do is be reached by another replica, and on a multi-replica deployment that is worse than it sounds: a **worker whose tunnel lands on this replica is unroutable from every other replica**, because the ownership lookup only accepts an owner that is registered and live. This replica serves that worker fine; the others answer requests for it with `no route from this replica to that worker`. Behind a round-robin load balancer with N replicas, that is (N-1)/N of the traffic for that worker. +Because that symptom looks like a broken **worker** and not a misconfigured **frontend**, the replica repeats itself every five minutes for as long as it runs, and names the workers it is currently costing: + +``` +ERROR This replica is not registered in the cluster and holds worker tunnels: those workers are + unroutable from every OTHER replica, and requests for their models fail there with no route. + The workers are healthy; this replica is invisible workers=[node-a node-b] worker_count=2 +``` + +If you are chasing a worker that answers on one replica and 5xxs on the others, grep the frontend logs for that line before looking at the worker. Until a worker's tunnel lands here the same line appears at `WARN` with no workers named, which is the same misconfiguration not yet costing anything. + A single-replica deployment is unaffected: it has no peers, and it holds every tunnel itself. Set the address explicitly to fix a multi-replica one: ```yaml @@ -104,6 +115,10 @@ environment: The peer link is served at `/api/cluster/peer` and authenticates with `LOCALAI_REGISTRATION_TOKEN`, the same shared secret workers register with. Replicas that disagree about it cannot link. A replica that stops heartbeating for 30 seconds is dropped from the table by the others, along with the worker-connection rows it owned. +{{% notice note %}} +**The peer link has no per-replica credential yet.** It checks the shared registration token and takes the replica id in `?id=` on trust. Anything already holding that token - every worker holds it - can therefore open a peer link, relay through it to every worker tunnel a replica owns, and by declaring another replica's id displace that replica's inbound link. Treat `LOCALAI_REGISTRATION_TOKEN` as a cluster-wide secret with the blast radius of the whole fleet: give it its own value per deployment, do not reuse it elsewhere, and keep `/api/cluster/peer` on a network only your replicas and workers can reach. Per-replica credentials for this route are planned. +{{% /notice %}} + ### Worker tunnels A worker can open one long-lived, multiplexed tunnel to the frontend instead of listening on a port of its own. It dials `GET /api/cluster/connect?id=`, the connection is upgraded to a WebSocket, and every subsequent request the frontend makes to that worker travels as a stream inside it. Nothing dials *into* the worker, so a worker behind NAT, in another Kubernetes cluster or on a laptop needs no inbound port and no reachable address. @@ -175,22 +190,32 @@ These outcomes are kept apart on purpose, because they call for different action | Not the owner | The routing was stale | Resolve the owner again | | Peer unreachable | A replica exists and will not answer | Retry | | No relay path | This replica cannot reach the owner at all | Report; requests here fail until it can | -| The worker refused | The worker answered and said no | Report; the worker is connected | +| The worker refused | The worker answered and said no | Report, and **act on it**; the worker is connected and speaking about its own backend | + +**None of the first four is absence.** A worker's presence is its **heartbeat**, and a route to it is a separate fact that can be false while the worker is registered, heartbeating and serving every request another replica sends it. So the frontend answers "no route", never "this worker is gone", and none of the first four causes a model to be rescheduled or a `node_models` row to be deleted. -**None of them is absence.** A worker's presence is its **heartbeat**, and a route to it is a separate fact that can be false while the worker is registered, heartbeating and serving every request another replica sends it. So the frontend answers "no route", never "this worker is gone", and nothing on this list causes a model to be rescheduled or a `node_models` row to be deleted. +The fifth is different, and deliberately so. A worker that **refuses** a stream has answered, which proves it is connected; what it is refusing is the stream to one backend process on it. That is the ordinary shape of a crashed backend now that workers listen on nothing: the worker's own dial to the process fails and it says so. The frontend treats that as evidence about the backend, so the model's row is reaped and the replica is reloaded, exactly as a dead local backend would be. Without that, a crashed backend on a healthy worker would leave a row nothing could ever delete, its replica slot permanently occupied. A refusal code the frontend does not recognise - a newer worker's vocabulary - is treated as "no route" instead, so a version skew costs a retry rather than a reaped replica. That distinction is the whole point rather than a nicety. A scheduler told that a connected worker has gone away stops its backend and reclaims every model it is running, and the events that produce "no route" are ordinary ones: a frontend replica restarting, an ownership row a moment stale, a worker that has not dialled its tunnel yet. A worker is treated as absent only when its **heartbeat** goes stale, which is a separate mechanism with its own threshold (see `--stale-node-threshold`). -#### There is no frontend-side fallback, and upgrade order matters +#### There is no frontend-side fallback + +`LOCALAI_WORKER_TUNNEL=false` is a **fatal startup error** on this release. It is not a degraded mode and not a rollback switch: the worker refuses to boot and prints why. Nothing else would be honest, because the setting stops the worker dialling its tunnel while **no frontend path dials a worker's advertised address**, and a worker on this release advertises none and listens on no routable interface, so a worker that started with it off would register, heartbeat, be scheduled onto, and fail every request. The rollback is to run the previous release on both sides. + +#### Upgrade the frontends first + +**Upgrade every frontend replica, then restart the workers one at a time.** -`LOCALAI_WORKER_TUNNEL=false` still stops a worker dialling its tunnel, but it no longer has a frontend counterpart: **no frontend path dials a worker's advertised address**, and a worker on this release advertises none and listens on no routable interface. A worker with the tunnel off is a worker nothing can reach. Setting it is not a rollback. The rollback is to run the previous release on both sides. +- **Frontends first (correct).** Old workers keep running, keep heartbeating and keep their `node_models` rows: the new frontend reports them as unroutable rather than as gone, so nothing is rescheduled and nothing is reaped. What fails is requests for models on a worker that has not been restarted yet. That is a real degraded window, but it is bounded by how fast you roll the workers, it heals itself as each one comes back, and no state is lost. + - **What you will see while it lasts:** requests for models on a not-yet-restarted worker fail with "no route to the worker", while `GET /api/nodes` still shows that node healthy and heartbeating and its models still listed. Restart the worker and it clears. Nothing needs fixing; you are watching the window close. +- **Workers first (this fails, do not do it).** An old frontend has no `/api/cluster/connect` route for the worker to dial *and* rejects the new worker's registration outright, because the worker no longer sends an address and the old frontend requires one. A 4xx is a verdict rather than an outage, so the worker reports the reason on the **first** attempt and exits instead of retrying. Every worker you restart is a worker you take out of the fleet until the frontends are upgraded. + - **What you will see if you do it anyway:** each restarted worker exits within a second or two of starting, with `registration failed with status 400: address is required for backend workers: the frontend refused this registration`. The fleet drains one node per restart, and the nodes that are left are the ones you have not touched yet. -That makes upgrade order matter, in one direction only: +A worker that cannot reach its frontend *at the network level* retries with exponential backoff and never gives up, so restarting a worker is all that is needed to close the frontend-first window. A worker whose registration is **rejected** does not retry, which is what makes the wrong order destructive rather than slow. -- **Upgrade the workers first, then the frontends.** A worker on this build dials its tunnel, and a frontend of either version reaches it through that. An old frontend that would have dialled its advertised address no longer gets one, so this order is what keeps the fleet routable throughout. -- **Upgrading the frontends first** leaves every not-yet-restarted worker unroutable until it restarts. Those workers keep running their models and keep heartbeating, and the frontend reports them as unroutable rather than as gone: their `node_models` rows are left alone, nothing is rescheduled, and requests for those models fail loudly with "no route" until the worker reconnects. It is a degraded window, not an eviction, but it is a window, and doing it the other way round has none. +##### Rolling a frontend back requires restarting every worker -A worker that cannot reach its frontend retries with exponential backoff and never gives up, so restarting a worker is all that is needed to close the window. +Registering against an upgraded frontend **clears** a node's `address` and `http_address` columns in the shared database, and re-registration is the only thing that ever writes them back. So a partial rollback does not restore the previous behaviour on its own: the old frontend code reads an empty address for every node that has registered since the upgrade and dials nothing. Roll the frontends back *and then restart every worker* so each one re-registers and repopulates its address. Rolling back is not a frontend-only operation. #### Workers bind nothing routable @@ -427,7 +452,7 @@ local-ai worker \ | `--registration-require-auth` | `LOCALAI_REGISTRATION_REQUIRE_AUTH` | `false` | Refuse to start the HTTP file-transfer server when no registration token is set (it would otherwise fail open) | | `--distributed-require-auth` | `LOCALAI_DISTRIBUTED_REQUIRE_AUTH` | `false` | Umbrella switch implying both `--registration-require-auth` and `--nats-require-auth` | | `--heartbeat-interval` | `LOCALAI_HEARTBEAT_INTERVAL` | `10s` | Interval between heartbeat pings | -| `--worker-tunnel` | `LOCALAI_WORKER_TUNNEL` | `true` | Hold one outbound multiplexed tunnel to the frontend and serve its requests over it, so this worker needs no inbound port (see [Worker tunnels](#worker-tunnels)). Turning it off makes the worker unreachable: the frontend has no path that dials a worker's advertised address. | +| `--worker-tunnel` | `LOCALAI_WORKER_TUNNEL` | `true` | Hold one outbound multiplexed tunnel to the frontend and serve its requests over it, so this worker needs no inbound port (see [Worker tunnels](#worker-tunnels)). Setting it to `false` is a **fatal startup error**, not a degraded mode: the frontend has no path that dials a worker's advertised address, so a worker without its tunnel is a worker nothing can reach. To run without tunnels, run the pre-tunnel release on both the worker and the frontend. | | `--nats-url` | `LOCALAI_NATS_URL` | *(required)* | NATS URL for backend installation and file staging | | `--nats-jwt` | `LOCALAI_NATS_JWT` | *(empty)* | Optional override for the `nats_jwt` returned at registration | | `--nats-user-seed` | `LOCALAI_NATS_USER_SEED` | *(empty)* | Optional override for `nats_user_seed` from registration | diff --git a/pkg/model/connection_evicting_client.go b/pkg/model/connection_evicting_client.go index f957051b2459..a7176358938e 100644 --- a/pkg/model/connection_evicting_client.go +++ b/pkg/model/connection_evicting_client.go @@ -45,7 +45,13 @@ func (c *ConnectionEvictingClient) checkErr(err error) { // stops a model that is loaded and serving, on a worker that is // heartbeating. A locally spawned backend has no custom transport, so this // reports nil and the behaviour there is exactly what it always was. - if dialErr := grpc.LastDialErrorOf(c.Backend); dialErr != nil { + // transportFailure and not LastDialErrorOf: a refusal the WORKER wrote is + // the worker answering that it could not reach the process, which is what a + // crashed backend produces now that a worker listens on nothing. Treating + // that as a transport failure kept a genuinely dead model loaded and + // failing every request, which is the mirror image of the mistake this + // guard exists to prevent. + if dialErr := transportFailure(c.Backend); dialErr != nil { xlog.Warn("Inference failed because the worker could not be reached; keeping the model", "model", c.modelID, "error", dialErr) return diff --git a/pkg/model/loader.go b/pkg/model/loader.go index 57784ae88e3f..12d46910a48a 100644 --- a/pkg/model/loader.go +++ b/pkg/model/loader.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + "github.com/mudler/LocalAI/core/services/cluster" grpc "github.com/mudler/LocalAI/pkg/grpc" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/system" @@ -752,5 +753,20 @@ func transportFailure(client grpc.Backend) error { // over a *FileStagingClient, and an assertion on the outermost type reads // nil for both: they embed grpc.Backend, which does not declare // LastDialError. That is exactly how this guard shipped inert. - return grpc.LastDialErrorOf(client) + dialErr := grpc.LastDialErrorOf(client) + if dialErr == nil { + return nil + } + // A refusal WRITTEN BY THE WORKER is not a transport failure, however much + // it looks like one from here: the tunnel carried the request, the worker + // read it and answered that it could not reach the process the stream + // named. That is the ordinary shape of a crashed backend now that a worker + // listens on nothing, and reporting it as "could not reach the worker" + // pinned the model in this cache forever. cluster.Dial keeps these three + // out of its no-route umbrella for exactly this question; see + // cluster.IsWorkerAnswer. + if cluster.IsWorkerAnswer(dialErr) { + return nil + } + return dialErr } diff --git a/pkg/model/remote_unroutable_internal_test.go b/pkg/model/remote_unroutable_internal_test.go index 06883176f796..eb060458b261 100644 --- a/pkg/model/remote_unroutable_internal_test.go +++ b/pkg/model/remote_unroutable_internal_test.go @@ -3,18 +3,39 @@ package model import ( + "bytes" "context" "errors" + "fmt" "net" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/mudler/LocalAI/core/services/cluster" grpc "github.com/mudler/LocalAI/pkg/grpc" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/system" ) +// refusalFromWorker builds the error the frontend's dialler really returns when +// a worker refuses one of its streams, by writing the refusal with the worker's +// own writer and reading it back with the frontend's own reader. +// +// It matters that this goes over the wire rather than taking the sentinel +// directly. A worker that answers is CONNECTED, so its refusal is not a +// transport failure however much it looks like one from here, and a spec +// asserting against a hand-made value would not notice if the wire stopped +// carrying the distinction. +func refusalFromWorker(reason error) error { + GinkgoHelper() + var frame bytes.Buffer + Expect(cluster.WriteStreamRefusal(&frame, reason)).To(Succeed()) + readBack := cluster.ReadStreamReply(&frame) + Expect(readBack).To(MatchError(reason)) + return fmt.Errorf("opening %q on node %q: %w", "grpc", "node-1", readBack) +} + var _ = Describe("the health check on a remote model whose transport failed", func() { // The fourth site of the same shape as the reconciler, the health monitor // and the router, found by sweeping rather than by being named. @@ -47,6 +68,24 @@ var _ = Describe("the health check on a remote model whose transport failed", fu Expect(stillThere).To(BeTrue()) }) + It("evicts a remote model whose backend the WORKER ITSELF could not reach", func() { + // The shape a crashed backend takes since workers stopped listening: + // the tunnel carried the request, the worker read it and answered that + // nothing is listening on that port. That is the worker speaking about + // its backend, not a transport failure, and reading it as one pinned a + // genuinely dead model in this cache forever, failing every request. + client := grpc.NewClientWithDialer("10.0.0.1:9001", false, nil, false, "", + func(context.Context, string) (net.Conn, error) { + return nil, refusalFromWorker(cluster.ErrStreamTargetUnavailable) + }) + m := NewModelWithClient("refused-model", "10.0.0.1:9001", client) + ml.store.Set("refused-model", m) + + Expect(ml.checkIsLoaded("refused-model")).To(BeNil()) + _, stillThere := ml.store.Get("refused-model") + Expect(stillThere).To(BeFalse()) + }) + It("still evicts a remote model whose worker WAS reached and did not answer", func() { // The other direction, so the new check cannot pass by never evicting. // No custom dialer, so the transport reports nothing and a connection @@ -85,6 +124,36 @@ var _ = Describe("the eviction wrapper on a remote model whose transport failed" "a worker this frontend cannot route to must not have its backend stopped and its rows deleted") }) + It("evicts when the WORKER ITSELF refused the stream to the backend", func() { + // The same rule on the INFERENCE path. A refusal the worker wrote is + // the worker reporting its backend gone, so the model must be evicted + // here exactly as a locally spawned one would be; the guard is for a + // route this frontend lost, which is a different condition. + evicted := 0 + client := newConnectionEvictingClient( + failingDial(refusalFromWorker(cluster.ErrStreamTargetUnavailable)), + "refused-model", func() { evicted++ }) + + _, err := client.Predict(context.Background(), &pb.PredictOptions{}) + Expect(err).To(HaveOccurred()) + Expect(evicted).To(Equal(1)) + }) + + It("does not evict on a refusal code this frontend does not recognise", func() { + // The boundary. An unrecognised code is a newer worker's vocabulary, + // which WorkerDialer reports under the no-route umbrella, and acting on + // it would let a worker upgrade stop models that are running. + evicted := 0 + client := newConnectionEvictingClient( + failingDial(fmt.Errorf("reaching node %q: %w: opening %q: tunnel stream refused with unrecognised code %q: %s", + "node-1", cluster.ErrNoRoute, "grpc", "quiesced", "this worker is draining")), + "remote-model", func() { evicted++ }) + + _, err := client.Predict(context.Background(), &pb.PredictOptions{}) + Expect(err).To(HaveOccurred()) + Expect(evicted).To(BeZero()) + }) + It("still evicts when the worker WAS reached and the connection failed", func() { // The other direction. No custom dialer, so nothing reports a transport // failure and a connection error means what it always meant. From 00a26450ac77576ce8833f956718e81bfc7c324a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 04:48:25 +0000 Subject: [PATCH 064/153] fix(cluster): stop a late request frame reading as the worker's verdict Making a worker's refusal reaping evidence created a defect one layer along, at the producer. The worker refused a ReadStreamRequest failure with ErrStreamRequestInvalid and its own comment said "Includes the deadline above expiring", which was harmless while every refusal reached the frontend as "no route" and became a reap the moment one of them did not. So a request frame that had merely not ARRIVED yet was reported as a non-transient verdict about a backend. It is reachable on the relay path, which carries most production traffic: the worker's header timer starts when the OWNING replica opens the stream, while the frame is written by the DIALLING replica only after the relay's acceptance travels back to it, so a whole peer-link round trip runs inside that window, on a link this design deliberately loads with multi-gigabyte artifacts beside token streams. For a long-deadline caller the endpoint is ConnectionEvictingClient, which stops the model across the fleet. It also falsified the "neither clears on its own" argument that licensed the reap. There is now a fourth refusal, ErrStreamNotServed, for what a worker could not serve for a reason of its OWN. It is deliberately outside IsWorkerAnswer, so it reaches a consumer under the no-route umbrella and reaps nothing, which is the same treatment an unrecognised code already gets. Four producers move onto it: a request frame that timed out (a malformed one stays a verdict, because that is a frontend bug no retry fixes), both SetReadDeadline failures, which are facts about the stream and not about a target nothing has dialled yet, and WriteStreamRefusal's default for a reason nobody classified. classifyServiceFailure keeps ErrStreamTargetUnavailable as its default on purpose: inverting it would make errno enumeration the single point of failure for the reap, and a miss there is a row nothing can ever delete. What it gains is a deny-list of two causes that are provably this worker's own clock or its own context. Also: - The read-site caller-deadline guard in the handshake was unpinned: the existing seam spends the budget before the handshake starts, so only the write could ever fail. A spec whose deadline falls between the request and the reply pins it, and each guard now reddens on its own. - The documented worker-first failure line omitted the JSON error envelope the old frontend returns, so an operator grepping it found nothing. - The peer-link disclosure names the aimable per-session receive window in all four places, and LastDialErrorOf records why a third consumer must go through IsWorkerAnswer rather than roll its own list. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/cluster/peer.go | 16 ++- core/services/cluster/dialer.go | 16 ++- core/services/cluster/dialer_test.go | 56 +++++++++ core/services/cluster/tunnelproto.go | 60 +++++++-- core/services/cluster/tunnelproto_test.go | 57 +++++++-- core/services/nodes/interfaces.go | 18 ++- core/services/nodes/reconciler_prober_test.go | 36 +++++- core/services/nodes/registry.go | 7 +- core/services/worker/tunnel.go | 73 +++++++++-- core/services/worker/tunnel_test.go | 116 ++++++++++++++++++ docs/content/features/distributed-mode.md | 10 +- pkg/grpc/backend.go | 14 +++ 12 files changed, 432 insertions(+), 47 deletions(-) diff --git a/core/http/endpoints/cluster/peer.go b/core/http/endpoints/cluster/peer.go index ca8ef06fd6fd..9dcaea8b38de 100644 --- a/core/http/endpoints/cluster/peer.go +++ b/core/http/endpoints/cluster/peer.go @@ -47,12 +47,16 @@ func PeerHandler(token string, onSession func(peerID string, sess *yamux.Session // token (every worker holds it, and it is the same token that // authenticates registration): it can relay to every worker tunnel this // replica owns, reaching every backend gRPC process and every worker's - // file-transfer server; and by declaring a legitimate replica's id it - // can make SessionStore.Accept evict that replica's inbound link, at - // will. Neither is a new capability in KIND - before workers stopped - // listening, a holder of that token could already dial any worker's - // advertised ports directly - but the token is now the only thing - // between an attacker and the whole fleet's tunnels. + // file-transfer server; by declaring a legitimate replica's id it can + // make SessionStore.Accept evict that replica's inbound link, at will; + // and it can aim the per-session receive window, which PeerLinkConfig + // sizes at roughly 31 GiB of unread data per session, at one replica's + // memory. The first two are not new capabilities in KIND - before + // workers stopped listening, a holder of that token could already dial + // any worker's advertised ports directly - but the token is now the + // only thing between an attacker and the whole fleet's tunnels, and the + // third is a figure written down as sizing guidance that is also a + // budget on a route this open. // // It is deferred rather than patched, because the cheap patch does not // work: checking ?id= against the instances table stops an invented id diff --git a/core/services/cluster/dialer.go b/core/services/cluster/dialer.go index 3460dee3abd3..86c6a8bf0d43 100644 --- a/core/services/cluster/dialer.go +++ b/core/services/cluster/dialer.go @@ -119,10 +119,18 @@ func isAbsenceClaim(err error) bool { // the worker actually wrote. Everything else it returns is a failure to read // one, which is the tunnel breaking rather than the worker speaking. // -// A reply carrying a code this frontend does not recognise is deliberately NOT -// counted here, even though a worker did send it. Classifying it as an answer -// would let a newer worker's vocabulary be read by an older frontend as -// evidence about a backend, and the consequence of guessing wrong in that +// ErrStreamNotServed is deliberately NOT here, even though it is the fourth +// refusal and a worker plainly sent it. It is the code a worker uses to say it +// learned nothing: a request frame that never arrived in time, a stream whose +// deadline could not be armed, anything it could not classify. Those clear on +// their own, so they must reach a consumer as "no route" and cost a retry, not +// a row. Keeping the fourth code out of this predicate is what makes it +// possible for the worker to answer honestly at all. +// +// A reply carrying a code this frontend does not recognise is NOT counted here +// either, for the same reason at the next version boundary. Classifying it as +// an answer would let a newer worker's vocabulary be read by an older frontend +// as evidence about a backend, and the consequence of guessing wrong in that // direction is a reaped replica; guessing wrong the other way costs a retry. // // It is EXPORTED because it is half of a contract, not an implementation diff --git a/core/services/cluster/dialer_test.go b/core/services/cluster/dialer_test.go index e9ce9b243d20..8edd28e5215e 100644 --- a/core/services/cluster/dialer_test.go +++ b/core/services/cluster/dialer_test.go @@ -325,6 +325,62 @@ var _ = Describe("The worker dialer", func() { expectNoRoute(out.err) }) + It("blames the caller's spent budget when it runs out BETWEEN the request and the reply", func() { + // R2: the read-site guard, which its sibling above cannot reach. + // + // deadlinePassed makes the budget already spent when handshake + // starts, so the WRITE is what fails and only the write-site guard + // fires. The guard that matters in production is the other one: a + // caller with a real budget writes its request successfully, the + // worker takes longer than the remainder to answer, and the READ + // ends on the deadline handshakeDeadline armed from that same + // budget. Deleting the read-site guard left the whole cluster suite + // green, which is exactly the "pinned at one of three sites" gap + // this branch has now closed twice. + // + // Deterministic without a sleep: the worker reads the request and + // then never answers, so nothing but the caller's own deadline can + // end the exchange, and the spec waits on the request having been + // SEEN before waiting on the dial. Seeing it is also what proves + // the write succeeded and therefore that this is the read site. + frontend, worker := workerTunnel() + _, err := mine.Attach(ctx, "w1", frontend) + Expect(err).ToNot(HaveOccurred()) + + seen := make(chan servedRequest, 1) + go func() { + defer GinkgoRecover() + stream, err := worker.AcceptStream() + if err != nil { + seen <- servedRequest{err: err} + return + } + tag, target, err := cluster.ReadStreamRequest(stream) + seen <- servedRequest{tag: tag, target: target, stream: stream, err: err} + // Deliberately no reply. The caller's deadline is the only + // thing left that can end this handshake. + }() + + budgeted, cancel := context.WithTimeout(ctx, 750*time.Millisecond) + defer cancel() + d := cluster.NewWorkerDialer(mine, nil) + result := dialAsync(d, budgeted, "w1", cluster.StreamTagGRPC, "127.0.0.1:41000") + + var req servedRequest + Eventually(seen, "10s").Should(Receive(&req)) + Expect(req.err).ToNot(HaveOccurred(), + "the request frame must have been written and read, or this spec is testing the write site") + + var out dialResult + Eventually(result, "10s").Should(Receive(&out)) + Expect(out.err).To(MatchError(context.DeadlineExceeded), + "the caller ran out waiting for a reply; the worker never gave a verdict") + Expect(out.err.Error()).To(ContainSubstring("the caller's own budget ran out")) + Expect(out.err.Error()).To(ContainSubstring(`opening "grpc"`), + "this is the READ site; the write site says \"asking for\"") + expectNoRoute(out.err) + }) + It("reports a broken tunnel held here as itself, not as a routing fact", func() { // ErrNotOwner tells a caller to look for the worker elsewhere. For // a tunnel held right here that sends it back to this replica, and diff --git a/core/services/cluster/tunnelproto.go b/core/services/cluster/tunnelproto.go index c6433459f78c..33daac5db809 100644 --- a/core/services/cluster/tunnelproto.go +++ b/core/services/cluster/tunnelproto.go @@ -59,19 +59,24 @@ const ( replyCodeUnknownTag = "unknown-tag" replyCodeUnavailable = "unavailable" replyCodeBadRequest = "bad-request" + replyCodeNotServed = "not-served" replyPrefixRefused = "err " streamRequestSeparator = " " ) -// The three refusals a worker can send, kept apart on purpose. +// The four refusals a worker can send, kept apart on purpose. // // This is the phase's standing rule in its wire form. An unknown tag is a fact // about what this worker SERVES and will not change until the worker is -// upgraded; an unavailable target is an infrastructure failure that may well -// succeed on the next attempt; a bad request is this frontend's own bug. A -// caller retries the second, gives up on the first, and reports the third. -// Collapsing them into one error would make a frontend retry a stream that can -// never work, or abandon a backend that was merely restarting. +// upgraded; an unavailable target is the worker's own dial to the named +// process failing, which is what a backend that died looks like from inside +// the worker; a bad request is this frontend's own bug. A caller gives up on +// the first, acts on the second, and reports the third. Collapsing them into +// one error would make a frontend retry a stream that can never work, or +// abandon a backend that was merely restarting. +// +// The fourth is the one that says NOTHING, and it exists because the first +// three all say something a consumer now acts on. See ErrStreamNotServed. // // None of them wraps a node-absence error, and none must ever be built over // one: a refusal is proof the worker is CONNECTED and answered. @@ -79,6 +84,33 @@ var ( ErrStreamTagUnknown = errors.New("cluster: the worker does not serve that stream tag") ErrStreamTargetUnavailable = errors.New("cluster: the worker could not reach the local service for that stream") ErrStreamRequestInvalid = errors.New("cluster: the worker rejected the stream request as malformed") + + // ErrStreamNotServed reports that the worker could not serve the stream for + // a reason of ITS OWN, which is not a statement about the backend the + // stream named. + // + // It is the refusal a worker sends when it has learned nothing. The other + // three are evidence a frontend ACTS on: IsWorkerAnswer exempts them from + // the no-route umbrella, and nodes.unroutable then lets a reap guard delete + // the row. This one is deliberately OUTSIDE that predicate, so it reaches a + // consumer as ErrNoRoute and nothing is reaped. + // + // It exists because of a defect this phase created and then found: the + // worker used to answer a request frame that merely arrived LATE with + // ErrStreamRequestInvalid, which was harmless while the frontend treated + // every refusal as "no route", and became a reap the moment a frontend + // started acting on refusals. A delivery timeout clears as soon as the link + // drains; a malformed frame does not. Merging them was safe only while + // nothing downstream could tell them apart, and something downstream now + // can. Anything the worker cannot classify as one of the other three + // belongs here, because an unclassified failure is by definition not a + // verdict about a backend. + // + // A frontend too old to know this code reads it as an unrecognised reply, + // which ReadStreamReply already returns as a plain error and which + // IsWorkerAnswer already declines to count as an answer. So the safe + // behaviour is what a mixed-version deployment gets for free. + ErrStreamNotServed = errors.New("cluster: the worker could not serve that stream, for a reason that is not about the backend") ) // WriteStreamRequest sends the opening frame naming what the stream is for. @@ -127,12 +159,20 @@ func WriteStreamAccepted(w io.Writer) error { // WriteStreamRefusal reports why a stream will not be served. The caller closes // the stream afterwards; this only says why. // -// An unrecognised reason is sent as bad-request with its text attached rather +// An unrecognised reason is sent as NOT-SERVED with its text attached rather // than being dropped, because a refusal a frontend cannot read is // indistinguishable from a worker that hung up, and those are different // problems. +// +// The default is not-served and not bad-request, and the difference is the +// whole point of the fourth code. The other three are evidence a frontend acts +// on, up to and including deleting a model's row; an error that reached here +// without carrying one of them is by construction an error nobody classified, +// and an unclassified failure must never become a verdict by default. This +// default used to be bad-request, which was harmless while no consumer +// distinguished the codes and became a reap-by-omission when one did. func WriteStreamRefusal(w io.Writer, reason error) error { - code := replyCodeBadRequest + code := replyCodeNotServed switch { case errors.Is(reason, ErrStreamTagUnknown): code = replyCodeUnknownTag @@ -140,6 +180,8 @@ func WriteStreamRefusal(w io.Writer, reason error) error { code = replyCodeUnavailable case errors.Is(reason, ErrStreamRequestInvalid): code = replyCodeBadRequest + case errors.Is(reason, ErrStreamNotServed): + code = replyCodeNotServed } text := "" @@ -186,6 +228,8 @@ func ReadStreamReply(r io.Reader) error { return fmt.Errorf("%w: %s", ErrStreamTargetUnavailable, text) case replyCodeBadRequest: return fmt.Errorf("%w: %s", ErrStreamRequestInvalid, text) + case replyCodeNotServed: + return fmt.Errorf("%w: %s", ErrStreamNotServed, text) default: // A code from a newer worker. Reported as an error carrying the code // rather than mapped onto the nearest known one, so a frontend does not diff --git a/core/services/cluster/tunnelproto_test.go b/core/services/cluster/tunnelproto_test.go index 2664d9d509f5..2c7e8d5d061f 100644 --- a/core/services/cluster/tunnelproto_test.go +++ b/core/services/cluster/tunnelproto_test.go @@ -3,6 +3,7 @@ package cluster_test import ( "bytes" "encoding/binary" + "errors" "io" "strings" "unicode/utf8" @@ -100,27 +101,66 @@ var _ = Describe("Worker tunnel stream framing", func() { Expect(cluster.ReadStreamReply(&buf)).To(Succeed()) }) - DescribeTable("keeps the three refusals apart", + DescribeTable("keeps the four refusals apart", func(sent error, others []error) { var buf bytes.Buffer Expect(cluster.WriteStreamRefusal(&buf, sent)).To(Succeed()) got := cluster.ReadStreamReply(&buf) Expect(got).To(MatchError(sent)) - // The whole point. A caller gives up on an unknown tag, retries - // an unavailable target, and reports a bad request as its own - // bug; collapsing any pair makes one of those wrong. + // The whole point. A caller gives up on an unknown tag, acts on + // an unavailable target, reports a bad request as its own bug, + // and learns NOTHING from a not-served; collapsing any pair + // makes one of those wrong, and three of the four pairs end in + // a reaped replica. for _, other := range others { Expect(got).ToNot(MatchError(other)) } }, Entry("unknown tag", cluster.ErrStreamTagUnknown, - []error{cluster.ErrStreamTargetUnavailable, cluster.ErrStreamRequestInvalid}), + []error{cluster.ErrStreamTargetUnavailable, cluster.ErrStreamRequestInvalid, cluster.ErrStreamNotServed}), Entry("unavailable target", cluster.ErrStreamTargetUnavailable, - []error{cluster.ErrStreamTagUnknown, cluster.ErrStreamRequestInvalid}), + []error{cluster.ErrStreamTagUnknown, cluster.ErrStreamRequestInvalid, cluster.ErrStreamNotServed}), Entry("invalid request", cluster.ErrStreamRequestInvalid, - []error{cluster.ErrStreamTagUnknown, cluster.ErrStreamTargetUnavailable}), + []error{cluster.ErrStreamTagUnknown, cluster.ErrStreamTargetUnavailable, cluster.ErrStreamNotServed}), + Entry("nothing learned", cluster.ErrStreamNotServed, + []error{cluster.ErrStreamTagUnknown, cluster.ErrStreamTargetUnavailable, cluster.ErrStreamRequestInvalid}), ) + It("sends a reason it cannot classify as not-served, never as a verdict", func() { + // The default, and it is a safety default rather than a formality. + // Three of the four codes are evidence a frontend now ACTS on, up + // to deleting a model's row; an error that reached WriteStreamRefusal + // without carrying a sentinel is by construction one nobody + // classified. This default used to be bad-request, which was + // harmless while no consumer distinguished the codes and became a + // reap-by-omission the moment one did. + var buf bytes.Buffer + Expect(cluster.WriteStreamRefusal(&buf, errors.New("something nobody thought about"))).To(Succeed()) + got := cluster.ReadStreamReply(&buf) + Expect(got).To(MatchError(cluster.ErrStreamNotServed)) + Expect(got).ToNot(MatchError(cluster.ErrStreamRequestInvalid)) + Expect(cluster.IsWorkerAnswer(got)).To(BeFalse(), + "an unclassified failure must never become the worker's verdict about a backend") + Expect(got.Error()).To(ContainSubstring("something nobody thought about")) + }) + + It("keeps not-served OUT of the answers a frontend acts on", func() { + // The predicate is the seam between what the worker says and what + // the frontend does with it. The other three are exempted from the + // no-route umbrella so a crashed backend can be reaped; this one + // must not be, or every transient worker-side failure reaps. + var buf bytes.Buffer + Expect(cluster.WriteStreamRefusal(&buf, cluster.ErrStreamNotServed)).To(Succeed()) + Expect(cluster.IsWorkerAnswer(cluster.ReadStreamReply(&buf))).To(BeFalse()) + + for _, verdict := range []error{cluster.ErrStreamTagUnknown, cluster.ErrStreamTargetUnavailable, cluster.ErrStreamRequestInvalid} { + buf.Reset() + Expect(cluster.WriteStreamRefusal(&buf, verdict)).To(Succeed()) + Expect(cluster.IsWorkerAnswer(cluster.ReadStreamReply(&buf))).To(BeTrue(), + "a verdict that stopped being an answer makes a dead backend unreapable") + } + }) + It("carries the reason text to the far side", func() { var buf bytes.Buffer Expect(cluster.WriteStreamRefusal(&buf, wrapReason(cluster.ErrStreamTagUnknown, "no-such-tag"))).To(Succeed()) @@ -139,6 +179,8 @@ var _ = Describe("Worker tunnel stream framing", func() { Expect(got).ToNot(MatchError(cluster.ErrStreamTagUnknown)) Expect(got).ToNot(MatchError(cluster.ErrStreamTargetUnavailable)) Expect(got).ToNot(MatchError(cluster.ErrStreamRequestInvalid)) + Expect(got).ToNot(MatchError(cluster.ErrStreamNotServed)) + Expect(cluster.IsWorkerAnswer(got)).To(BeFalse()) }) It("reports a failure to READ the reply as itself, never as a refusal", func() { @@ -150,6 +192,7 @@ var _ = Describe("Worker tunnel stream framing", func() { Expect(got).ToNot(MatchError(cluster.ErrStreamTagUnknown)) Expect(got).ToNot(MatchError(cluster.ErrStreamTargetUnavailable)) Expect(got).ToNot(MatchError(cluster.ErrStreamRequestInvalid)) + Expect(got).ToNot(MatchError(cluster.ErrStreamNotServed)) }) It("truncates an over-long reason on a rune boundary, keeping it decodable", func() { diff --git a/core/services/nodes/interfaces.go b/core/services/nodes/interfaces.go index a3ceb0b9e242..b68a0dd63c00 100644 --- a/core/services/nodes/interfaces.go +++ b/core/services/nodes/interfaces.go @@ -242,9 +242,21 @@ var ErrNoWorkerDialer = fmt.Errorf("%w: no worker tunnel dialer is configured", // EVERY replica for as long as it exists, and reaping it converges: the model // is reloaded somewhere that works and re-registers a usable address. The // condition the phase refuses to reap on is a TRANSIENT one, and none of these -// is transient. A reply code this frontend does not recognise is deliberately -// not in the set (see cluster.IsWorkerAnswer), so a newer worker's vocabulary -// reaches an older frontend as "no route" and costs a retry rather than a row. +// is transient. +// +// That last sentence is a claim about the WORKER, not about this file, and it +// held only after the worker stopped answering a request frame that merely +// arrived late with ErrStreamRequestInvalid. It did, and the frontend's half of +// that contract is that a transient condition arrives as the fourth code: +// cluster.ErrStreamNotServed is not in cluster.IsWorkerAnswer, so it reaches +// here under the no-route umbrella and reaps nothing. If a worker ever starts +// sending one of the three for something that clears on its own, this comment +// becomes false and a live model gets evicted; the guard against that is at the +// worker, in Tunnel.accept and classifyServiceFailure, and it is stated there. +// +// A reply code this frontend does not recognise is deliberately not in the set +// either (see cluster.IsWorkerAnswer), so a newer worker's vocabulary reaches +// an older frontend as "no route" and costs a retry rather than a row. func unroutable(client grpc.Backend) error { // LastDialErrorOf and not a type assertion: the assertion could not see // past a decorator, and SmartRouter hands every routed client out wrapped. diff --git a/core/services/nodes/reconciler_prober_test.go b/core/services/nodes/reconciler_prober_test.go index a45a13093eba..b8cb80750987 100644 --- a/core/services/nodes/reconciler_prober_test.go +++ b/core/services/nodes/reconciler_prober_test.go @@ -26,9 +26,16 @@ import ( // refusal is written with the worker's own writer and read back with the // frontend's own reader, so a reason the protocol cannot carry, or a code // mapping that stopped round-tripping, reddens these specs instead of leaving -// them asserting against a value production never produces. The final wrap is -// the shape WorkerDialer.handshake returns for a worker answer, umbrella and -// all: there is none, which is the property under test. +// them asserting against a value production never produces. +// +// The wrap is chosen by cluster.IsWorkerAnswer, which is what +// WorkerDialer.handshake does, so the spec exercises the real branch rather +// than a transcription of it. That is not circular: the helper decides the +// SHAPE of the error and the table below states the OUTCOME independently, so +// moving a sentinel into or out of the predicate reddens the table. In +// particular, adding ErrStreamNotServed to the predicate would strip its +// umbrella here and turn its ProbeUnknown entry red, which is the property that +// keeps a transient worker-side failure from reaping a live model. func refusalFromWorker(reason error) error { GinkgoHelper() var frame bytes.Buffer @@ -36,7 +43,10 @@ func refusalFromWorker(reason error) error { readBack := cluster.ReadStreamReply(&frame) Expect(readBack).To(MatchError(reason), "the refusal must survive its own round trip") Expect(readBack).ToNot(MatchError(cluster.ErrNoRoute)) - return fmt.Errorf("opening %q on node %q: %w", "grpc", "node-1", readBack) + if cluster.IsWorkerAnswer(readBack) { + return fmt.Errorf("opening %q on node %q: %w", "grpc", "node-1", readBack) + } + return fmt.Errorf("reaching node %q: %w: opening %q: %w", "node-1", cluster.ErrNoRoute, "grpc", readBack) } // proberFactory hands the prober one client, and records what it was asked for. @@ -122,6 +132,24 @@ var _ = Describe("the reconciler's gRPC model prober", func() { Entry("the worker rejected the stored address", cluster.ErrStreamRequestInvalid), ) + It("answers ProbeUnknown when the worker refused but said it learned nothing", func() { + // The fourth refusal, and the boundary that keeps the three above safe + // to act on. A worker answers with it for its OWN transient conditions: + // a request frame that never arrived in time, a stream whose deadline + // would not arm, a local dial that ended on the session going away. + // Those clear on a reconnect, so acting on them would convert + // peer-link congestion into a reaped row, and on the inference path + // into a model stopped across the fleet. + // + // This is not hypothetical: the header-timeout case USED to arrive as + // ErrStreamRequestInvalid, which the table above reaps on. + Expect(probe(&proberFactory{client: &fakeBackendClient{ + healthy: false, + err: status.Error(codes.Unavailable, "connection error: transport"), + dialErr: refusalFromWorker(cluster.ErrStreamNotServed), + }})).To(Equal(ProbeUnknown)) + }) + It("answers ProbeUnknown for a refusal code this frontend does not recognise", func() { // The other direction, and the boundary of the exemption above. A // newer worker's vocabulary must not be read as evidence about a diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index 95238d2b021c..a3ba4eff74a6 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -53,9 +53,10 @@ type BackendNode struct { // still true. A leaked registration token no longer lets its holder BE a // worker; it still lets its holder REACH every worker, because // GET /api/cluster/peer authenticates with the shared cluster token and - // takes its ?id= on trust (see core/http/endpoints/cluster/peer.go). Per - // replica-to-replica credentials are a named phase-3 item, not something - // this column already delivers. + // takes its ?id= on trust (see core/http/endpoints/cluster/peer.go), which + // also puts the ~31 GiB per-session peer receive window inside reach of + // anything holding it. Per replica-to-replica credentials are a named + // phase-3 item, not something this column already delivers. // // Empty means no tunnel credential has been minted for this node yet, which // is what a node registered by an older LocalAI looks like. Such a node diff --git a/core/services/worker/tunnel.go b/core/services/worker/tunnel.go index 0ff28ea85a6f..cfd33a770deb 100644 --- a/core/services/worker/tunnel.go +++ b/core/services/worker/tunnel.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "net/url" + "os" "strconv" "strings" "sync" @@ -335,17 +336,32 @@ func (t *Tunnel) accept(ctx context.Context, stream net.Conn) (net.Conn, bool) { // Deliberately time.Now and not t.now: this is an I/O deadline, and the // clock seam exists only to measure how long a session lasted. if err := stream.SetReadDeadline(time.Now().Add(t.headerTimeout)); err != nil { - // Nothing is readable on a stream whose deadline cannot be set, so this - // is reported as an infrastructure failure rather than pushed past. - t.refuse(stream, fmt.Errorf("%w: arming the request deadline: %v", cluster.ErrStreamTargetUnavailable, err)) + // NotServed and not TargetUnavailable: this is a fact about the STREAM, + // which would not take a deadline, and no local service has been named + // yet, let alone dialled. Reporting it as an unreachable target would + // tell the frontend a backend it has not asked about is gone. + t.refuse(stream, fmt.Errorf("%w: arming the request deadline: %v", cluster.ErrStreamNotServed, err)) return nil, false } tag, target, err := cluster.ReadStreamRequest(stream) if err != nil { - // Includes the deadline above expiring. Both are "this stream never - // told me what it wanted", which is the frontend's problem to fix, not - // something a retry against this worker resolves. + // The two causes are SEPARATED here, and merging them was a real + // defect. A malformed frame is the frontend's own bug and does not + // clear on its own, so it stays a verdict the frontend acts on. The + // deadline above expiring is a frame that has not ARRIVED yet, which + // clears the moment the link drains; on the relay path the worker's + // timer starts when the OWNING replica opens the stream, while the + // frame is written by the DIALLING replica only after the relay's + // acceptance has travelled back to it, so a whole peer-link round trip + // runs inside this window, on a link this design deliberately loads + // with multi-gigabyte artifacts. Reported as a malformed request it + // became reaping evidence, and for a long-deadline caller that is a + // model evicted across the fleet by nothing but congestion. + if isReadTimeout(err) { + t.refuse(stream, fmt.Errorf("%w: %v", cluster.ErrStreamNotServed, err)) + return nil, false + } t.refuse(stream, fmt.Errorf("%w: %v", cluster.ErrStreamRequestInvalid, err)) return nil, false } @@ -365,7 +381,9 @@ func (t *Tunnel) accept(ctx context.Context, stream net.Conn) (net.Conn, bool) { // its own deadlines, and one left armed here would abort a long inference // stream in the middle. if err := stream.SetReadDeadline(time.Time{}); err != nil { - t.refuse(stream, fmt.Errorf("%w: clearing the request deadline: %v", cluster.ErrStreamTargetUnavailable, err)) + // NotServed for the same reason as arming it: the stream is what + // failed, and this worker has said nothing about the target. + t.refuse(stream, fmt.Errorf("%w: clearing the request deadline: %v", cluster.ErrStreamNotServed, err)) return nil, false } @@ -397,16 +415,51 @@ func (t *Tunnel) accept(ctx context.Context, stream net.Conn) (net.Conn, bool) { // retry something that can never work; reporting the second as the first makes // it give up on a backend that is merely starting. // -// Anything unclassified is infrastructure, because that is what an unadorned -// dial failure is, and it must never become the unknown-tag refusal: a tag this -// worker serves does not stop being served because one dial failed. +// The default is TargetUnavailable and stays that way, which is the deliberate +// half of this function now that the frontend acts on that code. A dial to the +// named process that came back with anything is the closest thing to evidence +// this worker can produce, and the direction of a mis-classification decides +// which mistake is made: defaulting the other way would mean a genuinely dead +// backend whose errno nobody enumerated becomes a permanently unreapable row, +// which is the exact defect this phase spent a round removing. So the +// exemptions below are a DENY-list of causes that are provably not about the +// target, not an allow-list of causes that are. +// +// The two exempted are the caller's context ending and a read/write deadline +// firing. Neither is the target answering: the context here is the SESSION's, +// so it ends when this worker's tunnel is torn down, and a deadline is this +// process's own timer. Both clear on a reconnect. They are exempted rather +// than argued away as unreachable because "unreachable" was the argument that +// made the request-frame merge look safe. +// +// It must never become the unknown-tag refusal either: a tag this worker serves +// does not stop being served because one dial failed. func classifyServiceFailure(err error) error { if errors.Is(err, cluster.ErrStreamRequestInvalid) { return err } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || isReadTimeout(err) { + return fmt.Errorf("%w: %v", cluster.ErrStreamNotServed, err) + } return fmt.Errorf("%w: %v", cluster.ErrStreamTargetUnavailable, err) } +// isReadTimeout reports whether err is an I/O deadline firing rather than a +// peer saying anything. +// +// net.Error's Timeout is asked as well as os.ErrDeadlineExceeded because the +// two are not the same set: a yamux stream returns its own timeout value from +// a Read whose deadline expired, and a net.OpError over a socket returns +// os.ErrDeadlineExceeded. Missing either would put a timeout back on the +// verdict path, which is the defect this predicate exists to keep closed. +func isReadTimeout(err error) bool { + if errors.Is(err, os.ErrDeadlineExceeded) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + // refuse reports why a stream will not be served and then ENDS it. // // The close is the part that matters and it is not optional. A worker that says diff --git a/core/services/worker/tunnel_test.go b/core/services/worker/tunnel_test.go index 8a59f686fef2..743cccf726d3 100644 --- a/core/services/worker/tunnel_test.go +++ b/core/services/worker/tunnel_test.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "sync/atomic" + "syscall" "time" "github.com/gorilla/websocket" @@ -349,6 +350,121 @@ var _ = Describe("Worker tunnel client", func() { }) Eventually(ended, "10s").Should(Receive(BeNil())) }) + + It("says it learned NOTHING when the request frame never arrived in time", func() { + // The producer side of the phase's worst self-inflicted defect. + // + // This refusal used to be ErrStreamRequestInvalid, merged with a + // genuinely malformed frame on the grounds that both are "this + // stream never told me what it wanted". That was safe only while + // the frontend treated every refusal as "no route". Once + // nodes.unroutable started exempting worker answers so a crashed + // backend could be reaped, this became reaping evidence for a frame + // that had merely not ARRIVED yet. + // + // It is reachable: on the relay path the worker's header timer + // starts when the OWNING replica opens the stream, while the frame + // is written by the DIALLING replica only after the relay + // acceptance travels back, so a peer-link round trip runs inside + // this window on a link that also carries multi-gigabyte artifacts. + // For a long-deadline caller the endpoint is + // ConnectionEvictingClient, which stops the model across the fleet. + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services[cluster.StreamTagGRPC] = dialLocalTCP + c.headerTimeout = 50 * time.Millisecond + }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + // Nothing is written, so only the header timer can end this. + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + var got error + Eventually(reply, "10s").Should(Receive(&got)) + + Expect(got).To(MatchError(cluster.ErrStreamNotServed)) + // The three assertions that make this bite. Each of the other + // sentinels is evidence the frontend acts on, and the predicate is + // the single place the two lists are kept identical. + Expect(got).ToNot(MatchError(cluster.ErrStreamRequestInvalid), + "a frame that arrived late is not a malformed frame, and this one reaps") + Expect(got).ToNot(MatchError(cluster.ErrStreamTargetUnavailable)) + Expect(cluster.IsWorkerAnswer(got)).To(BeFalse(), + "a timeout must reach a reap guard as no-route, never as the worker's verdict") + }) + + It("still calls a MALFORMED request frame malformed, which is a verdict", func() { + // The other direction. Separating the timeout out must not turn the + // verdict off: a frontend that writes a frame this worker cannot + // parse has a bug that no retry fixes, and the refusal has to keep + // saying so. + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services[cluster.StreamTagGRPC] = dialLocalTCP + c.headerTimeout = time.Minute + }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + // A frame whose declared length exceeds what the reader will take, + // so the failure is the frame's shape and not the clock. + Expect(binary.Write(stream, binary.BigEndian, uint16(60000))).To(Succeed()) + + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + var got error + Eventually(reply, "10s").Should(Receive(&got)) + + Expect(got).To(MatchError(cluster.ErrStreamRequestInvalid)) + Expect(got).ToNot(MatchError(cluster.ErrStreamNotServed)) + Expect(cluster.IsWorkerAnswer(got)).To(BeTrue()) + }) + + It("says it learned nothing when the local dial ended on the session going away", func() { + // classifyServiceFailure's deny-list. The default there is + // TargetUnavailable and stays that way, because a mis-classified + // dial failure must fall towards "reapable" rather than towards a + // row nothing can ever delete. What is exempted is the pair of + // causes that are provably not the target answering: this worker's + // own session context ending, and its own I/O deadline firing. + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services[cluster.StreamTagGRPC] = func(ctx context.Context, _ string) (net.Conn, error) { + return nil, fmt.Errorf("dialing the backend: %w", context.Canceled) + } + }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagGRPC, "127.0.0.1:41000")).To(Succeed()) + + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + var got error + Eventually(reply, "10s").Should(Receive(&got)) + Expect(got).To(MatchError(cluster.ErrStreamNotServed)) + Expect(cluster.IsWorkerAnswer(got)).To(BeFalse()) + }) + + It("still reports a refused local dial as an unavailable target, which reaps", func() { + // The other direction for the deny-list: the ordinary shape of a + // crashed backend must keep producing the code the reap guards act + // on, or the ghost rows come back. + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services[cluster.StreamTagGRPC] = func(context.Context, string) (net.Conn, error) { + return nil, fmt.Errorf("dial tcp 127.0.0.1:41000: connect: %w", syscall.ECONNREFUSED) + } + }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagGRPC, "127.0.0.1:41000")).To(Succeed()) + + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + var got error + Eventually(reply, "10s").Should(Receive(&got)) + Expect(got).To(MatchError(cluster.ErrStreamTargetUnavailable)) + Expect(cluster.IsWorkerAnswer(got)).To(BeTrue()) + }) }) Describe("surviving a bad stream", func() { diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 6c4399eb4818..aa2443c82cd6 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -116,7 +116,7 @@ environment: The peer link is served at `/api/cluster/peer` and authenticates with `LOCALAI_REGISTRATION_TOKEN`, the same shared secret workers register with. Replicas that disagree about it cannot link. A replica that stops heartbeating for 30 seconds is dropped from the table by the others, along with the worker-connection rows it owned. {{% notice note %}} -**The peer link has no per-replica credential yet.** It checks the shared registration token and takes the replica id in `?id=` on trust. Anything already holding that token - every worker holds it - can therefore open a peer link, relay through it to every worker tunnel a replica owns, and by declaring another replica's id displace that replica's inbound link. Treat `LOCALAI_REGISTRATION_TOKEN` as a cluster-wide secret with the blast radius of the whole fleet: give it its own value per deployment, do not reuse it elsewhere, and keep `/api/cluster/peer` on a network only your replicas and workers can reach. Per-replica credentials for this route are planned. +**The peer link has no per-replica credential yet.** It checks the shared registration token and takes the replica id in `?id=` on trust. Anything already holding that token - every worker holds it - can therefore open a peer link, relay through it to every worker tunnel a replica owns, by declaring another replica's id displace that replica's inbound link, and hold sessions open against the per-session receive window, which the peer-link code sizes at roughly 31 GiB of unread data per session and which on this route is also a memory budget an attacker can point at one replica. Treat `LOCALAI_REGISTRATION_TOKEN` as a cluster-wide secret with the blast radius of the whole fleet: give it its own value per deployment, do not reuse it elsewhere, and keep `/api/cluster/peer` on a network only your replicas and workers can reach. Per-replica credentials for this route are planned. {{% /notice %}} ### Worker tunnels @@ -209,7 +209,13 @@ That distinction is the whole point rather than a nicety. A scheduler told that - **Frontends first (correct).** Old workers keep running, keep heartbeating and keep their `node_models` rows: the new frontend reports them as unroutable rather than as gone, so nothing is rescheduled and nothing is reaped. What fails is requests for models on a worker that has not been restarted yet. That is a real degraded window, but it is bounded by how fast you roll the workers, it heals itself as each one comes back, and no state is lost. - **What you will see while it lasts:** requests for models on a not-yet-restarted worker fail with "no route to the worker", while `GET /api/nodes` still shows that node healthy and heartbeating and its models still listed. Restart the worker and it clears. Nothing needs fixing; you are watching the window close. - **Workers first (this fails, do not do it).** An old frontend has no `/api/cluster/connect` route for the worker to dial *and* rejects the new worker's registration outright, because the worker no longer sends an address and the old frontend requires one. A 4xx is a verdict rather than an outage, so the worker reports the reason on the **first** attempt and exits instead of retrying. Every worker you restart is a worker you take out of the fleet until the frontends are upgraded. - - **What you will see if you do it anyway:** each restarted worker exits within a second or two of starting, with `registration failed with status 400: address is required for backend workers: the frontend refused this registration`. The fleet drains one node per restart, and the nodes that are left are the ones you have not touched yet. + - **What you will see if you do it anyway:** each restarted worker exits within a second or two of starting, with + + ``` + registration failed with status 400: {"error":{"code":400,"message":"address is required for backend workers","type":"node_error"}}: the frontend refused this registration + ``` + + The fleet drains one node per restart, and the nodes that are left are the ones you have not touched yet. Grep for `address is required for backend workers` if your log collector reflows the line. A worker that cannot reach its frontend *at the network level* retries with exponential backoff and never gives up, so restarting a worker is all that is needed to close the frontend-first window. A worker whose registration is **rejected** does not retry, which is what makes the wrong order destructive rather than slow. diff --git a/pkg/grpc/backend.go b/pkg/grpc/backend.go index 8448afcb92ef..fd032bdbceea 100644 --- a/pkg/grpc/backend.go +++ b/pkg/grpc/backend.go @@ -133,6 +133,20 @@ const maxBackendUnwrapDepth = 16 // pkg/model is an *InFlightTrackingClient over a *FileStagingClient over the // real one, so both callers were asking a wrapper that had no answer and // reading nil as "the transport was fine". +// +// WHAT IT ANSWERS IS NOT "was this the transport's fault". It answers "what did +// the dialler last return", and in distributed mode some of those values are a +// WORKER'S OWN REFUSAL, which means the tunnel worked and the worker spoke. +// Telling those apart is cluster.IsWorkerAnswer, and the two production callers +// (nodes.unroutable, model.transportFailure) both go through it. A new caller +// that matches on sentinels of its own would be re-creating the collapse this +// phase spent two rounds removing: the reap guards and the dialler would stop +// agreeing on which errors are evidence. +// +// Nothing structural prevents that, unlike the WrappedBackend rule in +// hack/lint/ which makes decorator transparency impossible to forget. With two +// callers, both funnelling through one predicate, a ruleguard rule is not worth +// its false positives; if a third appears, it is. Recorded as a phase-3 note. func LastDialErrorOf(b Backend) error { for range maxBackendUnwrapDepth { if b == nil { From e2094922039f896900e0e51c7d8d02675f35f182 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 06:10:26 +0000 Subject: [PATCH 065/153] fix(cluster): keep the refusal vocabulary in one table The worker re-classified a failure a local service had already classified. classifyServiceFailure preserved exactly one of the four refusal codes, which was faithful to its own comment for as long as there was one worth keeping; once ErrStreamNotServed existed, a service returning the code whose whole job is to say "I learned nothing" had it promoted to ErrStreamTargetUnavailable, which every reap guard acts on. ErrStreamTagUnknown was promoted too, and cost nothing only because both sides of that one reap. No in-tree service produces either, which is the same "unreachable, therefore safe" argument that let the request-frame merge survive a whole phase, and LocalService is exported. The cause was a fifth site enumerating the vocabulary by hand, so the fix is one table. streamRefusals pairs each sentinel with its wire code and with whether a frontend may act on it as evidence about a backend, and the writer, the reader, IsWorkerAnswer and the new IsStreamRefusal all read it. A fifth code is now taught to every one of them at once. The codes are also pinned against literals written out in a spec, the way this branch already pinned the NATS vocabulary. The round-trip table cannot see a rename, because a rename moves the writer and the reader together; an unrecognised code is deliberately not the worker's answer, so renaming "unavailable" would turn every crashed backend on a tunnelled worker into a row nothing can ever reap, silently and with the suite green. Three comments the previous fix falsified, corrected: - tunnelHeaderTimeout still said the window bounds only framing the frontend writes immediately after opening the stream. That is true on the direct path and false on the relay path, and it was the argument for treating an expiry as the frontend's fault. - classifyServiceFailure's deny-list is three causes, not two: on a dial error net.Error.Timeout also covers ETIMEDOUT and EAGAIN. Both are kept deliberately, because reaping a wedged or resource-starved backend is the eviction this phase exists to prevent, and ECONNREFUSED still reaps. isReadTimeout is renamed reportsTimeout, which is what it asks. - The operator table named three refusals and said a refusal is acted on. It now lists four, with when each is sent and whether the row is reaped. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/dialer.go | 13 +- core/services/cluster/tunnelproto.go | 84 +++++++---- .../services/cluster/tunnelproto_wire_test.go | 130 ++++++++++++++++++ core/services/worker/tunnel.go | 93 +++++++++---- core/services/worker/tunnel_test.go | 42 ++++++ docs/content/features/distributed-mode.md | 17 ++- 6 files changed, 328 insertions(+), 51 deletions(-) create mode 100644 core/services/cluster/tunnelproto_wire_test.go diff --git a/core/services/cluster/dialer.go b/core/services/cluster/dialer.go index 86c6a8bf0d43..cba17d206f03 100644 --- a/core/services/cluster/dialer.go +++ b/core/services/cluster/dialer.go @@ -143,9 +143,16 @@ func isAbsenceClaim(err error) bool { // model.transportFailure, whose job is to answer "did this call reach a // backend?" and for whom a refusal means it did. func IsWorkerAnswer(err error) bool { - return errors.Is(err, ErrStreamTagUnknown) || - errors.Is(err, ErrStreamTargetUnavailable) || - errors.Is(err, ErrStreamRequestInvalid) + // Read off the vocabulary table rather than enumerated here, so this + // predicate and the wire codes cannot disagree about which refusals exist. + // Enumerating them by hand is what let a fifth site promote the fourth code + // into a verdict; see streamRefusals. + for _, r := range streamRefusals { + if r.evidence && errors.Is(err, r.sentinel) { + return true + } + } + return false } // dialHandshakeTimeout bounds the request/reply exchange that opens every diff --git a/core/services/cluster/tunnelproto.go b/core/services/cluster/tunnelproto.go index 33daac5db809..2e404b247c2a 100644 --- a/core/services/cluster/tunnelproto.go +++ b/core/services/cluster/tunnelproto.go @@ -113,6 +113,53 @@ var ( ErrStreamNotServed = errors.New("cluster: the worker could not serve that stream, for a reason that is not about the backend") ) +// streamRefusals is the whole refusal vocabulary, in ONE table. +// +// The writer, the reader, IsWorkerAnswer and IsStreamRefusal all read it, so a +// fifth refusal cannot be taught to some of them and forgotten in the others. +// That is not hypothetical tidiness: the fourth code was added to the writer, +// the reader and the consumer predicate, and a fifth place that enumerated the +// sentinels by hand (the worker's classifyServiceFailure) silently PROMOTED it +// to a reaping verdict. One table is what makes the next such site impossible +// to write. +// +// ORDER MATTERS for the writer: matching is by errors.Is and the first hit +// wins, so a reason wrapping two sentinels resolves the same way every time. +// +// evidence is the half a consumer acts on, and it is a property OF THE CODE +// rather than of the consumer asking. Three of the four are statements about a +// backend that a frontend may reap on; ErrStreamNotServed is the worker saying +// it learned nothing, and folding it in turns every transient worker-side +// failure into an eviction. See IsWorkerAnswer. +var streamRefusals = []struct { + sentinel error + code string + evidence bool +}{ + {ErrStreamTagUnknown, replyCodeUnknownTag, true}, + {ErrStreamTargetUnavailable, replyCodeUnavailable, true}, + {ErrStreamRequestInvalid, replyCodeBadRequest, true}, + {ErrStreamNotServed, replyCodeNotServed, false}, +} + +// IsStreamRefusal reports whether err ALREADY carries one of this vocabulary's +// classifications. +// +// It answers "has something already decided what this failure is", which is a +// different question from IsWorkerAnswer's "may a consumer act on it". A worker +// that re-classifies an error which already carries a sentinel overwrites a +// decision made closer to the failure, and when the overwrite lands on one of +// the three evidence codes it manufactures a verdict out of something that was +// explicitly not one. +func IsStreamRefusal(err error) bool { + for _, r := range streamRefusals { + if errors.Is(err, r.sentinel) { + return true + } + } + return false +} + // WriteStreamRequest sends the opening frame naming what the stream is for. // // An empty tag is refused here rather than on the wire, because the worker @@ -173,15 +220,11 @@ func WriteStreamAccepted(w io.Writer) error { // distinguished the codes and became a reap-by-omission when one did. func WriteStreamRefusal(w io.Writer, reason error) error { code := replyCodeNotServed - switch { - case errors.Is(reason, ErrStreamTagUnknown): - code = replyCodeUnknownTag - case errors.Is(reason, ErrStreamTargetUnavailable): - code = replyCodeUnavailable - case errors.Is(reason, ErrStreamRequestInvalid): - code = replyCodeBadRequest - case errors.Is(reason, ErrStreamNotServed): - code = replyCodeNotServed + for _, r := range streamRefusals { + if errors.Is(reason, r.sentinel) { + code = r.code + break + } } text := "" @@ -221,21 +264,16 @@ func ReadStreamReply(r io.Reader) error { return fmt.Errorf("reading a tunnel stream reply: unrecognised reply %q", payload) } code, text, _ := strings.Cut(rest, streamRequestSeparator) - switch code { - case replyCodeUnknownTag: - return fmt.Errorf("%w: %s", ErrStreamTagUnknown, text) - case replyCodeUnavailable: - return fmt.Errorf("%w: %s", ErrStreamTargetUnavailable, text) - case replyCodeBadRequest: - return fmt.Errorf("%w: %s", ErrStreamRequestInvalid, text) - case replyCodeNotServed: - return fmt.Errorf("%w: %s", ErrStreamNotServed, text) - default: - // A code from a newer worker. Reported as an error carrying the code - // rather than mapped onto the nearest known one, so a frontend does not - // retry forever against a refusal that means something else entirely. - return fmt.Errorf("tunnel stream refused with unrecognised code %q: %s", code, text) + for _, r := range streamRefusals { + if code == r.code { + return fmt.Errorf("%w: %s", r.sentinel, text) + } } + // A code from a newer worker. Reported as an error carrying the code rather + // than mapped onto the nearest known one, so a frontend does not retry + // forever against a refusal that means something else entirely, and so + // IsWorkerAnswer reports false for it and nothing reaps. + return fmt.Errorf("tunnel stream refused with unrecognised code %q: %s", code, text) } // truncateRunes cuts s to at most limit BYTES, on a rune boundary. diff --git a/core/services/cluster/tunnelproto_wire_test.go b/core/services/cluster/tunnelproto_wire_test.go new file mode 100644 index 000000000000..0745444dad08 --- /dev/null +++ b/core/services/cluster/tunnelproto_wire_test.go @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: MIT + +package cluster + +// In-package, and that is the point: these specs assert the BYTES a refusal +// puts on the wire, against literals written out here rather than against the +// constants the code uses. A spec that round-trips through this process's own +// writer and reader cannot see a rename, because a rename moves both sides at +// once; the DescribeTable in tunnelproto_test.go is exactly that spec and it +// stays green through any renaming of the four codes. +// +// A wire code is a cross-version contract. A worker and a frontend built from +// different commits talk to each other over it, and the consequence of them +// disagreeing is not a parse error: an unrecognised code is deliberately +// treated as "not the worker's answer", so renaming `unavailable` would turn +// every crashed backend on a tunnelled worker into a row nothing can ever reap, +// silently and with the whole suite green. That is the exact defect this phase +// spent two rounds removing. +// +// This branch set the precedent for pinning a vocabulary against literals in +// core/services/messaging/subjects_wire_test.go, for the same reason. Nothing +// has shipped yet, so no value here is load bearing across a release boundary +// today; that is why the literals may still be changed, and why the change has +// to be deliberate rather than incidental. + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// frameBytes returns the payload of the single frame w wrote, so a spec can +// assert on the bytes rather than on what the reader makes of them. +func frameBytes(write func(w *bytes.Buffer) error) string { + GinkgoHelper() + var buf bytes.Buffer + Expect(write(&buf)).To(Succeed()) + raw := buf.Bytes() + Expect(len(raw)).To(BeNumerically(">=", 2)) + Expect(binary.BigEndian.Uint16(raw[:2])).To(Equal(uint16(len(raw) - 2))) + return string(raw[2:]) +} + +var _ = Describe("the tunnel refusal vocabulary on the wire", func() { + DescribeTable("writes the exact code an older build reads", + func(sentinel error, wantCode string) { + payload := frameBytes(func(w *bytes.Buffer) error { + return WriteStreamRefusal(w, fmt.Errorf("%w: because", sentinel)) + }) + Expect(payload).To(Equal("err " + wantCode + " " + sentinel.Error() + ": because")) + }, + Entry("unknown tag", ErrStreamTagUnknown, "unknown-tag"), + Entry("unavailable target", ErrStreamTargetUnavailable, "unavailable"), + Entry("invalid request", ErrStreamRequestInvalid, "bad-request"), + Entry("nothing learned", ErrStreamNotServed, "not-served"), + ) + + DescribeTable("reads the exact code an older build writes", + func(rawCode string, want error) { + var buf bytes.Buffer + Expect(writeFrame(&buf, "err "+rawCode+" some reason")).To(Succeed()) + Expect(ReadStreamReply(&buf)).To(MatchError(want)) + }, + Entry("unknown tag", "unknown-tag", ErrStreamTagUnknown), + Entry("unavailable target", "unavailable", ErrStreamTargetUnavailable), + Entry("invalid request", "bad-request", ErrStreamRequestInvalid), + Entry("nothing learned", "not-served", ErrStreamNotServed), + ) + + It("accepts a stream with the literal an older build sends", func() { + // The success case has a literal too, and a rename of it would refuse + // every stream rather than mis-classify one, which is at least loud. + Expect(frameBytes(func(w *bytes.Buffer) error { return WriteStreamAccepted(w) })).To(Equal("ok")) + }) + + It("names the two stream tags with the literals the worker routes on", func() { + // The worker's routing table is keyed by these, so a rename here is a + // worker that serves nothing while reporting an unknown tag, which is a + // verdict a frontend acts on. + Expect(StreamTagGRPC).To(Equal("grpc")) + Expect(StreamTagHTTP).To(Equal("http")) + }) + + It("pins which codes a frontend may act on as evidence about a backend", func() { + // The half of the table that decides whether a row is deleted. It is + // asserted against the literals, not against IsWorkerAnswer's own + // output, so moving a code between the two columns reddens here as well + // as at the consumer. + evidence := map[string]bool{} + for _, r := range streamRefusals { + evidence[r.code] = r.evidence + } + Expect(evidence).To(Equal(map[string]bool{ + "unknown-tag": true, + "unavailable": true, + "bad-request": true, + "not-served": false, + }), "a code that changed column changes whether a live model gets evicted") + }) + + It("has exactly one entry per sentinel, and no duplicate codes", func() { + // A duplicate code makes the reader's first match win and the writer's + // first match win, which need not be the same entry. + codes := map[string]int{} + for _, r := range streamRefusals { + Expect(r.sentinel).ToNot(BeNil()) + codes[r.code]++ + } + Expect(codes).To(HaveLen(len(streamRefusals))) + for code, n := range codes { + Expect(n).To(Equal(1), "code %q appears %d times", code, n) + } + }) + + It("classifies every sentinel as a refusal, and nothing else as one", func() { + // IsStreamRefusal is what stops the worker re-classifying a decision + // something closer to the failure already made. Derived from the table + // so a fifth code joins it automatically; asserted here so that + // derivation cannot quietly stop. + for _, r := range streamRefusals { + Expect(IsStreamRefusal(fmt.Errorf("wrapped: %w", r.sentinel))).To(BeTrue(), r.code) + } + Expect(IsStreamRefusal(errors.New("a plain failure"))).To(BeFalse()) + Expect(IsStreamRefusal(nil)).To(BeFalse()) + }) +}) diff --git a/core/services/worker/tunnel.go b/core/services/worker/tunnel.go index cfd33a770deb..dab799cb0af7 100644 --- a/core/services/worker/tunnel.go +++ b/core/services/worker/tunnel.go @@ -65,11 +65,24 @@ const ( tunnelHandshakeTimeout = 10 * time.Second // tunnelHeaderTimeout bounds how long a stream may go without sending the - // request frame that says what it is for. It is generous because it bounds - // only the frontend's own framing, which it writes immediately after - // opening the stream; it is present because without it a stream that sends - // nothing holds a goroutine and one of the session's stream slots for as - // long as the tunnel lives. + // request frame that says what it is for. It is present because without it + // a stream that sends nothing holds a goroutine and one of the session's + // stream slots for as long as the tunnel lives. + // + // It is generous because it does NOT bound only the frontend's own framing. + // An earlier version of this comment said it did, which is true on the + // direct path and false on the relay path that carries most of a + // multi-replica deployment's traffic: this timer starts when the OWNING + // replica opens the stream, while the frame is written by the DIALLING + // replica only after the relay's acceptance has travelled back to it. A + // whole peer-link round trip therefore runs inside this window, on a link + // deliberately loaded with multi-gigabyte artifacts beside token streams. + // + // The comment mattered because it was the argument for treating an expiry + // as the frontend's fault: framing written immediately can only be late if + // something is wrong with the frontend. It cannot, so an expiry is refused + // with cluster.ErrStreamNotServed and says nothing about a backend. See + // Tunnel.accept. tunnelHeaderTimeout = 15 * time.Second ) @@ -358,7 +371,7 @@ func (t *Tunnel) accept(ctx context.Context, stream net.Conn) (net.Conn, bool) { // with multi-gigabyte artifacts. Reported as a malformed request it // became reaping evidence, and for a long-deadline caller that is a // model evicted across the fleet by nothing but congestion. - if isReadTimeout(err) { + if reportsTimeout(err) { t.refuse(stream, fmt.Errorf("%w: %v", cluster.ErrStreamNotServed, err)) return nil, false } @@ -408,51 +421,85 @@ func (t *Tunnel) accept(ctx context.Context, stream net.Conn) (net.Conn, bool) { // classifyServiceFailure decides which refusal a local service's error is. // // A service that has ALREADY classified its own failure keeps that -// classification. loopbackService does, and the distinction is not cosmetic: a +// classification, and that is asked of the whole vocabulary rather than of one +// sentinel. loopbackService classifies, and the distinction is not cosmetic: a // target outside this worker's backend port range is a request this worker will // never serve, while a backend that is not listening yet is a condition that // clears on its own. Reporting the first as the second tells a frontend to // retry something that can never work; reporting the second as the first makes // it give up on a backend that is merely starting. // +// This used to preserve ONE of the four codes, which was true to its own +// comment for exactly as long as there was one classification worth keeping. +// Once ErrStreamNotServed existed, a service returning the code whose entire +// job is to say "I learned nothing" had it PROMOTED here into +// ErrStreamTargetUnavailable, which every reap guard acts on. No in-tree +// service produced it, which is the same "unreachable, therefore safe" +// argument that let the request-frame merge survive a whole phase; LocalService +// and TunnelConfig.Services are both exported, so out-of-tree is a real place. +// cluster.IsStreamRefusal reads the vocabulary table, so a fifth code is +// preserved here without anyone remembering to come back. +// // The default is TargetUnavailable and stays that way, which is the deliberate // half of this function now that the frontend acts on that code. A dial to the // named process that came back with anything is the closest thing to evidence // this worker can produce, and the direction of a mis-classification decides -// which mistake is made: defaulting the other way would mean a genuinely dead -// backend whose errno nobody enumerated becomes a permanently unreapable row, -// which is the exact defect this phase spent a round removing. So the -// exemptions below are a DENY-list of causes that are provably not about the -// target, not an allow-list of causes that are. +// which mistake is made. A wrong reap is ACTIVE: it deletes rows and, on the +// inference path, runs ShutdownModel on a model that is loaded and serving. A +// wrong retention is passive, bounded to one replica slot, and clears on a +// restart or an eviction. An allow-list of reapable causes would also fail +// SILENTLY and PERMANENTLY when it missed one, where a deny-list that misses +// fails loudly. So the exemptions below are a DENY-list of causes that are not +// the target answering, not an allow-list of causes that are. // -// The two exempted are the caller's context ending and a read/write deadline -// firing. Neither is the target answering: the context here is the SESSION's, -// so it ends when this worker's tunnel is torn down, and a deadline is this -// process's own timer. Both clear on a reconnect. They are exempted rather -// than argued away as unreachable because "unreachable" was the argument that -// made the request-frame merge look safe. +// The exempted causes, stated exactly, because an earlier version of this +// comment said "two" and the predicate covered more than it named: +// +// - The context ending. Here that is the SESSION's context, cancelled while +// stream goroutines are still running, so it means this worker's tunnel is +// being torn down and will reconnect. +// - This process's own I/O deadline (os.ErrDeadlineExceeded). +// - Anything else reporting itself as a net.Error timeout, which on a DIAL +// also covers syscall.ETIMEDOUT and syscall.EAGAIN. Those are kept +// deliberately. EAGAIN is this worker running out of resources, which is +// plainly not about the target. ETIMEDOUT from connect(2) IS an observation +// about the target, but it is the observation "it did not finish the +// handshake", which is a wedged or backlogged listener rather than an +// absent one, and reaping an overloaded backend is the eviction this whole +// phase exists to prevent. ECONNREFUSED, the shape of a process that is +// genuinely gone, is not a timeout and still reaps. +// +// They are exempted rather than argued away as unreachable because +// "unreachable" was the argument that made the request-frame merge look safe. // // It must never become the unknown-tag refusal either: a tag this worker serves // does not stop being served because one dial failed. func classifyServiceFailure(err error) error { - if errors.Is(err, cluster.ErrStreamRequestInvalid) { + if cluster.IsStreamRefusal(err) { return err } - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || isReadTimeout(err) { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || reportsTimeout(err) { return fmt.Errorf("%w: %v", cluster.ErrStreamNotServed, err) } return fmt.Errorf("%w: %v", cluster.ErrStreamTargetUnavailable, err) } -// isReadTimeout reports whether err is an I/O deadline firing rather than a -// peer saying anything. +// reportsTimeout reports whether err says of ITSELF that it is a timeout. +// +// Named for what it asks rather than for where it is asked, because it is asked +// in two places that mean different things. On a stream READ it is a deadline +// this process armed. On a DIAL it is wider: Go's syscall.Errno.Timeout is true +// for ETIMEDOUT and EAGAIN as well, and net.OpError passes that through. Both +// call sites want the same ANSWER (not the target speaking, so not evidence +// about a backend), which is why one predicate serves both; see +// classifyServiceFailure for why the wider set is kept deliberately. // // net.Error's Timeout is asked as well as os.ErrDeadlineExceeded because the // two are not the same set: a yamux stream returns its own timeout value from // a Read whose deadline expired, and a net.OpError over a socket returns // os.ErrDeadlineExceeded. Missing either would put a timeout back on the // verdict path, which is the defect this predicate exists to keep closed. -func isReadTimeout(err error) bool { +func reportsTimeout(err error) bool { if errors.Is(err, os.ErrDeadlineExceeded) { return true } diff --git a/core/services/worker/tunnel_test.go b/core/services/worker/tunnel_test.go index 743cccf726d3..6afafaa20189 100644 --- a/core/services/worker/tunnel_test.go +++ b/core/services/worker/tunnel_test.go @@ -444,6 +444,48 @@ var _ = Describe("Worker tunnel client", func() { Expect(cluster.IsWorkerAnswer(got)).To(BeFalse()) }) + DescribeTable("keeps a classification the local service already made", + // The latent instance of the same shape, found by the gate rather + // than by anything reaching it. This function preserved exactly ONE + // of the four codes, which was faithful to its own comment for as + // long as there was one worth keeping. Once ErrStreamNotServed + // existed, a service returning the code whose whole job is to say + // "I learned nothing" had it PROMOTED to ErrStreamTargetUnavailable, + // which every reap guard acts on. + // + // No in-tree service produced it, which is the "unreachable, + // therefore safe" argument that let the request-frame merge survive + // a whole phase. LocalService and TunnelConfig.Services are both + // exported, so out-of-tree is a real place, and the same standard + // applies. + func(classified error, wantEvidence bool) { + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services[cluster.StreamTagGRPC] = func(context.Context, string) (net.Conn, error) { + return nil, fmt.Errorf("the service decided for itself: %w", classified) + } + }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagGRPC, "127.0.0.1:41000")).To(Succeed()) + + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + var got error + Eventually(reply, "10s").Should(Receive(&got)) + Expect(got).To(MatchError(classified), + "re-classifying overwrites a decision made closer to the failure") + Expect(cluster.IsWorkerAnswer(got)).To(Equal(wantEvidence)) + }, + // The one the promotion broke: not evidence before, evidence after. + Entry("I learned nothing", cluster.ErrStreamNotServed, false), + // Promoted too. Both sides reap, so it cost nothing, which is + // exactly why nothing caught it. + Entry("I do not serve that tag", cluster.ErrStreamTagUnknown, true), + Entry("that request was malformed", cluster.ErrStreamRequestInvalid, true), + Entry("I could not reach the target", cluster.ErrStreamTargetUnavailable, true), + ) + It("still reports a refused local dial as an unavailable target, which reaps", func() { // The other direction for the deny-list: the ordinary shape of a // crashed backend must keep producing the code the reap guards act diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index aa2443c82cd6..cb528ea305e8 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -190,11 +190,24 @@ These outcomes are kept apart on purpose, because they call for different action | Not the owner | The routing was stale | Resolve the owner again | | Peer unreachable | A replica exists and will not answer | Retry | | No relay path | This replica cannot reach the owner at all | Report; requests here fail until it can | -| The worker refused | The worker answered and said no | Report, and **act on it**; the worker is connected and speaking about its own backend | +| The worker refused | The worker answered and said no | Depends on WHICH refusal; see below | **None of the first four is absence.** A worker's presence is its **heartbeat**, and a route to it is a separate fact that can be false while the worker is registered, heartbeating and serving every request another replica sends it. So the frontend answers "no route", never "this worker is gone", and none of the first four causes a model to be rescheduled or a `node_models` row to be deleted. -The fifth is different, and deliberately so. A worker that **refuses** a stream has answered, which proves it is connected; what it is refusing is the stream to one backend process on it. That is the ordinary shape of a crashed backend now that workers listen on nothing: the worker's own dial to the process fails and it says so. The frontend treats that as evidence about the backend, so the model's row is reaped and the replica is reloaded, exactly as a dead local backend would be. Without that, a crashed backend on a healthy worker would leave a row nothing could ever delete, its replica slot permanently occupied. A refusal code the frontend does not recognise - a newer worker's vocabulary - is treated as "no route" instead, so a version skew costs a retry rather than a reaped replica. +The fifth is different, and deliberately so. A worker that **refuses** a stream has answered, which proves it is connected; what it is refusing is the stream to one backend process on it. That is the ordinary shape of a crashed backend now that workers listen on nothing: the worker's own dial to the process fails and it says so. + +There are **four** refusals, and only three of them are evidence about a backend. The distinction decides whether a model's row is deleted, so an operator reading one of these in a log can tell what will happen next: + +| Refusal a worker sends | When | Row reaped? | +|---|---|---| +| `the worker could not reach the local service for that stream` | The worker's own dial to the backend process was refused. A crashed backend | **Yes.** Reloaded elsewhere, as a dead local backend would be | +| `the worker does not serve that stream tag` | The worker does not serve that kind of stream at all | **Yes.** Nothing clears this until the worker is upgraded, and the model re-registers somewhere that works | +| `the worker rejected the stream request as malformed` | The stored backend address is not a port in this worker's range | **Yes.** The row can never be reached, so reaping lets the model re-register a usable address | +| `the worker could not serve that stream, for a reason that is not about the backend` | The request frame did not arrive in the worker's 15s window, the worker's tunnel was being torn down, or it ran out of a local resource | **No.** These clear on their own; the request fails with "no route" and is retried | + +The fourth exists because the other three are acted on. A relayed request crosses a peer link before its frame reaches the worker, so on a congested link a frame can arrive late through nobody's fault; reported as one of the first three, that would evict a model that is loaded and serving. If you see the fourth in your logs, look at peer-link congestion or a worker that is reconnecting, not at the backend it names. + +A refusal code the frontend does not recognise - a newer worker's vocabulary - is treated as "no route" as well, so a version skew costs a retry rather than a reaped replica. That distinction is the whole point rather than a nicety. A scheduler told that a connected worker has gone away stops its backend and reclaims every model it is running, and the events that produce "no route" are ordinary ones: a frontend replica restarting, an ownership row a moment stale, a worker that has not dialled its tunnel yet. A worker is treated as absent only when its **heartbeat** goes stale, which is a separate mechanism with its own threshold (see `--stale-node-threshold`). From db31cf521be3a8f3dca0347cd76b5275ab9f25fe Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 12:03:55 +0000 Subject: [PATCH 066/153] feat(cluster): record a departure instead of erasing the connection Releasing a worker tunnel deleted its node_connections row, so "this worker's link dropped a moment ago" and "this worker has never connected here" were one observation: no row. Nothing above could tell a worker re-homing between replicas from a worker that is gone, and any grace period built on top would have had nothing to measure from. The row now survives a departure. Release clears owner_instance_id and stamps disconnected_at on the database clock; the membership sweep and Deregister do the same for every connection a dead or departing replica held; Claim clears the stamp in the same upsert that writes the owner, so a reconnect is never observed half-applied. PurgeDepartedBefore deletes a departure once it is older than DepartedRetention, and the membership tick owns that schedule. Owner and OwnerRow report a departed row as ErrNoConnection, through the one predicate connectionIsHeld, the way instanceIsLive is the one predicate for replica liveness. This change records the departure and does not interpret it: how long ago it happened is nobody's answer yet. The sweep only clears rows that are still held. An empty owner is in no instance's id, so without that filter every heartbeat would restamp every departed row and no departure could ever age out. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/membership.go | 94 ++++++--- core/services/cluster/membership_test.go | 67 ++++++- core/services/cluster/ownership.go | 152 +++++++++++++-- core/services/cluster/ownership_test.go | 225 ++++++++++++++++++++-- docs/content/features/distributed-mode.md | 2 +- 5 files changed, 481 insertions(+), 59 deletions(-) diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index 6a8650927b15..84544e7134bb 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -29,6 +29,18 @@ const ( // deregisterTimeout bounds the deregistration Stop performs. Shutdown is // not the place to wait on a database. deregisterTimeout = 5 * time.Second + + // DepartedRetention is how long a connection row outlives the tunnel it + // recorded before the sweep deletes it. + // + // A multiple of the liveness window rather than the window itself. The row + // survives so that how long ago a tunnel went can be answered at all, and a + // retention as short as the window a reader compares that age against would + // let the purge delete the row out from under the reader, turning a worker + // that is re-dialling back into a worker that was never here. Ten windows + // is far enough clear of any such comparison, and short enough that a + // worker retired for good does not sit in the table for a day. + DepartedRetention = 10 * InstanceLiveness ) // Membership publishes this replica's address and keeps the instances table @@ -207,6 +219,19 @@ func (m *Membership) tick(ctx context.Context) { if instances > 0 || connections > 0 { xlog.Info("Reaped cluster state left by dead replicas", "instances", instances, "connections", connections) } + + // The retention has an owner, and it is this sweep. A departure that + // nothing ever deletes is a row per worker that ever dialled this + // deployment, and it is the same loop that decides a replica is gone, so + // there is one schedule rather than two. + purged, err := m.reg.PurgeDepartedBefore(ctx, DepartedRetention) + if err != nil { + xlog.Warn("Purging departed worker connections failed", "error", err) + return + } + if purged > 0 { + xlog.Info("Purged worker connections whose departure aged out", "connections", purged, "retention", DepartedRetention) + } } // reclaimTunnels re-writes a claim for every worker tunnel this replica still @@ -233,13 +258,18 @@ func (m *Membership) reclaimTunnels(ctx context.Context) { } } -// Deregister removes one replica and the connections it owned. +// Deregister removes one replica and records a departure for every connection +// it owned. // -// It deletes both, in one transaction, for the same reason ReapStale does: a -// replica that is gone owns nothing, and leaving its connection rows behind -// would point every reader at an owner that no longer exists. This is the +// Both in one transaction, for the same reason ReapStale does it in one: a +// replica that is gone owns nothing, and leaving its connection rows pointing +// at it would point every reader at an owner that no longer exists. This is the // announced form of what the sweeper does by inference, and the two must not -// disagree about what "gone" removes. +// disagree about what "gone" leaves behind. +// +// The connection rows are cleared, not deleted: a worker whose frontend shut +// down is about to re-dial the load balancer, and erasing its row would make +// the seconds in between look like a worker that had never connected. // // Instances first, then connections, which is deliberate and is the same order // ReapStale takes. The two paths run concurrently in the ordinary case, a @@ -255,8 +285,11 @@ func (r *Registry) Deregister(ctx context.Context, id string) error { if err := tx.Where("id = ?", id).Delete(&Instance{}).Error; err != nil { return fmt.Errorf("deleting instance %q: %w", id, err) } - if err := tx.Where("owner_instance_id = ?", id).Delete(&NodeConnection{}).Error; err != nil { - return fmt.Errorf("deleting connections owned by %q: %w", id, err) + // No held-ness filter: only rows this replica owns match, and a + // departed row carries no owner, so it cannot be restamped here. + if err := tx.Model(&NodeConnection{}).Where("owner_instance_id = ?", id). + Updates(departure()).Error; err != nil { + return fmt.Errorf("recording departures for connections owned by %q: %w", id, err) } return nil }); err != nil { @@ -266,33 +299,39 @@ func (r *Registry) Deregister(ctx context.Context, id string) error { } // ReapStale deletes the replicas that have not heartbeated within the liveness -// window, and the connection rows whose owner is no longer among the survivors. +// window, and records a departure for every connection row whose owner is no +// longer among the survivors. // -// The two deletes are one sweeper on purpose. A connection row is only ever -// orphaned by its owner dying, so the moment that is decided is the moment to -// clean up after it; a second sweeper with its own schedule would either lag -// this one or race it, and would need its own answer to "is that replica -// alive", which is the one fact this table already owns. +// The two are one sweeper on purpose. A connection row is only ever orphaned by +// its owner dying, so the moment that is decided is the moment to clean up +// after it; a second sweeper with its own schedule would either lag this one or +// race it, and would need its own answer to "is that replica alive", which is +// the one fact this table already owns. +// +// The connection rows are cleared and not deleted, and the returned count is of +// rows this sweep cleared. A worker whose owning replica died has not gone +// anywhere: it is re-dialling the load balancer, and deleting its row would +// erase the departure that says how long ago that started. // // self is never reaped. This process may fail to heartbeat for longer than the // window (a long stall, a database blip) and still be serving: deleting its own -// row would then delete the connections of workers that are, at that moment, +// row would then mark as departed the workers that are, at that moment, // connected to it. // // That protection is one-sided. A replica that stalls long enough is reaped BY -// ANOTHER replica, taking its connection rows with it, and Register rebuilds -// the instance row and nothing else. What restores the rest is the re-claim in -// tick, which writes a fresh claim for every tunnel the tunnel registry still -// holds; until it runs, this replica holds sockets the table records nobody -// holding. +// ANOTHER replica, which records a departure for every connection it held, and +// Register rebuilds the instance row and nothing else. What restores the rest +// is the re-claim in tick, which writes a fresh claim for every tunnel the +// tunnel registry still holds; until it runs, this replica holds sockets the +// table records nobody holding. // // PostgreSQL only, like Live: distributed mode requires it, and the interval // arithmetic is measured on the database's clock because liveness is compared // across replicas. // -// Instances are deleted before connections, and Deregister takes the same order +// Instances are written before connections, and Deregister takes the same order // on purpose, so the two paths cannot deadlock against each other. Here the -// order is also forced: the connection delete asks which instance rows survived, +// order is also forced: the connection sweep asks which instance rows survived, // so it has to run second. func (r *Registry) ReapStale(ctx context.Context, self string, within time.Duration) (instances int64, connections int64, err error) { err = r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { @@ -309,10 +348,17 @@ func (r *Registry) ReapStale(ctx context.Context, self string, within time.Durat // Whatever survived the delete above is the live set, so this needs no // second liveness rule and cannot disagree with the first one. - res = tx.Where("owner_instance_id NOT IN (SELECT id FROM instances)"). - Delete(&NodeConnection{}) + // + // Held rows only. An empty owner is in no instance's id, so a departed + // row matches the set difference too, and re-clearing it every sweep + // would push its departure forward five seconds at a time: it would + // never age out of any window measured from it, and every sweep would + // report clearing a connection that had already gone. + res = tx.Model(&NodeConnection{}). + Where("owner_instance_id NOT IN (SELECT id FROM instances) AND " + connectionIsHeld). + Updates(departure()) if res.Error != nil { - return fmt.Errorf("deleting orphaned node connections: %w", res.Error) + return fmt.Errorf("recording departures for orphaned node connections: %w", res.Error) } connections = res.RowsAffected return nil diff --git a/core/services/cluster/membership_test.go b/core/services/cluster/membership_test.go index c331eafe23d2..22bd37f006f8 100644 --- a/core/services/cluster/membership_test.go +++ b/core/services/cluster/membership_test.go @@ -51,6 +51,69 @@ var _ = Describe("Reaping dead replicas", func() { Expect(err).To(MatchError(cluster.ErrNoConnection)) }) + It("records a departure rather than erasing the connection when a stale replica is swept", func() { + // The row is what tells "this worker's owner died a moment ago" from + // "this worker has never connected". Deleting it on the sweep erased + // exactly the departure a reconnect grace has to be measured from. + Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "dead") + Expect(err).ToNot(HaveOccurred()) + age("dead", time.Hour) + + _, _, err = reg.ReapStale(ctx, "live", time.Minute) + Expect(err).ToNot(HaveOccurred()) + + var row cluster.NodeConnection + Expect(db.WithContext(ctx).Where("node_id = ?", "w1").First(&row).Error).To(Succeed(), + "the sweep deleted the row, so the worker its owner left behind looks like one that never dialled") + Expect(row.OwnerInstanceID).To(BeEmpty()) + Expect(row.DisconnectedAt).ToNot(BeNil()) + }) + + It("does not restamp a departure it has already recorded", func() { + // The sweep runs every heartbeat. Re-clearing a row it already cleared + // would push its departure forward on every pass, so the departure + // would never age out of any window and the row would be reported as + // swept for as long as it existed. + Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "dead") + Expect(err).ToNot(HaveOccurred()) + age("dead", time.Hour) + + _, connections, err := reg.ReapStale(ctx, "live", time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(connections).To(Equal(int64(1))) + var first cluster.NodeConnection + Expect(db.WithContext(ctx).Where("node_id = ?", "w1").First(&first).Error).To(Succeed()) + + _, connections, err = reg.ReapStale(ctx, "live", time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(connections).To(BeZero(), "the sweeper reported clearing a connection that was already departed") + + var second cluster.NodeConnection + Expect(db.WithContext(ctx).Where("node_id = ?", "w1").First(&second).Error).To(Succeed()) + Expect(second.DisconnectedAt).ToNot(BeNil()) + Expect(*second.DisconnectedAt).To(Equal(*first.DisconnectedAt), + "the second sweep moved the departure forward, so it can never age past a grace window") + }) + + It("records a departure rather than erasing the connection when a replica deregisters", func() { + // The announced form of the same thing the sweeper does by inference, + // and the two must not disagree about what "gone" leaves behind. + Expect(reg.Register(ctx, "leaving", "10.0.0.2:8080", "v1")).To(Succeed()) + _, err := reg.Claim(ctx, "w1", "leaving") + Expect(err).ToNot(HaveOccurred()) + + Expect(reg.Deregister(ctx, "leaving")).To(Succeed()) + + var row cluster.NodeConnection + Expect(db.WithContext(ctx).Where("node_id = ?", "w1").First(&row).Error).To(Succeed()) + Expect(row.OwnerInstanceID).To(BeEmpty()) + Expect(row.DisconnectedAt).ToNot(BeNil()) + }) + It("leaves the connections of a live replica alone", func() { Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1")).To(Succeed()) Expect(reg.Register(ctx, "other", "10.0.0.2:8080", "v1")).To(Succeed()) @@ -150,8 +213,8 @@ var _ = Describe("Reaping dead replicas", func() { ReapStale(ctx, "sweeper", time.Minute) Expect(err).ToNot(HaveOccurred()) - Expect(deregRec.deleteOrder()).To(Equal([]string{"instances", "node_connections"})) - Expect(reapRec.deleteOrder()).To(Equal(deregRec.deleteOrder()), + Expect(deregRec.writeOrder()).To(Equal([]string{"instances", "node_connections"})) + Expect(reapRec.writeOrder()).To(Equal(deregRec.writeOrder()), "the sweeper and deregistration must lock the same two tables in the same order") }) diff --git a/core/services/cluster/ownership.go b/core/services/cluster/ownership.go index 8a80adede74b..d4a544fcc201 100644 --- a/core/services/cluster/ownership.go +++ b/core/services/cluster/ownership.go @@ -28,11 +28,12 @@ func isPostgres(db *gorm.DB) bool { } // epochSequence is the PostgreSQL sequence every claim draws its epoch from. -// A sequence rather than a per-row counter because a released row is deleted: -// with `epoch = epoch + 1` the numbering restarts at 1 for the next claim, so a +// A sequence rather than a per-row counter because a connection row does not +// outlive the deployment: a departure is purged once it is old enough, and with +// `epoch = epoch + 1` the numbering would restart at 1 for the next claim, so a // replica that claimed, lost the worker, and claimed again could be handed an // epoch it already held, and a delayed cleanup from the first claim would then -// match, and delete, the live one. +// match, and clear, the live one. const epochSequence = "node_connection_epochs" // NodeConnection records which frontend replica currently holds a worker's @@ -50,15 +51,46 @@ const epochSequence = "node_connection_epochs" // one is answered by the epoch; a second liveness clock for the same fact would // only drift from the first. type NodeConnection struct { - NodeID string `gorm:"primaryKey;size:36" json:"node_id"` + NodeID string `gorm:"primaryKey;size:36" json:"node_id"` + // Empty when nobody holds this tunnel. A departed row keeps the empty + // string rather than NULL so there is one spelling of "held by nobody"; + // connectionIsHeld is the only place that spelling is written down. OwnerInstanceID string `gorm:"size:36;index;not null" json:"owner_instance_id"` Epoch int64 `gorm:"not null" json:"epoch"` // No column DEFAULT: now() is PostgreSQL syntax and would reach the DDL, // which breaks AutoMigrate on the SQLite single-binary path. Claim writes // the database clock as an expression instead, the way Register does. ConnectedAt time.Time `gorm:"not null" json:"connected_at"` + // DisconnectedAt is when the tunnel LEFT, and it exists because "gone for + // thirty seconds" and "never here" have to be different answers. + // + // A released row used to be deleted, so a worker re-homing between replicas + // was indistinguishable from one that had never dialled, and any grace + // period built on top would have had nothing to measure from. It is NOT a + // liveness clock for the owner: whether the OWNING replica is alive is + // still Instance.LastSeen and nothing else. It ticks once, on departure. + // + // Null while the row is held, and stamped on the database clock by whoever + // records the departure, so a row with an owner never also carries one. + DisconnectedAt *time.Time `gorm:"index" json:"disconnected_at,omitempty"` } +// connectionIsHeld is the one predicate that separates a row recording a tunnel +// somebody holds from a row recording a departure. Everything that asks that +// question asks it here, or asks for its negation: Owner and OwnerRow refuse a +// row it rejects, ReapStale clears only rows it accepts, and +// PurgeDepartedBefore deletes only rows it rejects. Two spellings of one fact +// drift, and the drift here would show up as a departed worker resolving to a +// replica that holds nothing. +// +// Deregister is the one write that does not use it, because it matches an owner +// id, which is narrower: a departed row carries no owner and so cannot match. +// +// The column is table-qualified because Owner reads it across a join, where an +// unqualified name is ambiguous; qualifying it costs the single-table readers +// nothing. +const connectionIsHeld = `node_connections.owner_instance_id <> ''` + // Migrate creates every table and sequence this package owns. It is the one // call a caller has to remember: gorm's AutoMigrate models tables and columns // but has no notion of a sequence, and the connection fence draws its epochs @@ -145,6 +177,10 @@ func (r *Registry) Claim(ctx context.Context, nodeID, ownerID string) (int64, er "owner_instance_id": ownerID, "epoch": nextEpoch, "connected_at": gorm.Expr("now()"), + // In the SAME statement as the owner, so a reconnect is never + // observed half-applied: a row that names a holder while still + // carrying a departure would answer "held" and "gone" at once. + "disconnected_at": nil, }), }, clause.Returning{Columns: []clause.Column{{Name: "epoch"}}}, @@ -179,10 +215,15 @@ func (r *Registry) Claim(ctx context.Context, nodeID, ownerID string) (int64, er // liveness. Its callers today are this package's specs, including the one that // holds the two reads apart, and the e2e cluster spec that watches ownership // move between replicas. No production caller reads it, and the sweeper is not -// one: ReapStale deletes orphans with a set difference in SQL. +// one: ReapStale finds orphans with a set difference in SQL. +// +// A row that records a departure is ErrNoConnection here too. The row survives +// so that something can decide how old the departure is; what it says about who +// holds the tunnel is nobody, and this read reports exactly that. func (r *Registry) OwnerRow(ctx context.Context, nodeID string) (string, int64, error) { var conn NodeConnection - err := r.db.WithContext(ctx).Where("node_id = ?", nodeID).First(&conn).Error + err := r.db.WithContext(ctx). + Where("node_id = ? AND "+connectionIsHeld, nodeID).First(&conn).Error if errors.Is(err, gorm.ErrRecordNotFound) { return "", 0, fmt.Errorf("looking up owner of node %q: %w", nodeID, ErrNoConnection) } @@ -202,10 +243,15 @@ func (r *Registry) OwnerRow(ctx context.Context, nodeID string) (string, int64, // is gone, and a relay built on it would dial a corpse and report the worker as // unreachable rather than as absent. // -// A missing row and a dead owner are one answer on purpose. Both mean "no -// replica here holds this worker's tunnel", which is what a caller decides on; -// they differ only in which sweep has already run, and that is the sweeper's -// business rather than the caller's. +// A missing row, a departed row and a dead owner are one answer on purpose. All +// three mean "no live replica holds this worker's tunnel", which is what a +// caller decides on; they differ only in which sweep has already run, and that +// is the sweeper's business rather than the caller's. +// +// That answer is emphatically not "this worker is gone". How long ago the +// departure was recorded is what separates a worker re-homing between replicas +// from one that has left, and this read does not report it: a caller that has +// to tell those apart reads the departure, not this. // // One statement, joined, not a row read followed by an instance lookup: between // two statements the owner can die, and the caller would act on an owner the @@ -235,7 +281,11 @@ func (r *Registry) Owner(ctx context.Context, nodeID string) (string, int64, err // is here to filter and the row scanned back must stay this table's. Select("node_connections.*"). Joins("JOIN instances ON instances.id = node_connections.owner_instance_id AND "+instanceIsLive, InstanceLiveness.Seconds()). - Where("node_connections.node_id = ?", nodeID). + // The departure filter is this query's own, not the join's. An empty + // owner matches no instance today only because no replica registers + // under an empty id, which is an accident of who registers rather than + // a property of ownership. + Where("node_connections.node_id = ? AND "+connectionIsHeld, nodeID). Take(&conn).Error if errors.Is(err, gorm.ErrRecordNotFound) { return "", 0, fmt.Errorf("looking up live owner of node %q: %w", nodeID, ErrNoConnection) @@ -246,19 +296,52 @@ func (r *Registry) Owner(ctx context.Context, nodeID string) (string, int64, err return conn.OwnerInstanceID, conn.Epoch, nil } -// Release drops the claim identified by ownerID and epoch. Both are in the -// WHERE so a replica that has only just noticed its dead socket cannot delete -// the claim a later reconnect established elsewhere: the row it is trying to -// clean up no longer exists, and deleting the live one would strand a worker -// that is in fact connected. A claim that is no longer the live one is reported -// as ErrNoConnection rather than silently ignored, because the caller learning -// it has been fenced out is the point. +// departure is the single assignment that turns a held row into a departed one. +// Release writes it for one claim and the membership sweep writes it for every +// claim a dead replica held, and the two must not disagree about what a +// departure looks like: a row cleared without a stamp, or stamped without being +// cleared, is a state every reader here has no answer for. +// +// The timestamp is the database's, never this process's, for the same reason +// instance liveness is: it is compared across replicas, so it has to be +// measured on the one clock they share. +func departure() map[string]any { + return map[string]any{ + "owner_instance_id": "", + "disconnected_at": gorm.Expr("now()"), + } +} + +// Release drops the claim identified by ownerID and epoch, and records the +// departure: the row survives with no owner and a disconnected_at stamp. +// +// An UPDATE and not a DELETE, because the row is the only place a departure can +// be recorded. Deleting it made a worker re-homing between replicas look like +// one that had never connected, so nothing above could tell a two-second blip +// from a worker that is gone. +// +// Both ownerID and epoch are in the WHERE so a replica that has only just +// noticed its dead socket cannot touch the claim a later reconnect established +// elsewhere: it must not clear a live claim, and it must not stamp a departure +// onto one. A claim that is no longer the live one is reported as +// ErrNoConnection rather than silently ignored, because the caller learning it +// has been fenced out is the point. func (r *Registry) Release(ctx context.Context, nodeID, ownerID string, epoch int64) error { + // Refused rather than attempted, for the reason Claim refuses: the + // departure is stamped with now(), which on the single-binary SQLite path + // fails as a missing function and reads as a missing migration. It is + // deliberately not ErrNoConnection: a deployment with no cluster holds no + // claims, and reporting a fenced-out release would tell the caller it lost + // one. + if !isPostgres(r.db) { + return fmt.Errorf("releasing connection for node %q held by %q: connection ownership requires PostgreSQL, this deployment runs on %q", nodeID, ownerID, r.db.Dialector.Name()) + } // gorm reports no error when a Where matches nothing, so the miss has to be // read off RowsAffected. res := r.db.WithContext(ctx). + Model(&NodeConnection{}). Where("node_id = ? AND owner_instance_id = ? AND epoch = ?", nodeID, ownerID, epoch). - Delete(&NodeConnection{}) + Updates(departure()) if res.Error != nil { return fmt.Errorf("releasing connection for node %q held by %q at epoch %d: %w", nodeID, ownerID, epoch, res.Error) } @@ -267,3 +350,34 @@ func (r *Registry) Release(ctx context.Context, nodeID, ownerID string, epoch in } return nil } + +// PurgeDepartedBefore deletes the connection rows whose departure is older than +// olderThan, and returns how many went. +// +// Departures are kept so that absence can be decided, not kept forever. The +// retention has to outlast every window measured from a departure by enough +// that a purge can never turn a worker inside its reconnect grace into a worker +// that was never here; the caller passes a multiple of that grace, never the +// grace itself. +// +// A held row is never touched, whatever timestamp it carries: deleting one +// strands a worker that is connected at that moment. The age is measured by the +// database, like every other window here, so no replica's clock decides how old +// another replica's departure is. +func (r *Registry) PurgeDepartedBefore(ctx context.Context, olderThan time.Duration) (int64, error) { + // Refused rather than attempted, for the reason Claim refuses: now() and + // make_interval are PostgreSQL, and on the single-binary SQLite path this + // would fail with "no such function: now", which reads as a missing + // migration. + if !isPostgres(r.db) { + return 0, fmt.Errorf("purging departed connections: connection ownership requires PostgreSQL, this deployment runs on %q", r.db.Dialector.Name()) + } + res := r.db.WithContext(ctx). + Where("NOT ("+connectionIsHeld+") AND node_connections.disconnected_at < now() - make_interval(secs => ?)", + olderThan.Seconds()). + Delete(&NodeConnection{}) + if res.Error != nil { + return 0, fmt.Errorf("purging departed connections: %w", res.Error) + } + return res.RowsAffected, nil +} diff --git a/core/services/cluster/ownership_test.go b/core/services/cluster/ownership_test.go index cd422199c638..17f88617bad4 100644 --- a/core/services/cluster/ownership_test.go +++ b/core/services/cluster/ownership_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "regexp" "strings" "sync" "time" @@ -57,25 +58,28 @@ func (r *sqlRecorder) only() string { return r.statements[0] } -// deleteOrder returns the tables the recorded statements deleted from, in the -// order they were issued. It is how a spec pins a lock order: the order two -// paths take the same tables in is a property of the SQL, and asserting it on -// an outcome instead would mean racing two transactions into a real deadlock. -func (r *sqlRecorder) deleteOrder() []string { +// writeTarget matches the table a statement writes to, anchored at the verb so +// the UPDATE inside an upsert's ON CONFLICT clause cannot be mistaken for one. +var writeTarget = regexp.MustCompile(`^\s*(?i:delete\s+from|update)\s+"?([a-z_]+)"?`) + +// writeOrder returns the tables the recorded statements wrote to, in the order +// they were issued. It is how a spec pins a lock order: the order two paths +// take the same tables in is a property of the SQL, and asserting it on an +// outcome instead would mean racing two transactions into a real deadlock. +// +// Deletes and updates count alike. What deadlocks two transactions is the order +// they take row locks in, and an update takes the same lock a delete does, so a +// path that stopped deleting a table and started updating it would still have +// to keep the order. +func (r *sqlRecorder) writeOrder() []string { r.mu.Lock() defer r.mu.Unlock() ExpectWithOffset(1, r.errs).To(BeEmpty(), "a recorded statement failed") var tables []string for _, stmt := range r.statements { - idx := strings.Index(strings.ToLower(stmt), "delete from ") - if idx < 0 { - continue + if m := writeTarget.FindStringSubmatch(stmt); m != nil { + tables = append(tables, m[1]) } - fields := strings.Fields(stmt[idx+len("delete from "):]) - if len(fields) == 0 { - continue - } - tables = append(tables, strings.Trim(fields[0], `"`)) } return tables } @@ -380,6 +384,179 @@ var _ = Describe("Connection ownership", func() { Expect(rows).To(HaveLen(1)) Expect(seen).To(HaveKey(rows[0].Epoch), "the stored epoch was never handed to any claimant") }) + + // A tunnel that goes away has to leave a mark. Deleting the row made "this + // worker's link dropped a moment ago" and "this worker has never connected + // here" one observation, so nothing above could tell a reconnect in flight + // from a departure, and every grace period built on top would have had + // nothing to measure from. + Describe("recording a departure", func() { + It("keeps the row and stamps disconnected_at when a claim is released", func() { + epoch, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + + Expect(reg.Release(ctx, "w1", "inst-a", epoch)).To(Succeed()) + + var row cluster.NodeConnection + Expect(db.WithContext(ctx).Where("node_id = ?", "w1").First(&row).Error).To(Succeed(), + "the released row was deleted, so a worker that just left is indistinguishable from one that never dialled") + Expect(row.OwnerInstanceID).To(BeEmpty()) + Expect(row.DisconnectedAt).ToNot(BeNil()) + }) + + It("records the departure with one update measured on the database clock", func() { + epoch, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + + rec := newSQLRecorder() + recording := cluster.NewRegistry(db.Session(&gorm.Session{Logger: rec})) + Expect(recording.Release(ctx, "w1", "inst-a", epoch)).To(Succeed()) + + sql := strings.ToLower(rec.only()) + Expect(sql).To(ContainSubstring("update")) + Expect(sql).ToNot(ContainSubstring("delete")) + // The departure is compared against a grace window by other + // replicas, so it has to be stamped on the one clock they share. A + // Go-side time.Now() would appear as a bound parameter instead, and + // clock skew would then widen or narrow every window built on it. + Expect(sql).To(ContainSubstring("now()")) + Expect(sql).ToNot(MatchRegexp(`disconnected_at"?\s*=\s*'`), + "the departure must not be a literal timestamp from this process's clock") + }) + + It("reports a released row as no connection from both reads", func() { + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + epoch, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "w1", "inst-a", epoch)).To(Succeed()) + + // Both reads: the row surviving is for whoever decides how old the + // departure is, and until something does, a departed row is no + // connection to a dialer and no connection to an observer. + _, _, err = reg.Owner(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + _, _, err = reg.OwnerRow(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + }) + + It("does not resolve a departed row through an instance whose id is empty", func() { + // Owner rejects a departed row itself rather than leaning on the + // instances join to miss it. The join only misses an empty owner + // for as long as no instance row carries an empty id, which is an + // accident of who registers rather than a property of ownership. + Expect(reg.Register(ctx, "", "10.0.0.1:8080", "v1")).To(Succeed()) + epoch, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "w1", "inst-a", epoch)).To(Succeed()) + + _, _, err = reg.Owner(ctx, "w1") + Expect(err).To(MatchError(cluster.ErrNoConnection)) + }) + + It("clears the departure when the worker reconnects", func() { + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + epoch, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "w1", "inst-a", epoch)).To(Succeed()) + + again, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(again).ToNot(Equal(epoch), "uniqueness, never ordering") + + var row cluster.NodeConnection + Expect(db.WithContext(ctx).Where("node_id = ?", "w1").First(&row).Error).To(Succeed()) + Expect(row.DisconnectedAt).To(BeNil(), + "a row with an owner still carried a departure, so a reader has two answers to choose from") + owner, _, err := reg.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("inst-a")) + }) + + It("does not let a fenced-out replica stamp a departure onto the live claim", func() { + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + stale, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + live, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + + Expect(reg.Release(ctx, "w1", "inst-a", stale)).To(MatchError(cluster.ErrNoConnection)) + + owner, epoch, err := reg.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred()) + Expect(owner).To(Equal("inst-a")) + Expect(epoch).To(Equal(live)) + var row cluster.NodeConnection + Expect(db.WithContext(ctx).Where("node_id = ?", "w1").First(&row).Error).To(Succeed()) + Expect(row.DisconnectedAt).To(BeNil(), + "a replica that only just noticed its dead socket marked a live tunnel as departed") + }) + + It("purges a departure older than the retention and keeps a recent one", func() { + oldEpoch, err := reg.Claim(ctx, "old", "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "old", "inst-a", oldEpoch)).To(Succeed()) + // Aged on the database clock, which is the clock the purge measures + // on. Sleeping out a retention window in a spec is forbidden, and + // would be measuring this process's clock rather than the query. + Expect(db.WithContext(ctx).Exec( + `UPDATE node_connections SET disconnected_at = now() - make_interval(secs => ?) WHERE node_id = ?`, + 3600, "old").Error).To(Succeed()) + + freshEpoch, err := reg.Claim(ctx, "fresh", "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "fresh", "inst-a", freshEpoch)).To(Succeed()) + + purged, err := reg.PurgeDepartedBefore(ctx, 10*time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(purged).To(Equal(int64(1))) + + var remaining []cluster.NodeConnection + Expect(db.WithContext(ctx).Find(&remaining).Error).To(Succeed()) + Expect(remaining).To(HaveLen(1)) + Expect(remaining[0].NodeID).To(Equal("fresh"), + "the purge took a departure that is still inside every window built on it") + }) + + It("measures the retention on the database clock", func() { + rec := newSQLRecorder() + recording := cluster.NewRegistry(db.Session(&gorm.Session{Logger: rec})) + _, err := recording.PurgeDepartedBefore(ctx, 10*time.Minute) + Expect(err).ToNot(HaveOccurred()) + + sql := strings.ToLower(rec.only()) + // Asserted on the statement because no outcome can separate the two + // here: every replica in a spec shares this host's clock, so a + // cutoff computed in Go agrees with the database's until two + // machines disagree, and then one replica purges departures its + // peers still consider recent. + Expect(sql).To(ContainSubstring("now()")) + Expect(sql).To(ContainSubstring("make_interval")) + Expect(sql).ToNot(MatchRegexp(`disconnected_at"?\s*<\s*'`), + "the retention cutoff must not be a literal timestamp from this process's clock") + }) + + It("never purges a row that is still held, whatever timestamp it carries", func() { + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + claimed, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + // No writer produces a held row carrying a departure, so it is + // written here directly: the purge has to be pinned on the owner + // column rather than on the timestamp happening to be null, because + // deleting a held row strands a worker that is connected right now. + Expect(db.WithContext(ctx).Exec( + `UPDATE node_connections SET disconnected_at = now() - make_interval(secs => ?) WHERE node_id = ?`, + 3600, "w1").Error).To(Succeed()) + + purged, err := reg.PurgeDepartedBefore(ctx, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(purged).To(BeZero()) + + owner, epoch, err := reg.Owner(ctx, "w1") + Expect(err).ToNot(HaveOccurred(), "the purge deleted the row of a tunnel somebody holds") + Expect(owner).To(Equal("inst-a")) + Expect(epoch).To(Equal(claimed)) + }) + }) }) var _ = Describe("Connection ownership on a non-PostgreSQL dialect", func() { @@ -420,4 +597,26 @@ var _ = Describe("Connection ownership on a non-PostgreSQL dialect", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("requires PostgreSQL")) }) + + It("refuses to release, rather than clearing a claim outside the fence", func() { + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + + err := cluster.NewRegistry(db).Release(ctx, "w1", "inst-a", 1) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("requires PostgreSQL")) + Expect(err.Error()).ToNot(ContainSubstring("no such function"), + "a dialect that cannot record a departure must say so, not surface as a missing migration") + Expect(err).ToNot(MatchError(cluster.ErrNoConnection), + "a deployment with no cluster holds no claims; reporting a fenced-out release would let a caller conclude it lost one") + }) + + It("refuses to purge departures, rather than failing as a missing function", func() { + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + + _, err := cluster.NewRegistry(db).PurgeDepartedBefore(ctx, time.Minute) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("requires PostgreSQL")) + Expect(err.Error()).ToNot(ContainSubstring("no such function"), + "a dialect that cannot answer must say so, not surface as a missing migration") + }) }) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index cb528ea305e8..8efa7aa1fc33 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -139,7 +139,7 @@ A tunnel credential does not replace `LOCALAI_REGISTRATION_TOKEN`. Without one, Only **backend** nodes are issued one. An agent worker has no inbound surface for the tunnel to replace and no client for it, so minting one would widen the credential surface for nothing; its row keeps an empty tunnel credential and the tunnel route refuses it like any other node without one. -The tunnel lands on exactly one frontend replica, and that replica records itself as the owner of the worker's connection in the `node_connections` table. When the socket dies the claim is dropped with it. If the replica stalls long enough for its peers to reap it, it re-claims the tunnels it still holds on a live session as soon as it re-registers, skipping any whose socket has already gone. That re-claim needs the replica to have an advertised address: without one it never had an instance row to begin with, and its tunnels are usable only by the replica holding them. +The tunnel lands on exactly one frontend replica, and that replica records itself as the owner of the worker's connection in the `node_connections` table. When the socket dies the claim is dropped, but the row stays behind with no owner and a `disconnected_at` stamp, so a worker that is re-dialling the load balancer can be told from one that has never connected. The row is deleted once that departure is older than ten liveness windows (five minutes). If the replica stalls long enough for its peers to reap it, it re-claims the tunnels it still holds on a live session as soon as it re-registers, skipping any whose socket has already gone. That re-claim needs the replica to have an advertised address: without one it never had an instance row to begin with, and its tunnels are usable only by the replica holding them. | Method | Path | Description | |--------|------|-------------| From f8bece308f9211693da8b0507cf9697f579b58e8 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 14:26:38 +0000 Subject: [PATCH 067/153] fix(cluster): pin the departure retention to the loop that applies it The membership tick's call to PurgeDepartedBefore was the only production wiring this change introduced, and removing it left the suite green. A retention nothing applies is a departure that never ages out, which is the state the sweep's held-ness filter exists to make reachable at all. A spec now ages a released row past DepartedRetention on the database clock, starts a real Membership, and waits for the row to go. Release and Deregister leaned on "no owner id is ever empty" to avoid touching an already-departed row, which is the accident Owner refuses to lean on. A departed row keeps its epoch and carries an empty owner, so a release or a deregistration naming an empty id matched it and stamped a fresh departure over the old one, making a worker that left long ago look like one that has only just gone. Both now filter on connectionIsHeld. The comment on DisconnectedAt claimed a held row never carries a departure. A binary from before this column existed claims without clearing the stamp, so a rolling upgrade produces exactly that row. The comment now says what holds, and says to ask held-ness first and read the stamp second. The sweep's vocabulary follows the code: it records departures where the comments still said it deleted rows, and its log line separates the instance rows it deleted from the connection rows it left behind. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/cluster/membership.go | 36 ++++++++++------- core/services/cluster/membership_test.go | 50 ++++++++++++++++++++++++ core/services/cluster/ownership.go | 25 +++++++++--- core/services/cluster/ownership_test.go | 20 ++++++++++ 4 files changed, 112 insertions(+), 19 deletions(-) diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index 84544e7134bb..8dad478a9bf1 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -20,10 +20,11 @@ const ( // InstanceLiveness is how long a replica may go without a heartbeat before // its peers treat it as gone: six consecutive misses. // - // The window is generous on purpose. Declaring a replica dead deletes the - // connection rows it owned, and a worker whose row is deleted while its - // owner is merely slow has to be re-homed for nothing. The cost of waiting - // is bounded and symmetric: traffic for that worker is retried, not lost. + // The window is generous on purpose. Declaring a replica dead records a + // departure for every connection row it owned, and a worker recorded as + // departed while its owner is merely slow has to be re-homed for nothing. + // The cost of waiting is bounded and symmetric: traffic for that worker is + // retried, not lost. InstanceLiveness = 30 * time.Second // deregisterTimeout bounds the deregistration Stop performs. Shutdown is @@ -192,10 +193,10 @@ func (m *Membership) tick(ctx context.Context) { // carries no address, so the row has to be rebuilt from scratch. // // Register rebuilds the instance row ONLY. The sweep that removed it - // removed the connections this replica owned in the same transaction, - // so the tunnels still held here have to be claimed again or this - // replica serves workers that, as far as every other replica can see, - // are connected nowhere. + // recorded a departure for every connection this replica owned, in the + // same transaction, so the tunnels still held here have to be claimed + // again or this replica serves workers that, as far as every other + // replica can see, are connected nowhere. xlog.Warn("Cluster instance row was reaped, re-registering", "id", m.id) if err := m.reg.Register(ctx, m.id, m.addr, m.version); err == nil { m.reclaimTunnels(ctx) @@ -217,7 +218,10 @@ func (m *Membership) tick(ctx context.Context) { return } if instances > 0 || connections > 0 { - xlog.Info("Reaped cluster state left by dead replicas", "instances", instances, "connections", connections) + // Two verbs because the sweep does two things: the instance rows are + // gone, the connection rows are still there and now record a departure. + xlog.Info("Swept cluster state left by dead replicas", + "instances_deleted", instances, "connections_departed", connections) } // The retention has an owner, and it is this sweep. A departure that @@ -235,8 +239,9 @@ func (m *Membership) tick(ctx context.Context) { } // reclaimTunnels re-writes a claim for every worker tunnel this replica still -// holds, after the sweep that deleted them. It is separate from tick only so -// the lock around the registry reference is not held across the database work. +// holds, after the sweep that recorded them as departed. It is separate from +// tick only so the lock around the registry reference is not held across the +// database work. func (m *Membership) reclaimTunnels(ctx context.Context) { m.mu.Lock() tunnels := m.tunnels @@ -285,9 +290,12 @@ func (r *Registry) Deregister(ctx context.Context, id string) error { if err := tx.Where("id = ?", id).Delete(&Instance{}).Error; err != nil { return fmt.Errorf("deleting instance %q: %w", id, err) } - // No held-ness filter: only rows this replica owns match, and a - // departed row carries no owner, so it cannot be restamped here. - if err := tx.Model(&NodeConnection{}).Where("owner_instance_id = ?", id). + // Held rows only, for the reason Owner filters rather than leaning on + // the join: an id column that is never empty is an accident of who + // registers, not a property, and an empty id here would match every + // departed row in the table and reset every departure's age. + if err := tx.Model(&NodeConnection{}). + Where("owner_instance_id = ? AND "+connectionIsHeld, id). Updates(departure()).Error; err != nil { return fmt.Errorf("recording departures for connections owned by %q: %w", id, err) } diff --git a/core/services/cluster/membership_test.go b/core/services/cluster/membership_test.go index 22bd37f006f8..653726bee2ea 100644 --- a/core/services/cluster/membership_test.go +++ b/core/services/cluster/membership_test.go @@ -178,6 +178,56 @@ var _ = Describe("Reaping dead replicas", func() { Expect(owner).To(Equal("staying"), "deregistering one replica took another replica's claim") }) + It("does not restamp a departure when a replica with no id deregisters", func() { + // Deregister matches an owner id, and a departed row carries an empty + // one, so an empty id would match every departure in the table and reset + // each one's age. Nothing generates an empty id today, which is exactly + // why the filter has to be in the query rather than in that habit. + Expect(reg.Register(ctx, "leaving", "10.0.0.2:8080", "v1")).To(Succeed()) + epoch, err := reg.Claim(ctx, "w1", "leaving") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "w1", "leaving", epoch)).To(Succeed()) + var before cluster.NodeConnection + Expect(db.WithContext(ctx).Where("node_id = ?", "w1").First(&before).Error).To(Succeed()) + + Expect(reg.Deregister(ctx, "")).To(Succeed()) + + var after cluster.NodeConnection + Expect(db.WithContext(ctx).Where("node_id = ?", "w1").First(&after).Error).To(Succeed()) + Expect(after.DisconnectedAt).ToNot(BeNil()) + Expect(*after.DisconnectedAt).To(Equal(*before.DisconnectedAt), + "a departure that had already been recorded was stamped again, so its age restarted") + }) + + It("purges a departure that has aged past the retention, from the loop that sweeps it", func() { + // The retention exists only if something applies it, and this loop is + // the only thing that does. A PurgeDepartedBefore nobody calls leaves a + // row per worker that ever dialled this deployment, which is the state + // the sweep's held-ness filter exists to make reachable in the first + // place. + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + epoch, err := reg.Claim(ctx, "w1", "me") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "w1", "me", epoch)).To(Succeed()) + // Aged on the database clock, past the retention the loop applies. + Expect(db.WithContext(ctx).Exec( + `UPDATE node_connections SET disconnected_at = now() - make_interval(secs => ?) WHERE node_id = ?`, + (cluster.DepartedRetention + time.Minute).Seconds(), "w1").Error).To(Succeed()) + + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + Expect(membership.Start(ctx)).To(Succeed()) + DeferCleanup(membership.Stop) + + // Polled rather than slept: the loop ticks on its own schedule, and the + // spec is about the call happening at all, not about when. + Eventually(func() (int64, error) { + var rows int64 + err := db.WithContext(ctx).Model(&cluster.NodeConnection{}).Count(&rows).Error + return rows, err + }, "20s", "250ms").Should(BeZero(), + "the loop that sweeps dead replicas never applied the departure retention") + }) + It("deregisters when the membership loop stops", func() { membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") Expect(membership.Start(ctx)).To(Succeed()) diff --git a/core/services/cluster/ownership.go b/core/services/cluster/ownership.go index d4a544fcc201..1f253cd073f3 100644 --- a/core/services/cluster/ownership.go +++ b/core/services/cluster/ownership.go @@ -70,8 +70,16 @@ type NodeConnection struct { // liveness clock for the owner: whether the OWNING replica is alive is // still Instance.LastSeen and nothing else. It ticks once, on departure. // - // Null while the row is held, and stamped on the database clock by whoever - // records the departure, so a row with an owner never also carries one. + // Every writer in this package leaves it null while the row is held: Claim + // clears it in the statement that writes the owner, and only a departure + // sets it. That is a property of these writers, not of the table, and a + // mixed-version deployment breaks it: a replica running a binary from + // before this column existed claims without clearing the stamp, so a held + // row can carry the departure a newer replica recorded. + // + // So held-ness is the question, and the stamp only refines it. Ask + // connectionIsHeld first and read this second; a reader that reads the + // stamp alone reports a connected worker as gone. DisconnectedAt *time.Time `gorm:"index" json:"disconnected_at,omitempty"` } @@ -83,8 +91,11 @@ type NodeConnection struct { // drift, and the drift here would show up as a departed worker resolving to a // replica that holds nothing. // -// Deregister is the one write that does not use it, because it matches an owner -// id, which is narrower: a departed row carries no owner and so cannot match. +// The writes ask it too, and for the same reason Owner does rather than leaning +// on its join: Release and Deregister already match an owner id, but that only +// excludes a departed row while no owner id is ever empty, which is an accident +// of who registers. An empty one would match every departure in the table and +// reset its age. // // The column is table-qualified because Owner reads it across a join, where an // unqualified name is ambiguous; qualifying it costs the single-table readers @@ -340,7 +351,11 @@ func (r *Registry) Release(ctx context.Context, nodeID, ownerID string, epoch in // read off RowsAffected. res := r.db.WithContext(ctx). Model(&NodeConnection{}). - Where("node_id = ? AND owner_instance_id = ? AND epoch = ?", nodeID, ownerID, epoch). + // The held-ness filter is not implied by the owner match: a departed row + // keeps its epoch, so a release naming an empty owner would match it and + // stamp a fresh departure over the old one, making a worker that left + // long ago look like one that has only just gone. + Where("node_id = ? AND owner_instance_id = ? AND epoch = ? AND "+connectionIsHeld, nodeID, ownerID, epoch). Updates(departure()) if res.Error != nil { return fmt.Errorf("releasing connection for node %q held by %q at epoch %d: %w", nodeID, ownerID, epoch, res.Error) diff --git a/core/services/cluster/ownership_test.go b/core/services/cluster/ownership_test.go index 17f88617bad4..54195cfd86a1 100644 --- a/core/services/cluster/ownership_test.go +++ b/core/services/cluster/ownership_test.go @@ -491,6 +491,26 @@ var _ = Describe("Connection ownership", func() { "a replica that only just noticed its dead socket marked a live tunnel as departed") }) + It("refuses a release that names no owner, so a recorded departure cannot be aged backwards", func() { + // A departed row keeps the epoch of the claim that left, and its + // owner is the empty string, so a release naming an empty owner + // matches it on both columns. Stamping a fresh departure there would + // make a worker that left long ago look like one that has only just + // gone, which is the difference every window above is measured from. + epoch, err := reg.Claim(ctx, "w1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "w1", "inst-a", epoch)).To(Succeed()) + var before cluster.NodeConnection + Expect(db.WithContext(ctx).Where("node_id = ?", "w1").First(&before).Error).To(Succeed()) + + Expect(reg.Release(ctx, "w1", "", epoch)).To(MatchError(cluster.ErrNoConnection)) + + var after cluster.NodeConnection + Expect(db.WithContext(ctx).Where("node_id = ?", "w1").First(&after).Error).To(Succeed()) + Expect(after.DisconnectedAt).ToNot(BeNil()) + Expect(*after.DisconnectedAt).To(Equal(*before.DisconnectedAt)) + }) + It("purges a departure older than the retention and keeps a recent one", func() { oldEpoch, err := reg.Claim(ctx, "old", "inst-a") Expect(err).ToNot(HaveOccurred()) From c4e558796f738e0bb962477be7afcf3a90806419 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 14:44:50 +0000 Subject: [PATCH 068/153] feat(cluster): answer presence with four values on the database clock A worker whose tunnel is gone is not, by that fact, a worker that has left. Absence is what makes the scheduler stop placing work, reap the worker's rows and evict its models, and one of those paths runs during inference, so the deployment needs to tell a worker re-homing between frontend replicas from one that is really gone before anything acts. Registry.Presence answers that in one joined statement, with four values and not a boolean: unknown when there is no row at all (this package cannot tell a worker that has never dialled from one whose departure aged out, and must not guess), connected while a live replica holds the tunnel, reconnecting while the departure is inside the grace, and gone once it is older. Only the last is a verdict a caller may act on. Held-ness is asked FIRST and the departure only refines it, in the SQL and again in the switch that reads it. Every writer here clears disconnected_at in the statement that writes the owner, but that is a property of these writers rather than of the table: a replica running a binary from before the column existed re-claims without clearing the stamp, so during a rolling upgrade a held row carries an old departure, and a read that consults the stamp first reports a connected worker as gone for the whole upgrade. Both windows are computed by the database, for the reason every other window in this package is: they are compared across replicas, and replicas disagreeing about whether a worker is gone is the flapping this branch exists to remove. No behavioural spec can see the difference, since the test container shares the host clock, so the statement shape is pinned instead. The grace is an operator's knob, defaulting to twice the worker tunnel's maximum reconnect backoff. That made the fixed departure retention wrong: an operator raising the grace past it gets a purge that deletes departures before the grace elapses, so a worker that is gone reads as unknown forever and nothing ever reaps it. The retention is now derived from the grace, with the old constant as its floor. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 3 + core/cli/run.go | 8 + core/config/distributed_config.go | 54 ++++- core/config/distributed_config_test.go | 29 +++ core/services/cluster/membership.go | 78 ++++++- core/services/cluster/presence.go | 170 ++++++++++++++ core/services/cluster/presence_test.go | 263 ++++++++++++++++++++++ docs/content/features/distributed-mode.md | 19 ++ 8 files changed, 611 insertions(+), 13 deletions(-) create mode 100644 core/services/cluster/presence.go create mode 100644 core/services/cluster/presence_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index a50ca76044e0..f4666b5bf3cf 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -256,6 +256,9 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade "error", err, "knob", "LOCALAI_DISTRIBUTED_ADVERTISE_ADDR") } else { membership = cluster.NewMembership(clusterRegistry, cfg.Distributed.InstanceID, advertised, internal.PrintableVersion()) + // Before Start, so the first sweep already purges on the retention this + // deployment's grace requires rather than on the floor. + membership.SetReconnectGrace(cfg.Distributed.ReconnectGraceOrDefault()) if err := membership.Start(cfg.Context); err != nil { return nil, fmt.Errorf("registering this replica in the cluster: %w", err) } diff --git a/core/cli/run.go b/core/cli/run.go index f613e1da85d8..de407b75c67e 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -185,6 +185,7 @@ type RunCMD struct { ModelLoadWait string `env:"LOCALAI_MODEL_LOAD_WAIT" help:"How long an inference request waits for a model that is still cold-loading onto a worker before it is answered with 503, a Retry-After header and live staging progress (default 60s). The request is served the moment the model becomes ready, so a model already most of the way staged needs no client retry. Set to 0 to wait as long as the load takes — only safe when no ingress or load balancer with an idle timeout sits in front." group:"distributed"` StaleNodeThreshold string `env:"LOCALAI_STALE_NODE_THRESHOLD" help:"How long a worker node may go without a durable heartbeat before the health monitor marks it offline (default 5m). Because a beat that only carries a fresher timestamp is held back by --node-heartbeat-checkpoint, this must stay comfortably wider than that interval; raise both together. Dead-node detection through the per-model gRPC health check and through request-time failure is unaffected by this knob." group:"distributed"` NodeHeartbeatCheckpoint string `env:"LOCALAI_NODE_HEARTBEAT_CHECKPOINT" help:"Minimum gap between durable heartbeat writes for a worker node (default 60s). A beat that only carries a fresher timestamp is dropped until this interval elapses; every field is compared against the value last written, so a node's first beat, a changed total VRAM/total disk/GPU vendor, and a free VRAM/RAM/disk reading that has moved more than 256 MiB from the written value all still write immediately, and a node that is not active is never suppressed. Set below the worker heartbeat interval to write on every beat." group:"distributed"` + WorkerReconnectGrace string `env:"LOCALAI_WORKER_RECONNECT_GRACE" help:"How long a worker whose tunnel was lost is treated as reconnecting rather than gone (default 60s, twice the worker's own maximum reconnect backoff). Only after this window may the scheduler stop placing work on that worker and clean up its rows, so a value below the worker's backoff condemns workers that are re-homing normally; raise it to make a rolling frontend restart safer, lower it to reap a genuinely dead worker sooner. Measured on the database clock, so every replica agrees." group:"distributed"` NatsAccountSeed string `env:"LOCALAI_NATS_ACCOUNT_SEED" help:"NATS account signing seed (SU...) used to mint per-node worker JWTs at registration" group:"distributed"` NatsServiceJWT string `env:"LOCALAI_NATS_SERVICE_JWT" help:"NATS user JWT for the frontend (and agent workers) to publish control-plane messages" group:"distributed"` NatsServiceSeed string `env:"LOCALAI_NATS_SERVICE_SEED" help:"NATS user signing seed (SU...) paired with LOCALAI_NATS_SERVICE_JWT" group:"distributed"` @@ -424,6 +425,13 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { } opts = append(opts, config.WithNodeHeartbeatCheckpoint(d)) } + if r.WorkerReconnectGrace != "" { + d, err := parseDistributedDuration("LOCALAI_WORKER_RECONNECT_GRACE", r.WorkerReconnectGrace) + if err != nil { + return err + } + opts = append(opts, config.WithWorkerReconnectGrace(d)) + } if r.RegistrationToken != "" { opts = append(opts, config.WithRegistrationToken(r.RegistrationToken)) } diff --git a/core/config/distributed_config.go b/core/config/distributed_config.go index 92e31df6df34..3514fa7269a5 100644 --- a/core/config/distributed_config.go +++ b/core/config/distributed_config.go @@ -82,6 +82,17 @@ type DistributedConfig struct { MCPCIJobTimeout time.Duration // MCP CI job execution timeout (default 10m) + // WorkerReconnectGrace is how long a worker whose tunnel was lost is + // treated as reconnecting rather than gone. It is the ONLY thing that + // separates a worker re-homing between frontend replicas from one that has + // left, and absence is what makes the scheduler stop placing work and reap + // the worker's rows, so a grace shorter than the worker's own reconnect + // backoff condemns workers that are behaving exactly as designed. + // + // Zero means unset (DefaultWorkerReconnectGrace applies). Measured on the + // database clock, so every replica agrees on when the window ends. + WorkerReconnectGrace time.Duration // LOCALAI_WORKER_RECONNECT_GRACE + BackendInstallTimeout time.Duration // NATS round-trip timeout for backend.install (default 15m) BackendUpgradeTimeout time.Duration // NATS round-trip timeout for backend.upgrade (default 15m) // ModelLoadTimeout is the gRPC deadline for the remote LoadModel call the @@ -184,6 +195,7 @@ func (c DistributedConfig) Validate() error { FlagBackendInstallTimeout: c.BackendInstallTimeout, FlagBackendUpgradeTimeout: c.BackendUpgradeTimeout, FlagModelLoadTimeout: c.ModelLoadTimeout, + FlagWorkerReconnectGrace: c.WorkerReconnectGrace, } { if d < 0 { return fmt.Errorf("%s must not be negative", name) @@ -324,6 +336,14 @@ func WithStorageSecretKey(key string) AppOption { } } +// WithWorkerReconnectGrace sets how long a lost worker tunnel is read as +// reconnecting rather than gone (see DistributedConfig.WorkerReconnectGrace). +func WithWorkerReconnectGrace(d time.Duration) AppOption { + return func(o *ApplicationConfig) { + o.Distributed.WorkerReconnectGrace = d + } +} + func WithBackendInstallTimeout(d time.Duration) AppOption { return func(o *ApplicationConfig) { o.Distributed.BackendInstallTimeout = d @@ -440,7 +460,10 @@ const ( FlagBackendInstallTimeout = "backend-install-timeout" FlagBackendUpgradeTimeout = "backend-upgrade-timeout" FlagModelLoadTimeout = "model-load-timeout" - FlagModelLoadWait = "model-load-wait" + // FlagWorkerReconnectGrace names the reconnect-grace knob. Validate quotes + // it when the operator hands it a negative duration. + FlagWorkerReconnectGrace = "worker-reconnect-grace" + FlagModelLoadWait = "model-load-wait" // FlagDiskHeadroomCheck names the disk-headroom toggle. It is quoted in // the warning the check emits while disabled, so the operator reading a // log line knows exactly which knob produced it. @@ -471,6 +494,18 @@ const ( // LocalAI (with progress the client can act on) rather than from a proxy // dropping the connection. DefaultModelLoadWait = 60 * time.Second + // DefaultWorkerReconnectGrace is twice the worker tunnel's maximum + // reconnect backoff (30s; tunnelBackoffMax in core/services/worker), so a + // worker that misses one reconnect at the ceiling and lands on the next is + // never condemned for it. + // + // Raising it makes a rolling frontend restart safer, because a worker + // re-homing across it stays "reconnecting" for longer. Lowering it makes a + // worker that really has gone reap sooner. There is no value at which a + // live worker is reported as gone and no value at which a dead one is + // reported as connected; the trade is only in how long the deployment waits + // before it is willing to say "gone". + DefaultWorkerReconnectGrace = 60 * time.Second ) // ModelLoadWaitUnbounded records LOCALAI_MODEL_LOAD_WAIT=0 — "wait as long as @@ -519,6 +554,23 @@ func (c DistributedConfig) NatsAuthConfig() natsauth.Config { } } +// ReconnectGraceOrDefault returns the configured worker reconnect grace or the +// default. +// +// A non-positive value falls back rather than being taken verbatim, which is +// the opposite of what the timeout knobs above do, and deliberately. A +// negative grace makes every departure older than the window the instant it is +// stamped, so a worker two seconds into a normal reconnect reports as GONE, and +// gone is the one answer a caller may reap and evict on. Validate rejects a +// negative duration at startup; this is the second line, for a config built in +// code that never went through it. +func (c DistributedConfig) ReconnectGraceOrDefault() time.Duration { + if c.WorkerReconnectGrace <= 0 { + return DefaultWorkerReconnectGrace + } + return c.WorkerReconnectGrace +} + // BackendInstallTimeoutOrDefault returns the configured timeout or the default. func (c DistributedConfig) BackendInstallTimeoutOrDefault() time.Duration { return cmp.Or(c.BackendInstallTimeout, DefaultBackendInstallTimeout) diff --git a/core/config/distributed_config_test.go b/core/config/distributed_config_test.go index dc98c09bb4e9..938ba5cc8aef 100644 --- a/core/config/distributed_config_test.go +++ b/core/config/distributed_config_test.go @@ -198,3 +198,32 @@ var _ = Describe("DistributedConfig.Validate registration auth", func() { Expect(err.Error()).To(ContainSubstring("LOCALAI_NATS_REQUIRE_AUTH")) }) }) + +var _ = Describe("DistributedConfig worker reconnect grace", func() { + It("defaults to twice the worker tunnel's maximum reconnect backoff", func() { + // The tunnel's ceiling is 30s (core/services/worker, tunnelBackoffMax), + // so 60s covers a worker that misses one reconnect at the ceiling and + // lands on the next. A grace shorter than that condemns workers that + // are re-homing exactly as designed. + Expect(config.DistributedConfig{}.ReconnectGraceOrDefault()).To(Equal(60 * time.Second)) + }) + + It("takes a configured worker reconnect grace verbatim", func() { + cfg := config.DistributedConfig{WorkerReconnectGrace: 5 * time.Minute} + Expect(cfg.ReconnectGraceOrDefault()).To(Equal(5 * time.Minute)) + }) + + It("falls back to the default rather than condemning every worker on a negative value", func() { + // A negative grace makes every departure older than the window the + // instant it is stamped, which reports a worker that has been gone for + // two seconds as GONE, and gone is the one value a caller may reap on. + cfg := config.DistributedConfig{WorkerReconnectGrace: -1 * time.Second} + Expect(cfg.ReconnectGraceOrDefault()).To(Equal(config.DefaultWorkerReconnectGrace)) + }) + + It("is settable through the application option", func() { + o := &config.ApplicationConfig{} + config.WithWorkerReconnectGrace(90 * time.Second)(o) + Expect(o.Distributed.ReconnectGraceOrDefault()).To(Equal(90 * time.Second)) + }) +}) diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index 8dad478a9bf1..1520d41b3560 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -31,8 +31,9 @@ const ( // not the place to wait on a database. deregisterTimeout = 5 * time.Second - // DepartedRetention is how long a connection row outlives the tunnel it - // recorded before the sweep deletes it. + // DepartedRetention is the FLOOR on how long a connection row outlives the + // tunnel it recorded before the sweep deletes it. DepartedRetentionFor is + // what the sweep actually applies; this is what it never goes below. // // A multiple of the liveness window rather than the window itself. The row // survives so that how long ago a tunnel went can be answered at all, and a @@ -42,8 +43,35 @@ const ( // is far enough clear of any such comparison, and short enough that a // worker retired for good does not sit in the table for a day. DepartedRetention = 10 * InstanceLiveness + + // departedRetentionGraceFactor is how many reconnect graces a departure is + // kept for once the grace is the larger of the two. Five, so the purge + // window is nowhere near the window a reader measures against, the same + // margin DepartedRetention keeps over the liveness window. + departedRetentionGraceFactor = 5 ) +// DepartedRetentionFor returns how long a departure must be kept, given the +// reconnect grace the readers of that departure measure against it. +// +// It exists because the two windows are set by different people. The retention +// is this package's, the grace is an operator's +// (DistributedConfig.WorkerReconnectGrace), and a fixed retention is only +// correct while nobody raises the grace past it. An operator who does gets a +// purge that deletes departures BEFORE the grace has elapsed, so Presence stops +// answering PresenceGone for that worker and answers PresenceUnknown forever +// instead. Nobody may act on unknown, so nothing is misreported; but nothing +// ever reaps that worker's rows either, and the leak is silent. +// +// So the retention is derived rather than declared, and the ordering is a +// property of this function rather than of two constants that happen to agree. +func DepartedRetentionFor(grace time.Duration) time.Duration { + if scaled := departedRetentionGraceFactor * grace; scaled > DepartedRetention { + return scaled + } + return DepartedRetention +} + // Membership publishes this replica's address and keeps the instances table // free of replicas that have stopped answering. // @@ -58,6 +86,11 @@ type Membership struct { interval time.Duration liveness time.Duration + // retention is how long a departure is kept before the purge deletes it. + // It is derived from the reconnect grace rather than fixed, because the + // grace is the window Presence measures departures against and the purge + // must never outrun it (see DepartedRetentionFor). + retention time.Duration stop chan struct{} done chan struct{} @@ -75,14 +108,15 @@ type Membership struct { // address this process binds. func NewMembership(reg *Registry, id, addr, version string) *Membership { return &Membership{ - reg: reg, - id: id, - addr: addr, - version: version, - interval: InstanceHeartbeat, - liveness: InstanceLiveness, - stop: make(chan struct{}), - done: make(chan struct{}), + reg: reg, + id: id, + addr: addr, + version: version, + interval: InstanceHeartbeat, + liveness: InstanceLiveness, + retention: DepartedRetention, + stop: make(chan struct{}), + done: make(chan struct{}), } } @@ -108,6 +142,23 @@ func (m *Membership) SetTunnels(t *TunnelRegistry) { m.tunnels = t } +// SetReconnectGrace tells the loop the window Presence measures a departure +// against, so the purge keeps departures for longer than any reader needs them. +// A Membership that is never told one purges on the DepartedRetention floor, +// which is correct for every grace at or below it. +// +// It is a setter for the reason SetTunnels is: this package deliberately +// produces a nil *Membership (core/application/distributed.go leaves it nil +// when no peer-reachable address can be derived), so it is safe on one. +func (m *Membership) SetReconnectGrace(grace time.Duration) { + if m == nil { + return + } + m.mu.Lock() + defer m.mu.Unlock() + m.retention = DepartedRetentionFor(grace) +} + // Start registers this replica and begins heartbeating and sweeping. The first // registration is synchronous and its failure is returned: a replica whose // address never reaches the table is invisible to its peers, and starting @@ -228,13 +279,16 @@ func (m *Membership) tick(ctx context.Context) { // nothing ever deletes is a row per worker that ever dialled this // deployment, and it is the same loop that decides a replica is gone, so // there is one schedule rather than two. - purged, err := m.reg.PurgeDepartedBefore(ctx, DepartedRetention) + m.mu.Lock() + retention := m.retention + m.mu.Unlock() + purged, err := m.reg.PurgeDepartedBefore(ctx, retention) if err != nil { xlog.Warn("Purging departed worker connections failed", "error", err) return } if purged > 0 { - xlog.Info("Purged worker connections whose departure aged out", "connections", purged, "retention", DepartedRetention) + xlog.Info("Purged worker connections whose departure aged out", "connections", purged, "retention", retention) } } diff --git a/core/services/cluster/presence.go b/core/services/cluster/presence.go new file mode 100644 index 000000000000..bcdd10e7d0fe --- /dev/null +++ b/core/services/cluster/presence.go @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: MIT + +package cluster + +import ( + "context" + "fmt" + "time" +) + +// Presence is what this package can say about a worker's tunnel. +// +// Four values and not a boolean, because the conditions underneath are four and +// only ONE of them is a verdict a caller may act on. Collapsing any pair is the +// defect this phase exists to prevent: absence is what makes the scheduler stop +// placing work, reap node rows and evict models, and one of those paths runs +// during inference, so a worker misreported as absent loses the models it is +// serving at that moment. +// +// What this type does NOT cover is the two conditions that live on the dialing +// path: a replica that would not answer, and the worker's own reply (including +// its "that backend is not there"). Those are IsWorkerAnswer and ErrNoRoute in +// dialer.go, and they are deliberately not spelled again here. +type Presence uint8 + +const ( + // PresenceUnknown reports that no connection row exists for this node at + // all. This package cannot tell a worker that has never dialled from one + // whose departure aged out of retention, and it must not guess: the first + // is a worker still starting up and the second is one long gone. + // + // It is the zero value on purpose, so a Presence returned alongside an + // error is the value nobody may act on rather than one that reads as a + // verdict. + PresenceUnknown Presence = iota + // PresenceConnected reports that a LIVE replica holds this worker's tunnel. + PresenceConnected + // PresenceReconnecting reports that no live replica holds the tunnel and + // the grace has not run out: either the departure is recent, or the owning + // replica died and no departure has been stamped yet. A retry, never a + // verdict; nobody may act on it. + PresenceReconnecting + // PresenceGone reports that no live replica holds the tunnel and the + // departure is older than the grace. This is the ONLY value a caller may + // read as absence. + PresenceGone +) + +// String names the value for a log line. The default case names the number +// rather than falling through to a real value: a caller reading "gone" out of a +// log for a value that does not exist would be reading the one answer it is +// allowed to reap on. +func (p Presence) String() string { + switch p { + case PresenceUnknown: + return "unknown" + case PresenceConnected: + return "connected" + case PresenceReconnecting: + return "reconnecting" + case PresenceGone: + return "gone" + default: + return fmt.Sprintf("presence(%d)", uint8(p)) + } +} + +// presenceQuery answers all three questions about one node in one statement: +// does a row exist, does a LIVE replica hold it, and is its departure older +// than the grace. +// +// One statement rather than a read followed by an instance lookup, for the +// reason Owner is one statement: between two reads the owning replica can die, +// and the answer would then be assembled from two different snapshots of the +// cluster. Phase 2 shipped exactly that as a dialer relaying into a corpse for +// a whole liveness window. +// +// HELD-NESS IS ASKED FIRST, and the departure only refines it. Every writer in +// this binary clears disconnected_at in the same statement that writes the +// owner, but that is a property of these writers rather than of the table: a +// replica running a binary from before the column existed re-claims WITHOUT +// clearing the stamp, so during a rolling upgrade a HELD row can carry a +// departure older than any grace. Reading the stamp first, or treating a +// non-null stamp as evidence of absence on its own, reports a worker that is +// connected right now as gone. That is why `departed` carries the negation of +// connectionIsHeld rather than standing on the timestamp alone. +// +// Both windows are computed by the DATABASE. They are compared across replicas, +// so a Go-side cutoff would make the effective window depend on each replica's +// clock skew, and replicas disagreeing about whether a worker is gone is the +// flapping this branch exists to remove. No behavioural spec can see the +// difference (the test container shares the host clock), which is why the +// statement shape is pinned instead. +// +// The predicates are the package's own, not copies. Two spellings of "live" or +// of "held" drift, and the drift here reads as a worker that one query calls +// connected and another calls gone. +// +// The bind order is the order the placeholders appear in the text: the grace, +// then the liveness window, then the node id. +const presenceQuery = ` +SELECT + (` + connectionIsHeld + ` AND instances.id IS NOT NULL) AS held, + (NOT (` + connectionIsHeld + `) + AND node_connections.disconnected_at IS NOT NULL + AND node_connections.disconnected_at < now() - make_interval(secs => ?)) AS departed +FROM node_connections +LEFT JOIN instances + ON instances.id = node_connections.owner_instance_id + AND ` + instanceIsLive + ` +WHERE node_connections.node_id = ?` + +// Presence reports what this deployment can say about nodeID's tunnel, with +// grace as the window a departure has to outlive before it becomes a verdict. +// +// The grace is a parameter rather than a constant because it is an operator's +// trade (see DistributedConfig.WorkerReconnectGrace), where the liveness window +// inside the query is not: that one has to be the window the membership loop +// sweeps with, or a reader would keep trusting an owner the sweeper has already +// declared dead. +// +// A held row whose owner is no longer live is PresenceReconnecting and not +// PresenceGone, deliberately. Nothing has stamped a departure on it yet, so +// there is no age to compare and the grace clock has not started; the +// membership sweep is what starts it. A replica dying must not condemn every +// worker it held. +func (r *Registry) Presence(ctx context.Context, nodeID string, grace time.Duration) (Presence, error) { + // Refused rather than attempted, for the reason Owner refuses: now() and + // make_interval are PostgreSQL, so on the single-binary SQLite path this + // would fail with "no such function: now", which reads as a missing + // migration. The refusal carries PresenceUnknown, the one value nobody may + // act on: a deployment with no cluster has no answer about a worker's + // tunnel, and reporting PresenceGone would license a reap. + if !isPostgres(r.db) { + return PresenceUnknown, fmt.Errorf("reading presence of node %q: connection ownership requires PostgreSQL, this deployment runs on %q", nodeID, r.db.Dialector.Name()) + } + var row struct { + Held bool + Departed bool + } + // gorm's Scan leaves the destination zero-valued when nothing matched and + // reports no error, so "no row" and "a row whose booleans are both false" + // are told apart by RowsAffected and by nothing else. Scan does not produce + // gorm.ErrRecordNotFound, so matching on that would silently turn every + // missing row into PresenceReconnecting. + res := r.db.WithContext(ctx).Raw(presenceQuery, + grace.Seconds(), InstanceLiveness.Seconds(), nodeID, + ).Scan(&row) + if res.Error != nil { + return PresenceUnknown, fmt.Errorf("reading presence of node %q: %w", nodeID, res.Error) + } + if res.RowsAffected == 0 { + return PresenceUnknown, nil + } + switch { + case row.Held: + // First, and redundantly so: the query already excludes held rows from + // `departed`, so either gate alone answers correctly and no behavioural + // spec can tell which one is doing the work. The redundancy is kept + // because held-ness-first is the invariant the phase rests on, and this + // is what still answers correctly if the SQL gate is ever edited away. + // The SQL gate has its own assertion in the statement-shape spec, so + // removing it is caught there rather than here. + return PresenceConnected, nil + case row.Departed: + return PresenceGone, nil + default: + return PresenceReconnecting, nil + } +} diff --git a/core/services/cluster/presence_test.go b/core/services/cluster/presence_test.go new file mode 100644 index 000000000000..0aceca2b1963 --- /dev/null +++ b/core/services/cluster/presence_test.go @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: MIT + +package cluster_test + +import ( + "context" + "path/filepath" + "strings" + "time" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// age pushes a departure or a heartbeat into the past ON THE DATABASE CLOCK. +// Writing a Go-side time.Now().Add(-d) would age the row against this process's +// clock and then compare it against the database's, which is precisely the skew +// every window in this package is written to be immune to; a helper that did +// that would make the specs agree with an implementation that has the bug. +func age(ctx context.Context, db *gorm.DB, table, column, keyColumn, key string, by time.Duration) { + GinkgoHelper() + Expect(db.WithContext(ctx).Exec( + `UPDATE `+table+` SET `+column+` = now() - make_interval(secs => ?) WHERE `+keyColumn+` = ?`, + by.Seconds(), key).Error).To(Succeed()) +} + +var _ = Describe("Presence", func() { + var ( + ctx context.Context + db *gorm.DB + reg *cluster.Registry + ) + + // The grace every spec measures against, so the ageing below can sit just + // over the edge of it rather than an order of magnitude past it: a window + // only ever aged ten times its own width is a window nothing pins, and + // widening it stays green. + const grace = 60 * time.Second + + BeforeEach(func() { + ctx = context.Background() + db = testutil.SetupTestDB() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg = cluster.NewRegistry(db) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + }) + + It("reports unknown for a node with no connection row", func() { + // Not "gone". This package cannot tell a worker that has never dialled + // from one whose departure aged out of retention, and a caller that + // reaps on absence would reap a worker that is still starting up. + p, err := reg.Presence(ctx, "never-seen", grace) + Expect(err).ToNot(HaveOccurred()) + Expect(p).To(Equal(cluster.PresenceUnknown)) + }) + + It("reports connected while a live replica holds the tunnel", func() { + _, err := reg.Claim(ctx, "node-1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + + p, err := reg.Presence(ctx, "node-1", grace) + Expect(err).ToNot(HaveOccurred()) + Expect(p).To(Equal(cluster.PresenceConnected)) + }) + + It("reports connected for a held row that still carries an old departure stamp", func() { + // The rolling-upgrade case, and the one thing this read must not get + // wrong. Every writer in THIS binary clears disconnected_at in the same + // statement that writes the owner, but a replica running a binary from + // before the column existed re-claims WITHOUT clearing it, so a held row + // can carry a departure older than any grace. Held-ness is the question + // and the stamp only refines it; a read that consults the stamp first, + // or treats a non-null stamp as evidence of absence, reports a worker + // that is connected right now as gone, for the whole upgrade. + _, err := reg.Claim(ctx, "node-1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + age(ctx, db, "node_connections", "disconnected_at", "node_id", "node-1", 10*grace) + + p, err := reg.Presence(ctx, "node-1", grace) + Expect(err).ToNot(HaveOccurred()) + Expect(p).To(Equal(cluster.PresenceConnected), + "a row whose connection is HELD is present whatever its departure stamp says") + }) + + It("reports reconnecting immediately after the tunnel is released", func() { + epoch, err := reg.Claim(ctx, "node-1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "node-1", "inst-a", epoch)).To(Succeed()) + + p, err := reg.Presence(ctx, "node-1", grace) + Expect(err).ToNot(HaveOccurred()) + Expect(p).To(Equal(cluster.PresenceReconnecting), + "a departure inside the grace is a retry, not a verdict; nobody may act on it") + }) + + It("reports gone once the departure is older than the grace", func() { + epoch, err := reg.Claim(ctx, "node-1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "node-1", "inst-a", epoch)).To(Succeed()) + // One second past the edge, deliberately. Ageing by ten graces would + // leave a widened window green, and a window nothing pins is the defect + // class this spec exists for. + age(ctx, db, "node_connections", "disconnected_at", "node_id", "node-1", grace+time.Second) + + p, err := reg.Presence(ctx, "node-1", grace) + Expect(err).ToNot(HaveOccurred()) + Expect(p).To(Equal(cluster.PresenceGone)) + }) + + It("still reports reconnecting one second inside the grace", func() { + // The other edge. Without it, a read that treats any departure as gone + // passes the spec above and condemns every worker re-homing normally. + epoch, err := reg.Claim(ctx, "node-1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "node-1", "inst-a", epoch)).To(Succeed()) + age(ctx, db, "node_connections", "disconnected_at", "node_id", "node-1", grace-time.Second) + + p, err := reg.Presence(ctx, "node-1", grace) + Expect(err).ToNot(HaveOccurred()) + Expect(p).To(Equal(cluster.PresenceReconnecting)) + }) + + It("reports reconnecting, not gone, when the OWNING REPLICA dies and the row is still held", func() { + // A replica dying must not condemn every worker it held. The row still + // names inst-a and inst-a is no longer live, so nothing holds the + // tunnel; but no departure has been stamped, so the grace clock has not + // started and there is no age to compare. That is a retry. + _, err := reg.Claim(ctx, "node-1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + age(ctx, db, "instances", "last_seen", "id", "inst-a", cluster.InstanceLiveness+5*time.Second) + + p, err := reg.Presence(ctx, "node-1", grace) + Expect(err).ToNot(HaveOccurred()) + Expect(p).To(Equal(cluster.PresenceReconnecting)) + }) + + It("reports gone after the membership sweep has stamped the departure and the grace has passed", func() { + // The full path a dead replica's worker takes: the sweep records the + // departure, and only then does the grace start running. + _, err := reg.Claim(ctx, "node-1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + age(ctx, db, "instances", "last_seen", "id", "inst-a", cluster.InstanceLiveness+5*time.Second) + + _, connections, err := reg.ReapStale(ctx, "inst-sweeper", cluster.InstanceLiveness) + Expect(err).ToNot(HaveOccurred()) + Expect(connections).To(Equal(int64(1)), "the sweep must have stamped the departure this spec then ages") + age(ctx, db, "node_connections", "disconnected_at", "node_id", "node-1", grace+time.Second) + + p, err := reg.Presence(ctx, "node-1", grace) + Expect(err).ToNot(HaveOccurred()) + Expect(p).To(Equal(cluster.PresenceGone)) + }) + + It("answers in one joined statement measured on the database clock", func() { + _, err := reg.Claim(ctx, "node-1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + + rec := newSQLRecorder() + recording := cluster.NewRegistry(db.Session(&gorm.Session{Logger: rec})) + _, err = recording.Presence(ctx, "node-1", grace) + Expect(err).ToNot(HaveOccurred()) + + sql := strings.ToLower(rec.only()) + // only() rules out the read-then-look-up shape: between two statements + // the owning replica can die, and the answer would then be built from + // two different snapshots of the cluster. + Expect(sql).To(ContainSubstring("join")) + Expect(sql).To(ContainSubstring("instances")) + // The departure test is gated on held-ness IN THE SQL, and not only by + // the order of the switch that reads these two booleans. Either gate + // alone gives the right answer today, so no behavioural spec can tell + // which one is doing the work, and a single-layer regression would sit + // here unnoticed until the second layer moved too. Held-ness first is + // the invariant the whole phase rests on, so both layers are pinned. + Expect(strings.Join(strings.Fields(sql), " ")).To(ContainSubstring( + "not (node_connections.owner_instance_id <> '') and node_connections.disconnected_at is not null")) + // Both windows are computed by the database. A Go-side comparison would + // appear here as a bound literal, and replica clock skew would then move + // the grace and the liveness window per replica. The test container + // shares this host's clock, so no behavioural spec can catch that; the + // statement shape is the only place it is visible. + Expect(strings.Count(sql, "make_interval")).To(Equal(2), + "both the liveness window and the reconnect grace must be computed by the database") + Expect(sql).To(ContainSubstring("now()")) + Expect(sql).ToNot(MatchRegexp(`disconnected_at\s*<\s*'`), + "the grace cutoff must not be a literal timestamp from this process's clock") + Expect(sql).ToNot(MatchRegexp(`last_seen\s*>\s*'`), + "the liveness cutoff must not be a literal timestamp from this process's clock") + }) + + Describe("naming the values", func() { + It("gives every value a name a log line can carry", func() { + Expect(cluster.PresenceUnknown.String()).To(Equal("unknown")) + Expect(cluster.PresenceConnected.String()).To(Equal("connected")) + Expect(cluster.PresenceReconnecting.String()).To(Equal("reconnecting")) + Expect(cluster.PresenceGone.String()).To(Equal("gone")) + }) + + It("does not name an unknown value after a real one", func() { + // A default branch that fell through to "gone" would put the one + // value callers may act on into a log line for a value that does + // not exist. + Expect(cluster.Presence(200).String()).To(ContainSubstring("200")) + }) + }) + + Describe("the retention that outlives the grace", func() { + // The retention is not pinned to any one grace value on purpose. It is + // the purge's window and the grace is a reader's window, and the only + // property that matters is that the first always outlasts the second: + // a purge that deletes a departure a reader is still measuring against + // turns a worker inside its reconnect grace back into a worker that was + // never here, and an operator raising the grace past a FIXED retention + // would leave a genuinely gone worker reading as unknown forever, so + // nothing ever reaps it. + DescribeTable("outlasts the grace a reader measures against", + func(grace time.Duration) { + Expect(cluster.DepartedRetentionFor(grace)).To(BeNumerically(">", grace)) + }, + Entry("a grace of one second", time.Second), + Entry("the default grace", 60*time.Second), + Entry("a grace at the retention floor", cluster.DepartedRetention), + Entry("a grace far past the floor", 10*cluster.DepartedRetention), + ) + + It("never shortens below the floor for a very small grace", func() { + // A tiny grace must not shrink the retention to match: the row is also + // what a restarting deployment reads to tell a re-dialling worker from + // one it has never seen. + Expect(cluster.DepartedRetentionFor(time.Second)).To(Equal(cluster.DepartedRetention)) + }) + }) +}) + +var _ = Describe("Presence on a non-PostgreSQL dialect", func() { + var ( + db *gorm.DB + ctx context.Context + ) + + BeforeEach(func() { + var err error + ctx = context.Background() + db, err = gorm.Open(sqlite.Open(filepath.Join(GinkgoT().TempDir(), "cluster.db")), &gorm.Config{}) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + }) + + It("refuses to answer, rather than failing as a missing function", func() { + p, err := cluster.NewRegistry(db).Presence(ctx, "node-1", time.Minute) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("requires PostgreSQL")) + Expect(err.Error()).ToNot(ContainSubstring("no such function"), + "a dialect that cannot answer must say so, not surface as a missing migration") + Expect(p).To(Equal(cluster.PresenceUnknown), + "a refusal must carry the value nobody may act on, never one that reads as absence") + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 8efa7aa1fc33..bf326bdbec32 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -245,6 +245,25 @@ A worker on this release opens **no inbound listener on a routable interface**. - **`LOCALAI_ADDR` and `LOCALAI_SERVE_ADDR` are read for their port only.** The port is the base of the backend port range, and `port-1` is the HTTP file-transfer port. The host half names an interface nothing binds. - The node's `address` and `http_address` fields in `GET /api/nodes` are empty, and are cleared for nodes that reported them before the upgrade. +#### A lost tunnel is a departure, not an absence + +When a worker's tunnel goes, the frontend does **not** forget the worker. It records *when* the tunnel went, and for a grace period after that the worker is reported as **reconnecting**, not as gone. Only once the departure is older than the grace may anything act on the worker's absence: stop scheduling work onto it, clean up its rows, release its models. + +That distinction exists because a worker loses its tunnel for entirely ordinary reasons. A frontend replica restarting during a rolling upgrade drops every tunnel it held, and each of those workers immediately re-dials the load balancer and lands on another replica. Treating that as "the worker is gone" would evict models mid-upgrade for a fleet that never actually went anywhere. + +| Flag | Env var | Default | Description | +|------|---------|---------|-------------| +| `--worker-reconnect-grace` | `LOCALAI_WORKER_RECONNECT_GRACE` | `60s` | How long a worker whose tunnel was lost is treated as reconnecting rather than gone. | + +The default is **twice the worker's own maximum reconnect backoff** (the worker retries with exponential backoff capped at 30s), so a worker that misses one attempt at the ceiling and succeeds on the next is never condemned for it. + +What you trade by changing it: + +- **Lower**: a worker that has genuinely gone away is declared absent sooner, so its rows are cleaned and its models released sooner. Set it below the worker's backoff ceiling and you will condemn workers that are re-homing exactly as designed. +- **Higher**: a rolling frontend restart is safer, because workers crossing it stay "reconnecting" for longer. The cost is that a worker that really has died keeps its rows for longer. + +The window is measured on the **database clock**, not on any frontend's own clock, so every replica agrees to the second on when a worker's grace ran out. A departure row is kept well past the grace before it is purged; once purged, the worker reads as *unknown* rather than *gone*, and nothing acts on unknown. + ### The model load deadline scales with the checkpoint The `LoadModel` deadline starts *after* the backend is installed and the model files are staged, so it covers only the worker backend's own checkpoint read and pipeline init. That work is proportional to the bytes on disk, which makes any fixed deadline a model-size cliff rather than a timeout: a 70 GB video checkpoint on a Jetson Thor worker failed reproducibly against the old fixed 5m default (`rpc error: code = DeadlineExceeded` after 953.5s of wall clock, roughly 11m of which was backend install and staging), and simply raising the constant would only move the cliff to the next larger model while making a genuinely wedged *small* model hang for the whole inflated duration. From 29a0fd371084df00f4a2c301420e765d39f9dcad Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 15:21:04 +0000 Subject: [PATCH 069/153] fix(cluster): pin the derived retention to the sweep that applies it The retention a worker's departure is kept for is now derived from the reconnect grace, so a purge can never outrun the window Presence measures against. Nothing pinned that. The sweep could be reverted to pass the constant, or the setter emptied out, and the suite stayed green either way: the specs covered the arithmetic helper, and the fix is the wiring. The loop now has a spec of its own. It departs two workers either side of the difference between the floor and the derived retention, and the row that must go is what witnesses the sweep running at all, so the row that must stay cannot survive by nothing happening. The default grace goes from 60s to 90s. Two of the worker's ceiling backoffs is 60s, but the failed dial between them costs its handshake timeout too, which puts the worst case at 70s, and the backoff resets only after a session long enough that a replica accepting a dial and then dying denies it. So the ceiling is reachable exactly during the rolling restart this window exists for, and 60s sat on the edge of it. Too short reports a live worker as gone and costs a model reload; too long reaps a dead one later. The cheaper mistake is the long one. A held row whose owner is dead and whose stamp is stale is the state a rolling upgrade actually produces, and it was the one state no spec built. It has an answer now, and the two ways to get this wrong land either side of it: reading the stamp first says gone, reading held-ness without the liveness join says connected. Two comments claimed more than the code did. There IS a grace at which a live worker is reported as gone, which is the point of it being a duration; and the switch that reads held-ness first is only a partial second gate, since with the SQL gate gone and a dead owner it answers gone rather than reconnecting. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/cli/run.go | 2 +- core/config/distributed_config.go | 34 ++++++++++------ core/config/distributed_config_test.go | 32 ++++++++++++--- core/services/cluster/membership.go | 9 +++++ core/services/cluster/membership_test.go | 47 +++++++++++++++++++++++ core/services/cluster/presence.go | 15 +++++--- core/services/cluster/presence_test.go | 43 +++++++++++++++++++++ core/services/cluster/tunnel_test.go | 11 ++++++ docs/content/features/distributed-mode.md | 5 ++- 9 files changed, 172 insertions(+), 26 deletions(-) diff --git a/core/cli/run.go b/core/cli/run.go index de407b75c67e..410e2f465106 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -185,7 +185,7 @@ type RunCMD struct { ModelLoadWait string `env:"LOCALAI_MODEL_LOAD_WAIT" help:"How long an inference request waits for a model that is still cold-loading onto a worker before it is answered with 503, a Retry-After header and live staging progress (default 60s). The request is served the moment the model becomes ready, so a model already most of the way staged needs no client retry. Set to 0 to wait as long as the load takes — only safe when no ingress or load balancer with an idle timeout sits in front." group:"distributed"` StaleNodeThreshold string `env:"LOCALAI_STALE_NODE_THRESHOLD" help:"How long a worker node may go without a durable heartbeat before the health monitor marks it offline (default 5m). Because a beat that only carries a fresher timestamp is held back by --node-heartbeat-checkpoint, this must stay comfortably wider than that interval; raise both together. Dead-node detection through the per-model gRPC health check and through request-time failure is unaffected by this knob." group:"distributed"` NodeHeartbeatCheckpoint string `env:"LOCALAI_NODE_HEARTBEAT_CHECKPOINT" help:"Minimum gap between durable heartbeat writes for a worker node (default 60s). A beat that only carries a fresher timestamp is dropped until this interval elapses; every field is compared against the value last written, so a node's first beat, a changed total VRAM/total disk/GPU vendor, and a free VRAM/RAM/disk reading that has moved more than 256 MiB from the written value all still write immediately, and a node that is not active is never suppressed. Set below the worker heartbeat interval to write on every beat." group:"distributed"` - WorkerReconnectGrace string `env:"LOCALAI_WORKER_RECONNECT_GRACE" help:"How long a worker whose tunnel was lost is treated as reconnecting rather than gone (default 60s, twice the worker's own maximum reconnect backoff). Only after this window may the scheduler stop placing work on that worker and clean up its rows, so a value below the worker's backoff condemns workers that are re-homing normally; raise it to make a rolling frontend restart safer, lower it to reap a genuinely dead worker sooner. Measured on the database clock, so every replica agrees." group:"distributed"` + WorkerReconnectGrace string `env:"LOCALAI_WORKER_RECONNECT_GRACE" help:"How long a worker whose tunnel was lost is treated as reconnecting rather than gone (default 90s, clear of two of the worker's own ceiling backoffs plus the dial between them). Only after this window may the scheduler stop placing work on that worker and clean up its rows, so a value below the worker's backoff condemns workers that are re-homing normally; raise it to make a rolling frontend restart safer, lower it to reap a genuinely dead worker sooner. Measured on the database clock, so every replica agrees." group:"distributed"` NatsAccountSeed string `env:"LOCALAI_NATS_ACCOUNT_SEED" help:"NATS account signing seed (SU...) used to mint per-node worker JWTs at registration" group:"distributed"` NatsServiceJWT string `env:"LOCALAI_NATS_SERVICE_JWT" help:"NATS user JWT for the frontend (and agent workers) to publish control-plane messages" group:"distributed"` NatsServiceSeed string `env:"LOCALAI_NATS_SERVICE_SEED" help:"NATS user signing seed (SU...) paired with LOCALAI_NATS_SERVICE_JWT" group:"distributed"` diff --git a/core/config/distributed_config.go b/core/config/distributed_config.go index 3514fa7269a5..976bb0caa3d2 100644 --- a/core/config/distributed_config.go +++ b/core/config/distributed_config.go @@ -494,18 +494,30 @@ const ( // LocalAI (with progress the client can act on) rather than from a proxy // dropping the connection. DefaultModelLoadWait = 60 * time.Second - // DefaultWorkerReconnectGrace is twice the worker tunnel's maximum - // reconnect backoff (30s; tunnelBackoffMax in core/services/worker), so a - // worker that misses one reconnect at the ceiling and lands on the next is - // never condemned for it. + // DefaultWorkerReconnectGrace covers a worker that misses one reconnect at + // the ceiling and lands on the next, with margin. The worker's own numbers + // (core/services/worker/tunnel.go) are a 30s backoff ceiling + // (tunnelBackoffMax) and a 10s dial budget (tunnelHandshakeTimeout), so two + // ceiling waits with a hung dial between them puts the worker back at 70s, + // not 60s: two waits alone is the boundary, not a bound. // - // Raising it makes a rolling frontend restart safer, because a worker - // re-homing across it stays "reconnecting" for longer. Lowering it makes a - // worker that really has gone reap sooner. There is no value at which a - // live worker is reported as gone and no value at which a dead one is - // reported as connected; the trade is only in how long the deployment waits - // before it is willing to say "gone". - DefaultWorkerReconnectGrace = 60 * time.Second + // The ceiling is reachable precisely when it matters. The backoff resets + // only after a session that lasted tunnelHealthyAfter (30s), which a + // replica accepting a dial and then dying denies, so a worker crossing a + // rolling frontend restart climbs to the ceiling rather than sitting near + // the 500ms floor. + // + // 90s therefore has margin where 60s sat on the edge. The asymmetry is + // deliberate: too short and a worker that is reconnecting exactly as + // designed is reported GONE, which licenses a reap and costs a model + // reload; too long and a worker that really has died is reaped later. The + // second is cheaper, so the default errs long. + // + // Raising it further makes a rolling frontend restart safer still; lowering + // it reaps a dead worker sooner. There IS a value at which a live worker is + // reported as gone: any grace shorter than that worker's actual reconnect. + // That is why this is a duration and not a boolean. + DefaultWorkerReconnectGrace = 90 * time.Second ) // ModelLoadWaitUnbounded records LOCALAI_MODEL_LOAD_WAIT=0 — "wait as long as diff --git a/core/config/distributed_config_test.go b/core/config/distributed_config_test.go index 938ba5cc8aef..031d455ea6ca 100644 --- a/core/config/distributed_config_test.go +++ b/core/config/distributed_config_test.go @@ -200,12 +200,15 @@ var _ = Describe("DistributedConfig.Validate registration auth", func() { }) var _ = Describe("DistributedConfig worker reconnect grace", func() { - It("defaults to twice the worker tunnel's maximum reconnect backoff", func() { - // The tunnel's ceiling is 30s (core/services/worker, tunnelBackoffMax), - // so 60s covers a worker that misses one reconnect at the ceiling and - // lands on the next. A grace shorter than that condemns workers that - // are re-homing exactly as designed. - Expect(config.DistributedConfig{}.ReconnectGraceOrDefault()).To(Equal(60 * time.Second)) + It("defaults clear of two ceiling backoffs plus the dial between them", func() { + // The worker's own numbers (core/services/worker/tunnel.go): a 30s + // backoff ceiling and a 10s dial budget, so two ceiling waits with a + // hung dial between them puts the worker back at 70s. 60s would sit + // under that and condemn a worker reconnecting exactly as designed; + // 90s clears it with margin. + Expect(config.DistributedConfig{}.ReconnectGraceOrDefault()).To(Equal(90 * time.Second)) + Expect(config.DefaultWorkerReconnectGrace).To(BeNumerically(">", 70*time.Second), + "the default must clear two ceiling backoffs plus one handshake timeout") }) It("takes a configured worker reconnect grace verbatim", func() { @@ -221,6 +224,23 @@ var _ = Describe("DistributedConfig worker reconnect grace", func() { Expect(cfg.ReconnectGraceOrDefault()).To(Equal(config.DefaultWorkerReconnectGrace)) }) + It("refuses to start on a negative grace rather than reaping on it", func() { + // The flag is in Validate's negative-duration table, and this is what + // says so. A negative grace makes every departure older than the window + // the instant it is stamped, so the deployment would answer GONE for + // every worker that has ever lost a tunnel, and gone is the one answer + // a caller may reap and evict on. + c := config.DistributedConfig{ + Enabled: true, + NatsURL: "nats://localhost:4222", + RegistrationToken: "tok", + WorkerReconnectGrace: -1 * time.Second, + } + err := c.Validate() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(config.FlagWorkerReconnectGrace)) + }) + It("is settable through the application option", func() { o := &config.ApplicationConfig{} config.WithWorkerReconnectGrace(90 * time.Second)(o) diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index 1520d41b3560..b144afca2cdc 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -6,6 +6,7 @@ import ( "context" "errors" "fmt" + "math" "sync" "time" @@ -66,6 +67,14 @@ const ( // So the retention is derived rather than declared, and the ordering is a // property of this function rather than of two constants that happen to agree. func DepartedRetentionFor(grace time.Duration) time.Duration { + // Guarded before the multiply, not after. A grace above roughly 58 years + // overflows time.Duration's int64 nanoseconds, and the product comes back + // negative or small, so a plain `scaled > DepartedRetention` would fall + // through to the floor and reinstate the very defect this function exists + // to remove, silently and only for the operator who set the largest window. + if grace > time.Duration(math.MaxInt64)/departedRetentionGraceFactor { + return time.Duration(math.MaxInt64) + } if scaled := departedRetentionGraceFactor * grace; scaled > DepartedRetention { return scaled } diff --git a/core/services/cluster/membership_test.go b/core/services/cluster/membership_test.go index 653726bee2ea..188295bb494f 100644 --- a/core/services/cluster/membership_test.go +++ b/core/services/cluster/membership_test.go @@ -228,6 +228,53 @@ var _ = Describe("Reaping dead replicas", func() { "the loop that sweeps dead replicas never applied the departure retention") }) + It("purges on the retention its reconnect grace requires, not on the floor", func() { + // The wiring, not the arithmetic. DepartedRetentionFor is pinned on its + // own in presence_test.go, and a sweep that ignored it and passed the + // DepartedRetention constant would satisfy every one of those + // expectations: the derived value is only worth anything if the loop + // that deletes rows is the thing reading it. + // + // A grace of 10m derives a retention of 50m, well past the 5m floor, so + // the two rows below straddle the difference and one answer cannot + // stand in for the other. + const grace = 10 * time.Minute + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + depart := func(node string, ago time.Duration) { + epoch, err := reg.Claim(ctx, node, "me") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, node, "me", epoch)).To(Succeed()) + // Aged on the database clock, like every window in this package. + Expect(db.WithContext(ctx).Exec( + `UPDATE node_connections SET disconnected_at = now() - make_interval(secs => ?) WHERE node_id = ?`, + ago.Seconds(), node).Error).To(Succeed()) + } + // Past the 5m floor and inside the 50m the grace derives. A sweep + // applying the floor deletes this row, which is the whole defect: the + // purge would be outrunning the window Presence measures against, and + // a worker that really is gone would read as unknown for ever after. + depart("w-inside-derived-retention", cluster.DepartedRetention+time.Minute) + // Past both, so its deletion is the WITNESS that the sweep ran at all. + // Without it "the row survived" would also be true of a loop that never + // ticked, and the spec would pass on nothing happening. + depart("w-past-derived-retention", cluster.DepartedRetentionFor(grace)+time.Minute) + + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + membership.SetReconnectGrace(grace) + Expect(membership.Start(ctx)).To(Succeed()) + DeferCleanup(membership.Stop) + + // One assertion for both halves: rows are only ever deleted, so the set + // settles, and the settled set says which retention the loop applied. + Eventually(func() ([]string, error) { + var ids []string + err := db.WithContext(ctx).Model(&cluster.NodeConnection{}). + Order("node_id").Pluck("node_id", &ids).Error + return ids, err + }, "20s", "250ms").Should(Equal([]string{"w-inside-derived-retention"}), + "the sweep applied a retention that is not the one this replica's reconnect grace derives") + }) + It("deregisters when the membership loop stops", func() { membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") Expect(membership.Start(ctx)).To(Succeed()) diff --git a/core/services/cluster/presence.go b/core/services/cluster/presence.go index bcdd10e7d0fe..118136b36593 100644 --- a/core/services/cluster/presence.go +++ b/core/services/cluster/presence.go @@ -155,12 +155,15 @@ func (r *Registry) Presence(ctx context.Context, nodeID string, grace time.Durat switch { case row.Held: // First, and redundantly so: the query already excludes held rows from - // `departed`, so either gate alone answers correctly and no behavioural - // spec can tell which one is doing the work. The redundancy is kept - // because held-ness-first is the invariant the phase rests on, and this - // is what still answers correctly if the SQL gate is ever edited away. - // The SQL gate has its own assertion in the statement-shape spec, so - // removing it is caught there rather than here. + // `departed` (on the RAW column, before the join), so either gate alone + // answers Connected for a held row with a live owner, and no + // behavioural spec can tell which one is doing the work. + // + // The redundancy is PARTIAL, not a second complete gate. With the SQL + // gate removed and the owner DEAD, `held` is false and `departed` is + // true, so this ordering yields Gone where Reconnecting is correct. + // That is why the SQL gate carries its own assertion in the + // statement-shape spec: removing it is caught there, not here. return PresenceConnected, nil case row.Departed: return PresenceGone, nil diff --git a/core/services/cluster/presence_test.go b/core/services/cluster/presence_test.go index 0aceca2b1963..eeb66ca06935 100644 --- a/core/services/cluster/presence_test.go +++ b/core/services/cluster/presence_test.go @@ -4,6 +4,7 @@ package cluster_test import ( "context" + "math" "path/filepath" "strings" "time" @@ -87,6 +88,34 @@ var _ = Describe("Presence", func() { "a row whose connection is HELD is present whatever its departure stamp says") }) + It("reports reconnecting for a held row whose owner is dead and whose stamp is stale", func() { + // The full mixed-version rolling-upgrade state, which no other spec + // constructs: the row is HELD, the owner is NOT live, and the stamp is + // older than the grace. It is reachable exactly once, during an upgrade + // where a pre-column replica re-claimed without clearing the stamp and + // then died before the sweep reached it. + // + // Reconnecting is the only defensible answer. The worker is re-dialling + // the load balancer right now, and the stamp on this row records a + // departure from a DIFFERENT, earlier session, so its age says nothing + // about the current one; the grace clock starts when the sweep stamps a + // departure for THIS session, and it has not run yet. + // + // Both of the ways this task can be got wrong land on a different + // answer here, which is why the state is worth constructing rather than + // reasoning about. Reading the stamp first gives Gone. Reading + // held-ness without the liveness join gives Connected. + _, err := reg.Claim(ctx, "node-1", "inst-a") + Expect(err).ToNot(HaveOccurred()) + age(ctx, db, "node_connections", "disconnected_at", "node_id", "node-1", 10*grace) + age(ctx, db, "instances", "last_seen", "id", "inst-a", cluster.InstanceLiveness+5*time.Second) + + p, err := reg.Presence(ctx, "node-1", grace) + Expect(err).ToNot(HaveOccurred()) + Expect(p).To(Equal(cluster.PresenceReconnecting), + "a held row with a dead owner is a retry; its stale stamp belongs to an earlier session and must not become a verdict") + }) + It("reports reconnecting immediately after the tunnel is released", func() { epoch, err := reg.Claim(ctx, "node-1", "inst-a") Expect(err).ToNot(HaveOccurred()) @@ -228,6 +257,20 @@ var _ = Describe("Presence", func() { Entry("a grace far past the floor", 10*cluster.DepartedRetention), ) + It("does not wrap back to the floor for a grace so large the multiply overflows", func() { + // time.Duration is int64 nanoseconds, so five times a grace above + // roughly 58 years wraps. An unguarded multiply comes back negative + // or small, falls through to the floor, and reinstates exactly the + // defect this function exists to remove, silently and only for the + // operator who set the largest window. + huge := time.Duration(math.MaxInt64) + Expect(cluster.DepartedRetentionFor(huge)).To(BeNumerically(">=", huge)) + // And one just past the threshold, where the wrap is easiest to + // miss because the input still looks like an ordinary duration. + overflowing := time.Duration(math.MaxInt64/5) + time.Second + Expect(cluster.DepartedRetentionFor(overflowing)).To(BeNumerically(">=", overflowing)) + }) + It("never shortens below the floor for a very small grace", func() { // A tiny grace must not shrink the retention to match: the row is also // what a restarting deployment reads to tell a re-dialling worker from diff --git a/core/services/cluster/tunnel_test.go b/core/services/cluster/tunnel_test.go index ca4455541f9f..26d70350a1df 100644 --- a/core/services/cluster/tunnel_test.go +++ b/core/services/cluster/tunnel_test.go @@ -850,6 +850,17 @@ var _ = Describe("The worker tunnel registry's claim gate", func() { }) }) +var _ = Describe("Membership.SetReconnectGrace", func() { + It("is safe on a nil receiver, like SetTunnels and Stop", func() { + // Same reason SetTunnels is: core/application/distributed.go + // deliberately produces a nil *Membership when no peer-reachable + // address can be derived, so a setter that panicked on one would be a + // trap for the next caller rather than an impossibility. + var m *cluster.Membership + Expect(func() { m.SetReconnectGrace(time.Minute) }).ToNot(Panic()) + }) +}) + var _ = Describe("Membership.SetTunnels", func() { It("is safe on a nil receiver, like Stop", func() { // A nil *Membership is a value this codebase deliberately produces when diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index bf326bdbec32..c48b9b9c2adc 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -79,6 +79,7 @@ The frontend is a standard LocalAI instance with distributed mode enabled. These | *(env only)* | `LOCALAI_MODEL_LOAD_WAIT` | `60s` | How long an inference request waits for a model that is still cold-loading onto a worker before it is answered with `503`, a `Retry-After` header and live staging progress. The request is served the moment the model becomes ready, so a model already most of the way staged needs no client retry. Set to `0` to wait as long as the load takes — only safe when no ingress or load balancer with an idle timeout sits in front. See [Requests for a model that is still loading](#requests-for-a-model-that-is-still-loading). | | `--node-heartbeat-checkpoint` | `LOCALAI_NODE_HEARTBEAT_CHECKPOINT` | `60s` | Minimum gap between **durable** heartbeat writes for a worker node. A beat that only carries a fresher timestamp is kept in memory until this interval elapses instead of being written to PostgreSQL; every reported field is compared against the value last written rather than merely tested for presence, so a node's first beat, a changed total VRAM / total disk / GPU vendor, and a free VRAM / RAM / disk reading that has moved more than 256 MiB from the written value all still write immediately, and a node that is not active is never suppressed. Set it below the worker's `--heartbeat-interval` to restore a write per beat. See [Heartbeat writes and stale-node detection](#heartbeat-writes-and-stale-node-detection). | | `--stale-node-threshold` | `LOCALAI_STALE_NODE_THRESHOLD` | `5m` | How long a node may go without a **durable** heartbeat before the health monitor marks it `offline`. Because `--node-heartbeat-checkpoint` holds back a beat that only carries a fresher timestamp, this has to stay comfortably wider than that interval: raising the checkpoint without raising this marks healthy, beating nodes offline. Neither the per-model gRPC health check nor request-time failure reads `last_heartbeat`, so neither is affected by this knob. See [Heartbeat writes and stale-node detection](#heartbeat-writes-and-stale-node-detection). | +| `--worker-reconnect-grace` | `LOCALAI_WORKER_RECONNECT_GRACE` | `90s` | How long a worker whose tunnel was lost is treated as **reconnecting** rather than **gone**. Only after this window may the scheduler stop placing work on that worker, clean up its rows and release its models. Set it below the worker's own reconnect worst case and you will condemn workers that are re-homing exactly as designed. Measured on the database clock, so every replica agrees. See [A lost tunnel is a departure, not an absence](#a-lost-tunnel-is-a-departure-not-an-absence). | | `--expose-node-header` | `LOCALAI_EXPOSE_NODE_HEADER` | `false` | When enabled, inference responses carry an `X-LocalAI-Node` header with the ID of the worker node that served the request. Coverage spans the OpenAI-compatible endpoints (chat completions, completions, embeddings, audio transcriptions, audio speech / TTS, image generations, image inpainting), the Jina rerank endpoint (`/v1/rerank`), the VAD endpoints (`/v1/vad`, `/vad`), and the Anthropic Messages (`/v1/messages`) and Ollama (`/api/chat`, `/api/generate`, `/api/embed`) shims. Useful for debugging, observability and load-balancer attribution. Off by default: the node ID reveals internal cluster topology and should not be exposed on a public endpoint. Best-effort: under heavy concurrency for the same model across multiple replicas, the header may reflect a recent routing decision rather than this exact request's. Acceptable for observability and debugging. | ### Replica peer links @@ -253,9 +254,9 @@ That distinction exists because a worker loses its tunnel for entirely ordinary | Flag | Env var | Default | Description | |------|---------|---------|-------------| -| `--worker-reconnect-grace` | `LOCALAI_WORKER_RECONNECT_GRACE` | `60s` | How long a worker whose tunnel was lost is treated as reconnecting rather than gone. | +| `--worker-reconnect-grace` | `LOCALAI_WORKER_RECONNECT_GRACE` | `90s` | How long a worker whose tunnel was lost is treated as reconnecting rather than gone. | -The default is **twice the worker's own maximum reconnect backoff** (the worker retries with exponential backoff capped at 30s), so a worker that misses one attempt at the ceiling and succeeds on the next is never condemned for it. +The default clears the worker's own worst-case reconnect. A worker retries with exponential backoff capped at **30s**, and each attempt has a **10s** dial budget, so a worker that waits the ceiling, hangs a dial, and waits the ceiling again is back at **70s**. The backoff also only resets after a session that lasted 30s, which a replica accepting a dial and then dying denies, so a worker crossing a rolling restart really does climb to the ceiling rather than sitting near the 500ms floor. 90s clears that worst case with margin; 60s would sit under it. What you trade by changing it: From 9b611f04f3b3a1b431b0036eb24fc82142261805 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 15:51:58 +0000 Subject: [PATCH 070/153] feat(worker): serve the control plane over the tunnel, not over NATS Ten NATS subscriptions on the worker become ten HTTP routes under /v1/control/, served on the loopback HTTP server the worker already runs and reached only through the tunnel's existing `http` stream tag. The carrier is the tag that already exists rather than a new one. A new tag would have had to invent correlation, per-request deadlines, unbounded payloads and a progress stream, and each of those is a place this branch has already put a defect. It would also have added a fifth entry to the worker's stream-refusal vocabulary, which decides what a frontend reaps on and took eight fixes to settle. Riding `http` means a control RPC to a worker another replica holds takes the same relay the inference path takes, which is the path that has been measured. The request and reply DTOs are untouched, so a body on a control route is byte-for-byte what the corresponding subject carried. No subject was deleted: agent workers still subscribe to nodes..backend.stop. Install and upgrade stream. They answer application/x-ndjson: zero or more {"progress":...} lines carrying the same event the per-op NATS subject carried, then exactly one {"reply":...} line, always last. That deletes the 8000-byte notification cap structurally instead of reproducing it on a new carrier: a progress line is written into the response the caller is already reading, so there is nothing to size and no subscribe-before-request window. The debouncer is shared with the NATS publisher rather than forked, so the ~4/s tick bound is one fact. A verb's own failure is a 200 with Error set, never a 5xx. The frontend maps a transport failure onto "no route to that worker", which nothing may act on, and the worker's answer onto evidence a reap guard may act on; answering 500 for a failed install would put the worker's verdict in the bucket reserved for a broken link. Only a request that could not be read or routed is non-2xx. Control RPCs carry the caller's budget. r.Context() replaces four context.Background() calls at the gallery-install sites, and the one pre-existing fixed timeout on model.unload is now derived from the caller's context so a shorter budget is honoured. No timeout is invented. The inner `go func()` in the install and upgrade handlers is deleted rather than nested: it existed because one subscription served every install, and over HTTP each request already has its own goroutine. Per-backend serialization stays lockBackend, which is what actually prevented two requests racing the gallery directory. Bounds against a boundary the worker now serves: every body is capped at 8 MiB before any decode; the 404 echoes at most 128 bytes of the request path, cut on a rune boundary so a half rune cannot travel downstream as a replacement character; non-POST is refused before the body is read so a probe cannot fire a command; the streaming responses set nosniff. The routes mount through nodes.AuthenticatedRoutes, which hands the registrar a private mux and puts the whole prefix behind the same constant-time bearer check as the file routes. The worker's HTTP server now takes the supervisor as a required parameter, so there is no way to start it without the control plane mounted. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .../nodes/authenticated_routes_test.go | 92 +++ core/services/nodes/file_transfer_server.go | 64 ++- .../nodes/install_progress_publisher.go | 59 +- core/services/worker/control_routes.go | 382 +++++++++++++ core/services/worker/control_routes_test.go | 524 ++++++++++++++++++ core/services/worker/install.go | 51 +- core/services/worker/lifecycle.go | 336 +++-------- core/services/worker/model_stop_test.go | 24 +- core/services/worker/models_running.go | 8 - core/services/worker/supervisor.go | 8 +- core/services/worker/worker.go | 98 ++-- core/services/workerctl/paths.go | 71 +++ core/services/workerctl/paths_test.go | 72 +++ .../workerctl/workerctl_suite_test.go | 13 + docs/content/features/distributed-mode.md | 34 +- 15 files changed, 1487 insertions(+), 349 deletions(-) create mode 100644 core/services/nodes/authenticated_routes_test.go create mode 100644 core/services/worker/control_routes.go create mode 100644 core/services/worker/control_routes_test.go create mode 100644 core/services/workerctl/paths.go create mode 100644 core/services/workerctl/paths_test.go create mode 100644 core/services/workerctl/workerctl_suite_test.go diff --git a/core/services/nodes/authenticated_routes_test.go b/core/services/nodes/authenticated_routes_test.go new file mode 100644 index 000000000000..db6c2a72b821 --- /dev/null +++ b/core/services/nodes/authenticated_routes_test.go @@ -0,0 +1,92 @@ +package nodes + +import ( + "net" + "net/http" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("extra authenticated routes on the worker HTTP server", func() { + const token = "s3cr3t" + + var ( + srv *http.Server + base string + ) + + BeforeEach(func() { + lis, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + dir := GinkgoT().TempDir() + srv, err = StartFileTransferServerWithRoutes(lis, dir, dir, dir, token, 0, nil, + &AuthenticatedRoutes{ + Prefix: "/v1/control/", + Register: func(mux *http.ServeMux) { + mux.HandleFunc("/v1/control/ping", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("pong")) + }) + }, + }) + Expect(err).NotTo(HaveOccurred()) + base = "http://" + lis.Addr().String() + DeferCleanup(func() { ShutdownFileTransferServer(srv) }) + }) + + get := func(path, bearer string) *http.Response { + GinkgoHelper() + req, err := http.NewRequest(http.MethodGet, base+path, nil) + Expect(err).NotTo(HaveOccurred()) + if bearer != "" { + req.Header.Set("Authorization", "Bearer "+bearer) + } + resp, err := http.DefaultClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + return resp + } + + It("serves a registered extra route to a caller carrying the token", func() { + Expect(get("/v1/control/ping", token).StatusCode).To(Equal(http.StatusOK)) + }) + + It("refuses an unauthenticated request on an extra route", func() { + // The control plane must not be a second authentication path: an extra + // route that forgot its own check would be an unauthenticated command + // on the boundary the tunnel exposes. + Expect(get("/v1/control/ping", "").StatusCode).To(Equal(http.StatusUnauthorized)) + }) + + It("refuses a wrong token on an extra route", func() { + Expect(get("/v1/control/ping", "wrong").StatusCode).To(Equal(http.StatusUnauthorized)) + }) + + It("checks the token before the route exists, so an unknown control path leaks nothing", func() { + Expect(get("/v1/control/no-such-verb", "").StatusCode).To(Equal(http.StatusUnauthorized)) + }) + + It("leaves the file routes reachable alongside the extra ones", func() { + Expect(get("/healthz", "").StatusCode).To(Equal(http.StatusOK)) + }) + + It("mounts nothing when no route set is given", func() { + lis, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + dir := GinkgoT().TempDir() + bare, err := StartFileTransferServerWithRoutes(lis, dir, dir, dir, token, 0, nil, nil) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { ShutdownFileTransferServer(bare) }) + + req, err := http.NewRequest(http.MethodGet, "http://"+lis.Addr().String()+"/v1/control/ping", nil) + Expect(err).NotTo(HaveOccurred()) + req.Header.Set("Authorization", "Bearer "+token) + resp, err := http.DefaultClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) + Expect(strings.TrimSpace(resp.Status)).NotTo(BeEmpty()) + }) +}) diff --git a/core/services/nodes/file_transfer_server.go b/core/services/nodes/file_transfer_server.go index 676cb84bd63f..df8fecd159c4 100644 --- a/core/services/nodes/file_transfer_server.go +++ b/core/services/nodes/file_transfer_server.go @@ -45,23 +45,47 @@ const ( targetSidecarSuffix = ".sha256.target" ) +// AuthenticatedRoutes is a set of extra routes to serve on the worker's HTTP +// server, mounted under Prefix and behind the SAME bearer check as the file +// routes. +// +// It is a mount request rather than a plain func(*http.ServeMux) so this +// package, and not its caller, owns the authentication. A caller handed the +// server's own mux would be registering handlers alongside the file routes, and +// forgetting the token check in one of them would be an unauthenticated verb +// rather than a compile error. Register is instead given a mux of its own, +// which is reachable only through the check below. +type AuthenticatedRoutes struct { + // Prefix is the single path prefix every registered route lives under. + Prefix string + // Register mounts the routes on a mux private to this route set. + Register func(*http.ServeMux) +} + // StartFileTransferServer starts a small HTTP server for file transfer in distributed mode. // It provides PUT/GET/POST endpoints for uploading, downloading, and allocating temp files, // as well as backend log REST and WebSocket endpoints when logStore is non-nil. // Auth is via Bearer token (registration token), using constant-time comparison. // A nil readiness fails open, keeping /readyz's historical always-200 answer. -func StartFileTransferServer(addr, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, logStore ...*model.BackendLogStore) (*http.Server, error) { - return StartFileTransferServerWithCapacity(addr, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, nil, logStore...) +// A nil extra mounts no additional routes. +func StartFileTransferServer(addr, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, extra *AuthenticatedRoutes, logStore ...*model.BackendLogStore) (*http.Server, error) { + return StartFileTransferServerWithCapacityAndRoutes(addr, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, nil, extra, logStore...) } // StartFileTransferServerWithCapacity starts the file transfer server with a // worker-local guard for per-request ephemeral inputs. func StartFileTransferServerWithCapacity(addr, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, capacity EphemeralCapacity, logStore ...*model.BackendLogStore) (*http.Server, error) { + return StartFileTransferServerWithCapacityAndRoutes(addr, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, capacity, nil, logStore...) +} + +// StartFileTransferServerWithCapacityAndRoutes combines capacity admission +// with authenticated control routes. +func StartFileTransferServerWithCapacityAndRoutes(addr, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, capacity EphemeralCapacity, extra *AuthenticatedRoutes, logStore ...*model.BackendLogStore) (*http.Server, error) { listener, err := net.Listen("tcp", addr) if err != nil { return nil, fmt.Errorf("listen %s: %w", addr, err) } - return startFileTransferServer(listener, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, capacity, logStore...) + return startFileTransferServerWithRoutes(listener, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, capacity, extra, logStore...) } // StartFileTransferServerWithListener starts the server on an existing listener. @@ -75,10 +99,23 @@ func StartFileTransferServerWithListener(lis net.Listener, stagingDir, modelsDir // the probe keeps its historical always-200 behaviour for callers that have no // meaningful readiness signal to report. func StartFileTransferServerWithReadiness(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, logStore ...*model.BackendLogStore) (*http.Server, error) { - return startFileTransferServer(lis, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, nil, logStore...) + return StartFileTransferServerWithRoutes(lis, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, nil, logStore...) +} + +// StartFileTransferServerWithRoutes is StartFileTransferServerWithReadiness +// plus an extra authenticated route set. See AuthenticatedRoutes. +func StartFileTransferServerWithRoutes(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, extra *AuthenticatedRoutes, logStore ...*model.BackendLogStore) (*http.Server, error) { + return startFileTransferServerWithRoutes(lis, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, nil, extra, logStore...) } func startFileTransferServer(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, capacity EphemeralCapacity, logStore ...*model.BackendLogStore) (*http.Server, error) { + return startFileTransferServerWithRoutes(lis, stagingDir, modelsDir, dataDir, token, maxUploadSize, readiness, capacity, nil, logStore...) +} + +func startFileTransferServerWithRoutes(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, capacity EphemeralCapacity, extra *AuthenticatedRoutes, logStore ...*model.BackendLogStore) (*http.Server, error) { + if err := validateAuthenticatedRoutes(extra); err != nil { + return nil, err + } if err := os.MkdirAll(stagingDir, 0750); err != nil { return nil, fmt.Errorf("creating staging dir %s: %w", stagingDir, err) } @@ -194,8 +231,27 @@ func startFileTransferServer(lis net.Listener, stagingDir, modelsDir, dataDir, t // Readiness: "can this worker actually accept work?" See WorkerReadiness. mux.HandleFunc("/readyz", probe(readiness.Check)) + if extra != nil && extra.Register != nil && extra.Prefix != "" { + extraMux := http.NewServeMux() + extra.Register(extraMux) + mux.Handle(extra.Prefix, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !checkBearerToken(r, token) { + xlog.Debug("worker HTTP server: unauthorized request on an extra route", + "method", r.Method, "path", r.URL.Path, "remote", r.RemoteAddr) + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + extraMux.ServeHTTP(w, r) + })) + } + addr := lis.Addr().String() server := &http.Server{ + // Addr is informational here: Serve takes the listener, not this + // field. It is set so a caller that asked for port 0 can learn the + // port it actually got without threading a second return value + // through every wrapper above. + Addr: addr, Handler: mux, ReadHeaderTimeout: 30 * time.Second, // prevent slowloris; does not affect body reads } diff --git a/core/services/nodes/install_progress_publisher.go b/core/services/nodes/install_progress_publisher.go index 60eacb711935..88a2d16eef2d 100644 --- a/core/services/nodes/install_progress_publisher.go +++ b/core/services/nodes/install_progress_publisher.go @@ -8,9 +8,14 @@ import ( ) // DebouncedInstallProgressPublisher buffers backend-install download ticks -// and publishes them to the per-op NATS progress subject at most once per -// `interval`. Always publishes the final event on Flush so the UI sees the -// terminal percentage. +// and hands them to its emit sink at most once per `interval`. Always emits the +// final event on Flush so the UI sees the terminal percentage. +// +// The sink is a function rather than a NATS subject because the same debounce +// now bounds two carriers: the NATS progress subject an agent node still +// publishes on, and a line written into the streaming HTTP response a worker +// serves over its tunnel. Keeping one debouncer keeps the ~4/s tick bound a +// single fact instead of one per carrier. // // Behavior: leading-edge debounce. The first OnDownload after a quiet window // publishes immediately; subsequent ticks within `interval` only buffer the @@ -18,13 +23,12 @@ import ( // keeps the wire chatter bounded (~4 events per second at 250ms) while // still surfacing every meaningful percentage jump. // -// Lock ordering: never hold p.mu across a Publish call. Publish hits the -// NATS client which may block on a slow link, and we don't want a stalled -// network to stall the underlying gallery download loop. +// Lock ordering: never hold p.mu across an emit call. The sink writes to the +// network, which may block on a slow link, and we don't want a stalled network +// to stall the underlying gallery download loop. type DebouncedInstallProgressPublisher struct { mu sync.Mutex - client messaging.MessagingClient - subject string + emit func(messaging.BackendInstallProgressEvent) nodeID string opID string backend string @@ -34,13 +38,24 @@ type DebouncedInstallProgressPublisher struct { timer *time.Timer } -// NewDebouncedInstallProgressPublisher constructs a publisher for one -// install operation. interval is the leading-edge debounce window -// (~250ms in production). +// NewDebouncedInstallProgressPublisher constructs a publisher for one install +// operation that publishes to the per-op NATS progress subject. interval is the +// leading-edge debounce window (~250ms in production). func NewDebouncedInstallProgressPublisher(client messaging.MessagingClient, nodeID, opID, backend string, interval time.Duration) *DebouncedInstallProgressPublisher { + subject := messaging.SubjectNodeBackendInstallProgress(nodeID, opID) + return NewDebouncedInstallProgressSink(func(ev messaging.BackendInstallProgressEvent) { + _ = client.Publish(subject, ev) + }, nodeID, opID, backend, interval) +} + +// NewDebouncedInstallProgressSink constructs a publisher for one install +// operation that hands each debounced event to emit. +// +// emit is called with p.mu released, so a sink that blocks on a slow link +// cannot stall the gallery download loop that feeds it. +func NewDebouncedInstallProgressSink(emit func(messaging.BackendInstallProgressEvent), nodeID, opID, backend string, interval time.Duration) *DebouncedInstallProgressPublisher { return &DebouncedInstallProgressPublisher{ - client: client, - subject: messaging.SubjectNodeBackendInstallProgress(nodeID, opID), + emit: emit, nodeID: nodeID, opID: opID, backend: backend, @@ -70,7 +85,7 @@ func (p *DebouncedInstallProgressPublisher) OnDownload(file, current, total stri p.lastPublishedAt = now p.pending = nil p.mu.Unlock() - _ = p.client.Publish(p.subject, ev) + p.emit(ev) return } // Within the window: buffer the latest event and arm a trailing @@ -85,8 +100,8 @@ func (p *DebouncedInstallProgressPublisher) OnDownload(file, current, total stri } // flushPending is the trailing-edge publisher fired by the AfterFunc timer. -// It clears the pending slot under the lock, then publishes outside the -// lock so Publish never blocks an in-progress OnDownload call. +// It clears the pending slot under the lock, then emits outside the lock so the +// sink never blocks an in-progress OnDownload call. func (p *DebouncedInstallProgressPublisher) flushPending() { p.mu.Lock() p.timer = nil @@ -97,14 +112,14 @@ func (p *DebouncedInstallProgressPublisher) flushPending() { } p.mu.Unlock() if pending != nil { - _ = p.client.Publish(p.subject, *pending) + p.emit(*pending) } } -// Flush publishes any pending buffered event synchronously and stops the -// pending timer. Safe to call multiple times. Callers MUST defer Flush -// after constructing the publisher so the terminal percentage reaches the -// master even on error returns. +// Flush emits any pending buffered event synchronously and stops the pending +// timer. Safe to call multiple times. Callers MUST defer Flush after +// constructing the publisher so the terminal percentage reaches the master even +// on error returns. func (p *DebouncedInstallProgressPublisher) Flush() { p.mu.Lock() if p.timer != nil { @@ -115,6 +130,6 @@ func (p *DebouncedInstallProgressPublisher) Flush() { p.pending = nil p.mu.Unlock() if pending != nil { - _ = p.client.Publish(p.subject, *pending) + p.emit(*pending) } } diff --git a/core/services/worker/control_routes.go b/core/services/worker/control_routes.go new file mode 100644 index 000000000000..57e3b6fcef63 --- /dev/null +++ b/core/services/worker/control_routes.go @@ -0,0 +1,382 @@ +package worker + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sync" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" + "github.com/mudler/xlog" +) + +// The worker's control plane, served as ordinary HTTP on the loopback server +// the worker already runs, and reached only through the tunnel's `http` stream +// tag. It replaces ten NATS subscriptions. +// +// HTTP and not a new stream tag, and the reason is not convenience. Every one +// of correlation, per-request deadlines, unbounded payloads and a progress +// stream is something a new tag would have had to invent, and each is a place +// this branch has already put a defect. Riding the tag that already exists also +// adds nothing to the worker's stream-refusal vocabulary, which is the table a +// frontend decides what to reap on. And a control RPC to a worker another +// replica holds takes the SAME relay the inference path takes, which is the +// path that has been measured, rather than a second one that has not. +// +// A handler's own failure is a 200 carrying a reply with Error set, NOT a 5xx. +// The distinction is load-bearing: the frontend maps a transport failure onto +// "this frontend has no route to that worker", which nothing may reap on, and a +// worker's answer onto evidence a reap guard MAY act on. A handler that +// answered 500 for "the install failed" would put the worker's own verdict into +// the bucket reserved for a broken link. Only a failure to read or route the +// request is a non-2xx. + +// maxControlRequestBytes bounds a control request body. +// +// The largest real body is BackendInstallRequest.BackendGalleries, a serialized +// gallery list of a few hundred kilobytes. Eight megabytes is therefore not a +// size the protocol needs: it is a defence against a body that never ends, +// arriving on a boundary this worker now serves. +const maxControlRequestBytes = 8 << 20 + +// maxEchoedPathBytes bounds how much of an unknown control path the 404 body +// repeats back. The path is caller-controlled and the answer exists to be read +// in a log line, so a caller cannot make this worker echo a request-sized +// string into one. +const maxEchoedPathBytes = 128 + +// installFunc and upgradeFunc are the shapes of the two long-running verbs. +// They exist as named types so the fields that override them below read as one +// thing rather than as two inline signatures. +type installFunc func(ctx context.Context, req messaging.BackendInstallRequest, force bool, + onProgress func(messaging.BackendInstallProgressEvent)) (string, error) + +type upgradeFunc func(ctx context.Context, req messaging.BackendUpgradeRequest, + onProgress func(messaging.BackendInstallProgressEvent)) ([]string, error) + +// installer and upgrader return the implementation the streaming verbs call. +// +// The override fields exist so the ROUTING can be specced without a gallery, a +// registry or a real download, which is the same argument tunnelServices was +// extracted under: the part of this file that can put a worker's verdict in the +// wrong bucket is the part that has nothing to do with installing anything. +func (s *backendSupervisor) installer() installFunc { + if s.installFn != nil { + return s.installFn + } + return s.installBackend +} + +func (s *backendSupervisor) upgrader() upgradeFunc { + if s.upgradeFn != nil { + return s.upgradeFn + } + return s.upgradeBackend +} + +// RegisterControlRoutes mounts every control verb on mux. +// +// The caller is responsible for putting mux behind authentication; see +// nodes.AuthenticatedRoutes, which is how the worker mounts this so the control +// plane shares one bearer check with the file routes rather than growing a +// second one. +func (s *backendSupervisor) RegisterControlRoutes(mux *http.ServeMux) { + // post registers one JSON-in, JSON-out verb. reply may be nil, in which + // case the verb answers 204: that is the shape the two former + // publish-no-reply subjects take. + post := func(path string, h func(ctx context.Context, body []byte) (any, error)) { + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + body, ok := readControlBody(w, r) + if !ok { + return + } + reply, err := h(r.Context(), body) + if err != nil { + // Reaching here means the request could not be READ or routed, + // which is this worker failing rather than answering. A verb's + // own failure never reaches here: it comes back as a reply with + // Error set, and a 200. + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if reply == nil { + w.WriteHeader(http.StatusNoContent) + return + } + w.Header().Set("Content-Type", "application/json") + if encErr := json.NewEncoder(w).Encode(reply); encErr != nil { + xlog.Debug("worker control reply could not be written", "path", path, "error", encErr) + } + }) + } + + post(workerctl.PathModelsRunning, func(context.Context, []byte) (any, error) { + return messaging.ModelsRunningReply{Models: s.runningModels()}, nil + }) + + post(workerctl.PathModelStop, func(_ context.Context, body []byte) (any, error) { + var req messaging.ModelStopRequest + if err := json.Unmarshal(body, &req); err != nil { + return nil, fmt.Errorf("invalid model.stop request: %w", err) + } + return s.stopModelExact(req), nil + }) + + post(workerctl.PathBackendList, func(context.Context, []byte) (any, error) { + return s.backendList(), nil + }) + + post(workerctl.PathBackendDelete, func(_ context.Context, body []byte) (any, error) { + var req messaging.BackendDeleteRequest + if err := json.Unmarshal(body, &req); err != nil { + return nil, fmt.Errorf("invalid backend.delete request: %w", err) + } + return s.deleteBackend(req), nil + }) + + post(workerctl.PathModelUnload, func(ctx context.Context, body []byte) (any, error) { + var req messaging.ModelUnloadRequest + if err := json.Unmarshal(body, &req); err != nil { + return nil, fmt.Errorf("invalid model.unload request: %w", err) + } + return s.unloadModel(ctx, req), nil + }) + + post(workerctl.PathModelDelete, func(_ context.Context, body []byte) (any, error) { + var req messaging.ModelDeleteRequest + if err := json.Unmarshal(body, &req); err != nil { + return nil, fmt.Errorf("invalid model.delete request: %w", err) + } + return s.deleteModel(req), nil + }) + + post(workerctl.PathBackendStop, func(_ context.Context, body []byte) (any, error) { + req, stopAll, err := decodeBackendStopRequest(body) + if err != nil { + return nil, fmt.Errorf("invalid backend.stop request: %w", err) + } + return s.stopBackends(req, stopAll), nil + }) + + post(workerctl.PathNodeStop, func(context.Context, []byte) (any, error) { + // The signal is sent before the 204 is written, and that ordering is + // safe rather than lucky: sigCh is buffered and this send never blocks, + // and the shutdown it starts is a graceful one that waits for this + // request to finish before closing the listener. + s.signalNodeStop() + return nil, nil + }) + + // The two streaming verbs. They write NDJSON rather than one JSON object, + // so they do not go through post. + mux.HandleFunc(workerctl.PathBackendInstall, s.serveInstall) + mux.HandleFunc(workerctl.PathBackendUpgrade, s.serveUpgrade) + + mux.HandleFunc(workerctl.Prefix, func(w http.ResponseWriter, r *http.Request) { + // The catch-all. A path under the control prefix that no verb claims is + // a frontend newer than this worker, and the body says so, because a + // bare 404 through a tunnel is indistinguishable from a proxy fault. + http.Error(w, "unknown worker control path "+truncate(r.URL.Path, maxEchoedPathBytes), http.StatusNotFound) + }) +} + +// readControlBody enforces the two things every control verb requires of a +// request: that it is a POST, and that its body is bounded. +// +// A GET is refused rather than served because a control verb is a command, and +// a liveness probe, a link prefetch or a browser address bar must not be able +// to stop a node. +func readControlBody(w http.ResponseWriter, r *http.Request) ([]byte, bool) { + if r.Method != http.MethodPost { + http.Error(w, "control verbs are POST only", http.StatusMethodNotAllowed) + return nil, false + } + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxControlRequestBytes)) + if err != nil { + // A body this worker could not READ is not an answer about any + // backend, so it must not look like one: 400 is what the frontend maps + // onto "the request was rejected", never onto "that model is gone". + http.Error(w, "reading the control request body: "+err.Error(), http.StatusBadRequest) + return nil, false + } + return body, true +} + +// truncate bounds a caller-controlled string that is about to be echoed. +// It cuts on a rune boundary: a byte-wise cut can split a multi-byte rune, and +// the half-rune then travels as a replacement character through every log and +// UI that reads it. +func truncate(s string, max int) string { + if len(s) <= max { + return s + } + cut := max + for cut > 0 && !isRuneStart(s[cut]) { + cut-- + } + return s[:cut] + "…" +} + +// isRuneStart reports whether b begins a UTF-8 rune (i.e. is not a +// continuation byte). +func isRuneStart(b byte) bool { return b&0xC0 != 0x80 } + +// ndjsonStream writes the Envelope lines of one streaming control response. +// +// The mutex is not optional. A progress line can be emitted from the debounce +// timer's own goroutine, so without serialization a progress write can +// interleave with the terminal reply write and put a torn line on the wire, +// and the frontend's contract is that the reply line is the LAST thing on the +// body. done is what enforces that half: once the reply is written, a late +// progress line is dropped rather than appended after it. +type ndjsonStream struct { + mu sync.Mutex + w http.ResponseWriter + enc *json.Encoder + done bool +} + +func newNDJSONStream(w http.ResponseWriter) *ndjsonStream { + w.Header().Set("Content-Type", workerctl.ContentTypeStream) + // A streaming body must not be buffered into a guessed content type: the + // caller reads line by line and the first line may be minutes before the + // last. + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + return &ndjsonStream{w: w, enc: json.NewEncoder(w)} +} + +// progress writes one progress line. It is a no-op once the reply has been +// written. +func (n *ndjsonStream) progress(ev messaging.BackendInstallProgressEvent) { + raw, err := json.Marshal(ev) + if err != nil { + xlog.Debug("worker control progress event could not be marshalled", "error", err) + return + } + n.mu.Lock() + defer n.mu.Unlock() + if n.done { + return + } + n.write(workerctl.Envelope{Progress: raw}) +} + +// reply writes the single terminal reply line and closes the stream to any +// further progress. +func (n *ndjsonStream) reply(v any) { + raw, err := json.Marshal(v) + if err != nil { + // The caller is waiting for a terminal line and will otherwise read to + // EOF without one, which it cannot tell apart from a truncated + // response. Send a reply it can decode as a failure of THIS worker's + // own making rather than sending nothing. + xlog.Error("worker control reply could not be marshalled", "error", err) + raw = json.RawMessage(`{"success":false,"error":"the worker could not encode its own reply"}`) + } + n.mu.Lock() + defer n.mu.Unlock() + if n.done { + return + } + n.write(workerctl.Envelope{Reply: raw}) + n.done = true +} + +// write encodes one envelope and flushes it. Callers hold n.mu. +func (n *ndjsonStream) write(env workerctl.Envelope) { + if err := n.enc.Encode(env); err != nil { + xlog.Debug("worker control stream line could not be written", "error", err) + return + } + if f, ok := n.w.(http.Flusher); ok { + f.Flush() + } +} + +// serveInstall answers backend.install, streaming download progress ahead of +// the single terminal reply. +// +// The NATS handler this replaces ran its work on a fresh goroutine so a slow +// install could not head-of-line-block the one subscription every install +// arrived on. Over HTTP each request already has its own goroutine, so there is +// nothing left to block and no goroutine is started here. Per-backend +// serialization is unchanged and still comes from lockBackend, which is what +// actually prevented two requests racing the gallery directory. +func (s *backendSupervisor) serveInstall(w http.ResponseWriter, r *http.Request) { + body, ok := readControlBody(w, r) + if !ok { + return + } + var req messaging.BackendInstallRequest + if err := json.Unmarshal(body, &req); err != nil { + http.Error(w, fmt.Sprintf("invalid backend.install request: %v", err), http.StatusBadRequest) + return + } + xlog.Info("Serving backend.install", "backend", req.Backend, "model", req.ModelID) + + stream := newNDJSONStream(w) + + release := s.lockBackend(req.Backend) + defer release() + + // req.Force=true is the legacy path used by pre-2026-05-08 masters that + // don't know about backend.upgrade. Honor it so a rolling update with new + // worker + old master keeps working; new masters send to backend.upgrade + // instead. + addr, err := s.installer()(r.Context(), req, req.Force, stream.progress) + if err != nil { + xlog.Error("Failed to install backend", "error", err) + stream.reply(messaging.BackendInstallReply{Success: false, Error: err.Error()}) + return + } + + // The address goes back exactly as the process listens on it. It used to be + // rewritten onto this worker's advertise host, which made the reply the + // worker's third advertisement site; the frontend now reads only the port + // out of it and dials nothing. + stream.reply(messaging.BackendInstallReply{Success: true, WorkerLocalAddress: addr}) +} + +// serveUpgrade answers backend.upgrade, a force-reinstall, on the same +// streaming shape as install. +func (s *backendSupervisor) serveUpgrade(w http.ResponseWriter, r *http.Request) { + body, ok := readControlBody(w, r) + if !ok { + return + } + var req messaging.BackendUpgradeRequest + if err := json.Unmarshal(body, &req); err != nil { + http.Error(w, fmt.Sprintf("invalid backend.upgrade request: %v", err), http.StatusBadRequest) + return + } + xlog.Info("Serving backend.upgrade", "backend", req.Backend) + + stream := newNDJSONStream(w) + + release := s.lockBackend(req.Backend) + defer release() + + // stopped is meaningful even on the error paths: it lists processes already + // terminated (and ports already recycled) before the failure, so the + // controller must drop those rows regardless of the outcome. + stopped, err := s.upgrader()(r.Context(), req, stream.progress) + if err != nil { + xlog.Error("Failed to upgrade backend", "error", err) + stream.reply(messaging.BackendUpgradeReply{ + Success: false, + Error: err.Error(), + StoppedProcessKeys: stopped, + ReportsStoppedProcesses: true, + }) + return + } + stream.reply(messaging.BackendUpgradeReply{ + Success: true, + StoppedProcessKeys: stopped, + ReportsStoppedProcesses: true, + }) +} diff --git a/core/services/worker/control_routes_test.go b/core/services/worker/control_routes_test.go new file mode 100644 index 000000000000..fb8576e71836 --- /dev/null +++ b/core/services/worker/control_routes_test.go @@ -0,0 +1,524 @@ +package worker + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/workerctl" + "github.com/mudler/LocalAI/pkg/system" +) + +// lastReplyOf drains an NDJSON control body and decodes the single terminal +// reply line into out. It fails the spec when the body carries no reply line, +// which is the shape the frontend cannot recover from: it would sit reading a +// body that already ended. +func lastReplyOf(body io.Reader, out any) error { + GinkgoHelper() + dec := json.NewDecoder(body) + var reply json.RawMessage + for { + var env workerctl.Envelope + err := dec.Decode(&env) + if errors.Is(err, io.EOF) { + break + } + Expect(err).NotTo(HaveOccurred()) + if env.Reply != nil { + reply = env.Reply + } + } + Expect(reply).NotTo(BeNil(), "the control body carried no terminal reply line") + return json.Unmarshal(reply, out) +} + +// envelopeKindsOf reports the order of progress/reply lines in an NDJSON body. +func envelopeKindsOf(body io.Reader) []string { + GinkgoHelper() + var kinds []string + dec := json.NewDecoder(body) + for { + var env workerctl.Envelope + err := dec.Decode(&env) + if errors.Is(err, io.EOF) { + break + } + Expect(err).NotTo(HaveOccurred()) + if env.Reply != nil { + kinds = append(kinds, "reply") + continue + } + Expect(env.Progress).NotTo(BeNil(), "an envelope carried neither progress nor reply") + kinds = append(kinds, "progress") + } + return kinds +} + +var _ = Describe("worker control routes", func() { + var ( + sup *backendSupervisor + srv *httptest.Server + sigCh chan os.Signal + ) + + BeforeEach(func() { + sigCh = make(chan os.Signal, 1) + sup = &backendSupervisor{ + cfg: &Config{}, + nodeID: "node-under-test", + sigCh: sigCh, + processes: map[string]*backendProcess{}, + } + mux := http.NewServeMux() + sup.RegisterControlRoutes(mux) + srv = httptest.NewServer(mux) + DeferCleanup(srv.Close) + }) + + // post marshals, POSTs, and hands back the response. + post := func(path string, body any) *http.Response { + GinkgoHelper() + buf, err := json.Marshal(body) + Expect(err).NotTo(HaveOccurred()) + req, err := http.NewRequest(http.MethodPost, srv.URL+path, bytes.NewReader(buf)) + Expect(err).NotTo(HaveOccurred()) + req.Header.Set("Content-Type", "application/json") + resp, err := srv.Client().Do(req) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + return resp + } + + It("answers models.running with the worker's process table", func() { + resp := post(workerctl.PathModelsRunning, messaging.ModelsRunningRequest{}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply messaging.ModelsRunningReply + Expect(json.NewDecoder(resp.Body).Decode(&reply)).To(Succeed()) + Expect(reply.Models).To(BeEmpty()) + }) + + It("refuses a GET on a control route, so a probe cannot fire a command", func() { + resp, err := srv.Client().Get(srv.URL + workerctl.PathNodeStop) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusMethodNotAllowed)) + Expect(sigCh).NotTo(Receive(), "a GET must not have signalled shutdown") + }) + + It("refuses a GET on the streaming install route too", func() { + resp, err := srv.Client().Get(srv.URL + workerctl.PathBackendInstall) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusMethodNotAllowed)) + }) + + It("answers an unknown control path with 404 and a body that names the prefix", func() { + resp := post(workerctl.Prefix+"no-such-verb", struct{}{}) + Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) + body, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + // A mixed-version deployment has to be diagnosable from one line in a + // log, not from a bare 404 that looks like a proxy. + Expect(string(body)).To(ContainSubstring("control")) + }) + + It("bounds the unknown path it echoes back, so a long URL cannot be reflected wholesale", func() { + long := strings.Repeat("a", 4096) + resp := post(workerctl.Prefix+long, struct{}{}) + Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) + body, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(len(body)).To(BeNumerically("<", 512)) + }) + + It("reports a malformed request body as 400 and does not touch the process table", func() { + resp, err := srv.Client().Post(srv.URL+workerctl.PathModelStop, "application/json", + strings.NewReader("{not json")) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + }) + + It("refuses a request body larger than the control bound", func() { + // The body is DELIBERATELY well-formed JSON for the verb it is sent to. + // A body of garbage would be rejected by the decoder whether or not the + // bound exists, so a spec built on one is green with the bound removed + // and pins nothing. This one is refused only because of the bound. + oversized := []byte(`{"process_key":"` + strings.Repeat("a", maxControlRequestBytes) + `"}`) + Expect(len(oversized)).To(BeNumerically(">", maxControlRequestBytes)) + resp, err := srv.Client().Post(srv.URL+workerctl.PathModelStop, "application/json", + bytes.NewReader(oversized)) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + }) + + It("accepts a large but in-bound body, so the bound is a ceiling and not a shape", func() { + // BackendInstallRequest.BackendGalleries is a serialized gallery list of + // a few hundred kilobytes on a real cluster, so the bound has to sit + // well above that rather than at the size of a typical request. + sup.installFn = func(context.Context, messaging.BackendInstallRequest, bool, + func(messaging.BackendInstallProgressEvent)) (string, error) { + return "127.0.0.1:1", nil + } + big := []byte(`{"backend":"mock","backend_galleries":"` + strings.Repeat("a", 1<<20) + `"}`) + Expect(len(big)).To(BeNumerically("<", maxControlRequestBytes)) + resp, err := srv.Client().Post(srv.URL+workerctl.PathBackendInstall, "application/json", + bytes.NewReader(big)) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + }) + + It("answers backend.stop with 204, the shape a fire-and-forget verb takes", func() { + resp := post(workerctl.PathBackendStop, messaging.BackendStopRequest{Backend: "no-such-backend"}) + Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) + }) + + It("answers node.stop with 204 and signals shutdown", func() { + resp := post(workerctl.PathNodeStop, struct{}{}) + Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) + Eventually(sigCh).Should(Receive(Equal(os.Signal(syscall.SIGTERM)))) + }) + + It("answers model.unload with the worker's own reply rather than a transport error", func() { + resp := post(workerctl.PathModelUnload, messaging.ModelUnloadRequest{ModelName: "m"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply messaging.ModelUnloadReply + Expect(json.NewDecoder(resp.Body).Decode(&reply)).To(Succeed()) + Expect(reply.Success).To(BeTrue()) + }) + + It("answers model.stop for an unknown process with the worker's verdict, not a 5xx", func() { + // The whole invariant of the phase: "that backend is not there" is the + // WORKER answering, and it must never arrive as the status code a + // frontend reads as "I could not reach that worker". + resp := post(workerctl.PathModelStop, messaging.ModelStopRequest{ProcessKey: "ghost#0"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply messaging.ModelStopReply + Expect(json.NewDecoder(resp.Body).Decode(&reply)).To(Succeed()) + Expect(reply.Matched).To(BeFalse()) + Expect(reply.ProcessKey).To(Equal("ghost#0")) + }) + + Context("streaming install", func() { + It("writes every progress line before the single terminal reply line", func() { + sup.installFn = func(_ context.Context, _ messaging.BackendInstallRequest, _ bool, + progress func(messaging.BackendInstallProgressEvent)) (string, error) { + progress(messaging.BackendInstallProgressEvent{Percentage: 50}) + progress(messaging.BackendInstallProgressEvent{Percentage: 100}) + return "127.0.0.1:50051", nil + } + resp := post(workerctl.PathBackendInstall, messaging.BackendInstallRequest{Backend: "mock", OpID: "op-1"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + Expect(resp.Header.Get("Content-Type")).To(Equal(workerctl.ContentTypeStream)) + Expect(envelopeKindsOf(resp.Body)).To(Equal([]string{"progress", "progress", "reply"})) + }) + + It("carries the address the backend actually listens on back in the reply", func() { + sup.installFn = func(_ context.Context, _ messaging.BackendInstallRequest, _ bool, + _ func(messaging.BackendInstallProgressEvent)) (string, error) { + return "127.0.0.1:50099", nil + } + resp := post(workerctl.PathBackendInstall, messaging.BackendInstallRequest{Backend: "mock"}) + var reply messaging.BackendInstallReply + Expect(lastReplyOf(resp.Body, &reply)).To(Succeed()) + Expect(reply.Success).To(BeTrue()) + Expect(reply.WorkerLocalAddress).To(Equal("127.0.0.1:50099")) + }) + + It("still writes a terminal reply line when the install fails", func() { + sup.installFn = func(_ context.Context, _ messaging.BackendInstallRequest, _ bool, + _ func(messaging.BackendInstallProgressEvent)) (string, error) { + return "", errors.New("boom") + } + resp := post(workerctl.PathBackendInstall, messaging.BackendInstallRequest{Backend: "mock", OpID: "op-2"}) + // 200 with a failed reply, not 500: the WORKER answered, and a 5xx + // is what the frontend reads as the worker not answering at all. + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply messaging.BackendInstallReply + Expect(lastReplyOf(resp.Body, &reply)).To(Succeed()) + Expect(reply.Success).To(BeFalse()) + Expect(reply.Error).To(ContainSubstring("boom")) + }) + + It("passes the request's Force flag through, which is the legacy upgrade path", func() { + forced := make(chan bool, 1) + sup.installFn = func(_ context.Context, _ messaging.BackendInstallRequest, force bool, + _ func(messaging.BackendInstallProgressEvent)) (string, error) { + forced <- force + return "127.0.0.1:1", nil + } + post(workerctl.PathBackendInstall, messaging.BackendInstallRequest{Backend: "mock", Force: true}) + Expect(forced).To(Receive(BeTrue())) + }) + + It("gives the install the caller's context, so a frontend that gave up stops the work", func() { + started := make(chan struct{}) + observed := make(chan error, 1) + // abandon releases the handler if the context never arrives, so a + // build where the caller's budget was dropped fails this spec + // instead of parking a goroutine until the suite times out. + abandon := make(chan struct{}) + DeferCleanup(func() { close(abandon) }) + sup.installFn = func(ctx context.Context, _ messaging.BackendInstallRequest, _ bool, + _ func(messaging.BackendInstallProgressEvent)) (string, error) { + close(started) + select { + case <-ctx.Done(): + case <-abandon: + } + observed <- ctx.Err() + return "", ctx.Err() + } + ctx, cancel := context.WithCancel(context.Background()) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + srv.URL+workerctl.PathBackendInstall, strings.NewReader(`{"backend":"mock"}`)) + Expect(err).NotTo(HaveOccurred()) + go func() { + defer GinkgoRecover() + resp, derr := srv.Client().Do(req) + if derr == nil { + _ = resp.Body.Close() + } + }() + Eventually(started).Should(BeClosed()) + cancel() + Eventually(observed).Should(Receive(MatchError(context.Canceled))) + }) + + It("reports a malformed install body as 400 rather than as a failed install", func() { + resp, err := srv.Client().Post(srv.URL+workerctl.PathBackendInstall, "application/json", + strings.NewReader("{not json")) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + }) + }) + + Context("streaming upgrade", func() { + It("writes progress before the terminal reply and reports what it stopped", func() { + sup.upgradeFn = func(_ context.Context, _ messaging.BackendUpgradeRequest, + progress func(messaging.BackendInstallProgressEvent)) ([]string, error) { + progress(messaging.BackendInstallProgressEvent{Percentage: 10}) + return []string{"m#0"}, nil + } + resp := post(workerctl.PathBackendUpgrade, messaging.BackendUpgradeRequest{Backend: "mock", OpID: "op-3"}) + Expect(resp.Header.Get("Content-Type")).To(Equal(workerctl.ContentTypeStream)) + body, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(envelopeKindsOf(bytes.NewReader(body))).To(Equal([]string{"progress", "reply"})) + var reply messaging.BackendUpgradeReply + Expect(lastReplyOf(bytes.NewReader(body), &reply)).To(Succeed()) + Expect(reply.Success).To(BeTrue()) + Expect(reply.StoppedProcessKeys).To(Equal([]string{"m#0"})) + Expect(reply.ReportsStoppedProcesses).To(BeTrue()) + }) + + It("reports the processes it stopped even when the upgrade then failed", func() { + // stopped is meaningful on the error path: those ports are already + // recycled, so the controller must drop their rows regardless. + sup.upgradeFn = func(_ context.Context, _ messaging.BackendUpgradeRequest, + _ func(messaging.BackendInstallProgressEvent)) ([]string, error) { + return []string{"m#0"}, errors.New("upgrade boom") + } + resp := post(workerctl.PathBackendUpgrade, messaging.BackendUpgradeRequest{Backend: "mock"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply messaging.BackendUpgradeReply + Expect(lastReplyOf(resp.Body, &reply)).To(Succeed()) + Expect(reply.Success).To(BeFalse()) + Expect(reply.Error).To(ContainSubstring("upgrade boom")) + Expect(reply.StoppedProcessKeys).To(Equal([]string{"m#0"})) + Expect(reply.ReportsStoppedProcesses).To(BeTrue()) + }) + }) +}) + +// These specs pin the PRODUCTION wiring rather than the handler. Every defect +// this branch has shipped in the last two tasks was a call site no spec +// touched, and "the control plane is mounted, on the same listener, behind the +// same token" is exactly that kind of fact: the handlers can be perfect and a +// worker that never mounts them answers 404 to every command while looking +// healthy on /healthz. +var _ = Describe("the worker's HTTP server", func() { + const token = "worker-token" + + var ( + srv *http.Server + base string + sup *backendSupervisor + ) + + BeforeEach(func() { + dir := GinkgoT().TempDir() + st, err := system.GetSystemState( + system.WithModelPath(dir), + system.WithBackendPath(filepath.Join(dir, "backends")), + system.WithBackendSystemPath(filepath.Join(dir, "backends-system")), + ) + Expect(err).NotTo(HaveOccurred()) + sup = &backendSupervisor{ + cfg: &Config{ModelsPath: dir}, + systemState: st, + nodeID: "node-under-test", + sigCh: make(chan os.Signal, 1), + processes: map[string]*backendProcess{}, + // This Describe is about MOUNTING, so the two verbs that would + // otherwise reach a gallery are scripted. Everything else runs its + // real body against an empty worker. + installFn: func(context.Context, messaging.BackendInstallRequest, bool, + func(messaging.BackendInstallProgressEvent)) (string, error) { + return "127.0.0.1:50051", nil + }, + upgradeFn: func(context.Context, messaging.BackendUpgradeRequest, + func(messaging.BackendInstallProgressEvent)) ([]string, error) { + return nil, nil + }, + } + srv, err = startWorkerHTTPServer("127.0.0.1:0", filepath.Join(dir, "staging"), dir, + filepath.Join(dir, "data"), token, &nodes.WorkerReadiness{}, sup, nil) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { nodes.ShutdownFileTransferServer(srv) }) + Expect(srv.Addr).NotTo(BeEmpty(), "the worker HTTP server must report the address it bound") + base = "http://" + srv.Addr + }) + + postCtl := func(path, bearer string) *http.Response { + GinkgoHelper() + req, err := http.NewRequest(http.MethodPost, base+path, strings.NewReader("{}")) + Expect(err).NotTo(HaveOccurred()) + if bearer != "" { + req.Header.Set("Authorization", "Bearer "+bearer) + } + resp, err := http.DefaultClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + return resp + } + + It("serves the control plane on the same listener as the file routes", func() { + resp := postCtl(workerctl.PathModelsRunning, token) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply messaging.ModelsRunningReply + Expect(json.NewDecoder(resp.Body).Decode(&reply)).To(Succeed()) + Expect(reply.Models).To(BeEmpty()) + }) + + It("puts the control plane behind the registration token", func() { + Expect(postCtl(workerctl.PathModelsRunning, "").StatusCode).To(Equal(http.StatusUnauthorized)) + Expect(postCtl(workerctl.PathModelsRunning, "wrong").StatusCode).To(Equal(http.StatusUnauthorized)) + }) + + It("mounts every control verb, not just the one this spec reads", func() { + for _, p := range workerctl.AllPaths() { + if p == workerctl.PathNodeStop { + // Firing it would tear down the worker under the other specs. + continue + } + resp := postCtl(p, token) + Expect(resp.StatusCode).NotTo(Equal(http.StatusNotFound), "control verb %q is not mounted", p) + Expect(resp.StatusCode).NotTo(Equal(http.StatusUnauthorized), "control verb %q rejected a valid token", p) + } + }) +}) + +// lockedRecorder is a ResponseWriter safe for concurrent writes. httptest's +// recorder is not, and the concurrency spec below is about what ndjsonStream +// serializes, not about what the recorder does. +type lockedRecorder struct { + mu sync.Mutex + buf bytes.Buffer + hdr http.Header +} + +func newLockedRecorder() *lockedRecorder { return &lockedRecorder{hdr: http.Header{}} } + +func (l *lockedRecorder) Header() http.Header { return l.hdr } + +func (l *lockedRecorder) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.buf.Write(p) +} + +func (l *lockedRecorder) WriteHeader(int) {} + +func (l *lockedRecorder) body() []byte { + l.mu.Lock() + defer l.mu.Unlock() + return append([]byte(nil), l.buf.Bytes()...) +} + +var _ = Describe("the NDJSON control stream", func() { + It("drops a progress line that arrives after the reply, so the reply stays last", func() { + // The debounce timer emits from its own goroutine and can fire after + // the install has already returned. Appending that line would break the + // one thing the frontend relies on to stop reading. + rec := httptest.NewRecorder() + stream := newNDJSONStream(rec) + stream.progress(messaging.BackendInstallProgressEvent{Percentage: 10}) + stream.reply(messaging.BackendInstallReply{Success: true}) + stream.progress(messaging.BackendInstallProgressEvent{Percentage: 100}) + + Expect(envelopeKindsOf(bytes.NewReader(rec.Body.Bytes()))).To(Equal([]string{"progress", "reply"})) + }) + + It("writes at most one reply line even when reply is called twice", func() { + rec := httptest.NewRecorder() + stream := newNDJSONStream(rec) + stream.reply(messaging.BackendInstallReply{Success: true}) + stream.reply(messaging.BackendInstallReply{Success: false, Error: "second"}) + + Expect(envelopeKindsOf(bytes.NewReader(rec.Body.Bytes()))).To(Equal([]string{"reply"})) + Expect(rec.Body.String()).NotTo(ContainSubstring("second")) + }) + + It("names the streaming content type and refuses to let it be sniffed", func() { + rec := httptest.NewRecorder() + newNDJSONStream(rec) + Expect(rec.Header().Get("Content-Type")).To(Equal(workerctl.ContentTypeStream)) + Expect(rec.Header().Get("X-Content-Type-Options")).To(Equal("nosniff")) + }) + + It("serializes concurrent progress against the reply, so no line is torn", func() { + rec := newLockedRecorder() + stream := newNDJSONStream(rec) + + const writers = 16 + start := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(writers) + for i := 0; i < writers; i++ { + go func(n int) { + defer GinkgoRecover() + defer wg.Done() + <-start + stream.progress(messaging.BackendInstallProgressEvent{Percentage: float64(n)}) + }(i) + } + close(start) + stream.reply(messaging.BackendInstallReply{Success: true}) + wg.Wait() + + kinds := envelopeKindsOf(bytes.NewReader(rec.body())) + Expect(kinds).NotTo(BeEmpty()) + Expect(kinds[len(kinds)-1]).To(Equal("reply"), "the reply line must be the last line on the body") + for _, k := range kinds[:len(kinds)-1] { + Expect(k).To(Equal("progress")) + } + }) +}) diff --git a/core/services/worker/install.go b/core/services/worker/install.go index 122b5d266c9c..2390ca28d25c 100644 --- a/core/services/worker/install.go +++ b/core/services/worker/install.go @@ -56,11 +56,16 @@ func buildProcessKey(modelID, backend string, replicaIndex int) string { // // Returns the gRPC address of the backend process. // +// ctx is the CALLER's budget, carried in from the control request, so a +// frontend that stopped reading stops the download rather than leaving the +// worker pulling gigabytes for a response nobody will read. onProgress receives +// debounced download ticks and may be nil. +// // ProcessKey includes the replica index so a worker with MaxReplicasPerModel>1 // can host multiple processes for the same model on distinct ports. Old // controllers (no replica_index in the request) implicitly target replica 0, // which preserves single-replica behavior. -func (s *backendSupervisor) installBackend(req messaging.BackendInstallRequest, force bool) (string, error) { +func (s *backendSupervisor) installBackend(ctx context.Context, req messaging.BackendInstallRequest, force bool, onProgress func(messaging.BackendInstallProgressEvent)) (string, error) { processKey := buildProcessKey(req.ModelID, req.Backend, int(req.ReplicaIndex)) if !force { @@ -129,16 +134,16 @@ func (s *backendSupervisor) installBackend(req messaging.BackendInstallRequest, galleries = reqGalleries } - // When the master tagged this install with an OpID, stream the - // gallery download progress back to it on the per-op NATS subject. - // Old masters that omit OpID stay on the silent path so they keep - // working without changes. The publisher releases its mutex before - // every Publish so a slow link never stalls the download loop, and - // the deferred Flush guarantees a terminal-percentage event reaches - // the master even when the install errors out. + // When the caller tagged this install with an OpID and is listening for + // progress, stream the gallery download ticks back on the response the + // caller is already reading. Callers that omit OpID stay on the silent path. + // The publisher releases its mutex before every emit so a slow link never + // stalls the download loop, and the deferred Flush guarantees a + // terminal-percentage event reaches the caller even when the install errors + // out. var downloadCb func(file, current, total string, percentage float64) - if req.OpID != "" && s.nats != nil { - publisher := nodes.NewDebouncedInstallProgressPublisher(s.nats, s.nodeID, req.OpID, req.Backend, installProgressDebounce) + if req.OpID != "" && onProgress != nil { + publisher := nodes.NewDebouncedInstallProgressSink(onProgress, s.nodeID, req.OpID, req.Backend, installProgressDebounce) downloadCb = publisher.OnDownload defer publisher.Flush() } @@ -155,14 +160,14 @@ func (s *backendSupervisor) installBackend(req messaging.BackendInstallRequest, if req.URI != "" { xlog.Info("Installing backend from external URI", "backend", req.Backend, "uri", req.URI, "force", force) if err := galleryop.InstallExternalBackend( - context.Background(), galleries, s.systemState, s.ml, downloadCb, req.URI, req.Name, req.Alias, force, s.cfg.RequireBackendIntegrity, + ctx, galleries, s.systemState, s.ml, downloadCb, req.URI, req.Name, req.Alias, force, s.cfg.RequireBackendIntegrity, ); err != nil { return "", fmt.Errorf("installing backend from gallery: %w", err) } } else { xlog.Info("Installing backend from gallery", "backend", req.Backend, "force", force) if err := gallery.InstallBackendFromGallery( - context.Background(), galleries, s.systemState, s.ml, req.Backend, downloadCb, force, s.cfg.RequireBackendIntegrity, + ctx, galleries, s.systemState, s.ml, req.Backend, downloadCb, force, s.cfg.RequireBackendIntegrity, ); err != nil { return "", fmt.Errorf("installing backend from gallery: %w", err) } @@ -191,13 +196,15 @@ func (s *backendSupervisor) installBackend(req messaging.BackendInstallRequest, // It does NOT start any new gRPC process — the next routine model load via // backend.install will spawn a fresh process picking up the new binary. // -// The caller is responsible for holding s.lockBackend(req.Backend). +// The caller is responsible for holding s.lockBackend(req.Backend). ctx is the +// caller's budget and onProgress its progress sink, with the same meaning as on +// installBackend. // // It returns the process keys it terminated so the controller can drop the // NodeModel rows addressing them: an upgrade stops every process using the // binary and starts none back up, recycling their gRPC ports while the rows // still point at those addresses. -func (s *backendSupervisor) upgradeBackend(req messaging.BackendUpgradeRequest) ([]string, error) { +func (s *backendSupervisor) upgradeBackend(ctx context.Context, req messaging.BackendUpgradeRequest, onProgress func(messaging.BackendInstallProgressEvent)) ([]string, error) { // Stop every live process for this backend (peer replicas + the bare // processKey). Same logic as the force branch in installBackend. toStop := s.resolveProcessKeysForBackend(s.backendIdentity(req.Backend)) @@ -228,14 +235,14 @@ func (s *backendSupervisor) upgradeBackend(req messaging.BackendUpgradeRequest) galleries = reqGalleries } - // When the master tagged this upgrade with an OpID, stream gallery download - // progress back on the per-op subject (reused from install — an upgrade is a - // force-reinstall). Old masters omit OpID and stay on the silent path. The + // When the caller tagged this upgrade with an OpID, stream gallery download + // progress back on the same sink install uses — an upgrade IS a + // force-reinstall. Callers that omit OpID stay on the silent path. The // deferred Flush guarantees a terminal-percentage event even if the upgrade - // errors out, so the master's per-node bar never hangs mid-download. + // errors out, so the caller's per-node bar never hangs mid-download. var downloadCb func(file, current, total string, percentage float64) - if req.OpID != "" && s.nats != nil { - publisher := nodes.NewDebouncedInstallProgressPublisher(s.nats, s.nodeID, req.OpID, req.Backend, installProgressDebounce) + if req.OpID != "" && onProgress != nil { + publisher := nodes.NewDebouncedInstallProgressSink(onProgress, s.nodeID, req.OpID, req.Backend, installProgressDebounce) downloadCb = publisher.OnDownload defer publisher.Flush() } @@ -243,14 +250,14 @@ func (s *backendSupervisor) upgradeBackend(req messaging.BackendUpgradeRequest) if req.URI != "" { xlog.Info("Upgrading backend from external URI", "backend", req.Backend, "uri", req.URI) if err := galleryop.InstallExternalBackend( - context.Background(), galleries, s.systemState, s.ml, downloadCb, req.URI, req.Name, req.Alias, true, s.cfg.RequireBackendIntegrity, + ctx, galleries, s.systemState, s.ml, downloadCb, req.URI, req.Name, req.Alias, true, s.cfg.RequireBackendIntegrity, ); err != nil { return stopped, fmt.Errorf("upgrading backend from external URI: %w", err) } } else { xlog.Info("Upgrading backend from gallery", "backend", req.Backend) if err := gallery.InstallBackendFromGallery( - context.Background(), galleries, s.systemState, s.ml, req.Backend, downloadCb, true, /* force */ + ctx, galleries, s.systemState, s.ml, req.Backend, downloadCb, true, /* force */ s.cfg.RequireBackendIntegrity, ); err != nil { return stopped, fmt.Errorf("upgrading backend from gallery: %w", err) diff --git a/core/services/worker/lifecycle.go b/core/services/worker/lifecycle.go index b864c8e7c749..89da5419ad59 100644 --- a/core/services/worker/lifecycle.go +++ b/core/services/worker/lifecycle.go @@ -15,170 +15,70 @@ import ( "github.com/mudler/xlog" ) -// subscribeLifecycleEvents wires every NATS subject this worker accepts to its -// per-event handler method. Each handler lives on *backendSupervisor below; -// keeping the dispatcher to a single line per subject makes adding a new -// subject a 2-line patch (one line here, one new method) instead of grafting -// onto a monolith. -func (s *backendSupervisor) subscribeLifecycleEvents() error { - if _, err := s.nats.SubscribeReply(messaging.SubjectNodeBackendInstall(s.nodeID), s.handleBackendInstall); err != nil { - return fmt.Errorf("subscribing to backend install events: %w", err) - } - if _, err := s.nats.SubscribeReply(messaging.SubjectNodeBackendUpgrade(s.nodeID), s.handleBackendUpgrade); err != nil { - return fmt.Errorf("subscribing to backend upgrade events: %w", err) - } - if _, err := s.nats.SubscribeReply(messaging.SubjectNodeBackendStop(s.nodeID), s.handleBackendStop); err != nil { - return fmt.Errorf("subscribing to backend stop events: %w", err) - } - if _, err := s.nats.SubscribeReply(messaging.SubjectNodeBackendDelete(s.nodeID), s.handleBackendDelete); err != nil { - return fmt.Errorf("subscribing to backend delete events: %w", err) - } - if _, err := s.nats.SubscribeReply(messaging.SubjectNodeBackendList(s.nodeID), s.handleBackendList); err != nil { - return fmt.Errorf("subscribing to backend list events: %w", err) - } - if _, err := s.nats.SubscribeReply(messaging.SubjectNodeModelsRunning(s.nodeID), s.handleModelsRunning); err != nil { - return fmt.Errorf("subscribing to models running events: %w", err) - } - if _, err := s.nats.SubscribeReply(messaging.SubjectNodeModelUnload(s.nodeID), s.handleModelUnload); err != nil { - return fmt.Errorf("subscribing to model unload events: %w", err) - } - if _, err := s.nats.SubscribeReply(messaging.SubjectNodeModelStop(s.nodeID), s.handleModelStop); err != nil { - return fmt.Errorf("subscribing to model stop events: %w", err) - } - if _, err := s.nats.SubscribeReply(messaging.SubjectNodeModelDelete(s.nodeID), s.handleModelDelete); err != nil { - return fmt.Errorf("subscribing to model delete events: %w", err) - } - if _, err := s.nats.Subscribe(messaging.SubjectNodeStop(s.nodeID), s.handleNodeStop); err != nil { - return fmt.Errorf("subscribing to node stop events: %w", err) - } - return nil -} - -func (s *backendSupervisor) handleModelStop(data []byte, reply func([]byte)) { - var req messaging.ModelStopRequest - if err := json.Unmarshal(data, &req); err != nil { - replyJSON(reply, messaging.ModelStopReply{Error: fmt.Sprintf("invalid request: %v", err)}) - return +// The worker's lifecycle verbs. +// +// Each takes a decoded request and returns a reply value. They carry no +// carrier: the HTTP control plane in control_routes.go is what decodes the +// request, calls one of these, and encodes what comes back. Keeping the verb +// free of its transport is what let the ten NATS subscriptions these replaced +// be deleted without touching a line of what they actually do. + +// backendList answers backend.list with the backends installed in this node's +// gallery. +func (s *backendSupervisor) backendList() messaging.BackendListReply { + xlog.Info("Serving backend.list") + backends, err := gallery.ListSystemBackends(s.systemState) + if err != nil { + return messaging.BackendListReply{Error: err.Error()} } - replyJSON(reply, s.stopModelExact(req)) -} -// handleBackendInstall is the NATS callback for backend.install — install -// backend (idempotent: skips download if binary exists on disk) + start gRPC -// process (request-reply). -// -// Each request runs in its own goroutine so that a slow install on one -// backend does NOT head-of-line-block install requests for unrelated -// backends arriving on the same subscription. Per-backend serialization -// is provided by lockBackend so two requests targeting the same on-disk -// artifact don't race the gallery directory. -func (s *backendSupervisor) handleBackendInstall(data []byte, reply func([]byte)) { - go func() { - xlog.Info("Received NATS backend.install event") - var req messaging.BackendInstallRequest - if err := json.Unmarshal(data, &req); err != nil { - resp := messaging.BackendInstallReply{Success: false, Error: fmt.Sprintf("invalid request: %v", err)} - replyJSON(reply, resp) - return + var infos []messaging.NodeBackendInfo + for name, b := range backends { + // Drop synthetic alias rows: ListSystemBackends emits an entry + // keyed by the alias name that re-uses the chosen concrete's + // metadata. The frontend can't reconstruct that aliasing + // faithfully from a flat NodeBackendInfo, and for upgrade + // detection it would surface as a phantom `` install + // pointing at the dev concrete's URI/digest — tricking the + // upgrade check into flagging the non-dev gallery entry of the + // same alias. Concrete and meta entries always have + // `name == b.Metadata.Name`, so this drops aliases only. + if b.Metadata != nil && b.Metadata.Name != "" && name != b.Metadata.Name { + continue } - - release := s.lockBackend(req.Backend) - defer release() - - // req.Force=true is the legacy path used by pre-2026-05-08 masters - // that don't know about backend.upgrade. Honor it so a rolling - // update with new worker + old master keeps working; new masters - // send to backend.upgrade instead. - addr, err := s.installBackend(req, req.Force) - if err != nil { - xlog.Error("Failed to install backend via NATS", "error", err) - resp := messaging.BackendInstallReply{Success: false, Error: err.Error()} - replyJSON(reply, resp) - return + info := messaging.NodeBackendInfo{ + Name: name, + IsSystem: b.IsSystem, + IsMeta: b.IsMeta, } - - // The address goes back exactly as the process listens on it. It used - // to be rewritten onto this worker's advertise host, which made the - // reply the worker's third advertisement site; the frontend now reads - // only the port out of it and dials nothing. - // - // The rewrite was also wrong in a way nothing caught: the worker - // records the loopback address and stopModelExact refuses a stop whose - // ExpectedAddress does not match it, so on any worker whose advertise - // host was not 127.0.0.1 every acknowledged model stop failed with an - // address mismatch. - replyJSON(reply, messaging.BackendInstallReply{Success: true, WorkerLocalAddress: addr}) - }() -} - -// handleBackendUpgrade is the NATS callback for backend.upgrade — force-reinstall -// a backend (request-reply). Lives on its own subscription so a multi-minute -// download here does NOT block the install fast-path subscription on the same -// worker. -func (s *backendSupervisor) handleBackendUpgrade(data []byte, reply func([]byte)) { - go func() { - xlog.Info("Received NATS backend.upgrade event") - var req messaging.BackendUpgradeRequest - if err := json.Unmarshal(data, &req); err != nil { - resp := messaging.BackendUpgradeReply{Success: false, Error: fmt.Sprintf("invalid request: %v", err)} - replyJSON(reply, resp) - return + if b.Metadata != nil { + info.InstalledAt = b.Metadata.InstalledAt + info.GalleryURL = b.Metadata.GalleryURL + info.Version = b.Metadata.Version + info.URI = b.Metadata.URI + info.Digest = b.Metadata.Digest } + infos = append(infos, info) + } - release := s.lockBackend(req.Backend) - defer release() - - // stopped is meaningful even on the error paths: it lists processes - // already terminated (and ports already recycled) before the failure, so - // the controller must drop those rows regardless of the outcome. - stopped, err := s.upgradeBackend(req) - if err != nil { - xlog.Error("Failed to upgrade backend via NATS", "error", err) - replyJSON(reply, messaging.BackendUpgradeReply{ - Success: false, - Error: err.Error(), - StoppedProcessKeys: stopped, - ReportsStoppedProcesses: true, - }) - return - } - replyJSON(reply, messaging.BackendUpgradeReply{ - Success: true, - StoppedProcessKeys: stopped, - ReportsStoppedProcesses: true, - }) - }() + return messaging.BackendListReply{Backends: infos} } -// handleBackendStop is the NATS callback for backend.stop — stop a specific -// backend process and report what it terminated. -// -// The reply is what lets the controller tell a stop that worked from one that -// matched nothing or failed. Callers that publish without a reply subject (an -// older controller) still work: SubscribeReply drops the response. -func (s *backendSupervisor) handleBackendStop(data []byte, reply func([]byte)) { - req, stopAll, err := decodeBackendStopRequest(data) - if err != nil { - xlog.Error("Ignoring malformed NATS backend.stop event", "error", err) - replyJSON(reply, messaging.BackendStopReply{ - Error: fmt.Sprintf("invalid request: %v", err), - ReportsStoppedProcesses: true, - }) - return - } +// stopBackends serves backend.stop: it terminates the processes the request +// names, or every process when it names none. +func (s *backendSupervisor) stopBackends(req messaging.BackendStopRequest, stopAll bool) messaging.BackendStopReply { if stopAll { - xlog.Info("Received NATS backend.stop event (all)", "force", req.Force) + xlog.Info("Serving backend.stop (all)", "force", req.Force) stopped := s.stopAllBackends(req.Force) - replyJSON(reply, messaging.BackendStopReply{ + return messaging.BackendStopReply{ Success: true, StoppedProcessKeys: stopped, ReportsStoppedProcesses: true, - }) - return + } } - xlog.Info("Received NATS backend.stop event", "backend", req.Backend, "force", req.Force) + xlog.Info("Serving backend.stop", "backend", req.Backend, "force", req.Force) // The identifier may be a backend name, a model name, or an exact - // modelID#replica key depending on the publisher; resolveStopTargets + // modelID#replica key depending on the caller; resolveStopTargets // handles all three. stopBackend alone resolves only the model meanings. var stopped []string var failures []string @@ -202,9 +102,12 @@ func (s *backendSupervisor) handleBackendStop(data []byte, reply func([]byte)) { if len(failures) > 0 { res.Error = strings.Join(failures, "; ") } - replyJSON(reply, res) + return res } +// decodeBackendStopRequest reads a backend.stop body. An EMPTY body means "stop +// everything", which is the shape the frontend uses to drain a node, so it is +// not a decode failure. func decodeBackendStopRequest(data []byte) (messaging.BackendStopRequest, bool, error) { if len(data) == 0 { return messaging.BackendStopRequest{}, true, nil @@ -216,16 +119,10 @@ func decodeBackendStopRequest(data []byte) (messaging.BackendStopRequest, bool, return req, req.Backend == "", nil } -// handleBackendDelete is the NATS callback for backend.delete — stop the -// backend process if running, then remove its files from disk (request-reply). -func (s *backendSupervisor) handleBackendDelete(data []byte, reply func([]byte)) { - var req messaging.BackendDeleteRequest - if err := json.Unmarshal(data, &req); err != nil { - resp := messaging.BackendDeleteReply{Success: false, Error: fmt.Sprintf("invalid request: %v", err)} - replyJSON(reply, resp) - return - } - xlog.Info("Received NATS backend.delete event", "backend", req.Backend) +// deleteBackend serves backend.delete: stop the backend's processes if running, +// then remove its files from disk. +func (s *backendSupervisor) deleteBackend(req messaging.BackendDeleteRequest) messaging.BackendDeleteReply { + xlog.Info("Serving backend.delete", "backend", req.Backend) // Resolve the backend's identity (concrete name + alias) BEFORE touching // the filesystem: DeleteBackendFromSystem removes the metadata.json that @@ -267,8 +164,7 @@ func (s *backendSupervisor) handleBackendDelete(data []byte, reply func([]byte)) // "backend deleted" while the process keeps serving requests. xlog.Error("Failed to stop backend process during delete; aborting delete", "backend", req.Backend, "processKey", key, "error", err) - replyJSON(reply, deleteReply(false, fmt.Sprintf("could not stop running process %s: %v", key, err))) - return + return deleteReply(false, fmt.Sprintf("could not stop running process %s: %v", key, err)) } stopped = append(stopped, key) } @@ -276,74 +172,22 @@ func (s *backendSupervisor) handleBackendDelete(data []byte, reply func([]byte)) // Delete the backend files if err := gallery.DeleteBackendFromSystem(s.systemState, req.Backend); err != nil { xlog.Warn("Failed to delete backend files", "backend", req.Backend, "error", err) - replyJSON(reply, deleteReply(false, err.Error())) - return + return deleteReply(false, err.Error()) } // Re-register backends after deletion if err := gallery.RegisterBackends(s.systemState, s.ml); err != nil { xlog.Error("Failed to refresh registered backends after deletion", "backend", req.Backend, "error", err) - replyJSON(reply, deleteReply(false, err.Error())) - return - } - - replyJSON(reply, deleteReply(true, "")) -} - -// handleBackendList is the NATS callback for backend.list — reply with the -// installed backends from this node's gallery (request-reply). -func (s *backendSupervisor) handleBackendList(data []byte, reply func([]byte)) { - xlog.Info("Received NATS backend.list event") - backends, err := gallery.ListSystemBackends(s.systemState) - if err != nil { - resp := messaging.BackendListReply{Error: err.Error()} - replyJSON(reply, resp) - return - } - - var infos []messaging.NodeBackendInfo - for name, b := range backends { - // Drop synthetic alias rows: ListSystemBackends emits an entry - // keyed by the alias name that re-uses the chosen concrete's - // metadata. The frontend can't reconstruct that aliasing - // faithfully from a flat NodeBackendInfo, and for upgrade - // detection it would surface as a phantom `` install - // pointing at the dev concrete's URI/digest — tricking the - // upgrade check into flagging the non-dev gallery entry of the - // same alias. Concrete and meta entries always have - // `name == b.Metadata.Name`, so this drops aliases only. - if b.Metadata != nil && b.Metadata.Name != "" && name != b.Metadata.Name { - continue - } - info := messaging.NodeBackendInfo{ - Name: name, - IsSystem: b.IsSystem, - IsMeta: b.IsMeta, - } - if b.Metadata != nil { - info.InstalledAt = b.Metadata.InstalledAt - info.GalleryURL = b.Metadata.GalleryURL - info.Version = b.Metadata.Version - info.URI = b.Metadata.URI - info.Digest = b.Metadata.Digest - } - infos = append(infos, info) + return deleteReply(false, err.Error()) } - resp := messaging.BackendListReply{Backends: infos} - replyJSON(reply, resp) + return deleteReply(true, "") } -// handleModelUnload is the NATS callback for model.unload — call gRPC Free() -// to release GPU memory without killing the backend process (request-reply). -func (s *backendSupervisor) handleModelUnload(data []byte, reply func([]byte)) { - xlog.Info("Received NATS model.unload event") - var req messaging.ModelUnloadRequest - if err := json.Unmarshal(data, &req); err != nil { - resp := messaging.ModelUnloadReply{Success: false, Error: fmt.Sprintf("invalid request: %v", err)} - replyJSON(reply, resp) - return - } +// unloadModel serves model.unload: a gRPC Free() that releases GPU memory +// without killing the backend process. +func (s *backendSupervisor) unloadModel(ctx context.Context, req messaging.ModelUnloadRequest) messaging.ModelUnloadReply { + xlog.Info("Serving model.unload") // Find the backend address for this model's backend type // The request includes an Address field if the router knows which process to target @@ -358,44 +202,40 @@ func (s *backendSupervisor) handleModelUnload(data []byte, reply func([]byte)) { s.mu.Unlock() } - if targetAddr != "" { - // Best-effort bounded gRPC Free(). A model.unload request must not - // occupy the NATS reply handler forever when a backend is wedged. - client := grpc.NewClientWithToken(targetAddr, false, nil, false, s.cfg.RegistrationToken) - freeCtx, cancel := context.WithTimeout(context.Background(), workerBackendFreeTimeout) - if err := client.Free(freeCtx); err != nil { - xlog.Warn("Free() failed during model.unload", "error", err, "addr", targetAddr) - } - cancel() + if targetAddr == "" { + return messaging.ModelUnloadReply{Success: true} } - resp := messaging.ModelUnloadReply{Success: true} - replyJSON(reply, resp) + client := grpc.NewClientWithToken(targetAddr, false, nil, false, s.cfg.RegistrationToken) + freeCtx, cancel := context.WithTimeout(ctx, workerBackendFreeTimeout) + err := client.Free(freeCtx) + cancel() + if err != nil { + xlog.Warn("Free() failed during model.unload", "error", err, "addr", targetAddr) + return messaging.ModelUnloadReply{ + Success: false, + Error: fmt.Sprintf("freeing model on %s: %v", targetAddr, err), + } + } + return messaging.ModelUnloadReply{Success: true} } -// handleModelDelete is the NATS callback for model.delete — remove model -// files from disk (request-reply). -func (s *backendSupervisor) handleModelDelete(data []byte, reply func([]byte)) { - xlog.Info("Received NATS model.delete event") - var req messaging.ModelDeleteRequest - if err := json.Unmarshal(data, &req); err != nil { - replyJSON(reply, messaging.ModelDeleteReply{Success: false, Error: "invalid request"}) - return - } +// deleteModel serves model.delete: remove a model's staged files from disk. +func (s *backendSupervisor) deleteModel(req messaging.ModelDeleteRequest) messaging.ModelDeleteReply { + xlog.Info("Serving model.delete", "model", req.ModelName) if err := gallery.DeleteStagedModelFiles(s.cfg.ModelsPath, req.ModelName); err != nil { xlog.Warn("Failed to delete model files", "model", req.ModelName, "error", err) - replyJSON(reply, messaging.ModelDeleteReply{Success: false, Error: err.Error()}) - return + return messaging.ModelDeleteReply{Success: false, Error: err.Error()} } - replyJSON(reply, messaging.ModelDeleteReply{Success: true}) + return messaging.ModelDeleteReply{Success: true} } -// handleNodeStop is the NATS callback for node.stop — trigger the normal -// shutdown path via sigCh so deferred cleanup runs (fire-and-forget). -func (s *backendSupervisor) handleNodeStop(data []byte) { - xlog.Info("Received NATS stop event — signaling shutdown") +// signalNodeStop serves node.stop: it triggers the normal shutdown path via +// sigCh so deferred cleanup runs, rather than exiting the process here. +func (s *backendSupervisor) signalNodeStop() { + xlog.Info("Serving node.stop — signaling shutdown") select { case s.sigCh <- syscall.SIGTERM: default: diff --git a/core/services/worker/model_stop_test.go b/core/services/worker/model_stop_test.go index 345b61a30976..c54aec821b6f 100644 --- a/core/services/worker/model_stop_test.go +++ b/core/services/worker/model_stop_test.go @@ -1,13 +1,17 @@ package worker import ( + "bytes" "context" "encoding/json" "errors" "net" + "net/http" + "net/http/httptest" "sync/atomic" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" process "github.com/mudler/go-processmanager" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -42,13 +46,27 @@ func startModelStopProcess() *process.Process { return proc } +// requestModelStop drives the model.stop verb over the HTTP control plane the +// worker actually serves, rather than calling the method directly. These specs +// are the ones that pin the acknowledged-stop contract, so routing them through +// the carrier is what keeps a routing mistake from passing them. func requestModelStop(s *backendSupervisor, req messaging.ModelStopRequest) messaging.ModelStopReply { + GinkgoHelper() data, err := json.Marshal(req) Expect(err).NotTo(HaveOccurred()) - var response []byte - s.handleModelStop(data, func(data []byte) { response = append([]byte(nil), data...) }) + + mux := http.NewServeMux() + s.RegisterControlRoutes(mux) + srv := httptest.NewServer(mux) + defer srv.Close() + + resp, err := srv.Client().Post(srv.URL+workerctl.PathModelStop, "application/json", bytes.NewReader(data)) + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = resp.Body.Close() }() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply messaging.ModelStopReply - Expect(json.Unmarshal(response, &reply)).To(Succeed()) + Expect(json.NewDecoder(resp.Body).Decode(&reply)).To(Succeed()) return reply } diff --git a/core/services/worker/models_running.go b/core/services/worker/models_running.go index efc4700a084a..4cdb08edb15d 100644 --- a/core/services/worker/models_running.go +++ b/core/services/worker/models_running.go @@ -55,11 +55,3 @@ func (s *backendSupervisor) runningModels() []messaging.RunningModelInfo { } return running } - -// handleModelsRunning answers a models.running request with this worker's live -// process set. -func (s *backendSupervisor) handleModelsRunning(_ []byte, reply func([]byte)) { - running := s.runningModels() - xlog.Debug("Answering models.running", "nodeID", s.nodeID, "count", len(running)) - replyJSON(reply, messaging.ModelsRunningReply{Models: running}) -} diff --git a/core/services/worker/supervisor.go b/core/services/worker/supervisor.go index 9dba6a7ecbdf..9a860c4e8cbc 100644 --- a/core/services/worker/supervisor.go +++ b/core/services/worker/supervisor.go @@ -129,9 +129,15 @@ type backendSupervisor struct { systemState *system.SystemState galleries []config.Gallery nodeID string - nats messaging.MessagingClient sigCh chan<- os.Signal // send shutdown signal instead of os.Exit + // installFn and upgradeFn override the two long-running verbs. Non-nil + // only in specs: they exist so the control plane's ROUTING can be exercised + // without a gallery, a registry or a real download, which is the same + // argument tunnelServices was extracted under. See installer/upgrader. + installFn installFunc + upgradeFn upgradeFunc + mu sync.Mutex processes map[string]*backendProcess // key: backend name nextPort int // next unhanded-out port; grows within [minPort, maxPort] diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index dd8f974eb07c..ea1dc2f82024 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "net" + "net/http" "os" "os/signal" "path/filepath" @@ -19,6 +20,7 @@ import ( "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/workerctl" grpc "github.com/mudler/LocalAI/pkg/grpc" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/sanitize" @@ -26,8 +28,8 @@ import ( "github.com/mudler/xlog" ) -// Run starts the distributed agent worker: registers with the frontend, -// subscribes to NATS lifecycle subjects, and blocks on signals. +// Run starts the distributed agent worker: registers with the frontend, opens +// its tunnel, serves its control plane on that tunnel, and blocks on signals. func Run(ctx *cliContext.Context, cfg *Config) error { xlog.Info("Starting worker", "basePort", cfg.effectiveBasePort()) @@ -211,7 +213,42 @@ func Run(ctx *cliContext.Context, cfg *Config) error { // means the worker has already registered with the frontend, so it is // mid-startup rather than broken. readiness := &nodes.WorkerReadiness{} - httpServer, err := nodes.StartFileTransferServerWithCapacity(httpAddr, stagingDir, cfg.ModelsPath, dataDir, cfg.RegistrationToken, config.DefaultMaxUploadSize, readiness, ephemeralCapacity, ml.BackendLogs()) + + // The supervisor is built BEFORE the HTTP server, not after, because the + // server is what serves its control plane. Ten NATS subscriptions used to + // be attached to it later, and could be, because the bus buffered nothing + // the worker had not subscribed to; a control route that is not mounted + // when the tunnel comes up is instead a 404 the frontend reads as a worker + // that does not implement the verb. + basePort := cfg.effectiveBasePort() + // Buffered so the node.stop verb can signal without blocking its response. + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + + // Set the registration token once before any backends are started. + if cfg.RegistrationToken != "" { + if err := os.Setenv(grpc.AuthTokenEnvVar, cfg.RegistrationToken); err != nil { + return fmt.Errorf("setting backend authentication token: %w", err) + } + } + + // Process supervisor — manages multiple backend gRPC processes on different ports + supervisor := &backendSupervisor{ + cfg: cfg, + ml: ml, + systemState: systemState, + galleries: galleries, + nodeID: nodeID, + sigCh: sigCh, + processes: make(map[string]*backendProcess), + portAffinity: make(map[string]portOwnership), + nextPort: basePort, + minPort: basePort, + maxPort: cfg.effectiveMaxPort(basePort), + } + + httpServer, err := startWorkerHTTPServer(httpAddr, stagingDir, cfg.ModelsPath, dataDir, + cfg.RegistrationToken, readiness, supervisor, ephemeralCapacity, ml.BackendLogs()) if err != nil { return fmt.Errorf("starting HTTP file transfer server: %w", err) } @@ -281,35 +318,6 @@ func Run(ctx *cliContext.Context, cfg *Config) error { } }() - // Process supervisor — manages multiple backend gRPC processes on different ports - basePort := cfg.effectiveBasePort() - // Buffered so NATS stop handler can send without blocking - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - - // Set the registration token once before any backends are started - if cfg.RegistrationToken != "" { - if err := os.Setenv(grpc.AuthTokenEnvVar, cfg.RegistrationToken); err != nil { - nodes.ShutdownFileTransferServer(httpServer) - return fmt.Errorf("setting backend authentication token: %w", err) - } - } - - supervisor := &backendSupervisor{ - cfg: cfg, - ml: ml, - systemState: systemState, - galleries: galleries, - nodeID: nodeID, - nats: natsClient, - sigCh: sigCh, - processes: make(map[string]*backendProcess), - portAffinity: make(map[string]portOwnership), - nextPort: basePort, - minPort: basePort, - maxPort: cfg.effectiveMaxPort(basePort), - } - // Arm the readiness gate now that the worker can actually receive work. // NATS is already connected at this point, so a worker that is up but cut // off from the bus reports 503 instead of a meaningless 200 (#10987). @@ -328,11 +336,6 @@ func Run(ctx *cliContext.Context, cfg *Config) error { }), )) - if err := supervisor.subscribeLifecycleEvents(); err != nil { - nodes.ShutdownFileTransferServer(httpServer) - return fmt.Errorf("subscribing to worker lifecycle events: %w", err) - } - // Subscribe to file staging NATS subjects if S3 is configured if cfg.StorageURL != "" { if err := cfg.subscribeFileStaging(natsClient, nodeID, ephemeralCapacity); err != nil { @@ -341,7 +344,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error { } } - xlog.Info("Worker ready, waiting for backend.install events") + xlog.Info("Worker ready, serving its control plane over the tunnel") // Exit on an OS signal or on an internal fatal condition (e.g. NATS // credentials became unrenewable), so the worker restarts and re-acquires // rather than lingering unable to serve. @@ -360,3 +363,22 @@ func Run(ctx *cliContext.Context, cfg *Config) error { nodes.ShutdownFileTransferServer(httpServer) return runErr } + +// startWorkerHTTPServer starts the worker's loopback HTTP server with sup's +// control plane mounted on it. +// +// It takes the supervisor rather than an optional route set on purpose: the +// control plane and the file routes are served by one listener behind one +// bearer check, and there is no worker that wants the second without the first. +// Making the supervisor a parameter is what stops a future edit from starting +// the server without the control plane and producing a worker that looks +// healthy while answering 404 to every command. +func startWorkerHTTPServer(addr, stagingDir, modelsDir, dataDir, token string, + readiness *nodes.WorkerReadiness, sup *backendSupervisor, capacity nodes.EphemeralCapacity, + logStore *model.BackendLogStore) (*http.Server, error) { + return nodes.StartFileTransferServerWithCapacityAndRoutes(addr, stagingDir, modelsDir, dataDir, token, + config.DefaultMaxUploadSize, readiness, capacity, &nodes.AuthenticatedRoutes{ + Prefix: workerctl.Prefix, + Register: sup.RegisterControlRoutes, + }, logStore) +} diff --git a/core/services/workerctl/paths.go b/core/services/workerctl/paths.go new file mode 100644 index 000000000000..e76b5566dacd --- /dev/null +++ b/core/services/workerctl/paths.go @@ -0,0 +1,71 @@ +// Package workerctl names the HTTP control plane a worker serves to the +// frontends that manage it. +// +// It is a leaf on the standard library alone, and deliberately so: the worker +// registers these paths and the frontend calls them, so both sides must agree +// on the literals without either importing the other's package. +package workerctl + +import "encoding/json" + +// Prefix is the one path prefix the worker mounts its whole control plane +// under. Everything the frontend may command a worker to do lives below it, +// which is what lets the worker put the control plane behind a single +// authentication check instead of one per verb. +const Prefix = "/v1/control/" + +// The control verbs. Each replaces one NATS subject; the request and reply +// bodies are the messaging DTOs those subjects already carried, unchanged, so +// a worker still reachable over NATS and one reachable over the tunnel answer +// with the same bytes. +const ( + PathBackendInstall = "/v1/control/backend/install" + PathBackendUpgrade = "/v1/control/backend/upgrade" + PathBackendList = "/v1/control/backend/list" + PathBackendStop = "/v1/control/backend/stop" + PathBackendDelete = "/v1/control/backend/delete" + PathModelStop = "/v1/control/model/stop" + PathModelUnload = "/v1/control/model/unload" + PathModelDelete = "/v1/control/model/delete" + PathModelsRunning = "/v1/control/models/running" + PathNodeStop = "/v1/control/node/stop" +) + +// AllPaths returns every control verb's path. +// +// It exists so a spec can assert a property of the whole set rather than of a +// list it re-types, which would go stale the moment a verb is added. +func AllPaths() []string { + return []string{ + PathBackendInstall, + PathBackendUpgrade, + PathBackendList, + PathBackendStop, + PathBackendDelete, + PathModelStop, + PathModelUnload, + PathModelDelete, + PathModelsRunning, + PathNodeStop, + } +} + +// Envelope is one line of a streaming control response. +// +// Exactly one of the two is set. Zero or more Progress lines are followed by +// exactly ONE Reply line, and the Reply line is the last thing on the body. +// That ordering is the contract: it is what lets the frontend stop reading, and +// it is what replaces the subscribe-before-request dance the NATS carrier +// needed, since progress and reply now share one response and nothing can +// arrive before the caller is listening. +// +// Progress carrying the reply's own bytes is also why the 8000-byte +// notification cap that bounded the NATS progress subject has no analogue here: +// a line is written into the response body the caller is already reading. +type Envelope struct { + Progress json.RawMessage `json:"progress,omitempty"` + Reply json.RawMessage `json:"reply,omitempty"` +} + +// ContentTypeStream is the media type of a streaming control response. +const ContentTypeStream = "application/x-ndjson" diff --git a/core/services/workerctl/paths_test.go b/core/services/workerctl/paths_test.go new file mode 100644 index 000000000000..c5685a99dde2 --- /dev/null +++ b/core/services/workerctl/paths_test.go @@ -0,0 +1,72 @@ +package workerctl_test + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// The literals below are written out BY HAND and are deliberately not derived +// from the constants under test. A spec that says PathBackendInstall equals +// PathBackendInstall pins nothing: a rename moves both sides at once and stays +// green. What these paths actually are is a cross-version contract, because a +// frontend and a worker built from different commits reach each other over +// them, and a renamed path is a 404 that looks exactly like a broken tunnel. +var _ = Describe("control plane paths on the wire", func() { + DescribeTable("is the exact path both sides agree on", + func(got, want string) { Expect(got).To(Equal(want)) }, + Entry("install", workerctl.PathBackendInstall, "/v1/control/backend/install"), + Entry("upgrade", workerctl.PathBackendUpgrade, "/v1/control/backend/upgrade"), + Entry("list", workerctl.PathBackendList, "/v1/control/backend/list"), + Entry("backend stop", workerctl.PathBackendStop, "/v1/control/backend/stop"), + Entry("backend delete", workerctl.PathBackendDelete, "/v1/control/backend/delete"), + Entry("model stop", workerctl.PathModelStop, "/v1/control/model/stop"), + Entry("model unload", workerctl.PathModelUnload, "/v1/control/model/unload"), + Entry("model delete", workerctl.PathModelDelete, "/v1/control/model/delete"), + Entry("models running", workerctl.PathModelsRunning, "/v1/control/models/running"), + Entry("node stop", workerctl.PathNodeStop, "/v1/control/node/stop"), + ) + + It("names the prefix exactly, since the worker mounts its whole control plane behind it", func() { + Expect(workerctl.Prefix).To(Equal("/v1/control/")) + }) + + It("puts every path under the one prefix", func() { + for _, p := range workerctl.AllPaths() { + Expect(p).To(HavePrefix(workerctl.Prefix)) + } + }) + + It("enumerates every verb, so a new one cannot be added without the prefix check seeing it", func() { + Expect(workerctl.AllPaths()).To(HaveLen(10)) + Expect(workerctl.AllPaths()).To(ContainElement(workerctl.PathBackendInstall)) + Expect(workerctl.AllPaths()).To(ContainElement(workerctl.PathNodeStop)) + }) + + It("gives each verb a distinct path", func() { + seen := map[string]bool{} + for _, p := range workerctl.AllPaths() { + Expect(seen[p]).To(BeFalse(), "duplicate control path %q", p) + seen[p] = true + } + }) + + It("marshals an envelope with exactly one populated field", func() { + b, err := json.Marshal(workerctl.Envelope{Reply: json.RawMessage(`{"success":true}`)}) + Expect(err).NotTo(HaveOccurred()) + Expect(string(b)).To(Equal(`{"reply":{"success":true}}`)) + }) + + It("marshals a progress envelope without a reply key, which is what ends the body", func() { + b, err := json.Marshal(workerctl.Envelope{Progress: json.RawMessage(`{"percentage":50}`)}) + Expect(err).NotTo(HaveOccurred()) + Expect(string(b)).To(Equal(`{"progress":{"percentage":50}}`)) + }) + + It("names the streaming media type", func() { + Expect(workerctl.ContentTypeStream).To(Equal("application/x-ndjson")) + }) +}) diff --git a/core/services/workerctl/workerctl_suite_test.go b/core/services/workerctl/workerctl_suite_test.go new file mode 100644 index 000000000000..a47f1c8ffdfc --- /dev/null +++ b/core/services/workerctl/workerctl_suite_test.go @@ -0,0 +1,13 @@ +package workerctl_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestWorkerctl(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Workerctl test suite") +} diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index c48b9b9c2adc..c0ce0b6d3152 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -30,7 +30,7 @@ The SmartRouter uses **idle-first** scheduling with **preemptive eviction**: 4. Fall back to idle nodes (zero models), then least-loaded nodes 5. If no node has capacity → **evict the least-recently-used model with zero in-flight requests** to free a node 6. If all models are busy → wait (with timeout) for a model to become idle, then evict -7. Send `backend.install` NATS event with backend name + model ID → worker starts a new gRPC process on a dynamic port +7. `POST /v1/control/backend/install` through the worker's tunnel with backend name + model ID → worker starts a new gRPC process on a dynamic port 8. SmartRouter calls gRPC `LoadModel` on the model-specific port, records in DB Each model gets its own gRPC backend process, so a single worker can serve multiple models simultaneously (e.g., a chat model and an embedding model). @@ -172,6 +172,34 @@ Every connection the frontend makes to a worker now goes through that worker's t | Inference, model load, health checks | gRPC to a backend process | `grpc` | | Model file staging, backend-log listing | HTTP to the worker's own server | `http` | | Live backend-log streaming | WebSocket to the same server | `http` | +| Backend install/upgrade/list/delete, model stop/unload/delete, node stop | HTTP to the worker's own server | `http` | + +#### The worker control plane + +A serve-backend worker serves the commands the frontend gives it as ordinary +HTTP routes under `/v1/control/`, on the same loopback server that already +carries file staging and backend logs, behind the same `LOCALAI_REGISTRATION_TOKEN` +bearer check. They replace the ten `nodes..*` NATS subjects a worker used to +subscribe to; the request and reply bodies are unchanged, so nothing an operator +inspects on the wire has a new shape. Agent workers still take `nodes..backend.stop` +over NATS. + +Two of the routes stream. `POST /v1/control/backend/install` and +`/v1/control/backend/upgrade` answer with `application/x-ndjson`: zero or more +`{"progress":{...}}` lines carrying the same download-progress payload the +per-op NATS subject carried, followed by exactly one `{"reply":{...}}` line, +which is always the last line on the body. An install that FAILS is still a +`200` with a reply whose `success` is `false`. That is deliberate, and it is the +same distinction the refusal table above draws: a non-2xx means the frontend +could not get the request to the worker, which nothing may act on, while the +worker's own verdict, including "there is no such backend", is evidence a reap +guard may act on. A worker that answered `500` for a failed install would put +its own verdict in the bucket reserved for a broken link. + +A control request carries the caller's deadline and nothing else: the worker +does not impose a timeout of its own on an install, and a caller that gives up +cancels the download rather than leaving the worker pulling gigabytes for a +response nobody will read. The address the frontend holds for a backend (the per-replica port a worker reports after an install) is still what identifies it, and it is still what appears in logs and errors. What it no longer is, is somewhere the frontend connects to: it travels inside the tunnel as the stream's target, and the worker decides what to do with it. @@ -343,7 +371,7 @@ A frontend replica that dies mid-load does not wedge the model: the job row carr ### NATS JWT authentication (recommended for production) -By default, NATS connections are anonymous: any client that can reach port `4222` may publish control-plane subjects such as `nodes..backend.install`. Enable JWT auth to scope workers to their own node subjects and give the frontend a dedicated service credential. +By default, NATS connections are anonymous: any client that can reach port `4222` may publish the subjects still carried on it. Serve-backend workers no longer subscribe to `nodes..backend.install` and its nine siblings - those are HTTP routes on the worker's tunnel now, see [The worker control plane](#the-worker-control-plane) - but agent workers, file staging and the frontend's own service credential still use NATS. Enable JWT auth to scope workers to their own node subjects and give the frontend a dedicated service credential. | Flag | Env Var | Description | |------|---------|-------------| @@ -427,7 +455,7 @@ during installation as well as the committed snapshot. {{% /notice %}} {{% notice warning %}} -The worker HTTP file transfer server is authenticated by `LOCALAI_REGISTRATION_TOKEN`. If the token is **empty**, the server **fails open** - anyone who can reach the port gets read/write access to the worker's models/staging/data directories (a remote model-poisoning / exfiltration vector). The worker logs a loud warning at startup in this case. Always set `LOCALAI_REGISTRATION_TOKEN` in distributed mode, and set `LOCALAI_DISTRIBUTED_REQUIRE_AUTH=true` (frontend **and** workers) to make a missing token *or* missing NATS credentials a hard startup error rather than a silent fail-open. +The worker HTTP file transfer server is authenticated by `LOCALAI_REGISTRATION_TOKEN`. If the token is **empty**, the server **fails open** - anyone who can reach the port gets read/write access to the worker's models/staging/data directories (a remote model-poisoning / exfiltration vector), **and to the `/v1/control/` routes that install, upgrade and delete backends and stop the node**. The worker logs a loud warning at startup in this case. Always set `LOCALAI_REGISTRATION_TOKEN` in distributed mode, and set `LOCALAI_DISTRIBUTED_REQUIRE_AUTH=true` (frontend **and** workers) to make a missing token *or* missing NATS credentials a hard startup error rather than a silent fail-open. By default the server binds loopback, so "anyone who can reach the port" means a process on the worker host, and no firewall rule is required. Setting `LOCALAI_HTTP_ADDR` to a routable address opts back out of that and puts the fail-open case back on the network - if you do it, firewall the port. {{% /notice %}} From 3b836bce36212413607bf49b60fc5d18be6b6127 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 16:21:41 +0000 Subject: [PATCH 071/153] fix(worker): pin the rune cut, answer unload honestly, drop the dead publisher Review fix round 1. Seven non-blocking findings; the blocking one is a merge gate for Task 4 rather than anything in this diff, and the report's concern about it is corrected: until Task 4 lands, PingNode probes two subjects no serve-backend worker subscribes to any more, so every healthy worker reads as absent and is marked unhealthy on the scheduling path. The rune-boundary cut in truncate was true behaviour with nothing holding it: a byte-wise mutation survived all 201 specs. isRuneStart is replaced by utf8.RuneStart, the same predicate the cluster package uses for this rule, and two specs pin it, one with a rune straddling the bound and one with a rune ending exactly on it so the fix cannot be "always walk back". unloadModel answered Success:true whatever Free did. That is the worker saying "done" about work it did not do, and the frontend's only caller is EvictLRU, so a false yes told the scheduler VRAM had been released and let it place the next model on a node still holding the old one. It now reports the failure, following stopModelExact, which is the honest pattern already in this package. Still a 200: the worker answered, only its verdict is negative. An address with nothing loaded still answers success, which is a true answer rather than a claim about work done. NewDebouncedInstallProgressPublisher had no production caller after the last commit, only its own spec. Deleted rather than wired: wiring it would publish every event on two carriers, which is what the carrier decision exists to avoid. Its specs now run against the sink, plus one that pins the identity stamped on each event, since the subject used to carry the op and node id and now nothing but the body does. The install progress wiring was exercised by no spec, because with no gallery nothing ever invokes the download callback. The guard moves into startProgress, shared by install and upgrade, which also emits one resolving event before any gallery work. That is worth having on its own: a cold install spends minutes on a manifest and a progress stream with nothing on it is indistinguishable from a broken one. It also makes the wiring observable end to end, and four specs now drive the real installBackend and upgradeBackend over HTTP with no override. model/stop and backend/stop keep taking Background rather than the caller's context, and the sites now say why. model/stop is the acknowledged stop path: it reserves the process, frees it, kills it, waits for exit and releases the port, and abandoning that because the caller hung up would leave a process marked stopping, a port not returned to the allocator and a row nothing reconciles. In stopBackendExact the Free is a courtesy before a kill that happens anyway. model/unload differs because Free IS the operation there. A route set with no prefix or no registrar is now a startup error rather than a silent no-op: a server that comes up healthy while every route the caller registered answers 404 is, through a tunnel, indistinguishable from a version skew. And the AllPaths spec no longer claims to catch a constant that was never added to the set, which it cannot; it asserts the whole set instead, which catches a verb dropped from it. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .../nodes/authenticated_routes_test.go | 20 ++ core/services/nodes/file_transfer_server.go | 16 +- .../nodes/install_progress_publisher.go | 25 +-- .../nodes/install_progress_publisher_test.go | 69 +++++-- core/services/worker/control_routes.go | 32 ++- core/services/worker/control_routes_test.go | 192 ++++++++++++++++++ core/services/worker/install.go | 64 +++--- core/services/worker/lifecycle.go | 16 +- core/services/worker/supervisor.go | 8 + core/services/workerctl/paths_test.go | 22 +- docs/content/features/distributed-mode.md | 6 +- 11 files changed, 397 insertions(+), 73 deletions(-) diff --git a/core/services/nodes/authenticated_routes_test.go b/core/services/nodes/authenticated_routes_test.go index db6c2a72b821..215e2bead38d 100644 --- a/core/services/nodes/authenticated_routes_test.go +++ b/core/services/nodes/authenticated_routes_test.go @@ -72,6 +72,26 @@ var _ = Describe("extra authenticated routes on the worker HTTP server", func() Expect(get("/healthz", "").StatusCode).To(Equal(http.StatusOK)) }) + It("refuses to start when a route set names no prefix", func() { + lis, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = lis.Close() }) + dir := GinkgoT().TempDir() + _, err = StartFileTransferServerWithRoutes(lis, dir, dir, dir, token, 0, nil, + &AuthenticatedRoutes{Register: func(*http.ServeMux) {}}) + Expect(err).To(MatchError(ContainSubstring("prefix"))) + }) + + It("refuses to start when a route set names no registrar", func() { + lis, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = lis.Close() }) + dir := GinkgoT().TempDir() + _, err = StartFileTransferServerWithRoutes(lis, dir, dir, dir, token, 0, nil, + &AuthenticatedRoutes{Prefix: "/v1/control/"}) + Expect(err).To(MatchError(ContainSubstring("registrar"))) + }) + It("mounts nothing when no route set is given", func() { lis, err := net.Listen("tcp", "127.0.0.1:0") Expect(err).NotTo(HaveOccurred()) diff --git a/core/services/nodes/file_transfer_server.go b/core/services/nodes/file_transfer_server.go index df8fecd159c4..c68c2f05bdf8 100644 --- a/core/services/nodes/file_transfer_server.go +++ b/core/services/nodes/file_transfer_server.go @@ -113,8 +113,18 @@ func startFileTransferServer(lis net.Listener, stagingDir, modelsDir, dataDir, t } func startFileTransferServerWithRoutes(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, capacity EphemeralCapacity, extra *AuthenticatedRoutes, logStore ...*model.BackendLogStore) (*http.Server, error) { - if err := validateAuthenticatedRoutes(extra); err != nil { - return nil, err + // Checked before anything is created. A route set that names no prefix or + // no registrar is a caller bug, and mounting nothing for it would be the + // worst possible answer: the server comes up healthy and every route the + // caller believes it registered answers 404, which through a tunnel is + // indistinguishable from a version skew. + if extra != nil { + if extra.Prefix == "" { + return nil, fmt.Errorf("extra routes were given no prefix to mount under") + } + if extra.Register == nil { + return nil, fmt.Errorf("extra routes under %q were given no registrar", extra.Prefix) + } } if err := os.MkdirAll(stagingDir, 0750); err != nil { return nil, fmt.Errorf("creating staging dir %s: %w", stagingDir, err) @@ -231,7 +241,7 @@ func startFileTransferServerWithRoutes(lis net.Listener, stagingDir, modelsDir, // Readiness: "can this worker actually accept work?" See WorkerReadiness. mux.HandleFunc("/readyz", probe(readiness.Check)) - if extra != nil && extra.Register != nil && extra.Prefix != "" { + if extra != nil { extraMux := http.NewServeMux() extra.Register(extraMux) mux.Handle(extra.Prefix, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/core/services/nodes/install_progress_publisher.go b/core/services/nodes/install_progress_publisher.go index 88a2d16eef2d..23bd67195faf 100644 --- a/core/services/nodes/install_progress_publisher.go +++ b/core/services/nodes/install_progress_publisher.go @@ -11,11 +11,10 @@ import ( // and hands them to its emit sink at most once per `interval`. Always emits the // final event on Flush so the UI sees the terminal percentage. // -// The sink is a function rather than a NATS subject because the same debounce -// now bounds two carriers: the NATS progress subject an agent node still -// publishes on, and a line written into the streaming HTTP response a worker -// serves over its tunnel. Keeping one debouncer keeps the ~4/s tick bound a -// single fact instead of one per carrier. +// The sink is a function rather than a NATS subject because the events now ride +// the streaming HTTP response the worker serves over its tunnel. Leaving the +// debounce here, rather than at the sink, is what keeps the ~4/s tick bound a +// property of install progress itself and not of whatever carries it. // // Behavior: leading-edge debounce. The first OnDownload after a quiet window // publishes immediately; subsequent ticks within `interval` only buffer the @@ -38,21 +37,17 @@ type DebouncedInstallProgressPublisher struct { timer *time.Timer } -// NewDebouncedInstallProgressPublisher constructs a publisher for one install -// operation that publishes to the per-op NATS progress subject. interval is the -// leading-edge debounce window (~250ms in production). -func NewDebouncedInstallProgressPublisher(client messaging.MessagingClient, nodeID, opID, backend string, interval time.Duration) *DebouncedInstallProgressPublisher { - subject := messaging.SubjectNodeBackendInstallProgress(nodeID, opID) - return NewDebouncedInstallProgressSink(func(ev messaging.BackendInstallProgressEvent) { - _ = client.Publish(subject, ev) - }, nodeID, opID, backend, interval) -} - // NewDebouncedInstallProgressSink constructs a publisher for one install // operation that hands each debounced event to emit. // // emit is called with p.mu released, so a sink that blocks on a slow link // cannot stall the gallery download loop that feeds it. +// +// There was a sibling constructor that published to the per-op NATS progress +// subject. It is gone rather than kept: a worker streams these events inside +// the install response now, so the NATS publisher had no caller left, and a +// constructor alive only for its own spec is a carrier a reader would believe +// still runs. func NewDebouncedInstallProgressSink(emit func(messaging.BackendInstallProgressEvent), nodeID, opID, backend string, interval time.Duration) *DebouncedInstallProgressPublisher { return &DebouncedInstallProgressPublisher{ emit: emit, diff --git a/core/services/nodes/install_progress_publisher_test.go b/core/services/nodes/install_progress_publisher_test.go index 04073cebeef0..a0c42c9fd479 100644 --- a/core/services/nodes/install_progress_publisher_test.go +++ b/core/services/nodes/install_progress_publisher_test.go @@ -1,6 +1,7 @@ package nodes import ( + "sync" "time" . "github.com/onsi/ginkgo/v2" @@ -9,10 +10,29 @@ import ( "github.com/mudler/LocalAI/core/services/messaging" ) +// collectingSink records every event the debouncer emits. Emits arrive from the +// trailing timer's own goroutine as well as from OnDownload, so it locks. +type collectingSink struct { + mu sync.Mutex + events []messaging.BackendInstallProgressEvent +} + +func (c *collectingSink) emit(ev messaging.BackendInstallProgressEvent) { + c.mu.Lock() + defer c.mu.Unlock() + c.events = append(c.events, ev) +} + +func (c *collectingSink) snapshot() []messaging.BackendInstallProgressEvent { + c.mu.Lock() + defer c.mu.Unlock() + return append([]messaging.BackendInstallProgressEvent(nil), c.events...) +} + var _ = Describe("DebouncedInstallProgressPublisher", func() { - It("publishes the first event immediately and debounces subsequent ones within the window", func() { - mc := newScriptedMessagingClient() - pub := NewDebouncedInstallProgressPublisher(mc, "n1", "op1", "vllm", 50*time.Millisecond) + It("emits the first event immediately and debounces subsequent ones within the window", func() { + sink := &collectingSink{} + pub := NewDebouncedInstallProgressSink(sink.emit, "n1", "op1", "vllm", 50*time.Millisecond) // Three rapid-fire ticks within the debounce window. pub.OnDownload("vllm.tar.zst", "100 MB", "1 GB", 10.0) @@ -20,29 +40,44 @@ var _ = Describe("DebouncedInstallProgressPublisher", func() { pub.OnDownload("vllm.tar.zst", "300 MB", "1 GB", 30.0) pub.Flush() - // First event publishes immediately; the others coalesce; Flush guarantees a final. - // So we expect at least 2 publishes and at most 4 (lead + final + any window-bounded). - Eventually(func() int { - return len(mc.publishCalls(messaging.SubjectNodeBackendInstallProgress("n1", "op1"))) - }, "1s").Should(BeNumerically(">=", 2)) - calls := mc.publishCalls(messaging.SubjectNodeBackendInstallProgress("n1", "op1")) - Expect(len(calls)).To(BeNumerically("<=", 4), - "three ticks within the debounce window should produce at most ~4 publishes") + // First event emits immediately; the others coalesce; Flush guarantees a final. + // So we expect at least 2 emits and at most 4 (lead + final + any window-bounded). + Eventually(func() int { return len(sink.snapshot()) }, "1s").Should(BeNumerically(">=", 2)) + Expect(len(sink.snapshot())).To(BeNumerically("<=", 4), + "three ticks within the debounce window should produce at most ~4 emits") }) - It("publishes the final event after Flush with the latest percentage", func() { - mc := newScriptedMessagingClient() - pub := NewDebouncedInstallProgressPublisher(mc, "n1", "op1", "vllm", 50*time.Millisecond) + It("emits the final event after Flush with the latest percentage", func() { + sink := &collectingSink{} + pub := NewDebouncedInstallProgressSink(sink.emit, "n1", "op1", "vllm", 50*time.Millisecond) pub.OnDownload("vllm.tar.zst", "1 GB", "1 GB", 100.0) pub.Flush() Eventually(func() float64 { - calls := mc.publishCalls(messaging.SubjectNodeBackendInstallProgress("n1", "op1")) - if len(calls) == 0 { + events := sink.snapshot() + if len(events) == 0 { return -1 } - return calls[len(calls)-1].Percentage + return events[len(events)-1].Percentage }, "1s").Should(Equal(100.0)) }) + + It("stamps every event with the identity the frontend correlates on", func() { + // The op id and node id are what a frontend matches a progress line to + // an operation with; the subject used to carry them and now nothing + // else does, so the event body has to. + sink := &collectingSink{} + pub := NewDebouncedInstallProgressSink(sink.emit, "node-7", "op-42", "vllm", time.Millisecond) + pub.OnDownload("vllm.tar.zst", "1 GB", "1 GB", 100.0) + pub.Flush() + + Eventually(func() int { return len(sink.snapshot()) }, "1s").Should(BeNumerically(">=", 1)) + ev := sink.snapshot()[0] + Expect(ev.NodeID).To(Equal("node-7")) + Expect(ev.OpID).To(Equal("op-42")) + Expect(ev.Backend).To(Equal("vllm")) + Expect(ev.FileName).To(Equal("vllm.tar.zst")) + Expect(ev.Phase).To(Equal(messaging.PhaseDownloading)) + }) }) diff --git a/core/services/worker/control_routes.go b/core/services/worker/control_routes.go index 57e3b6fcef63..31ddbfb7e332 100644 --- a/core/services/worker/control_routes.go +++ b/core/services/worker/control_routes.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "sync" + "unicode/utf8" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/workerctl" @@ -117,6 +118,14 @@ func (s *backendSupervisor) RegisterControlRoutes(mux *http.ServeMux) { return messaging.ModelsRunningReply{Models: s.runningModels()}, nil }) + // model.stop deliberately does NOT take the caller's context, and the + // asymmetry with model.unload below is the point. This is the ACKNOWLEDGED + // stop path: it reserves the process, frees it, kills it, waits for it to + // exit and releases its port. Abandoning that half way because the caller + // stopped listening would leave a process the worker has marked stopping, + // a port not returned to the allocator, and a controller row nothing ever + // reconciles. The stop has to finish whether or not anyone reads the + // answer; its own bounds are internal and already in place. post(workerctl.PathModelStop, func(_ context.Context, body []byte) (any, error) { var req messaging.ModelStopRequest if err := json.Unmarshal(body, &req); err != nil { @@ -137,6 +146,10 @@ func (s *backendSupervisor) RegisterControlRoutes(mux *http.ServeMux) { return s.deleteBackend(req), nil }) + // model.unload is the one verb that DOES take the caller's context. Free is + // the whole operation here rather than a courtesy before a kill: nothing is + // terminated, no port is released, and abandoning it leaves the worker + // exactly as it was. So the caller's budget is the operation's budget. post(workerctl.PathModelUnload, func(ctx context.Context, body []byte) (any, error) { var req messaging.ModelUnloadRequest if err := json.Unmarshal(body, &req); err != nil { @@ -153,6 +166,9 @@ func (s *backendSupervisor) RegisterControlRoutes(mux *http.ServeMux) { return s.deleteModel(req), nil }) + // backend.stop drops the caller's context for the same reason model.stop + // does, and more plainly: it is fire-and-forget, so there is no answer for + // the caller to still be waiting on. post(workerctl.PathBackendStop, func(_ context.Context, body []byte) (any, error) { req, stopAll, err := decodeBackendStopRequest(body) if err != nil { @@ -206,24 +222,24 @@ func readControlBody(w http.ResponseWriter, r *http.Request) ([]byte, bool) { } // truncate bounds a caller-controlled string that is about to be echoed. -// It cuts on a rune boundary: a byte-wise cut can split a multi-byte rune, and -// the half-rune then travels as a replacement character through every log and -// UI that reads it. +// +// It cuts on a rune boundary. A byte-wise cut can split a multi-byte rune, and +// the half rune then travels as a replacement character through every log and +// UI that reads it; phase 2 shipped exactly that defect on a refusal reason and +// pinned the rule afterwards. utf8.RuneStart is the same predicate the cluster +// package uses for it, so the two are one rule rather than two hand-rolled +// copies that can drift. func truncate(s string, max int) string { if len(s) <= max { return s } cut := max - for cut > 0 && !isRuneStart(s[cut]) { + for cut > 0 && !utf8.RuneStart(s[cut]) { cut-- } return s[:cut] + "…" } -// isRuneStart reports whether b begins a UTF-8 rune (i.e. is not a -// continuation byte). -func isRuneStart(b byte) bool { return b&0xC0 != 0x80 } - // ndjsonStream writes the Envelope lines of one streaming control response. // // The mutex is not optional. A progress line can be emitted from the debounce diff --git a/core/services/worker/control_routes_test.go b/core/services/worker/control_routes_test.go index fb8576e71836..f57793324bc8 100644 --- a/core/services/worker/control_routes_test.go +++ b/core/services/worker/control_routes_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "io" + "net" "net/http" "net/http/httptest" "os" @@ -13,6 +14,7 @@ import ( "strings" "sync" "syscall" + "unicode/utf8" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -20,6 +22,7 @@ import ( "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/workerctl" + "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" ) @@ -136,6 +139,39 @@ var _ = Describe("worker control routes", func() { Expect(string(body)).To(ContainSubstring("control")) }) + It("cuts the echoed path on a rune boundary, so no half rune reaches a log", func() { + // The multi-byte rune is placed so that it STRADDLES the cut: a + // byte-wise cut lands inside it and the body carries a replacement + // character. Phase 2 shipped this exact defect on a refusal reason. + // "a" is one byte and "€" is three. The echoed string is the WHOLE + // path, prefix included, so the padding is sized against the prefix to + // put the rune across bytes max-1, max and max+1. + lead := maxEchoedPathBytes - len(workerctl.Prefix) - 1 + straddle := strings.Repeat("a", lead) + "€" + strings.Repeat("b", 64) + resp := post(workerctl.Prefix+straddle, struct{}{}) + Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) + body, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + + Expect(utf8.ValidString(string(body))).To(BeTrue(), "the 404 body must stay valid UTF-8") + Expect(string(body)).NotTo(ContainSubstring("\uFFFD"), "the cut split a rune") + // And the whole rune is dropped rather than kept past the bound. + Expect(string(body)).NotTo(ContainSubstring("€")) + }) + + It("keeps a rune that ends exactly on the bound, so the cut is not off by one", func() { + // Here the rune's last byte is at maxEchoedPathBytes-1, so it fits + // entirely and must survive. A cut that walked back unconditionally + // would drop it and this spec would catch that. + lead := maxEchoedPathBytes - len(workerctl.Prefix) - 3 + exact := strings.Repeat("a", lead) + "€" + strings.Repeat("b", 64) + resp := post(workerctl.Prefix+exact, struct{}{}) + body, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(utf8.ValidString(string(body))).To(BeTrue()) + Expect(string(body)).To(ContainSubstring("€")) + }) + It("bounds the unknown path it echoes back, so a long URL cannot be reflected wholesale", func() { long := strings.Repeat("a", 4096) resp := post(workerctl.Prefix+long, struct{}{}) @@ -196,6 +232,8 @@ var _ = Describe("worker control routes", func() { }) It("answers model.unload with the worker's own reply rather than a transport error", func() { + // Nothing is loaded, so there is nothing to free and the true answer is + // success. resp := post(workerctl.PathModelUnload, messaging.ModelUnloadRequest{ModelName: "m"}) Expect(resp.StatusCode).To(Equal(http.StatusOK)) var reply messaging.ModelUnloadReply @@ -203,6 +241,26 @@ var _ = Describe("worker control routes", func() { Expect(reply.Success).To(BeTrue()) }) + It("reports a failed Free as a failed unload, not as success", func() { + // A worker that answers "done" about work it did not do is the fourth + // kind of answer in this programme's taxonomy, and it is the one acted + // on: the frontend's only caller of unload is EvictLRU, so a false yes + // tells the scheduler VRAM was released and lets it place the next + // model on a node still holding the old one. + dead, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + addr := dead.Addr().String() + Expect(dead.Close()).To(Succeed()) + + resp := post(workerctl.PathModelUnload, messaging.ModelUnloadRequest{ModelName: "m", Address: addr}) + // Still a 200: the WORKER answered. Only the verdict inside is negative. + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply messaging.ModelUnloadReply + Expect(json.NewDecoder(resp.Body).Decode(&reply)).To(Succeed()) + Expect(reply.Success).To(BeFalse()) + Expect(reply.Error).To(ContainSubstring(addr)) + }) + It("answers model.stop for an unknown process with the worker's verdict, not a 5xx", func() { // The whole invariant of the phase: "that backend is not there" is the // WORKER answering, and it must never arrive as the status code a @@ -522,3 +580,137 @@ var _ = Describe("the NDJSON control stream", func() { } }) }) + +// These specs drive the REAL installBackend and upgradeBackend, with no +// installFn override, so the progress wiring inside them is exercised end to +// end through the carrier. Every other install spec scripts installFn, which +// leaves the one line that decides whether a caller sees any progress at all +// pinned by nothing. +var _ = Describe("the real install progress wiring", func() { + var ( + sup *backendSupervisor + srv *httptest.Server + ) + + BeforeEach(func() { + dir := GinkgoT().TempDir() + st, err := system.GetSystemState( + system.WithModelPath(dir), + system.WithBackendPath(filepath.Join(dir, "backends")), + system.WithBackendSystemPath(filepath.Join(dir, "backends-system")), + ) + Expect(err).NotTo(HaveOccurred()) + sup = &backendSupervisor{ + cfg: &Config{ModelsPath: dir, BackendsPath: filepath.Join(dir, "backends")}, + systemState: st, + ml: model.NewModelLoader(st), + nodeID: "node-under-test", + sigCh: make(chan os.Signal, 1), + processes: map[string]*backendProcess{}, + // Empty on purpose. The install cannot succeed, which is the point: + // what is under test is that the caller is told what the worker is + // doing and then told it failed, not that a download works. + galleries: nil, + } + mux := http.NewServeMux() + sup.RegisterControlRoutes(mux) + srv = httptest.NewServer(mux) + DeferCleanup(srv.Close) + }) + + postJSON := func(path string, body any) *http.Response { + GinkgoHelper() + buf, err := json.Marshal(body) + Expect(err).NotTo(HaveOccurred()) + resp, err := srv.Client().Post(srv.URL+path, "application/json", bytes.NewReader(buf)) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + return resp + } + + // envelopesOf decodes a whole NDJSON body into its envelopes. + envelopesOf := func(body io.Reader) []workerctl.Envelope { + GinkgoHelper() + var out []workerctl.Envelope + dec := json.NewDecoder(body) + for { + var env workerctl.Envelope + err := dec.Decode(&env) + if errors.Is(err, io.EOF) { + break + } + Expect(err).NotTo(HaveOccurred()) + out = append(out, env) + } + return out + } + + It("streams a resolving line before the failure, from the real installBackend", func() { + resp := postJSON(workerctl.PathBackendInstall, + messaging.BackendInstallRequest{Backend: "no-such-backend", OpID: "op-real"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + envs := envelopesOf(resp.Body) + Expect(envs).NotTo(BeEmpty()) + Expect(envs[0].Progress).NotTo(BeNil(), "the caller must see a line before the gallery work starts") + + var ev messaging.BackendInstallProgressEvent + Expect(json.Unmarshal(envs[0].Progress, &ev)).To(Succeed()) + Expect(ev.Phase).To(Equal(messaging.PhaseResolving)) + Expect(ev.OpID).To(Equal("op-real")) + Expect(ev.NodeID).To(Equal("node-under-test")) + Expect(ev.Backend).To(Equal("no-such-backend")) + + last := envs[len(envs)-1] + Expect(last.Reply).NotTo(BeNil()) + var reply messaging.BackendInstallReply + Expect(json.Unmarshal(last.Reply, &reply)).To(Succeed()) + Expect(reply.Success).To(BeFalse()) + }) + + It("streams nothing but the reply when the caller asked for no progress", func() { + // An empty OpID is a reconciler-driven retry. It must not be given a + // progress stream, and it must still get its terminal line. + resp := postJSON(workerctl.PathBackendInstall, + messaging.BackendInstallRequest{Backend: "no-such-backend"}) + Expect(envelopeKindsOf(resp.Body)).To(Equal([]string{"reply"})) + }) + + It("streams a resolving line from the real upgradeBackend too", func() { + resp := postJSON(workerctl.PathBackendUpgrade, + messaging.BackendUpgradeRequest{Backend: "no-such-backend", OpID: "op-real-upgrade"}) + envs := envelopesOf(resp.Body) + Expect(envs).NotTo(BeEmpty()) + Expect(envs[0].Progress).NotTo(BeNil()) + + var ev messaging.BackendInstallProgressEvent + Expect(json.Unmarshal(envs[0].Progress, &ev)).To(Succeed()) + Expect(ev.Phase).To(Equal(messaging.PhaseResolving)) + Expect(ev.OpID).To(Equal("op-real-upgrade")) + + var reply messaging.BackendUpgradeReply + Expect(json.Unmarshal(envs[len(envs)-1].Reply, &reply)).To(Succeed()) + Expect(reply.Success).To(BeFalse()) + }) + + It("hands the gallery no download callback when either half of the guard is missing", func() { + // A nil callback is what puts the gallery on its silent path, and both + // halves have to be checked: a caller with an OpID but no sink is the + // shape that would nil-panic on the resolving emit. + collected := func(messaging.BackendInstallProgressEvent) {} + + cb, flush := sup.startProgress("", "b", collected) + Expect(cb).To(BeNil()) + Expect(flush).NotTo(BeNil()) + flush() + + cb, flush = sup.startProgress("op", "b", nil) + Expect(cb).To(BeNil()) + Expect(flush).NotTo(BeNil()) + flush() + + cb, flush = sup.startProgress("op", "b", collected) + Expect(cb).NotTo(BeNil()) + flush() + }) +}) diff --git a/core/services/worker/install.go b/core/services/worker/install.go index 2390ca28d25c..1fc3238f4ae6 100644 --- a/core/services/worker/install.go +++ b/core/services/worker/install.go @@ -134,19 +134,11 @@ func (s *backendSupervisor) installBackend(ctx context.Context, req messaging.Ba galleries = reqGalleries } - // When the caller tagged this install with an OpID and is listening for - // progress, stream the gallery download ticks back on the response the - // caller is already reading. Callers that omit OpID stay on the silent path. - // The publisher releases its mutex before every emit so a slow link never - // stalls the download loop, and the deferred Flush guarantees a - // terminal-percentage event reaches the caller even when the install errors - // out. - var downloadCb func(file, current, total string, percentage float64) - if req.OpID != "" && onProgress != nil { - publisher := nodes.NewDebouncedInstallProgressSink(onProgress, s.nodeID, req.OpID, req.Backend, installProgressDebounce) - downloadCb = publisher.OnDownload - defer publisher.Flush() - } + // Gallery download ticks go back on the response the caller is already + // reading. See startProgress for the guard and for why the flush is + // deferred. + downloadCb, flushProgress := s.startProgress(req.OpID, req.Backend, onProgress) + defer flushProgress() // On upgrade, run the gallery install path even if the binary already // exists on disk: findBackend would otherwise short-circuit and we'd @@ -235,17 +227,10 @@ func (s *backendSupervisor) upgradeBackend(ctx context.Context, req messaging.Ba galleries = reqGalleries } - // When the caller tagged this upgrade with an OpID, stream gallery download - // progress back on the same sink install uses — an upgrade IS a - // force-reinstall. Callers that omit OpID stay on the silent path. The - // deferred Flush guarantees a terminal-percentage event even if the upgrade - // errors out, so the caller's per-node bar never hangs mid-download. - var downloadCb func(file, current, total string, percentage float64) - if req.OpID != "" && onProgress != nil { - publisher := nodes.NewDebouncedInstallProgressSink(onProgress, s.nodeID, req.OpID, req.Backend, installProgressDebounce) - downloadCb = publisher.OnDownload - defer publisher.Flush() - } + // The same sink install uses: an upgrade IS a force-reinstall, so its + // progress is install progress. + downloadCb, flushProgress := s.startProgress(req.OpID, req.Backend, onProgress) + defer flushProgress() if req.URI != "" { xlog.Info("Upgrading backend from external URI", "backend", req.Backend, "uri", req.URI) @@ -309,3 +294,34 @@ func (s *backendSupervisor) lockBackend(name string) func() { m.Lock() return m.Unlock } + +// startProgress wires one install or upgrade to its caller's progress sink. +// +// It returns the download callback the gallery takes, and a flush the caller +// must defer: the debouncer buffers within its window, and the deferred flush +// is what gets the terminal percentage to the caller even when the install +// errors out. +// +// Both halves of the guard matter. An empty OpID means the caller is a +// reconciler-driven retry that asked for no progress, and a nil sink means the +// caller is not reading a stream at all; either way the gallery gets a nil +// callback and takes its silent path. +// +// The resolving event is emitted here, before any gallery work, so the caller +// sees a line as soon as the worker has accepted the job rather than only once +// bytes start moving. A cold install spends minutes resolving a manifest, and +// during that time a progress stream with nothing on it is indistinguishable +// from one that is broken. +func (s *backendSupervisor) startProgress(opID, backend string, onProgress func(messaging.BackendInstallProgressEvent)) (func(file, current, total string, percentage float64), func()) { + if opID == "" || onProgress == nil { + return nil, func() {} + } + publisher := nodes.NewDebouncedInstallProgressSink(onProgress, s.nodeID, opID, backend, installProgressDebounce) + onProgress(messaging.BackendInstallProgressEvent{ + OpID: opID, + NodeID: s.nodeID, + Backend: backend, + Phase: messaging.PhaseResolving, + }) + return publisher.OnDownload, publisher.Flush +} diff --git a/core/services/worker/lifecycle.go b/core/services/worker/lifecycle.go index 89da5419ad59..c3504429fff9 100644 --- a/core/services/worker/lifecycle.go +++ b/core/services/worker/lifecycle.go @@ -203,14 +203,28 @@ func (s *backendSupervisor) unloadModel(ctx context.Context, req messaging.Model } if targetAddr == "" { + // Nothing is loaded here, so there is nothing to free. That is the + // worker's own answer and it is a true one, not a claim about work it + // performed. return messaging.ModelUnloadReply{Success: true} } + // Bounded gRPC Free(). A model.unload request must not occupy the handler + // forever when a backend is wedged. The bound is derived from the caller's + // own budget, so a caller that allowed less than workerBackendFreeTimeout + // is not made to wait longer than it asked for. client := grpc.NewClientWithToken(targetAddr, false, nil, false, s.cfg.RegistrationToken) freeCtx, cancel := context.WithTimeout(ctx, workerBackendFreeTimeout) err := client.Free(freeCtx) cancel() if err != nil { + // Reported, not swallowed. This used to answer Success:true whatever + // Free did, which is the worker saying "done" about something it did + // not do: the frontend's only caller is EvictLRU, so a false yes told + // the scheduler VRAM had been released and let it place the next model + // on a node still holding the old one. A failure here is the worker's + // own verdict about one Free call and nothing more; it is not a + // statement that the node or the model is gone. xlog.Warn("Free() failed during model.unload", "error", err, "addr", targetAddr) return messaging.ModelUnloadReply{ Success: false, @@ -235,7 +249,7 @@ func (s *backendSupervisor) deleteModel(req messaging.ModelDeleteRequest) messag // signalNodeStop serves node.stop: it triggers the normal shutdown path via // sigCh so deferred cleanup runs, rather than exiting the process here. func (s *backendSupervisor) signalNodeStop() { - xlog.Info("Serving node.stop — signaling shutdown") + xlog.Info("Serving node.stop, signaling shutdown") select { case s.sigCh <- syscall.SIGTERM: default: diff --git a/core/services/worker/supervisor.go b/core/services/worker/supervisor.go index 9a860c4e8cbc..b3f180c609f8 100644 --- a/core/services/worker/supervisor.go +++ b/core/services/worker/supervisor.go @@ -870,6 +870,12 @@ func (s *backendSupervisor) stopBackendExact(key string, force bool) error { } if !force { + // Background and not a caller's context on purpose. This Free is a + // courtesy before the process is killed anyway, and the stop that + // follows must complete for the port to be released, so binding it to + // a caller that may have gone would buy nothing and could abandon a + // half-finished stop. model.unload, where Free IS the operation, takes + // the caller's budget instead; see RegisterControlRoutes. client := grpc.NewClientWithToken(bp.addr, false, nil, false, s.cfg.RegistrationToken) freeCtx, cancel := context.WithTimeout(context.Background(), workerBackendFreeTimeout) xlog.Debug("Calling bounded Free() before stopping backend", "backend", key, "timeout", workerBackendFreeTimeout) @@ -916,6 +922,8 @@ func (s *backendSupervisor) stopModelExact(req messaging.ModelStopRequest) messa s.mu.Unlock() if !req.Force { + // Background, for the reason given in stopBackendExact: this is the + // acknowledged stop and it has to run to completion. client := grpc.NewClientWithToken(bp.addr, false, nil, false, s.cfg.RegistrationToken) freeCtx, cancel := context.WithTimeout(context.Background(), workerBackendFreeTimeout) freeErr := client.Free(freeCtx) diff --git a/core/services/workerctl/paths_test.go b/core/services/workerctl/paths_test.go index c5685a99dde2..8ffabb2e3cd9 100644 --- a/core/services/workerctl/paths_test.go +++ b/core/services/workerctl/paths_test.go @@ -40,10 +40,24 @@ var _ = Describe("control plane paths on the wire", func() { } }) - It("enumerates every verb, so a new one cannot be added without the prefix check seeing it", func() { - Expect(workerctl.AllPaths()).To(HaveLen(10)) - Expect(workerctl.AllPaths()).To(ContainElement(workerctl.PathBackendInstall)) - Expect(workerctl.AllPaths()).To(ContainElement(workerctl.PathNodeStop)) + It("lists every verb this package names, so none can be dropped from the set", func() { + // The claim is bounded on purpose. Go constants are not enumerable, so + // nothing here can see a NEW constant that was never added to AllPaths; + // what this catches is an EXISTING verb going missing from it, which + // matters because the prefix check above and the worker's mounting spec + // both iterate AllPaths and would silently stop covering it. + Expect(workerctl.AllPaths()).To(ConsistOf( + workerctl.PathBackendInstall, + workerctl.PathBackendUpgrade, + workerctl.PathBackendList, + workerctl.PathBackendStop, + workerctl.PathBackendDelete, + workerctl.PathModelStop, + workerctl.PathModelUnload, + workerctl.PathModelDelete, + workerctl.PathModelsRunning, + workerctl.PathNodeStop, + )) }) It("gives each verb a distinct path", func() { diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index c0ce0b6d3152..92f699495207 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -188,7 +188,11 @@ Two of the routes stream. `POST /v1/control/backend/install` and `/v1/control/backend/upgrade` answer with `application/x-ndjson`: zero or more `{"progress":{...}}` lines carrying the same download-progress payload the per-op NATS subject carried, followed by exactly one `{"reply":{...}}` line, -which is always the last line on the body. An install that FAILS is still a +which is always the last line on the body. When the request carries an `op_id`, +the first progress line has phase `resolving` and is written before any gallery +work begins, so a cold install that spends minutes on a manifest is +distinguishable from a stream that is broken. Nothing publishes install progress +over NATS any more. An install that FAILS is still a `200` with a reply whose `success` is `false`. That is deliberate, and it is the same distinction the refusal table above draws: a non-2xx means the frontend could not get the request to the worker, which nothing may act on, while the From 96e259f90859c6ca9539f456e496184d9c22d693 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 19:43:53 +0000 Subject: [PATCH 072/153] feat(distributed): call the worker's control routes instead of the bus The ten backend and model lifecycle verbs stop being NATS requests and become HTTP calls on the worker's own control routes, reached through that worker's tunnel on the `http` stream tag that already carries file staging. Nine subject builders and the per-op install-progress subject are deleted with their entries in the worker's NATS permissions; the request and reply DTOs are untouched, so a body on the wire is byte for byte what the subject carried. This closes the merge gate Task 3 left open, which was worse than lost commands. Once the worker stopped subscribing, PingNode was still asking nodes..backend.list and nodes..models.running, so EVERY healthy worker answered no-responders, nodeAnswersOnBus read it as absence and pickReachableNode demoted it on the scheduling path. PingNode is a control RPC now, and no control RPC can produce ErrNoResponders, which is the only error that exclusion acts on. Two specs drive pickReachableNode against a real adapter and a worker answering over its control plane, which is the only arrangement that can see the difference: the router's own double never touches a transport and stayed green for the whole window the defect was live. How a control RPC FAILS is the whole of this change, so it is decided in ONE function reading ONE table. A worker's answer passes through unwrapped, so cluster.IsWorkerAnswer still sees it and a reap guard may act on it; everything else is wrapped in ErrWorkerUnroutable so nothing can. There is no third branch, because a third branch is how the eight collapses on this branch happened: each was a site that decided for itself which errors were evidence. A 404 under the prefix is its own sentinel, because it is the worker stating a deployment fact about ITSELF rather than a verdict about a backend, and only the legacy upgrade fallback may act on it. The caller's budget is checked FIRST. A timeout is not a verdict: a refusal arriving in the instant a deadline expires would otherwise be reported as the worker's non-transient answer, which reaps a row, and nothing orders the two timers. A 5xx and an undecodable body are transport failures, not answers. An empty ModelsRunningReply means "this worker is running nothing", which the reconciler acts on, so it must never be manufactured from a body that would not parse. A stream that ends before its reply line is the same rule one layer up: a tunnel dying mid-install is not the worker saying the install failed. backend.stop is split by node type rather than moved. Agent workers hold no tunnel, so they have no control plane to serve, and they still subscribe to nodes..backend.stop to drop cached MCP sessions; that subject and its agent permission both survive. It is the honest intermediate state until agent workers hold tunnels too. A failed control RPC no longer demotes a node anywhere. ErrNoResponders meant "not on the bus"; a control failure means "this frontend could not route to it", which is equally what a healthy worker re-homing its tunnel between replicas produces. Absence is a fact read from the database, and the scheduler starts reading it in a later task. The rolling-update fallback re-fires a DESTRUCTIVE force-reinstall, so it runs only on the worker's own 404. Its negative direction was pinned at the admin call site and unpinned at the reconciler's, where widening the condition to any error left all 676 specs green: a background drain nobody is watching would then force-reinstall every queued backend the moment a replica lost its tunnels. Three specs cover it, arranged so the force install IS reachable in the negative case and a fallback that fired would show as a call and a drained row. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 7 + .../messaging/backend_install_progress.go | 16 +- .../backend_install_progress_test.go | 18 - core/services/messaging/subjects.go | 151 ++---- .../messaging/subjects_upgrade_test.go | 16 +- core/services/nodes/control_client.go | 307 +++++++++++ core/services/nodes/control_client_test.go | 317 +++++++++++ .../nodes/control_worker_fake_test.go | 273 ++++++++++ core/services/nodes/interfaces.go | 7 + .../nodes/managers_agent_node_test.go | 51 +- core/services/nodes/managers_distributed.go | 46 +- .../nodes/managers_distributed_test.go | 429 ++++----------- core/services/nodes/reconciler.go | 35 +- core/services/nodes/reconciler_test.go | 91 ++++ core/services/nodes/router_liveness.go | 25 +- core/services/nodes/unloader.go | 494 +++++++++--------- core/services/nodes/unloader_ping_test.go | 127 ++++- .../nodes/unloader_stale_rows_test.go | 39 +- core/services/nodes/unloader_test.go | 469 ++++++++++------- core/services/nodes/unloader_upgrade_test.go | 66 ++- .../worker/control_client_roundtrip_test.go | 151 ++++++ docs/content/features/distributed-mode.md | 46 +- pkg/natsauth/mint_test.go | 8 +- pkg/natsauth/permissions.go | 12 +- pkg/natsauth/permissions_coverage_test.go | 36 +- tests/e2e/distributed/control_workers_test.go | 112 ++++ .../distributed/distributed_full_flow_test.go | 57 +- tests/e2e/distributed/managers_test.go | 94 ++-- tests/e2e/distributed/node_lifecycle_test.go | 115 ++-- tests/e2e/distributed/router_tracking_test.go | 25 +- 30 files changed, 2387 insertions(+), 1253 deletions(-) create mode 100644 core/services/nodes/control_client.go create mode 100644 core/services/nodes/control_client_test.go create mode 100644 core/services/nodes/control_worker_fake_test.go create mode 100644 core/services/worker/control_client_roundtrip_test.go create mode 100644 tests/e2e/distributed/control_workers_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index f4666b5bf3cf..2586e403ee57 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -422,10 +422,17 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade }, cfg.Distributed.RegistrationToken, workerHTTPDialer) xlog.Info("File stager initialized (HTTP direct transfer)") } + // The frontend's control plane client. It reaches every worker over that + // worker's own tunnel, on the same `http` stream tag the file stager above + // uses, so a control RPC to a worker another replica holds is relayed the + // way an inference request is. + controlClient := nodes.NewControlClient(workerHTTPDialer, cfg.Distributed.RegistrationToken) + // Create RemoteUnloaderAdapter — needed by SmartRouter and startup.go remoteUnloader := nodes.NewRemoteUnloaderAdapter( registry, natsClient, + controlClient, cfg.Distributed.BackendInstallTimeoutOrDefault(), cfg.Distributed.BackendUpgradeTimeoutOrDefault(), ) diff --git a/core/services/messaging/backend_install_progress.go b/core/services/messaging/backend_install_progress.go index 268ef86b909b..52065bf64c7f 100644 --- a/core/services/messaging/backend_install_progress.go +++ b/core/services/messaging/backend_install_progress.go @@ -11,10 +11,11 @@ const ( PhaseStarting = "starting" // worker is spawning the gRPC backend process ) -// BackendInstallProgressEvent is the wire payload published by a worker to -// nodes..backend.install..progress while a long-running install -// is in flight. Transient: dropped events are acceptable, the master relies -// on BackendInstallReply for ground truth on success/failure. +// BackendInstallProgressEvent is the wire payload a worker writes as a progress +// line of its backend.install and backend.upgrade responses while a +// long-running install is in flight. Transient: a line the frontend cannot read +// is acceptable, and BackendInstallReply is the ground truth on +// success/failure. // // Phase holds one of the Phase* constants above. type BackendInstallProgressEvent struct { @@ -27,10 +28,3 @@ type BackendInstallProgressEvent struct { Percentage float64 `json:"percentage"` Phase string `json:"phase,omitempty"` } - -// SubjectNodeBackendInstallProgress returns the NATS subject for transient -// progress events emitted by a worker during a single backend.install run. -// Per-op so multiple concurrent installs on the same node never alias. -func SubjectNodeBackendInstallProgress(nodeID, opID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".backend.install." + sanitizeSubjectToken(opID) + ".progress" -} diff --git a/core/services/messaging/backend_install_progress_test.go b/core/services/messaging/backend_install_progress_test.go index ec45f4619f69..e57c5505f788 100644 --- a/core/services/messaging/backend_install_progress_test.go +++ b/core/services/messaging/backend_install_progress_test.go @@ -2,7 +2,6 @@ package messaging_test import ( "encoding/json" - "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,23 +25,6 @@ var _ = Describe("Phase constants", func() { }) var _ = Describe("BackendInstallProgress", func() { - Context("SubjectNodeBackendInstallProgress", func() { - It("composes the per-op progress subject", func() { - Expect(messaging.SubjectNodeBackendInstallProgress("node-abc", "op-123")). - To(Equal("nodes.node-abc.backend.install.op-123.progress")) - }) - - It("sanitizes NATS-reserved characters in node and op tokens", func() { - // '.' is the NATS hierarchy delimiter, '*' and '>' are wildcards, - // and whitespace must be stripped - sanitizeSubjectToken replaces - // all of them with '-'. The resulting subject must still parse as - // exactly six hierarchy segments: nodes//backend/install//progress. - subj := messaging.SubjectNodeBackendInstallProgress("a.b c", "x.y z") - Expect(subj).ToNot(ContainSubstring(" ")) - Expect(strings.Count(subj, ".")).To(Equal(5)) - }) - }) - Context("BackendInstallProgressEvent", func() { It("JSON round-trips with all known fields", func() { ev := messaging.BackendInstallProgressEvent{ diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index 43366bced1c6..bcfc9009d178 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -141,26 +141,29 @@ func SubjectResponseCancel(responseID string) string { return subjectResponseCancelPrefix + sanitizeSubjectToken(responseID) + ".cancel" } -// Node Backend Lifecycle (Pub/Sub — targeted to specific nodes) +// Node Backend Lifecycle // -// These subjects control the backend *process* lifecycle on a serve-backend node, -// mirroring how the local ModelLoader uses startProcess() / deleteProcess(). +// The frontend's control plane no longer travels on NATS. The ten verbs that +// drove a worker's backend and model lifecycle are HTTP routes under +// workerctl.Prefix, served on the worker's own loopback server and reached +// through its tunnel, so a subject builder for any of them would be a subject +// nothing publishes and nothing subscribes to. // -// Model loading (LoadModel gRPC) is done via direct gRPC calls to the node's -// address — no NATS needed for that, same as local mode. +// ONE survives: backend.stop, and only for AGENT workers. They hold no tunnel, +// so they have no control plane to serve, and they subscribe to it to drop the +// MCP sessions cached for a backend that is going away. See +// nodes.RemoteUnloaderAdapter.stopBackend for the split, and +// core/cli/agent_worker.go for the subscriber. +// +// The request and reply types below are UNCHANGED and still live here: they are +// the wire format of the control routes, byte for byte what the subjects +// carried, so a worker and a frontend from different releases still understand +// each other. const ( subjectNodePrefix = "nodes." ) -// SubjectNodeBackendInstall tells a worker node to install a backend and start its gRPC process. -// Uses NATS request-reply: the SmartRouter sends the request, the worker installs -// the backend from gallery (if not already installed), starts the gRPC process, -// and replies when ready. -func SubjectNodeBackendInstall(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".backend.install" -} - -// BackendInstallRequest is the payload for a backend.install NATS request. +// BackendInstallRequest is the payload for a backend.install control request. type BackendInstallRequest struct { Backend string `json:"backend"` ModelID string `json:"model_id,omitempty"` @@ -178,20 +181,20 @@ type BackendInstallRequest struct { ReplicaIndex int32 `json:"replica_index,omitempty"` // Force is retained on the wire only for backward compatibility with // pre-2026-05-08 masters that did not know about backend.upgrade. New - // callers MUST send to SubjectNodeBackendUpgrade instead. Workers continue + // callers MUST use workerctl.PathBackendUpgrade instead. Workers continue // to honor Force=true here so a rolling update with new master + old // worker still works (the master's install fallback path also uses this - // when backend.upgrade returns nats.ErrNoResponders). + // when the worker answers that it does not serve the upgrade verb). Force bool `json:"force,omitempty"` - // OpID identifies the admin-side operation. When non-empty the worker - // publishes BackendInstallProgressEvent values to - // SubjectNodeBackendInstallProgress(nodeID, OpID) while the install is - // running, debounced to roughly 250ms. Empty means the caller is a - // reconciler-driven retry that does not need progress streamed. + // OpID identifies the admin-side operation. It travels so the worker can + // name the operation on the BackendInstallProgressEvent values it writes + // into the install response ahead of the reply, debounced to roughly 250ms. + // Empty means the caller is a reconciler-driven retry that does not need + // progress streamed. OpID string `json:"op_id,omitempty"` } -// BackendInstallReply is the response from a backend.install NATS request. +// BackendInstallReply is the response from a backend.install control request. type BackendInstallReply struct { Success bool `json:"success"` // WorkerLocalAddress is where the backend process listens ON THE WORKER, @@ -207,17 +210,7 @@ type BackendInstallReply struct { Error string `json:"error,omitempty"` } -// SubjectNodeBackendUpgrade tells a worker node to force-reinstall a backend -// from the gallery, stop every running process for that backend, and restart. -// Uses NATS request-reply with a long deadline (gallery image pulls can take -// many minutes on slow links). Routine model loads use SubjectNodeBackendInstall -// instead — this subject exists so the slow path doesn't head-of-line-block -// the fast one through a shared subscription goroutine. -func SubjectNodeBackendUpgrade(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".backend.upgrade" -} - -// BackendUpgradeRequest is the payload for a backend.upgrade NATS request. +// BackendUpgradeRequest is the payload for a backend.upgrade control request. // It is intentionally a strict subset of BackendInstallRequest — there is no // Force field because the upgrade subject IS the force semantics; no ModelID // because upgrade is backend-scoped (it stops every replica using the binary @@ -232,13 +225,9 @@ type BackendUpgradeRequest struct { // but the field lets future per-replica metadata (e.g. progress reporting // scoped to a slot) ride the same wire without a v3 type. ReplicaIndex int32 `json:"replica_index,omitempty"` - // OpID identifies the admin-side operation. When non-empty the worker - // publishes BackendInstallProgressEvent values to - // SubjectNodeBackendInstallProgress(nodeID, OpID) while the force-reinstall - // runs, so the master can stream per-node progress for upgrades exactly as - // it already does for installs (an upgrade IS a force-reinstall, so the - // install-progress subject is reused rather than minting a new one — no new - // NATS permission or rolling-update compat surface). Empty on legacy callers. + // OpID identifies the admin-side operation, so an upgrade streams per-node + // progress in its own response exactly as an install does. Empty on legacy + // callers. OpID string `json:"op_id,omitempty"` } @@ -258,16 +247,10 @@ type BackendUpgradeReply struct { ReportsStoppedProcesses bool `json:"reports_stopped_processes,omitempty"` } -// SubjectNodeBackendList queries a worker node for its installed backends. -// Uses NATS request-reply. -func SubjectNodeBackendList(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".backend.list" -} - -// BackendListRequest is the payload for a backend.list NATS request. +// BackendListRequest is the payload for a backend.list control request. type BackendListRequest struct{} -// BackendListReply is the response from a backend.list NATS request. +// BackendListReply is the response from a backend.list control request. type BackendListReply struct { Backends []NodeBackendInfo `json:"backends"` Error string `json:"error,omitempty"` @@ -320,25 +303,17 @@ type BackendStopReply struct { ReportsStoppedProcesses bool `json:"reports_stopped_processes,omitempty"` } -// SubjectNodeBackendStop tells a worker node to stop its gRPC backend process. -// Equivalent to the local deleteProcess(). The node will: -// 1. Best-effort bounded Free() via gRPC (unless Force is true) -// 2. Kill the backend process -// 3. Can be restarted via another backend.start event. +// SubjectNodeBackendStop tells an AGENT worker that a backend is going away, so +// it can close the MCP sessions it cached for that backend. // -// Request-reply, answered with a BackendStopReply. A worker that predates that -// reply never answers, so the controller must treat a timeout as "unconfirmed" -// rather than "failed" — see RemoteUnloaderAdapter.stopBackend. +// It is the one node subject left, and it is addressed only to agent nodes. A +// BACKEND worker takes its stop on workerctl.PathBackendStop over its tunnel, +// where it also kills the process and recycles the port; an agent worker runs +// no backend processes and only needs to hear that one went. func SubjectNodeBackendStop(nodeID string) string { return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".backend.stop" } -// SubjectNodeModelStop targets one supervisor process and acknowledges only -// after that process has exited and its worker-side resources are released. -func SubjectNodeModelStop(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".model.stop" -} - type ModelStopRequest struct { ModelName string `json:"model_name"` ProcessKey string `json:"process_key"` @@ -356,18 +331,12 @@ type ModelStopReply struct { Error string `json:"error,omitempty"` } -// SubjectNodeBackendDelete tells a worker node to delete a backend (stop + remove files). -// Uses NATS request-reply. -func SubjectNodeBackendDelete(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".backend.delete" -} - -// BackendDeleteRequest is the payload for a backend.delete NATS request. +// BackendDeleteRequest is the payload for a backend.delete control request. type BackendDeleteRequest struct { Backend string `json:"backend"` } -// BackendDeleteReply is the response from a backend.delete NATS request. +// BackendDeleteReply is the response from a backend.delete control request. type BackendDeleteReply struct { Success bool `json:"success"` Error string `json:"error,omitempty"` @@ -390,57 +359,33 @@ type BackendDeleteReply struct { ReportsStoppedProcesses bool `json:"reports_stopped_processes,omitempty"` } -// SubjectNodeModelUnload tells a worker node to unload a model (gRPC Free) without killing the backend. -// Uses NATS request-reply. -func SubjectNodeModelUnload(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".model.unload" -} - -// ModelUnloadRequest is the payload for a model.unload NATS request. +// ModelUnloadRequest is the payload for a model.unload control request. type ModelUnloadRequest struct { ModelName string `json:"model_name"` Address string `json:"address,omitempty"` // gRPC address of the backend process to unload from } -// ModelUnloadReply is the response from a model.unload NATS request. +// ModelUnloadReply is the response from a model.unload control request. type ModelUnloadReply struct { Success bool `json:"success"` Error string `json:"error,omitempty"` } -// SubjectNodeModelDelete tells a worker node to delete model files from disk. -// Uses NATS request-reply. -func SubjectNodeModelDelete(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".model.delete" -} - -// ModelDeleteRequest is the payload for a model.delete NATS request. +// ModelDeleteRequest is the payload for a model.delete control request. type ModelDeleteRequest struct { ModelName string `json:"model_name"` } -// ModelDeleteReply is the response from a model.delete NATS request. +// ModelDeleteReply is the response from a model.delete control request. type ModelDeleteReply struct { Success bool `json:"success"` Error string `json:"error,omitempty"` } -// SubjectNodeModelsRunning asks a worker node which model backend processes it -// currently has running. Uses NATS request-reply. -// -// This is the authoritative answer to "is this replica still alive". The worker -// owns the process table, so unlike a health probe against the backend's own -// serving port, its reply does not depend on whether that backend happens to be -// busy: a model mid-generation cannot answer a gRPC health check for minutes at -// a time, but the worker answers immediately either way. -func SubjectNodeModelsRunning(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".models.running" -} - -// ModelsRunningRequest is the payload for a models.running NATS request. +// ModelsRunningRequest is the payload for a models.running control request. type ModelsRunningRequest struct{} -// ModelsRunningReply is the response from a models.running NATS request. +// ModelsRunningReply is the response from a models.running control request. type ModelsRunningReply struct { Models []RunningModelInfo `json:"models"` Error string `json:"error,omitempty"` @@ -455,12 +400,6 @@ type RunningModelInfo struct { Address string `json:"address,omitempty"` } -// SubjectNodeStop tells a serve-backend node to shut down entirely -// (deregister + exit). The node will not restart the backend process. -func SubjectNodeStop(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".stop" -} - // File Staging (Request-Reply — targeted to specific nodes) // These subjects use request-reply for synchronous file operations. diff --git a/core/services/messaging/subjects_upgrade_test.go b/core/services/messaging/subjects_upgrade_test.go index e60369cfca0a..e1a059fac98d 100644 --- a/core/services/messaging/subjects_upgrade_test.go +++ b/core/services/messaging/subjects_upgrade_test.go @@ -7,15 +7,19 @@ import ( "github.com/mudler/LocalAI/core/services/messaging" ) -var _ = Describe("SubjectNodeBackendUpgrade", func() { - It("returns the per-node upgrade subject", func() { - Expect(messaging.SubjectNodeBackendUpgrade("abc")). - To(Equal("nodes.abc.backend.upgrade")) +// The surviving node subject. It is written out BY HAND and not derived from +// the builder: an agent worker built from another commit subscribes to this +// literal, and a renamed subject is silence that looks exactly like a worker +// that never started. +var _ = Describe("SubjectNodeBackendStop", func() { + It("returns the per-node stop subject an agent worker subscribes to", func() { + Expect(messaging.SubjectNodeBackendStop("abc")). + To(Equal("nodes.abc.backend.stop")) }) It("sanitizes reserved NATS tokens in the node id", func() { - Expect(messaging.SubjectNodeBackendUpgrade("a.b*c")). - To(Equal("nodes.a-b-c.backend.upgrade")) + Expect(messaging.SubjectNodeBackendStop("a.b*c")). + To(Equal("nodes.a-b-c.backend.stop")) }) }) diff --git a/core/services/nodes/control_client.go b/core/services/nodes/control_client.go new file mode 100644 index 000000000000..6c943892b426 --- /dev/null +++ b/core/services/nodes/control_client.go @@ -0,0 +1,307 @@ +package nodes + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "sync" + "time" + + "github.com/mudler/xlog" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" + "github.com/mudler/LocalAI/pkg/httpclient" +) + +// ErrWorkerControlUnsupported reports that the worker answered, and does not +// serve that control verb. +// +// It is a deployment fact and not a verdict about anything: the worker is +// running a build older than this frontend, so a path this frontend knows is a +// 404 there. The one caller that may act on it is the upgrade fallback, which +// re-issues the legacy force-install the older build does understand. +// +// It is a SPECIALISATION of ErrWorkerUnroutable for the same reason +// ErrNoWorkerDialer is: everything that DELETES a node_models row makes exactly +// one check, and a 404 must not pass it. The worker spoke, but it said nothing +// about any backend, and reaping on it would evict healthy work over a version +// skew. Callers that want the fallback match this sentinel specifically, which +// ErrWorkerUnroutable does not imply in that direction. +var ErrWorkerControlUnsupported = fmt.Errorf("%w: the worker does not serve that control verb", ErrWorkerUnroutable) + +// ControlClient issues the frontend's control RPCs to a worker over that +// worker's tunnel. +// +// It is the frontend half of core/services/workerctl: the worker mounts those +// paths on the loopback HTTP server it already runs, and this reaches them +// through the `http` stream tag, so a control RPC to a worker another replica +// holds is relayed exactly like an inference request. +// +// There is one of these per frontend rather than one per verb, because the +// mapping from a failed RPC onto the four conditions this system must never +// confuse belongs in ONE place. See controlFailure. +type ControlClient struct { + // dialFor supplies the transport for one worker. It is per node because a + // worker is reached over ITS OWN tunnel and an http.Transport carries one + // DialContext, so a single shared transport could only ever reach one + // worker. nil means no tunnel dialer is wired and every call is refused; + // see ErrNoWorkerDialer for why that is not a fallback to a direct dial. + dialFor WorkerNetDialerFor + token string + + // clients caches one *http.Client per node, so a verb issued twice in a + // row reuses the tunnel stream its transport already holds instead of + // opening a new one. + // + // Entries are never pruned, and that is judged rather than overlooked: the + // map is bounded by the number of distinct workers this frontend has ever + // commanded, which is bounded by the fleet, and a departed worker's entry + // holds a map slot plus a transport whose idle connections IdleConnTimeout + // reclaims. It is the same shape, and would need the same missing + // node-departure signal to fix, as HTTPFileStager.clients. + clientsMu sync.Mutex + clients map[string]*http.Client +} + +// NewControlClient returns the control client for the workers dialFor can +// reach, authenticating with the deployment's registration token. +func NewControlClient(dialFor WorkerNetDialerFor, token string) *ControlClient { + return &ControlClient{dialFor: dialFor, token: token, clients: map[string]*http.Client{}} +} + +// clientFor returns the HTTP client that reaches one worker, building it on +// first use. +// +// The transport mirrors HTTPFileStager.clientFor, which is the other consumer +// of a worker's own HTTP server, and differs only where control traffic +// differs. HTTP/2 stays OFF for the reason it always was: its flow control +// stalls large transfers, and the install verb streams for minutes. The +// stager's 256 KB socket buffers are dropped because a control body is a +// handful of kilobytes and two 256 KB buffers per worker would be paid for +// every node in the fleet. +// +// The net.Dialer's connect timeout and keepalive have nothing to act on here: +// there is no TCP connect to time out, and liveness on the link is the yamux +// session's keepalive rather than the socket's. No client.Timeout is set +// either, because it would bound the response BODY, and the install verb's +// body stays open for as long as the install runs. What bounds a call is the +// context its caller passes. +func (c *ControlClient) clientFor(nodeID string) (*http.Client, error) { + if c == nil || c.dialFor == nil { + return nil, fmt.Errorf("control rpc to node %q: %w", nodeID, ErrNoWorkerDialer) + } + c.clientsMu.Lock() + defer c.clientsMu.Unlock() + if cl, ok := c.clients[nodeID]; ok { + return cl, nil + } + dial := c.dialFor(nodeID) + if dial == nil { + return nil, fmt.Errorf("control rpc to node %q: %w", nodeID, ErrNoWorkerDialer) + } + transport := &http.Transport{ + DialContext: dial, + ForceAttemptHTTP2: false, // HTTP/2 flow control can stall a long streaming verb + MaxIdleConns: 10, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + cl := httpclient.New(httpclient.WithTransport(transport)) + c.clients[nodeID] = cl + return cl, nil +} + +// Call issues one control RPC and decodes its reply. reply may be nil for the +// verbs that answer 204. +// +// A reply that arrives with its Error field set is NOT an error here: the +// worker answered, and reading its answer is the caller's job. Only a failure +// to reach the worker, or an answer this frontend cannot read, comes back as an +// error, and every one of those is mapped by controlFailure. +func (c *ControlClient) Call(ctx context.Context, nodeID, path string, req, reply any) error { + resp, err := c.do(ctx, nodeID, path, req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNoContent || reply == nil { + // A verb that answers 204 sends no body at all; a caller that asked for + // no reply may still have been sent one, and draining it is what lets + // the transport keep the tunnel stream for the next verb instead of + // tearing it down. + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxControlErrorBodyBytes)) + return nil + } + if err := json.NewDecoder(resp.Body).Decode(reply); err != nil { + // A body this frontend cannot read is not the worker's verdict about + // anything, so it must not be reported as one. + return controlFailure(ctx, nodeID, fmt.Errorf("decoding the %s reply: %w", path, err)) + } + return nil +} + +// CallStreaming issues a control RPC whose response is an NDJSON envelope +// stream, invoking onProgress for each progress line and decoding the single +// terminal reply line into reply. onProgress may be nil. +// +// onProgress runs SYNCHRONOUSLY, on this goroutine. The NATS carrier ran each +// progress callback on a goroutine of its own because a slow callback there +// stalled the one reader thread every worker's events arrived on; here the only +// thing a slow callback holds up is this request's own body, which is the +// caller's business. Dropping the guard is also what makes the events arrive in +// the order the worker sent them. +func (c *ControlClient) CallStreaming(ctx context.Context, nodeID, path string, + req, reply any, onProgress func(messaging.BackendInstallProgressEvent)) error { + resp, err := c.do(ctx, nodeID, path, req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + + dec := json.NewDecoder(resp.Body) + var terminal json.RawMessage + for terminal == nil { + var env workerctl.Envelope + decErr := dec.Decode(&env) + if decErr != nil { + // EOF before a reply line is the tunnel dying mid-verb, and reading + // it as "the install failed" is precisely the collapse this phase + // exists to prevent: nothing was learned about the backend, so this + // is unroutable and a caller may not act on it. + if errors.Is(decErr, io.EOF) { + decErr = fmt.Errorf("the %s stream ended before its reply line: %w", path, io.ErrUnexpectedEOF) + } + return controlFailure(ctx, nodeID, decErr) + } + if env.Reply != nil { + terminal = env.Reply + break + } + if env.Progress == nil || onProgress == nil { + continue + } + var ev messaging.BackendInstallProgressEvent + if err := json.Unmarshal(env.Progress, &ev); err != nil { + // Progress is transient by contract, so a line this frontend cannot + // read costs a tick and never the operation. + xlog.Debug("unreadable control progress line", "node", nodeID, "path", path, "error", err) + continue + } + onProgress(ev) + } + if reply == nil { + return nil + } + if err := json.Unmarshal(terminal, reply); err != nil { + return controlFailure(ctx, nodeID, fmt.Errorf("decoding the %s reply line: %w", path, err)) + } + return nil +} + +// do issues the request and returns the response for any status this frontend +// can read a body from. Every other outcome is already mapped. +// +// The caller owns closing the body. +func (c *ControlClient) do(ctx context.Context, nodeID, path string, req any) (*http.Response, error) { + client, err := c.clientFor(nodeID) + if err != nil { + // ErrNoWorkerDialer already carries ErrWorkerUnroutable, so it must not + // go through controlFailure, which would wrap the umbrella twice. + return nil, err + } + body, err := json.Marshal(req) + if err != nil { + return nil, controlFailure(ctx, nodeID, fmt.Errorf("encoding the %s request: %w", path, err)) + } + + // The host is a name that resolves nowhere. What carries the request is the + // transport's DialContext, which opens a stream on this worker's tunnel; + // the host exists because an http.Request needs one, and it names the node + // so a log line is diagnosable. See WorkerHTTPHost. + url := "http://" + WorkerHTTPHost(nodeID, "") + path + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return nil, controlFailure(ctx, nodeID, fmt.Errorf("building the %s request: %w", path, err)) + } + httpReq.Header.Set("Content-Type", "application/json") + if c.token != "" { + httpReq.Header.Set("Authorization", "Bearer "+c.token) + } + + resp, err := client.Do(httpReq) + if err != nil { + return nil, controlFailure(ctx, nodeID, err) + } + + switch { + case resp.StatusCode == http.StatusOK, resp.StatusCode == http.StatusNoContent: + return resp, nil + case resp.StatusCode == http.StatusNotFound: + // The worker answered, about ITSELF rather than about a backend: it is + // older than this frontend and serves no such verb. The catch-all under + // the control prefix is what makes this distinguishable from a proxy's + // bare 404. + _ = resp.Body.Close() + return nil, fmt.Errorf("control rpc %s to node %q: %w", path, nodeID, ErrWorkerControlUnsupported) + default: + // A verb's own failure arrives as 200 with Error set, so a non-2xx is + // the worker failing to serve the request rather than answering it: a + // body it could not read, a method it refuses, a handler that panicked. + // None of those is evidence about a backend. + detail := readErrorBody(resp) + _ = resp.Body.Close() + return nil, controlFailure(ctx, nodeID, fmt.Errorf("the %s verb answered HTTP %d: %s", path, resp.StatusCode, detail)) + } +} + +// maxControlErrorBodyBytes bounds how much of a non-2xx body reaches a log +// line. The body is written by the worker and lands in this frontend's logs, so +// it is bounded for the same reason the worker bounds the path it echoes. +const maxControlErrorBodyBytes = 512 + +// readErrorBody reads the diagnostic text off a non-2xx control response. +func readErrorBody(resp *http.Response) string { + raw, err := io.ReadAll(io.LimitReader(resp.Body, maxControlErrorBodyBytes)) + if err != nil { + return "" + } + return string(bytes.TrimSpace(raw)) +} + +// controlFailure maps a control RPC's failure onto exactly one of the four +// conditions this system must never confuse, and it is the only place that +// mapping is made. +// +// The rule it enforces, stated as the code enforces it rather than as advice: a +// WORKER'S ANSWER passes through unwrapped so cluster.IsWorkerAnswer still sees +// it and a reap guard may act on it, and EVERYTHING ELSE is wrapped in +// ErrWorkerUnroutable so nothing can. There is no third branch, because a third +// branch is how the eight collapses on this branch happened: each was a place +// that decided for itself which errors were evidence. +// +// The caller's spent budget is checked FIRST, and that ordering is the same one +// cluster.WorkerDialer.handshake takes for the same reason. A worker's refusal +// that arrives in the instant the deadline expires would otherwise be reported +// as the worker's non-transient verdict and reap a row, and nothing orders the +// two timers: an expiry is never evidence about a backend, so it is answered as +// the caller's own timeout rather than as an answer. +func controlFailure(ctx context.Context, nodeID string, err error) error { + if err == nil { + return nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("control rpc to node %q: %w: %w", nodeID, ErrWorkerUnroutable, ctxErr) + } + if cluster.IsWorkerAnswer(err) { + return err + } + return fmt.Errorf("control rpc to node %q: %w: %w", nodeID, ErrWorkerUnroutable, err) +} diff --git a/core/services/nodes/control_client_test.go b/core/services/nodes/control_client_test.go new file mode 100644 index 000000000000..d8abc1197a5d --- /dev/null +++ b/core/services/nodes/control_client_test.go @@ -0,0 +1,317 @@ +package nodes + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// The whole of this task is the table controlFailure implements: every way a +// control RPC can fail lands in exactly one of four conditions, and none of +// them may be reported as another. The rows, and what each permits: +// +// - the WORKER'S OWN ANSWER, which a reap guard may act on; +// - an UNREACHABLE PEER or a lost route, which nobody may act on; +// - a SPENT BUDGET, which nobody may act on; +// - the worker answering that it is OLDER than this frontend, which only the +// legacy upgrade fallback may act on. +var _ = Describe("ControlClient", func() { + Context("error mapping", func() { + dialerReturning := func(err error) WorkerNetDialerFor { + return func(string) func(context.Context, string, string) (net.Conn, error) { + return func(context.Context, string, string) (net.Conn, error) { return nil, err } + } + } + + It("keeps a worker's own refusal matchable and does NOT wrap it as unroutable", func() { + c := NewControlClient(dialerReturning( + fmt.Errorf("%w: no such process", cluster.ErrStreamTargetUnavailable)), "tok") + err := c.Call(context.Background(), "n1", workerctl.PathModelsRunning, struct{}{}, &struct{}{}) + Expect(cluster.IsWorkerAnswer(err)).To(BeTrue()) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeFalse()) + }) + + DescribeTable("reports every non-answer as unroutable and never as a worker verdict", + func(cause error) { + c := NewControlClient(dialerReturning(cause), "tok") + err := c.Call(context.Background(), "n1", workerctl.PathModelsRunning, struct{}{}, &struct{}{}) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + }, + Entry("no route", fmt.Errorf("reaching node: %w", cluster.ErrNoRoute)), + Entry("no connection recorded", cluster.ErrNoConnection), + Entry("peer unreachable", cluster.ErrPeerUnreachable), + Entry("no relay path", cluster.ErrNoRelayPath), + // The fourth refusal code. The worker DID send it, and it still + // must not be a verdict: it is what a worker says when it learned + // nothing, and those clear on their own. + Entry("the worker learned nothing", fmt.Errorf("%w: late frame", cluster.ErrStreamNotServed)), + Entry("a reply code this frontend does not know", errors.New(`tunnel stream refused with unrecognised code "from-the-future": x`)), + ) + + It("refuses without a dialer rather than reaching for an address", func() { + c := NewControlClient(nil, "tok") + err := c.Call(context.Background(), "n1", workerctl.PathModelsRunning, struct{}{}, &struct{}{}) + Expect(err).To(MatchError(ErrNoWorkerDialer)) + Expect(err).To(MatchError(ErrWorkerUnroutable)) + }) + + It("refuses when the dialer has nothing for that node", func() { + c := NewControlClient(func(string) func(context.Context, string, string) (net.Conn, error) { + return nil + }, "tok") + err := c.Call(context.Background(), "n1", workerctl.PathModelsRunning, struct{}{}, &struct{}{}) + Expect(err).To(MatchError(ErrNoWorkerDialer)) + }) + + It("reports a spent budget as unroutable, so nothing acts on an expiry", func() { + expired, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + + c := NewControlClient(dialerReturning(errors.New("never reached")), "tok") + err := c.Call(expired, "n1", workerctl.PathModelsRunning, struct{}{}, &struct{}{}) + + Expect(errors.Is(err, context.DeadlineExceeded)).To(BeTrue()) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + }) + + // A timeout is not a verdict, and this is the ordering that enforces + // it. Phase 2's final defect was a worker refusal that arrived in the + // instant the budget expired and was reported as the worker's + // non-transient answer, which reaps a row. + // + // Asserted on the mapping function directly, and deliberately so. + // Nothing orders the two timers, so a spec that drove this through Call + // could only ever produce one of the two orderings by luck: with a + // context already spent, the HTTP client returns before the dialler is + // even asked, and the refusal this is about never happens. Reaching for + // the collapse through the public call was tried and left the suite + // green under the mutation that removes the guard. + DescribeTable("decides between a worker's refusal and a spent budget by the BUDGET first", + func(spent bool, wantAnswer bool) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if spent { + var stop context.CancelFunc + ctx, stop = context.WithDeadline(ctx, time.Now().Add(-time.Second)) + defer stop() + } + refusal := fmt.Errorf("%w: no such process", cluster.ErrStreamTargetUnavailable) + err := controlFailure(ctx, "n1", refusal) + + Expect(cluster.IsWorkerAnswer(err)).To(Equal(wantAnswer)) + Expect(errors.Is(err, context.DeadlineExceeded)).To(Equal(spent)) + }, + // The negative control: with budget left, the very same refusal is + // the worker speaking and a reap guard may act on it. Without it + // this table would pass on a mapping that never reports an answer. + Entry("budget left: the worker spoke", false, true), + Entry("budget spent: the caller's own timeout", true, false), + ) + }) + + Context("against a worker's HTTP answers", func() { + var ( + srv *httptest.Server + handler http.HandlerFunc + client *ControlClient + ) + + BeforeEach(func() { + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler(w, r) + })) + DeferCleanup(srv.Close) + addr := srv.Listener.Addr().String() + client = NewControlClient(func(string) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "tcp", addr) + } + }, "tok") + }) + + It("addresses the node in the URL host and carries the bearer token", func() { + var gotHost, gotAuth, gotPath, gotMethod string + handler = func(w http.ResponseWriter, r *http.Request) { + gotHost, gotAuth, gotPath, gotMethod = r.Host, r.Header.Get("Authorization"), r.URL.Path, r.Method + _, _ = w.Write([]byte(`{}`)) + } + Expect(client.Call(context.Background(), "n1", workerctl.PathModelsRunning, struct{}{}, &struct{}{})).To(Succeed()) + + Expect(gotHost).To(Equal("n1" + unroutableHostSuffix)) + Expect(gotAuth).To(Equal("Bearer tok")) + Expect(gotPath).To(Equal(workerctl.PathModelsRunning)) + // POST, because a control verb is a command: the worker refuses a + // GET so a probe cannot fire one, and a client that sent GET would + // be refused rather than served. + Expect(gotMethod).To(Equal(http.MethodPost)) + }) + + It("hands the worker's own reply back with its Error field intact", func() { + // A verb's own failure is a 200 with Error set. The CALLER reads + // it; this is not an error here, and reporting it as one would put + // the worker's verdict in the bucket reserved for a broken link. + handler = func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"success":false,"error":"disk full"}`)) + } + var reply messaging.BackendDeleteReply + Expect(client.Call(context.Background(), "n1", workerctl.PathBackendDelete, struct{}{}, &reply)).To(Succeed()) + Expect(reply.Success).To(BeFalse()) + Expect(reply.Error).To(Equal("disk full")) + }) + + It("accepts a 204 for the verbs that answer nothing", func() { + handler = func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) } + Expect(client.Call(context.Background(), "n1", workerctl.PathNodeStop, struct{}{}, nil)).To(Succeed()) + }) + + It("reports an unknown control verb as unsupported, and not as absence", func() { + handler = func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "unknown worker control path "+r.URL.Path, http.StatusNotFound) + } + err := client.Call(context.Background(), "n1", workerctl.Prefix+"invented", struct{}{}, &struct{}{}) + Expect(err).To(MatchError(ErrWorkerControlUnsupported)) + Expect(errors.Is(err, cluster.ErrNoRoute)).To(BeFalse()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + }) + + DescribeTable("reports a worker that failed to SERVE the request as unroutable, never as a verdict", + func(status int) { + handler = func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "boom", status) } + err := client.Call(context.Background(), "n1", workerctl.PathBackendList, struct{}{}, &struct{}{}) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + Expect(errors.Is(err, ErrWorkerControlUnsupported)).To(BeFalse()) + }, + Entry("the handler failed", http.StatusInternalServerError), + Entry("the body could not be read", http.StatusBadRequest), + Entry("the method was refused", http.StatusMethodNotAllowed), + Entry("a proxy in the path", http.StatusBadGateway), + ) + + It("reports a reply it cannot decode as unroutable, not as an empty answer", func() { + // An empty ModelsRunningReply says "this worker is running nothing", + // which the reconciler acts on. It must never be manufactured from + // a body that would not parse. + handler = func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`not json`)) } + var reply messaging.ModelsRunningReply + err := client.Call(context.Background(), "n1", workerctl.PathModelsRunning, struct{}{}, &reply) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + }) + + It("streams install progress in order and returns the terminal reply", func() { + handler = func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", workerctl.ContentTypeStream) + enc := json.NewEncoder(w) + _ = enc.Encode(workerctl.Envelope{Progress: json.RawMessage(`{"percentage":50}`)}) + _ = enc.Encode(workerctl.Envelope{Progress: json.RawMessage(`{"percentage":100}`)}) + _ = enc.Encode(workerctl.Envelope{Reply: json.RawMessage(`{"success":true}`)}) + } + var seen []float64 + var reply messaging.BackendInstallReply + err := client.CallStreaming(context.Background(), "n1", workerctl.PathBackendInstall, + messaging.BackendInstallRequest{Backend: "mock", OpID: "op-1"}, &reply, + func(ev messaging.BackendInstallProgressEvent) { seen = append(seen, ev.Percentage) }) + Expect(err).NotTo(HaveOccurred()) + Expect(seen).To(Equal([]float64{50, 100})) + Expect(reply.Success).To(BeTrue()) + }) + + It("stops reading at the reply line, so nothing after it can be taken for progress", func() { + handler = func(w http.ResponseWriter, _ *http.Request) { + enc := json.NewEncoder(w) + _ = enc.Encode(workerctl.Envelope{Reply: json.RawMessage(`{"success":true}`)}) + _ = enc.Encode(workerctl.Envelope{Progress: json.RawMessage(`{"percentage":10}`)}) + } + var seen []float64 + var reply messaging.BackendInstallReply + Expect(client.CallStreaming(context.Background(), "n1", workerctl.PathBackendInstall, + struct{}{}, &reply, + func(ev messaging.BackendInstallProgressEvent) { seen = append(seen, ev.Percentage) })).To(Succeed()) + Expect(seen).To(BeEmpty()) + Expect(reply.Success).To(BeTrue()) + }) + + It("reports a stream that ends before its reply line as unroutable, not as a failed install", func() { + // A tunnel that dies mid-install must not be read as the worker + // saying the install failed. This is the collapse the phase exists + // to prevent, one layer up from where phase 2 fixed it. + handler = func(w http.ResponseWriter, _ *http.Request) { + enc := json.NewEncoder(w) + _ = enc.Encode(workerctl.Envelope{Progress: json.RawMessage(`{"percentage":50}`)}) + hijackAndClose(w) + } + var reply messaging.BackendInstallReply + err := client.CallStreaming(context.Background(), "n1", workerctl.PathBackendInstall, + struct{}{}, &reply, nil) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + Expect(reply.Success).To(BeFalse()) + }) + + It("reports a stream with no lines at all as unroutable", func() { + handler = func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) } + var reply messaging.BackendInstallReply + err := client.CallStreaming(context.Background(), "n1", workerctl.PathBackendInstall, + struct{}{}, &reply, nil) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) + Expect(errors.Is(err, io.ErrUnexpectedEOF)).To(BeTrue()) + }) + + It("keeps going past a progress line it cannot read, since progress is transient", func() { + handler = func(w http.ResponseWriter, _ *http.Request) { + enc := json.NewEncoder(w) + _ = enc.Encode(workerctl.Envelope{Progress: json.RawMessage(`{"percentage":"not a number"}`)}) + _ = enc.Encode(workerctl.Envelope{Progress: json.RawMessage(`{"percentage":70}`)}) + _ = enc.Encode(workerctl.Envelope{Reply: json.RawMessage(`{"success":true}`)}) + } + var seen []float64 + var reply messaging.BackendInstallReply + Expect(client.CallStreaming(context.Background(), "n1", workerctl.PathBackendInstall, + struct{}{}, &reply, + func(ev messaging.BackendInstallProgressEvent) { seen = append(seen, ev.Percentage) })).To(Succeed()) + Expect(seen).To(Equal([]float64{70})) + Expect(reply.Success).To(BeTrue()) + }) + + It("reports a streaming 404 as unsupported rather than as a truncated stream", func() { + handler = func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "unknown worker control path "+r.URL.Path, http.StatusNotFound) + } + var reply messaging.BackendUpgradeReply + err := client.CallStreaming(context.Background(), "n1", workerctl.PathBackendUpgrade, struct{}{}, &reply, nil) + Expect(err).To(MatchError(ErrWorkerControlUnsupported)) + }) + }) +}) + +// hijackAndClose ends a response mid-body without the chunked terminator, which +// is what a tunnel dying under an in-flight verb looks like to the reader. +func hijackAndClose(w http.ResponseWriter) { + GinkgoHelper() + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + hj, ok := w.(http.Hijacker) + Expect(ok).To(BeTrue(), "the test server must support hijacking") + conn, buf, err := hj.Hijack() + Expect(err).NotTo(HaveOccurred()) + _ = buf.Flush() + _ = conn.Close() +} diff --git a/core/services/nodes/control_worker_fake_test.go b/core/services/nodes/control_worker_fake_test.go new file mode 100644 index 000000000000..013634324d2b --- /dev/null +++ b/core/services/nodes/control_worker_fake_test.go @@ -0,0 +1,273 @@ +package nodes + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// controlKey names one control verb on one node. +// +// It replaces the NATS subject the scripted double used to key on, and it is +// the same pair the real client addresses: the node picks the tunnel and the +// path picks the verb. +func controlKey(nodeID, path string) string { return nodeID + " " + path } + +// scriptedControlWorkers is a fleet of fake workers reachable only the way a +// real one is: through a per-node dialer, over HTTP, on the control paths. +// +// One HTTP server serves every node, and which node a request is FOR is read +// off the Host header, because that is what ControlClient puts there. That is +// not a convenience: it means a client addressing the wrong node's URL through +// the right node's tunnel would be visible here, which a per-node server could +// not see. +type scriptedControlWorkers struct { + mu sync.Mutex + srv *httptest.Server + + replies map[string][]byte + unsupported map[string]bool + matched map[string][]matchedControlReply + progress map[string][]messaging.BackendInstallProgressEvent + + // unreachable and expired are keyed by NODE, not by verb, because they are + // failures of the ROUTE and a route belongs to a node. They are what the + // dialer answers with; see scriptUnroutable and scriptTimeout. + unreachable map[string]bool + expired map[string]bool + + // hangs names the verbs whose handler never answers, so the only thing + // that ends the call is the caller's own budget. + hangs map[string]bool + + // calls records every request that reached a worker, in order. + calls []requestCall +} + +// matchedControlReply is a canned reply that fires only for a request matching +// pred. It exists so a spec can tell "install with Force=true" (the legacy +// upgrade fallback) from an ordinary install on the same verb. +type matchedControlReply struct { + pred func(messaging.BackendInstallRequest) bool + reply []byte +} + +func newScriptedControlWorkers() *scriptedControlWorkers { + s := &scriptedControlWorkers{ + replies: map[string][]byte{}, + unsupported: map[string]bool{}, + matched: map[string][]matchedControlReply{}, + progress: map[string][]messaging.BackendInstallProgressEvent{}, + unreachable: map[string]bool{}, + expired: map[string]bool{}, + hangs: map[string]bool{}, + } + mux := http.NewServeMux() + mux.HandleFunc(workerctl.Prefix, s.serve) + s.srv = httptest.NewServer(mux) + DeferCleanup(s.srv.Close) + return s +} + +// dialer hands ControlClient the per-node transport it expects. +// +// A node scripted unreachable or expired never reaches the server, which is +// what a real route failure looks like: nothing is asked of the worker and +// nothing is learned about it. +func (s *scriptedControlWorkers) dialer() WorkerNetDialerFor { + return func(nodeID string) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, _, _ string) (net.Conn, error) { + s.mu.Lock() + unreachable, expired := s.unreachable[nodeID], s.expired[nodeID] + s.mu.Unlock() + switch { + case expired: + return nil, context.DeadlineExceeded + case unreachable: + return nil, fmt.Errorf("reaching node %q: %w", nodeID, cluster.ErrNoRoute) + } + var d net.Dialer + return d.DialContext(ctx, "tcp", s.srv.Listener.Addr().String()) + } + } +} + +func (s *scriptedControlWorkers) controlClient() *ControlClient { + return NewControlClient(s.dialer(), "test-token") +} + +// nodeOf recovers the node a request was addressed to from its Host header. +func nodeOf(host string) string { + return strings.TrimSuffix(host, unroutableHostSuffix) +} + +func (s *scriptedControlWorkers) serve(w http.ResponseWriter, r *http.Request) { + body, readErr := io.ReadAll(r.Body) + Expect(readErr).ToNot(HaveOccurred()) + key := controlKey(nodeOf(r.Host), r.URL.Path) + + s.mu.Lock() + s.calls = append(s.calls, requestCall{Subject: key, Data: body}) + unsupported := s.unsupported[key] + hang := s.hangs[key] + reply := s.replies[key] + matchers := s.matched[key] + ticks := s.progress[key] + s.mu.Unlock() + + if hang { + // The worker took the request and never answered. Only the caller's + // own budget ends this, which is what makes the budget observable. + <-r.Context().Done() + return + } + if unsupported { + http.Error(w, "unknown worker control path "+r.URL.Path, http.StatusNotFound) + return + } + if len(matchers) > 0 { + var req messaging.BackendInstallRequest + _ = json.Unmarshal(body, &req) + for _, m := range matchers { + if m.pred(req) { + reply = m.reply + break + } + } + } + if reply == nil { + // A verb no spec scripted. Answered LOUDLY rather than plausibly: a + // forgotten script must be a red spec, never a worker that looks absent. + http.Error(w, "this spec scripted no answer for "+key, http.StatusInternalServerError) + return + } + + if r.URL.Path != workerctl.PathBackendInstall && r.URL.Path != workerctl.PathBackendUpgrade { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(reply) + return + } + + // The two streaming verbs: zero or more progress lines, then exactly one + // reply line, always last. + w.Header().Set("Content-Type", workerctl.ContentTypeStream) + w.WriteHeader(http.StatusOK) + enc := json.NewEncoder(w) + for _, ev := range ticks { + raw, err := json.Marshal(ev) + if err != nil { + continue + } + _ = enc.Encode(workerctl.Envelope{Progress: raw}) + } + _ = enc.Encode(workerctl.Envelope{Reply: reply}) +} + +func (s *scriptedControlWorkers) scriptReply(key string, reply any) { + raw, err := json.Marshal(reply) + Expect(err).ToNot(HaveOccurred()) + s.mu.Lock() + defer s.mu.Unlock() + s.replies[key] = raw +} + +// scriptRawReply scripts a reply byte for byte, so a spec can express what a +// worker on a DIFFERENT build would send rather than what today's struct +// marshals to. +func (s *scriptedControlWorkers) scriptRawReply(key string, raw []byte) { + s.mu.Lock() + defer s.mu.Unlock() + s.replies[key] = raw +} + +// scriptUnsupported makes the worker answer 404 for one verb, which is what a +// build older than this frontend does. +func (s *scriptedControlWorkers) scriptUnsupported(key string) { + s.mu.Lock() + defer s.mu.Unlock() + s.unsupported[key] = true +} + +// scriptUnroutable makes every route to a node fail without reaching it. The +// worker is asked nothing, so nothing is learned about it. +func (s *scriptedControlWorkers) scriptUnroutable(nodeID string) { + s.mu.Lock() + defer s.mu.Unlock() + s.unreachable[nodeID] = true +} + +// scriptTimeout makes a node's RPCs end with the caller's budget spent. +// +// Injected at the dial rather than by making a handler sleep, and the +// difference is only in how long the spec takes: a real worker that answers too +// late ends the same way, with the caller's own deadline, because +// cluster.WorkerDialer reports a handshake that outlives the budget as exactly +// this. Answering it instantly keeps the adapter's real install timeout, which +// the retry-scheduling assertions read. +func (s *scriptedControlWorkers) scriptTimeout(nodeID string) { + s.mu.Lock() + defer s.mu.Unlock() + s.expired[nodeID] = true +} + +// clearTimeout lets a node answer again, as a worker that finished a long +// install in the background does. +func (s *scriptedControlWorkers) clearTimeout(nodeID string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.expired, nodeID) +} + +func (s *scriptedControlWorkers) scriptReplyMatching(key string, pred func(messaging.BackendInstallRequest) bool, reply messaging.BackendInstallReply) { + raw, err := json.Marshal(reply) + Expect(err).ToNot(HaveOccurred()) + s.mu.Lock() + defer s.mu.Unlock() + s.matched[key] = append(s.matched[key], matchedControlReply{pred: pred, reply: raw}) +} + +// scriptProgress queues the progress lines a streaming verb writes ahead of its +// reply. There is no window to miss them in and nothing to subscribe to: they +// are part of the response the caller is already reading. +func (s *scriptedControlWorkers) scriptProgress(key string, events []messaging.BackendInstallProgressEvent) { + s.mu.Lock() + defer s.mu.Unlock() + s.progress[key] = events +} + +// scriptHang makes one verb on one node accept the request and never answer, +// so the call ends only when the caller's budget does. +// +// It is how a spec observes which budget a verb was given. HTTP carries no +// deadline and the transport dials on a context of its own, so the budget is +// invisible from the far side; how long the client is willing to wait is the +// only thing that shows it. +func (s *scriptedControlWorkers) scriptHang(key string) { + s.mu.Lock() + defer s.mu.Unlock() + s.hangs[key] = true +} + +// callSubjects reports the (node, verb) pairs that reached a worker, in order. +func (s *scriptedControlWorkers) callSubjects() []string { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]string, 0, len(s.calls)) + for _, c := range s.calls { + out = append(out, c.Subject) + } + return out +} diff --git a/core/services/nodes/interfaces.go b/core/services/nodes/interfaces.go index b68a0dd63c00..cd1ba03e5c11 100644 --- a/core/services/nodes/interfaces.go +++ b/core/services/nodes/interfaces.go @@ -110,10 +110,17 @@ type NodeHealthStore interface { } // ModelLocator is used by RemoteUnloaderAdapter for model discovery. +// +// Get is here for one reason: backend.stop is the only control verb whose +// carrier depends on what KIND of worker it is addressed to, because agent +// workers hold no tunnel and still take it over the bus. The callers that come +// through NodeCommandSender carry a node id and nothing else, so the type is +// read here rather than threaded through every one of them. type ModelLocator interface { FindNodesWithModel(ctx context.Context, modelName string) ([]BackendNode, error) RemoveNodeModel(ctx context.Context, nodeID, modelName string, replicaIndex int) error RemoveAllNodeModelReplicas(ctx context.Context, nodeID, modelName string) error + Get(ctx context.Context, nodeID string) (*BackendNode, error) } // ModelLookup is used by DistributedModelStore for model existence queries. diff --git a/core/services/nodes/managers_agent_node_test.go b/core/services/nodes/managers_agent_node_test.go index 8ee95c083f8e..5666c093d0f0 100644 --- a/core/services/nodes/managers_agent_node_test.go +++ b/core/services/nodes/managers_agent_node_test.go @@ -11,19 +11,20 @@ import ( "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/testutil" + "github.com/mudler/LocalAI/core/services/workerctl" ) -// Agent workers do not subscribe to the backend.* subjects, so asking one to -// list its backends can only answer "no responders". ListBackends read that as -// a node that had gone away and marked it unhealthy; the node's next heartbeat -// marked it healthy again. Every poll of the backends view therefore flapped -// every agent node in the cluster, and while it was unhealthy the router would -// not schedule onto it. +// Agent workers hold no tunnel and serve no control plane, so asking one to +// list its backends can only fail. ListBackends read that failure as a node +// that had gone away and marked it unhealthy; the node's next heartbeat marked +// it healthy again. Every poll of the backends view therefore flapped every +// agent node in the cluster, and while it was unhealthy the router would not +// schedule onto it. var _ = Describe("Backend listing across mixed node types", func() { var ( db *gorm.DB registry *NodeRegistry - mc *scriptedMessagingClient + mc *scriptedControlWorkers mgr *DistributedBackendManager ctx context.Context ) @@ -36,10 +37,10 @@ var _ = Describe("Backend listing across mixed node types", func() { var err error registry, err = NewNodeRegistry(db) Expect(err).ToNot(HaveOccurred()) - mc = newScriptedMessagingClient() + mc = newScriptedControlWorkers() mgr = &DistributedBackendManager{ local: stubLocalBackendManager{}, - adapter: NewRemoteUnloaderAdapter(nil, mc, 3*time.Minute, 15*time.Minute), + adapter: NewRemoteUnloaderAdapter(nil, nil, mc.controlClient(), 3*time.Minute, 15*time.Minute), registry: registry, } ctx = context.Background() @@ -62,23 +63,43 @@ var _ = Describe("Backend listing across mixed node types", func() { It("leaves an agent node healthy instead of flapping it", func() { agent := register("agent-worker-1", NodeTypeAgent) - mc.scriptNoResponders(messaging.SubjectNodeBackendList(agent.ID)) + mc.scriptUnroutable(agent.ID) _, err := mgr.ListBackends() Expect(err).ToNot(HaveOccurred()) Expect(statusOf(agent.ID)).To(Equal(StatusHealthy), - "an agent node cannot answer backend.list and must not be judged on it") + "an agent node serves no control plane and must not be judged on it") + Expect(mc.callSubjects()).To(BeEmpty(), + "an agent node must not be asked a backend-worker verb at all") }) - It("still marks a backend node unhealthy when it does not answer", func() { + // The direction changed with the carrier, and the change is the safety + // property rather than a regression. "No responders" meant the worker was + // not on the bus; a failed control RPC means THIS frontend could not route + // to it, which is equally what a healthy worker re-homing its tunnel + // between frontend replicas produces. Demoting on that is the fleet-wide + // eviction this phase exists to prevent, so a node that does not answer is + // skipped and left alone. cluster.Presence, read identically on every + // replica from the database, is what may say a worker is gone. + It("leaves a BACKEND node healthy when its control RPC could not be routed", func() { backendNode := register("worker-a", NodeTypeBackend) - mc.scriptNoResponders(messaging.SubjectNodeBackendList(backendNode.ID)) + mc.scriptUnroutable(backendNode.ID) _, err := mgr.ListBackends() Expect(err).ToNot(HaveOccurred()) - Expect(statusOf(backendNode.ID)).To(Equal(StatusUnhealthy), - "a backend worker that does not answer is genuinely gone") + Expect(statusOf(backendNode.ID)).To(Equal(StatusHealthy), + "a route this frontend could not open is not evidence the worker has gone") + }) + + It("still reports the backends of a node that does answer", func() { + backendNode := register("worker-b", NodeTypeBackend) + mc.scriptReply(controlKey(backendNode.ID, workerctl.PathBackendList), + messaging.BackendListReply{Backends: []messaging.NodeBackendInfo{{Name: "vllm"}}}) + + backends, err := mgr.ListBackends() + Expect(err).ToNot(HaveOccurred()) + Expect(backends).To(HaveKey("vllm")) }) }) diff --git a/core/services/nodes/managers_distributed.go b/core/services/nodes/managers_distributed.go index 4132eca797db..e3b6ae5b5094 100644 --- a/core/services/nodes/managers_distributed.go +++ b/core/services/nodes/managers_distributed.go @@ -14,7 +14,6 @@ import ( "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/xlog" - "github.com/nats-io/nats.go" ) // DistributedModelManager wraps a local ModelManager and adds NATS fan-out @@ -213,8 +212,7 @@ func (d *DistributedBackendManager) enqueueAndDrainBackendOp(ctx context.Context continue } - // Record failure for backoff. If it's an ErrNoResponders, the node's - // gone AWOL - mark unhealthy so the router stops picking it too. + // Record failure for backoff. errMsg := applyErr.Error() // Worker-still-installing is a "soft" failure: the worker is most @@ -234,10 +232,15 @@ func (d *DistributedBackendManager) enqueueAndDrainBackendOp(ctx context.Context continue } - if errors.Is(applyErr, nats.ErrNoResponders) { - xlog.Warn("No NATS responders for node, marking unhealthy", "node", node.Name, "nodeID", node.ID) - d.registry.MarkUnhealthy(ctx, node.ID) - } + // A failed control RPC no longer demotes the node, and that is the + // point rather than an omission. It used to, on nats.ErrNoResponders, + // which meant "not on the bus"; the control plane's failures mean "this + // frontend could not route to it", which is equally true of a worker + // that is heartbeating, serving another replica and re-homing its + // tunnel. Demoting on that is the fleet-wide eviction this phase exists + // to prevent. Absence is a fact read from the database, identically on + // every replica, and the scheduler starts reading it in the task that + // replaces this signal with cluster.Presence. if id, err := d.findPendingRow(ctx, node.ID, backend, op); err == nil { _ = d.registry.RecordPendingBackendOpFailure(ctx, id, errMsg) } @@ -331,9 +334,9 @@ func (d *DistributedBackendManager) DeleteBackendDetailed(ctx context.Context, n // populated from the first node seen so single-node-minded callers still work. // // Pending/offline/draining nodes are skipped because they aren't expected to -// answer NATS requests, and so are non-backend workers, which do not subscribe -// to backend.list at all; unhealthy backend nodes are still queried — -// ErrNoResponders then marks them unhealthy and the loop continues. +// answer, and so are non-backend workers, which serve no control plane at all; +// unhealthy backend nodes are still queried, and a node that does not answer is +// skipped rather than demoted. func (d *DistributedBackendManager) ListBackends() (gallery.SystemBackends, error) { result := make(gallery.SystemBackends) allNodes, err := d.registry.List(context.Background()) @@ -345,9 +348,9 @@ func (d *DistributedBackendManager) ListBackends() (gallery.SystemBackends, erro if node.Status == StatusPending || node.Status == StatusOffline || node.Status == StatusDraining { continue } - // Only backend workers subscribe to backend.list. Asking an agent - // worker can only answer "no responders", which the error handling - // below reads as a node that has gone away, so every poll of this view + // Only backend workers serve backend.list. An agent worker holds no + // tunnel, so asking one can only fail, and the failure handling used to + // read that as a node that had gone away: every poll of this view // marked every agent node unhealthy and its next heartbeat marked it // healthy again. The backend-op fan-out skips them for the same reason. if node.NodeType != "" && node.NodeType != NodeTypeBackend { @@ -355,11 +358,9 @@ func (d *DistributedBackendManager) ListBackends() (gallery.SystemBackends, erro } reply, err := d.adapter.ListBackends(node.ID) if err != nil { - if errors.Is(err, nats.ErrNoResponders) { - xlog.Warn("No NATS responders for node, marking unhealthy", "node", node.Name, "nodeID", node.ID) - d.registry.MarkUnhealthy(context.Background(), node.ID) - continue - } + // Skipped, never demoted. Listing a node's backends is a read, and + // a read this frontend could not route says nothing about whether + // the worker is there; see the fan-out above for the same rule. xlog.Warn("Failed to list backends on worker", "node", node.Name, "error", err) continue } @@ -538,7 +539,7 @@ func (d *DistributedBackendManager) InstallBackend(ctx context.Context, op *gall // worker has no platform variant for a linux-only backend) and leaves a // forever-retrying pending_backend_ops row. // -// Rolling-update fallback: when a worker returns nats.ErrNoResponders on +// Rolling-update fallback: when a worker answers that it does not serve // backend.upgrade, we try the legacy backend.install Force=true path so a // new master + old worker still converges. Drop the fallback once every // worker in the fleet is on 2026-05-08 or newer. @@ -600,8 +601,11 @@ func (d *DistributedBackendManager) UpgradeBackend(ctx context.Context, op *gall reply, err := d.adapter.UpgradeBackend(node.ID, name, string(galleriesJSON), "", "", "", 0, opID, onProgressArg) if err != nil { // Rolling-update fallback: an older worker doesn't know - // backend.upgrade. Try the legacy install-with-force path. - if errors.Is(err, nats.ErrNoResponders) { + // backend.upgrade and answers 404 for it. ONLY that answer + // triggers the fallback: a worker this frontend merely could not + // route to has said nothing, and re-firing a force-reinstall at it + // would turn a lost route into a destructive retry. + if errors.Is(err, ErrWorkerControlUnsupported) { instReply, instErr := d.adapter.installWithForceFallback(node.ID, name, string(galleriesJSON), "", "", "", 0, opID, onProgressArg) if instErr != nil { return instErr diff --git a/core/services/nodes/managers_distributed_test.go b/core/services/nodes/managers_distributed_test.go index a72707ea2871..62abd3fc9a0e 100644 --- a/core/services/nodes/managers_distributed_test.go +++ b/core/services/nodes/managers_distributed_test.go @@ -2,13 +2,11 @@ package nodes import ( "context" - "encoding/json" "errors" "runtime" "sync" "time" - "github.com/nats-io/nats.go" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "gorm.io/gorm" @@ -18,245 +16,9 @@ import ( "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/testutil" + "github.com/mudler/LocalAI/core/services/workerctl" ) -// scriptedMessagingClient maps a NATS subject to a canned reply payload -// (or error). Used so each fan-out request can simulate a different worker -// outcome without spinning up real NATS. -type scriptedMessagingClient struct { - mu sync.Mutex - replies map[string][]byte - errs map[string]error - calls []requestCall - matchedReplies map[string][]matchedReply - publishes []progressPublishCall - scheduledProgressPublishes []scheduledProgressPublish - subscribes []string -} - -// progressPublishCall records a single Publish invocation. The progress -// publisher tests assert on the sequence of BackendInstallProgressEvent -// values written to a per-op subject, so we capture both subject and the -// decoded event. Named to avoid clashing with the simpler `publishCall` -// already defined in unloader_test.go (which stores raw JSON bytes for -// non-progress assertions). -type progressPublishCall struct { - Subject string - Event messaging.BackendInstallProgressEvent -} - -// scheduledProgressPublish queues a batch of BackendInstallProgressEvent -// values to be delivered the next time Subscribe is called with the matching -// subject. This lets master-side tests assert that the adapter installs its -// handler BEFORE publishing the install request, by scripting events to be -// delivered as soon as the subscription appears. -type scheduledProgressPublish struct { - subject string - events []messaging.BackendInstallProgressEvent -} - -// matchedReply lets a test script a canned reply that only fires when the -// inbound request matches a predicate. Used by scriptReplyMatching to -// distinguish "install Force=true" (the fallback) from "install Force=false" -// on the same subject. -type matchedReply struct { - pred func(messaging.BackendInstallRequest) bool - reply []byte - fallback []byte - fallbackErr error -} - -func newScriptedMessagingClient() *scriptedMessagingClient { - return &scriptedMessagingClient{ - replies: map[string][]byte{}, - errs: map[string]error{}, - } -} - -func (s *scriptedMessagingClient) scriptReply(subject string, reply any) { - raw, err := json.Marshal(reply) - Expect(err).ToNot(HaveOccurred()) - s.mu.Lock() - defer s.mu.Unlock() - s.replies[subject] = raw -} - -func (s *scriptedMessagingClient) scriptErr(subject string, err error) { - s.mu.Lock() - defer s.mu.Unlock() - s.errs[subject] = err -} - -// scriptNoResponders scripts a nats.ErrNoResponders error for `subject` so -// tests can simulate "old worker without backend.upgrade subscription" -// scenarios. Uses the real nats sentinel so errors.Is(...) works at the -// caller (the manager's NoResponders fallback path). -func (s *scriptedMessagingClient) scriptNoResponders(subject string) { - s.mu.Lock() - defer s.mu.Unlock() - s.errs[subject] = nats.ErrNoResponders -} - -// scriptReplyMatching is like scriptReply but the canned reply only fires -// when the inbound request payload matches `pred(req)`. Lets tests -// differentiate "install with Force=true" from "install Force=false" on -// the same subject — useful for asserting the rolling-update fallback -// path actually sets Force=true on its retry. -// -// If `pred` returns false (or the unmarshal of the payload into the -// predicate's expected type fails), the subject falls through to whatever -// was scripted before (or to the unscripted default ErrNoResponders). -func (s *scriptedMessagingClient) scriptReplyMatching(subject string, pred func(messaging.BackendInstallRequest) bool, reply messaging.BackendInstallReply) { - raw, err := json.Marshal(reply) - Expect(err).ToNot(HaveOccurred()) - s.mu.Lock() - defer s.mu.Unlock() - prev := s.replies[subject] // may be nil — that's fine - prevErr := s.errs[subject] // may be nil — that's fine - if s.matchedReplies == nil { - s.matchedReplies = map[string][]matchedReply{} - } - s.matchedReplies[subject] = append(s.matchedReplies[subject], matchedReply{ - pred: pred, - reply: raw, - fallback: prev, - fallbackErr: prevErr, - }) -} - -func (s *scriptedMessagingClient) Request(subject string, data []byte, timeout time.Duration) ([]byte, error) { - s.mu.Lock() - defer s.mu.Unlock() - s.calls = append(s.calls, requestCall{Subject: subject, Data: data, Timeout: timeout}) - - // Predicate-matched replies take precedence over flat scriptReply. - if matchers, ok := s.matchedReplies[subject]; ok { - var req messaging.BackendInstallRequest - _ = json.Unmarshal(data, &req) - for _, m := range matchers { - if m.pred(req) { - return m.reply, nil - } - } - // No predicate matched — fall through to the recorded fallback - // (whatever was scripted before scriptReplyMatching took over). - if matchers[0].fallback != nil { - return matchers[0].fallback, nil - } - if matchers[0].fallbackErr != nil { - return nil, matchers[0].fallbackErr - } - // No fallback either — default to ErrNoResponders. - return nil, nats.ErrNoResponders - } - - if err, ok := s.errs[subject]; ok && err != nil { - return nil, err - } - if reply, ok := s.replies[subject]; ok { - return reply, nil - } - // Simulate ErrNoResponders for any unscripted subject so tests fail - // loudly when they forget to script a node. - return nil, &fakeNoRespondersErr{} -} - -// Publish records each call so progress-publisher tests can assert on the -// stream of events written to a subject. The real messaging.Client JSON -// encodes the payload before sending, but our publisher hands a typed -// struct directly, so we handle both shapes. -func (s *scriptedMessagingClient) Publish(subject string, data any) error { - s.mu.Lock() - defer s.mu.Unlock() - switch ev := data.(type) { - case messaging.BackendInstallProgressEvent: - s.publishes = append(s.publishes, progressPublishCall{Subject: subject, Event: ev}) - case []byte: - var e messaging.BackendInstallProgressEvent - _ = json.Unmarshal(ev, &e) - s.publishes = append(s.publishes, progressPublishCall{Subject: subject, Event: e}) - } - return nil -} - -// publishCalls returns every BackendInstallProgressEvent that was published -// to `subject`, in order. Lets tests assert on debounce behavior without -// depending on internal Publish timing. -func (s *scriptedMessagingClient) publishCalls(subject string) []messaging.BackendInstallProgressEvent { - s.mu.Lock() - defer s.mu.Unlock() - out := make([]messaging.BackendInstallProgressEvent, 0) - for _, c := range s.publishes { - if c.Subject != subject { - continue - } - out = append(out, c.Event) - } - return out -} - -// scheduleProgressPublish queues a set of BackendInstallProgressEvent values -// to be delivered on the next Subscribe call matching the per-op progress -// subject. A short delay before delivery gives the subscriber time to install -// its message handler before the events arrive. -func (s *scriptedMessagingClient) scheduleProgressPublish(nodeID, opID string, events []messaging.BackendInstallProgressEvent) { - s.mu.Lock() - defer s.mu.Unlock() - s.scheduledProgressPublishes = append(s.scheduledProgressPublishes, scheduledProgressPublish{ - subject: messaging.SubjectNodeBackendInstallProgress(nodeID, opID), - events: events, - }) -} - -// subscribeCalls returns the subjects on which Subscribe was invoked. -// Used to confirm the master skipped subscription when onProgress was nil. -func (s *scriptedMessagingClient) subscribeCalls() []string { - s.mu.Lock() - defer s.mu.Unlock() - out := make([]string, len(s.subscribes)) - copy(out, s.subscribes) - return out -} - -func (s *scriptedMessagingClient) Subscribe(subject string, handler func([]byte)) (messaging.Subscription, error) { - s.mu.Lock() - s.subscribes = append(s.subscribes, subject) - matched := []scheduledProgressPublish{} - remaining := s.scheduledProgressPublishes[:0] - for _, sp := range s.scheduledProgressPublishes { - if sp.subject == subject { - matched = append(matched, sp) - } else { - remaining = append(remaining, sp) - } - } - s.scheduledProgressPublishes = remaining - s.mu.Unlock() - - go func() { - time.Sleep(20 * time.Millisecond) - for _, sp := range matched { - for _, ev := range sp.events { - raw, _ := json.Marshal(ev) - handler(raw) - } - } - }() - - return &fakeSubscription{}, nil -} -func (s *scriptedMessagingClient) QueueSubscribe(_ string, _ string, _ func([]byte)) (messaging.Subscription, error) { - return &fakeSubscription{}, nil -} -func (s *scriptedMessagingClient) QueueSubscribeReply(_ string, _ string, _ func([]byte, func([]byte))) (messaging.Subscription, error) { - return &fakeSubscription{}, nil -} -func (s *scriptedMessagingClient) SubscribeReply(_ string, _ func([]byte, func([]byte))) (messaging.Subscription, error) { - return &fakeSubscription{}, nil -} -func (s *scriptedMessagingClient) IsConnected() bool { return true } -func (s *scriptedMessagingClient) Close() {} - // recordingNodeCall captures a single UpdateNodeProgress invocation so // per-node OpStatus tests can assert on the sequence of writes the // DistributedBackendManager fans out into the sink. @@ -292,16 +54,6 @@ func (r *recordingProgressSink) callsFor(opID, nodeID string) []galleryop.NodePr return out } -// fakeNoRespondersErr is the unscripted-subject default. It matches -// nats.ErrNoResponders by string only - used when a test forgets to script -// a node so the failure is loud but doesn't tickle errors.Is(...) sentinel -// paths the test wasn't deliberately exercising. Tests that DO want the -// real sentinel (e.g. to drive the manager's NoResponders fallback) call -// scriptNoResponders instead, which scripts nats.ErrNoResponders directly. -type fakeNoRespondersErr struct{} - -func (e *fakeNoRespondersErr) Error() string { return "no responders" } - // stubLocalBackendManager satisfies galleryop.BackendManager for the // distributed manager's `local` field. The DeleteBackend path expects to // call into local first; in distributed mode the frontend rarely has @@ -329,7 +81,7 @@ var _ = Describe("DistributedBackendManager", func() { var ( db *gorm.DB registry *NodeRegistry - mc *scriptedMessagingClient + mc *scriptedControlWorkers adapter *RemoteUnloaderAdapter mgr *DistributedBackendManager ctx context.Context @@ -344,8 +96,8 @@ var _ = Describe("DistributedBackendManager", func() { registry, err = NewNodeRegistry(db) Expect(err).ToNot(HaveOccurred()) - mc = newScriptedMessagingClient() - adapter = NewRemoteUnloaderAdapter(nil, mc, 3*time.Minute, 15*time.Minute) + mc = newScriptedControlWorkers() + adapter = NewRemoteUnloaderAdapter(nil, nil, mc.controlClient(), 3*time.Minute, 15*time.Minute) mgr = &DistributedBackendManager{ local: stubLocalBackendManager{}, adapter: adapter, @@ -385,9 +137,9 @@ var _ = Describe("DistributedBackendManager", func() { n1 := registerHealthyBackend("worker-a", "10.0.0.1:50051") n2 := registerHealthyBackend("worker-b", "10.0.0.2:50051") - mc.scriptReply(messaging.SubjectNodeBackendInstall(n1.ID), + mc.scriptReply(controlKey(n1.ID, workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:50100"}) - mc.scriptReply(messaging.SubjectNodeBackendInstall(n2.ID), + mc.scriptReply(controlKey(n2.ID, workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.2:50100"}) Expect(mgr.InstallBackend(ctx, op("vllm-development"), nil)).To(Succeed()) @@ -399,9 +151,9 @@ var _ = Describe("DistributedBackendManager", func() { n1 := registerHealthyBackend("dgx-casa", "10.0.0.1:50051") n2 := registerHealthyBackend("nvidia-thor", "10.0.0.2:50051") - mc.scriptReply(messaging.SubjectNodeBackendInstall(n1.ID), + mc.scriptReply(controlKey(n1.ID, workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: false, Error: "no child with platform linux/arm64 in index quay.io/...master-cpu-vllm"}) - mc.scriptReply(messaging.SubjectNodeBackendInstall(n2.ID), + mc.scriptReply(controlKey(n2.ID, workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: false, Error: "disk full"}) err := mgr.InstallBackend(ctx, op("vllm-development"), nil) @@ -419,9 +171,9 @@ var _ = Describe("DistributedBackendManager", func() { ok := registerHealthyBackend("worker-ok", "10.0.0.1:50051") bad := registerHealthyBackend("worker-bad", "10.0.0.2:50051") - mc.scriptReply(messaging.SubjectNodeBackendInstall(ok.ID), + mc.scriptReply(controlKey(ok.ID, workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:50100"}) - mc.scriptReply(messaging.SubjectNodeBackendInstall(bad.ID), + mc.scriptReply(controlKey(bad.ID, workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: false, Error: "out of memory"}) err := mgr.InstallBackend(ctx, op("vllm-development"), nil) @@ -437,8 +189,8 @@ var _ = Describe("DistributedBackendManager", func() { registerUnhealthyBackend("worker-a", "10.0.0.1:50051") registerUnhealthyBackend("worker-b", "10.0.0.2:50051") - // No replies scripted: if the manager tried to call Request, - // it would hit the "no responders" default and we'd see it. + // No replies scripted: if the manager issued a control RPC at + // all, the unscripted-verb default answers 500 and we'd see it. Expect(mgr.InstallBackend(ctx, op("vllm-development"), nil)).To(Succeed()) mc.mu.Lock() calls := len(mc.calls) @@ -458,11 +210,10 @@ var _ = Describe("DistributedBackendManager", func() { target := registerHealthyBackend("worker-target", "10.0.0.1:50051") other := registerHealthyBackend("worker-other", "10.0.0.2:50051") - mc.scriptReply(messaging.SubjectNodeBackendInstall(target.ID), + mc.scriptReply(controlKey(target.ID, workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:50100"}) - // No reply scripted for `other`: if InstallBackend fans out - // to it, the fakeNoRespondersErr default would surface and - // the test would fail. + // No reply scripted for `other`: if InstallBackend fans out to + // it, the unscripted-verb default surfaces and the test fails. targetedOp := &galleryop.ManagementOp[gallery.GalleryBackend, any]{ GalleryElementName: "llama-cpp", @@ -473,13 +224,13 @@ var _ = Describe("DistributedBackendManager", func() { mc.mu.Lock() defer mc.mu.Unlock() Expect(mc.calls).To(HaveLen(1)) - Expect(mc.calls[0].Subject).To(Equal(messaging.SubjectNodeBackendInstall(target.ID))) - Expect(mc.calls[0].Subject).ToNot(Equal(messaging.SubjectNodeBackendInstall(other.ID))) + Expect(mc.calls[0].Subject).To(Equal(controlKey(target.ID, workerctl.PathBackendInstall))) + Expect(mc.calls[0].Subject).ToNot(Equal(controlKey(other.ID, workerctl.PathBackendInstall))) }) }) Context("when op.TargetNodeID is set to a node that does not exist", func() { - It("returns nil without sending any NATS request", func() { + It("returns nil without sending any control request", func() { registerHealthyBackend("worker-a", "10.0.0.1:50051") ghostOp := &galleryop.ManagementOp[gallery.GalleryBackend, any]{ @@ -498,10 +249,10 @@ var _ = Describe("DistributedBackendManager", func() { It("returns galleryop.ErrWorkerStillInstalling and keeps the queue row with NextRetryAt pushed out", func() { n := registerHealthyBackend("slow", "10.0.0.1:50051") - // Script a NATS timeout on the install subject. The adapter - // wraps this into galleryop.ErrWorkerStillInstalling, which - // the manager should treat as a soft failure. - mc.scriptErr(messaging.SubjectNodeBackendInstall(n.ID), nats.ErrTimeout) + // The install RPC ends with the caller's budget spent. The + // adapter wraps that into galleryop.ErrWorkerStillInstalling, + // which the manager treats as a soft failure. + mc.scriptTimeout(n.ID) err := mgr.InstallBackend(ctx, op("vllm"), nil) Expect(err).To(HaveOccurred()) @@ -513,7 +264,8 @@ var _ = Describe("DistributedBackendManager", func() { Expect(rows).To(HaveLen(1)) Expect(rows[0].Backend).To(Equal("vllm")) // The adapter is configured with a 3m install timeout in this - // suite (NewRemoteUnloaderAdapter above). NextRetryAt should + // suite (NewRemoteUnloaderAdapter above), and the RPC failed + // instantly rather than waiting it out. NextRetryAt should // be ~now+3m; a > now+2m bound is safe-but-tight enough to // catch the buggy short default (30s exponential backoff). Expect(rows[0].NextRetryAt).To(BeTemporally(">", time.Now().Add(2*time.Minute)), @@ -521,17 +273,17 @@ var _ = Describe("DistributedBackendManager", func() { }) }) - Context("end-to-end: timeout then successful reconcile via backend.list", func() { + Context("end-to-end: budget spent, then successful reconcile via backend.list", func() { It("surfaces the install in ListBackends after the worker finishes", func() { // Use the same node-registration helper the Task 5 test uses // so the test fixture is identical to the prior context. node := registerHealthyBackend("jetson", "10.0.0.2:50051") - // First install attempt: NATS times out. The adapter wraps + // First install attempt: the budget runs out. The adapter wraps // this as galleryop.ErrWorkerStillInstalling and the manager // keeps the pending_backend_ops row alive with NextRetryAt // pushed out (asserted in the previous context). - mc.scriptErr(messaging.SubjectNodeBackendInstall(node.ID), nats.ErrTimeout) + mc.scriptTimeout(node.ID) err := mgr.InstallBackend(ctx, op("vllm"), nil) Expect(err).To(HaveOccurred()) @@ -542,10 +294,11 @@ var _ = Describe("DistributedBackendManager", func() { Expect(listErr).ToNot(HaveOccurred()) Expect(rows).To(HaveLen(1)) - // The worker finished installing in the background. Script - // backend.list on the same scriptedMessagingClient so the - // manager's ListBackends fan-out reports the backend. - mc.scriptReply(messaging.SubjectNodeBackendList(node.ID), messaging.BackendListReply{ + // The worker finished installing in the background and answers + // again, so the manager's ListBackends fan-out reports the + // backend. + mc.clearTimeout(node.ID) + mc.scriptReply(controlKey(node.ID, workerctl.PathBackendList), messaging.BackendListReply{ Backends: []messaging.NodeBackendInfo{{Name: "vllm"}}, }) @@ -569,9 +322,9 @@ var _ = Describe("DistributedBackendManager", func() { It("deletes the pending_backend_ops install row when the backend is reported installed on its target node", func() { node := registerHealthyBackend("worker-a", "10.0.0.5:50051") - // Pre-stage: simulate an admin install that timed out at the NATS - // round-trip, leaving an install row in the queue. - mc.scriptErr(messaging.SubjectNodeBackendInstall(node.ID), nats.ErrTimeout) + // Pre-stage: simulate an admin install whose control RPC ran out + // of budget, leaving an install row in the queue. + mc.scriptTimeout(node.ID) err := mgr.InstallBackend(ctx, op("vllm"), nil) Expect(err).To(HaveOccurred()) Expect(errors.Is(err, galleryop.ErrWorkerStillInstalling)).To(BeTrue()) @@ -581,7 +334,8 @@ var _ = Describe("DistributedBackendManager", func() { // Worker finishes installing in the background. backend.list now // confirms presence; ListBackends should proactively clear the row. - mc.scriptReply(messaging.SubjectNodeBackendList(node.ID), messaging.BackendListReply{ + mc.clearTimeout(node.ID) + mc.scriptReply(controlKey(node.ID, workerctl.PathBackendList), messaging.BackendListReply{ Backends: []messaging.NodeBackendInfo{{Name: "vllm"}}, }) @@ -599,7 +353,7 @@ var _ = Describe("DistributedBackendManager", func() { Expect(registry.UpsertPendingBackendOp(ctx, node.ID, "vllm", OpBackendUpgrade, []byte("[]"))).To(Succeed()) - mc.scriptReply(messaging.SubjectNodeBackendList(node.ID), messaging.BackendListReply{ + mc.scriptReply(controlKey(node.ID, workerctl.PathBackendList), messaging.BackendListReply{ Backends: []messaging.NodeBackendInfo{{Name: "vllm"}}, }) @@ -615,8 +369,8 @@ var _ = Describe("DistributedBackendManager", func() { It("invokes progressCb once per worker-published progress event", func() { node := registerHealthyBackend("worker-prog", "10.0.0.7:50051") - mc.scriptReply(messaging.SubjectNodeBackendInstall(node.ID), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.7:50051"}) - mc.scheduleProgressPublish(node.ID, "op-prog-1", []messaging.BackendInstallProgressEvent{ + mc.scriptReply(controlKey(node.ID, workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.7:50051"}) + mc.scriptProgress(controlKey(node.ID, workerctl.PathBackendInstall), []messaging.BackendInstallProgressEvent{ {OpID: "op-prog-1", NodeID: node.ID, Backend: "vllm", FileName: "vllm.tar", Current: "100 MB", Total: "1 GB", Percentage: 10}, {OpID: "op-prog-1", NodeID: node.ID, Backend: "vllm", FileName: "vllm.tar", Current: "1 GB", Total: "1 GB", Percentage: 100}, }) @@ -646,21 +400,20 @@ var _ = Describe("DistributedBackendManager", func() { }, "1s").Should(Equal(2)) mu.Lock() defer mu.Unlock() - // The adapter dispatches each progress event to its own goroutine - // (see unloader.go: `go onProgress(ev)`) so two events emitted back - // to back can land at the bridge in either order. Assert the set of - // percentages observed contains both ticks, rather than depending - // on goroutine scheduling for ordering. - pcts := []float64{pcCalls[0].Percentage, pcCalls[1].Percentage} - Expect(pcts).To(ConsistOf(10.0, 100.0)) + // ORDER, not a set. Progress lines are read off the install + // response on the caller's own goroutine, so the order the + // worker wrote them in is the order the bridge sees; the + // goroutine-per-event dispatch that made this best-effort is + // gone with the subscription it existed for. + Expect([]float64{pcCalls[0].Percentage, pcCalls[1].Percentage}).To(Equal([]float64{10.0, 100.0})) }) }) Context("InstallBackend tolerates silent (pre-Phase-2) workers", func() { It("completes successfully even when no progress events are ever published", func() { node := registerHealthyBackend("worker-silent", "10.0.0.8:50051") - mc.scriptReply(messaging.SubjectNodeBackendInstall(node.ID), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.8:50051"}) - // NO scheduleProgressPublish call - silent worker. + mc.scriptReply(controlKey(node.ID, workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.8:50051"}) + // NO scriptProgress call - silent worker. var ticks int var mu sync.Mutex @@ -695,13 +448,13 @@ var _ = Describe("DistributedBackendManager", func() { mgr = NewDistributedBackendManager(appCfg, nil, adapter, registry, sink) // stubLocalBackendManager mirrors the production behaviour // where the frontend node rarely has the backend installed - // locally - the NATS fan-out is what these specs verify. + // locally - the control-plane fan-out is what these specs verify. mgr.local = stubLocalBackendManager{} }) It("emits a success entry for each healthy node visited", func() { node := registerHealthyBackend("worker-ok", "10.0.0.9:50051") - mc.scriptReply(messaging.SubjectNodeBackendInstall(node.ID), + mc.scriptReply(controlKey(node.ID, workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.9:50051"}) opVal := op("vllm") @@ -714,9 +467,9 @@ var _ = Describe("DistributedBackendManager", func() { Expect(calls[len(calls)-1].NodeName).To(Equal("worker-ok")) }) - It("emits a running_on_worker entry when NATS times out", func() { + It("emits a running_on_worker entry when the install RPC runs out of budget", func() { node := registerHealthyBackend("worker-slow", "10.0.0.10:50051") - mc.scriptErr(messaging.SubjectNodeBackendInstall(node.ID), nats.ErrTimeout) + mc.scriptTimeout(node.ID) opVal := op("vllm") opVal.ID = "op-node-slow" @@ -730,9 +483,9 @@ var _ = Describe("DistributedBackendManager", func() { It("emits downloading entries from progress events", func() { node := registerHealthyBackend("worker-dl", "10.0.0.11:50051") - mc.scriptReply(messaging.SubjectNodeBackendInstall(node.ID), + mc.scriptReply(controlKey(node.ID, workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: true}) - mc.scheduleProgressPublish(node.ID, "op-node-dl", []messaging.BackendInstallProgressEvent{ + mc.scriptProgress(controlKey(node.ID, workerctl.PathBackendInstall), []messaging.BackendInstallProgressEvent{ {OpID: "op-node-dl", NodeID: node.ID, Backend: "vllm", FileName: "vllm.tar", Current: "1 GB", Total: "1 GB", Percentage: 100, Phase: messaging.PhaseDownloading}, }) @@ -765,13 +518,13 @@ var _ = Describe("DistributedBackendManager", func() { // upgrade should skip it. scriptInstalled := func(backend string, nodeIDs ...string) { for _, id := range nodeIDs { - mc.scriptReply(messaging.SubjectNodeBackendList(id), + mc.scriptReply(controlKey(id, workerctl.PathBackendList), messaging.BackendListReply{Backends: []messaging.NodeBackendInfo{{Name: backend}}}) } } scriptNoBackends := func(nodeIDs ...string) { for _, id := range nodeIDs { - mc.scriptReply(messaging.SubjectNodeBackendList(id), + mc.scriptReply(controlKey(id, workerctl.PathBackendList), messaging.BackendListReply{Backends: nil}) } } @@ -782,9 +535,9 @@ var _ = Describe("DistributedBackendManager", func() { n2 := registerHealthyBackend("worker-b", "10.0.0.2:50051") scriptInstalled("vllm-development", n1.ID, n2.ID) - mc.scriptReply(messaging.SubjectNodeBackendUpgrade(n1.ID), + mc.scriptReply(controlKey(n1.ID, workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: false, Error: "image manifest not found"}) - mc.scriptReply(messaging.SubjectNodeBackendUpgrade(n2.ID), + mc.scriptReply(controlKey(n2.ID, workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: false, Error: "registry unauthorized"}) err := mgr.UpgradeBackend(ctx, upgradeOp("vllm-development"), nil) @@ -800,7 +553,7 @@ var _ = Describe("DistributedBackendManager", func() { It("returns nil", func() { n1 := registerHealthyBackend("worker-a", "10.0.0.1:50051") scriptInstalled("vllm-development", n1.ID) - mc.scriptReply(messaging.SubjectNodeBackendUpgrade(n1.ID), + mc.scriptReply(controlKey(n1.ID, workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: true}) Expect(mgr.UpgradeBackend(ctx, upgradeOp("vllm-development"), nil)).To(Succeed()) }) @@ -818,7 +571,7 @@ var _ = Describe("DistributedBackendManager", func() { scriptInstalled("cpu-insightface-development", has.ID) scriptNoBackends(lacks.ID) - mc.scriptReply(messaging.SubjectNodeBackendUpgrade(has.ID), + mc.scriptReply(controlKey(has.ID, workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: true}) // Deliberately don't script SubjectNodeBackendUpgrade for `lacks`: // if the manager attempts it, the scripted-client default returns @@ -829,7 +582,7 @@ var _ = Describe("DistributedBackendManager", func() { mc.mu.Lock() defer mc.mu.Unlock() for _, call := range mc.calls { - Expect(call.Subject).ToNot(Equal(messaging.SubjectNodeBackendUpgrade(lacks.ID)), + Expect(call.Subject).ToNot(Equal(controlKey(lacks.ID, workerctl.PathBackendUpgrade)), "upgrade leaked to %s which does not have the backend installed", lacks.Name) } }) @@ -846,9 +599,9 @@ var _ = Describe("DistributedBackendManager", func() { n2 := registerHealthyBackend("worker-b", "10.0.0.2:50051") scriptInstalled("vllm-development", n1.ID, n2.ID) - mc.scriptReply(messaging.SubjectNodeBackendUpgrade(n1.ID), + mc.scriptReply(controlKey(n1.ID, workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: true}) - mc.scriptReply(messaging.SubjectNodeBackendUpgrade(n2.ID), + mc.scriptReply(controlKey(n2.ID, workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: true}) op := upgradeOp("vllm-development") @@ -859,10 +612,10 @@ var _ = Describe("DistributedBackendManager", func() { defer mc.mu.Unlock() upgraded := map[string]bool{} for _, call := range mc.calls { - if call.Subject == messaging.SubjectNodeBackendUpgrade(n1.ID) { + if call.Subject == controlKey(n1.ID, workerctl.PathBackendUpgrade) { upgraded[n1.ID] = true } - if call.Subject == messaging.SubjectNodeBackendUpgrade(n2.ID) { + if call.Subject == controlKey(n2.ID, workerctl.PathBackendUpgrade) { upgraded[n2.ID] = true } } @@ -876,7 +629,7 @@ var _ = Describe("DistributedBackendManager", func() { scriptInstalled("vllm-development", has.ID) scriptNoBackends(lacks.ID) - mc.scriptReply(messaging.SubjectNodeBackendUpgrade(has.ID), + mc.scriptReply(controlKey(has.ID, workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: true}) op := upgradeOp("vllm-development") @@ -888,9 +641,9 @@ var _ = Describe("DistributedBackendManager", func() { mc.mu.Lock() defer mc.mu.Unlock() for _, call := range mc.calls { - Expect(call.Subject).ToNot(Equal(messaging.SubjectNodeBackendUpgrade(has.ID)), + Expect(call.Subject).ToNot(Equal(controlKey(has.ID, workerctl.PathBackendUpgrade)), "a node-scoped upgrade for %s must not touch other nodes", lacks.Name) - Expect(call.Subject).ToNot(Equal(messaging.SubjectNodeBackendUpgrade(lacks.ID)), + Expect(call.Subject).ToNot(Equal(controlKey(lacks.ID, workerctl.PathBackendUpgrade)), "the target node lacks the backend; nothing should be sent") } }) @@ -908,37 +661,57 @@ var _ = Describe("DistributedBackendManager", func() { mc.mu.Lock() defer mc.mu.Unlock() for _, call := range mc.calls { - Expect(call.Subject).ToNot(Equal(messaging.SubjectNodeBackendUpgrade(n1.ID))) - Expect(call.Subject).ToNot(Equal(messaging.SubjectNodeBackendInstall(n1.ID))) + Expect(call.Subject).ToNot(Equal(controlKey(n1.ID, workerctl.PathBackendUpgrade))) + Expect(call.Subject).ToNot(Equal(controlKey(n1.ID, workerctl.PathBackendInstall))) } }) }) - // Rolling-update fallback: pre-2026-05-08 workers don't subscribe to - // backend.upgrade, so the manager catches nats.ErrNoResponders and + // Rolling-update fallback: pre-2026-05-08 workers do not serve + // backend.upgrade, so the manager catches the worker's own 404 and // re-fires the legacy backend.install Force=true on the same node. // Drop these specs once the fallback path itself is removed (see // managers_distributed.go UpgradeBackend godoc for the deprecation). Context("rolling-update fallback", func() { - It("falls back to backend.install Force=true when upgrade returns ErrNoResponders", func() { + It("falls back to backend.install Force=true when the worker does not serve the upgrade verb", func() { n := registerHealthyBackend("worker-old", "10.0.0.1:50051") scriptInstalled("vllm-development", n.ID) - // Old worker: no subscriber on backend.upgrade. - mc.scriptNoResponders(messaging.SubjectNodeBackendUpgrade(n.ID)) + // Old worker: it answers 404 for a verb it does not serve. + mc.scriptUnsupported(controlKey(n.ID, workerctl.PathBackendUpgrade)) // Fallback re-fires legacy backend.install with Force=true. - mc.scriptReplyMatching(messaging.SubjectNodeBackendInstall(n.ID), + mc.scriptReplyMatching(controlKey(n.ID, workerctl.PathBackendInstall), func(req messaging.BackendInstallRequest) bool { return req.Force }, messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:50100"}) Expect(mgr.UpgradeBackend(ctx, upgradeOp("vllm-development"), nil)).To(Succeed()) }) - It("returns the upgrade error when it is not ErrNoResponders", func() { + // The negative direction, and it is the one that matters: the + // fallback re-fires a DESTRUCTIVE force-reinstall, so it may run + // only on the worker's own "I do not serve that verb". A worker + // this frontend merely failed to reach has said nothing, and + // retrying a force-reinstall on silence is how a lost route becomes + // a reinstall of every backend on the fleet. + It("does NOT fall back to the legacy force install when the upgrade could not be routed", func() { + n := registerHealthyBackend("worker-unroutable", "10.0.0.1:50051") + scriptInstalled("vllm-development", n.ID) + // backend.upgrade is deliberately not scripted, so the worker + // fails to SERVE it rather than answering that it lacks it. + // backend.install is not scripted either, so a fallback would + // be visible in the calls below. + + err := mgr.UpgradeBackend(ctx, upgradeOp("vllm-development"), nil) + Expect(err).To(HaveOccurred()) + Expect(mc.callSubjects()).ToNot(ContainElement(controlKey(n.ID, workerctl.PathBackendInstall)), + "an unroutable upgrade must not re-fire a force-reinstall") + }) + + It("returns the upgrade error when the worker served the verb and refused", func() { n := registerHealthyBackend("worker-bad", "10.0.0.1:50051") scriptInstalled("vllm-development", n.ID) - mc.scriptReply(messaging.SubjectNodeBackendUpgrade(n.ID), + mc.scriptReply(controlKey(n.ID, workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: false, Error: "disk full"}) err := mgr.UpgradeBackend(ctx, upgradeOp("vllm-development"), nil) @@ -954,9 +727,9 @@ var _ = Describe("DistributedBackendManager", func() { n1 := registerHealthyBackend("worker-a", "10.0.0.1:50051") n2 := registerHealthyBackend("worker-b", "10.0.0.2:50051") - mc.scriptReply(messaging.SubjectNodeBackendDelete(n1.ID), + mc.scriptReply(controlKey(n1.ID, workerctl.PathBackendDelete), messaging.BackendDeleteReply{Success: false, Error: "backend not installed"}) - mc.scriptReply(messaging.SubjectNodeBackendDelete(n2.ID), + mc.scriptReply(controlKey(n2.ID, workerctl.PathBackendDelete), messaging.BackendDeleteReply{Success: false, Error: "permission denied"}) err := mgr.DeleteBackend("vllm-development") @@ -971,7 +744,7 @@ var _ = Describe("DistributedBackendManager", func() { Context("when every node succeeds", func() { It("returns nil", func() { n1 := registerHealthyBackend("worker-a", "10.0.0.1:50051") - mc.scriptReply(messaging.SubjectNodeBackendDelete(n1.ID), + mc.scriptReply(controlKey(n1.ID, workerctl.PathBackendDelete), messaging.BackendDeleteReply{Success: true}) Expect(mgr.DeleteBackend("vllm-development")).To(Succeed()) }) diff --git a/core/services/nodes/reconciler.go b/core/services/nodes/reconciler.go index 3a506905997d..9bee9219d8fe 100644 --- a/core/services/nodes/reconciler.go +++ b/core/services/nodes/reconciler.go @@ -13,7 +13,6 @@ import ( "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes/prefixcache" "github.com/mudler/xlog" - "github.com/nats-io/nats.go" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "gorm.io/gorm" @@ -405,14 +404,17 @@ func (rc *ReplicaReconciler) drainPendingBackendOps(ctx context.Context) { // Pending-op drain for admin upgrade — fires backend.upgrade so // the slow re-pull doesn't head-of-line-block install traffic on // the same worker. Falls back to the legacy backend.install - // Force=true path on nats.ErrNoResponders for old workers that - // don't subscribe to backend.upgrade yet (rolling-update window). - // Reconciler retries are background reconciliation with no live - // admin watching a progress bar, so opID/onProgress are empty — - // the adapter skips the progress subscription entirely. + // Force=true path when the worker answers that it does not serve + // backend.upgrade (rolling-update window). Reconciler retries are + // background reconciliation with no live admin watching a progress + // bar, so opID/onProgress are empty and no progress is streamed. reply, err := rc.adapter.UpgradeBackend(op.NodeID, op.Backend, string(op.Galleries), "", "", "", 0, "", nil) if err != nil { - if errors.Is(err, nats.ErrNoResponders) { + // Only the worker's own "I do not serve that verb" may + // re-fire a force-reinstall. An unroutable worker has said + // nothing, and retrying a destructive verb on silence is how a + // lost route becomes a reinstall. + if errors.Is(err, ErrWorkerControlUnsupported) { instReply, instErr := rc.adapter.installWithForceFallback(op.NodeID, op.Backend, string(op.Galleries), "", "", "", 0, "", nil) if instErr != nil { applyErr = instErr @@ -440,17 +442,14 @@ func (rc *ReplicaReconciler) drainPendingBackendOps(ctx context.Context) { continue } - // ErrNoResponders means the node has no active NATS subscription for - // this subject. Either its connection dropped, or it's the wrong - // node type entirely. Mark unhealthy so the health monitor's - // heartbeat-only pass doesn't immediately flip it back — and so - // ListDuePendingBackendOps (which filters by status=healthy) stops - // picking the row until the node genuinely recovers. - if errors.Is(applyErr, nats.ErrNoResponders) { - xlog.Warn("Reconciler: no NATS responders — marking node unhealthy", - "op", op.Op, "backend", op.Backend, "node", op.NodeID) - _ = rc.registry.MarkUnhealthy(ctx, op.NodeID) - } + // A failed op no longer demotes the node. It used to, on + // nats.ErrNoResponders, which said the worker was not on the bus; a + // control RPC fails when THIS frontend cannot route to the worker, + // which a worker re-homing its tunnel does while it is heartbeating and + // serving. Demoting on it would take the node out of + // ListDuePendingBackendOps and out of scheduling for a reason that has + // nothing to do with the node. The row keeps its backoff and its + // dead-letter cap, and cluster.Presence becomes the absence signal. // Dead-letter cap: after maxAttempts the row is the reconciler // equivalent of a poison message. Delete it loudly so the queue diff --git a/core/services/nodes/reconciler_test.go b/core/services/nodes/reconciler_test.go index b91cacc29c26..075fee25fee7 100644 --- a/core/services/nodes/reconciler_test.go +++ b/core/services/nodes/reconciler_test.go @@ -9,8 +9,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes/prefixcache" "github.com/mudler/LocalAI/core/services/testutil" + "github.com/mudler/LocalAI/core/services/workerctl" "gorm.io/gorm" ) @@ -872,6 +874,95 @@ var _ = Describe("ReplicaReconciler — state reconciliation", func() { }) }) + // The pending-op drain runs the SAME rolling-update fallback the admin + // path does, and that fallback re-fires a DESTRUCTIVE force-reinstall. + // Only the worker's own "I do not serve that verb" may trigger it. A + // worker this frontend could not reach has said nothing about the verb, + // and a BACKGROUND drain that force-reinstalls on silence is worse than + // the admin path doing it: nobody is watching, it retries on every tick, + // and a frontend replica that has just lost its tunnels would reinstall + // every queued backend on the fleet. + // + // The admin path's negative direction is pinned in + // managers_distributed_test.go; this one is the second call site of the + // same rule and was unpinned, so the condition here could be widened to + // any error with the whole suite still green. + Describe("draining a pending backend upgrade", func() { + var ( + workers *scriptedControlWorkers + node *BackendNode + rc *ReplicaReconciler + ) + + BeforeEach(func() { + workers = newScriptedControlWorkers() + node = &BackendNode{Name: "worker-drain", NodeType: NodeTypeBackend, Address: "10.0.0.9:50051"} + Expect(registry.Register(context.Background(), node, true)).To(Succeed()) + Expect(registry.UpsertPendingBackendOp(context.Background(), node.ID, "vllm", OpBackendUpgrade, []byte("[]"))).To(Succeed()) + rc = NewReplicaReconciler(ReplicaReconcilerOptions{ + Registry: registry, + Scheduler: &fakeScheduler{}, + DB: db, + Adapter: NewRemoteUnloaderAdapter(registry, nil, workers.controlClient(), time.Minute, time.Minute), + }) + }) + + queuedOps := func() []PendingBackendOp { + var rows []PendingBackendOp + Expect(db.Find(&rows).Error).To(Succeed()) + return rows + } + + It("falls back to the legacy force install when the worker does not serve the upgrade verb", func() { + workers.scriptUnsupported(controlKey(node.ID, workerctl.PathBackendUpgrade)) + workers.scriptReplyMatching(controlKey(node.ID, workerctl.PathBackendInstall), + func(req messaging.BackendInstallRequest) bool { return req.Force }, + messaging.BackendInstallReply{Success: true}) + + rc.drainPendingBackendOps(context.Background()) + + Expect(workers.callSubjects()).To(Equal([]string{ + controlKey(node.ID, workerctl.PathBackendUpgrade), + controlKey(node.ID, workerctl.PathBackendInstall), + })) + Expect(queuedOps()).To(BeEmpty(), "an op the fallback converged is drained") + }) + + // The negative direction, arranged so it cannot pass vacuously: the + // force install IS scripted and IS reachable here, so a fallback that + // fired would show up as a call AND as a drained row. What the worker + // fails to do is SERVE the upgrade verb, which is a 5xx and not the + // 404 that means it lacks it. + It("does NOT fall back to the legacy force install when the worker failed to serve the upgrade", func() { + workers.scriptReplyMatching(controlKey(node.ID, workerctl.PathBackendInstall), + func(req messaging.BackendInstallRequest) bool { return req.Force }, + messaging.BackendInstallReply{Success: true}) + + rc.drainPendingBackendOps(context.Background()) + + Expect(workers.callSubjects()).ToNot(ContainElement(controlKey(node.ID, workerctl.PathBackendInstall)), + "an upgrade the worker failed to serve must not re-fire a force-reinstall") + rows := queuedOps() + Expect(rows).To(HaveLen(1), "the op keeps its backoff and is retried, not converged") + Expect(rows[0].Attempts).To(Equal(1)) + }) + + // The same rule for the other non-answer: no route at all. The install + // is unreachable too here, so the witness is the error the drain + // RECORDED, which names the verb that actually failed. + It("records the upgrade's own failure when the worker could not be routed to", func() { + workers.scriptUnroutable(node.ID) + + rc.drainPendingBackendOps(context.Background()) + + rows := queuedOps() + Expect(rows).To(HaveLen(1)) + Expect(rows[0].LastError).To(ContainSubstring(workerctl.PathBackendUpgrade)) + Expect(rows[0].LastError).ToNot(ContainSubstring(workerctl.PathBackendInstall), + "a fallback that fired would have replaced the upgrade's error with the install's") + }) + }) + Describe("NewNodeRegistry malformed-row pruning", func() { It("drops queue rows for agent nodes and non-existent nodes on startup", func() { agent := &BackendNode{Name: "agent-1", NodeType: NodeTypeAgent, Address: "x"} diff --git a/core/services/nodes/router_liveness.go b/core/services/nodes/router_liveness.go index 88646162fde2..9809d1b2d715 100644 --- a/core/services/nodes/router_liveness.go +++ b/core/services/nodes/router_liveness.go @@ -14,14 +14,25 @@ import ( // whole fleet. const maxNodeLivenessRetries = 3 -// nodeAnswersOnBus reports whether a node still has a live subscription. +// nodeAnswersOnBus reports whether a node may still be given work. // -// Only nats.ErrNoResponders means "absent". Any other outcome, a timeout or a -// transport hiccup, leaves the node eligible: wrongly excluding a node that is -// merely slow costs real capacity, while the install that follows already -// reports its own failure. When no command sender is configured there is no bus -// to consult and every node is treated as reachable, which preserves the -// behaviour of deployments that do not run one. +// It probes the node's CONTROL PLANE, over that node's tunnel, and nothing it +// can learn there is proof of absence. A worker that answers is present by +// demonstration; every other outcome is this frontend failing to route, which +// is equally what a healthy worker re-homing its tunnel between frontend +// replicas produces, so excluding on it would demote a node that is +// heartbeating and serving. Only nats.ErrNoResponders excludes, and no control +// RPC produces it: the exclusion is therefore inert here, and stays only until +// the scheduler reads cluster.Presence, which is the one fact in the deployment +// that can say a worker is gone and says it identically on every replica. +// +// Before that carrier change this asked two NATS subjects the worker no longer +// subscribes to, so EVERY healthy worker answered "no responders" and was +// marked unhealthy on the scheduling path. +// +// When no command sender is configured there is nothing to consult and every +// node is treated as reachable, which preserves the behaviour of deployments +// that do not run one. func (r *SmartRouter) nodeAnswersOnBus(node *BackendNode) bool { if r.unloader == nil || node == nil { return true diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index a4187e94afef..3d961fbb5617 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -2,22 +2,22 @@ package nodes import ( "context" - "encoding/json" "errors" "fmt" - "strings" "time" "github.com/nats-io/nats.go" "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/xlog" ) -// NodeCommandSender abstracts NATS-based commands to worker nodes. -// Used by HTTP endpoint handlers to avoid coupling to the concrete RemoteUnloaderAdapter. +// NodeCommandSender abstracts the control commands a frontend issues to a +// worker node. They travel over the worker's tunnel as HTTP under +// workerctl.Prefix; see RemoteUnloaderAdapter. // // InstallBackend is idempotent: the worker short-circuits if the backend is // already running for the requested (modelID, replica) slot. Routine model @@ -27,8 +27,8 @@ import ( // every live process for the backend, re-pulls the gallery artifact, and // replies. Caller (DistributedBackendManager.UpgradeBackend) handles // rolling-update fallback to the legacy install Force=true path on -// nats.ErrNoResponders for old workers that don't subscribe to the new -// backend.upgrade subject. +// ErrWorkerControlUnsupported, which is what a worker older than the +// backend.upgrade verb answers. type NodeCommandSender interface { InstallBackend(nodeID, backendType, modelID, galleriesJSON, uri, name, alias string, replicaIndex int, opID string, onProgress func(messaging.BackendInstallProgressEvent)) (*messaging.BackendInstallReply, error) UpgradeBackend(nodeID, backendType, galleriesJSON, uri, name, alias string, replicaIndex int, opID string, onProgress func(messaging.BackendInstallProgressEvent)) (*messaging.BackendUpgradeReply, error) @@ -36,34 +36,46 @@ type NodeCommandSender interface { ListBackends(nodeID string) (*messaging.BackendListReply, error) StopBackend(nodeID, backend string) error UnloadModelOnNode(nodeID, modelName string) error - // PingNode reports whether the node is still subscribed on the bus. It - // returns nats.ErrNoResponders when nothing answers for the node, which is - // the only condition callers may read as "this node cannot be given work". + // PingNode reports whether the worker answered a control RPC. Only an + // answer is evidence; every other outcome is this frontend failing to + // route, which callers must not read as the node having gone. Task 6 + // replaces it with a presence read on the database clock, which is the + // only thing in this deployment that CAN say a worker is gone. PingNode(nodeID string) error } -// RemoteUnloaderAdapter implements NodeCommandSender and model.RemoteModelUnloader -// by publishing NATS events for backend process lifecycle. The worker process -// subscribes and handles the actual process start/stop. +// RemoteUnloaderAdapter implements NodeCommandSender and +// model.RemoteModelUnloader by issuing control RPCs to the worker over its +// tunnel. The worker serves them on the loopback HTTP server it already runs +// (see core/services/worker/control_routes.go) and handles the actual process +// start/stop. // -// This mirrors the local ModelLoader's startProcess()/deleteProcess() but -// over NATS for remote nodes. +// This mirrors the local ModelLoader's startProcess()/deleteProcess() but for +// remote nodes. +// +// One verb is still carried by the bus, and only for one KIND of node: +// backend.stop to an AGENT node. Agent workers hold no tunnel yet, so they have +// nothing to serve a control route on, and they subscribe to +// nodes..backend.stop to drop cached MCP sessions. Removing that publish +// would strand them; see stopBackend. type RemoteUnloaderAdapter struct { registry ModelLocator nats messaging.MessagingClient + control *ControlClient installTimeout time.Duration upgradeTimeout time.Duration } -// NewRemoteUnloaderAdapter creates a new adapter. installTimeout and -// upgradeTimeout govern the NATS request-reply deadlines for backend.install -// and backend.upgrade respectively. Use -// DistributedConfig.BackendInstallTimeoutOrDefault() / +// NewRemoteUnloaderAdapter creates a new adapter. control carries every verb +// except backend.stop to an agent node, which stays on nats. installTimeout and +// upgradeTimeout bound the backend.install and backend.upgrade RPCs +// respectively; use DistributedConfig.BackendInstallTimeoutOrDefault() / // BackendUpgradeTimeoutOrDefault() at construction. -func NewRemoteUnloaderAdapter(registry ModelLocator, nats messaging.MessagingClient, installTimeout, upgradeTimeout time.Duration) *RemoteUnloaderAdapter { +func NewRemoteUnloaderAdapter(registry ModelLocator, nats messaging.MessagingClient, control *ControlClient, installTimeout, upgradeTimeout time.Duration) *RemoteUnloaderAdapter { return &RemoteUnloaderAdapter{ registry: registry, nats: nats, + control: control, installTimeout: installTimeout, upgradeTimeout: upgradeTimeout, } @@ -93,6 +105,13 @@ const exactModelStopTimeout = 10 * time.Second // StopModelReplica stops only the process represented by replica. Configuration // cleanup intentionally has no backend.stop fallback: an old worker that does // not understand this request leaves the quarantine row for a later retry. +// +// The caller's context is carried into the RPC rather than run alongside it on +// a goroutine, which is what the request/reply carrier needed because it took a +// timeout and not a context. Abandoning the request is safe: the worker's +// model.stop handler deliberately drops the caller's context and runs the stop +// to completion, so a frontend that gives up cannot leave a half-stopped +// process or an unreturned port behind. func (a *RemoteUnloaderAdapter) StopModelReplica(ctx context.Context, nodeID string, replica NodeModel, force bool) (messaging.ModelStopReply, error) { if ctx == nil { ctx = context.Background() @@ -100,31 +119,18 @@ func (a *RemoteUnloaderAdapter) StopModelReplica(ctx context.Context, nodeID str ctx, cancel := context.WithTimeout(ctx, exactModelStopTimeout) defer cancel() - type result struct { - reply *messaging.ModelStopReply - err error - } - done := make(chan result, 1) - go func() { - reply, err := messaging.RequestJSON[messaging.ModelStopRequest, messaging.ModelStopReply](a.nats, messaging.SubjectNodeModelStop(nodeID), messaging.ModelStopRequest{ - ModelName: replica.ModelName, - ProcessKey: model.BackendProcessKey(replica.ModelName, replica.ReplicaIndex), - ExpectedAddress: replica.WorkerLocalAddress, - Force: force, - ConfigRevision: replica.ConfigRevision, - }, exactModelStopTimeout) - done <- result{reply: reply, err: err} - }() - - select { - case <-ctx.Done(): - return messaging.ModelStopReply{}, ctx.Err() - case result := <-done: - if result.err != nil { - return messaging.ModelStopReply{}, result.err - } - return *result.reply, nil + var reply messaging.ModelStopReply + err := a.control.Call(ctx, nodeID, workerctl.PathModelStop, messaging.ModelStopRequest{ + ModelName: replica.ModelName, + ProcessKey: model.BackendProcessKey(replica.ModelName, replica.ReplicaIndex), + ExpectedAddress: replica.WorkerLocalAddress, + Force: force, + ConfigRevision: replica.ConfigRevision, + }, &reply) + if err != nil { + return messaging.ModelStopReply{}, err } + return reply, nil } // UnloadRemoteModel finds the node(s) hosting the given model and tells them @@ -178,8 +184,8 @@ func (a *RemoteUnloaderAdapter) UnloadRemoteModelContext(ctx context.Context, mo continue } seenNodeIDs[node.ID] = struct{}{} - xlog.Info("Sending NATS backend.stop to node", "model", modelName, "node", node.Name, "nodeID", node.ID, "force", force) - if err := a.stopBackend(node.ID, modelName, force); err != nil { + xlog.Info("Sending backend.stop to node", "model", modelName, "node", node.Name, "nodeID", node.ID, "force", force) + if err := a.stopBackend(ctx, node.ID, node.NodeType, modelName, force); err != nil { xlog.Warn("Failed to send backend.stop", "node", node.Name, "error", err) unloadErr = errors.Join(unloadErr, fmt.Errorf("stopping model on node %s: %w", node.ID, err)) continue @@ -194,7 +200,7 @@ func (a *RemoteUnloaderAdapter) UnloadRemoteModelContext(ctx context.Context, mo return unloadErr } -// InstallBackend sends a backend.install request-reply to a worker node. +// InstallBackend asks a worker node to install a backend and start its process. // Idempotent on the worker: if the (modelID, replica) process is already // running, the worker short-circuits and returns its address; if the binary // is on disk, the worker just spawns a process; only a missing binary @@ -206,23 +212,24 @@ func (a *RemoteUnloaderAdapter) UnloadRemoteModelContext(ctx context.Context, mo // case on slow links (Jetson Wi-Fi, multi-GB CUDA images) while still // failing fast enough to surface real worker hangs. // -// For force-reinstall (admin-driven Upgrade), use UpgradeBackend instead - -// it lives on a different NATS subject so it cannot head-of-line-block -// routine load traffic on the same worker. +// Progress needs no subscription and no window to miss events in: the worker +// writes its download ticks into THIS response ahead of the terminal reply, so +// there is nothing to arrange before the request is sent. +// +// For force-reinstall (admin-driven Upgrade), use UpgradeBackend instead. func (a *RemoteUnloaderAdapter) InstallBackend( nodeID, backendType, modelID, galleriesJSON, uri, name, alias string, replicaIndex int, opID string, onProgress func(messaging.BackendInstallProgressEvent), ) (*messaging.BackendInstallReply, error) { - subject := messaging.SubjectNodeBackendInstall(nodeID) - xlog.Info("Sending NATS backend.install", "nodeID", nodeID, "backend", backendType, "modelID", modelID, "replica", replicaIndex, "opID", opID) + xlog.Info("Sending backend.install", "nodeID", nodeID, "backend", backendType, "modelID", modelID, "replica", replicaIndex, "opID", opID) - // Subscribe to the per-op progress subject BEFORE publishing the install - // request so we don't miss early events. - sub := a.subscribeProgress(nodeID, opID, onProgress) + ctx, cancel := context.WithTimeout(context.Background(), a.installTimeout) + defer cancel() - reply, err := messaging.RequestJSON[messaging.BackendInstallRequest, messaging.BackendInstallReply](a.nats, subject, messaging.BackendInstallRequest{ + var reply messaging.BackendInstallReply + err := a.control.CallStreaming(ctx, nodeID, workerctl.PathBackendInstall, messaging.BackendInstallRequest{ Backend: backendType, ModelID: modelID, BackendGalleries: galleriesJSON, @@ -231,75 +238,34 @@ func (a *RemoteUnloaderAdapter) InstallBackend( Alias: alias, ReplicaIndex: int32(replicaIndex), OpID: opID, - }, a.installTimeout) - - if sub != nil { - if unsubscribeErr := sub.Unsubscribe(); unsubscribeErr != nil { - xlog.Warn("Failed to unsubscribe from backend install progress", "nodeID", nodeID, "backend", backendType, "opID", opID, "error", unsubscribeErr) - } - } - - if err != nil && isNATSTimeout(err) { - return nil, fmt.Errorf("%w (subject=%s nodeID=%s backend=%s): %v", - galleryop.ErrWorkerStillInstalling, subject, nodeID, backendType, err) - } - return reply, err -} - -// subscribeProgress subscribes to the per-op backend-install progress subject -// so the master can stream per-node download ticks while a worker installs or -// upgrades. Returns nil (and subscribes to nothing) when onProgress is nil or -// opID is empty — the reconciler-driven retry path and legacy callers stay -// silent at no cost. Shared by InstallBackend, UpgradeBackend, and the legacy -// force-install fallback: an upgrade is a force-reinstall, so it reuses the -// install-progress subject rather than minting a new one (no new NATS -// permission, no new rolling-update compat surface). Caller must Unsubscribe -// the returned subscription after the request completes. -func (a *RemoteUnloaderAdapter) subscribeProgress(nodeID, opID string, onProgress func(messaging.BackendInstallProgressEvent)) messaging.Subscription { - if onProgress == nil || opID == "" { - return nil - } - progressSubject := messaging.SubjectNodeBackendInstallProgress(nodeID, opID) - s, subErr := a.nats.Subscribe(progressSubject, func(raw []byte) { - var ev messaging.BackendInstallProgressEvent - if err := json.Unmarshal(raw, &ev); err != nil { - xlog.Debug("malformed backend progress event", "subject", progressSubject, "error", err) - return + }, &reply, onProgress) + if err != nil { + if isRequestTimeout(err) { + return nil, fmt.Errorf("%w (nodeID=%s backend=%s): %v", + galleryop.ErrWorkerStillInstalling, nodeID, backendType, err) } - // Goroutine guard: a slow onProgress callback must not stall the NATS - // reader thread. Events spawn one goroutine each, so ordering at the - // consumer is best-effort; the worker debounces to ~250ms which dwarfs - // goroutine scheduling jitter, and its final Flush() is the terminal tick. - go onProgress(ev) - }) - if subErr != nil { - xlog.Warn("Failed to subscribe to backend progress subject; proceeding without progress streaming", - "subject", progressSubject, "error", subErr) - return nil + return nil, err } - return s + return &reply, nil } -// UpgradeBackend sends a backend.upgrade request-reply to a worker node. +// UpgradeBackend asks a worker node to force-reinstall a backend. // The worker stops every live process for this backend, force-reinstalls // from the gallery (overwriting the on-disk artifact), and replies. The // next routine InstallBackend call spawns a fresh process with the new // binary - upgrade itself does not start a process. // -// When opID is non-empty and onProgress is set, the master subscribes to the -// per-op progress subject before firing the request so a long force-reinstall -// streams per-node download ticks instead of blocking opaque at progress 0. -// // Timeout: configured via DistributedConfig.BackendUpgradeTimeoutOrDefault // (default 15m). Real-world worst case observed: 8-10 minutes for large // CUDA-l4t backend images on Jetson over WiFi. func (a *RemoteUnloaderAdapter) UpgradeBackend(nodeID, backendType, galleriesJSON, uri, name, alias string, replicaIndex int, opID string, onProgress func(messaging.BackendInstallProgressEvent)) (*messaging.BackendUpgradeReply, error) { - subject := messaging.SubjectNodeBackendUpgrade(nodeID) - xlog.Info("Sending NATS backend.upgrade", "nodeID", nodeID, "backend", backendType, "replica", replicaIndex, "opID", opID) + xlog.Info("Sending backend.upgrade", "nodeID", nodeID, "backend", backendType, "replica", replicaIndex, "opID", opID) - sub := a.subscribeProgress(nodeID, opID, onProgress) + ctx, cancel := context.WithTimeout(context.Background(), a.upgradeTimeout) + defer cancel() - reply, err := messaging.RequestJSON[messaging.BackendUpgradeRequest, messaging.BackendUpgradeReply](a.nats, subject, messaging.BackendUpgradeRequest{ + var reply messaging.BackendUpgradeReply + err := a.control.CallStreaming(ctx, nodeID, workerctl.PathBackendUpgrade, messaging.BackendUpgradeRequest{ Backend: backendType, BackendGalleries: galleriesJSON, URI: uri, @@ -307,37 +273,31 @@ func (a *RemoteUnloaderAdapter) UpgradeBackend(nodeID, backendType, galleriesJSO Alias: alias, ReplicaIndex: int32(replicaIndex), OpID: opID, - }, a.upgradeTimeout) - - if sub != nil { - if unsubscribeErr := sub.Unsubscribe(); unsubscribeErr != nil { - xlog.Warn("Failed to unsubscribe from backend upgrade progress", "nodeID", nodeID, "backend", backendType, "opID", opID, "error", unsubscribeErr) + }, &reply, onProgress) + if err != nil { + if isRequestTimeout(err) { + return nil, fmt.Errorf("%w (nodeID=%s backend=%s): %v", + galleryop.ErrWorkerStillInstalling, nodeID, backendType, err) } + return nil, err } - - if err != nil && isNATSTimeout(err) { - return nil, fmt.Errorf("%w (subject=%s nodeID=%s backend=%s): %v", - galleryop.ErrWorkerStillInstalling, subject, nodeID, backendType, err) - } - if err == nil { - a.dropStoppedReplicaRows(nodeID, "backend.upgrade", backendType, reply.StoppedProcessKeys, reply.ReportsStoppedProcesses) - } - return reply, err + a.dropStoppedReplicaRows(nodeID, "backend.upgrade", backendType, reply.StoppedProcessKeys, reply.ReportsStoppedProcesses) + return &reply, nil } // installWithForceFallback is the rolling-update fallback used by -// DistributedBackendManager.UpgradeBackend when backend.upgrade returns -// nats.ErrNoResponders (the worker is on a pre-2026-05-08 build that -// doesn't subscribe to the new subject). It re-fires the legacy -// backend.install with Force=true. Drop this once every worker is on -// 2026-05-08 or newer. +// DistributedBackendManager.UpgradeBackend when backend.upgrade reports that +// the worker does not serve that verb (a pre-2026-05-08 build). It re-fires +// the legacy backend.install with Force=true. Drop this once every worker is +// on 2026-05-08 or newer. func (a *RemoteUnloaderAdapter) installWithForceFallback(nodeID, backendType, galleriesJSON, uri, name, alias string, replicaIndex int, opID string, onProgress func(messaging.BackendInstallProgressEvent)) (*messaging.BackendInstallReply, error) { - subject := messaging.SubjectNodeBackendInstall(nodeID) xlog.Warn("Falling back to legacy backend.install Force=true (old worker)", "nodeID", nodeID, "backend", backendType) - sub := a.subscribeProgress(nodeID, opID, onProgress) + ctx, cancel := context.WithTimeout(context.Background(), a.upgradeTimeout) + defer cancel() - reply, err := messaging.RequestJSON[messaging.BackendInstallRequest, messaging.BackendInstallReply](a.nats, subject, messaging.BackendInstallRequest{ + var reply messaging.BackendInstallReply + err := a.control.CallStreaming(ctx, nodeID, workerctl.PathBackendInstall, messaging.BackendInstallRequest{ Backend: backendType, BackendGalleries: galleriesJSON, URI: uri, @@ -346,160 +306,176 @@ func (a *RemoteUnloaderAdapter) installWithForceFallback(nodeID, backendType, ga ReplicaIndex: int32(replicaIndex), Force: true, OpID: opID, - }, a.upgradeTimeout) - - if sub != nil { - if unsubscribeErr := sub.Unsubscribe(); unsubscribeErr != nil { - xlog.Warn("Failed to unsubscribe from legacy backend install progress", "nodeID", nodeID, "backend", backendType, "opID", opID, "error", unsubscribeErr) + }, &reply, onProgress) + if err != nil { + if isRequestTimeout(err) { + return nil, fmt.Errorf("%w (nodeID=%s backend=%s): %v", + galleryop.ErrWorkerStillInstalling, nodeID, backendType, err) } + return nil, err } - - if err != nil && isNATSTimeout(err) { - return nil, fmt.Errorf("%w (subject=%s nodeID=%s backend=%s): %v", - galleryop.ErrWorkerStillInstalling, subject, nodeID, backendType, err) - } - return reply, err + return &reply, nil } -// ListBackends queries a worker node for its installed backends via NATS request-reply. +// Control-RPC budgets. Each is the deadline the corresponding NATS +// request/reply carried, kept unchanged so this cutover changes the carrier and +// not how long the frontend waits. +const ( + backendListTimeout = 30 * time.Second + modelsRunningTimeout = 10 * time.Second + backendStopTimeout = 30 * time.Second + backendDeleteTimeout = 2 * time.Minute + modelUnloadTimeout = 30 * time.Second + modelDeleteTimeout = 30 * time.Second + nodeStopTimeout = 30 * time.Second + nodePingTimeout = 5 * time.Second +) + +// ListBackends queries a worker node for its installed backends. func (a *RemoteUnloaderAdapter) ListBackends(nodeID string) (*messaging.BackendListReply, error) { - subject := messaging.SubjectNodeBackendList(nodeID) - xlog.Debug("Sending NATS backend.list", "nodeID", nodeID) + xlog.Debug("Sending backend.list", "nodeID", nodeID) - return messaging.RequestJSON[messaging.BackendListRequest, messaging.BackendListReply](a.nats, subject, messaging.BackendListRequest{}, 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), backendListTimeout) + defer cancel() + + var reply messaging.BackendListReply + if err := a.control.Call(ctx, nodeID, workerctl.PathBackendList, messaging.BackendListRequest{}, &reply); err != nil { + return nil, err + } + return &reply, nil } -// PingNode checks that a worker still has a live subscription on the bus. +// PingNode asks a worker one cheap control verb and reports what came back. // // A node's status in the database comes from its HTTP heartbeat, which is a -// separate channel from NATS. A worker that has died stops answering on NATS -// at once but keeps its healthy status until the heartbeat ages out, so the -// scheduler could pick a node that could not be given work and the request -// failed with "no responders available". +// separate channel, so a worker that has died keeps its healthy status until +// the heartbeat ages out and the scheduler could commit to a node it cannot +// reach. // -// The subject asked has to be one every worker in the fleet subscribes to, or -// this check condemns the workers that do not. models.running was the obvious -// choice and the wrong one: it arrived in 4.6, so a 4.5 worker that is alive -// and serving never answers it, and a model pinned to that node could never be -// scheduled. backend.list has been part of the worker protocol far longer, so -// it is the safer question to ask. +// What this can and cannot conclude changed with the carrier, and the +// difference is the whole point. A worker that ANSWERS is present by +// demonstration, whatever it answered. Every other outcome is this frontend +// failing to route to it, which is true of a worker that is heartbeating and +// serving another replica's requests while re-homing its tunnel; reported as +// absence it would have the scheduler demote a healthy node. So the error is +// ErrWorkerUnroutable, callers must not read it as the node having gone, and +// nothing here returns an absence sentinel at all. Task 6 replaces this with +// cluster.Registry.Presence, which is the one fact in the deployment that can +// say a worker is gone, and says it identically on every replica. // -// A worker that answers anything is alive. Only when every subject reports no -// responders is the node treated as absent, so adding a newer subject here can -// never condemn an older worker. +// backend.list rather than models.running for the reason it always was: it has +// been part of the worker protocol far longer, so it is the safer question to +// ask of a worker that may be older than this frontend. func (a *RemoteUnloaderAdapter) PingNode(nodeID string) error { - subjects := []string{ - messaging.SubjectNodeBackendList(nodeID), - messaging.SubjectNodeModelsRunning(nodeID), - } - var lastErr error - for _, subject := range subjects { - _, err := messaging.RequestJSON[messaging.BackendListRequest, messaging.BackendListReply]( - a.nats, subject, messaging.BackendListRequest{}, 5*time.Second) - if err == nil { - return nil - } - if !errors.Is(err, nats.ErrNoResponders) { - // Reached someone, or failed for a reason that is not absence. - // Either way the node is not proven gone. - return nil - } - lastErr = err + ctx, cancel := context.WithTimeout(context.Background(), nodePingTimeout) + defer cancel() + + var reply messaging.BackendListReply + err := a.control.Call(ctx, nodeID, workerctl.PathBackendList, messaging.BackendListRequest{}, &reply) + if errors.Is(err, ErrWorkerControlUnsupported) { + // The worker answered, from a build that predates this verb. That is a + // worker that is very much there. + return nil } - return lastErr + return err } // ListRunningModels asks a worker node which model backend processes it -// currently has running, via NATS request-reply. +// currently has running. // // The timeout is short on purpose: the worker answers straight out of its // in-memory process table, so a slow reply means the worker itself is in // trouble, and the caller treats no-answer as "don't know" rather than as // "nothing running". func (a *RemoteUnloaderAdapter) ListRunningModels(nodeID string) (*messaging.ModelsRunningReply, error) { - subject := messaging.SubjectNodeModelsRunning(nodeID) - return messaging.RequestJSON[messaging.ModelsRunningRequest, messaging.ModelsRunningReply]( - a.nats, subject, messaging.ModelsRunningRequest{}, 10*time.Second) -} + ctx, cancel := context.WithTimeout(context.Background(), modelsRunningTimeout) + defer cancel() -// backendStopAckTimeout bounds the wait for a worker's backend.stop reply. -// -// A single stop is bounded by the worker's 5s best-effort Free plus the kill, -// so this leaves comfortable headroom. It is also the stall a worker that -// predates the reply imposes on every stop, which is why it is not generous: -// such a worker performs the stop and simply never answers, so the controller -// waits out the full budget before falling back to the old assumption. -const backendStopAckTimeout = 15 * time.Second + var reply messaging.ModelsRunningReply + if err := a.control.Call(ctx, nodeID, workerctl.PathModelsRunning, messaging.ModelsRunningRequest{}, &reply); err != nil { + return nil, err + } + return &reply, nil +} // StopBackend tells a worker node to stop a specific gRPC backend process. // If backend is empty, the worker stops ALL backends. // The node stays registered and can receive another InstallBackend later. func (a *RemoteUnloaderAdapter) StopBackend(nodeID, backend string) error { - return a.stopBackend(nodeID, backend, false) + ctx, cancel := context.WithTimeout(context.Background(), backendStopTimeout) + defer cancel() + return a.stopBackend(ctx, nodeID, a.nodeTypeOf(ctx, nodeID), backend, false) } -// stopBackend asks a worker to stop a backend and waits for it to say what it -// did. -// -// This was a bare Publish, which returned nil as soon as the local publish -// succeeded and so reported success for a stop that killed nothing or failed -// outright. The unload endpoint answered HTTP 200 while the backend kept -// running and holding its VRAM, and the endpoint's own "backend stop failed" -// branch was unreachable. +// nodeTypeOf answers which KIND of worker a node id names, so stopBackend can +// pick the carrier that node actually listens on. // -// A silent worker is NOT treated as a failure. A worker that predates -// BackendStopReply still receives the request and still stops the backend; it -// only lacks the reply. Failing here would break every stop against a fleet -// that has not been upgraded yet, so a timeout degrades to the previous -// assumption and says so in the log. +// A lookup that fails answers NodeTypeBackend, matching the column's own +// default and the "empty means backend" reading every other node-type branch in +// this package takes. The failure directions are not symmetric: sending a +// backend node's stop over the bus is silently lost, because nothing subscribes +// to it any more, while sending an agent node's stop over the tunnel returns an +// error the caller sees. +func (a *RemoteUnloaderAdapter) nodeTypeOf(ctx context.Context, nodeID string) string { + if a.registry == nil { + return NodeTypeBackend + } + node, err := a.registry.Get(ctx, nodeID) + if err != nil || node == nil { + xlog.Debug("Could not resolve node type for a backend stop; assuming a backend worker", + "nodeID", nodeID, "error", err) + return NodeTypeBackend + } + return node.NodeType +} + +// stopBackend sends one backend.stop, over the carrier that kind of worker +// listens on. // -// Only silence degrades. A transport error — the connection is closed, nothing -// subscribes to this node's subject at all — is reported, because under the -// old Publish it was reported too, and callers such as UnloadRemoteModel rely -// on that to skip the registry cleanup for a node they could not reach. -func (a *RemoteUnloaderAdapter) stopBackend(nodeID, backend string, force bool) error { - subject := messaging.SubjectNodeBackendStop(nodeID) - req := messaging.BackendStopRequest{Backend: backend, Force: force} - - reply, err := messaging.RequestJSON[messaging.BackendStopRequest, messaging.BackendStopReply]( - a.nats, subject, req, backendStopAckTimeout) - if err != nil { - if errors.Is(err, nats.ErrTimeout) { - xlog.Warn("Worker did not acknowledge backend.stop; assuming an older worker delivered it", - "nodeID", nodeID, "backend", backend, "force", force) - return nil +// An AGENT node keeps the bus. It holds no tunnel, so it has no control route +// to serve, and it subscribes to nodes..backend.stop to drop the MCP +// sessions cached for a backend that is going away. This is the one verb of the +// ten that is split rather than moved, and the split is the honest intermediate +// state until agent workers hold tunnels too. +func (a *RemoteUnloaderAdapter) stopBackend(ctx context.Context, nodeID, nodeType, backend string, force bool) error { + if nodeType == NodeTypeAgent { + subject := messaging.SubjectNodeBackendStop(nodeID) + if backend == "" && !force { + return a.nats.Publish(subject, nil) } - return fmt.Errorf("backend.stop on node %s: %w", nodeID, err) + return a.nats.Publish(subject, messaging.BackendStopRequest{Backend: backend, Force: force}) + } + // An empty Backend is what the worker reads as "stop everything", the same + // meaning the bus carried as an empty payload; see decodeBackendStopRequest. + var reply messaging.BackendStopReply + if err := a.control.Call(ctx, nodeID, workerctl.PathBackendStop, + messaging.BackendStopRequest{Backend: backend, Force: force}, &reply); err != nil { + return err } if !reply.Success { return fmt.Errorf("backend.stop on node %s: %s", nodeID, reply.Error) } - // An empty list from a worker that enumerates what it stopped is the answer - // to "was anything actually running under that name" — and the answer is - // no. That is not an error: stopping a backend that is not running leaves - // the caller in the state it asked for. It is worth saying out loud, - // because a caller that expected to reclaim VRAM did not. if reply.ReportsStoppedProcesses && len(reply.StoppedProcessKeys) == 0 { - xlog.Warn("backend.stop matched no running process on the worker", - "nodeID", nodeID, "backend", backend) + xlog.Warn("backend.stop matched no running process on the worker", "nodeID", nodeID, "backend", backend) return nil } - xlog.Info("Worker stopped backend processes", - "nodeID", nodeID, "backend", backend, "stopped", reply.StoppedProcessKeys) + xlog.Info("Worker stopped backend processes", "nodeID", nodeID, "backend", backend, "stopped", reply.StoppedProcessKeys) return nil } // DeleteBackend tells a worker node to delete a backend (stop + remove files). func (a *RemoteUnloaderAdapter) DeleteBackend(nodeID, backendName string) (*messaging.BackendDeleteReply, error) { - subject := messaging.SubjectNodeBackendDelete(nodeID) - xlog.Info("Sending NATS backend.delete", "nodeID", nodeID, "backend", backendName) + xlog.Info("Sending backend.delete", "nodeID", nodeID, "backend", backendName) - reply, err := messaging.RequestJSON[messaging.BackendDeleteRequest, messaging.BackendDeleteReply](a.nats, subject, messaging.BackendDeleteRequest{Backend: backendName}, 2*time.Minute) - if err != nil { - return reply, err + ctx, cancel := context.WithTimeout(context.Background(), backendDeleteTimeout) + defer cancel() + + var reply messaging.BackendDeleteReply + if err := a.control.Call(ctx, nodeID, workerctl.PathBackendDelete, messaging.BackendDeleteRequest{Backend: backendName}, &reply); err != nil { + return nil, err } a.dropStoppedReplicaRows(nodeID, "backend.delete", backendName, reply.StoppedProcessKeys, reply.ReportsStoppedProcesses) - return reply, nil + return &reply, nil } // dropStoppedReplicaRows removes the NodeModel rows addressing processes a @@ -548,11 +524,13 @@ func (a *RemoteUnloaderAdapter) dropStoppedReplicaRows(nodeID, op, backendName s // UnloadModelOnNode sends a model.unload request to a specific node. // The worker calls gRPC Free() to release GPU memory. func (a *RemoteUnloaderAdapter) UnloadModelOnNode(nodeID, modelName string) error { - subject := messaging.SubjectNodeModelUnload(nodeID) - xlog.Info("Sending NATS model.unload", "nodeID", nodeID, "model", modelName) + xlog.Info("Sending model.unload", "nodeID", nodeID, "model", modelName) - reply, err := messaging.RequestJSON[messaging.ModelUnloadRequest, messaging.ModelUnloadReply](a.nats, subject, messaging.ModelUnloadRequest{ModelName: modelName}, 30*time.Second) - if err != nil { + ctx, cancel := context.WithTimeout(context.Background(), modelUnloadTimeout) + defer cancel() + + var reply messaging.ModelUnloadReply + if err := a.control.Call(ctx, nodeID, workerctl.PathModelUnload, messaging.ModelUnloadRequest{ModelName: modelName}, &reply); err != nil { return err } if !reply.Success { @@ -564,18 +542,20 @@ func (a *RemoteUnloaderAdapter) UnloadModelOnNode(nodeID, modelName string) erro // DeleteModelFiles sends model.delete to all nodes that have the model cached. // This removes model files from worker disks. func (a *RemoteUnloaderAdapter) DeleteModelFiles(modelName string) error { - nodes, err := a.registry.FindNodesWithModel(context.Background(), modelName) + ctx, cancel := context.WithTimeout(context.Background(), modelDeleteTimeout) + defer cancel() + + nodes, err := a.registry.FindNodesWithModel(ctx, modelName) if err != nil || len(nodes) == 0 { xlog.Debug("No nodes with model for file deletion", "model", modelName) return nil } for _, node := range nodes { - subject := messaging.SubjectNodeModelDelete(node.ID) - xlog.Info("Sending NATS model.delete", "nodeID", node.ID, "model", modelName) + xlog.Info("Sending model.delete", "nodeID", node.ID, "model", modelName) - reply, err := messaging.RequestJSON[messaging.ModelDeleteRequest, messaging.ModelDeleteReply](a.nats, subject, messaging.ModelDeleteRequest{ModelName: modelName}, 30*time.Second) - if err != nil { + var reply messaging.ModelDeleteReply + if err := a.control.Call(ctx, node.ID, workerctl.PathModelDelete, messaging.ModelDeleteRequest{ModelName: modelName}, &reply); err != nil { xlog.Warn("model.delete failed on node", "node", node.Name, "error", err) continue } @@ -588,17 +568,19 @@ func (a *RemoteUnloaderAdapter) DeleteModelFiles(modelName string) error { // StopNode tells a worker node to shut down entirely (deregister + exit). func (a *RemoteUnloaderAdapter) StopNode(nodeID string) error { - subject := messaging.SubjectNodeStop(nodeID) - return a.nats.Publish(subject, nil) + ctx, cancel := context.WithTimeout(context.Background(), nodeStopTimeout) + defer cancel() + return a.control.Call(ctx, nodeID, workerctl.PathNodeStop, struct{}{}, nil) } -// isNATSTimeout returns true if err looks like a NATS request-reply timeout. -// nats.ErrTimeout is the canonical sentinel; context.DeadlineExceeded can -// also surface depending on the client's path; we accept both, plus a -// string-match fallback for clients that return a bare error. -func isNATSTimeout(err error) bool { - if errors.Is(err, nats.ErrTimeout) || errors.Is(err, context.DeadlineExceeded) { - return true - } - return err != nil && strings.Contains(err.Error(), "nats: timeout") +// isRequestTimeout reports whether a control RPC ended because its budget ran +// out rather than because the worker said anything. +// +// context.DeadlineExceeded is the one signal, and it is matchable because +// controlFailure wraps the caller's own ctx.Err(). nats.ErrTimeout is still +// accepted for as long as any verb reaches a worker over the bus. The string +// match the NATS carrier needed is deliberately NOT reproduced: a message that +// merely quotes a timeout is not one. +func isRequestTimeout(err error) bool { + return errors.Is(err, nats.ErrTimeout) || errors.Is(err, context.DeadlineExceeded) } diff --git a/core/services/nodes/unloader_ping_test.go b/core/services/nodes/unloader_ping_test.go index a9b3a5889469..ae2f594a060f 100644 --- a/core/services/nodes/unloader_ping_test.go +++ b/core/services/nodes/unloader_ping_test.go @@ -1,51 +1,134 @@ package nodes import ( + "context" "errors" "time" + "github.com/nats-io/nats.go" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/nats-io/nats.go" + "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" ) -// The scheduler's liveness probe asks a worker a question over NATS and treats -// "no responders" as proof the worker is gone. That is only sound if every -// worker in the fleet subscribes to the subject asked. +// The scheduler asks a worker one cheap question before committing work to it, +// and then acts on the answer by DEMOTING the node. That is only sound if the +// question is one every worker in the fleet answers. // -// It originally asked models.running, which arrived in 4.6. A 4.5 worker is -// perfectly alive and serving, answers backend.list, and never subscribes to -// models.running, so the probe condemned it on every scheduling attempt. A -// model pinned to such a node could then never be placed at all. -var _ = Describe("Node liveness probe subject", func() { +// It has been the wrong question twice. It first asked models.running, which +// arrived in 4.6, so a 4.5 worker that was alive and serving never answered and +// a model pinned to it could never be placed. It then asked two NATS subjects +// the worker had stopped subscribing to when its control plane moved onto the +// tunnel, so EVERY healthy worker answered "no responders" and was marked +// unhealthy on the scheduling path. +// +// These specs pin both halves of the answer: what counts as the worker +// speaking, and what must never be read as the worker being gone. +var _ = Describe("Node liveness probe over the control plane", func() { var ( - mc *scriptedMessagingClient + workers *scriptedControlWorkers adapter *RemoteUnloaderAdapter ) const nodeID = "11111111-2222-3333-4444-555555555555" BeforeEach(func() { - mc = newScriptedMessagingClient() - adapter = NewRemoteUnloaderAdapter(nil, mc, 3*time.Minute, 15*time.Minute) + workers = newScriptedControlWorkers() + adapter = NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) }) It("treats a worker that answers backend.list as alive", func() { - // A worker old enough to predate models.running: it answers the - // long-standing backend.list subject and nothing else. - mc.scriptReply(messaging.SubjectNodeBackendList(nodeID), messaging.BackendListReply{}) - mc.scriptNoResponders(messaging.SubjectNodeModelsRunning(nodeID)) + workers.scriptReply(controlKey(nodeID, workerctl.PathBackendList), messaging.BackendListReply{}) + + Expect(adapter.PingNode(nodeID)).To(Succeed()) + }) + + It("treats a worker too old to serve the verb as alive, since it answered", func() { + // A 404 under the control prefix is the worker saying it is older than + // this frontend. It is a deployment fact, and a worker that can state + // one is a worker that is there. + workers.scriptUnsupported(controlKey(nodeID, workerctl.PathBackendList)) + + Expect(adapter.PingNode(nodeID)).To(Succeed()) + }) + + It("reports a worker it cannot route to as unroutable, and never as absent", func() { + workers.scriptUnroutable(nodeID) + + err := adapter.PingNode(nodeID) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) + // The two things a caller must not be able to conclude from it: that + // the worker spoke, and that it is gone. + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + Expect(errors.Is(err, nats.ErrNoResponders)).To(BeFalse()) + }) + + It("refuses without a tunnel dialer rather than reaching for an address", func() { + plain := NewRemoteUnloaderAdapter(nil, nil, NewControlClient(nil, "tok"), time.Minute, time.Minute) + + err := plain.PingNode(nodeID) + Expect(err).To(MatchError(ErrNoWorkerDialer)) + Expect(err).To(MatchError(ErrWorkerUnroutable)) + }) +}) + +// The merge gate this task exists to clear, asserted through the SCHEDULING +// path with the real adapter rather than through the router's own double. +// +// router_liveness_test.go drives a test-owned map of dead nodes and never +// touches a transport, so it stayed green for the whole window in which every +// healthy worker read as absent. These specs run pickReachableNode against a +// real RemoteUnloaderAdapter and a worker that answers over its control plane, +// which is the only arrangement that can see the difference. +var _ = Describe("Scheduling onto a worker reached over its control plane", func() { + var ( + workers *scriptedControlWorkers + reg *fakeModelRouter + router *SmartRouter + ) + + const nodeID = "22222222-3333-4444-5555-666666666666" + + BeforeEach(func() { + workers = newScriptedControlWorkers() + reg = &fakeModelRouter{} + router = NewSmartRouter(reg, SmartRouterOptions{ + Unloader: NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), time.Minute, time.Minute), + }) + }) + + selectOnce := func(node *BackendNode) func() *BackendNode { + handed := false + return func() *BackendNode { + if handed { + return nil + } + handed = true + return node + } + } + + It("schedules onto a healthy worker instead of demoting it", func() { + node := &BackendNode{ID: nodeID, Name: "healthy", Address: "10.0.0.1:50051"} + workers.scriptReply(controlKey(nodeID, workerctl.PathBackendList), messaging.BackendListReply{}) - Expect(errors.Is(adapter.PingNode(nodeID), nats.ErrNoResponders)).To(BeFalse(), - "a worker answering backend.list is alive regardless of newer subjects") + Expect(router.pickReachableNode(context.Background(), selectOnce(node))).To(Equal(node)) + Expect(reg.markedUnhealthy).To(BeEmpty()) }) - It("still reports a worker that answers nothing as absent", func() { - mc.scriptNoResponders(messaging.SubjectNodeBackendList(nodeID)) - mc.scriptNoResponders(messaging.SubjectNodeModelsRunning(nodeID)) + It("schedules onto a worker this frontend could not route to, rather than demoting it", func() { + // A worker re-homing its tunnel between frontend replicas, or one whose + // owning replica is restarting, is unroutable from here while it is + // heartbeating and serving. Excluding it costs capacity; demoting it + // tells every other scheduler the same wrong thing. + node := &BackendNode{ID: nodeID, Name: "re-homing", Address: "10.0.0.2:50051"} + workers.scriptUnroutable(nodeID) - Expect(errors.Is(adapter.PingNode(nodeID), nats.ErrNoResponders)).To(BeTrue()) + Expect(router.pickReachableNode(context.Background(), selectOnce(node))).To(Equal(node)) + Expect(reg.markedUnhealthy).To(BeEmpty()) }) }) diff --git a/core/services/nodes/unloader_stale_rows_test.go b/core/services/nodes/unloader_stale_rows_test.go index 0fff87bafe20..9be312917aa1 100644 --- a/core/services/nodes/unloader_stale_rows_test.go +++ b/core/services/nodes/unloader_stale_rows_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/gomega" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" ) // Replies are handed to the adapter as raw JSON rather than as marshalled @@ -18,14 +19,16 @@ import ( var _ = Describe("RemoteUnloaderAdapter stale replica rows", func() { var ( locator *fakeModelLocator - mc *fakeMessagingClient + workers *scriptedControlWorkers adapter *RemoteUnloaderAdapter ) + const nodeID = "node-1" + BeforeEach(func() { locator = &fakeModelLocator{} - mc = &fakeMessagingClient{} - adapter = NewRemoteUnloaderAdapter(locator, mc, 3*time.Minute, 15*time.Minute) + workers = newScriptedControlWorkers() + adapter = NewRemoteUnloaderAdapter(locator, nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) }) Describe("DeleteBackend", func() { @@ -35,11 +38,11 @@ var _ = Describe("RemoteUnloaderAdapter stale replica rows", func() { // addresses will pass probeHealth as soon as an unrelated backend // binds the recycled port, and the request is then silently served // by the wrong backend. - mc.requestReply = []byte(`{ + workers.scriptRawReply(controlKey(nodeID, workerctl.PathBackendDelete), []byte(`{ "success": true, "reports_stopped_processes": true, "stopped_process_keys": ["qwen3-0.6b#0", "qwen3-0.6b#2"] - }`) + }`)) reply, err := adapter.DeleteBackend("node-1", "llama-cpp") Expect(err).NotTo(HaveOccurred()) @@ -56,11 +59,11 @@ var _ = Describe("RemoteUnloaderAdapter stale replica rows", func() { // contain '#' themselves, so the split must be anchored at the last // separator. Splitting at the first one addresses a row that does // not exist and leaves the real stale row in place. - mc.requestReply = []byte(`{ + workers.scriptRawReply(controlKey(nodeID, workerctl.PathBackendDelete), []byte(`{ "success": true, "reports_stopped_processes": true, "stopped_process_keys": ["weird#name#3"] - }`) + }`)) _, err := adapter.DeleteBackend("node-1", "llama-cpp") Expect(err).NotTo(HaveOccurred()) @@ -70,7 +73,7 @@ var _ = Describe("RemoteUnloaderAdapter stale replica rows", func() { It("removes nothing when a worker that reports stopped processes stopped none", func() { // Deleting a backend that was never loaded is routine. The reply is // authoritative here, so "no keys" genuinely means "no rows". - mc.requestReply = []byte(`{"success": true, "reports_stopped_processes": true}`) + workers.scriptRawReply(controlKey(nodeID, workerctl.PathBackendDelete), []byte(`{"success": true, "reports_stopped_processes": true}`)) _, err := adapter.DeleteBackend("node-1", "llama-cpp") Expect(err).NotTo(HaveOccurred()) @@ -84,7 +87,7 @@ var _ = Describe("RemoteUnloaderAdapter stale replica rows", func() { // not guess at rows to delete. It falls back to the probe-based // self-heal in SmartRouter.probeHealth, which is exactly the // pre-change behavior. - mc.requestReply = []byte(`{"success": true}`) + workers.scriptRawReply(controlKey(nodeID, workerctl.PathBackendDelete), []byte(`{"success": true}`)) reply, err := adapter.DeleteBackend("node-1", "llama-cpp") Expect(err).NotTo(HaveOccurred()) @@ -98,11 +101,11 @@ var _ = Describe("RemoteUnloaderAdapter stale replica rows", func() { // The worker aborts the delete without listing the key it failed to // kill: that process survived, so its address is still correct and // dropping the row would force a needless reload of a live replica. - mc.requestReply = []byte(`{ + workers.scriptRawReply(controlKey(nodeID, workerctl.PathBackendDelete), []byte(`{ "success": false, "error": "could not stop running process qwen3-0.6b#0", "reports_stopped_processes": true - }`) + }`)) reply, err := adapter.DeleteBackend("node-1", "llama-cpp") Expect(err).NotTo(HaveOccurred()) @@ -115,12 +118,12 @@ var _ = Describe("RemoteUnloaderAdapter stale replica rows", func() { // so a failure further along (removing files, re-registering) does // not make the already-recycled ports any less dangerous. Gating // removal on overall success would strand exactly those rows. - mc.requestReply = []byte(`{ + workers.scriptRawReply(controlKey(nodeID, workerctl.PathBackendDelete), []byte(`{ "success": false, "error": "failed to delete backend files", "reports_stopped_processes": true, "stopped_process_keys": ["qwen3-0.6b#0"] - }`) + }`)) _, err := adapter.DeleteBackend("node-1", "llama-cpp") Expect(err).NotTo(HaveOccurred()) @@ -128,11 +131,11 @@ var _ = Describe("RemoteUnloaderAdapter stale replica rows", func() { }) It("ignores malformed process keys instead of removing a wrong row", func() { - mc.requestReply = []byte(`{ + workers.scriptRawReply(controlKey(nodeID, workerctl.PathBackendDelete), []byte(`{ "success": true, "reports_stopped_processes": true, "stopped_process_keys": ["no-replica-suffix", "qwen3-0.6b#notanumber", "good#1"] - }`) + }`)) _, err := adapter.DeleteBackend("node-1", "llama-cpp") Expect(err).NotTo(HaveOccurred()) @@ -145,11 +148,11 @@ var _ = Describe("RemoteUnloaderAdapter stale replica rows", func() { // An upgrade force-stops every process using the binary and starts // none of them back up, so it recycles ports exactly as delete does // while leaving the same rows behind. - mc.requestReply = []byte(`{ + workers.scriptRawReply(controlKey(nodeID, workerctl.PathBackendUpgrade), []byte(`{ "success": true, "reports_stopped_processes": true, "stopped_process_keys": ["whisper#0", "whisper#1"] - }`) + }`)) reply, err := adapter.UpgradeBackend("node-1", "whisper", "", "", "", "", 0, "", nil) Expect(err).NotTo(HaveOccurred()) @@ -162,7 +165,7 @@ var _ = Describe("RemoteUnloaderAdapter stale replica rows", func() { }) It("does not read an old worker's silence as a completed cleanup", func() { - mc.requestReply = []byte(`{"success": true}`) + workers.scriptRawReply(controlKey(nodeID, workerctl.PathBackendUpgrade), []byte(`{"success": true}`)) reply, err := adapter.UpgradeBackend("node-1", "whisper", "", "", "", "", 0, "", nil) Expect(err).NotTo(HaveOccurred()) diff --git a/core/services/nodes/unloader_test.go b/core/services/nodes/unloader_test.go index f801ff529558..b83d8abae49e 100644 --- a/core/services/nodes/unloader_test.go +++ b/core/services/nodes/unloader_test.go @@ -4,16 +4,15 @@ import ( "context" "encoding/json" "errors" - "fmt" "sync" "time" - "github.com/nats-io/nats.go" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" ) // --- Fakes --- @@ -22,6 +21,7 @@ import ( type fakeModelLocator struct { nodes []BackendNode findErr error + getErr error removedPairs []modelNodePair // records RemoveNodeModel calls removedReplicas []modelReplicaRef // records RemoveNodeModel calls including the replica index } @@ -46,6 +46,21 @@ func (f *fakeModelLocator) FindNodesWithModel(_ context.Context, _ string) ([]Ba return f.nodes, f.findErr } +// Get answers out of the same node list the locator hands out, so a spec that +// registers an AGENT node gets an agent node back and the carrier split is +// driven by the fixture rather than by a second thing to keep in step. +func (f *fakeModelLocator) Get(_ context.Context, nodeID string) (*BackendNode, error) { + if f.getErr != nil { + return nil, f.getErr + } + for i := range f.nodes { + if f.nodes[i].ID == nodeID { + return &f.nodes[i], nil + } + } + return nil, errors.New("no such node") +} + func (f *fakeModelLocator) RemoveNodeModel(_ context.Context, nodeID, modelName string, replicaIndex int) error { f.removedPairs = append(f.removedPairs, modelNodePair{nodeID, modelName}) f.removedReplicas = append(f.removedReplicas, modelReplicaRef{nodeID, modelName, replicaIndex}) @@ -59,6 +74,10 @@ func (f *fakeModelLocator) RemoveAllNodeModelReplicas(_ context.Context, nodeID, // fakeMessagingClient implements messaging.MessagingClient, recording Publish // and Request calls so we can assert on subjects and payloads. +// +// Only ONE verb still reaches it: backend.stop to an agent node. Every other +// control verb travels over the tunnel, so a publish recorded here for a +// backend node is a bug, and several specs below assert exactly that. type fakeMessagingClient struct { mu sync.Mutex published []publishCall @@ -120,38 +139,46 @@ func (f *fakeMessagingClient) Request(subject string, data []byte, timeout time. func (f *fakeMessagingClient) IsConnected() bool { return true } func (f *fakeMessagingClient) Close() {} +// publishedSubjects reports what actually reached the bus. +func (f *fakeMessagingClient) publishedSubjects() []string { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]string, 0, len(f.published)) + for _, c := range f.published { + out = append(out, c.Subject) + } + return out +} + type fakeSubscription struct{} func (f *fakeSubscription) Unsubscribe() error { return nil } -func mustJSON(v any) []byte { - data, err := json.Marshal(v) - Expect(err).ToNot(HaveOccurred()) - return data -} - // --- Tests --- var _ = Describe("RemoteUnloaderAdapter", func() { var ( locator *fakeModelLocator - mc *fakeMessagingClient + bus *fakeMessagingClient + workers *scriptedControlWorkers adapter *RemoteUnloaderAdapter ) BeforeEach(func() { locator = &fakeModelLocator{} - mc = &fakeMessagingClient{} - // backend.stop is request-reply, so the default fake must answer the - // way a current worker does. Specs that care about the reply override - // requestReply themselves. - mc.requestReply = mustJSON(messaging.BackendStopReply{ + bus = &fakeMessagingClient{} + workers = newScriptedControlWorkers() + adapter = NewRemoteUnloaderAdapter(locator, bus, workers.controlClient(), 3*time.Minute, 15*time.Minute) + }) + + // scriptStop lets a backend node acknowledge the tunnelled backend.stop. + scriptStop := func(nodeID string) { + workers.scriptReply(controlKey(nodeID, workerctl.PathBackendStop), messaging.BackendStopReply{ Success: true, StoppedProcessKeys: []string{"llama#0"}, ReportsStoppedProcesses: true, }) - adapter = NewRemoteUnloaderAdapter(locator, mc, 3*time.Minute, 15*time.Minute) - }) + } // HasRemoteModel carries the distinction that UnloadRemoteModel // deliberately does not, so ShutdownModel can answer 404 for a model that @@ -193,20 +220,25 @@ var _ = Describe("RemoteUnloaderAdapter", func() { // tests/e2e/distributed/node_lifecycle_test.go — keep them in step. locator.nodes = nil Expect(adapter.UnloadRemoteModel("my-model")).To(Succeed()) - Expect(mc.requestCalls).To(BeEmpty()) + Expect(workers.callSubjects()).To(BeEmpty()) + Expect(bus.publishedSubjects()).To(BeEmpty()) }) - It("broadcasts to all nodes with model", func() { + It("stops the backend on every node holding the model, over their tunnels", func() { locator.nodes = []BackendNode{ - {ID: "node-1", Name: "worker-1"}, - {ID: "node-2", Name: "worker-2"}, + {ID: "node-1", Name: "worker-1", NodeType: NodeTypeBackend}, + {ID: "node-2", Name: "worker-2", NodeType: NodeTypeBackend}, } + scriptStop("node-1") + scriptStop("node-2") + Expect(adapter.UnloadRemoteModel("llama")).To(Succeed()) - // Should have asked each node to stop the backend. - Expect(mc.requestCalls).To(HaveLen(2)) - Expect(mc.requestCalls[0].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-1"))) - Expect(mc.requestCalls[1].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-2"))) + Expect(workers.callSubjects()).To(Equal([]string{ + controlKey("node-1", workerctl.PathBackendStop), + controlKey("node-2", workerctl.PathBackendStop), + })) + Expect(bus.publishedSubjects()).To(BeEmpty()) // Should have removed the model from each node in the registry. Expect(locator.removedPairs).To(HaveLen(2)) @@ -216,15 +248,18 @@ var _ = Describe("RemoteUnloaderAdapter", func() { It("stops each node once when the registry returns multiple replicas", func() { locator.nodes = []BackendNode{ - {ID: "node-1", Name: "worker-1"}, - {ID: "node-1", Name: "worker-1"}, - {ID: "node-2", Name: "worker-2"}, + {ID: "node-1", Name: "worker-1", NodeType: NodeTypeBackend}, + {ID: "node-1", Name: "worker-1", NodeType: NodeTypeBackend}, + {ID: "node-2", Name: "worker-2", NodeType: NodeTypeBackend}, } + scriptStop("node-1") + scriptStop("node-2") Expect(adapter.UnloadRemoteModel("llama")).To(Succeed()) - Expect(mc.requestCalls).To(HaveLen(2)) - Expect(mc.requestCalls[0].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-1"))) - Expect(mc.requestCalls[1].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-2"))) + Expect(workers.callSubjects()).To(Equal([]string{ + controlKey("node-1", workerctl.PathBackendStop), + controlKey("node-2", workerctl.PathBackendStop), + })) Expect(locator.removedPairs).To(ConsistOf( modelNodePair{"node-1", "llama"}, modelNodePair{"node-2", "llama"}, @@ -233,91 +268,113 @@ var _ = Describe("RemoteUnloaderAdapter", func() { It("continues when one node fails", func() { locator.nodes = []BackendNode{ - {ID: "node-fail", Name: "worker-fail"}, - {ID: "node-ok", Name: "worker-ok"}, + {ID: "node-fail", Name: "worker-fail", NodeType: NodeTypeBackend}, + {ID: "node-ok", Name: "worker-ok", NodeType: NodeTypeBackend}, } - // Use a messaging client that fails the first Request call only. - failOnce := &failOnceMessagingClient{inner: mc, failOn: 0} - adapter = NewRemoteUnloaderAdapter(locator, failOnce, 3*time.Minute, 15*time.Minute) + workers.scriptUnroutable("node-fail") + scriptStop("node-ok") Expect(adapter.UnloadRemoteModel("llama")).To(HaveOccurred()) - // The second node should still have been processed. - // The first node's StopBackend errored, so RemoveNodeModel was NOT called for it. - // The second node's StopBackend succeeded, so RemoveNodeModel WAS called. + // The second node should still have been processed. The first + // node's stop errored, so its row was NOT dropped: a row deleted + // on a route this frontend could not open is a model reclaimed + // while it is still loaded. Expect(locator.removedPairs).To(HaveLen(1)) Expect(locator.removedPairs[0].nodeID).To(Equal("node-ok")) }) It("propagates forced shutdown to every worker", func() { - locator.nodes = []BackendNode{{ID: "node-1", Name: "worker-1"}} + locator.nodes = []BackendNode{{ID: "node-1", Name: "worker-1", NodeType: NodeTypeBackend}} + scriptStop("node-1") + Expect(adapter.UnloadRemoteModelContext(context.Background(), "llama", true)).To(Succeed()) + workers.mu.Lock() + defer workers.mu.Unlock() + Expect(workers.calls).To(HaveLen(1)) var payload messaging.BackendStopRequest - Expect(json.Unmarshal(mc.requestCalls[0].Data, &payload)).To(Succeed()) + Expect(json.Unmarshal(workers.calls[0].Data, &payload)).To(Succeed()) Expect(payload).To(Equal(messaging.BackendStopRequest{Backend: "llama", Force: true})) }) }) - Describe("StopBackend", func() { - It("with empty backend asks the worker to stop everything", func() { - Expect(adapter.StopBackend("node-1", "")).To(Succeed()) - Expect(mc.requestCalls).To(HaveLen(1)) - Expect(mc.requestCalls[0].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-1"))) - - // An empty Backend is the wire signal for "stop all"; the worker's - // decodeBackendStopRequest reads it the same way it read the bare - // nil payload this replaced. - var payload messaging.BackendStopRequest - Expect(json.Unmarshal(mc.requestCalls[0].Data, &payload)).To(Succeed()) - Expect(payload.Backend).To(BeEmpty()) - }) - - // The bug this reply exists for: the worker could not stop what was - // asked, and the caller was told everything was fine. - It("reports a stop the worker could not carry out", func() { - mc.requestReply = mustJSON(messaging.BackendStopReply{ + // The carrier split. It is the one verb of the ten that is decided by the + // KIND of worker, and getting it wrong is silent in both directions: a + // backend node's stop published on the bus reaches nothing, and an agent + // node's stop sent over a tunnel it does not hold reaches nothing either. + Describe("StopBackend and node type", func() { + It("surfaces a backend worker's stop failure", func() { + locator.nodes = []BackendNode{{ID: "backend-1", Name: "gpu-1", NodeType: NodeTypeBackend}} + workers.scriptReply(controlKey("backend-1", workerctl.PathBackendStop), messaging.BackendStopReply{ Success: false, Error: "llama#0: process refused to die", ReportsStoppedProcesses: true, }) - err := adapter.StopBackend("node-1", "llama-backend") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("process refused to die")) + + err := adapter.StopBackend("backend-1", "llama-backend") + Expect(err).To(MatchError(ContainSubstring("process refused to die"))) }) - // Nothing running under that name is the state the caller asked for, so - // it stays a success — eviction and cleanup paths stop models that are - // already gone all the time. - It("succeeds when the worker matched no running process", func() { - mc.requestReply = mustJSON(messaging.BackendStopReply{ - Success: true, - ReportsStoppedProcesses: true, - }) - Expect(adapter.StopBackend("node-1", "llama-backend")).To(Succeed()) + It("sends a backend stop to a BACKEND node over the tunnel and not over the bus", func() { + locator.nodes = []BackendNode{{ID: "backend-1", Name: "gpu-1", NodeType: NodeTypeBackend}} + scriptStop("backend-1") + + Expect(adapter.StopBackend("backend-1", "llama-backend")).To(Succeed()) + + Expect(workers.callSubjects()).To(ContainElement(controlKey("backend-1", workerctl.PathBackendStop))) + Expect(bus.publishedSubjects()).To(BeEmpty()) }) - // A worker built before BackendStopReply performs the stop and never - // answers. Failing here would break every stop on a fleet mid-upgrade. - It("assumes delivery when an older worker never answers", func() { - mc.requestErr = nats.ErrTimeout - Expect(adapter.StopBackend("node-1", "llama-backend")).To(Succeed()) + It("sends a backend stop to an AGENT node over the bus, because agent workers hold no tunnel", func() { + locator.nodes = []BackendNode{{ID: "agent-1", Name: "agent", NodeType: NodeTypeAgent}} + + Expect(adapter.StopBackend("agent-1", "llama-backend")).To(Succeed()) + + Expect(bus.publishedSubjects()).To(ContainElement(messaging.SubjectNodeBackendStop("agent-1"))) + Expect(workers.callSubjects()).ToNot(ContainElement(controlKey("agent-1", workerctl.PathBackendStop))) }) - // A closed connection is not an old worker, and callers depend on - // hearing about it: UnloadRemoteModel skips the registry cleanup for a - // node it could not reach. - It("reports a transport failure rather than assuming delivery", func() { - mc.requestErr = nats.ErrConnectionClosed - Expect(adapter.StopBackend("node-1", "llama-backend")).To(HaveOccurred()) + It("treats a node whose type cannot be read as a backend worker", func() { + // The column defaults to backend and every other node-type branch + // in this package reads an empty value the same way. The lookup + // failing must not silently move a stop onto a carrier nothing is + // listening on. + locator.getErr = errors.New("database is down") + scriptStop("unknown-1") + + Expect(adapter.StopBackend("unknown-1", "llama-backend")).To(Succeed()) + + Expect(workers.callSubjects()).To(ContainElement(controlKey("unknown-1", workerctl.PathBackendStop))) + Expect(bus.publishedSubjects()).To(BeEmpty()) }) - It("with backend name sends JSON", func() { - Expect(adapter.StopBackend("node-1", "llama-backend")).To(Succeed()) - Expect(mc.requestCalls).To(HaveLen(1)) + It("with an empty backend asks the worker to stop everything", func() { + locator.nodes = []BackendNode{{ID: "backend-1", Name: "gpu-1", NodeType: NodeTypeBackend}} + scriptStop("backend-1") + Expect(adapter.StopBackend("backend-1", "")).To(Succeed()) + + workers.mu.Lock() + defer workers.mu.Unlock() var payload messaging.BackendStopRequest - Expect(json.Unmarshal(mc.requestCalls[0].Data, &payload)).To(Succeed()) + Expect(json.Unmarshal(workers.calls[0].Data, &payload)).To(Succeed()) + // An empty Backend is what the worker reads as "stop everything"; + // see decodeBackendStopRequest. + Expect(payload.Backend).To(BeEmpty()) + Expect(payload.Force).To(BeFalse()) + }) + + It("names the backend when one is given", func() { + locator.nodes = []BackendNode{{ID: "backend-1", Name: "gpu-1", NodeType: NodeTypeBackend}} + scriptStop("backend-1") + + Expect(adapter.StopBackend("backend-1", "llama-backend")).To(Succeed()) + + workers.mu.Lock() + defer workers.mu.Unlock() + var payload messaging.BackendStopRequest + Expect(json.Unmarshal(workers.calls[0].Data, &payload)).To(Succeed()) Expect(payload.Backend).To(Equal("llama-backend")) Expect(payload.Force).To(BeFalse()) }) @@ -325,30 +382,46 @@ var _ = Describe("RemoteUnloaderAdapter", func() { Describe("StopModelReplica", func() { It("requests an acknowledged stop for the exact process", func() { - mc.requestReply, _ = json.Marshal(messaging.ModelStopReply{Matched: true, Terminated: true, ProcessKey: "llama#2"}) + workers.scriptReply(controlKey("node-1", workerctl.PathModelStop), + messaging.ModelStopReply{Matched: true, Terminated: true, ProcessKey: "llama#2"}) replica := NodeModel{ModelName: "llama", ReplicaIndex: 2, WorkerLocalAddress: "127.0.0.1:5002", ConfigRevision: "rev-1"} reply, err := adapter.StopModelReplica(context.Background(), "node-1", replica, true) Expect(err).NotTo(HaveOccurred()) Expect(reply.Terminated).To(BeTrue()) - Expect(mc.requestCalls).To(HaveLen(1)) - Expect(mc.requestCalls[0].Subject).To(Equal(messaging.SubjectNodeModelStop("node-1"))) - Expect(mc.requestCalls[0].Timeout).To(BeNumerically(">", 0)) + + workers.mu.Lock() + defer workers.mu.Unlock() + Expect(workers.calls).To(HaveLen(1)) + Expect(workers.calls[0].Subject).To(Equal(controlKey("node-1", workerctl.PathModelStop))) var request messaging.ModelStopRequest - Expect(json.Unmarshal(mc.requestCalls[0].Data, &request)).To(Succeed()) + Expect(json.Unmarshal(workers.calls[0].Data, &request)).To(Succeed()) Expect(request).To(Equal(messaging.ModelStopRequest{ ModelName: "llama", ProcessKey: "llama#2", ExpectedAddress: "127.0.0.1:5002", Force: true, ConfigRevision: "rev-1", })) }) + + It("reports an unroutable worker without inventing a stop reply", func() { + // A zero ModelStopReply reads as Matched=false, which the cleanup + // path treats as "there was nothing to stop" and drops the row. It + // must only ever be paired with an error. + workers.scriptUnroutable("node-gone") + + reply, err := adapter.StopModelReplica(context.Background(), "node-gone", NodeModel{ModelName: "llama"}, false) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) + Expect(reply).To(Equal(messaging.ModelStopReply{})) + }) }) Describe("StopNode", func() { - It("publishes to correct subject", func() { + It("asks the worker to shut down over its tunnel", func() { + workers.scriptRawReply(controlKey("node-abc", workerctl.PathNodeStop), []byte(`{}`)) + Expect(adapter.StopNode("node-abc")).To(Succeed()) - Expect(mc.published).To(HaveLen(1)) - Expect(mc.published[0].Subject).To(Equal(messaging.SubjectNodeStop("node-abc"))) - Expect(mc.published[0].Data).To(BeNil()) + + Expect(workers.callSubjects()).To(Equal([]string{controlKey("node-abc", workerctl.PathNodeStop)})) + Expect(bus.publishedSubjects()).To(BeEmpty()) }) }) @@ -360,101 +433,87 @@ var _ = Describe("RemoteUnloaderAdapter", func() { It("continues on failure", func() { locator.nodes = []BackendNode{ - {ID: "node-1", Name: "w1"}, - {ID: "node-2", Name: "w2"}, + {ID: "node-1", Name: "w1", NodeType: NodeTypeBackend}, + {ID: "node-2", Name: "w2", NodeType: NodeTypeBackend}, } - // Request will fail for all calls. - mc.requestErr = fmt.Errorf("timeout") + // Neither node is scripted, so both answer the loud + // unscripted-verb default. Both must still be attempted. Expect(adapter.DeleteModelFiles("my-model")).To(Succeed()) - // Both nodes attempted. - Expect(mc.requestCalls).To(HaveLen(2)) - Expect(mc.requestCalls[0].Subject).To(Equal(messaging.SubjectNodeModelDelete("node-1"))) - Expect(mc.requestCalls[1].Subject).To(Equal(messaging.SubjectNodeModelDelete("node-2"))) + Expect(workers.callSubjects()).To(Equal([]string{ + controlKey("node-1", workerctl.PathModelDelete), + controlKey("node-2", workerctl.PathModelDelete), + })) }) }) -}) - -// failOnceMessagingClient wraps fakeMessagingClient but fails the Publish call -// at index failOn (0-based) and succeeds all others. -type failOnceMessagingClient struct { - inner *fakeMessagingClient - failOn int - callIdx int - mu sync.Mutex -} - -func (f *failOnceMessagingClient) Publish(subject string, data any) error { - f.mu.Lock() - idx := f.callIdx - f.callIdx++ - f.mu.Unlock() - if idx == f.failOn { - return fmt.Errorf("simulated failure") - } - return f.inner.Publish(subject, data) -} - -func (f *failOnceMessagingClient) Subscribe(subject string, handler func([]byte)) (messaging.Subscription, error) { - return f.inner.Subscribe(subject, handler) -} - -func (f *failOnceMessagingClient) QueueSubscribe(subject, queue string, handler func([]byte)) (messaging.Subscription, error) { - return f.inner.QueueSubscribe(subject, queue, handler) -} - -func (f *failOnceMessagingClient) QueueSubscribeReply(subject, queue string, handler func(data []byte, reply func([]byte))) (messaging.Subscription, error) { - return f.inner.QueueSubscribeReply(subject, queue, handler) -} -func (f *failOnceMessagingClient) SubscribeReply(subject string, handler func(data []byte, reply func([]byte))) (messaging.Subscription, error) { - return f.inner.SubscribeReply(subject, handler) -} + Describe("UnloadModelOnNode", func() { + It("succeeds when the worker reports the model freed", func() { + workers.scriptReply(controlKey("node-1", workerctl.PathModelUnload), messaging.ModelUnloadReply{Success: true}) + Expect(adapter.UnloadModelOnNode("node-1", "llama")).To(Succeed()) + }) -func (f *failOnceMessagingClient) Request(subject string, data []byte, timeout time.Duration) ([]byte, error) { - f.mu.Lock() - idx := f.callIdx - f.callIdx++ - f.mu.Unlock() - if idx == f.failOn { - return nil, fmt.Errorf("simulated failure") - } - return f.inner.Request(subject, data, timeout) -} + It("surfaces the worker's own refusal, which is an answer and not a lost route", func() { + workers.scriptReply(controlKey("node-1", workerctl.PathModelUnload), + messaging.ModelUnloadReply{Success: false, Error: "Free failed"}) -func (f *failOnceMessagingClient) IsConnected() bool { return true } -func (f *failOnceMessagingClient) Close() {} + err := adapter.UnloadModelOnNode("node-1", "llama") + Expect(err).To(MatchError(ContainSubstring("Free failed"))) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeFalse()) + }) + }) +}) var _ = Describe("RemoteUnloaderAdapter timeout configuration", func() { - It("passes the configured install timeout to the messaging client", func() { - mc := newScriptedMessagingClient() - mc.scriptReply(messaging.SubjectNodeBackendInstall("n1"), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "127.0.0.1:0"}) - adapter := NewRemoteUnloaderAdapter(nil, mc, 7*time.Minute, 11*time.Minute) + // Each verb must be given ITS OWN configured budget, and the budget is + // observed the only way it can be: by how long the client waits on a worker + // that took the request and never answered. HTTP carries no deadline, and + // the transport dials on a context of its own, so nothing on the far side + // can see it. + // + // The two budgets are far apart so a swap cannot pass: install would wait + // the upgrade budget and upgrade would wait the install one, and each + // assertion below is on the wrong side of the divide for the other. + const ( + installBudget = 150 * time.Millisecond + upgradeBudget = 600 * time.Millisecond + divide = 400 * time.Millisecond + ) + It("gives backend.install the configured install timeout", func() { + workers := newScriptedControlWorkers() + workers.scriptHang(controlKey("n1", workerctl.PathBackendInstall)) + adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), installBudget, upgradeBudget) + + started := time.Now() _, err := adapter.InstallBackend("n1", "llama-cpp", "", "[]", "", "", "", 0, "", nil) - Expect(err).ToNot(HaveOccurred()) + elapsed := time.Since(started) - Expect(mc.calls).To(HaveLen(1)) - Expect(mc.calls[0].Timeout).To(Equal(7 * time.Minute)) + Expect(errors.Is(err, galleryop.ErrWorkerStillInstalling)).To(BeTrue(), + "a spent budget is reported as still-installing, got %v", err) + Expect(elapsed).To(BeNumerically(">=", installBudget)) + Expect(elapsed).To(BeNumerically("<", divide)) }) - It("passes the configured upgrade timeout to the messaging client", func() { - mc := newScriptedMessagingClient() - mc.scriptReply(messaging.SubjectNodeBackendUpgrade("n1"), messaging.BackendUpgradeReply{Success: true}) - adapter := NewRemoteUnloaderAdapter(nil, mc, 7*time.Minute, 11*time.Minute) + It("gives backend.upgrade the configured upgrade timeout", func() { + workers := newScriptedControlWorkers() + workers.scriptHang(controlKey("n1", workerctl.PathBackendUpgrade)) + adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), installBudget, upgradeBudget) + started := time.Now() _, err := adapter.UpgradeBackend("n1", "llama-cpp", "[]", "", "", "", 0, "", nil) - Expect(err).ToNot(HaveOccurred()) + elapsed := time.Since(started) - Expect(mc.calls).To(HaveLen(1)) - Expect(mc.calls[0].Timeout).To(Equal(11 * time.Minute)) + Expect(errors.Is(err, galleryop.ErrWorkerStillInstalling)).To(BeTrue(), + "a spent budget is reported as still-installing, got %v", err) + Expect(elapsed).To(BeNumerically(">", divide)) }) }) -var _ = Describe("RemoteUnloaderAdapter NATS timeout handling", func() { - It("wraps nats.ErrTimeout from InstallBackend in galleryop.ErrWorkerStillInstalling", func() { - mc := newScriptedMessagingClient() - mc.scriptErr(messaging.SubjectNodeBackendInstall("n1"), nats.ErrTimeout) - adapter := NewRemoteUnloaderAdapter(nil, mc, 100*time.Millisecond, 1*time.Second) +var _ = Describe("RemoteUnloaderAdapter timeout handling", func() { + It("reports a spent budget as still-installing, so the operation shows as running on the worker", func() { + workers := newScriptedControlWorkers() + workers.scriptTimeout("n1") + adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 100*time.Millisecond, 1*time.Second) _, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "", nil) Expect(err).To(HaveOccurred()) @@ -462,57 +521,71 @@ var _ = Describe("RemoteUnloaderAdapter NATS timeout handling", func() { "expected wrapped ErrWorkerStillInstalling, got %v", err) }) - It("does NOT wrap non-timeout errors", func() { - mc := newScriptedMessagingClient() - mc.scriptErr(messaging.SubjectNodeBackendInstall("n1"), nats.ErrNoResponders) - adapter := NewRemoteUnloaderAdapter(nil, mc, 100*time.Millisecond, 1*time.Second) + It("does NOT report an unroutable worker as still installing", func() { + // The two are different facts and the operator UI shows them + // differently: one keeps the queue row and pushes the retry out, the + // other is a plain failure. Both are non-verdicts, so neither may reap. + workers := newScriptedControlWorkers() + workers.scriptUnroutable("n1") + adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 100*time.Millisecond, 1*time.Second) _, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "", nil) Expect(err).To(HaveOccurred()) Expect(errors.Is(err, galleryop.ErrWorkerStillInstalling)).To(BeFalse()) - Expect(errors.Is(err, nats.ErrNoResponders)).To(BeTrue()) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) + }) + + It("does not read a message merely containing the words nats timeout as a timeout", func() { + // The string match the bus carrier needed would have matched a worker + // error that quoted the phrase, and would have turned a real failure + // into "still installing" forever. + workers := newScriptedControlWorkers() + workers.scriptReply(controlKey("n1", workerctl.PathBackendInstall), + messaging.BackendInstallReply{Success: false, Error: `the worker said "nats: timeout" in its log`}) + adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), time.Minute, time.Minute) + + reply, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "", nil) + Expect(err).ToNot(HaveOccurred()) + Expect(reply.Success).To(BeFalse()) }) }) var _ = Describe("RemoteUnloaderAdapter install progress streaming", func() { - It("forwards BackendInstallProgressEvent values into the onProgress callback when the worker publishes them", func() { - mc := newScriptedMessagingClient() - mc.scriptReply(messaging.SubjectNodeBackendInstall("n1"), messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "127.0.0.1:0"}) - mc.scheduleProgressPublish("n1", "op-abc", []messaging.BackendInstallProgressEvent{ + It("forwards the worker's progress lines to onProgress, in the order it wrote them", func() { + workers := newScriptedControlWorkers() + workers.scriptReply(controlKey("n1", workerctl.PathBackendInstall), + messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "127.0.0.1:0"}) + workers.scriptProgress(controlKey("n1", workerctl.PathBackendInstall), []messaging.BackendInstallProgressEvent{ {OpID: "op-abc", NodeID: "n1", Backend: "vllm", FileName: "vllm.tar.zst", Current: "100 MB", Total: "1 GB", Percentage: 10}, {OpID: "op-abc", NodeID: "n1", Backend: "vllm", FileName: "vllm.tar.zst", Current: "500 MB", Total: "1 GB", Percentage: 50}, }) - adapter := NewRemoteUnloaderAdapter(nil, mc, 1*time.Second, 1*time.Second) - var ( - received []messaging.BackendInstallProgressEvent - mu sync.Mutex - ) + adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), time.Second, time.Second) + var received []messaging.BackendInstallProgressEvent onProgress := func(ev messaging.BackendInstallProgressEvent) { - mu.Lock() - defer mu.Unlock() + // No lock, and that is an assertion in itself: the callback runs + // synchronously on the caller's own goroutine, so a race detector + // run would fail here if it did not. received = append(received, ev) } _, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "op-abc", onProgress) Expect(err).ToNot(HaveOccurred()) - Eventually(func() int { - mu.Lock() - defer mu.Unlock() - return len(received) - }, "1s").Should(Equal(2)) + Expect(received).To(HaveLen(2)) + Expect([]float64{received[0].Percentage, received[1].Percentage}).To(Equal([]float64{10, 50})) }) - It("does NOT subscribe when onProgress is nil (reconciler retry path)", func() { - mc := newScriptedMessagingClient() - mc.scriptReply(messaging.SubjectNodeBackendInstall("n1"), messaging.BackendInstallReply{Success: true}) + It("completes when the caller wants no progress at all (reconciler retry path)", func() { + workers := newScriptedControlWorkers() + workers.scriptReply(controlKey("n1", workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: true}) + workers.scriptProgress(controlKey("n1", workerctl.PathBackendInstall), []messaging.BackendInstallProgressEvent{ + {OpID: "", NodeID: "n1", Percentage: 42}, + }) - adapter := NewRemoteUnloaderAdapter(nil, mc, 1*time.Second, 1*time.Second) - _, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "", nil) + adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), time.Second, time.Second) + reply, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "", nil) Expect(err).ToNot(HaveOccurred()) - - Expect(mc.subscribeCalls()).To(BeEmpty(), - "reconciler-driven retries must not subscribe to the per-op progress subject") + Expect(reply.Success).To(BeTrue()) }) }) diff --git a/core/services/nodes/unloader_upgrade_test.go b/core/services/nodes/unloader_upgrade_test.go index bad8f9ed5062..41fe7316f16e 100644 --- a/core/services/nodes/unloader_upgrade_test.go +++ b/core/services/nodes/unloader_upgrade_test.go @@ -1,82 +1,78 @@ package nodes import ( - "sync" + "errors" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" ) var _ = Describe("RemoteUnloaderAdapter.UpgradeBackend", func() { - It("fires a NATS request to the backend.upgrade subject and returns the reply", func() { - mc := newScriptedMessagingClient() + It("calls the backend.upgrade verb on the worker and returns the reply", func() { + workers := newScriptedControlWorkers() nodeID := "node-x" - mc.scriptReply(messaging.SubjectNodeBackendUpgrade(nodeID), + workers.scriptReply(controlKey(nodeID, workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: true}) - adapter := NewRemoteUnloaderAdapter(nil, mc, 3*time.Minute, 15*time.Minute) + adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) reply, err := adapter.UpgradeBackend(nodeID, "llama-cpp", `[{"name":"x"}]`, "", "", "", 0, "", nil) Expect(err).ToNot(HaveOccurred()) Expect(reply.Success).To(BeTrue()) + Expect(workers.callSubjects()).To(Equal([]string{controlKey(nodeID, workerctl.PathBackendUpgrade)})) }) - It("returns the underlying error when the subject has no responders", func() { - mc := newScriptedMessagingClient() // unscripted subject => fakeNoRespondersErr by harness convention + It("reports a worker it cannot reach as unroutable rather than as a failed upgrade", func() { + workers := newScriptedControlWorkers() + workers.scriptUnroutable("missing-node") - adapter := NewRemoteUnloaderAdapter(nil, mc, 3*time.Minute, 15*time.Minute) + adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) _, err := adapter.UpgradeBackend("missing-node", "llama-cpp", "", "", "", "", 0, "", nil) - Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) + // Not the worker's 404 either: nothing was asked of it, so it did not + // say it lacks the verb, and the legacy force-install fallback must not + // fire on this. + Expect(errors.Is(err, ErrWorkerControlUnsupported)).To(BeFalse()) }) // Reproducer for "upgrade reports progress:0 the whole time" (Bug B). The // install path streamed per-node download ticks; the upgrade path did a bare - // request→single-reply with no progress subscription, so a long force-reinstall - // blocked opaque. The adapter must subscribe to the per-op progress subject - // (reused from install) BEFORE the request and deliver each tick to onProgress. + // request→single-reply with no progress at all, so a long force-reinstall + // blocked opaque. Both verbs now stream on the same envelope shape. It("streams per-node progress ticks during the upgrade", func() { - mc := newScriptedMessagingClient() + workers := newScriptedControlWorkers() nodeID := "node-slow" opID := "op-upgrade-1" - mc.scriptReply(messaging.SubjectNodeBackendUpgrade(nodeID), + workers.scriptReply(controlKey(nodeID, workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: true}) - // The worker would publish these while force-reinstalling. The harness - // replays them as soon as the adapter subscribes to the per-op subject. - mc.scheduleProgressPublish(nodeID, opID, []messaging.BackendInstallProgressEvent{ + workers.scriptProgress(controlKey(nodeID, workerctl.PathBackendUpgrade), []messaging.BackendInstallProgressEvent{ {NodeID: nodeID, FileName: "llama-cpp.tar", Current: "10 MB", Total: "100 MB", Percentage: 10}, {NodeID: nodeID, FileName: "llama-cpp.tar", Current: "100 MB", Total: "100 MB", Percentage: 100}, }) - var mu sync.Mutex var got []messaging.BackendInstallProgressEvent onProgress := func(ev messaging.BackendInstallProgressEvent) { - mu.Lock() got = append(got, ev) - mu.Unlock() } - adapter := NewRemoteUnloaderAdapter(nil, mc, 3*time.Minute, 15*time.Minute) + adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) reply, err := adapter.UpgradeBackend(nodeID, "llama-cpp", `[{"name":"x"}]`, "", "", "", 0, opID, onProgress) Expect(err).ToNot(HaveOccurred()) Expect(reply.Success).To(BeTrue()) - // Confirm it subscribed to the (reused) install-progress subject for this op. - Expect(mc.subscribeCalls()).To(ContainElement(messaging.SubjectNodeBackendInstallProgress(nodeID, opID))) - - // Progress events are delivered asynchronously (goroutine-per-event), so - // poll for both and assert on the set — ordering is best-effort by design. - Eventually(func() []float64 { - mu.Lock() - defer mu.Unlock() - pcts := make([]float64, 0, len(got)) - for _, e := range got { - pcts = append(pcts, e.Percentage) - } - return pcts - }, 2*time.Second, 20*time.Millisecond).Should(ConsistOf(float64(10), float64(100))) + // Every tick, in the worker's order, and all of them BEFORE the reply + // the call returned: the reply line is the last thing on the body, so a + // tick arriving late is structurally impossible rather than merely + // unlikely. + pcts := make([]float64, 0, len(got)) + for _, e := range got { + pcts = append(pcts, e.Percentage) + } + Expect(pcts).To(Equal([]float64{10, 100})) }) }) diff --git a/core/services/worker/control_client_roundtrip_test.go b/core/services/worker/control_client_roundtrip_test.go new file mode 100644 index 000000000000..52046f320f1a --- /dev/null +++ b/core/services/worker/control_client_roundtrip_test.go @@ -0,0 +1,151 @@ +package worker + +import ( + "context" + "errors" + "net" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// The frontend's control client against the REAL worker control plane: the real +// supervisor's handlers, mounted through the real nodes.AuthenticatedRoutes so +// the real bearer check runs first, reached by a real nodes.ControlClient over +// a real HTTP transport. The only thing standing in for production is the +// transport's dial, which is the seam the tunnel occupies. +// +// It lives in this package because the dependency runs worker -> nodes: the +// frontend's client cannot be exercised against the real handler from the other +// side without an import cycle. Both halves of the contract are written by +// different packages, so a spec on either side alone can only prove that side +// agrees with itself; the paths, the envelope shape and the status codes are +// only pinned together here. +var _ = Describe("the frontend's control client against the real worker", func() { + const ( + token = "s3cret-registration-token" + nodeID = "worker-under-test" + ) + + var ( + sup *backendSupervisor + client *nodes.ControlClient + srvAddr string + sigCh chan os.Signal + ) + + // newClient builds a control client whose transport dials srvAddr, + // authenticating with the token given. + newClient := func(srvAddr, tok string) *nodes.ControlClient { + return nodes.NewControlClient(func(string) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "tcp", srvAddr) + } + }, tok) + } + + BeforeEach(func() { + sigCh = make(chan os.Signal, 1) + sup = &backendSupervisor{ + cfg: &Config{}, + nodeID: nodeID, + sigCh: sigCh, + processes: map[string]*backendProcess{}, + } + + dir := GinkgoT().TempDir() + lis, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + srv, err := nodes.StartFileTransferServerWithRoutes(lis, + filepath.Join(dir, "staging"), filepath.Join(dir, "models"), filepath.Join(dir, "data"), + token, config.DefaultMaxUploadSize, nil, + &nodes.AuthenticatedRoutes{Prefix: workerctl.Prefix, Register: sup.RegisterControlRoutes}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = srv.Close() }) + + srvAddr = lis.Addr().String() + client = newClient(srvAddr, token) + }) + + It("round-trips models.running end to end", func() { + var reply messaging.ModelsRunningReply + Expect(client.Call(context.Background(), nodeID, workerctl.PathModelsRunning, + messaging.ModelsRunningRequest{}, &reply)).To(Succeed()) + // Nothing is running, and an empty list is the worker's real answer + // rather than a decode that quietly produced nothing: the client + // reports a body it cannot read as unroutable instead. + Expect(reply.Models).To(BeEmpty()) + Expect(reply.Error).To(BeEmpty()) + }) + + It("streams install progress in order and returns the terminal reply", func() { + sup.installFn = func(_ context.Context, req messaging.BackendInstallRequest, _ bool, + onProgress func(messaging.BackendInstallProgressEvent)) (string, error) { + onProgress(messaging.BackendInstallProgressEvent{OpID: req.OpID, Percentage: 50}) + onProgress(messaging.BackendInstallProgressEvent{OpID: req.OpID, Percentage: 100}) + return "127.0.0.1:41234", nil + } + + var seen []float64 + var reply messaging.BackendInstallReply + err := client.CallStreaming(context.Background(), nodeID, workerctl.PathBackendInstall, + messaging.BackendInstallRequest{Backend: "mock", OpID: "op-1"}, &reply, + func(ev messaging.BackendInstallProgressEvent) { seen = append(seen, ev.Percentage) }) + Expect(err).NotTo(HaveOccurred()) + Expect(seen).To(Equal([]float64{50, 100})) + Expect(reply.Success).To(BeTrue()) + Expect(reply.WorkerLocalAddress).To(Equal("127.0.0.1:41234")) + }) + + It("reports a FAILED install as the worker's own answer, not as a transport failure", func() { + // The distinction the two sides exist to preserve: the worker answers + // 200 with Error set, so the frontend reads a verdict a caller may act + // on rather than a route it must not act on. + sup.installFn = func(context.Context, messaging.BackendInstallRequest, bool, + func(messaging.BackendInstallProgressEvent)) (string, error) { + return "", errors.New("no child with platform linux/arm64") + } + + var reply messaging.BackendInstallReply + err := client.CallStreaming(context.Background(), nodeID, workerctl.PathBackendInstall, + messaging.BackendInstallRequest{Backend: "mock"}, &reply, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(reply.Success).To(BeFalse()) + Expect(reply.Error).To(ContainSubstring("linux/arm64")) + }) + + It("round-trips a 204 verb, which carries no body at all", func() { + Expect(client.Call(context.Background(), nodeID, workerctl.PathBackendStop, + messaging.BackendStopRequest{Backend: "no-such-backend"}, nil)).To(Succeed()) + }) + + It("reports an unknown control verb as unsupported, and not as absence", func() { + err := client.Call(context.Background(), nodeID, workerctl.Prefix+"invented", struct{}{}, &struct{}{}) + Expect(err).To(MatchError(nodes.ErrWorkerControlUnsupported)) + Expect(errors.Is(err, cluster.ErrNoRoute)).To(BeFalse()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + }) + + It("reports a rejected token as unroutable, never as a worker verdict about a backend", func() { + // The bearer check runs before routing, so a wrong token is a 401 for + // every verb. It says nothing about any backend and nothing may reap on + // it; it also must not be read as the verb being unsupported, which + // would send the upgrade path into its destructive legacy fallback. + wrong := newClient(srvAddr, "not-the-token") + var reply messaging.BackendListReply + err := wrong.Call(context.Background(), nodeID, workerctl.PathBackendList, + messaging.BackendListRequest{}, &reply) + Expect(errors.Is(err, nodes.ErrWorkerUnroutable)).To(BeTrue()) + Expect(errors.Is(err, nodes.ErrWorkerControlUnsupported)).To(BeFalse()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 92f699495207..6f81ec891ed0 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -17,7 +17,7 @@ Distributed mode requires authentication enabled with a **PostgreSQL** database **Frontends** are stateless LocalAI instances that receive API requests and route them to worker nodes via the **SmartRouter**. All frontends share state through PostgreSQL and coordinate via NATS. -**Workers** are generic processes that self-register with a frontend. They don't have a fixed backend type - the SmartRouter dynamically installs the required backend via NATS `backend.install` events when a model request arrives. +**Workers** are generic processes that self-register with a frontend. They don't have a fixed backend type - the SmartRouter dynamically installs the required backend by calling the worker's `backend.install` control route through its tunnel when a model request arrives. ### Scheduling Algorithm @@ -165,14 +165,14 @@ Reconnects use exponential backoff with jitter: the interval doubles from 500ms #### What the frontend sends through it -Every connection the frontend makes to a worker now goes through that worker's tunnel. There are three, and all three are the same path underneath: +Every connection the frontend makes to a worker now goes through that worker's tunnel. There are four, and all four are the same path underneath: | What | Protocol | Stream tag | |------|----------|-----------| | Inference, model load, health checks | gRPC to a backend process | `grpc` | | Model file staging, backend-log listing | HTTP to the worker's own server | `http` | | Live backend-log streaming | WebSocket to the same server | `http` | -| Backend install/upgrade/list/delete, model stop/unload/delete, node stop | HTTP to the worker's own server | `http` | +| The control plane: backend install/upgrade/list/stop/delete, model stop/unload/delete, models running, node stop | HTTP to the worker's own server | `http` | #### The worker control plane @@ -205,6 +205,32 @@ does not impose a timeout of its own on an install, and a caller that gives up cancels the download rather than leaving the worker pulling gigabytes for a response nobody will read. +On the frontend's side the ten verbs are ordinary HTTP calls on the `http` +stream tag, so a control RPC to a worker **another replica holds is relayed +exactly like an inference request** - same lookup, same one hop, same budget +arithmetic as [Reaching a worker another replica holds](#reaching-a-worker-another-replica-holds). +There is nothing to subscribe to before an install: its progress lines share the +install's own response, so no event can arrive before the caller is listening +and there is no per-op subject to grant a permission for. + +How a control RPC can FAIL is where absence is decided for the whole control +plane, so the frontend maps every outcome onto exactly one row of the table +below and never onto another. Only the two rows in which the WORKER spoke may +be acted on; everything else is this frontend failing to reach it, which says +nothing about the worker at all: + +| What happened | How the frontend reports it | May anything reap on it? | +|---|---|---| +| The worker refused the stream with one of its three evidence codes | the refusal itself, unwrapped | **Yes.** The worker spoke. | +| No route: no live owner, an unreachable peer, no relay path, a refusal code this frontend does not recognise, or the worker saying it learned nothing | "this frontend has no route to that worker" | No | +| The call ran out of budget | a deadline, which install and upgrade report as *still installing on the worker* | No | +| `404` under `/v1/control/` | "the worker does not serve that control verb" - it is older than this frontend, and only the upgrade path acts on it, by re-issuing the legacy force-install | No | +| `200` with a reply whose `error` is set | the worker's own answer, handed to the caller as-is | **Yes**, by the caller | + +A `5xx`, or a body the frontend cannot decode, is in the second row and not the +last: a worker's verdict arrives as a `200`, so a `5xx` is the server failing +rather than answering. + The address the frontend holds for a backend (the per-replica port a worker reports after an install) is still what identifies it, and it is still what appears in logs and errors. What it no longer is, is somewhere the frontend connects to: it travels inside the tunnel as the stream's target, and the worker decides what to do with it. A frontend with no way to reach a worker says so and fails. It does **not** fall back to connecting to the worker's advertised address. That fallback is what the tunnel exists to remove, and it is the kind of defect that works on a one-replica developer box and fails in production, so it is an error everywhere. The consequences are deliberately narrow: a model whose worker cannot be reached is not reaped, and its row is left alone, because a frontend that cannot reach a worker has learned nothing about whether that worker is still running the model. @@ -692,7 +718,7 @@ The system automatically applies hardware-detected labels on registration: ### How Workers Operate -Workers start as generic processes with no backend installed. When the SmartRouter needs to load a model on a worker, it sends a NATS `backend.install` event with the backend name and model ID. The worker: +Workers start as generic processes with no backend installed. When the SmartRouter needs to load a model on a worker, it calls `POST /v1/control/backend/install` through that worker's tunnel with the backend name and model ID. The worker: 1. Installs the backend from the gallery (if not already installed) 2. Starts a **new gRPC backend process on a dynamic port** (each model gets its own process) @@ -1452,7 +1478,7 @@ Notes: | **Coordination** | Gossip protocol | NATS messaging | | **Node management** | Automatic | REST API + WebUI | | **Health monitoring** | Peer heartbeats | Centralized HealthMonitor | -| **Backend management** | Manual per node | Dynamic via NATS backend.install | +| **Backend management** | Manual per node | Dynamic via the worker's `backend.install` control route | | **Best for** | Ad-hoc clusters, community sharing | Production, Kubernetes, managed infrastructure | | **Setup complexity** | Minimal (share a token) | Requires PostgreSQL + NATS | @@ -1503,11 +1529,11 @@ Notes: - Liveness is decided by the load job's progress heartbeat, not by elapsed time. Staging a large checkpoint legitimately runs for a long time without touching the replica row, so a transfer that is still progressing is never reclaimed however long it takes. - `Reconciler: reclaimed a replica slot held by a load nobody is driving` names each row reclaimed this way. -**A request fails with `nats: no responders available for request`:** -- The chosen worker was not subscribed on the bus when the frontend tried to install the backend on it. A node's status comes from its HTTP heartbeat, which is a separate channel: a worker that stops stays `healthy` until that heartbeat ages out. -- The scheduler now checks that a node still answers on the bus before it commits to it, marks one that does not as unhealthy, and picks another. A request should therefore see this only when no reachable node is left. -- Only a no-responders answer counts as absent. A worker that answers slowly stays eligible, because excluding it would cost capacity that is really there. -- Check the worker process is running and its NATS connection is up. `Scheduled node is not answering on the bus` in the frontend log names each node demoted this way. +**A request fails with `this frontend has no route to that worker`:** +- The chosen worker's tunnel was not reachable from the replica that handled the request. A node's status comes from its HTTP heartbeat, which is a separate channel: a worker that stops stays `healthy` until that heartbeat ages out, and a worker that is very much alive can be unroutable for a moment while its tunnel re-homes between frontend replicas. +- It is **not** the same as the worker being gone, and nothing acts on it as if it were. A model on an unroutable worker is not reaped, its rows are left alone, and the node is not demoted: doing any of those on a lost route is how a rolling frontend restart turns into a fleet-wide eviction. +- Check the worker process is running and that it has an open tunnel (`opened a tunnelled stream to a worker` in the frontend log, and the worker's own dial/reconnect lines). A worker behind a load balancer that keeps reconnecting is usually an idle-timeout or WebSocket-upgrade problem at the proxy; see the tunnel section above. +- A `nats: no responders available for request` in a modern deployment concerns only the subjects still on the bus: agent-worker jobs, MCP, and file staging. It is no longer how a serve-backend worker is reached. **A worker fills its own disk over time:** - A request that carries a file (an image, an audio clip, a video) stages that file below the worker's HTTP staging or S3 cache `ephemeral/` directory. The frontend releases each request-owned input when inference finishes, and the worker reserves capacity before accepting it. diff --git a/pkg/natsauth/mint_test.go b/pkg/natsauth/mint_test.go index 4d5bb77cee62..e33992ec1898 100644 --- a/pkg/natsauth/mint_test.go +++ b/pkg/natsauth/mint_test.go @@ -37,7 +37,13 @@ var _ = Describe("MintWorkerJWT", func() { uc, err := jwt.DecodeUserClaims(token) Expect(err).NotTo(HaveOccurred()) Expect(uc.Permissions.Sub.Allow).To(ContainElement("nodes.550e8400-e29b-41d4-a716-446655440000.>")) - Expect(uc.Permissions.Pub.Allow).To(ContainElement("nodes.550e8400-e29b-41d4-a716-446655440000.backend.install.*.progress")) + Expect(uc.Permissions.Pub.Allow).To(ContainElement("nodes.550e8400-e29b-41d4-a716-446655440000.files.>")) + // The install-progress subject is gone with the carrier: progress is a + // line in the install response now, so a minted worker JWT must not + // still be granted a publish right for it. + for _, subj := range uc.Permissions.Pub.Allow { + Expect(subj).NotTo(ContainSubstring("backend.install")) + } }) It("mints agent permissions without backend install subscribe", func() { diff --git a/pkg/natsauth/permissions.go b/pkg/natsauth/permissions.go index d44e51990480..a411e05fd478 100644 --- a/pkg/natsauth/permissions.go +++ b/pkg/natsauth/permissions.go @@ -38,13 +38,21 @@ func WorkerPermissions(nodeID, nodeType string) (pubAllow, subAllow []string) { "_INBOX.>", } default: - // Backend worker: lifecycle + file staging on this node only. + // Backend worker: file staging on this node only. + // + // The backend and model lifecycle verbs left the bus: they are HTTP + // routes under workerctl.Prefix, served on the worker's own server and + // reached through its tunnel, so no subject is minted for them and none + // is allowed here. The wildcard stays because the file-staging subjects + // are still under this node's prefix; narrowing it to nodes..files.> + // is a separate change that would break a worker mid-upgrade. subAllow = []string{ prefix + ".>", "_INBOX.>", } + // backend.install.*.progress is gone with the subject: install progress + // is written into the install response the frontend is already reading. pubAllow = []string{ - prefix + ".backend.install.*.progress", prefix + ".files.>", "_INBOX.>", } diff --git a/pkg/natsauth/permissions_coverage_test.go b/pkg/natsauth/permissions_coverage_test.go index 51a60a86fa42..a66d4f74e019 100644 --- a/pkg/natsauth/permissions_coverage_test.go +++ b/pkg/natsauth/permissions_coverage_test.go @@ -33,6 +33,13 @@ func subjectMatches(pattern, subject string) bool { return len(p) == len(s) } +// workerSubjectTokenForTest mirrors the sanitizer both packages implement, so +// the negative assertion below names the exact prefix without reaching into +// either package's unexported copy. +func workerSubjectTokenForTest(nodeID string) string { + return strings.NewReplacer(".", "-", "*", "-", ">", "-", " ", "-", "\t", "-", "\n", "-").Replace(nodeID) +} + func anyAllows(allow []string, subject string) bool { for _, p := range allow { if subjectMatches(p, subject) { @@ -52,16 +59,11 @@ var _ = Describe("WorkerPermissions subject coverage", func() { Context("backend worker", func() { pub, sub := natsauth.WorkerPermissions(nodeID, "backend") - // Every subject core/services/worker/{lifecycle,file_staging}.go subscribes to. + // Every subject core/services/worker/file_staging.go subscribes to. + // The backend and model lifecycle verbs are NOT here: they left the bus + // for the worker's tunnelled control plane, so there is no subject to + // cover. See core/services/workerctl. subscribed := []string{ - messaging.SubjectNodeBackendInstall(nodeID), - messaging.SubjectNodeBackendUpgrade(nodeID), - messaging.SubjectNodeBackendStop(nodeID), - messaging.SubjectNodeBackendDelete(nodeID), - messaging.SubjectNodeBackendList(nodeID), - messaging.SubjectNodeModelUnload(nodeID), - messaging.SubjectNodeModelDelete(nodeID), - messaging.SubjectNodeStop(nodeID), messaging.SubjectNodeFilesEnsure(nodeID), messaging.SubjectNodeFilesStage(nodeID), messaging.SubjectNodeFilesTemp(nodeID), @@ -74,11 +76,21 @@ var _ = Describe("WorkerPermissions subject coverage", func() { }) } - It("allows publishing backend.install progress", func() { - subject := messaging.SubjectNodeBackendInstallProgress(nodeID, "op-123") + It("allows publishing file staging replies", func() { + subject := messaging.SubjectNodeFilesStage(nodeID) Expect(anyAllows(pub, subject)).To(BeTrue(), "backend JWT pub allow-list %v does not cover %s", pub, subject) }) + + // The negative half, and it is the one that would catch a verb quietly + // coming back to the bus: a backend worker is granted nothing to + // publish outside its own file-staging subtree and its inbox. + It("grants a backend worker no publish rights outside file staging and its inbox", func() { + Expect(pub).To(ConsistOf( + "nodes."+workerSubjectTokenForTest(nodeID)+".files.>", + "_INBOX.>", + )) + }) }) Context("agent worker", func() { @@ -116,7 +128,7 @@ var _ = Describe("Documented NATS service-user permissions", func() { messaging.SubjectPrefixCacheObserve, messaging.SubjectPrefixCacheInvalidate, messaging.SubjectPrefixCachePressure, - messaging.SubjectNodeBackendInstall("node-1"), + messaging.SubjectNodeBackendStop("node-1"), messaging.SubjectGalleryProgress("op-1"), } diff --git a/tests/e2e/distributed/control_workers_test.go b/tests/e2e/distributed/control_workers_test.go new file mode 100644 index 000000000000..15ef583b6088 --- /dev/null +++ b/tests/e2e/distributed/control_workers_test.go @@ -0,0 +1,112 @@ +package distributed_test + +import ( + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// ControlWorkers is a fleet of fake workers serving the tunnelled control +// plane, which is where the frontend's backend and model lifecycle verbs go +// now that they have left the bus. +// +// It replaces the NATS SubscribeReply fakes these suites used to stand up. One +// HTTP server answers for every node and reads which node a request is for off +// the Host header, because that is what nodes.ControlClient puts there; a +// client addressing the wrong node would be visible rather than silently +// served. +// +// It is deliberately NOT a real worker: these suites are about what the +// frontend does with a worker's answer. The real handlers are exercised against +// the real client in core/services/worker. +type ControlWorkers struct { + mu sync.Mutex + srv *httptest.Server + handlers map[string]func(nodeID string, body []byte) any +} + +// controlHandlerKey is the (node, verb) a handler is registered for. AnyNode +// registers one handler for every node, which is how a suite fakes a fleet +// whose ids it does not know up front. +const AnyNode = "*" + +func controlHandlerKey(nodeID, path string) string { return nodeID + " " + path } + +// NewControlWorkers starts the fleet and stops it when the spec ends. +func NewControlWorkers() *ControlWorkers { + c := &ControlWorkers{handlers: map[string]func(string, []byte) any{}} + mux := http.NewServeMux() + mux.HandleFunc(workerctl.Prefix, c.serve) + c.srv = httptest.NewServer(mux) + DeferCleanup(c.srv.Close) + return c +} + +// Client returns a control client that reaches this fleet. +func (c *ControlWorkers) Client() *nodes.ControlClient { + addr := c.srv.Listener.Addr().String() + return nodes.NewControlClient(func(string) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "tcp", addr) + } + }, "") +} + +// On registers what one node answers for one verb. Returning nil answers 204, +// which is the shape the fire-and-forget verbs take. +func (c *ControlWorkers) On(nodeID, path string, fn func(nodeID string, body []byte) any) { + c.mu.Lock() + defer c.mu.Unlock() + c.handlers[controlHandlerKey(nodeID, path)] = fn +} + +func (c *ControlWorkers) serve(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + Expect(err).ToNot(HaveOccurred()) + nodeID := strings.TrimSuffix(r.Host, ".worker.invalid:80") + + c.mu.Lock() + fn, ok := c.handlers[controlHandlerKey(nodeID, r.URL.Path)] + if !ok { + fn, ok = c.handlers[controlHandlerKey(AnyNode, r.URL.Path)] + } + c.mu.Unlock() + + if !ok { + // Loud, never plausible: a verb no spec scripted must be a red spec + // rather than a worker that looks absent. + http.Error(w, "no control handler registered for "+r.URL.Path+" on "+nodeID, http.StatusInternalServerError) + return + } + + reply := fn(nodeID, body) + streaming := r.URL.Path == workerctl.PathBackendInstall || r.URL.Path == workerctl.PathBackendUpgrade + if !streaming { + if reply == nil { + w.WriteHeader(http.StatusNoContent) + return + } + w.Header().Set("Content-Type", "application/json") + Expect(json.NewEncoder(w).Encode(reply)).To(Succeed()) + return + } + + raw, err := json.Marshal(reply) + Expect(err).ToNot(HaveOccurred()) + w.Header().Set("Content-Type", workerctl.ContentTypeStream) + w.WriteHeader(http.StatusOK) + // The reply line, and it is the last thing on the body by contract. + Expect(json.NewEncoder(w).Encode(workerctl.Envelope{Reply: raw})).To(Succeed()) +} diff --git a/tests/e2e/distributed/distributed_full_flow_test.go b/tests/e2e/distributed/distributed_full_flow_test.go index de26573790df..abf0cbb72288 100644 --- a/tests/e2e/distributed/distributed_full_flow_test.go +++ b/tests/e2e/distributed/distributed_full_flow_test.go @@ -2,7 +2,6 @@ package distributed_test import ( "context" - "encoding/json" "fmt" "io" "net" @@ -13,6 +12,7 @@ import ( "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/workerctl" "github.com/mudler/LocalAI/pkg/grpc/base" pb "github.com/mudler/LocalAI/pkg/grpc/proto" @@ -21,7 +21,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/nats-io/nats.go" "google.golang.org/grpc" pgdriver "gorm.io/driver/postgres" @@ -225,10 +224,21 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() cancel() }) - // newTestSmartRouter creates a SmartRouter with NATS wired up and a mock - // backend.install handler that always replies success for all registered nodes. + // newTestSmartRouter creates a SmartRouter reaching a fleet of fake workers + // over the tunnelled control plane, with a backend.install handler that + // always replies success for every registered node. newTestSmartRouter := func(reg *nodes.NodeRegistry, extraOpts ...nodes.SmartRouterOptions) *nodes.SmartRouter { - unloader := nodes.NewRemoteUnloaderAdapter(reg, infra.NC, 3*time.Minute, 15*time.Minute) + workers := NewControlWorkers() + workers.On(AnyNode, workerctl.PathBackendInstall, func(string, []byte) any { + return messaging.BackendInstallReply{Success: true} + }) + workers.On(AnyNode, workerctl.PathModelsRunning, func(string, []byte) any { + return messaging.ModelsRunningReply{} + }) + workers.On(AnyNode, workerctl.PathBackendList, func(string, []byte) any { + return messaging.BackendListReply{} + }) + unloader := nodes.NewRemoteUnloaderAdapter(reg, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) opts := nodes.SmartRouterOptions{ Unloader: unloader, @@ -251,22 +261,6 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() router := nodes.NewSmartRouter(reg, opts) - // Subscribe a mock backend.install handler that replies success for any node. - // We use a wildcard-style approach: subscribe to all nodes' install subjects - // by registering after each node. In practice, we rely on the test registering - // nodes before calling Route, so we subscribe to a catch-all pattern. - infra.NC.Conn().Subscribe("nodes.*.backend.install", func(msg *nats.Msg) { - reply := messaging.BackendInstallReply{Success: true} - data, _ := json.Marshal(reply) - msg.Respond(data) - }) - _, err := infra.NC.Conn().Subscribe("nodes.*.models.running", func(msg *nats.Msg) { - data, _ := json.Marshal(messaging.ModelsRunningReply{}) - _ = msg.Respond(data) - }) - Expect(err).NotTo(HaveOccurred()) - FlushNATS(infra.NC) - return router } // suppress unused warning in case some tests don't call it @@ -385,30 +379,25 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() result.Release() }) - It("should unload remote model via NATS", func() { + It("should unload a remote model over the worker's control plane", func() { // Register a node with a loaded model node := &nodes.BackendNode{Name: "gpu-unload", Address: "127.0.0.1:50099"} Expect(registry.Register(context.Background(), node, true)).To(Succeed()) Expect(registry.SetNodeModel(context.Background(), node.ID, "old-model", 0, "loaded", "", 0)).To(Succeed()) - // Subscribe to NATS backend.stop for this node - stopSubject := messaging.SubjectNodeBackendStop(node.ID) + // A worker serving backend.stop on its own control plane. received := make(chan struct{}, 1) - rawConn, err := nats.Connect(infra.NatsURL) - Expect(err).ToNot(HaveOccurred()) - defer rawConn.Close() - - _, err = rawConn.Subscribe(stopSubject, func(msg *nats.Msg) { + workers := NewControlWorkers() + workers.On(node.ID, workerctl.PathBackendStop, func(string, []byte) any { received <- struct{}{} + return nil }) - Expect(err).ToNot(HaveOccurred()) // Create RemoteUnloaderAdapter and unload model - unloader := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, 3*time.Minute, 15*time.Minute) - err = unloader.UnloadRemoteModel("old-model") - Expect(err).ToNot(HaveOccurred()) + unloader := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) + Expect(unloader.UnloadRemoteModel("old-model")).To(Succeed()) - // Verify NATS event received + // The worker got the stop over its tunnel, not over the bus. Eventually(received, 5*time.Second).Should(Receive()) // Verify model removed from registry diff --git a/tests/e2e/distributed/managers_test.go b/tests/e2e/distributed/managers_test.go index b4f51ef957b2..83080f0d72b5 100644 --- a/tests/e2e/distributed/managers_test.go +++ b/tests/e2e/distributed/managers_test.go @@ -12,6 +12,7 @@ import ( "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/workerctl" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" @@ -136,30 +137,18 @@ var _ = Describe("Model and Backend Managers", Label("Distributed"), func() { Expect(registry.SetNodeModel(context.Background(), node1.ID, "big-model", 0, "loaded", "", 0)).To(Succeed()) Expect(registry.SetNodeModel(context.Background(), node2.ID, "big-model", 0, "loaded", "", 0)).To(Succeed()) - // Subscribe to model.delete on both node subjects, track receipt + // Both workers serve model.delete on their own control plane. var deleteCount atomic.Int32 - sub1, err := infra.NC.SubscribeReply(messaging.SubjectNodeModelDelete(node1.ID), func(data []byte, reply func([]byte)) { - var req messaging.ModelDeleteRequest - json.Unmarshal(data, &req) - Expect(req.ModelName).To(Equal("big-model")) - deleteCount.Add(1) - resp, _ := json.Marshal(messaging.ModelDeleteReply{Success: true}) - reply(resp) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub1.Unsubscribe() - - sub2, err := infra.NC.SubscribeReply(messaging.SubjectNodeModelDelete(node2.ID), func(data []byte, reply func([]byte)) { - var req messaging.ModelDeleteRequest - json.Unmarshal(data, &req) - deleteCount.Add(1) - resp, _ := json.Marshal(messaging.ModelDeleteReply{Success: true}) - reply(resp) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub2.Unsubscribe() - - FlushNATS(infra.NC) + workers := NewControlWorkers() + for _, id := range []string{node1.ID, node2.ID} { + workers.On(id, workerctl.PathModelDelete, func(_ string, data []byte) any { + var req messaging.ModelDeleteRequest + Expect(json.Unmarshal(data, &req)).To(Succeed()) + Expect(req.ModelName).To(Equal("big-model")) + deleteCount.Add(1) + return messaging.ModelDeleteReply{Success: true} + }) + } // Create temp dir for local model files tempDir, err := os.MkdirTemp("", "dist-model-test-*") @@ -176,7 +165,7 @@ var _ = Describe("Model and Backend Managers", Label("Distributed"), func() { appCfg := config.NewApplicationConfig() appCfg.SystemState = ss - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) distMgr := nodes.NewDistributedModelManager(appCfg, ml, adapter) err = distMgr.DeleteModel("big-model") @@ -202,39 +191,24 @@ var _ = Describe("Model and Backend Managers", Label("Distributed"), func() { Expect(registry.Register(context.Background(), node3, true)).To(Succeed()) Expect(registry.MarkUnhealthy(context.Background(), node3.ID)).To(Succeed()) - // Subscribe to backend.delete on all 3 nodes + // All 3 workers serve backend.delete on their control plane. var deleteCount atomic.Int32 - sub1, err := infra.NC.SubscribeReply(messaging.SubjectNodeBackendDelete(node1.ID), func(data []byte, reply func([]byte)) { - var req messaging.BackendDeleteRequest - json.Unmarshal(data, &req) - Expect(req.Backend).To(Equal("my-backend")) - deleteCount.Add(1) - resp, _ := json.Marshal(messaging.BackendDeleteReply{Success: true}) - reply(resp) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub1.Unsubscribe() - - sub2, err := infra.NC.SubscribeReply(messaging.SubjectNodeBackendDelete(node2.ID), func(data []byte, reply func([]byte)) { - var req messaging.BackendDeleteRequest - json.Unmarshal(data, &req) - deleteCount.Add(1) - resp, _ := json.Marshal(messaging.BackendDeleteReply{Success: true}) - reply(resp) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub2.Unsubscribe() + workers := NewControlWorkers() + for _, id := range []string{node1.ID, node2.ID} { + workers.On(id, workerctl.PathBackendDelete, func(_ string, data []byte) any { + var req messaging.BackendDeleteRequest + Expect(json.Unmarshal(data, &req)).To(Succeed()) + Expect(req.Backend).To(Equal("my-backend")) + deleteCount.Add(1) + return messaging.BackendDeleteReply{Success: true} + }) + } var unhealthyReceived atomic.Int32 - sub3, err := infra.NC.SubscribeReply(messaging.SubjectNodeBackendDelete(node3.ID), func(data []byte, reply func([]byte)) { + workers.On(node3.ID, workerctl.PathBackendDelete, func(string, []byte) any { unhealthyReceived.Add(1) - resp, _ := json.Marshal(messaging.BackendDeleteReply{Success: true}) - reply(resp) + return messaging.BackendDeleteReply{Success: true} }) - Expect(err).ToNot(HaveOccurred()) - defer sub3.Unsubscribe() - - FlushNATS(infra.NC) // Create temp dir for local backend files tempDir, err := os.MkdirTemp("", "dist-backend-test-*") @@ -252,7 +226,7 @@ var _ = Describe("Model and Backend Managers", Label("Distributed"), func() { appCfg := config.NewApplicationConfig() appCfg.SystemState = ss - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) distMgr := nodes.NewDistributedBackendManager(appCfg, ml, adapter, registry, nil) err = distMgr.DeleteBackend("my-backend") @@ -275,18 +249,14 @@ var _ = Describe("Model and Backend Managers", Label("Distributed"), func() { Expect(registry.Register(context.Background(), node1, true)).To(Succeed()) var deleteCount atomic.Int32 - sub1, err := infra.NC.SubscribeReply(messaging.SubjectNodeBackendDelete(node1.ID), func(data []byte, reply func([]byte)) { + workers := NewControlWorkers() + workers.On(node1.ID, workerctl.PathBackendDelete, func(_ string, data []byte) any { var req messaging.BackendDeleteRequest - json.Unmarshal(data, &req) + Expect(json.Unmarshal(data, &req)).To(Succeed()) Expect(req.Backend).To(Equal("remote-only-backend")) deleteCount.Add(1) - resp, _ := json.Marshal(messaging.BackendDeleteReply{Success: true}) - reply(resp) + return messaging.BackendDeleteReply{Success: true} }) - Expect(err).ToNot(HaveOccurred()) - defer sub1.Unsubscribe() - - FlushNATS(infra.NC) // Use a temp dir with NO local backend directory — simulates frontend node tempDir, err := os.MkdirTemp("", "dist-backend-remote-only-*") @@ -299,7 +269,7 @@ var _ = Describe("Model and Backend Managers", Label("Distributed"), func() { appCfg := config.NewApplicationConfig() appCfg.SystemState = ss - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) distMgr := nodes.NewDistributedBackendManager(appCfg, ml, adapter, registry, nil) // Should NOT return an error even though the backend doesn't exist locally diff --git a/tests/e2e/distributed/node_lifecycle_test.go b/tests/e2e/distributed/node_lifecycle_test.go index 04b7342e794b..e3fdb1eefcb5 100644 --- a/tests/e2e/distributed/node_lifecycle_test.go +++ b/tests/e2e/distributed/node_lifecycle_test.go @@ -8,6 +8,7 @@ import ( "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/workerctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -17,7 +18,7 @@ import ( "gorm.io/gorm/logger" ) -var _ = Describe("Node Backend Lifecycle (NATS-driven)", Label("Distributed"), func() { +var _ = Describe("Node Backend Lifecycle over the worker control plane", Label("Distributed"), func() { var ( infra *TestInfra db *gorm.DB @@ -37,27 +38,23 @@ var _ = Describe("Node Backend Lifecycle (NATS-driven)", Label("Distributed"), f Expect(err).ToNot(HaveOccurred()) }) - Context("NATS backend.install events", func() { - It("should send backend.install request-reply to a specific node", func() { + Context("backend.install", func() { + It("should send backend.install to a specific node", func() { node := &nodes.BackendNode{ Name: "gpu-node-1", Address: "h1:50051", } Expect(registry.Register(context.Background(), node, true)).To(Succeed()) - // Simulate worker subscribing to backend.install and replying success - infra.NC.SubscribeReply(messaging.SubjectNodeBackendInstall(node.ID), func(data []byte, reply func([]byte)) { + // The worker serves backend.install on its own control plane. + workers := NewControlWorkers() + workers.On(node.ID, workerctl.PathBackendInstall, func(_ string, data []byte) any { var req messaging.BackendInstallRequest - json.Unmarshal(data, &req) + Expect(json.Unmarshal(data, &req)).To(Succeed()) Expect(req.Backend).To(Equal("llama-cpp")) - - resp := messaging.BackendInstallReply{Success: true} - respData, _ := json.Marshal(resp) - reply(respData) + return messaging.BackendInstallReply{Success: true} }) - FlushNATS(infra.NC) - - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) installReply, err := adapter.InstallBackend(node.ID, "llama-cpp", "", "", "", "", "", 0, "", nil) Expect(err).ToNot(HaveOccurred()) Expect(installReply.Success).To(BeTrue()) @@ -69,16 +66,13 @@ var _ = Describe("Node Backend Lifecycle (NATS-driven)", Label("Distributed"), f } Expect(registry.Register(context.Background(), node, true)).To(Succeed()) - // Simulate worker replying with error - infra.NC.SubscribeReply(messaging.SubjectNodeBackendInstall(node.ID), func(data []byte, reply func([]byte)) { - resp := messaging.BackendInstallReply{Success: false, Error: "backend not found"} - respData, _ := json.Marshal(resp) - reply(respData) + // The worker's own verdict: an answer, not a transport failure. + workers := NewControlWorkers() + workers.On(node.ID, workerctl.PathBackendInstall, func(string, []byte) any { + return messaging.BackendInstallReply{Success: false, Error: "backend not found"} }) - FlushNATS(infra.NC) - - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) installReply, err := adapter.InstallBackend(node.ID, "nonexistent", "", "", "", "", "", 0, "", nil) Expect(err).ToNot(HaveOccurred()) Expect(installReply.Success).To(BeFalse()) @@ -86,7 +80,7 @@ var _ = Describe("Node Backend Lifecycle (NATS-driven)", Label("Distributed"), f }) }) - Context("NATS backend.stop events (model unload)", func() { + Context("backend.stop (model unload)", func() { It("should send backend.stop to nodes hosting the model", func() { node := &nodes.BackendNode{ Name: "gpu-node-2", Address: "h2:50051", @@ -95,16 +89,14 @@ var _ = Describe("Node Backend Lifecycle (NATS-driven)", Label("Distributed"), f Expect(registry.SetNodeModel(context.Background(), node.ID, "whisper-large", 0, "loaded", "", 0)).To(Succeed()) var stopReceived atomic.Int32 - sub, err := infra.NC.Subscribe(messaging.SubjectNodeBackendStop(node.ID), func(data []byte) { + workers := NewControlWorkers() + workers.On(node.ID, workerctl.PathBackendStop, func(string, []byte) any { stopReceived.Add(1) + return nil }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - FlushNATS(infra.NC) // Frontend calls UnloadRemoteModel (triggered by UI "Stop" or WatchDog) - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) Expect(adapter.UnloadRemoteModel("whisper-large")).To(Succeed()) Eventually(func() int32 { return stopReceived.Load() }, "5s").Should(Equal(int32(1))) @@ -123,18 +115,15 @@ var _ = Describe("Node Backend Lifecycle (NATS-driven)", Label("Distributed"), f registry.SetNodeModel(context.Background(), node2.ID, "shared-model", 0, "loaded", "", 0) var count atomic.Int32 - sub1, _ := infra.NC.Subscribe(messaging.SubjectNodeBackendStop(node1.ID), func(data []byte) { - count.Add(1) - }) - sub2, _ := infra.NC.Subscribe(messaging.SubjectNodeBackendStop(node2.ID), func(data []byte) { - count.Add(1) - }) - defer sub1.Unsubscribe() - defer sub2.Unsubscribe() - - FlushNATS(infra.NC) + workers := NewControlWorkers() + for _, id := range []string{node1.ID, node2.ID} { + workers.On(id, workerctl.PathBackendStop, func(string, []byte) any { + count.Add(1) + return nil + }) + } - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) adapter.UnloadRemoteModel("shared-model") Eventually(func() int32 { return count.Load() }, "5s").Should(Equal(int32(2))) @@ -150,49 +139,57 @@ var _ = Describe("Node Backend Lifecycle (NATS-driven)", Label("Distributed"), f // The same contract is pinned at unit level by "with no nodes // returns nil" in core/services/nodes/unloader_test.go; keep them // in step. - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, NewControlWorkers().Client(), 3*time.Minute, 15*time.Minute) Expect(adapter.UnloadRemoteModel("nonexistent-model")).To(Succeed()) }) }) - Context("NATS node stop events (full shutdown)", func() { - It("should publish stop event to a node", func() { + Context("node.stop (full shutdown)", func() { + It("should ask the node to shut down over its control plane", func() { node := &nodes.BackendNode{ Name: "stop-me", Address: "h3:50051", } Expect(registry.Register(context.Background(), node, true)).To(Succeed()) var stopped atomic.Int32 - sub, err := infra.NC.Subscribe(messaging.SubjectNodeStop(node.ID), func(data []byte) { + workers := NewControlWorkers() + workers.On(node.ID, workerctl.PathNodeStop, func(string, []byte) any { stopped.Add(1) + return nil }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - FlushNATS(infra.NC) - - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) Expect(adapter.StopNode(node.ID)).To(Succeed()) Eventually(func() int32 { return stopped.Load() }, "5s").Should(Equal(int32(1))) }) }) - Context("NATS subject naming", func() { - It("should generate correct backend lifecycle subjects", func() { - Expect(messaging.SubjectNodeBackendInstall("node-abc")).To(Equal("nodes.node-abc.backend.install")) + Context("wire naming", func() { + // Written out BY HAND, and not derived from the constants: a frontend + // and a worker built from different commits reach each other over these + // literals, and a renamed path is a 404 that looks exactly like a + // broken tunnel. + It("should name the backend lifecycle control verbs", func() { + Expect(workerctl.PathBackendInstall).To(Equal("/v1/control/backend/install")) + Expect(workerctl.PathBackendStop).To(Equal("/v1/control/backend/stop")) + Expect(workerctl.PathNodeStop).To(Equal("/v1/control/node/stop")) + }) + + // The one node subject left, and it is addressed only to AGENT workers: + // they hold no tunnel and subscribe to it to drop cached MCP sessions. + It("should keep the agent worker's backend.stop subject", func() { Expect(messaging.SubjectNodeBackendStop("node-abc")).To(Equal("nodes.node-abc.backend.stop")) - Expect(messaging.SubjectNodeStop("node-abc")).To(Equal("nodes.node-abc.stop")) }) }) - // Design note: LoadModel is a direct gRPC call to node.Address, NOT a NATS event. - // NATS is used for backend.install (install + start process) and backend.stop. - // The SmartRouter calls grpc.NewClient(node.Address).LoadModel() directly. + // Design note: LoadModel is a gRPC call through the worker's tunnel, not a + // control verb. The control plane installs and stops the process; the model + // is loaded into it over the `grpc` stream tag. // // Flow: - // 1. NATS backend.install → worker installs backend + starts gRPC process - // 2. SmartRouter.Route() → gRPC LoadModel(node.Address) directly - // 3. [inference via gRPC] - // 4. NATS backend.stop → worker stops gRPC process + // 1. backend.install → worker installs backend + starts gRPC process + // 2. SmartRouter.Route() → LoadModel over the worker's tunnel + // 3. [inference over the tunnel] + // 4. backend.stop → worker stops gRPC process }) diff --git a/tests/e2e/distributed/router_tracking_test.go b/tests/e2e/distributed/router_tracking_test.go index 75895a372d96..6a4b545e0a3f 100644 --- a/tests/e2e/distributed/router_tracking_test.go +++ b/tests/e2e/distributed/router_tracking_test.go @@ -2,11 +2,11 @@ package distributed_test import ( "context" - "encoding/json" "time" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/workerctl" "github.com/mudler/LocalAI/pkg/grpc/base" pb "github.com/mudler/LocalAI/pkg/grpc/proto" @@ -15,8 +15,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/nats-io/nats.go" - pgdriver "gorm.io/driver/postgres" gormDB "gorm.io/gorm" "gorm.io/gorm/logger" @@ -60,18 +58,17 @@ var _ = Describe("SmartRouter trackingKey", Label("Distributed"), func() { registry, err = nodes.NewNodeRegistry(db) Expect(err).ToNot(HaveOccurred()) - // Mock backend.install handler — always replies success - infra.NC.Conn().Subscribe("nodes.*.backend.install", func(msg *nats.Msg) { - reply := messaging.BackendInstallReply{Success: true} - data, _ := json.Marshal(reply) - msg.Respond(data) + // Mock control plane — backend.install always replies success. + workers := NewControlWorkers() + workers.On(AnyNode, workerctl.PathBackendInstall, func(string, []byte) any { + return messaging.BackendInstallReply{Success: true} + }) + workers.On(AnyNode, workerctl.PathModelsRunning, func(string, []byte) any { + return messaging.ModelsRunningReply{} }) - _, err = infra.NC.Conn().Subscribe("nodes.*.models.running", func(msg *nats.Msg) { - data, _ := json.Marshal(messaging.ModelsRunningReply{}) - _ = msg.Respond(data) + workers.On(AnyNode, workerctl.PathBackendList, func(string, []byte) any { + return messaging.BackendListReply{} }) - Expect(err).NotTo(HaveOccurred()) - FlushNATS(infra.NC) // Start a mock gRPC backend using the same helper as full flow tests llm := &trackingTestLLM{} @@ -85,7 +82,7 @@ var _ = Describe("SmartRouter trackingKey", Label("Distributed"), func() { Expect(registry.Register(context.Background(), node, true)).To(Succeed()) nodeID = node.ID - unloader := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, 3*time.Minute, 15*time.Minute) + unloader := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) router = nodes.NewSmartRouter(registry, nodes.SmartRouterOptions{ Unloader: unloader, }) From 04942027e373da62ada76fe7f4cc52dd9ad467ca Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 20:45:01 +0000 Subject: [PATCH 073/153] test(distributed): pin the no-demotion rule at every call site it is stated Review fix round 1. Two blocking findings and seven non-blocking; both blocking ones are M12's shape again, and this time on the invariant itself. No production behaviour changes here: everything below was already correct and merely unpinned, so re-inserting the defect left all 679 specs green. The only non-comment edits are one struct-field comment and one log message. "A failed control RPC no longer demotes a node" is stated three times in this package and was pinned once, at ListBackends. Putting MarkUnhealthy back at either op-drain site passed. What that buys in production is the fleet-wide eviction this phase exists to prevent: MarkUnhealthy removes a node from ListDuePendingBackendOps AND from scheduling, so a frontend replica that has just lost its tunnels demotes every node it holds an op for, for a reason that is about the frontend. The reconciler's is the worse of the two, being a background loop nobody is watching. Both now have a spec, each with the recorded op failure as its negative control so "still healthy" cannot pass by nothing having happened. The sweep the review asked for found four more rules stated at more call sites than they were pinned at, and two the review had not: The still-installing surfacing at the manager layer has two call sites and was pinned at InstallBackend. Dropping it from UpgradeBackend reported a spent budget as GREEN SUCCESS: the admin sees the upgrade finished while the worker is still re-pulling gigabytes. The agent-node skip has two call sites and was pinned at ListBackends. Without it the fan-out enqueues a row for every agent node, and an agent worker serves no control plane, so that row can never drain: it retries until the dead-letter cap. The still-installing conversion has three call sites and was pinned at two; the legacy force-install fallback was the gap. Its budget was unpinned too, so the new spec asserts both, on the upgrade budget rather than the install one, since the fallback re-fires an install as part of an upgrade. The carrier split has two call sites and was pinned at one. Hardcoding NodeTypeBackend in UnloadRemoteModelContext passed, and an agent node holding a node_models row would then have its stop sent over a tunnel it does not hold, fail, and leave the row behind. The new spec unloads a model held by one node of each kind and asserts each stop went to that node's own carrier and to no other. router_nats_liveness_test.go asserted demote-on-absence, which production can no longer produce, and its header described the pre-cutover world. The exclusion is unreachable by construction rather than by argument: cluster, the package supplying every control-path dial error, does not link nats.go at all. The file now says that, and gains the assertion that IS load-bearing, a table naming each sentinel a control RPC can answer with and requiring that none of them excludes. Widening the exclusion to ErrWorkerUnroutable reddens four of its entries plus the real-adapter scheduling spec. unroutable keeps no budget-first guard and the reason is now written at it: unlike controlFailure it reads one already-recorded error rather than racing a live deadline, and an expiry is not in streamRefusals, so it falls to the umbrella without one. The two implement the same split at two layers and each now names the other. Fourteen comments still described the bus. Among them the reconciler saying a drain would "churn NATS every tick", a spec comment naming a subject builder this branch deleted, and the agent-skip comment explaining the skip by a subscription that no longer exists. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/nodes/interfaces.go | 6 ++ core/services/nodes/managers_distributed.go | 31 ++++--- .../nodes/managers_distributed_test.go | 90 ++++++++++++++++++- core/services/nodes/reconciler.go | 15 ++-- core/services/nodes/reconciler_test.go | 23 +++++ core/services/nodes/router_liveness.go | 9 +- .../nodes/router_nats_liveness_test.go | 58 ++++++++++-- core/services/nodes/unloader.go | 5 +- core/services/nodes/unloader_test.go | 64 +++++++++++++ 9 files changed, 267 insertions(+), 34 deletions(-) diff --git a/core/services/nodes/interfaces.go b/core/services/nodes/interfaces.go index cd1ba03e5c11..7231d5296510 100644 --- a/core/services/nodes/interfaces.go +++ b/core/services/nodes/interfaces.go @@ -264,6 +264,12 @@ var ErrNoWorkerDialer = fmt.Errorf("%w: no worker tunnel dialer is configured", // A reply code this frontend does not recognise is deliberately not in the set // either (see cluster.IsWorkerAnswer), so a newer worker's vocabulary reaches // an older frontend as "no route" and costs a retry rather than a row. +// The control plane's sibling is nodes.controlFailure, which splits the same +// two ways. That one checks the caller's remaining budget FIRST, because it +// races a live deadline against a reply that may arrive in the same instant. +// There is no such race here: this reads ONE error that was already recorded on +// the client, and an expiry is not in streamRefusals, so it falls to the +// umbrella below without a guard. Anyone changing the split must change both. func unroutable(client grpc.Backend) error { // LastDialErrorOf and not a type assertion: the assertion could not see // past a decorator, and SmartRouter hands every routed client out wrapped. diff --git a/core/services/nodes/managers_distributed.go b/core/services/nodes/managers_distributed.go index e3b6ae5b5094..2db66e9d0ef1 100644 --- a/core/services/nodes/managers_distributed.go +++ b/core/services/nodes/managers_distributed.go @@ -16,8 +16,9 @@ import ( "github.com/mudler/xlog" ) -// DistributedModelManager wraps a local ModelManager and adds NATS fan-out -// for model deletion so worker nodes clean up stale files. +// DistributedModelManager wraps a local ModelManager and fans model deletion +// out to the worker nodes so they clean up stale files. The fan-out is a +// control RPC over each worker's tunnel; see RemoteUnloaderAdapter. type DistributedModelManager struct { local galleryop.ModelManager adapter *RemoteUnloaderAdapter @@ -55,8 +56,9 @@ type nodeProgressSink interface { UpdateNodeProgress(opID, nodeID string, np galleryop.NodeProgress) } -// DistributedBackendManager wraps a local BackendManager and adds NATS fan-out -// for backend deletion so worker nodes clean up stale files. +// DistributedBackendManager wraps a local BackendManager and fans backend +// deletion out to the worker nodes so they clean up stale files. The fan-out is +// a control RPC over each worker's tunnel; see RemoteUnloaderAdapter. type DistributedBackendManager struct { local galleryop.BackendManager adapter *RemoteUnloaderAdapter @@ -121,7 +123,7 @@ func (r BackendOpResult) Err() error { // nodes get an immediate attempt; success deletes the row, failure records // the error and leaves the row for the reconciler to retry. // -// `apply` is the NATS round-trip for one node. Returning an error keeps the +// `apply` is the control RPC for one node. Returning an error keeps the // row in the queue and marks the per-node status as "error"; returning nil // deletes the row and reports "success". For non-healthy nodes the status // is "queued" — no attempt is made right now, reconciler will pick it up @@ -167,7 +169,7 @@ func (d *DistributedBackendManager) enqueueAndDrainBackendOp(ctx context.Context continue } // Backend lifecycle ops only make sense on backend-type workers. - // Agent workers don't subscribe to backend.install/delete/list, so + // Agent workers hold no tunnel and serve no control plane, so // enqueueing for them guarantees a forever-retrying row that the // reconciler can never drain. Silently skip - they aren't consumers. if node.NodeType != "" && node.NodeType != NodeTypeBackend { @@ -454,7 +456,7 @@ func (d *DistributedBackendManager) clearSatisfiedInstallRows(ctx context.Contex // InstallBackend fans out installation through the pending-ops queue so // non-healthy nodes get retried when they come back instead of being silently -// skipped. Reply success from the NATS round-trip deletes the queue row; +// skipped. Reply success from the control RPC deletes the queue row; // reply.Success==false is treated as an error so the row stays for retry. // // When op.TargetNodeID is set, only that node is visited - the same allowlist @@ -494,9 +496,10 @@ func (d *DistributedBackendManager) InstallBackend(ctx context.Context, op *gall }) } } - // nil-callback shortcut: when there is nothing to deliver to, - // hand the adapter a nil onProgress so it skips the per-op NATS - // subscription. Matches the pre-Phase-4 bridgeProgressCb semantics. + // nil-callback shortcut: when there is nothing to deliver to, hand the + // adapter a nil onProgress so it discards the worker's progress lines + // instead of decoding them. They ride the install response itself, so + // there is nothing to arrange either way. var onProgressArg func(messaging.BackendInstallProgressEvent) if progressCb != nil || d.progressSink != nil { onProgressArg = onProgress @@ -531,7 +534,7 @@ func (d *DistributedBackendManager) InstallBackend(ctx context.Context, op *gall return nil } -// UpgradeBackend uses a separate NATS subject (backend.upgrade) so the slow +// UpgradeBackend uses a separate control verb (backend.upgrade) so the slow // force-reinstall path doesn't head-of-line-block routine model loads on // the same worker. Only nodes that already report this backend as installed // are targeted — fanning out to every node would ask workers to "upgrade" @@ -629,8 +632,10 @@ func (d *DistributedBackendManager) UpgradeBackend(ctx context.Context, op *gall return hardErr } // Same in-progress surfacing as InstallBackend: a long-running worker - // upgrade that timed out the NATS round-trip must not be reported as - // green success. + // upgrade that outlived the caller's budget must not be reported as green + // success. Pinned by "reports an upgrade that ran out of budget as still + // installing", because this is the second of the rule's two call sites and + // dropping it here left the suite green. for _, n := range result.Nodes { if n.Status == galleryop.NodeStatusRunningOnWorker { return fmt.Errorf("%w: %s", galleryop.ErrWorkerStillInstalling, summarizeRunningOnWorker(result.Nodes)) diff --git a/core/services/nodes/managers_distributed_test.go b/core/services/nodes/managers_distributed_test.go index 62abd3fc9a0e..68b8b642006c 100644 --- a/core/services/nodes/managers_distributed_test.go +++ b/core/services/nodes/managers_distributed_test.go @@ -318,6 +318,67 @@ var _ = Describe("DistributedBackendManager", func() { }) }) + // The agent skip is stated at TWO call sites, the fan-out and + // ListBackends, and was pinned only at ListBackends. Agent workers + // serve no control plane, so a row enqueued for one can never be + // drained: it retries every reconciler tick until the dead-letter cap + // and shows in the UI as an operation that never finishes. + It("enqueues nothing for an agent node, which serves no control plane", func() { + backend := registerHealthyBackend("worker-a", "10.0.0.1:50051") + agent := &BackendNode{Name: "agent-a", NodeType: NodeTypeAgent, Address: "10.0.0.2:50051"} + Expect(registry.Register(ctx, agent, true)).To(Succeed()) + + mc.scriptReply(controlKey(backend.ID, workerctl.PathBackendInstall), + messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.1:50100"}) + + Expect(mgr.InstallBackend(ctx, op("vllm-development"), nil)).To(Succeed()) + + // The backend node WAS asked, which is the negative control: a + // fan-out that skipped everything would satisfy the agent + // assertion by doing nothing at all. + Expect(mc.callSubjects()).To(Equal([]string{controlKey(backend.ID, workerctl.PathBackendInstall)})) + rows, err := registry.ListPendingBackendOps(ctx) + Expect(err).ToNot(HaveOccurred()) + for _, row := range rows { + Expect(row.NodeID).ToNot(Equal(agent.ID), + "an agent node cannot drain a backend op, so it must never be given one") + } + }) + + // The admin fan-out is the third call site of the rule that a failed + // control RPC does not demote a node, and it was the unpinned one. The + // rule is stated in three places in this package and, before this spec, + // pinned only at ListBackends. + // + // What the demotion buys in production is the fleet-wide eviction this + // phase exists to prevent: MarkUnhealthy removes the node from + // ListDuePendingBackendOps AND from scheduling, so a frontend replica + // re-homing its tunnels would take out every node it has an op for. + Context("when the install RPC could not be routed", func() { + It("records the failure without demoting the node", func() { + node := registerHealthyBackend("worker-unroutable", "10.0.0.8:50051") + mc.scriptUnroutable(node.ID) + + err := mgr.InstallBackend(ctx, op("vllm"), nil) + Expect(err).To(HaveOccurred()) + // Not the still-installing soft path: that branch returns before + // the failure handling this spec is about, so without this the + // assertions below could pass on the wrong branch. + Expect(errors.Is(err, galleryop.ErrWorkerStillInstalling)).To(BeFalse()) + + // The recorded failure witnesses that the fan-out ran and + // reached the branch that used to demote. + rows, listErr := registry.ListPendingBackendOps(ctx) + Expect(listErr).ToNot(HaveOccurred()) + Expect(rows).To(HaveLen(1)) + Expect(rows[0].Attempts).To(Equal(1)) + + after, getErr := registry.Get(ctx, node.ID) + Expect(getErr).ToNot(HaveOccurred()) + Expect(after.Status).To(Equal(StatusHealthy)) + }) + }) + Context("ListBackends clears confirmed install rows", func() { It("deletes the pending_backend_ops install row when the backend is reported installed on its target node", func() { node := registerHealthyBackend("worker-a", "10.0.0.5:50051") @@ -573,7 +634,7 @@ var _ = Describe("DistributedBackendManager", func() { scriptNoBackends(lacks.ID) mc.scriptReply(controlKey(has.ID, workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: true}) - // Deliberately don't script SubjectNodeBackendUpgrade for `lacks`: + // Deliberately don't script backend.upgrade for `lacks`: // if the manager attempts it, the scripted-client default returns // fakeNoRespondersErr and the assertion below fails loudly. @@ -667,6 +728,33 @@ var _ = Describe("DistributedBackendManager", func() { }) }) + // The still-installing surfacing has TWO call sites, InstallBackend and + // this one, and was pinned only at InstallBackend. Dropping it here + // reports a spent budget as GREEN SUCCESS: the admin sees the upgrade + // finished while the worker is still re-pulling gigabytes, and the row + // the reconciler needs to confirm it is invisible in the UI. + It("reports an upgrade that ran out of budget as still installing, not as success", func() { + n := registerHealthyBackend("worker-slow", "10.0.0.1:50051") + scriptInstalled("vllm-development", n.ID) + // Only the upgrade verb hangs: backend.list must still answer, or + // the manager never gets as far as the node it would upgrade. + mc.scriptHang(controlKey(n.ID, workerctl.PathBackendUpgrade)) + slow := &DistributedBackendManager{ + local: stubLocalBackendManager{}, + adapter: NewRemoteUnloaderAdapter(nil, nil, mc.controlClient(), time.Minute, 200*time.Millisecond), + registry: registry, + } + + err := slow.UpgradeBackend(ctx, upgradeOp("vllm-development"), nil) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, galleryop.ErrWorkerStillInstalling)).To(BeTrue(), + "a spent budget must not read as a finished upgrade, got %v", err) + + rows, listErr := registry.ListPendingBackendOps(ctx) + Expect(listErr).ToNot(HaveOccurred()) + Expect(rows).To(HaveLen(1), "the row the reconciler confirms the outcome from must survive") + }) + // Rolling-update fallback: pre-2026-05-08 workers do not serve // backend.upgrade, so the manager catches the worker's own 404 and // re-fires the legacy backend.install Force=true on the same node. diff --git a/core/services/nodes/reconciler.go b/core/services/nodes/reconciler.go index 9bee9219d8fe..b493075ea83e 100644 --- a/core/services/nodes/reconciler.go +++ b/core/services/nodes/reconciler.go @@ -58,7 +58,7 @@ type ModelProber interface { } // NodeProcessLister asks a worker which model backend processes it currently -// has running. Implemented by RemoteUnloaderAdapter over NATS. +// has running. Implemented by RemoteUnloaderAdapter over the worker's tunnel. // // This is the sounder liveness signal: the worker owns the process table, so // its answer does not depend on whether a backend is busy. A health probe @@ -162,7 +162,7 @@ type ReplicaReconciler struct { registry *NodeRegistry scheduler ModelScheduler // interface for scheduling new models unloader NodeCommandSender - adapter *RemoteUnloaderAdapter // NATS sender for pending-op drain + adapter *RemoteUnloaderAdapter // control-RPC sender for the pending-op drain prober ModelProber // health probe for model gRPC addrs db *gorm.DB interval time.Duration @@ -213,7 +213,7 @@ type ReplicaReconcilerOptions struct { Registry *NodeRegistry Scheduler ModelScheduler Unloader NodeCommandSender - // Adapter is the NATS sender used to retry pending backend ops. When nil, + // Adapter is the control-RPC sender used to retry pending backend ops. When nil, // the state-reconciler pending-drain pass is a no-op (single-node mode). Adapter *RemoteUnloaderAdapter // RegistrationToken is the bearer token the default gRPC prober presents to @@ -453,8 +453,9 @@ func (rc *ReplicaReconciler) drainPendingBackendOps(ctx context.Context) { // Dead-letter cap: after maxAttempts the row is the reconciler // equivalent of a poison message. Delete it loudly so the queue - // doesn't churn NATS every tick forever — operators can re-issue - // the op from the UI if they still want it applied. + // doesn't churn a control RPC at the worker every tick forever — + // operators can re-issue the op from the UI if they still want it + // applied. if op.Attempts+1 >= maxPendingBackendOpAttempts { xlog.Error("Reconciler: abandoning pending backend op after max attempts", "op", op.Op, "backend", op.Backend, "node", op.NodeID, @@ -703,8 +704,8 @@ const workerMissesBeforeReap = 2 // from ever being mistaken for a dead one. // // A worker that cannot be reached is skipped rather than treated as empty. A -// messaging failure says nothing about the processes, and assuming the worst -// would delete a whole node's rows on a transient NATS blip; the port probe +// failure to route says nothing about the processes, and assuming the worst +// would delete a whole node's rows every time a tunnel re-homed; the port probe // remains as the fallback for those nodes. func (rc *ReplicaReconciler) reconcileNodeProcesses(ctx context.Context) { if rc.processLister == nil { diff --git a/core/services/nodes/reconciler_test.go b/core/services/nodes/reconciler_test.go index 075fee25fee7..64171d1859d3 100644 --- a/core/services/nodes/reconciler_test.go +++ b/core/services/nodes/reconciler_test.go @@ -947,6 +947,29 @@ var _ = Describe("ReplicaReconciler — state reconciliation", func() { Expect(rows[0].Attempts).To(Equal(1)) }) + // A failed op must not DEMOTE the node, and this is the second of the + // rule's three call sites. Marking unhealthy here takes the node out of + // ListDuePendingBackendOps and out of scheduling at the same time, so a + // frontend replica that has just lost its tunnels would evict every node + // with a queued op, fleet-wide, for a reason that is about the frontend. + // + // The surviving row is the negative control: it witnesses that the drain + // actually ran and actually failed, so "still healthy" cannot pass by + // nothing having happened. + It("does NOT demote a node whose queued op it could not route", func() { + workers.scriptUnroutable(node.ID) + + rc.drainPendingBackendOps(context.Background()) + + rows := queuedOps() + Expect(rows).To(HaveLen(1), "the drain must have run and failed") + Expect(rows[0].Attempts).To(Equal(1)) + + after, err := registry.Get(context.Background(), node.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(after.Status).To(Equal(StatusHealthy)) + }) + // The same rule for the other non-answer: no route at all. The install // is unreachable too here, so the witness is the error the drain // RECORDED, which names the verb that actually failed. diff --git a/core/services/nodes/router_liveness.go b/core/services/nodes/router_liveness.go index 9809d1b2d715..38c078384496 100644 --- a/core/services/nodes/router_liveness.go +++ b/core/services/nodes/router_liveness.go @@ -41,8 +41,11 @@ func (r *SmartRouter) nodeAnswersOnBus(node *BackendNode) bool { return !errors.Is(err, nats.ErrNoResponders) } -// pickReachableNode calls selectNode until it yields a node that still answers -// on the bus, and returns nil when it cannot find one. +// pickReachableNode calls selectNode until it yields a node that nodeAnswersOnBus +// does not exclude, and returns nil when it cannot find one. +// +// No control-RPC outcome excludes, so today this returns the first node the +// selector offers. The loop stays until the scheduler reads cluster.Presence. // // A node that does not answer is marked unhealthy before the next attempt. That // both removes it from the next selection, which queries only healthy nodes, @@ -57,7 +60,7 @@ func (r *SmartRouter) pickReachableNode(ctx context.Context, selectNode func() * if r.nodeAnswersOnBus(node) { return node } - xlog.Warn("Scheduled node is not answering on the bus, marking unhealthy and re-scheduling", + xlog.Warn("Scheduled node was excluded by the liveness probe, marking unhealthy and re-scheduling", "node", node.Name, "nodeID", node.ID) if err := r.registry.MarkUnhealthy(ctx, node.ID); err != nil { // Without the demotion the next selection would hand back the same diff --git a/core/services/nodes/router_nats_liveness_test.go b/core/services/nodes/router_nats_liveness_test.go index ec4820c9f0a8..d35d5a088a87 100644 --- a/core/services/nodes/router_nats_liveness_test.go +++ b/core/services/nodes/router_nats_liveness_test.go @@ -3,17 +3,34 @@ package nodes import ( "context" "errors" + "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/cluster" ) -// A node's stored status comes from its HTTP heartbeat, but work is dispatched -// over NATS. A worker that dies stops answering on the bus immediately and -// keeps its healthy status until the heartbeat ages out, so the scheduler could -// commit to a node it could not reach. The request then failed outright with -// "no responders available" rather than moving to a node that was actually up. -var _ = Describe("Scheduling past a node that left the bus", func() { +// pickReachableNode's exclusion branch, which no longer runs in production. +// +// It excludes on nats.ErrNoResponders alone, and since the control plane moved +// onto the tunnel nothing can produce that value: cluster, the package that +// supplies every control-path dial error, does not link nats.go at all. So the +// specs below that drive fakeUnloader.deadNodes exercise a branch a real +// adapter cannot enter. They are kept, rather than deleted, because the LOOP is +// still compiled and still executed on every scheduling decision, and its shape +// (the retry bound, and stopping when the demotion itself fails) is what stops a +// future re-wiring from spinning. They go with the exclusion in the task that +// gives the scheduler cluster.Presence. +// +// What is load-bearing TODAY is the opposite assertion, and it lives in two +// places: the DescribeTable at the bottom of this file, which pins that none of +// the sentinels a control RPC actually produces excludes anything, and the +// scheduling specs in unloader_ping_test.go, which prove the same thing through +// a REAL RemoteUnloaderAdapter. This file's doubles cannot see a carrier +// change; that is why they stayed green through the window in which every +// healthy worker read as absent. +var _ = Describe("The retired bus-absence exclusion", func() { var ( reg *fakeModelRouter fake *fakeUnloader @@ -96,8 +113,8 @@ var _ = Describe("Scheduling past a node that left the bus", func() { Expect(fake.pingCalls).To(Equal([]string{"dead-node"})) }) - // Only a no-responders answer proves absence. Excluding a node that is - // merely slow would cost real capacity. + // The half of the branch that IS still reachable: every error a control + // RPC can produce lands here. It("keeps a node that answers slowly or errors for another reason", func() { slow := newNode("slow-node") fake.pingErr = errors.New("timeout waiting for reply") @@ -115,4 +132,29 @@ var _ = Describe("Scheduling past a node that left the bus", func() { Expect(plain.pickReachableNode(context.Background(), selectorReturning(node))).To(Equal(node)) }) + + // The rule that actually holds now, stated in the vocabulary a control RPC + // answers in. Widening the exclusion to any of these puts the scheduler + // back in the state this task was opened to fix: a worker that is + // heartbeating and serving, unroutable from THIS replica for a moment + // while its tunnel re-homes, demoted for every scheduler in the cluster. + // + // unloader_ping_test.go proves the same thing through a real adapter and a + // real transport. This table is the cheap negative control beside it: it + // names each sentinel, so a widening is attributed to the sentinel that + // caused it rather than to "something in the ping". + DescribeTable("never excludes a node on anything a control RPC can answer with", + func(cause error) { + fake.pingErr = cause + node := newNode("live-node") + + Expect(router.pickReachableNode(context.Background(), selectorReturning(node))).To(Equal(node)) + Expect(reg.markedUnhealthy).To(BeEmpty()) + }, + Entry("no route to the worker", fmt.Errorf("control rpc: %w: %w", ErrWorkerUnroutable, cluster.ErrNoRoute)), + Entry("no tunnel dialer wired", ErrNoWorkerDialer), + Entry("the worker is older than this frontend", ErrWorkerControlUnsupported), + Entry("the caller's budget ran out", fmt.Errorf("control rpc: %w: %w", ErrWorkerUnroutable, context.DeadlineExceeded)), + Entry("the worker's own refusal", fmt.Errorf("%w: no such process", cluster.ErrStreamTargetUnavailable)), + ) }) diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index 3d961fbb5617..8240af22dfde 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -133,8 +133,9 @@ func (a *RemoteUnloaderAdapter) StopModelReplica(ctx context.Context, nodeID str return reply, nil } -// UnloadRemoteModel finds the node(s) hosting the given model and tells them -// to stop their backend process via NATS backend.stop event. +// UnloadRemoteModel finds the node(s) hosting the given model and tells each +// to stop its backend process. The carrier is decided per node by its type, +// which is why stopBackend takes one: see stopBackend. // The worker process handles a bounded Free() followed by process termination; // forced shutdown skips Free(). // This is called by ModelLoader.deleteProcess() when process == nil (remote model). diff --git a/core/services/nodes/unloader_test.go b/core/services/nodes/unloader_test.go index b83d8abae49e..2f33ad09bfeb 100644 --- a/core/services/nodes/unloader_test.go +++ b/core/services/nodes/unloader_test.go @@ -297,6 +297,32 @@ var _ = Describe("RemoteUnloaderAdapter", func() { Expect(json.Unmarshal(workers.calls[0].Data, &payload)).To(Succeed()) Expect(payload).To(Equal(messaging.BackendStopRequest{Backend: "llama", Force: true})) }) + + // The carrier split has TWO call sites, which is why stopBackend takes + // nodeType as a parameter. StopBackend is pinned in both directions + // below; this is the other caller, and hardcoding NodeTypeBackend here + // used to leave the whole suite green. An agent node holding a + // node_models row would then have its stop sent over a tunnel it does + // not hold, the call would fail, and the replica row would be left + // behind. + It("routes each stop by ITS node's type, not by one choice for the unload", func() { + locator.nodes = []BackendNode{ + {ID: "agent-1", Name: "agent", NodeType: NodeTypeAgent}, + {ID: "backend-1", Name: "gpu-1", NodeType: NodeTypeBackend}, + } + scriptStop("backend-1") + + Expect(adapter.UnloadRemoteModelContext(context.Background(), "llama", false)).To(Succeed()) + + // The agent's stop went to the bus and only the agent's did; the + // backend's went over the tunnel and only the backend's did. + Expect(bus.publishedSubjects()).To(Equal([]string{messaging.SubjectNodeBackendStop("agent-1")})) + Expect(workers.callSubjects()).To(Equal([]string{controlKey("backend-1", workerctl.PathBackendStop)})) + // Both rows dropped, which is the negative control: a stop put on + // the carrier the other kind of worker listens on fails, and a + // failed stop keeps its row. + Expect(locator.removedPairs).To(HaveLen(2)) + }) }) // The carrier split. It is the one verb of the ten that is decided by the @@ -535,6 +561,44 @@ var _ = Describe("RemoteUnloaderAdapter timeout handling", func() { Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) }) + // The still-installing rule has THREE call sites: InstallBackend, + // UpgradeBackend and the legacy force-install fallback. The first two are + // pinned by the specs above and by the timeout-configuration pair; the + // fallback was the unpinned one, and widening its guard to any error left + // the whole suite green. A permanently unroutable worker on the fallback + // path would then report as still-installing forever, which galleryop + // treats as a soft failure: the retry is pushed out and the operator never + // sees the error. + It("reports a spent budget on the legacy fallback as still-installing, on the upgrade budget", func() { + workers := newScriptedControlWorkers() + workers.scriptHang(controlKey("n1", workerctl.PathBackendInstall)) + // Install and upgrade budgets far apart: the fallback re-fires an + // INSTALL but is part of an upgrade, so it must wait the upgrade + // budget. Waiting the install one would satisfy a bare + // still-installing assertion while carrying the wrong deadline. + adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 100*time.Millisecond, 500*time.Millisecond) + + started := time.Now() + _, err := adapter.installWithForceFallback("n1", "llama-cpp", "[]", "", "", "", 0, "", nil) + elapsed := time.Since(started) + + Expect(errors.Is(err, galleryop.ErrWorkerStillInstalling)).To(BeTrue(), + "a spent budget is reported as still-installing, got %v", err) + Expect(elapsed).To(BeNumerically(">=", 500*time.Millisecond)) + }) + + It("does NOT report an unroutable legacy fallback as still installing", func() { + workers := newScriptedControlWorkers() + // The verb is not scripted, so the worker fails to SERVE it rather + // than answering; that is a 5xx and lands under the no-route umbrella. + adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), time.Minute, time.Minute) + + _, err := adapter.installWithForceFallback("n1", "llama-cpp", "[]", "", "", "", 0, "", nil) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, galleryop.ErrWorkerStillInstalling)).To(BeFalse()) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) + }) + It("does not read a message merely containing the words nats timeout as a timeout", func() { // The string match the bus carrier needed would have matched a worker // error that quoted the phrase, and would have turned a real failure From dccc5368e6f32e49e03bc0742915c64b94732a92 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 21:05:28 +0000 Subject: [PATCH 074/153] feat(distributed): serve file staging over the worker tunnel The four nodes..files.* subjects were the last commands a serve-backend worker took off the bus. They are now HTTP routes under workerctl.Prefix, on the same loopback server and behind the same bearer check as the ten lifecycle verbs, so the frontend reaches them through the worker's tunnel. files.listdir is the verb this matters most for. Its reply had to fit a payload the bus would carry, which put a wide model directory close to the limit; a response body has no such ceiling, so nothing truncates the listing at either end. A short listing reads to the frontend as files the worker does not have. S3NATSFileStager becomes S3FileStager and calls ControlClient, which means every failure now lands in the bucket phase 3 exists to keep straight: a route this frontend could not use is unroutable and nothing may act on it, while the worker's own answer, including "that file is not there", is evidence a caller may act on. Each RPC's deadline is DERIVED FROM the caller's context rather than started fresh, at every one of the five call sites, so a caller that gave up stops the RPC too. A worker started without an object store mounts no file verb at all and answers 404, which is the same answer a build too old to know them gives. The subjects and the backend worker's files.> publish grant go with them; a backend worker now publishes nowhere but its own inbox. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 21 +- core/services/messaging/subjects.go | 89 +--- core/services/nodes/file_stager_s3.go | 142 +++--- core/services/nodes/file_stager_s3_test.go | 173 ++++++++ core/services/worker/control_files.go | 268 ++++++++++++ core/services/worker/control_files_test.go | 301 +++++++++++++ core/services/worker/control_routes.go | 69 +-- core/services/worker/control_routes_test.go | 36 +- .../worker/file_stager_roundtrip_test.go | 207 +++++++++ core/services/worker/file_staging.go | 405 ------------------ core/services/worker/reply.go | 17 - core/services/worker/worker.go | 44 +- core/services/workerctl/paths.go | 15 + core/services/workerctl/paths_test.go | 8 + docs/content/features/distributed-mode.md | 32 +- pkg/natsauth/mint_test.go | 5 +- pkg/natsauth/permissions.go | 19 +- pkg/natsauth/permissions_coverage_test.go | 49 +-- tests/e2e/distributed/file_staging_test.go | 29 +- 19 files changed, 1257 insertions(+), 672 deletions(-) create mode 100644 core/services/nodes/file_stager_s3_test.go create mode 100644 core/services/worker/control_files.go create mode 100644 core/services/worker/control_files_test.go create mode 100644 core/services/worker/file_stager_roundtrip_test.go delete mode 100644 core/services/worker/reply.go diff --git a/core/application/distributed.go b/core/application/distributed.go index 2586e403ee57..8c32d777c1c6 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -402,11 +402,22 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } xlog.Info("File manager initialized", "cacheDir", cacheDir) + // The frontend's control plane client. It reaches every worker over that + // worker's own tunnel, on the same `http` stream tag the file stager below + // uses, so a control RPC to a worker another replica holds is relayed the + // way an inference request is. + // + // ONE of these for the whole frontend, and the S3 file stager takes this + // one rather than minting a second. The client caches an http.Client per + // node, which is what keeps a worker's tunnel stream warm between verbs; a + // second client would open its own and the two would never share one. + controlClient := nodes.NewControlClient(workerHTTPDialer, cfg.Distributed.RegistrationToken) + // Create FileStager for distributed file transfer var fileStager nodes.FileStager if cfg.Distributed.StorageURL != "" { - fileStager = nodes.NewS3NATSFileStager(fileMgr, natsClient) - xlog.Info("File stager initialized (S3+NATS)") + fileStager = nodes.NewS3FileStager(fileMgr, controlClient) + xlog.Info("File stager initialized (object store + worker tunnel)") } else { fileStager = nodes.NewHTTPFileStager(func(nodeID string) (string, error) { node, err := registry.Get(context.Background(), nodeID) @@ -422,12 +433,6 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade }, cfg.Distributed.RegistrationToken, workerHTTPDialer) xlog.Info("File stager initialized (HTTP direct transfer)") } - // The frontend's control plane client. It reaches every worker over that - // worker's own tunnel, on the same `http` stream tag the file stager above - // uses, so a control RPC to a worker another replica holds is relayed the - // way an inference request is. - controlClient := nodes.NewControlClient(workerHTTPDialer, cfg.Distributed.RegistrationToken) - // Create RemoteUnloaderAdapter — needed by SmartRouter and startup.go remoteUnloader := nodes.NewRemoteUnloaderAdapter( registry, diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index bcfc9009d178..3bb8a49eb57d 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -279,30 +279,6 @@ type BackendStopRequest struct { Force bool `json:"force,omitempty"` } -// BackendStopReply is the worker's answer to a backend.stop request. -// -// backend.stop had no reply until this type existed. The controller published -// and returned success as soon as the local publish succeeded, so a stop that -// killed nothing, and a stop that failed outright, both looked identical to a -// stop that worked. An operator calling the unload endpoint got HTTP 200 while -// the backend kept running and holding its VRAM. -type BackendStopReply struct { - Success bool `json:"success"` - Error string `json:"error,omitempty"` - - // StoppedProcessKeys names every `modelID#replica` process the worker - // terminated while serving this request. - StoppedProcessKeys []string `json:"stopped_process_keys,omitempty"` - - // ReportsStoppedProcesses distinguishes "this worker enumerates what it - // stopped and stopped nothing" from "this worker predates the field", the - // same way BackendDeleteReply does. Both send an empty list and only the - // first is authoritative, so a controller that cannot tell them apart would - // read silence as a completed stop — the exact conclusion this reply exists - // to prevent. - ReportsStoppedProcesses bool `json:"reports_stopped_processes,omitempty"` -} - // SubjectNodeBackendStop tells an AGENT worker that a backend is going away, so // it can close the MCP sessions it cached for that backend. // @@ -400,38 +376,9 @@ type RunningModelInfo struct { Address string `json:"address,omitempty"` } -// File Staging (Request-Reply — targeted to specific nodes) -// These subjects use request-reply for synchronous file operations. - -// SubjectNodeFilesEnsure tells a serve-backend node to download an S3 key to its local cache. -// Reply: {local_path, error} -func SubjectNodeFilesEnsure(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".files.ensure" -} - -// SubjectNodeFilesStage tells a serve-backend node to upload a local file to S3. -// Reply: {key, error} -func SubjectNodeFilesStage(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".files.stage" -} - -// SubjectNodeFilesRelease tells a serve-backend node to evict one request's ephemeral cache keys. -// Reply: {error} -func SubjectNodeFilesRelease(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".files.release" -} - -// SubjectNodeFilesTemp tells a serve-backend node to allocate a temp file. -// Reply: {local_path, error} -func SubjectNodeFilesTemp(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".files.temp" -} - -// SubjectNodeFilesListDir tells a serve-backend node to list files in a directory. -// Reply: {files: [...], error} -func SubjectNodeFilesListDir(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".files.listdir" -} +// File staging is no longer carried here. The four nodes..files.* subjects +// are HTTP routes under workerctl.Prefix, served on the worker's own server and +// reached through its tunnel, so no subject is minted for them. // Cache Invalidation (Pub/Sub — broadcast to all instances) const ( @@ -484,29 +431,8 @@ const subjectSyncStatePrefix = "state." const ( SubjectPrefixCacheObserve = "prefixcache.observe" SubjectPrefixCacheInvalidate = "prefixcache.invalidate" - SubjectPrefixCachePressure = "prefixcache.pressure" - SubjectPrefixCacheResidency = "prefixcache.residency" -) - -// PrefixCacheOperation describes a backend-reported KV-cache residency change. -type PrefixCacheOperation string - -const ( - PrefixCacheStore PrefixCacheOperation = "store" - PrefixCacheRemove PrefixCacheOperation = "remove" - PrefixCacheClear PrefixCacheOperation = "clear" ) -// PrefixCacheResidencyEvent reports exact backend KV-cache residency. Chain is -// the compatible shallow-to-deep prefix hash chain used by the router. -type PrefixCacheResidencyEvent struct { - Operation PrefixCacheOperation `json:"operation"` - Model string `json:"model"` - NodeID string `json:"node_id"` - Replica int `json:"replica"` - Chain []uint64 `json:"chain,omitempty"` -} - // PrefixCacheObserveEvent announces that the replica (NodeID, Replica) served a // request whose prefix chain ends at the given hashes for model. Chain is the // full shallow-to-deep hash chain so peers can insert the same path. Affinity is @@ -528,12 +454,3 @@ type PrefixCacheInvalidateEvent struct { NodeID string `json:"node_id"` Replica int `json:"replica"` } - -// PrefixCachePressureEvent announces one forced-disturb observed by a frontend. -// ID lets the publisher ignore its own NATS echo and all frontends ignore -// redelivery without inflating the autoscale signal. -type PrefixCachePressureEvent struct { - ID string `json:"id"` - Model string `json:"model"` - Reset bool `json:"reset,omitempty"` -} diff --git a/core/services/nodes/file_stager_s3.go b/core/services/nodes/file_stager_s3.go index fff6d1859cbe..e60ddb5d1c8a 100644 --- a/core/services/nodes/file_stager_s3.go +++ b/core/services/nodes/file_stager_s3.go @@ -6,29 +6,47 @@ import ( "fmt" "time" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/storage" + "github.com/mudler/LocalAI/core/services/workerctl" "github.com/mudler/xlog" ) -// S3NATSFileStager implements FileStager using S3 for storage and NATS -// request-reply for coordination with backend nodes. Both frontend and -// backend nodes share the same S3 bucket. The flow is: +// S3FileStager implements FileStager using an object store for the bytes and +// the worker's tunnelled control plane for coordination. Both frontend and +// worker share the same bucket. The flow is: // -// 1. Frontend uploads file to S3 -// 2. Frontend sends NATS request to nodes.{nodeID}.files.ensure -// 3. Backend downloads from S3 to local cache, replies with local path -type S3NATSFileStager struct { - fm *storage.FileManager - nats messaging.MessagingClient +// 1. Frontend uploads the file to the object store +// 2. Frontend calls POST /v1/control/files/ensure on the worker's tunnel +// 3. Worker downloads from the store to its local cache and replies with the +// local path +type S3FileStager struct { + fm *storage.FileManager + control *ControlClient } -// NewS3NATSFileStager creates a new S3+NATS file stager. -func NewS3NATSFileStager(fm *storage.FileManager, nats messaging.MessagingClient) *S3NATSFileStager { - return &S3NATSFileStager{fm: fm, nats: nats} +// NewS3FileStager creates a file stager that moves bytes through fm and +// commands workers over control. +func NewS3FileStager(fm *storage.FileManager, control *ControlClient) *S3FileStager { + return &S3FileStager{fm: fm, control: control} } -// NATS request/reply message types +// The two budgets a file-staging RPC gets. They are the ones the NATS +// request-reply timeouts carried, kept verbatim: a transfer verb waits out a +// multi-gigabyte copy, and a metadata verb does not. +// +// They are CEILINGS on the caller's own budget rather than budgets of their +// own. Every call below derives its deadline from the caller's context, so a +// caller that has already given up stops the RPC too; deriving from a fresh +// background context would keep commanding a worker nobody is listening to and +// would let a late answer be read as a live one. +const ( + fileTransferRPCTimeout = 10 * time.Minute + fileMetadataRPCTimeout = 30 * time.Second +) + +// Control request/reply message types. Their JSON is the shape the +// nodes..files.* subjects carried, so the worker's handler bodies did not +// have to change when the carrier did. type fileEnsureRequest struct { Key string `json:"key"` @@ -74,10 +92,24 @@ type fileListDirReply struct { Error string `json:"error,omitempty"` } -// EnsureRemote uploads a local file to S3 (if not already there) and sends -// a NATS request-reply to the backend node to download it locally. -func (s *S3NATSFileStager) EnsureRemote(ctx context.Context, nodeID, localPath, key string) (string, error) { - // Upload to S3 if not already present +// callWorker issues one file-staging RPC under a deadline derived from the +// caller's context. +// +// It exists so the derivation is written ONCE. Five call sites each repeating +// context.WithTimeout is five chances for one of them to start from a +// background context instead, and that one site would then keep commanding a +// worker after its caller had gone, with nothing else in the suite any redder +// for it. +func (s *S3FileStager) callWorker(ctx context.Context, nodeID, path string, budget time.Duration, req, reply any) error { + rpcCtx, cancel := context.WithTimeout(ctx, budget) + defer cancel() + return s.control.Call(rpcCtx, nodeID, path, req, reply) +} + +// EnsureRemote uploads a local file to the object store (if not already there) +// and tells the worker to fetch it. +func (s *S3FileStager) EnsureRemote(ctx context.Context, nodeID, localPath, key string) (string, error) { + // Upload to the store if not already present exists, _ := s.fm.Exists(ctx, key) if !exists { // Wrap with progress reporting if a staging callback is available @@ -88,14 +120,13 @@ func (s *S3NATSFileStager) EnsureRemote(ctx context.Context, nodeID, localPath, } } if err := s.fm.UploadWithProgress(ctx, key, localPath, progressFn); err != nil { - return "", fmt.Errorf("uploading %s to S3: %w", localPath, err) + return "", fmt.Errorf("uploading %s to the object store: %w", localPath, err) } } - // Send NATS request-reply to backend - subject := messaging.SubjectNodeFilesEnsure(nodeID) - reply, err := messaging.RequestJSON[fileEnsureRequest, fileEnsureReply](s.nats, subject, fileEnsureRequest{Key: key}, 10*time.Minute) - if err != nil { + var reply fileEnsureReply + if err := s.callWorker(ctx, nodeID, workerctl.PathFilesEnsure, fileTransferRPCTimeout, + fileEnsureRequest{Key: key}, &reply); err != nil { return "", err } if reply.Error != "" { @@ -106,36 +137,38 @@ func (s *S3NATSFileStager) EnsureRemote(ctx context.Context, nodeID, localPath, return reply.LocalPath, nil } -// FetchRemote tells the backend to upload a file to S3, then downloads it locally. -func (s *S3NATSFileStager) FetchRemote(ctx context.Context, nodeID, remotePath, localDst string) error { - // Tell backend to upload to S3 +// FetchRemote tells the worker to upload a file to the object store, then +// downloads it locally. +func (s *S3FileStager) FetchRemote(ctx context.Context, nodeID, remotePath, localDst string) error { key := storage.EphemeralKey(remotePath, "fetch", "output") return s.fetchRemoteWithKey(ctx, nodeID, remotePath, key, localDst, true) } -// FetchRemoteByKey tells the backend to upload a file (identified by key) to S3, -// then downloads it locally. The key is used as-is for S3 routing. -func (s *S3NATSFileStager) FetchRemoteByKey(ctx context.Context, nodeID, key, localDst string) error { - // For S3 mode, we still need the remote path — derive it from the key. - // The backend serves the file from its data dir based on the key prefix. +// FetchRemoteByKey tells the worker to upload a file (identified by key) to the +// object store, then downloads it locally. The key is used as-is for routing. +func (s *S3FileStager) FetchRemoteByKey(ctx context.Context, nodeID, key, localDst string) error { + // The remote path is derived from the key: the worker serves the file from + // its data dir based on the key prefix. remotePath := "/" + key // e.g. "/data/quantization/{jobID}/model.gguf" return s.fetchRemoteWithKey(ctx, nodeID, remotePath, key, localDst, true) } -func (s *S3NATSFileStager) fetchRemoteWithKey(ctx context.Context, nodeID, remotePath, key, localDst string, cleanup bool) error { - subject := messaging.SubjectNodeFilesStage(nodeID) - reply, err := messaging.RequestJSON[fileStageRequest, fileStageReply](s.nats, subject, fileStageRequest{LocalPath: remotePath, Key: key}, 10*time.Minute) - if err != nil { +func (s *S3FileStager) fetchRemoteWithKey(ctx context.Context, nodeID, remotePath, key, localDst string, cleanup bool) error { + var reply fileStageReply + if err := s.callWorker(ctx, nodeID, workerctl.PathFilesStage, fileTransferRPCTimeout, + fileStageRequest{LocalPath: remotePath, Key: key}, &reply); err != nil { return err } if reply.Error != "" { return fmt.Errorf("backend stage failed: %s", reply.Error) } - // Download from S3 to local cache + // Download from the store to the local cache. The CALLER's context bounds + // this rather than the RPC's, because it is this frontend's own work and + // the RPC it belonged to has already finished. cachedPath, err := s.fm.Download(ctx, key) if err != nil { - return fmt.Errorf("downloading %s from S3: %w", key, err) + return fmt.Errorf("downloading %s from the object store: %w", key, err) } // Copy from cache to destination @@ -151,11 +184,11 @@ func (s *S3NATSFileStager) fetchRemoteWithKey(ctx context.Context, nodeID, remot return nil } -// AllocRemoteTemp asks the backend to allocate a temp file via NATS request-reply. -func (s *S3NATSFileStager) AllocRemoteTemp(ctx context.Context, nodeID string) (string, error) { - subject := messaging.SubjectNodeFilesTemp(nodeID) - reply, err := messaging.RequestJSON[fileTempRequest, fileTempReply](s.nats, subject, fileTempRequest{}, 30*time.Second) - if err != nil { +// AllocRemoteTemp asks the worker to allocate a temp file. +func (s *S3FileStager) AllocRemoteTemp(ctx context.Context, nodeID string) (string, error) { + var reply fileTempReply + if err := s.callWorker(ctx, nodeID, workerctl.PathFilesTemp, fileMetadataRPCTimeout, + fileTempRequest{}, &reply); err != nil { return "", err } if reply.Error != "" { @@ -165,10 +198,17 @@ func (s *S3NATSFileStager) AllocRemoteTemp(ctx context.Context, nodeID string) ( return reply.LocalPath, nil } -func (s *S3NATSFileStager) ListRemoteDir(ctx context.Context, nodeID, keyPrefix string) ([]string, error) { - subject := messaging.SubjectNodeFilesListDir(nodeID) - reply, err := messaging.RequestJSON[fileListDirRequest, fileListDirReply](s.nats, subject, fileListDirRequest{KeyPrefix: keyPrefix}, 30*time.Second) - if err != nil { +// ListRemoteDir returns the relative paths of every file under keyPrefix on the +// worker. +// +// Nothing truncates the answer, at either end. The bus this used to ride put a +// ceiling on how big a reply could be, and a wide model directory was the case +// that pushed against it; a response body has no such ceiling, and a short +// listing would read to the caller as files the worker does not have. +func (s *S3FileStager) ListRemoteDir(ctx context.Context, nodeID, keyPrefix string) ([]string, error) { + var reply fileListDirReply + if err := s.callWorker(ctx, nodeID, workerctl.PathFilesListDir, fileMetadataRPCTimeout, + fileListDirRequest{KeyPrefix: keyPrefix}, &reply); err != nil { return nil, err } if reply.Error != "" { @@ -178,11 +218,11 @@ func (s *S3NATSFileStager) ListRemoteDir(ctx context.Context, nodeID, keyPrefix return reply.Files, nil } -// StageRemoteToStore tells the backend to upload a local file to S3. -func (s *S3NATSFileStager) StageRemoteToStore(ctx context.Context, nodeID, remotePath, key string) error { - subject := messaging.SubjectNodeFilesStage(nodeID) - reply, err := messaging.RequestJSON[fileStageRequest, fileStageReply](s.nats, subject, fileStageRequest{LocalPath: remotePath, Key: key}, 10*time.Minute) - if err != nil { +// StageRemoteToStore tells the worker to upload a local file to shared storage. +func (s *S3FileStager) StageRemoteToStore(ctx context.Context, nodeID, remotePath, key string) error { + var reply fileStageReply + if err := s.callWorker(ctx, nodeID, workerctl.PathFilesStage, fileTransferRPCTimeout, + fileStageRequest{LocalPath: remotePath, Key: key}, &reply); err != nil { return err } if reply.Error != "" { diff --git a/core/services/nodes/file_stager_s3_test.go b/core/services/nodes/file_stager_s3_test.go new file mode 100644 index 000000000000..8fdc4ae01036 --- /dev/null +++ b/core/services/nodes/file_stager_s3_test.go @@ -0,0 +1,173 @@ +package nodes + +import ( + "context" + "errors" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/storage" + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// The S3 file stager's half of the file-staging contract: which verb each +// method addresses, and whose budget bounds it. +// +// The stager is reached through the real ControlClient over a real HTTP +// transport onto a scripted worker, because what these specs are about is +// transport behaviour: a double that never dials anything cannot fail the way +// a spent budget or a rejected route fails. +var _ = Describe("the S3 file stager's control RPCs", func() { + const nodeID = "stager-node" + + var ( + workers *scriptedControlWorkers + stager *S3FileStager + local string + ) + + BeforeEach(func() { + workers = newScriptedControlWorkers() + + dir := GinkgoT().TempDir() + store, err := storage.NewFilesystemStore(filepath.Join(dir, "objectstore")) + Expect(err).NotTo(HaveOccurred()) + fm, err := storage.NewFileManager(store, filepath.Join(dir, "cache")) + Expect(err).NotTo(HaveOccurred()) + stager = NewS3FileStager(fm, workers.controlClient()) + + local = filepath.Join(dir, "model.gguf") + Expect(os.WriteFile(local, []byte("weights"), 0o600)).To(Succeed()) + }) + + DescribeTable("addresses the verb that verb's path names", + func(path string, reply any, call func(*S3FileStager) error) { + workers.scriptReply(controlKey(nodeID, path), reply) + Expect(call(stager)).To(Succeed()) + Expect(workers.callSubjects()).To(ContainElement(controlKey(nodeID, path))) + }, + Entry("ensure", workerctl.PathFilesEnsure, fileEnsureReply{LocalPath: "/w/models/m.gguf"}, + func(s *S3FileStager) error { + _, err := s.EnsureRemote(context.Background(), nodeID, local, storage.ModelKey("m.gguf")) + return err + }), + Entry("temp", workerctl.PathFilesTemp, fileTempReply{LocalPath: "/w/tmp/x"}, + func(s *S3FileStager) error { + _, err := s.AllocRemoteTemp(context.Background(), nodeID) + return err + }), + Entry("listdir", workerctl.PathFilesListDir, fileListDirReply{Files: []string{"a", "b"}}, + func(s *S3FileStager) error { + _, err := s.ListRemoteDir(context.Background(), nodeID, "models/m") + return err + }), + Entry("stage", workerctl.PathFilesStage, fileStageReply{Key: "data/out"}, + func(s *S3FileStager) error { + return s.StageRemoteToStore(context.Background(), nodeID, "/w/models/out", "data/out") + }), + ) + + // THE rule of this change, and it is written out at five separate call + // sites: the RPC's budget is DERIVED FROM the caller's context, never + // started fresh from a background one. A site that started fresh would keep + // commanding a worker after its caller had given up, and would report the + // worker's late answer as a live one. Each site is pinned on its own, + // because five sites behind one spec is four sites nothing holds. + DescribeTable("never reaches the worker once the caller's context is spent", + func(call func(context.Context, *S3FileStager) error) { + // Every verb is scripted to answer, so the ONLY thing that can stop + // the call is the caller's own spent context. + for _, p := range []string{ + workerctl.PathFilesEnsure, workerctl.PathFilesStage, + workerctl.PathFilesTemp, workerctl.PathFilesListDir, + } { + workers.scriptRawReply(controlKey(nodeID, p), []byte(`{}`)) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := call(ctx, stager) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, context.Canceled)).To(BeTrue(), "got %v", err) + // An expiry is never evidence about a file, so it must arrive + // wearing the umbrella that stops a caller acting on it. + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue(), "got %v", err) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + Expect(workers.callSubjects()).To(BeEmpty()) + }, + Entry("ensure", func(ctx context.Context, s *S3FileStager) error { + _, err := s.EnsureRemote(ctx, nodeID, local, storage.ModelKey("m.gguf")) + return err + }), + Entry("temp", func(ctx context.Context, s *S3FileStager) error { + _, err := s.AllocRemoteTemp(ctx, nodeID) + return err + }), + Entry("listdir", func(ctx context.Context, s *S3FileStager) error { + _, err := s.ListRemoteDir(ctx, nodeID, "models/m") + return err + }), + Entry("stage to store", func(ctx context.Context, s *S3FileStager) error { + return s.StageRemoteToStore(ctx, nodeID, "/w/models/out", "data/out") + }), + Entry("fetch", func(ctx context.Context, s *S3FileStager) error { + return s.FetchRemote(ctx, nodeID, "/w/models/out", filepath.Join(GinkgoT().TempDir(), "dst")) + }), + Entry("fetch by key", func(ctx context.Context, s *S3FileStager) error { + return s.FetchRemoteByKey(ctx, nodeID, "data/out", filepath.Join(GinkgoT().TempDir(), "dst")) + }), + ) + + // The other half of the same distinction, also at every site: what the + // WORKER said comes back as the worker's answer, so a caller may act on it, + // and it must not be dressed up as a route failure. + DescribeTable("reports the worker's own refusal as the worker's answer", + func(path string, call func(*S3FileStager) error) { + workers.scriptRawReply(controlKey(nodeID, path), []byte(`{"error":"no space left on device"}`)) + err := call(stager) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no space left on device")) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeFalse(), "got %v", err) + }, + Entry("ensure", workerctl.PathFilesEnsure, func(s *S3FileStager) error { + _, err := s.EnsureRemote(context.Background(), nodeID, local, storage.ModelKey("m.gguf")) + return err + }), + Entry("temp", workerctl.PathFilesTemp, func(s *S3FileStager) error { + _, err := s.AllocRemoteTemp(context.Background(), nodeID) + return err + }), + Entry("listdir", workerctl.PathFilesListDir, func(s *S3FileStager) error { + _, err := s.ListRemoteDir(context.Background(), nodeID, "models/m") + return err + }), + Entry("stage to store", workerctl.PathFilesStage, func(s *S3FileStager) error { + return s.StageRemoteToStore(context.Background(), nodeID, "/w/models/out", "data/out") + }), + Entry("fetch", workerctl.PathFilesStage, func(s *S3FileStager) error { + return s.FetchRemote(context.Background(), nodeID, "/w/models/out", + filepath.Join(GinkgoT().TempDir(), "dst")) + }), + ) + + // A worker too old to serve a file verb answers 404. That is a DEPLOYMENT + // fact about the worker's build and says nothing about the file, so it must + // not reach a caller as "that file is not there". + It("reports a worker that serves no file verbs as unsupported, not as an absent file", func() { + workers.scriptUnsupported(controlKey(nodeID, workerctl.PathFilesListDir)) + _, err := stager.ListRemoteDir(context.Background(), nodeID, "models/m") + Expect(err).To(MatchError(ErrWorkerControlUnsupported)) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + }) + + It("reports a worker it has no route to as unroutable", func() { + workers.scriptUnroutable(nodeID) + _, err := stager.AllocRemoteTemp(context.Background(), nodeID) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue(), "got %v", err) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + }) +}) diff --git a/core/services/worker/control_files.go b/core/services/worker/control_files.go new file mode 100644 index 000000000000..2b05781a915f --- /dev/null +++ b/core/services/worker/control_files.go @@ -0,0 +1,268 @@ +package worker + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/mudler/LocalAI/core/services/storage" + "github.com/mudler/LocalAI/core/services/workerctl" + "github.com/mudler/xlog" +) + +// The worker's file-staging control plane: four verbs that move model and job +// artifacts between the object store both sides share and this worker's disk. +// They replace the four nodes..files.* NATS subjects. +// +// The reply shapes are the ones those subjects already carried, unchanged, so +// an operator reading the wire sees the same fields. What DID change is that a +// listing is no longer sized against a bus payload: it is a response body the +// caller is already reading, which is why nothing here truncates one. +// +// The 200-with-error-field shape is the same one the lifecycle verbs take, and +// for the same reason: "that file is not there" is the worker's own ANSWER, and +// a reap guard may act on it, while a non-2xx is this frontend failing to reach +// the worker, which nothing may act on. A handler that answered 500 for a +// failed upload would move its verdict into the bucket reserved for a broken +// link. + +// The file-staging reply bodies. They mirror the frontend's decode structs in +// core/services/nodes/file_stager_s3.go field for field; the two are written +// separately because neither package may import the other, and the roundtrip +// spec is what holds them together. +type fileEnsureReply struct { + LocalPath string `json:"local_path,omitempty"` + Error string `json:"error,omitempty"` +} + +type fileStageReply struct { + Key string `json:"key,omitempty"` + Error string `json:"error,omitempty"` +} + +type fileTempReply struct { + LocalPath string `json:"local_path,omitempty"` + Error string `json:"error,omitempty"` +} + +type fileListDirReply struct { + Files []string `json:"files,omitempty"` + Error string `json:"error,omitempty"` +} + +// stagingCacheDir is the one place the worker's staging cache directory is +// derived from its configuration. +// +// One place and not two, because the FileManager caches INTO this directory and +// the listdir and temp verbs resolve paths AGAINST it. Two derivations that +// drifted would give a worker that downloads a file to one directory and then +// reports it missing from another, and both halves would still be self +// consistent. +func (cfg *Config) stagingCacheDir() string { + return filepath.Join(cfg.ModelsPath, "..", "cache") +} + +// stagingDataDir is where keys under storage.DataKeyPrefix resolve, and it is +// derived from the cache directory for the same single-source reason. +func (cfg *Config) stagingDataDir() string { + return filepath.Join(cfg.stagingCacheDir(), "..", "data") +} + +// NewStagingFileManager builds the FileManager the file-staging verbs serve +// from, over the same object store the frontend uses. +// +// It returns an error rather than degrading, because a worker whose deployment +// asked for object storage and could not reach it would otherwise mount four +// verbs that fail every call, which the frontend cannot tell from a worker out +// of disk. +func (cfg *Config) NewStagingFileManager(ctx context.Context) (*storage.FileManager, error) { + s3Store, err := storage.NewS3Store(ctx, storage.S3Config{ + Endpoint: cfg.StorageURL, + Region: cfg.StorageRegion, + Bucket: cfg.StorageBucket, + AccessKeyID: cfg.StorageAccessKey, + SecretAccessKey: cfg.StorageSecretKey, + ForcePathStyle: true, + }) + if err != nil { + return nil, fmt.Errorf("initializing S3 store: %w", err) + } + fm, err := storage.NewFileManager(s3Store, cfg.stagingCacheDir()) + if err != nil { + return nil, fmt.Errorf("initializing file manager: %w", err) + } + return fm, nil +} + +// RegisterFileControlRoutes mounts the four file-staging verbs on mux. +// +// The caller is responsible for putting mux behind authentication; see +// nodes.AuthenticatedRoutes, which is how the worker mounts this so the file +// verbs share one bearer check with the lifecycle verbs and the file routes +// rather than growing a third one. +func (cfg *Config) RegisterFileControlRoutes(mux *http.ServeMux, fm *storage.FileManager) { + cacheDir := cfg.stagingCacheDir() + + // files.ensure: download an object-store key into this worker's cache and + // say where it landed. + // + // It takes the caller's context. Nothing is terminated and no resource is + // held if it is abandoned half way: the download simply stops, and the next + // attempt starts over. So the caller's budget is the operation's budget. + postControlVerb(mux, workerctl.PathFilesEnsure, func(ctx context.Context, body []byte) (any, error) { + var req struct { + Key string `json:"key"` + } + if err := json.Unmarshal(body, &req); err != nil { + return nil, fmt.Errorf("invalid files.ensure request: %w", err) + } + localPath, err := fm.Download(ctx, req.Key) + if err != nil { + xlog.Error("File ensure failed", "key", req.Key, "error", err) + return fileEnsureReply{Error: err.Error()}, nil + } + xlog.Debug("File ensured locally", "key", req.Key, "path", localPath) + return fileEnsureReply{LocalPath: localPath}, nil + }) + + // files.stage: upload one of this worker's files to the object store. + // + // The path allow-list is what keeps this verb from being an exfiltration + // primitive: the token holder can name any absolute path, so only the + // directories this worker stages out of are served. + postControlVerb(mux, workerctl.PathFilesStage, func(ctx context.Context, body []byte) (any, error) { + var req struct { + LocalPath string `json:"local_path"` + Key string `json:"key"` + } + if err := json.Unmarshal(body, &req); err != nil { + return nil, fmt.Errorf("invalid files.stage request: %w", err) + } + allowedDirs := []string{cacheDir} + if cfg.ModelsPath != "" { + allowedDirs = append(allowedDirs, cfg.ModelsPath) + } + if !isPathAllowed(req.LocalPath, allowedDirs) { + return fileStageReply{Error: "path outside allowed directories"}, nil + } + if err := fm.Upload(ctx, req.Key, req.LocalPath); err != nil { + xlog.Error("File stage failed", "path", req.LocalPath, "key", req.Key, "error", err) + return fileStageReply{Error: err.Error()}, nil + } + xlog.Debug("File staged to the object store", "path", req.LocalPath, "key", req.Key) + return fileStageReply{Key: req.Key}, nil + }) + + // files.temp: allocate an empty file the frontend may then upload into. + postControlVerb(mux, workerctl.PathFilesTemp, func(context.Context, []byte) (any, error) { + tmpDir := filepath.Join(cacheDir, "staging-tmp") + if err := os.MkdirAll(tmpDir, 0750); err != nil { + return fileTempReply{Error: fmt.Sprintf("creating temp dir: %v", err)}, nil + } + f, err := os.CreateTemp(tmpDir, "localai-staging-*.tmp") + if err != nil { + return fileTempReply{Error: fmt.Sprintf("creating temp file: %v", err)}, nil + } + localPath := f.Name() + if err := f.Close(); err != nil { + return fileTempReply{Error: fmt.Sprintf("closing temp file: %v", err)}, nil + } + xlog.Debug("Allocated temp file", "path", localPath) + return fileTempReply{LocalPath: localPath}, nil + }) + + // files.listdir: the relative paths of every file under one key prefix. + // + // Nothing here caps the answer. Over NATS the reply had to fit a payload + // the bus was willing to carry, and a wide model directory was the case + // that risked it; over HTTP the listing is written into a body the caller + // is already reading, so its size is no longer a property of the carrier. A + // cap would silently return a SHORT listing, which the frontend reads as + // files that are not there. + postControlVerb(mux, workerctl.PathFilesListDir, func(ctx context.Context, body []byte) (any, error) { + var req struct { + KeyPrefix string `json:"key_prefix"` + } + if err := json.Unmarshal(body, &req); err != nil { + return nil, fmt.Errorf("invalid files.listdir request: %w", err) + } + dirPath, ok := cfg.resolveStagingDir(req.KeyPrefix) + if !ok { + return fileListDirReply{Error: "invalid key prefix"}, nil + } + files, err := listStagedFiles(ctx, dirPath) + if err != nil { + xlog.Error("Failed to list staged files", "keyPrefix", req.KeyPrefix, "dirPath", dirPath, "error", err) + return fileListDirReply{Error: err.Error()}, nil + } + xlog.Debug("Listed remote dir", "keyPrefix", req.KeyPrefix, "dirPath", dirPath, "fileCount", len(files)) + return fileListDirReply{Files: files}, nil + }) +} + +// resolveStagingDir maps a storage key prefix onto the local directory it names, +// and reports whether that directory is one this worker serves. +// +// The second return is not an error string on purpose: a prefix that climbs out +// of the served directories is refused before anything touches the filesystem, +// so a crafted key_prefix cannot turn this verb into a directory reader for the +// whole host. +func (cfg *Config) resolveStagingDir(keyPrefix string) (string, bool) { + cacheDir := cfg.stagingCacheDir() + dataDir := cfg.stagingDataDir() + + dirPath := filepath.Join(cacheDir, keyPrefix) + if rel, ok := strings.CutPrefix(keyPrefix, storage.ModelKeyPrefix); ok && cfg.ModelsPath != "" { + dirPath = filepath.Join(cfg.ModelsPath, rel) + } else if rel, ok := strings.CutPrefix(keyPrefix, storage.DataKeyPrefix); ok { + dirPath = filepath.Join(dataDir, rel) + } + + dirPath = filepath.Clean(dirPath) + cleanCache := filepath.Clean(cacheDir) + cleanModels := filepath.Clean(cfg.ModelsPath) + cleanData := filepath.Clean(dataDir) + within := func(root string) bool { + return dirPath == root || strings.HasPrefix(dirPath, root+string(filepath.Separator)) + } + if within(cleanCache) || (cleanModels != "." && within(cleanModels)) || within(cleanData) { + return dirPath, true + } + return "", false +} + +// listStagedFiles walks dirPath and returns every file's path relative to it. +// +// The walk honours ctx because a very wide directory is real work, and a caller +// that has already given up should not keep this worker stat-ing files. The +// context error is returned as the walk's error, so it travels back as a +// FAILURE of the listing rather than as an empty listing, which the frontend +// would read as a directory with nothing in it. +func listStagedFiles(ctx context.Context, dirPath string) ([]string, error) { + var files []string + err := filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + if d.IsDir() { + return nil + } + rel, relErr := filepath.Rel(dirPath, path) + if relErr != nil { + return relErr + } + files = append(files, rel) + return nil + }) + if err != nil { + return nil, err + } + return files, nil +} diff --git a/core/services/worker/control_files_test.go b/core/services/worker/control_files_test.go new file mode 100644 index 000000000000..42a815c0501d --- /dev/null +++ b/core/services/worker/control_files_test.go @@ -0,0 +1,301 @@ +package worker + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/storage" + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// The four file-staging verbs on the worker's own control plane. +// +// They used to be NATS request-reply subjects, and the size of a listdir reply +// was a property of the carrier: a directory with enough files in it produced a +// payload the bus was not comfortable with. Over HTTP it is a response body, +// which is why one of the specs below asserts a listing far past any payload +// cap rather than asserting a cap of its own. +var _ = Describe("worker file-staging control routes", func() { + var ( + cfg *Config + srv *httptest.Server + fm *storage.FileManager + store *storage.FilesystemStore + modelsDir string + cacheDir string + ) + + // post issues one control verb the way the frontend does and hands back the + // raw response, so a spec can assert the STATUS as well as the body. The + // two carry different meanings and a helper that decoded only the body + // would hide the one this file exists to pin. + post := func(path string, body any) *http.Response { + GinkgoHelper() + raw, err := json.Marshal(body) + Expect(err).NotTo(HaveOccurred()) + resp, err := http.Post(srv.URL+path, "application/json", bytes.NewReader(raw)) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + return resp + } + + decode := func(resp *http.Response, out any) { + GinkgoHelper() + Expect(json.NewDecoder(resp.Body).Decode(out)).To(Succeed()) + } + + BeforeEach(func() { + dir := GinkgoT().TempDir() + modelsDir = filepath.Join(dir, "models") + Expect(os.MkdirAll(modelsDir, 0o750)).To(Succeed()) + cacheDir = filepath.Join(dir, "cache") + + var err error + store, err = storage.NewFilesystemStore(filepath.Join(dir, "objectstore")) + Expect(err).NotTo(HaveOccurred()) + fm, err = storage.NewFileManager(store, cacheDir) + Expect(err).NotTo(HaveOccurred()) + + cfg = &Config{ModelsPath: modelsDir} + mux := http.NewServeMux() + cfg.RegisterFileControlRoutes(mux, fm) + srv = httptest.NewServer(mux) + DeferCleanup(srv.Close) + }) + + It("allocates a temp path and returns it", func() { + resp := post(workerctl.PathFilesTemp, struct{}{}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply struct { + LocalPath string `json:"local_path"` + Error string `json:"error"` + } + decode(resp, &reply) + Expect(reply.Error).To(BeEmpty()) + Expect(reply.LocalPath).To(BeAnExistingFile()) + }) + + It("downloads a key the store already holds and reports where it landed", func() { + key := storage.ModelKey("ensure-me.gguf") + Expect(store.Put(context.Background(), key, strings.NewReader("weights"))).To(Succeed()) + + resp := post(workerctl.PathFilesEnsure, map[string]string{"key": key}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply struct { + LocalPath string `json:"local_path"` + Error string `json:"error"` + } + decode(resp, &reply) + Expect(reply.Error).To(BeEmpty()) + Expect(reply.LocalPath).To(BeAnExistingFile()) + Expect(os.ReadFile(reply.LocalPath)).To(Equal([]byte("weights"))) + }) + + It("uploads a file under an allowed directory and answers with its key", func() { + local := filepath.Join(modelsDir, "staged.bin") + Expect(os.WriteFile(local, []byte("output"), 0o600)).To(Succeed()) + + resp := post(workerctl.PathFilesStage, map[string]string{"local_path": local, "key": "data/out.bin"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply struct { + Key string `json:"key"` + Error string `json:"error"` + } + decode(resp, &reply) + Expect(reply.Error).To(BeEmpty()) + Expect(reply.Key).To(Equal("data/out.bin")) + exists, err := store.Exists(context.Background(), "data/out.bin") + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + + It("returns a listing longer than a NATS payload would have carried", func() { + // 4000 files at ~40 bytes of name each is ~160 KB, twenty times the + // NOTIFY cap and well past what the old carrier was comfortable with. + // The point of the spec is that size is no longer a property of the + // transport. + bigDir := filepath.Join(modelsDir, "big") + Expect(os.MkdirAll(bigDir, 0o750)).To(Succeed()) + for i := range 4000 { + name := fmt.Sprintf("shard-%030d.safetensors", i) + Expect(os.WriteFile(filepath.Join(bigDir, name), []byte("x"), 0o600)).To(Succeed()) + } + + resp := post(workerctl.PathFilesListDir, map[string]string{"key_prefix": "models/big"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply struct { + Files []string `json:"files"` + Error string `json:"error"` + } + decode(resp, &reply) + Expect(reply.Error).To(BeEmpty()) + Expect(reply.Files).To(HaveLen(4000)) + }) + + // The rule these four pin is ONE rule stated at four handlers: a verb's own + // failure is the worker's ANSWER and travels as a 200 with the error field + // set, never as a 5xx. A 5xx is what the frontend maps onto "this frontend + // could not reach that worker", which nothing may act on, so a handler that + // answered 500 would move its own verdict into the bucket reserved for a + // broken link. Each handler is pinned separately because each writes the + // rule out for itself. + It("reports a staging failure as a 200 with an error field, not as a 5xx", func() { + resp := post(workerctl.PathFilesStage, map[string]string{"local_path": "/nope", "key": "k"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply struct { + Error string `json:"error"` + } + decode(resp, &reply) + Expect(reply.Error).NotTo(BeEmpty()) + }) + + It("reports an upload that failed as a 200 with an error field, not as a 5xx", func() { + // The path is INSIDE an allowed directory and the file is simply not + // there, so this reaches the upload rather than stopping at the + // allow-list. The two are separate statements of the same rule inside + // one handler, and a spec that only ever reaches the allow-list leaves + // the upload free to answer 500 for the worker's own verdict. + missing := filepath.Join(modelsDir, "was-never-written.bin") + resp := post(workerctl.PathFilesStage, map[string]string{"local_path": missing, "key": "data/x"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply struct { + Key string `json:"key"` + Error string `json:"error"` + } + decode(resp, &reply) + Expect(reply.Error).NotTo(BeEmpty()) + Expect(reply.Error).NotTo(ContainSubstring("outside allowed directories")) + Expect(reply.Key).To(BeEmpty()) + }) + + It("reports an ensure of a key the store does not hold as a 200 with an error field", func() { + resp := post(workerctl.PathFilesEnsure, map[string]string{"key": "models/absent.gguf"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply struct { + LocalPath string `json:"local_path"` + Error string `json:"error"` + } + decode(resp, &reply) + Expect(reply.Error).NotTo(BeEmpty()) + Expect(reply.LocalPath).To(BeEmpty()) + }) + + It("reports a listdir of a directory that is not there as a 200 with an error field", func() { + resp := post(workerctl.PathFilesListDir, map[string]string{"key_prefix": "models/never-created"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply struct { + Files []string `json:"files"` + Error string `json:"error"` + } + decode(resp, &reply) + Expect(reply.Error).NotTo(BeEmpty()) + Expect(reply.Files).To(BeEmpty()) + }) + + It("reports a temp allocation it cannot make as a 200 with an error field", func() { + // A regular file where the staging directory must go: MkdirAll cannot + // create through it, for root as much as for anyone else, so the verb + // fails for a reason that is entirely this worker's own. + Expect(os.MkdirAll(cacheDir, 0o750)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(cacheDir, "staging-tmp"), []byte("not a dir"), 0o600)).To(Succeed()) + + resp := post(workerctl.PathFilesTemp, struct{}{}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply struct { + LocalPath string `json:"local_path"` + Error string `json:"error"` + } + decode(resp, &reply) + Expect(reply.Error).NotTo(BeEmpty()) + Expect(reply.LocalPath).To(BeEmpty()) + }) + + It("refuses a key prefix that climbs out of the directories it serves", func() { + resp := post(workerctl.PathFilesListDir, map[string]string{"key_prefix": "../../../etc"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply struct { + Files []string `json:"files"` + Error string `json:"error"` + } + decode(resp, &reply) + Expect(reply.Error).To(ContainSubstring("invalid key prefix")) + Expect(reply.Files).To(BeEmpty()) + }) + + It("refuses to upload a path outside the directories it serves", func() { + outside := filepath.Join(GinkgoT().TempDir(), "secret") + Expect(os.WriteFile(outside, []byte("nope"), 0o600)).To(Succeed()) + + resp := post(workerctl.PathFilesStage, map[string]string{"local_path": outside, "key": "data/leak"}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply struct { + Error string `json:"error"` + } + decode(resp, &reply) + Expect(reply.Error).To(ContainSubstring("outside allowed directories")) + }) + + // The bounded, POST-only entry point, pinned at every one of the four + // paths rather than at one of them. A route that skipped it would be a + // second, unbounded door onto a boundary this worker serves, and it would + // also let a liveness probe or an address bar run a command. + DescribeTable("refuses a method that is not POST", + func(path string) { + resp, err := http.Get(srv.URL + path) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusMethodNotAllowed)) + }, + Entry("ensure", workerctl.PathFilesEnsure), + Entry("stage", workerctl.PathFilesStage), + Entry("temp", workerctl.PathFilesTemp), + Entry("listdir", workerctl.PathFilesListDir), + ) + + DescribeTable("refuses a body past the control cap, and as a rejected request rather than an answer", + func(path string) { + // VALID JSON past the cap, deliberately. A body of filler is + // refused by the decoder whether or not anything bounds it, so a + // spec written that way passes for a reason that has nothing to do + // with the bound and would keep passing after the bound was + // removed. This one can only be refused by the bound. + oversized := append([]byte(`{"key":"`), bytes.Repeat([]byte("a"), maxControlRequestBytes+1)...) + oversized = append(oversized, []byte(`"}`)...) + resp, err := http.Post(srv.URL+path, "application/json", bytes.NewReader(oversized)) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + }, + Entry("ensure", workerctl.PathFilesEnsure), + Entry("stage", workerctl.PathFilesStage), + Entry("temp", workerctl.PathFilesTemp), + Entry("listdir", workerctl.PathFilesListDir), + ) + + It("reports a request body it cannot read as a rejection, never as a file that is not there", func() { + resp, err := http.Post(srv.URL+workerctl.PathFilesEnsure, "application/json", strings.NewReader("{not json")) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + }) + + It("mounts every file verb, so none can be dropped from the set the frontend calls", func() { + for _, path := range []string{ + workerctl.PathFilesEnsure, workerctl.PathFilesStage, + workerctl.PathFilesTemp, workerctl.PathFilesListDir, + } { + resp := post(path, struct{}{}) + Expect(resp.StatusCode).NotTo(Equal(http.StatusNotFound), "%s is not mounted", path) + } + }) +}) diff --git a/core/services/worker/control_routes.go b/core/services/worker/control_routes.go index 31ddbfb7e332..a31c9faff645 100644 --- a/core/services/worker/control_routes.go +++ b/core/services/worker/control_routes.go @@ -85,34 +85,7 @@ func (s *backendSupervisor) upgrader() upgradeFunc { // plane shares one bearer check with the file routes rather than growing a // second one. func (s *backendSupervisor) RegisterControlRoutes(mux *http.ServeMux) { - // post registers one JSON-in, JSON-out verb. reply may be nil, in which - // case the verb answers 204: that is the shape the two former - // publish-no-reply subjects take. - post := func(path string, h func(ctx context.Context, body []byte) (any, error)) { - mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { - body, ok := readControlBody(w, r) - if !ok { - return - } - reply, err := h(r.Context(), body) - if err != nil { - // Reaching here means the request could not be READ or routed, - // which is this worker failing rather than answering. A verb's - // own failure never reaches here: it comes back as a reply with - // Error set, and a 200. - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - if reply == nil { - w.WriteHeader(http.StatusNoContent) - return - } - w.Header().Set("Content-Type", "application/json") - if encErr := json.NewEncoder(w).Encode(reply); encErr != nil { - xlog.Debug("worker control reply could not be written", "path", path, "error", encErr) - } - }) - } + post := func(path string, h controlVerb) { postControlVerb(mux, path, h) } post(workerctl.PathModelsRunning, func(context.Context, []byte) (any, error) { return messaging.ModelsRunningReply{Models: s.runningModels()}, nil @@ -199,6 +172,46 @@ func (s *backendSupervisor) RegisterControlRoutes(mux *http.ServeMux) { }) } +// controlVerb is the shape of a JSON-in, JSON-out control handler. A nil reply +// means the verb answers 204, which is the shape the former publish-no-reply +// subjects take. +// +// An error returned here means the request could not be READ or routed, which +// is this worker FAILING rather than answering, and it becomes a non-2xx. A +// verb's own failure is never returned this way: it comes back as a reply with +// Error set, on a 200. See the note at the top of this file for why the two +// must not be confused. +type controlVerb func(ctx context.Context, body []byte) (any, error) + +// postControlVerb registers one control verb on mux. +// +// It is the single door every control route enters through, so the POST-only +// check, the bounded read and the 200-with-error-field shape are written once +// rather than once per route set. The file-staging routes mount through it for +// exactly that reason: a second registrar with its own copy of the rules is how +// one of them ends up unbounded, or answering 500 for a verdict. +func postControlVerb(mux *http.ServeMux, path string, h controlVerb) { + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + body, ok := readControlBody(w, r) + if !ok { + return + } + reply, err := h(r.Context(), body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if reply == nil { + w.WriteHeader(http.StatusNoContent) + return + } + w.Header().Set("Content-Type", "application/json") + if encErr := json.NewEncoder(w).Encode(reply); encErr != nil { + xlog.Debug("worker control reply could not be written", "path", path, "error", encErr) + } + }) +} + // readControlBody enforces the two things every control verb requires of a // request: that it is a POST, and that its body is bounded. // diff --git a/core/services/worker/control_routes_test.go b/core/services/worker/control_routes_test.go index f57793324bc8..584e118c2362 100644 --- a/core/services/worker/control_routes_test.go +++ b/core/services/worker/control_routes_test.go @@ -21,6 +21,7 @@ import ( "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/storage" "github.com/mudler/LocalAI/core/services/workerctl" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" @@ -447,8 +448,16 @@ var _ = Describe("the worker's HTTP server", func() { return nil, nil }, } + // A real object store, because the four file verbs are only mounted for + // a worker that has one, and the mounting assertion below walks every + // path this package names. + store, err := storage.NewFilesystemStore(filepath.Join(dir, "objectstore")) + Expect(err).NotTo(HaveOccurred()) + stagingFM, err := storage.NewFileManager(store, filepath.Join(dir, "..", "cache")) + Expect(err).NotTo(HaveOccurred()) + srv, err = startWorkerHTTPServer("127.0.0.1:0", filepath.Join(dir, "staging"), dir, - filepath.Join(dir, "data"), token, &nodes.WorkerReadiness{}, sup, nil) + filepath.Join(dir, "data"), token, &nodes.WorkerReadiness{}, sup, sup.cfg, stagingFM, nil) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { nodes.ShutdownFileTransferServer(srv) }) Expect(srv.Addr).NotTo(BeEmpty(), "the worker HTTP server must report the address it bound") @@ -481,6 +490,31 @@ var _ = Describe("the worker's HTTP server", func() { Expect(postCtl(workerctl.PathModelsRunning, "wrong").StatusCode).To(Equal(http.StatusUnauthorized)) }) + It("serves no file verb at all when the deployment configured no object store", func() { + // Not a degraded mount: a worker with nowhere to stage to answers the + // four file paths the way a build that never had them does, which is + // the 404 the frontend already reads as "this worker does not serve + // that verb" rather than as a file that is not there. + dir := GinkgoT().TempDir() + bare, err := startWorkerHTTPServer("127.0.0.1:0", filepath.Join(dir, "staging"), dir, + filepath.Join(dir, "data"), token, &nodes.WorkerReadiness{}, sup, sup.cfg, nil, nil) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { nodes.ShutdownFileTransferServer(bare) }) + + for _, p := range []string{ + workerctl.PathFilesEnsure, workerctl.PathFilesStage, + workerctl.PathFilesTemp, workerctl.PathFilesListDir, + } { + req, reqErr := http.NewRequest(http.MethodPost, "http://"+bare.Addr+p, strings.NewReader("{}")) + Expect(reqErr).NotTo(HaveOccurred()) + req.Header.Set("Authorization", "Bearer "+token) + resp, doErr := http.DefaultClient.Do(req) + Expect(doErr).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusNotFound), "%s answered without an object store", p) + } + }) + It("mounts every control verb, not just the one this spec reads", func() { for _, p := range workerctl.AllPaths() { if p == workerctl.PathNodeStop { diff --git a/core/services/worker/file_stager_roundtrip_test.go b/core/services/worker/file_stager_roundtrip_test.go new file mode 100644 index 000000000000..1c8e455305d1 --- /dev/null +++ b/core/services/worker/file_stager_roundtrip_test.go @@ -0,0 +1,207 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/storage" + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// The frontend's S3 file stager against the REAL worker file-staging routes: a +// real object store both sides share, the real handlers mounted through the +// real nodes.AuthenticatedRoutes so the real bearer check runs first, reached +// by a real nodes.ControlClient over a real HTTP transport. +// +// It lives in this package for the same reason the control-client roundtrip +// does: the dependency runs worker -> nodes, so the frontend half cannot be +// exercised against the real handler from the other side without an import +// cycle. A spec on either side alone proves only that side agrees with itself; +// the path literals, the JSON shapes and the status codes are pinned together +// only here. +var _ = Describe("the frontend's file stager against the real worker", func() { + const ( + token = "s3cret-registration-token" + nodeID = "staging-worker" + ) + + var ( + stager *nodes.S3FileStager + store *storage.FilesystemStore + workerFM *storage.FileManager + modelsDir string + srvAddr string + ) + + newStager := func(tok string) *nodes.S3FileStager { + GinkgoHelper() + frontendFM, err := storage.NewFileManager(store, GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + control := nodes.NewControlClient(func(string) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "tcp", srvAddr) + } + }, tok) + return nodes.NewS3FileStager(frontendFM, control) + } + + BeforeEach(func() { + dir := GinkgoT().TempDir() + modelsDir = filepath.Join(dir, "worker", "models") + Expect(os.MkdirAll(modelsDir, 0o750)).To(Succeed()) + + var err error + store, err = storage.NewFilesystemStore(filepath.Join(dir, "objectstore")) + Expect(err).NotTo(HaveOccurred()) + + cfg := &Config{ModelsPath: modelsDir} + workerFM, err = storage.NewFileManager(store, filepath.Join(dir, "worker", "cache")) + Expect(err).NotTo(HaveOccurred()) + + lis, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + srv, err := nodes.StartFileTransferServerWithRoutes(lis, + filepath.Join(dir, "worker", "staging"), modelsDir, filepath.Join(dir, "worker", "data"), + token, config.DefaultMaxUploadSize, nil, + &nodes.AuthenticatedRoutes{ + Prefix: workerctl.Prefix, + Register: func(mux *http.ServeMux) { + cfg.RegisterFileControlRoutes(mux, workerFM) + }, + }) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = srv.Close() }) + + srvAddr = lis.Addr().String() + stager = newStager(token) + }) + + It("puts a frontend file in the store and has the worker fetch it", func() { + local := filepath.Join(GinkgoT().TempDir(), "model.gguf") + Expect(os.WriteFile(local, []byte("checkpoint bytes"), 0o600)).To(Succeed()) + + remote, err := stager.EnsureRemote(context.Background(), nodeID, local, storage.ModelKey("rt/model.gguf")) + Expect(err).NotTo(HaveOccurred()) + Expect(remote).To(BeAnExistingFile()) + Expect(os.ReadFile(remote)).To(Equal([]byte("checkpoint bytes"))) + }) + + It("allocates a temp file on the worker", func() { + remote, err := stager.AllocRemoteTemp(context.Background(), nodeID) + Expect(err).NotTo(HaveOccurred()) + Expect(remote).To(BeAnExistingFile()) + }) + + It("allocates and downloads into the SAME cache directory", func() { + // The staging cache root is derived once and read by two things: the + // FileManager caches downloads into it, and the temp verb allocates + // inside it. Two derivations that drifted would each stay self + // consistent, so nothing else in this suite would notice; what would + // notice is an operator whose disk budget covers one of the two. + local := filepath.Join(GinkgoT().TempDir(), "same-root.gguf") + Expect(os.WriteFile(local, []byte("bytes"), 0o600)).To(Succeed()) + downloaded, err := stager.EnsureRemote(context.Background(), nodeID, local, storage.ModelKey("root/x.gguf")) + Expect(err).NotTo(HaveOccurred()) + + tmp, err := stager.AllocRemoteTemp(context.Background(), nodeID) + Expect(err).NotTo(HaveOccurred()) + + cacheRoot := filepath.Join(filepath.Dir(modelsDir), "cache") + Expect(downloaded).To(HavePrefix(cacheRoot + string(filepath.Separator))) + Expect(tmp).To(HavePrefix(cacheRoot + string(filepath.Separator))) + }) + + It("stages a worker file into the store", func() { + remote := filepath.Join(modelsDir, "result.bin") + Expect(os.WriteFile(remote, []byte("job output"), 0o600)).To(Succeed()) + + Expect(stager.StageRemoteToStore(context.Background(), nodeID, remote, "data/rt/result.bin")).To(Succeed()) + exists, err := store.Exists(context.Background(), "data/rt/result.bin") + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + + It("fetches a worker file back through the store", func() { + remote := filepath.Join(modelsDir, "fetched.bin") + Expect(os.WriteFile(remote, []byte("fetch me"), 0o600)).To(Succeed()) + dst := filepath.Join(GinkgoT().TempDir(), "local.bin") + + Expect(stager.FetchRemote(context.Background(), nodeID, remote, dst)).To(Succeed()) + Expect(os.ReadFile(dst)).To(Equal([]byte("fetch me"))) + }) + + It("lists a worker directory whose listing outgrows any bus payload", func() { + big := filepath.Join(modelsDir, "wide") + Expect(os.MkdirAll(big, 0o750)).To(Succeed()) + for i := range 4000 { + name := fmt.Sprintf("shard-%030d.safetensors", i) + Expect(os.WriteFile(filepath.Join(big, name), []byte("x"), 0o600)).To(Succeed()) + } + + files, err := stager.ListRemoteDir(context.Background(), nodeID, "models/wide") + Expect(err).NotTo(HaveOccurred()) + Expect(files).To(HaveLen(4000)) + }) + + It("reports the worker's own refusal as the worker's answer", func() { + _, err := stager.ListRemoteDir(context.Background(), nodeID, "../../../etc") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("backend listdir failed")) + // It said something about a file, not about the route, so it must not + // be wearing the umbrella that stops a caller acting on it. + Expect(errors.Is(err, nodes.ErrWorkerUnroutable)).To(BeFalse()) + }) + + // One rule, stated once per verb: every file-staging RPC goes through the + // control client, so a 401 is a failure of the ROUTE and never the worker + // saying a file is not there. Pinned at each verb because each verb writes + // the call out for itself, and a single verb that reached the worker some + // other way would still leave the other five green. + DescribeTable("reports a rejected token as unroutable, never as a verdict about a file", + func(call func(*nodes.S3FileStager) error) { + wrong := newStager("not-the-token") + err := call(wrong) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, nodes.ErrWorkerUnroutable)).To(BeTrue(), "got %v", err) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + Expect(errors.Is(err, nodes.ErrWorkerControlUnsupported)).To(BeFalse()) + }, + Entry("ensure", func(s *nodes.S3FileStager) error { + local := filepath.Join(GinkgoT().TempDir(), "m.gguf") + Expect(os.WriteFile(local, []byte("x"), 0o600)).To(Succeed()) + _, err := s.EnsureRemote(context.Background(), nodeID, local, storage.ModelKey("auth/m.gguf")) + return err + }), + Entry("temp", func(s *nodes.S3FileStager) error { + _, err := s.AllocRemoteTemp(context.Background(), nodeID) + return err + }), + Entry("listdir", func(s *nodes.S3FileStager) error { + _, err := s.ListRemoteDir(context.Background(), nodeID, "models/wide") + return err + }), + Entry("stage to store", func(s *nodes.S3FileStager) error { + return s.StageRemoteToStore(context.Background(), nodeID, filepath.Join(modelsDir, "x"), "data/x") + }), + Entry("fetch", func(s *nodes.S3FileStager) error { + return s.FetchRemote(context.Background(), nodeID, filepath.Join(modelsDir, "x"), + filepath.Join(GinkgoT().TempDir(), "out")) + }), + Entry("fetch by key", func(s *nodes.S3FileStager) error { + return s.FetchRemoteByKey(context.Background(), nodeID, "data/x", + filepath.Join(GinkgoT().TempDir(), "out")) + }), + ) +}) diff --git a/core/services/worker/file_staging.go b/core/services/worker/file_staging.go index 3eea3a9e6a79..9e65f2cedcb5 100644 --- a/core/services/worker/file_staging.go +++ b/core/services/worker/file_staging.go @@ -1,21 +1,8 @@ package worker import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path" "path/filepath" "strings" - "time" - - "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/core/services/storage" - "github.com/mudler/LocalAI/pkg/safefile" - "github.com/mudler/xlog" - "golang.org/x/sync/singleflight" ) // isPathAllowed checks if path is within one of the allowed directories. @@ -40,395 +27,3 @@ func isPathAllowed(path string, allowedDirs []string) bool { } return false } - -// subscribeFileStaging subscribes to NATS file staging subjects for this node. -func (cfg *Config) subscribeFileStaging(natsClient messaging.MessagingClient, nodeID string, capacity *EphemeralCapacityGuard) error { - // Create FileManager with same S3 config as the frontend - // TODO: propagate a caller-provided context once Config carries one - s3Store, err := storage.NewS3Store(context.Background(), storage.S3Config{ - Endpoint: cfg.StorageURL, - Region: cfg.StorageRegion, - Bucket: cfg.StorageBucket, - AccessKeyID: cfg.StorageAccessKey, - SecretAccessKey: cfg.StorageSecretKey, - ForcePathStyle: true, - }) - if err != nil { - return fmt.Errorf("initializing S3 store: %w", err) - } - - cacheDir := filepath.Join(cfg.ModelsPath, "..", "cache") - fm, err := storage.NewFileManager(s3Store, cacheDir) - if err != nil { - return fmt.Errorf("initializing file manager: %w", err) - } - if err := subscribeFileReleaseWithCapacity(natsClient, nodeID, fm, cacheDir, capacity); err != nil { - return err - } - var ensureGroup singleflight.Group - - // Subscribe: files.ensure — download S3 key to local, reply with local path - if _, err := natsClient.SubscribeReply(messaging.SubjectNodeFilesEnsure(nodeID), func(data []byte, reply func([]byte)) { - var req struct { - Key string `json:"key"` - } - if err := json.Unmarshal(data, &req); err != nil { - replyJSON(reply, map[string]string{"error": "invalid request"}) - return - } - - value, err, _ := ensureGroup.Do(req.Key, func() (any, error) { - return ensureWorkerFile(context.Background(), fm, capacity, req.Key) - }) - if err != nil { - xlog.Error("File ensure failed", "key", req.Key, "error", err) - replyJSON(reply, map[string]string{"error": err.Error()}) - return - } - localPath, ok := value.(string) - if !ok { - replyJSON(reply, map[string]string{"error": fmt.Sprintf("unexpected file ensure result %T", value)}) - return - } - - xlog.Debug("File ensured locally", "key", req.Key, "path", localPath) - replyJSON(reply, map[string]string{"local_path": localPath}) - }); err != nil { - return fmt.Errorf("subscribing to files.ensure events: %w", err) - } - - // Subscribe: files.stage — upload local path to S3, reply with key - if _, err := natsClient.SubscribeReply(messaging.SubjectNodeFilesStage(nodeID), func(data []byte, reply func([]byte)) { - var req struct { - LocalPath string `json:"local_path"` - Key string `json:"key"` - } - if err := json.Unmarshal(data, &req); err != nil { - replyJSON(reply, map[string]string{"error": "invalid request"}) - return - } - - allowedDirs := []string{cacheDir} - if cfg.ModelsPath != "" { - allowedDirs = append(allowedDirs, cfg.ModelsPath) - } - if !isPathAllowed(req.LocalPath, allowedDirs) { - replyJSON(reply, map[string]string{"error": "path outside allowed directories"}) - return - } - - if err := fm.Upload(context.Background(), req.Key, req.LocalPath); err != nil { - xlog.Error("File stage failed", "path", req.LocalPath, "key", req.Key, "error", err) - replyJSON(reply, map[string]string{"error": err.Error()}) - return - } - - xlog.Debug("File staged to S3", "path", req.LocalPath, "key", req.Key) - replyJSON(reply, map[string]string{"key": req.Key}) - }); err != nil { - return fmt.Errorf("subscribing to files.stage events: %w", err) - } - - // Subscribe: files.temp — allocate temp file, reply with local path - if _, err := natsClient.SubscribeReply(messaging.SubjectNodeFilesTemp(nodeID), func(data []byte, reply func([]byte)) { - tmpDir := filepath.Join(cacheDir, "staging-tmp") - if err := os.MkdirAll(tmpDir, 0750); err != nil { - replyJSON(reply, map[string]string{"error": fmt.Sprintf("creating temp dir: %v", err)}) - return - } - - f, err := os.CreateTemp(tmpDir, "localai-staging-*.tmp") - if err != nil { - replyJSON(reply, map[string]string{"error": fmt.Sprintf("creating temp file: %v", err)}) - return - } - localPath := f.Name() - if err := f.Close(); err != nil { - replyJSON(reply, map[string]string{"error": fmt.Sprintf("closing temp file: %v", err)}) - return - } - - xlog.Debug("Allocated temp file", "path", localPath) - replyJSON(reply, map[string]string{"local_path": localPath}) - }); err != nil { - return fmt.Errorf("subscribing to files.temp events: %w", err) - } - - // Subscribe: files.listdir — list files in a local directory, reply with relative paths - if _, err := natsClient.SubscribeReply(messaging.SubjectNodeFilesListDir(nodeID), func(data []byte, reply func([]byte)) { - var req struct { - KeyPrefix string `json:"key_prefix"` - } - if err := json.Unmarshal(data, &req); err != nil { - replyJSON(reply, map[string]any{"error": "invalid request"}) - return - } - - // Resolve key prefix to local directory - dirPath := filepath.Join(cacheDir, req.KeyPrefix) - if rel, ok := strings.CutPrefix(req.KeyPrefix, storage.ModelKeyPrefix); ok && cfg.ModelsPath != "" { - dirPath = filepath.Join(cfg.ModelsPath, rel) - } else if rel, ok := strings.CutPrefix(req.KeyPrefix, storage.DataKeyPrefix); ok { - dirPath = filepath.Join(cacheDir, "..", "data", rel) - } - - // Sanitize to prevent directory traversal via crafted key_prefix - dirPath = filepath.Clean(dirPath) - cleanCache := filepath.Clean(cacheDir) - cleanModels := filepath.Clean(cfg.ModelsPath) - cleanData := filepath.Clean(filepath.Join(cacheDir, "..", "data")) - if !(strings.HasPrefix(dirPath, cleanCache+string(filepath.Separator)) || - dirPath == cleanCache || - (cleanModels != "." && strings.HasPrefix(dirPath, cleanModels+string(filepath.Separator))) || - dirPath == cleanModels || - strings.HasPrefix(dirPath, cleanData+string(filepath.Separator)) || - dirPath == cleanData) { - replyJSON(reply, map[string]any{"error": "invalid key prefix"}) - return - } - - var files []string - if err := filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if !d.IsDir() { - rel, err := filepath.Rel(dirPath, path) - if err != nil { - return err - } - files = append(files, rel) - } - return nil - }); err != nil { - xlog.Error("Failed to list staged files", "keyPrefix", req.KeyPrefix, "dirPath", dirPath, "error", err) - replyJSON(reply, map[string]any{"error": err.Error()}) - return - } - - xlog.Debug("Listed remote dir", "keyPrefix", req.KeyPrefix, "dirPath", dirPath, "fileCount", len(files)) - replyJSON(reply, map[string]any{"files": files}) - }); err != nil { - return fmt.Errorf("subscribing to files.listdir events: %w", err) - } - - xlog.Info("Subscribed to file staging NATS subjects", "nodeID", nodeID) - return nil -} - -func subscribeFileRelease(natsClient messaging.MessagingClient, nodeID string, fm *storage.FileManager, cacheDir string) error { - return subscribeFileReleaseWithCapacity(natsClient, nodeID, fm, cacheDir, nil) -} - -func subscribeFileReleaseWithCapacity(natsClient messaging.MessagingClient, nodeID string, fm *storage.FileManager, cacheDir string, capacity *EphemeralCapacityGuard) error { - if _, err := natsClient.SubscribeReply(messaging.SubjectNodeFilesRelease(nodeID), func(data []byte, reply func([]byte)) { - var req struct { - Key string `json:"key"` - RequestID string `json:"request_id"` - } - if err := json.Unmarshal(data, &req); err != nil { - replyJSON(reply, map[string]string{"error": "invalid request"}) - return - } - var err error - if req.RequestID != "" { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - err = releaseEphemeralCacheRequest(ctx, cacheDir, req.RequestID, capacity) - cancel() - } else { - cachePath, cacheErr := fm.CachePath(req.Key) - err = cacheErr - if err == nil { - err = releaseEphemeralCachePathWithCapacity(cacheDir, req.Key, cachePath, capacity) - } - } - if err != nil { - replyJSON(reply, map[string]string{"error": err.Error()}) - return - } - replyJSON(reply, map[string]string{}) - }); err != nil { - return fmt.Errorf("subscribing to files.release events: %w", err) - } - return nil -} - -func releaseEphemeralCacheKey(cacheDir, key string) error { - return releaseEphemeralCachePath(cacheDir, key, filepath.Join(cacheDir, filepath.FromSlash(key))) -} - -func releaseEphemeralCachePath(cacheDir, key, filePath string) error { - return releaseEphemeralCachePathWithCapacity(cacheDir, key, filePath, nil) -} - -func releaseEphemeralCachePathWithCapacity(cacheDir, key, filePath string, capacity *EphemeralCapacityGuard) error { - if err := validateEphemeralCacheKey(key); err != nil { - return err - } - relativePath := filepath.FromSlash(key) - expectedPath := filepath.Join(cacheDir, relativePath) - if filepath.Clean(filePath) != expectedPath { - return fmt.Errorf("release path %q does not match key %q", filePath, key) - } - if err := safefile.RemoveExact(cacheDir, relativePath, []string{".sha256", ".sha256.target"}, 2); err != nil { - return err - } - for _, path := range []string{filePath, filePath + ".sha256", filePath + ".sha256.target"} { - if capacity != nil { - if err := capacity.Release(path); err != nil { - return err - } - } - } - return nil -} - -func releaseEphemeralCacheRequest(ctx context.Context, cacheDir, requestID string, capacity *EphemeralCapacityGuard) error { - if err := validateEphemeralCacheRequestID(requestID); err != nil { - return err - } - if capacity != nil { - if err := capacity.BeginRequestRelease(ctx, requestID); err != nil { - return fmt.Errorf("beginning release for request %q: %w", requestID, err) - } - defer capacity.EndRequestRelease(requestID) - } - root := filepath.Join(cacheDir, "ephemeral") - categories, err := os.ReadDir(root) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - var releaseErrors []error - for _, category := range categories { - if !category.IsDir() || category.Type()&os.ModeSymlink != 0 { - continue - } - requestDir := filepath.Join(root, category.Name(), requestID) - info, err := os.Lstat(requestDir) - if err != nil { - if !os.IsNotExist(err) { - releaseErrors = append(releaseErrors, fmt.Errorf("stating request directory %q: %w", requestDir, err)) - } - continue - } - if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { - releaseErrors = append(releaseErrors, fmt.Errorf("ephemeral request path %q is not a real directory", requestDir)) - continue - } - entries, err := os.ReadDir(requestDir) - if err != nil { - if !os.IsNotExist(err) { - releaseErrors = append(releaseErrors, fmt.Errorf("reading request directory %q: %w", requestDir, err)) - } - continue - } - for _, entry := range entries { - if entry.IsDir() { - releaseErrors = append(releaseErrors, fmt.Errorf("unexpected directory in ephemeral request %q", filepath.Join(requestDir, entry.Name()))) - continue - } - key := filepath.ToSlash(filepath.Join("ephemeral", category.Name(), requestID, entry.Name())) - filePath := filepath.Join(cacheDir, filepath.FromSlash(key)) - if err := releaseEphemeralCachePathWithCapacity(cacheDir, key, filePath, capacity); err != nil { - releaseErrors = append(releaseErrors, fmt.Errorf("releasing %q: %w", key, err)) - } - } - } - return errors.Join(releaseErrors...) -} - -type ephemeralStagingCapacity interface { - Reserve(path string, size int64) error - Commit(path string) error - Claim(path string) error - Release(path string) error -} - -func ensureWorkerFile(ctx context.Context, fm *storage.FileManager, capacity *EphemeralCapacityGuard, key string) (string, error) { - if capacity == nil { - return fm.Download(ctx, key) - } - if strings.HasPrefix(key, "ephemeral/") { - if err := validateEphemeralCacheKey(key); err != nil { - return "", err - } - requestID := strings.Split(key, "/")[2] - if err := capacity.BeginRequestOperation(requestID); err != nil { - return "", err - } - defer capacity.EndRequestOperation(requestID) - } - return ensureWorkerFileWithCapacity(ctx, fm, capacity, key) -} - -func ensureWorkerFileWithCapacity(ctx context.Context, fm *storage.FileManager, capacity ephemeralStagingCapacity, key string) (string, error) { - if capacity == nil || !strings.HasPrefix(key, "ephemeral/") { - return fm.Download(ctx, key) - } - if err := validateEphemeralCacheKey(key); err != nil { - return "", err - } - cachePath, err := fm.CachePath(key) - if err != nil { - return "", err - } - if info, statErr := os.Lstat(cachePath); statErr == nil { - if !info.Mode().IsRegular() { - return "", fmt.Errorf("ephemeral cache path %q is not a regular file", cachePath) - } - if err := capacity.Claim(cachePath); err != nil { - if !errors.Is(err, os.ErrNotExist) { - return "", err - } - } else { - return cachePath, nil - } - } else if !os.IsNotExist(statErr) { - return "", statErr - } - - meta, err := fm.Head(ctx, key) - if err != nil { - return "", fmt.Errorf("reading size for %s: %w", key, err) - } - if err := capacity.Reserve(cachePath, meta.Size); err != nil { - return "", err - } - localPath, err := fm.Download(ctx, key) - if err != nil { - _ = capacity.Release(cachePath) - return "", err - } - if err := capacity.Commit(cachePath); err != nil { - _ = fm.EvictCache(key) - _ = capacity.Release(cachePath) - return "", err - } - return localPath, nil -} - -func validateEphemeralCacheKey(key string) error { - if strings.Contains(key, "\\") || path.Clean(key) != key { - return fmt.Errorf("invalid ephemeral key %q", key) - } - parts := strings.Split(key, "/") - if len(parts) != 4 || parts[0] != "ephemeral" { - return fmt.Errorf("release key %q must identify one file below ephemeral/", key) - } - for _, part := range parts[1:] { - if part == "" || part == "." || part == ".." { - return fmt.Errorf("invalid ephemeral key %q", key) - } - } - return nil -} - -func validateEphemeralCacheRequestID(requestID string) error { - if requestID == "" || strings.ContainsAny(requestID, "/\\") || path.Clean(requestID) != requestID || requestID == "." || requestID == ".." { - return fmt.Errorf("invalid ephemeral request ID %q", requestID) - } - return nil -} diff --git a/core/services/worker/reply.go b/core/services/worker/reply.go deleted file mode 100644 index 9700f19fa47c..000000000000 --- a/core/services/worker/reply.go +++ /dev/null @@ -1,17 +0,0 @@ -package worker - -import ( - "encoding/json" - - "github.com/mudler/xlog" -) - -// replyJSON marshals v to JSON and calls the reply function. -func replyJSON(reply func([]byte), v any) { - data, err := json.Marshal(v) - if err != nil { - xlog.Error("Failed to marshal NATS reply", "error", err) - data = []byte(`{"error":"internal marshal error"}`) - } - reply(data) -} diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index ea1dc2f82024..50a74ede393d 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -20,6 +20,7 @@ import ( "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/storage" "github.com/mudler/LocalAI/core/services/workerctl" grpc "github.com/mudler/LocalAI/pkg/grpc" "github.com/mudler/LocalAI/pkg/model" @@ -247,8 +248,26 @@ func Run(ctx *cliContext.Context, cfg *Config) error { maxPort: cfg.effectiveMaxPort(basePort), } + // The file-staging FileManager is built BEFORE the server, for the same + // reason the supervisor is: the server is what serves those four verbs, and + // a route that is not mounted when the tunnel comes up is a 404 the + // frontend reads as a worker that does not implement the verb. It used to + // be built after NATS connected, which is a window that no longer exists. + // + // A worker with no object store configured mounts NO file verbs. That is + // the honest answer rather than a degraded one: it has nowhere to fetch + // from or stage to, and the frontend that reaches such a deployment uses + // the HTTP file stager, which does not call these paths at all. + var stagingFM *storage.FileManager + if cfg.StorageURL != "" { + stagingFM, err = cfg.NewStagingFileManager(shutdownCtx) + if err != nil { + return fmt.Errorf("initializing file staging: %w", err) + } + } + httpServer, err := startWorkerHTTPServer(httpAddr, stagingDir, cfg.ModelsPath, dataDir, - cfg.RegistrationToken, readiness, supervisor, ephemeralCapacity, ml.BackendLogs()) + cfg.RegistrationToken, readiness, supervisor, ephemeralCapacity, cfg, stagingFM, ml.BackendLogs()) if err != nil { return fmt.Errorf("starting HTTP file transfer server: %w", err) } @@ -335,15 +354,6 @@ func Run(ctx *cliContext.Context, cfg *Config) error { return conn.Close() }), )) - - // Subscribe to file staging NATS subjects if S3 is configured - if cfg.StorageURL != "" { - if err := cfg.subscribeFileStaging(natsClient, nodeID, ephemeralCapacity); err != nil { - nodes.ShutdownFileTransferServer(httpServer) - return fmt.Errorf("subscribing to file staging subjects: %w", err) - } - } - xlog.Info("Worker ready, serving its control plane over the tunnel") // Exit on an OS signal or on an internal fatal condition (e.g. NATS // credentials became unrenewable), so the worker restarts and re-acquires @@ -375,10 +385,18 @@ func Run(ctx *cliContext.Context, cfg *Config) error { // healthy while answering 404 to every command. func startWorkerHTTPServer(addr, stagingDir, modelsDir, dataDir, token string, readiness *nodes.WorkerReadiness, sup *backendSupervisor, capacity nodes.EphemeralCapacity, - logStore *model.BackendLogStore) (*http.Server, error) { + cfg *Config, stagingFM *storage.FileManager, logStore *model.BackendLogStore) (*http.Server, error) { return nodes.StartFileTransferServerWithCapacityAndRoutes(addr, stagingDir, modelsDir, dataDir, token, config.DefaultMaxUploadSize, readiness, capacity, &nodes.AuthenticatedRoutes{ - Prefix: workerctl.Prefix, - Register: sup.RegisterControlRoutes, + Prefix: workerctl.Prefix, + // One registrar for both route sets, because there is ONE control + // prefix and AuthenticatedRoutes mounts one mux behind one bearer + // check. A second route set would be a second check to forget. + Register: func(mux *http.ServeMux) { + sup.RegisterControlRoutes(mux) + if stagingFM != nil { + cfg.RegisterFileControlRoutes(mux, stagingFM) + } + }, }, logStore) } diff --git a/core/services/workerctl/paths.go b/core/services/workerctl/paths.go index e76b5566dacd..4af1164d843d 100644 --- a/core/services/workerctl/paths.go +++ b/core/services/workerctl/paths.go @@ -29,6 +29,17 @@ const ( PathModelDelete = "/v1/control/model/delete" PathModelsRunning = "/v1/control/models/running" PathNodeStop = "/v1/control/node/stop" + + // The file-staging verbs. They are only ever served by a worker whose + // deployment configured an object store, because without one there is + // nothing for them to move a file to or from; a worker without one mounts + // them not at all, and the catch-all under Prefix answers for them. That is + // the same 404 an older build gives, which is what the frontend already + // reads as "this worker does not serve that verb" rather than as absence. + PathFilesEnsure = "/v1/control/files/ensure" + PathFilesStage = "/v1/control/files/stage" + PathFilesTemp = "/v1/control/files/temp" + PathFilesListDir = "/v1/control/files/listdir" ) // AllPaths returns every control verb's path. @@ -47,6 +58,10 @@ func AllPaths() []string { PathModelDelete, PathModelsRunning, PathNodeStop, + PathFilesEnsure, + PathFilesStage, + PathFilesTemp, + PathFilesListDir, } } diff --git a/core/services/workerctl/paths_test.go b/core/services/workerctl/paths_test.go index 8ffabb2e3cd9..4d72124ec0c9 100644 --- a/core/services/workerctl/paths_test.go +++ b/core/services/workerctl/paths_test.go @@ -28,6 +28,10 @@ var _ = Describe("control plane paths on the wire", func() { Entry("model delete", workerctl.PathModelDelete, "/v1/control/model/delete"), Entry("models running", workerctl.PathModelsRunning, "/v1/control/models/running"), Entry("node stop", workerctl.PathNodeStop, "/v1/control/node/stop"), + Entry("files ensure", workerctl.PathFilesEnsure, "/v1/control/files/ensure"), + Entry("files stage", workerctl.PathFilesStage, "/v1/control/files/stage"), + Entry("files temp", workerctl.PathFilesTemp, "/v1/control/files/temp"), + Entry("files listdir", workerctl.PathFilesListDir, "/v1/control/files/listdir"), ) It("names the prefix exactly, since the worker mounts its whole control plane behind it", func() { @@ -57,6 +61,10 @@ var _ = Describe("control plane paths on the wire", func() { workerctl.PathModelDelete, workerctl.PathModelsRunning, workerctl.PathNodeStop, + workerctl.PathFilesEnsure, + workerctl.PathFilesStage, + workerctl.PathFilesListDir, + workerctl.PathFilesTemp, )) }) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 6f81ec891ed0..6b138b66b7ca 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -38,7 +38,7 @@ Each model gets its own gRPC backend process, so a single worker can serve multi ## Prerequisites - **PostgreSQL** (with pgvector extension recommended for RAG) - used for node registry, job store, auth, and shared state -- **NATS** server - used for real-time backend lifecycle events and file staging +- **NATS** server - used for agent-worker coordination and the frontend's own cross-replica events. Serve-backend workers no longer take any command over it: the backend and model lifecycle verbs and file staging are HTTP routes on the worker's tunnel. - All services must be on the same network (or reachable via configured URLs) ## Quick Start with Docker Compose @@ -172,17 +172,24 @@ Every connection the frontend makes to a worker now goes through that worker's t | Inference, model load, health checks | gRPC to a backend process | `grpc` | | Model file staging, backend-log listing | HTTP to the worker's own server | `http` | | Live backend-log streaming | WebSocket to the same server | `http` | -| The control plane: backend install/upgrade/list/stop/delete, model stop/unload/delete, models running, node stop | HTTP to the worker's own server | `http` | +| The control plane: backend install/upgrade/list/stop/delete, model stop/unload/delete, models running, node stop, and the four object-store staging verbs | HTTP to the worker's own server | `http` | #### The worker control plane A serve-backend worker serves the commands the frontend gives it as ordinary HTTP routes under `/v1/control/`, on the same loopback server that already carries file staging and backend logs, behind the same `LOCALAI_REGISTRATION_TOKEN` -bearer check. They replace the ten `nodes..*` NATS subjects a worker used to -subscribe to; the request and reply bodies are unchanged, so nothing an operator -inspects on the wire has a new shape. Agent workers still take `nodes..backend.stop` -over NATS. +bearer check. They replace the fourteen `nodes..*` NATS subjects a worker +used to subscribe to - the ten backend and model lifecycle verbs, plus the four +object-store staging verbs (`POST /v1/control/files/{ensure,stage,temp,listdir}`, +mounted only when the deployment configured an object store). The request and +reply bodies are unchanged, so nothing an operator inspects on the wire has a new +shape. Agent workers still take `nodes..backend.stop` over NATS. + +`files/listdir` is the verb the change is most visible on. Its reply used to be +sized against what the bus would carry, which put a wide model directory close to +the limit; it is now a response body the frontend is already reading, so the +listing is returned whole and nothing truncates it at either end. Two of the routes stream. `POST /v1/control/backend/install` and `/v1/control/backend/upgrade` answer with `application/x-ndjson`: zero or more @@ -401,7 +408,7 @@ A frontend replica that dies mid-load does not wedge the model: the job row carr ### NATS JWT authentication (recommended for production) -By default, NATS connections are anonymous: any client that can reach port `4222` may publish the subjects still carried on it. Serve-backend workers no longer subscribe to `nodes..backend.install` and its nine siblings - those are HTTP routes on the worker's tunnel now, see [The worker control plane](#the-worker-control-plane) - but agent workers, file staging and the frontend's own service credential still use NATS. Enable JWT auth to scope workers to their own node subjects and give the frontend a dedicated service credential. +By default, NATS connections are anonymous: any client that can reach port `4222` may publish the subjects still carried on it. Serve-backend workers no longer subscribe to `nodes..backend.install` and its nine siblings - those are HTTP routes on the worker's tunnel now, see [The worker control plane](#the-worker-control-plane) - but agent workers and the frontend's own service credential still use NATS. Enable JWT auth to scope workers to their own node subjects and give the frontend a dedicated service credential. | Flag | Env Var | Description | |------|---------|-------------| @@ -445,7 +452,7 @@ Generate operator/account material with [`scripts/nats-auth-setup.sh`](https://g ### Optional: S3 Object Storage -For multi-host deployments where workers don't share a filesystem, S3-compatible storage enables distributed file transfer (model files, configs): +For multi-host deployments where workers don't share a filesystem, S3-compatible storage enables distributed file transfer (model files, configs). The frontend uploads the file to the bucket and then tells the worker to fetch it, over that worker's tunnel (`POST /v1/control/files/ensure`); the reverse direction (`.../files/stage`) has the worker upload one of its own files for the frontend to pull down. The bytes travel through the bucket, never through the tunnel: | Flag | Env Var | Default | Description | |------|---------|---------|-------------| @@ -455,6 +462,8 @@ For multi-host deployments where workers don't share a filesystem, S3-compatible | `--storage-access-key` | `LOCALAI_STORAGE_ACCESS_KEY` | *(empty)* | S3 access key | | `--storage-secret-key` | `LOCALAI_STORAGE_SECRET_KEY` | *(empty)* | S3 secret key | +A worker started without `LOCALAI_STORAGE_URL` does not serve the four staging verbs at all, and answers `404` for them, which is the same answer a frontend gets from a worker too old to know them. + When S3 is not configured, model files are transferred directly from the frontend to workers via **HTTP** - no shared filesystem needed. Each worker runs a small HTTP file transfer server alongside the gRPC backend process. This is the default and works out of the box. For high-throughput or very large model files, S3 can be more efficient since it avoids streaming through the frontend. @@ -476,7 +485,10 @@ Hugging Face for managed artifacts. With `LOCALAI_DISTRIBUTED_SHARED_MODELS` enabled, workers use the shared absolute snapshot path and skip transfer. Otherwise, the controller stages the -complete snapshot tree to each worker before loading the backend. +complete snapshot tree to each worker before loading the backend. With an object +store configured the controller uploads to the bucket and commands the worker to +fetch over its tunnel; without one it pushes the files to the worker's HTTP file +transfer server directly. Neither path uses NATS. {{% notice warning %}} Every controller and worker must have enough disk space for its own snapshot @@ -550,7 +562,7 @@ local-ai worker \ | `--distributed-require-auth` | `LOCALAI_DISTRIBUTED_REQUIRE_AUTH` | `false` | Umbrella switch implying both `--registration-require-auth` and `--nats-require-auth` | | `--heartbeat-interval` | `LOCALAI_HEARTBEAT_INTERVAL` | `10s` | Interval between heartbeat pings | | `--worker-tunnel` | `LOCALAI_WORKER_TUNNEL` | `true` | Hold one outbound multiplexed tunnel to the frontend and serve its requests over it, so this worker needs no inbound port (see [Worker tunnels](#worker-tunnels)). Setting it to `false` is a **fatal startup error**, not a degraded mode: the frontend has no path that dials a worker's advertised address, so a worker without its tunnel is a worker nothing can reach. To run without tunnels, run the pre-tunnel release on both the worker and the frontend. | -| `--nats-url` | `LOCALAI_NATS_URL` | *(required)* | NATS URL for backend installation and file staging | +| `--nats-url` | `LOCALAI_NATS_URL` | *(required)* | NATS URL. A serve-backend worker takes no command over it - backend installation and file staging are HTTP routes on its tunnel - but it must still connect for the frontend to consider it healthy. | | `--nats-jwt` | `LOCALAI_NATS_JWT` | *(empty)* | Optional override for the `nats_jwt` returned at registration | | `--nats-user-seed` | `LOCALAI_NATS_USER_SEED` | *(empty)* | Optional override for `nats_user_seed` from registration | | `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | `false` | Require NATS JWT+seed (from registration or env) | diff --git a/pkg/natsauth/mint_test.go b/pkg/natsauth/mint_test.go index e33992ec1898..195f05b2c343 100644 --- a/pkg/natsauth/mint_test.go +++ b/pkg/natsauth/mint_test.go @@ -37,10 +37,11 @@ var _ = Describe("MintWorkerJWT", func() { uc, err := jwt.DecodeUserClaims(token) Expect(err).NotTo(HaveOccurred()) Expect(uc.Permissions.Sub.Allow).To(ContainElement("nodes.550e8400-e29b-41d4-a716-446655440000.>")) - Expect(uc.Permissions.Pub.Allow).To(ContainElement("nodes.550e8400-e29b-41d4-a716-446655440000.files.>")) // The install-progress subject is gone with the carrier: progress is a // line in the install response now, so a minted worker JWT must not - // still be granted a publish right for it. + // still be granted a publish right for it. File staging went the same + // way, so a minted worker JWT publishes nowhere but its own inbox. + Expect(uc.Permissions.Pub.Allow).To(ConsistOf("_INBOX.>")) for _, subj := range uc.Permissions.Pub.Allow { Expect(subj).NotTo(ContainSubstring("backend.install")) } diff --git a/pkg/natsauth/permissions.go b/pkg/natsauth/permissions.go index a411e05fd478..8e0fa985e22b 100644 --- a/pkg/natsauth/permissions.go +++ b/pkg/natsauth/permissions.go @@ -38,22 +38,23 @@ func WorkerPermissions(nodeID, nodeType string) (pubAllow, subAllow []string) { "_INBOX.>", } default: - // Backend worker: file staging on this node only. - // - // The backend and model lifecycle verbs left the bus: they are HTTP + // Backend worker. Every verb a frontend gives it left the bus: the + // backend and model lifecycle verbs and now file staging too are HTTP // routes under workerctl.Prefix, served on the worker's own server and // reached through its tunnel, so no subject is minted for them and none - // is allowed here. The wildcard stays because the file-staging subjects - // are still under this node's prefix; narrowing it to nodes..files.> - // is a separate change that would break a worker mid-upgrade. + // is allowed here. + // + // The subscribe wildcard stays for now. A worker subscribes to nothing + // under it on this build, but narrowing it is a change a worker + // mid-upgrade would feel, and the connection itself is what the next + // step of this removal deletes. subAllow = []string{ prefix + ".>", "_INBOX.>", } - // backend.install.*.progress is gone with the subject: install progress - // is written into the install response the frontend is already reading. + // Nothing left to publish. backend.install.*.progress went with the + // install subject, and the file-staging replies went with theirs. pubAllow = []string{ - prefix + ".files.>", "_INBOX.>", } } diff --git a/pkg/natsauth/permissions_coverage_test.go b/pkg/natsauth/permissions_coverage_test.go index a66d4f74e019..4a7ffa7ac748 100644 --- a/pkg/natsauth/permissions_coverage_test.go +++ b/pkg/natsauth/permissions_coverage_test.go @@ -59,37 +59,32 @@ var _ = Describe("WorkerPermissions subject coverage", func() { Context("backend worker", func() { pub, sub := natsauth.WorkerPermissions(nodeID, "backend") - // Every subject core/services/worker/file_staging.go subscribes to. - // The backend and model lifecycle verbs are NOT here: they left the bus - // for the worker's tunnelled control plane, so there is no subject to - // cover. See core/services/workerctl. - subscribed := []string{ - messaging.SubjectNodeFilesEnsure(nodeID), - messaging.SubjectNodeFilesStage(nodeID), - messaging.SubjectNodeFilesTemp(nodeID), - messaging.SubjectNodeFilesListDir(nodeID), - } - for _, subject := range subscribed { - It("allows subscribing to "+subject, func() { - Expect(anyAllows(sub, subject)).To(BeTrue(), - "backend JWT sub allow-list %v does not cover %s", sub, subject) - }) - } - - It("allows publishing file staging replies", func() { - subject := messaging.SubjectNodeFilesStage(nodeID) - Expect(anyAllows(pub, subject)).To(BeTrue(), - "backend JWT pub allow-list %v does not cover %s", pub, subject) + // A backend worker subscribes to no subject of its own on this build. + // Every verb a frontend gives it — the backend and model lifecycle ten, + // and now the four file-staging verbs — is an HTTP route on its + // tunnelled control plane, so there is no subject left to cover. See + // core/services/workerctl. + // + // The subscribe wildcard is asserted rather than removed because the + // grant is still minted and a worker mid-upgrade still uses it. + It("still grants a backend worker its own node subtree to subscribe on", func() { + Expect(sub).To(ConsistOf( + "nodes."+workerSubjectTokenForTest(nodeID)+".>", + "_INBOX.>", + )) }) // The negative half, and it is the one that would catch a verb quietly // coming back to the bus: a backend worker is granted nothing to - // publish outside its own file-staging subtree and its inbox. - It("grants a backend worker no publish rights outside file staging and its inbox", func() { - Expect(pub).To(ConsistOf( - "nodes."+workerSubjectTokenForTest(nodeID)+".files.>", - "_INBOX.>", - )) + // publish at all beyond its own inbox. File staging used to be the one + // exception and is not any more. + It("grants a backend worker no publish rights outside its inbox", func() { + Expect(pub).To(ConsistOf("_INBOX.>")) + }) + + It("no longer grants a backend worker the file-staging publish subtree", func() { + Expect(anyAllows(pub, "nodes."+workerSubjectTokenForTest(nodeID)+".files.stage")).To(BeFalse(), + "backend JWT pub allow-list %v still covers file staging", pub) }) }) diff --git a/tests/e2e/distributed/file_staging_test.go b/tests/e2e/distributed/file_staging_test.go index e765d87c3b87..a22a639c455b 100644 --- a/tests/e2e/distributed/file_staging_test.go +++ b/tests/e2e/distributed/file_staging_test.go @@ -7,9 +7,9 @@ import ( "path/filepath" "github.com/mudler/LocalAI/core/config" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/storage" + "github.com/mudler/LocalAI/core/services/workerctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -41,8 +41,8 @@ var _ = Describe("File Staging", Label("Distributed"), func() { Expect(err).ToNot(HaveOccurred()) }) - Context("S3NATSFileStager", func() { - It("should create S3NATSFileStager with valid config", func() { + Context("S3FileStager", func() { + It("should create S3FileStager with valid config", func() { storeDir := filepath.Join(tmpDir, "objectstore") cacheDir := filepath.Join(tmpDir, "cache") @@ -53,7 +53,7 @@ var _ = Describe("File Staging", Label("Distributed"), func() { Expect(err).ToNot(HaveOccurred()) Expect(fm.IsConfigured()).To(BeTrue()) - stager := nodes.NewS3NATSFileStager(fm, infra.NC) + stager := nodes.NewS3FileStager(fm, nodes.NewControlClient(directWorkerDialerFor, "")) Expect(stager).ToNot(BeNil()) }) }) @@ -83,8 +83,8 @@ var _ = Describe("File Staging", Label("Distributed"), func() { }) }) - Context("S3NATSFileStager with backend node simulation", func() { - It("should coordinate file staging via NATS request-reply", func() { + Context("S3FileStager with backend node simulation", func() { + It("should coordinate file staging over the worker's control plane", func() { storeDir := filepath.Join(tmpDir, "objectstore") cacheDir := filepath.Join(tmpDir, "cache") @@ -103,15 +103,14 @@ var _ = Describe("File Staging", Label("Distributed"), func() { } Expect(registry.Register(context.Background(), node, true)).To(Succeed()) - // Verify NATS file staging subjects are correctly formed - ensureSubj := messaging.SubjectNodeFilesEnsure(node.ID) - Expect(ensureSubj).To(ContainSubstring("files.ensure")) - - stageSubj := messaging.SubjectNodeFilesStage(node.ID) - Expect(stageSubj).To(ContainSubstring("files.stage")) - - tempSubj := messaging.SubjectNodeFilesTemp(node.ID) - Expect(tempSubj).To(ContainSubstring("files.temp")) + // The staging verbs are HTTP routes on the worker's own control + // plane now, so what a registered node is addressed by is its + // tunnel host and the path, not a subject. + Expect(nodes.WorkerHTTPHost(node.ID, "")).To(ContainSubstring(node.ID)) + Expect(workerctl.PathFilesEnsure).To(HavePrefix(workerctl.Prefix)) + Expect(workerctl.PathFilesStage).To(HavePrefix(workerctl.Prefix)) + Expect(workerctl.PathFilesTemp).To(HavePrefix(workerctl.Prefix)) + Expect(workerctl.PathFilesListDir).To(HavePrefix(workerctl.Prefix)) }) }) From fd1eb342a270fc96ad42787f60bdb0c0a39050c0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 21:54:04 +0000 Subject: [PATCH 075/153] test(distributed): pin the unreadable-request rule at all three file verbs The rule "a body this worker could not parse is a non-2xx, never the worker's answer" is written at three exits in control_files.go and only ensure was pinned. Turning stage's or listdir's decode exit into a 200-with-error left worker and nodes entirely green, and what that converts is a frontend's malformed request into the worker's own verdict about a file, which passes cluster.IsWorkerAnswer and reaches a reap guard. The production code was already right; nothing held it there. The e2e NATS JWT spec was asserting the opposite of the code and passing. It published nodes..files.in and called it an allowed subject after that grant was deleted, and it could not tell: a permission violation does not close the connection, so FlushTimeout and IsConnected both stay happy. It now reads LastError, the way its sibling always has, and asserts the denial plus the one publish right a backend worker has left. Also pinned, each mutation-verified alone: the CreateTemp branch (an existing staging-tmp at 0500 reaches it without a seam), the walk's context check (a caller that gave up must fail the listing, never be answered with a short one), and the cache and data directory layout. The data directory was derived twice, once in worker.go and once for the listdir verb; worker.go now reads the same helper, so a move cannot leave a verb listing files the file server does not serve. The per-verb RPC ceiling moves from an argument at five call sites into fileRPCBudget, so no site can name the wrong one, and the two values are asserted. The body-cap table now holds both directions locally and with two different claims: a body exactly at the cap proves the bound is a ceiling and not an off-by-one, and an absolute megabyte proves the cap stays above real gallery traffic. Only the second notices a cap shrunk to 64 KiB. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/nodes/file_stager.go | 7 +- core/services/nodes/file_stager_s3.go | 47 +++++--- core/services/nodes/file_stager_s3_test.go | 21 ++++ core/services/nodes/file_staging_client.go | 4 +- core/services/worker/control_files_test.go | 116 +++++++++++++++++++- core/services/worker/control_routes_test.go | 4 +- core/services/worker/worker.go | 4 +- docs/content/features/distributed-mode.md | 8 +- tests/e2e/distributed/nats_jwt_test.go | 26 ++++- 9 files changed, 204 insertions(+), 33 deletions(-) diff --git a/core/services/nodes/file_stager.go b/core/services/nodes/file_stager.go index 144c93165360..ee28a62b8b94 100644 --- a/core/services/nodes/file_stager.go +++ b/core/services/nodes/file_stager.go @@ -10,11 +10,14 @@ import ( // FileStager abstracts file transfer between frontend and backend nodes // in distributed mode. Two implementations exist: // -// 1. S3NATSFileStager (primary): Both sides have FileManager with same S3. -// Frontend uploads to S3, sends NATS request-reply to backend to download locally. +// 1. S3FileStager (primary): Both sides have FileManager with same S3. +// Frontend uploads to S3, then calls the worker's control plane over its +// tunnel to have it download locally. // // 2. HTTPFileStager (fallback): Frontend pushes/pulls files directly over // HTTP to a small file transfer server on the backend node (no S3 needed). +// +// Both reach the worker through its tunnel and neither uses NATS. type FileStager interface { // EnsureRemote ensures a local file is available on the remote node. // Returns the remote-local path. diff --git a/core/services/nodes/file_stager_s3.go b/core/services/nodes/file_stager_s3.go index e60ddb5d1c8a..18030368e0c9 100644 --- a/core/services/nodes/file_stager_s3.go +++ b/core/services/nodes/file_stager_s3.go @@ -44,6 +44,29 @@ const ( fileMetadataRPCTimeout = 30 * time.Second ) +// fileRPCBudget is the ceiling one file-staging verb's RPC gets. +// +// The mapping lives here rather than at the call sites, and that is the same +// argument callWorker is written under: five sites each naming their own +// constant is five chances to name the wrong one, and a stage verb given the +// metadata ceiling would abandon a multi-gigabyte copy after thirty seconds +// while the worker went on making it. +// +// A path this function does not know gets the SHORTER ceiling. That is the safe +// direction: a verb wrongly given 30 seconds fails visibly and is retried, +// while one wrongly given ten minutes parks a caller on a verb that was never +// meant to be slow. +func fileRPCBudget(path string) time.Duration { + switch path { + case workerctl.PathFilesEnsure, workerctl.PathFilesStage: + return fileTransferRPCTimeout + case workerctl.PathFilesTemp, workerctl.PathFilesListDir: + return fileMetadataRPCTimeout + default: + return fileMetadataRPCTimeout + } +} + // Control request/reply message types. Their JSON is the shape the // nodes..files.* subjects carried, so the worker's handler bodies did not // have to change when the carrier did. @@ -93,15 +116,16 @@ type fileListDirReply struct { } // callWorker issues one file-staging RPC under a deadline derived from the -// caller's context. +// caller's context and bounded by the verb's own ceiling. // // It exists so the derivation is written ONCE. Five call sites each repeating // context.WithTimeout is five chances for one of them to start from a // background context instead, and that one site would then keep commanding a // worker after its caller had gone, with nothing else in the suite any redder -// for it. -func (s *S3FileStager) callWorker(ctx context.Context, nodeID, path string, budget time.Duration, req, reply any) error { - rpcCtx, cancel := context.WithTimeout(ctx, budget) +// for it. The budget comes from the path rather than from an argument for the +// same reason; see fileRPCBudget. +func (s *S3FileStager) callWorker(ctx context.Context, nodeID, path string, req, reply any) error { + rpcCtx, cancel := context.WithTimeout(ctx, fileRPCBudget(path)) defer cancel() return s.control.Call(rpcCtx, nodeID, path, req, reply) } @@ -125,8 +149,7 @@ func (s *S3FileStager) EnsureRemote(ctx context.Context, nodeID, localPath, key } var reply fileEnsureReply - if err := s.callWorker(ctx, nodeID, workerctl.PathFilesEnsure, fileTransferRPCTimeout, - fileEnsureRequest{Key: key}, &reply); err != nil { + if err := s.callWorker(ctx, nodeID, workerctl.PathFilesEnsure, fileEnsureRequest{Key: key}, &reply); err != nil { return "", err } if reply.Error != "" { @@ -155,8 +178,7 @@ func (s *S3FileStager) FetchRemoteByKey(ctx context.Context, nodeID, key, localD func (s *S3FileStager) fetchRemoteWithKey(ctx context.Context, nodeID, remotePath, key, localDst string, cleanup bool) error { var reply fileStageReply - if err := s.callWorker(ctx, nodeID, workerctl.PathFilesStage, fileTransferRPCTimeout, - fileStageRequest{LocalPath: remotePath, Key: key}, &reply); err != nil { + if err := s.callWorker(ctx, nodeID, workerctl.PathFilesStage, fileStageRequest{LocalPath: remotePath, Key: key}, &reply); err != nil { return err } if reply.Error != "" { @@ -187,8 +209,7 @@ func (s *S3FileStager) fetchRemoteWithKey(ctx context.Context, nodeID, remotePat // AllocRemoteTemp asks the worker to allocate a temp file. func (s *S3FileStager) AllocRemoteTemp(ctx context.Context, nodeID string) (string, error) { var reply fileTempReply - if err := s.callWorker(ctx, nodeID, workerctl.PathFilesTemp, fileMetadataRPCTimeout, - fileTempRequest{}, &reply); err != nil { + if err := s.callWorker(ctx, nodeID, workerctl.PathFilesTemp, fileTempRequest{}, &reply); err != nil { return "", err } if reply.Error != "" { @@ -207,8 +228,7 @@ func (s *S3FileStager) AllocRemoteTemp(ctx context.Context, nodeID string) (stri // listing would read to the caller as files the worker does not have. func (s *S3FileStager) ListRemoteDir(ctx context.Context, nodeID, keyPrefix string) ([]string, error) { var reply fileListDirReply - if err := s.callWorker(ctx, nodeID, workerctl.PathFilesListDir, fileMetadataRPCTimeout, - fileListDirRequest{KeyPrefix: keyPrefix}, &reply); err != nil { + if err := s.callWorker(ctx, nodeID, workerctl.PathFilesListDir, fileListDirRequest{KeyPrefix: keyPrefix}, &reply); err != nil { return nil, err } if reply.Error != "" { @@ -221,8 +241,7 @@ func (s *S3FileStager) ListRemoteDir(ctx context.Context, nodeID, keyPrefix stri // StageRemoteToStore tells the worker to upload a local file to shared storage. func (s *S3FileStager) StageRemoteToStore(ctx context.Context, nodeID, remotePath, key string) error { var reply fileStageReply - if err := s.callWorker(ctx, nodeID, workerctl.PathFilesStage, fileTransferRPCTimeout, - fileStageRequest{LocalPath: remotePath, Key: key}, &reply); err != nil { + if err := s.callWorker(ctx, nodeID, workerctl.PathFilesStage, fileStageRequest{LocalPath: remotePath, Key: key}, &reply); err != nil { return err } if reply.Error != "" { diff --git a/core/services/nodes/file_stager_s3_test.go b/core/services/nodes/file_stager_s3_test.go index 8fdc4ae01036..beceeb2facb7 100644 --- a/core/services/nodes/file_stager_s3_test.go +++ b/core/services/nodes/file_stager_s3_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -44,6 +45,26 @@ var _ = Describe("the S3 file stager's control RPCs", func() { Expect(os.WriteFile(local, []byte("weights"), 0o600)).To(Succeed()) }) + // The budgets are written out BY HAND and deliberately not derived from the + // constants under test. They are the timeouts the NATS request-reply calls + // carried, and keeping them is a compatibility fact rather than a taste: + // an operator whose staging of a 35 GB checkpoint fits inside ten minutes + // today must not find it cut to thirty seconds by the carrier changing. + DescribeTable("gives each verb the ceiling its NATS timeout carried", + func(path string, want time.Duration) { Expect(fileRPCBudget(path)).To(Equal(want)) }, + Entry("ensure moves bytes", workerctl.PathFilesEnsure, 10*time.Minute), + Entry("stage moves bytes", workerctl.PathFilesStage, 10*time.Minute), + Entry("temp is metadata", workerctl.PathFilesTemp, 30*time.Second), + Entry("listdir is metadata", workerctl.PathFilesListDir, 30*time.Second), + ) + + It("gives a verb it does not know the shorter ceiling", func() { + // The safe direction: a verb wrongly given thirty seconds fails visibly + // and is retried, while one wrongly given ten minutes parks a caller on + // a verb that was never meant to be slow. + Expect(fileRPCBudget(workerctl.Prefix + "invented")).To(Equal(30 * time.Second)) + }) + DescribeTable("addresses the verb that verb's path names", func(path string, reply any, call func(*S3FileStager) error) { workers.scriptReply(controlKey(nodeID, path), reply) diff --git a/core/services/nodes/file_staging_client.go b/core/services/nodes/file_staging_client.go index f2599a3383cd..236012228271 100644 --- a/core/services/nodes/file_staging_client.go +++ b/core/services/nodes/file_staging_client.go @@ -27,7 +27,9 @@ const stagedInputReleaseTimeout = 30 * time.Second // for distributed mode. Input files are staged on the backend node before the // gRPC call. Output files are retrieved from the backend after the call. // -// Uses the FileStager interface — agnostic to transport (S3+NATS or gRPC). +// Uses the FileStager interface — agnostic to transport (an object store, or +// direct HTTP to the worker), and in both cases reached over the worker's +// tunnel. // The caller gets a grpc.Backend that behaves identically to a local one — // no changes needed in core/backend/*.go. // diff --git a/core/services/worker/control_files_test.go b/core/services/worker/control_files_test.go index 42a815c0501d..0668e2de6e7e 100644 --- a/core/services/worker/control_files_test.go +++ b/core/services/worker/control_files_test.go @@ -73,6 +73,17 @@ var _ = Describe("worker file-staging control routes", func() { DeferCleanup(srv.Close) }) + // The on-disk layout, written out BY HAND rather than derived from the + // helpers under test. What these directories ARE is an operator-facing + // contract: a deployment mounts a volume per directory and sizes it, so a + // join that moved would put staged bytes on a volume nobody provisioned. + // Deriving the expectation from the helper would pin nothing. + DescribeTable("derives its directories from the models path, once each", + func(got, want string) { Expect(got).To(Equal(want)) }, + Entry("cache", (&Config{ModelsPath: "/srv/localai/models"}).stagingCacheDir(), "/srv/localai/cache"), + Entry("data", (&Config{ModelsPath: "/srv/localai/models"}).stagingDataDir(), "/srv/localai/data"), + ) + It("allocates a temp path and returns it", func() { resp := post(workerctl.PathFilesTemp, struct{}{}) Expect(resp.StatusCode).To(Equal(http.StatusOK)) @@ -220,6 +231,25 @@ var _ = Describe("worker file-staging control routes", func() { Expect(reply.LocalPath).To(BeEmpty()) }) + It("reports a temp file it cannot create as a 200 with an error field", func() { + // The staging directory ALREADY EXISTS and is unwritable, so MkdirAll + // succeeds and CreateTemp is the branch that fails. It is a second + // statement of the same rule inside the same handler as the MkdirAll + // spec above, and a spec that only ever reaches MkdirAll leaves this + // one free to answer a non-2xx for the worker's own verdict. + Expect(os.MkdirAll(filepath.Join(cacheDir, "staging-tmp"), 0o500)).To(Succeed()) + + resp := post(workerctl.PathFilesTemp, struct{}{}) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply struct { + LocalPath string `json:"local_path"` + Error string `json:"error"` + } + decode(resp, &reply) + Expect(reply.Error).NotTo(BeEmpty()) + Expect(reply.LocalPath).To(BeEmpty()) + }) + It("refuses a key prefix that climbs out of the directories it serves", func() { resp := post(workerctl.PathFilesListDir, map[string]string{"key_prefix": "../../../etc"}) Expect(resp.StatusCode).To(Equal(http.StatusOK)) @@ -282,11 +312,87 @@ var _ = Describe("worker file-staging control routes", func() { Entry("listdir", workerctl.PathFilesListDir), ) - It("reports a request body it cannot read as a rejection, never as a file that is not there", func() { - resp, err := http.Post(srv.URL+workerctl.PathFilesEnsure, "application/json", strings.NewReader("{not json")) - Expect(err).NotTo(HaveOccurred()) - DeferCleanup(func() { _ = resp.Body.Close() }) - Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + // The other direction of the same bound, kept beside the refusals so the + // pair is local rather than spread across the suite. + // + // The two entries hold DIFFERENT halves and both are needed. The + // cap-relative body holds that the boundary is inclusive, so the bound is a + // ceiling and not an off-by-one; it moves with the cap and therefore says + // nothing about the cap's size. The absolute body holds that the cap is + // large enough for real traffic: BackendInstallRequest.BackendGalleries is + // a serialized gallery list of a few hundred kilobytes, so a cap tightened + // below a megabyte would start refusing ordinary requests, and only an + // entry written in absolute bytes notices that. + DescribeTable("serves a body that fits inside the control cap", + func(path string, size int) { + body := append([]byte(`{"key":"`), bytes.Repeat([]byte("a"), size-len(`{"key":""}`))...) + body = append(body, []byte(`"}`)...) + Expect(body).To(HaveLen(size)) + + resp, err := http.Post(srv.URL+path, "application/json", bytes.NewReader(body)) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + }, + Entry("ensure, exactly at the cap", workerctl.PathFilesEnsure, maxControlRequestBytes), + Entry("stage, exactly at the cap", workerctl.PathFilesStage, maxControlRequestBytes), + Entry("temp, exactly at the cap", workerctl.PathFilesTemp, maxControlRequestBytes), + Entry("listdir, exactly at the cap", workerctl.PathFilesListDir, maxControlRequestBytes), + Entry("ensure, a megabyte of real traffic", workerctl.PathFilesEnsure, 1<<20), + Entry("stage, a megabyte of real traffic", workerctl.PathFilesStage, 1<<20), + Entry("temp, a megabyte of real traffic", workerctl.PathFilesTemp, 1<<20), + Entry("listdir, a megabyte of real traffic", workerctl.PathFilesListDir, 1<<20), + ) + + // A body this worker could not PARSE is the frontend's request being wrong, + // not this worker's verdict about a file. The two live in opposite buckets: + // a non-2xx is mapped onto ErrWorkerUnroutable, which nothing may act on, + // while a 200 with the error field set passes through unwrapped so + // cluster.IsWorkerAnswer sees it and a reap guard MAY act on it. A handler + // that answered `{"error":"invalid request"}` for an unparseable body would + // hand a malformed request to a reap guard as evidence about a file. + // + // Pinned at every verb that decodes a body, because each one writes the + // exit out for itself. The base's NATS handlers shipped the wrong answer + // here; adopting the shared door fixed it, and this is what keeps it fixed. + DescribeTable("reports a request body it cannot read as a rejection, never as a file that is not there", + func(path string) { + resp, err := http.Post(srv.URL+path, "application/json", strings.NewReader("{not json")) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + + // The status alone is not the whole rule: a 400 whose body still + // decodes as a reply with an error field would be read correctly by + // today's client and wrongly by anything that reads the body first. + var reply struct { + Error string `json:"error"` + } + Expect(json.NewDecoder(resp.Body).Decode(&reply)).NotTo(Succeed()) + }, + Entry("ensure", workerctl.PathFilesEnsure), + Entry("stage", workerctl.PathFilesStage), + Entry("listdir", workerctl.PathFilesListDir), + // temp decodes no body, so it has no such exit to state. + ) + + It("fails a listing the caller abandoned rather than answering a short one", func() { + // A caller that gave up must not be answered with the files walked so + // far. A partial listing is the one shape the frontend cannot tell from + // a directory that really is that size, and it would read as files the + // worker does not have. So the walk returns the context error and the + // verb reports a FAILED listing. + dir := filepath.Join(modelsDir, "abandoned") + Expect(os.MkdirAll(dir, 0o750)).To(Succeed()) + for i := range 32 { + Expect(os.WriteFile(filepath.Join(dir, fmt.Sprintf("f-%02d", i)), []byte("x"), 0o600)).To(Succeed()) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + files, err := listStagedFiles(ctx, dir) + Expect(err).To(MatchError(context.Canceled)) + Expect(files).To(BeEmpty()) }) It("mounts every file verb, so none can be dropped from the set the frontend calls", func() { diff --git a/core/services/worker/control_routes_test.go b/core/services/worker/control_routes_test.go index 584e118c2362..c75e8548d99d 100644 --- a/core/services/worker/control_routes_test.go +++ b/core/services/worker/control_routes_test.go @@ -457,7 +457,7 @@ var _ = Describe("the worker's HTTP server", func() { Expect(err).NotTo(HaveOccurred()) srv, err = startWorkerHTTPServer("127.0.0.1:0", filepath.Join(dir, "staging"), dir, - filepath.Join(dir, "data"), token, &nodes.WorkerReadiness{}, sup, sup.cfg, stagingFM, nil) + filepath.Join(dir, "data"), token, &nodes.WorkerReadiness{}, sup, nil, sup.cfg, stagingFM, nil) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { nodes.ShutdownFileTransferServer(srv) }) Expect(srv.Addr).NotTo(BeEmpty(), "the worker HTTP server must report the address it bound") @@ -497,7 +497,7 @@ var _ = Describe("the worker's HTTP server", func() { // that verb" rather than as a file that is not there. dir := GinkgoT().TempDir() bare, err := startWorkerHTTPServer("127.0.0.1:0", filepath.Join(dir, "staging"), dir, - filepath.Join(dir, "data"), token, &nodes.WorkerReadiness{}, sup, sup.cfg, nil, nil) + filepath.Join(dir, "data"), token, &nodes.WorkerReadiness{}, sup, nil, sup.cfg, nil, nil) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { nodes.ShutdownFileTransferServer(bare) }) diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index 50a74ede393d..c608faf855c0 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -188,8 +188,8 @@ func Run(ctx *cliContext.Context, cfg *Config) error { // the top of Run so the worker fails before registering.) httpAddr := cfg.resolveHTTPAddr() stagingDir := filepath.Join(cfg.ModelsPath, "..", "staging") - cacheDir := filepath.Join(cfg.ModelsPath, "..", "cache") - dataDir := filepath.Join(cfg.ModelsPath, "..", "data") + cacheDir := cfg.stagingCacheDir() + dataDir := cfg.stagingDataDir() ephemeralRoots := []string{ filepath.Join(stagingDir, "ephemeral"), filepath.Join(cacheDir, "ephemeral"), diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 6b138b66b7ca..3b12b08c956f 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -182,9 +182,11 @@ carries file staging and backend logs, behind the same `LOCALAI_REGISTRATION_TOK bearer check. They replace the fourteen `nodes..*` NATS subjects a worker used to subscribe to - the ten backend and model lifecycle verbs, plus the four object-store staging verbs (`POST /v1/control/files/{ensure,stage,temp,listdir}`, -mounted only when the deployment configured an object store). The request and -reply bodies are unchanged, so nothing an operator inspects on the wire has a new -shape. Agent workers still take `nodes..backend.stop` over NATS. +mounted only when the deployment configured an object store). The request bodies +are unchanged and the reply fields keep their names and types, so nothing an +operator inspects on the wire has a new shape. The one difference is that a +worker now OMITS an empty reply field where the NATS handlers always emitted it, +which a client reading a missing field as the zero value cannot tell apart. Agent workers still take `nodes..backend.stop` over NATS. `files/listdir` is the verb the change is most visible on. Its reply used to be sized against what the bus would carry, which put a wide model directory close to diff --git a/tests/e2e/distributed/nats_jwt_test.go b/tests/e2e/distributed/nats_jwt_test.go index bf947e472ff2..40008c824eea 100644 --- a/tests/e2e/distributed/nats_jwt_test.go +++ b/tests/e2e/distributed/nats_jwt_test.go @@ -17,14 +17,32 @@ var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { infra = SetupJWTInfra() }) - It("connects with a minted backend worker JWT and publishes on allowed subjects", func() { - // Backend workers may publish under nodes..files.> (see pkg/natsauth permissions). - subject := nodeSubjectPrefix(infra.NodeID) + ".files.in" - Expect(infra.NC.Publish(subject, map[string]string{"path": "/tmp/model"})).To(Succeed()) + It("connects with a minted backend worker JWT and publishes on its one remaining allowed subject", func() { + // A backend worker's publish grant is `_INBOX.>` and nothing else now. + // Every verb a frontend gives it, file staging included, is an HTTP + // route on its tunnel, so the `nodes..files.>` grant went with the + // subjects. See pkg/natsauth.WorkerPermissions. + Expect(infra.NC.Publish("_INBOX.probe", map[string]string{"path": "/tmp/model"})).To(Succeed()) Expect(infra.NC.Conn().FlushTimeout(2 * time.Second)).To(Succeed()) + Expect(infra.NC.Conn().LastError()).ToNot(HaveOccurred()) Expect(infra.NC.Conn().IsConnected()).To(BeTrue()) }) + It("denies a backend worker the file-staging subjects it no longer serves", func() { + // This spec used to assert the OPPOSITE, and kept passing after the + // grant was deleted. A NATS permission violation does not close the + // connection, so a spec that checks only FlushTimeout and IsConnected + // cannot tell an allowed publish from a denied one; LastError is what + // actually reads the server's verdict, which is why the sibling below + // has always used it. + subject := nodeSubjectPrefix(infra.NodeID) + ".files.stage" + Expect(infra.NC.Publish(subject, map[string]string{"path": "/tmp/model"})).To(Succeed()) + Eventually(func() error { + _ = infra.NC.Conn().FlushTimeout(500 * time.Millisecond) + return infra.NC.Conn().LastError() + }, "3s", "50ms").Should(HaveOccurred()) + }) + It("allows backend subscribe on the node prefix", func() { wild := nodeSubjectPrefix(infra.NodeID) + ".>" sub, err := infra.NC.Subscribe(wild, func(_ []byte) {}) From 1b12d87f3d44d6306b86ba650dc3087f76033eef Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 2 Sep 2026 22:28:49 +0000 Subject: [PATCH 076/153] feat(distributed): read worker absence from the database, not from a bus timeout The scheduler decided whether a worker had gone away from nats.ErrNoResponders: one frontend's observation that nobody answered IT within a request budget. Two replicas asking in the same moment could disagree and demote each other's workers, and a worker re-homing its tunnel between replicas looked identical to one that had died. SmartRouter now reads cluster.Presence instead. Only PresenceGone -- no live replica holds the tunnel AND the departure has outlived the reconnect grace -- excludes a node from placement, and it is a fact every replica reads identically from the database. PresenceReconnecting, PresenceUnknown and a failed presence query are all non-verdicts and place work as normal: excluding on a database hiccup would cost the fleet its capacity for a reason that has nothing to do with any worker. nodeAnswersOnBus is deleted. It excluded on a sentinel no control RPC can produce, so it decided nothing while PingNode cost a relayed round trip per scheduling decision to feed it. PingNode goes with it, from the adapter and from NodeCommandSender. isRequestTimeout drops nats.ErrTimeout: every verb this adapter sends now travels over the worker's tunnel. The predicate is named nodeMayTakeWork rather than nodeHasRoute. "Route" is ErrWorkerUnroutable in this package, the condition nobody may act on; PresenceGone is the one a scheduler may. Spelling them the same way is the collapse this work exists to prevent. Also folds in ReapStale's return rename: it counts connection rows CLEARED, never rows deleted, and reading it as a delete count would make a worker that is re-dialling right now look forgotten. The spec pinning that a message merely quoting "nats: timeout" is not a timeout was scripting a SUCCESSFUL reply carrying the phrase, which comes back with a nil error and never reaches the classifier. Restoring the string match left it green. It now scripts a 5xx whose body carries the phrase, and asserts that the phrase reaches the classifier as a precondition. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 9 +- .../localai/nodes_backends_list_test.go | 2 - core/services/cluster/membership.go | 18 +- .../nodes/control_worker_fake_test.go | 36 ++- core/services/nodes/managers_distributed.go | 11 +- .../services/nodes/pending_op_cleanup_test.go | 5 +- core/services/nodes/reconciler.go | 15 +- core/services/nodes/registry.go | 24 +- core/services/nodes/router.go | 30 +- core/services/nodes/router_liveness.go | 92 +++--- core/services/nodes/router_liveness_test.go | 280 ++++++++++++++++++ .../nodes/router_nats_liveness_test.go | 160 ---------- core/services/nodes/router_test.go | 19 -- core/services/nodes/unloader.go | 59 +--- core/services/nodes/unloader_ping_test.go | 134 --------- core/services/nodes/unloader_test.go | 28 +- docs/content/features/distributed-mode.md | 9 +- 17 files changed, 477 insertions(+), 454 deletions(-) create mode 100644 core/services/nodes/router_liveness_test.go delete mode 100644 core/services/nodes/router_nats_liveness_test.go delete mode 100644 core/services/nodes/unloader_ping_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index 8c32d777c1c6..617196d960fe 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -543,7 +543,14 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } modelCleanup := nodes.NewModelCleanupService(registry, remoteUnloader) router := nodes.NewSmartRouter(registry, nodes.SmartRouterOptions{ - Unloader: remoteUnloader, + Unloader: remoteUnloader, + // Absence, and the only source of it the scheduler has. It is a fact + // read from the database, so every replica answers it identically; + // the bus sentinel it replaces was one frontend's observation that + // nobody answered IT within a budget, and two replicas asking at the + // same moment could disagree and demote each other's workers. + Presence: clusterRegistry, + ReconnectGrace: cfg.Distributed.ReconnectGraceOrDefault(), ModelCleanup: modelCleanup, FileStager: fileStager, GalleriesJSON: routerGalleriesJSON, diff --git a/core/http/endpoints/localai/nodes_backends_list_test.go b/core/http/endpoints/localai/nodes_backends_list_test.go index 636ab58b818e..c625e8e9510f 100644 --- a/core/http/endpoints/localai/nodes_backends_list_test.go +++ b/core/http/endpoints/localai/nodes_backends_list_test.go @@ -42,8 +42,6 @@ func (s *stubNodeCommandSender) StopBackend(_, _ string) error { return nil } func (s *stubNodeCommandSender) UnloadModelOnNode(_, _ string) error { return nil } -func (s *stubNodeCommandSender) PingNode(_ string) error { return nil } - var _ = Describe("ListBackendsOnNodeEndpoint", func() { var registry *nodes.NodeRegistry diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index b144afca2cdc..d76388160bee 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -272,16 +272,16 @@ func (m *Membership) tick(ctx context.Context) { xlog.Warn("Cluster instance heartbeat failed", "id", m.id, "error", err) } - instances, connections, err := m.reg.ReapStale(ctx, m.id, m.liveness) + instances, cleared, err := m.reg.ReapStale(ctx, m.id, m.liveness) if err != nil { xlog.Warn("Reaping stale cluster instances failed", "error", err) return } - if instances > 0 || connections > 0 { + if instances > 0 || cleared > 0 { // Two verbs because the sweep does two things: the instance rows are // gone, the connection rows are still there and now record a departure. xlog.Info("Swept cluster state left by dead replicas", - "instances_deleted", instances, "connections_departed", connections) + "instances_deleted", instances, "connections_departed", cleared) } // The retention has an owner, and it is this sweep. A departure that @@ -379,8 +379,10 @@ func (r *Registry) Deregister(ctx context.Context, id string) error { // race it, and would need its own answer to "is that replica alive", which is // the one fact this table already owns. // -// The connection rows are cleared and not deleted, and the returned count is of -// rows this sweep cleared. A worker whose owning replica died has not gone +// The connection rows are cleared and not deleted, which is why the second +// return is named for what it counts: rows this sweep CLEARED, never rows it +// removed. Reading it as a delete count would make a worker that is re-dialling +// right now look like one this deployment has forgotten. A worker whose owning replica died has not gone // anywhere: it is re-dialling the load balancer, and deleting its row would // erase the departure that says how long ago that started. // @@ -404,7 +406,7 @@ func (r *Registry) Deregister(ctx context.Context, id string) error { // on purpose, so the two paths cannot deadlock against each other. Here the // order is also forced: the connection sweep asks which instance rows survived, // so it has to run second. -func (r *Registry) ReapStale(ctx context.Context, self string, within time.Duration) (instances int64, connections int64, err error) { +func (r *Registry) ReapStale(ctx context.Context, self string, within time.Duration) (instances int64, cleared int64, err error) { err = r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { // Negated rather than spelled as its own comparison: "stale" has to be // exactly "not live", including how each treats a row whose last_seen @@ -431,11 +433,11 @@ func (r *Registry) ReapStale(ctx context.Context, self string, within time.Durat if res.Error != nil { return fmt.Errorf("recording departures for orphaned node connections: %w", res.Error) } - connections = res.RowsAffected + cleared = res.RowsAffected return nil }) if err != nil { return 0, 0, fmt.Errorf("reaping stale cluster state: %w", err) } - return instances, connections, nil + return instances, cleared, nil } diff --git a/core/services/nodes/control_worker_fake_test.go b/core/services/nodes/control_worker_fake_test.go index 013634324d2b..b76d59b02496 100644 --- a/core/services/nodes/control_worker_fake_test.go +++ b/core/services/nodes/control_worker_fake_test.go @@ -53,6 +53,9 @@ type scriptedControlWorkers struct { // that ends the call is the caller's own budget. hangs map[string]bool + // serverErrors maps a verb to the body of the 5xx it answers with. + serverErrors map[string]string + // calls records every request that reached a worker, in order. calls []requestCall } @@ -67,13 +70,14 @@ type matchedControlReply struct { func newScriptedControlWorkers() *scriptedControlWorkers { s := &scriptedControlWorkers{ - replies: map[string][]byte{}, - unsupported: map[string]bool{}, - matched: map[string][]matchedControlReply{}, - progress: map[string][]messaging.BackendInstallProgressEvent{}, - unreachable: map[string]bool{}, - expired: map[string]bool{}, - hangs: map[string]bool{}, + replies: map[string][]byte{}, + unsupported: map[string]bool{}, + matched: map[string][]matchedControlReply{}, + progress: map[string][]messaging.BackendInstallProgressEvent{}, + unreachable: map[string]bool{}, + expired: map[string]bool{}, + hangs: map[string]bool{}, + serverErrors: map[string]string{}, } mux := http.NewServeMux() mux.HandleFunc(workerctl.Prefix, s.serve) @@ -122,6 +126,7 @@ func (s *scriptedControlWorkers) serve(w http.ResponseWriter, r *http.Request) { s.mu.Lock() s.calls = append(s.calls, requestCall{Subject: key, Data: body}) unsupported := s.unsupported[key] + serverError, failing := s.serverErrors[key] hang := s.hangs[key] reply := s.replies[key] matchers := s.matched[key] @@ -138,6 +143,10 @@ func (s *scriptedControlWorkers) serve(w http.ResponseWriter, r *http.Request) { http.Error(w, "unknown worker control path "+r.URL.Path, http.StatusNotFound) return } + if failing { + http.Error(w, serverError, http.StatusInternalServerError) + return + } if len(matchers) > 0 { var req messaging.BackendInstallRequest _ = json.Unmarshal(body, &req) @@ -201,6 +210,19 @@ func (s *scriptedControlWorkers) scriptUnsupported(key string) { s.unsupported[key] = true } +// scriptServerError makes one verb answer a 5xx carrying body, which is the +// worker failing to SERVE the request rather than answering it. +// +// It exists so a spec can put chosen text inside the ERROR a control RPC +// produces. A reply whose Error field carries that text is a different thing +// entirely: it comes back with a nil error, so it never reaches the code that +// classifies failures, and a spec built on one cannot see a classifier widen. +func (s *scriptedControlWorkers) scriptServerError(key, body string) { + s.mu.Lock() + defer s.mu.Unlock() + s.serverErrors[key] = body +} + // scriptUnroutable makes every route to a node fail without reaching it. The // worker is asked nothing, so nothing is learned about it. func (s *scriptedControlWorkers) scriptUnroutable(nodeID string) { diff --git a/core/services/nodes/managers_distributed.go b/core/services/nodes/managers_distributed.go index 2db66e9d0ef1..c488d16e19a3 100644 --- a/core/services/nodes/managers_distributed.go +++ b/core/services/nodes/managers_distributed.go @@ -234,15 +234,14 @@ func (d *DistributedBackendManager) enqueueAndDrainBackendOp(ctx context.Context continue } - // A failed control RPC no longer demotes the node, and that is the - // point rather than an omission. It used to, on nats.ErrNoResponders, - // which meant "not on the bus"; the control plane's failures mean "this + // A failed control RPC does not demote the node, and that is the point + // rather than an omission. The control plane's failures mean "this // frontend could not route to it", which is equally true of a worker // that is heartbeating, serving another replica and re-homing its // tunnel. Demoting on that is the fleet-wide eviction this phase exists - // to prevent. Absence is a fact read from the database, identically on - // every replica, and the scheduler starts reading it in the task that - // replaces this signal with cluster.Presence. + // to prevent. Absence is a separate fact, read from the database + // identically on every replica; the scheduler reads it through + // cluster.Presence and nothing on this path does. if id, err := d.findPendingRow(ctx, node.ID, backend, op); err == nil { _ = d.registry.RecordPendingBackendOpFailure(ctx, id, errMsg) } diff --git a/core/services/nodes/pending_op_cleanup_test.go b/core/services/nodes/pending_op_cleanup_test.go index ad8610cc460c..af73ec0667b6 100644 --- a/core/services/nodes/pending_op_cleanup_test.go +++ b/core/services/nodes/pending_op_cleanup_test.go @@ -86,8 +86,9 @@ var _ = Describe("DeleteStalePendingBackendOps", func() { }) It("clears ops behind an unhealthy node with a stale heartbeat (never ages to offline)", func() { - // A node marked unhealthy on a NATS ErrNoResponders never transitions to - // offline, so its ops must be reaped via the same stale-heartbeat path. + // A node the scheduler demoted for a departed tunnel never transitions + // to offline, so its ops must be reaped via the same stale-heartbeat + // path. sick := registerBackend("agx-orin-sick", "10.0.0.7:50051") Expect(registry.UpsertPendingBackendOp(ctx, sick, "llama-cpp-development", OpBackendUpgrade, nil)).To(Succeed()) Expect(registry.MarkUnhealthy(ctx, sick)).To(Succeed()) diff --git a/core/services/nodes/reconciler.go b/core/services/nodes/reconciler.go index b493075ea83e..738eb35cc1c4 100644 --- a/core/services/nodes/reconciler.go +++ b/core/services/nodes/reconciler.go @@ -442,14 +442,13 @@ func (rc *ReplicaReconciler) drainPendingBackendOps(ctx context.Context) { continue } - // A failed op no longer demotes the node. It used to, on - // nats.ErrNoResponders, which said the worker was not on the bus; a - // control RPC fails when THIS frontend cannot route to the worker, - // which a worker re-homing its tunnel does while it is heartbeating and - // serving. Demoting on it would take the node out of - // ListDuePendingBackendOps and out of scheduling for a reason that has - // nothing to do with the node. The row keeps its backoff and its - // dead-letter cap, and cluster.Presence becomes the absence signal. + // A failed op does not demote the node. A control RPC fails when THIS + // frontend cannot route to the worker, which a worker re-homing its + // tunnel does while it is heartbeating and serving. Demoting on it + // would take the node out of ListDuePendingBackendOps and out of + // scheduling for a reason that has nothing to do with the node. The row + // keeps its backoff and its dead-letter cap; absence is a separate fact + // read from the database by the scheduler (cluster.Presence). // Dead-letter cap: after maxAttempts the row is the reconciler // equivalent of a poison message. Delete it loudly so the queue diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index a3ba4eff74a6..e93e184e7bc5 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -1375,13 +1375,19 @@ func (r *NodeRegistry) GetByName(ctx context.Context, name string) (*BackendNode return &node, nil } -// MarkUnhealthy sets a node status to unhealthy. Deliberately status-only: -// callers fire this on transient triggers (a single nats.ErrNoResponders from -// managers_distributed / reconciler) where the next heartbeat is expected to -// flip the node back to healthy, and cascade-deleting node_models here would -// force a full model reload on every brief NATS hiccup. Stale rows are reaped -// by the per-model health probe (on by default; see HealthMonitor) and by -// MarkOffline when the heartbeat really has gone away. +// MarkUnhealthy sets a node status to unhealthy. Deliberately status-only: it +// stops placement, it does not delete a row, and that is what keeps it inside +// what the scheduler's one demotion trigger licenses. +// +// That trigger is now a single fact rather than a class of transient failures: +// the scheduler demotes only a node cluster.Presence reports as GONE, meaning +// no live replica holds its tunnel and its departure has outlived the reconnect +// grace (see SmartRouter.nodeMayTakeWork). A failed control RPC no longer +// reaches here at all. Cascade-deleting node_models here would still be wrong +// even so: the node may be re-dialling, and reloading every model it held is +// exactly the fleet-wide eviction the tunnel work exists to prevent. Stale rows +// are reaped by the per-model health probe (on by default; see HealthMonitor) +// and by MarkOffline when the heartbeat really has gone away. func (r *NodeRegistry) MarkUnhealthy(ctx context.Context, nodeID string) error { return r.setStatus(ctx, nodeID, StatusUnhealthy) } @@ -2845,8 +2851,8 @@ func (r *NodeRegistry) DeleteStalePendingBackendOps(ctx context.Context, grace t cutoff := time.Now().Add(-grace) // Draining nodes are cleared immediately (admin action; model rows already // purged). Offline AND unhealthy nodes are cleared only once their heartbeat - // is older than the grace window: a node marked unhealthy on a NATS - // ErrNoResponders never transitions to offline (health.go skips re-marking + // is older than the grace window: a node the scheduler demoted for a + // departed tunnel never transitions to offline (health.go skips re-marking // it), so without including unhealthy here its ops would leak exactly like // the offline case. A node with a fresh heartbeat (last_heartbeat > cutoff) // is recovering and keeps its op for retry. diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index 8cdd96782eba..68cfa00c4cb7 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -40,6 +40,19 @@ var companionSuffixes = map[string][]string{ // Passing them at construction time eliminates data races from post-creation setters. type SmartRouterOptions struct { Unloader NodeCommandSender + // Presence answers whether a worker's tunnel is held, lost inside the + // reconnect grace, lost past it, or unknown. It is the ONLY thing the + // scheduler reads absence from; nil disables the check entirely and every + // node is treated as present, which is what a deployment with no cluster + // registry gets. See nodeMayTakeWork. + Presence NodePresenceReader + // ReconnectGrace is how long a lost tunnel is read as reconnecting rather + // than gone. It is the operator's trade rather than this package's, so + // callers pass config.DistributedConfig.ReconnectGraceOrDefault(). Zero + // selects the same default: a caller that wires Presence and forgets the + // grace gets the documented window instead of a zero one, which would + // condemn every worker the instant its tunnel dropped. + ReconnectGrace time.Duration // ModelCleanup performs acknowledged exact-process cleanup when a load // finishes after its configuration revision became stale. ModelCleanup *ModelCleanupService @@ -158,8 +171,12 @@ func ModelLoadCeilingFor(installTimeout, loadTimeout time.Duration) time.Duratio // SmartRouter routes inference requests to the best available backend node. // It uses the ModelRouter interface (backed by NodeRegistry in production) for routing decisions. type SmartRouter struct { - registry ModelRouter - unloader NodeCommandSender // optional, for NATS-driven load/unload + registry ModelRouter + unloader NodeCommandSender // optional, for control-plane load/unload + // presence is the scheduler's only source of absence, and reconnectGrace + // the window it measures a departure against. See nodeMayTakeWork. + presence NodePresenceReader + reconnectGrace time.Duration modelCleanup *ModelCleanupService fileStager FileStager // optional, for distributed file transfer galleriesJSON string // backend gallery config for dynamic installation @@ -243,9 +260,18 @@ func NewSmartRouter(registry ModelRouter, opts SmartRouterOptions) *SmartRouter if diskHeadroom == nil { diskHeadroom = func() bool { return true } } + // Zero means the caller wired a presence reader without an explicit grace. + // Defaulted here rather than left at zero, because a zero window makes + // every departure instantly a verdict. + grace := opts.ReconnectGrace + if grace <= 0 { + grace = config.DefaultWorkerReconnectGrace + } return &SmartRouter{ registry: registry, unloader: opts.Unloader, + presence: opts.Presence, + reconnectGrace: grace, modelCleanup: opts.ModelCleanup, fileStager: opts.FileStager, galleriesJSON: opts.GalleriesJSON, diff --git a/core/services/nodes/router_liveness.go b/core/services/nodes/router_liveness.go index 38c078384496..06c684623159 100644 --- a/core/services/nodes/router_liveness.go +++ b/core/services/nodes/router_liveness.go @@ -2,70 +2,92 @@ package nodes import ( "context" - "errors" + "time" + "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/xlog" - "github.com/nats-io/nats.go" ) -// maxNodeLivenessRetries bounds how many unreachable nodes a single scheduling +// maxNodeLivenessRetries bounds how many excluded nodes a single scheduling // attempt discards before giving up. Each discarded node is marked unhealthy, // so the bound only has to cover one burst of dead workers rather than the // whole fleet. const maxNodeLivenessRetries = 3 -// nodeAnswersOnBus reports whether a node may still be given work. +// NodePresenceReader answers what this deployment can say about one worker's +// tunnel, given the window a departure has to outlive before it counts. // -// It probes the node's CONTROL PLANE, over that node's tunnel, and nothing it -// can learn there is proof of absence. A worker that answers is present by -// demonstration; every other outcome is this frontend failing to route, which -// is equally what a healthy worker re-homing its tunnel between frontend -// replicas produces, so excluding on it would demote a node that is -// heartbeating and serving. Only nats.ErrNoResponders excludes, and no control -// RPC produces it: the exclusion is therefore inert here, and stays only until -// the scheduler reads cluster.Presence, which is the one fact in the deployment -// that can say a worker is gone and says it identically on every replica. +// A narrow port rather than the whole registry, because absence is the only +// thing the scheduler has any business reading from it, and a wider dependency +// here would be one every scheduling spec had to build. +// (*cluster.Registry).Presence has exactly this shape. +type NodePresenceReader interface { + Presence(ctx context.Context, nodeID string, grace time.Duration) (cluster.Presence, error) +} + +// nodeMayTakeWork reports whether the scheduler may place work on a node. +// +// Only cluster.PresenceGone excludes, and it is a fact every replica reads +// identically from the database rather than one this replica inferred from a +// timeout. That is the whole change: absence used to be nats.ErrNoResponders, +// which is one frontend's observation that nobody answered it within a budget, +// and two replicas asking at the same moment could disagree. +// +// The other three values are not absence and none of them may exclude. +// PresenceReconnecting is a worker re-homing its tunnel between replicas, or +// one whose owning replica just died holding it: excluding it costs capacity, +// and the demotion that follows is what turns a rolling frontend restart into a +// fleet-wide eviction. PresenceUnknown is a worker that has never dialled or +// whose departure aged out of retention, and the registry cannot say which, so +// the scheduler places work and the install that follows reports its own +// failure. A query that FAILS is not an answer at all: excluding on an +// infrastructure failure would evict capacity for a reason that has nothing to +// do with the worker, and a database hiccup would take out the fleet. // -// Before that carrier change this asked two NATS subjects the worker no longer -// subscribes to, so EVERY healthy worker answered "no responders" and was -// marked unhealthy on the scheduling path. +// It is deliberately NOT named for a route. A route to a worker is +// ErrWorkerUnroutable, a different condition that nobody may act on; this reads +// the one fact in the deployment that can say a worker is gone. // -// When no command sender is configured there is nothing to consult and every -// node is treated as reachable, which preserves the behaviour of deployments -// that do not run one. -func (r *SmartRouter) nodeAnswersOnBus(node *BackendNode) bool { - if r.unloader == nil || node == nil { +// When no presence reader is configured there is nothing to consult and every +// node may take work, which preserves the behaviour of deployments that run no +// cluster registry. +func (r *SmartRouter) nodeMayTakeWork(ctx context.Context, node *BackendNode) bool { + if r.presence == nil || node == nil { return true } - err := r.unloader.PingNode(node.ID) - return !errors.Is(err, nats.ErrNoResponders) + p, err := r.presence.Presence(ctx, node.ID, r.reconnectGrace) + if err != nil { + xlog.Warn("Could not read node presence; scheduling as if the node were present", + "node", node.Name, "nodeID", node.ID, "error", err) + return true + } + return p != cluster.PresenceGone } -// pickReachableNode calls selectNode until it yields a node that nodeAnswersOnBus +// pickReachableNode calls selectNode until it yields a node nodeMayTakeWork // does not exclude, and returns nil when it cannot find one. // -// No control-RPC outcome excludes, so today this returns the first node the -// selector offers. The loop stays until the scheduler reads cluster.Presence. -// -// A node that does not answer is marked unhealthy before the next attempt. That -// both removes it from the next selection, which queries only healthy nodes, -// and tells every other scheduler in the cluster what this one just learned, so -// the discovery is not repeated one failed request at a time. +// An excluded node is marked unhealthy before the next attempt. That both +// removes it from the next selection, which queries only healthy nodes, and +// tells every other scheduler in the cluster what this one just learned, so the +// discovery is not repeated one failed request at a time. The demotion is +// status-only (see NodeRegistry.MarkUnhealthy): it stops placement, it does not +// delete a row, so it stays inside what a routing fact licenses. func (r *SmartRouter) pickReachableNode(ctx context.Context, selectNode func() *BackendNode) *BackendNode { for range maxNodeLivenessRetries { node := selectNode() if node == nil { return nil } - if r.nodeAnswersOnBus(node) { + if r.nodeMayTakeWork(ctx, node) { return node } - xlog.Warn("Scheduled node was excluded by the liveness probe, marking unhealthy and re-scheduling", - "node", node.Name, "nodeID", node.ID) + xlog.Warn("Scheduled node has no tunnel and its departure outlived the reconnect grace, marking unhealthy and re-scheduling", + "node", node.Name, "nodeID", node.ID, "grace", r.reconnectGrace) if err := r.registry.MarkUnhealthy(ctx, node.ID); err != nil { // Without the demotion the next selection would hand back the same // node, so stop rather than spin. - xlog.Warn("Failed to mark unreachable node unhealthy", + xlog.Warn("Failed to mark departed node unhealthy", "node", node.Name, "nodeID", node.ID, "error", err) return nil } diff --git a/core/services/nodes/router_liveness_test.go b/core/services/nodes/router_liveness_test.go new file mode 100644 index 000000000000..d05ddc061e39 --- /dev/null +++ b/core/services/nodes/router_liveness_test.go @@ -0,0 +1,280 @@ +package nodes + +import ( + "context" + "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// stubPresence answers the scheduler's presence question without a database. +// +// It exists to NAME each of the four values, so a widening of the exclusion is +// attributed to the value that caused it. It cannot see a carrier change, which +// is why the real-registry specs at the bottom of this file exist beside it: +// the previous version of this suite drove a test-owned map of "dead nodes" and +// stayed green for the whole window in which every healthy worker read as +// absent. +type stubPresence struct { + answer cluster.Presence + answerFor map[string]cluster.Presence + err error + + nodes []string + graces []time.Duration +} + +func (s *stubPresence) Presence(_ context.Context, nodeID string, grace time.Duration) (cluster.Presence, error) { + s.nodes = append(s.nodes, nodeID) + s.graces = append(s.graces, grace) + if s.err != nil { + // PresenceUnknown alongside the error, the way the real registry + // answers: a value nobody may act on rather than one that reads as a + // verdict. + return cluster.PresenceUnknown, s.err + } + if p, ok := s.answerFor[nodeID]; ok { + return p, nil + } + return s.answer, nil +} + +// selectorReturning hands back each node in turn, mimicking a scheduler that +// re-picks after the previous choice was demoted. +func selectorReturning(nodes ...*BackendNode) func() *BackendNode { + i := 0 + return func() *BackendNode { + if i >= len(nodes) { + return nil + } + n := nodes[i] + i++ + return n + } +} + +var _ = Describe("Scheduling and node presence", func() { + var ( + reg *fakeModelRouter + presence *stubPresence + router *SmartRouter + ctx context.Context + ) + + // The grace every spec measures against. It is the operator's trade, so it + // is also what the scheduler must hand the registry: a scheduler that asked + // with a constant of its own would ignore --worker-reconnect-grace entirely + // while every spec below that only checks a Presence VALUE stayed green. + const grace = 60 * time.Second + + newNode := func(id string) *BackendNode { + return &BackendNode{ID: id, Name: id, Address: id + ":50051"} + } + + BeforeEach(func() { + ctx = context.Background() + reg = &fakeModelRouter{} + presence = &stubPresence{answer: cluster.PresenceConnected} + router = NewSmartRouter(reg, SmartRouterOptions{Presence: presence, ReconnectGrace: grace}) + }) + + DescribeTable("decides whether a node may be given work", + func(p cluster.Presence, eligible bool) { + presence.answer = p + Expect(router.nodeMayTakeWork(ctx, newNode("node-1"))).To(Equal(eligible)) + }, + Entry("connected: eligible", cluster.PresenceConnected, true), + // The catastrophe case. A worker re-homing between replicas, or a + // replica that just died holding it, must NOT be excluded: excluding it + // costs capacity, and marking it unhealthy is what turns a rolling + // frontend restart into a fleet-wide eviction. + Entry("reconnecting: still eligible", cluster.PresenceReconnecting, true), + // A node with no connection row has never dialled or its departure aged + // out. Presence cannot say which, so scheduling does not exclude it and + // the install that follows reports its own failure. + Entry("unknown: still eligible", cluster.PresenceUnknown, true), + Entry("gone: not eligible", cluster.PresenceGone, false), + ) + + It("asks with the operator's configured grace rather than a constant of its own", func() { + router.nodeMayTakeWork(ctx, newNode("node-1")) + + Expect(presence.graces).To(Equal([]time.Duration{grace})) + }) + + It("does not mark a reconnecting node unhealthy", func() { + node := newNode("re-homing") + presence.answer = cluster.PresenceReconnecting + + Expect(router.pickReachableNode(ctx, selectorReturning(node))).To(Equal(node)) + Expect(reg.markedUnhealthy).To(BeEmpty()) + }) + + It("marks a gone node unhealthy exactly once and reschedules", func() { + gone, live := newNode("gone-node"), newNode("live-node") + presence.answerFor = map[string]cluster.Presence{ + gone.ID: cluster.PresenceGone, + live.ID: cluster.PresenceConnected, + } + + picked := router.pickReachableNode(ctx, selectorReturning(gone, live)) + + Expect(picked).To(Equal(live)) + Expect(reg.markedUnhealthy).To(Equal([]string{gone.ID})) + Expect(presence.nodes).To(Equal([]string{gone.ID, live.ID})) + }) + + It("treats a presence query FAILURE as eligible, never as absence", func() { + // A database hiccup must not evict the fleet. The error carries + // PresenceUnknown, so a read that acted on the value while ignoring the + // error would still be eligible here; what separates the two is that + // nothing may be concluded, which the demotion assertion covers. + node := newNode("only-node") + presence.err = errors.New("connection reset") + + Expect(router.nodeMayTakeWork(ctx, node)).To(BeTrue()) + Expect(router.pickReachableNode(ctx, selectorReturning(node))).To(Equal(node)) + Expect(reg.markedUnhealthy).To(BeEmpty()) + }) + + It("treats every node as eligible when no presence reader is configured", func() { + // A single-node deployment has no cluster registry to ask, and nothing + // there has a tunnel to lose. + plain := NewSmartRouter(reg, SmartRouterOptions{}) + node := newNode("only-node") + + Expect(plain.pickReachableNode(ctx, selectorReturning(node))).To(Equal(node)) + Expect(reg.markedUnhealthy).To(BeEmpty()) + }) + + It("takes the first eligible node without asking about the rest", func() { + first, second := newNode("first"), newNode("second") + + Expect(router.pickReachableNode(ctx, selectorReturning(first, second))).To(Equal(first)) + Expect(presence.nodes).To(Equal([]string{"first"})) + }) + + It("gives up rather than spinning when every node is gone", func() { + presence.answer = cluster.PresenceGone + a, b, c, d := newNode("a"), newNode("b"), newNode("c"), newNode("d") + + picked := router.pickReachableNode(ctx, selectorReturning(a, b, c, d)) + + Expect(picked).To(BeNil()) + Expect(len(presence.nodes)).To(BeNumerically("<=", maxNodeLivenessRetries)) + }) + + It("stops when the demotion itself fails, so it cannot loop on one node", func() { + presence.answer = cluster.PresenceGone + dead := newNode("dead-node") + reg.markUnhealthyErr = errors.New("database is down") + + picked := router.pickReachableNode(ctx, selectorReturning(dead, dead, dead)) + + Expect(picked).To(BeNil()) + Expect(presence.nodes).To(Equal([]string{"dead-node"})) + }) +}) + +// The same decision, against the registry that answers it in production, on the +// database clock. +// +// The stub above can only prove that the scheduler acts correctly on a value it +// was handed. These prove that the value it is handed is the one the deployment +// holds: that Presence is actually consulted, that the operator's grace reaches +// it, and that a departure inside the grace and a departure past it are told +// apart by the DATABASE rather than by this process. +var _ = Describe("Scheduling against the cluster registry that answers presence", func() { + var ( + ctx context.Context + db *gorm.DB + clusterR *cluster.Registry + reg *fakeModelRouter + router *SmartRouter + ) + + const ( + grace = 60 * time.Second + instance = "inst-a" + ) + + // ageDeparture pushes a worker's departure into the past ON THE DATABASE + // CLOCK. A Go-side time.Now().Add(-d) would age the row against this + // process's clock and then compare it against the database's, which is the + // skew the whole window is written to be immune to. + ageDeparture := func(nodeID string, by time.Duration) { + GinkgoHelper() + Expect(db.WithContext(ctx).Exec( + `UPDATE node_connections SET disconnected_at = now() - make_interval(secs => ?) WHERE node_id = ?`, + by.Seconds(), nodeID).Error).To(Succeed()) + } + + BeforeEach(func() { + ctx = context.Background() + db = testutil.SetupTestDB() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + clusterR = cluster.NewRegistry(db) + Expect(clusterR.Register(ctx, instance, "10.0.0.1:8080", "v1")).To(Succeed()) + reg = &fakeModelRouter{} + router = NewSmartRouter(reg, SmartRouterOptions{Presence: clusterR, ReconnectGrace: grace}) + }) + + node := func(id string) *BackendNode { + return &BackendNode{ID: id, Name: id, Address: id + ":50051"} + } + + It("schedules onto a worker whose tunnel a live replica holds", func() { + _, err := clusterR.Claim(ctx, "worker-connected", instance) + Expect(err).ToNot(HaveOccurred()) + + n := node("worker-connected") + Expect(router.pickReachableNode(ctx, selectorReturning(n))).To(Equal(n)) + Expect(reg.markedUnhealthy).To(BeEmpty()) + }) + + It("schedules onto a worker whose tunnel was lost inside the grace, without demoting it", func() { + // The rolling-restart case in its real form: the row records a + // departure, and the worker is at this moment re-dialling the load + // balancer. + epoch, err := clusterR.Claim(ctx, "worker-rehoming", instance) + Expect(err).ToNot(HaveOccurred()) + Expect(clusterR.Release(ctx, "worker-rehoming", instance, epoch)).To(Succeed()) + ageDeparture("worker-rehoming", grace/2) + + n := node("worker-rehoming") + Expect(router.pickReachableNode(ctx, selectorReturning(n))).To(Equal(n)) + Expect(reg.markedUnhealthy).To(BeEmpty()) + }) + + It("excludes and demotes a worker whose departure has outlived the grace", func() { + epoch, err := clusterR.Claim(ctx, "worker-gone", instance) + Expect(err).ToNot(HaveOccurred()) + Expect(clusterR.Release(ctx, "worker-gone", instance, epoch)).To(Succeed()) + // Just past the edge, not an order of magnitude past it: a window only + // ever aged ten times its own width is a window nothing pins. + ageDeparture("worker-gone", grace+5*time.Second) + + _, err = clusterR.Claim(ctx, "worker-live", instance) + Expect(err).ToNot(HaveOccurred()) + gone, live := node("worker-gone"), node("worker-live") + + Expect(router.pickReachableNode(ctx, selectorReturning(gone, live))).To(Equal(live)) + Expect(reg.markedUnhealthy).To(Equal([]string{"worker-gone"})) + }) + + It("schedules onto a worker that has never dialled a tunnel", func() { + // No connection row at all. The registry cannot tell a worker still + // starting up from one whose departure aged out of retention, so it + // answers unknown and the scheduler places work; the install that + // follows reports its own failure if the worker really is not there. + n := node("worker-never-seen") + Expect(router.pickReachableNode(ctx, selectorReturning(n))).To(Equal(n)) + Expect(reg.markedUnhealthy).To(BeEmpty()) + }) +}) diff --git a/core/services/nodes/router_nats_liveness_test.go b/core/services/nodes/router_nats_liveness_test.go deleted file mode 100644 index d35d5a088a87..000000000000 --- a/core/services/nodes/router_nats_liveness_test.go +++ /dev/null @@ -1,160 +0,0 @@ -package nodes - -import ( - "context" - "errors" - "fmt" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/mudler/LocalAI/core/services/cluster" -) - -// pickReachableNode's exclusion branch, which no longer runs in production. -// -// It excludes on nats.ErrNoResponders alone, and since the control plane moved -// onto the tunnel nothing can produce that value: cluster, the package that -// supplies every control-path dial error, does not link nats.go at all. So the -// specs below that drive fakeUnloader.deadNodes exercise a branch a real -// adapter cannot enter. They are kept, rather than deleted, because the LOOP is -// still compiled and still executed on every scheduling decision, and its shape -// (the retry bound, and stopping when the demotion itself fails) is what stops a -// future re-wiring from spinning. They go with the exclusion in the task that -// gives the scheduler cluster.Presence. -// -// What is load-bearing TODAY is the opposite assertion, and it lives in two -// places: the DescribeTable at the bottom of this file, which pins that none of -// the sentinels a control RPC actually produces excludes anything, and the -// scheduling specs in unloader_ping_test.go, which prove the same thing through -// a REAL RemoteUnloaderAdapter. This file's doubles cannot see a carrier -// change; that is why they stayed green through the window in which every -// healthy worker read as absent. -var _ = Describe("The retired bus-absence exclusion", func() { - var ( - reg *fakeModelRouter - fake *fakeUnloader - router *SmartRouter - ) - - newNode := func(id string) *BackendNode { - return &BackendNode{ID: id, Name: id, Address: id + ":50051"} - } - - // selectorReturning hands back each node in turn, mimicking a scheduler - // that re-picks after the previous choice was demoted. - selectorReturning := func(nodes ...*BackendNode) func() *BackendNode { - i := 0 - return func() *BackendNode { - if i >= len(nodes) { - return nil - } - n := nodes[i] - i++ - return n - } - } - - BeforeEach(func() { - reg = &fakeModelRouter{} - fake = &fakeUnloader{deadNodes: map[string]bool{}} - router = NewSmartRouter(reg, SmartRouterOptions{Unloader: fake}) - }) - - It("passes over a node that no longer answers and takes one that does", func() { - dead, alive := newNode("dead-node"), newNode("alive-node") - fake.deadNodes["dead-node"] = true - - picked := router.pickReachableNode(context.Background(), selectorReturning(dead, alive)) - - Expect(picked).ToNot(BeNil()) - Expect(picked.ID).To(Equal("alive-node")) - Expect(fake.pingCalls).To(Equal([]string{"dead-node", "alive-node"})) - }) - - It("demotes the absent node so other schedulers stop choosing it", func() { - dead, alive := newNode("dead-node"), newNode("alive-node") - fake.deadNodes["dead-node"] = true - - router.pickReachableNode(context.Background(), selectorReturning(dead, alive)) - - Expect(reg.markedUnhealthy).To(Equal([]string{"dead-node"})) - }) - - It("takes the first node when it answers, without probing further", func() { - first, second := newNode("first"), newNode("second") - - picked := router.pickReachableNode(context.Background(), selectorReturning(first, second)) - - Expect(picked.ID).To(Equal("first")) - Expect(fake.pingCalls).To(Equal([]string{"first"})) - }) - - It("gives up rather than spinning when every node is gone", func() { - a, b, c, d := newNode("a"), newNode("b"), newNode("c"), newNode("d") - for _, id := range []string{"a", "b", "c", "d"} { - fake.deadNodes[id] = true - } - - picked := router.pickReachableNode(context.Background(), selectorReturning(a, b, c, d)) - - Expect(picked).To(BeNil()) - Expect(len(fake.pingCalls)).To(BeNumerically("<=", maxNodeLivenessRetries)) - }) - - It("stops when the demotion itself fails, so it cannot loop on one node", func() { - dead := newNode("dead-node") - fake.deadNodes["dead-node"] = true - reg.markUnhealthyErr = errors.New("database is down") - - picked := router.pickReachableNode(context.Background(), selectorReturning(dead, dead, dead)) - - Expect(picked).To(BeNil()) - Expect(fake.pingCalls).To(Equal([]string{"dead-node"})) - }) - - // The half of the branch that IS still reachable: every error a control - // RPC can produce lands here. - It("keeps a node that answers slowly or errors for another reason", func() { - slow := newNode("slow-node") - fake.pingErr = errors.New("timeout waiting for reply") - - picked := router.pickReachableNode(context.Background(), selectorReturning(slow)) - - Expect(picked).ToNot(BeNil()) - Expect(picked.ID).To(Equal("slow-node")) - Expect(reg.markedUnhealthy).To(BeEmpty()) - }) - - It("treats every node as reachable when no command sender is configured", func() { - plain := NewSmartRouter(reg, SmartRouterOptions{}) - node := newNode("only-node") - - Expect(plain.pickReachableNode(context.Background(), selectorReturning(node))).To(Equal(node)) - }) - - // The rule that actually holds now, stated in the vocabulary a control RPC - // answers in. Widening the exclusion to any of these puts the scheduler - // back in the state this task was opened to fix: a worker that is - // heartbeating and serving, unroutable from THIS replica for a moment - // while its tunnel re-homes, demoted for every scheduler in the cluster. - // - // unloader_ping_test.go proves the same thing through a real adapter and a - // real transport. This table is the cheap negative control beside it: it - // names each sentinel, so a widening is attributed to the sentinel that - // caused it rather than to "something in the ping". - DescribeTable("never excludes a node on anything a control RPC can answer with", - func(cause error) { - fake.pingErr = cause - node := newNode("live-node") - - Expect(router.pickReachableNode(context.Background(), selectorReturning(node))).To(Equal(node)) - Expect(reg.markedUnhealthy).To(BeEmpty()) - }, - Entry("no route to the worker", fmt.Errorf("control rpc: %w: %w", ErrWorkerUnroutable, cluster.ErrNoRoute)), - Entry("no tunnel dialer wired", ErrNoWorkerDialer), - Entry("the worker is older than this frontend", ErrWorkerControlUnsupported), - Entry("the caller's budget ran out", fmt.Errorf("control rpc: %w: %w", ErrWorkerUnroutable, context.DeadlineExceeded)), - Entry("the worker's own refusal", fmt.Errorf("%w: no such process", cluster.ErrStreamTargetUnavailable)), - ) -}) diff --git a/core/services/nodes/router_test.go b/core/services/nodes/router_test.go index 43b4353c12ed..80bf628c0c07 100644 --- a/core/services/nodes/router_test.go +++ b/core/services/nodes/router_test.go @@ -18,7 +18,6 @@ import ( "github.com/mudler/LocalAI/pkg/distributedhdr" grpc "github.com/mudler/LocalAI/pkg/grpc" pb "github.com/mudler/LocalAI/pkg/grpc/proto" - "github.com/nats-io/nats.go" ggrpc "google.golang.org/grpc" "google.golang.org/protobuf/proto" "gorm.io/gorm" @@ -510,13 +509,6 @@ type fakeUnloader struct { stopErr error unloadCalls []string - // deadNodes names the nodes PingNode reports as absent from the bus, and - // pingCalls records every node it was asked about, in order. - deadNodes map[string]bool - pingCalls []string - // pingErr is returned for nodes not in deadNodes, so a spec can model a - // node that is reachable but answering badly. - pingErr error unloadErr error } @@ -580,17 +572,6 @@ func (f *fakeModelRouter) MarkUnhealthy(_ context.Context, nodeID string) error return f.markUnhealthyErr } -func (f *fakeUnloader) PingNode(nodeID string) error { - f.mu.Lock() - f.pingCalls = append(f.pingCalls, nodeID) - dead := f.deadNodes[nodeID] - f.mu.Unlock() - if dead { - return nats.ErrNoResponders - } - return f.pingErr -} - // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index 8240af22dfde..4048b5a8cac2 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -6,8 +6,6 @@ import ( "fmt" "time" - "github.com/nats-io/nats.go" - "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/workerctl" @@ -36,12 +34,6 @@ type NodeCommandSender interface { ListBackends(nodeID string) (*messaging.BackendListReply, error) StopBackend(nodeID, backend string) error UnloadModelOnNode(nodeID, modelName string) error - // PingNode reports whether the worker answered a control RPC. Only an - // answer is evidence; every other outcome is this frontend failing to - // route, which callers must not read as the node having gone. Task 6 - // replaces it with a presence read on the database clock, which is the - // only thing in this deployment that CAN say a worker is gone. - PingNode(nodeID string) error } // RemoteUnloaderAdapter implements NodeCommandSender and @@ -329,7 +321,6 @@ const ( modelUnloadTimeout = 30 * time.Second modelDeleteTimeout = 30 * time.Second nodeStopTimeout = 30 * time.Second - nodePingTimeout = 5 * time.Second ) // ListBackends queries a worker node for its installed backends. @@ -346,41 +337,6 @@ func (a *RemoteUnloaderAdapter) ListBackends(nodeID string) (*messaging.BackendL return &reply, nil } -// PingNode asks a worker one cheap control verb and reports what came back. -// -// A node's status in the database comes from its HTTP heartbeat, which is a -// separate channel, so a worker that has died keeps its healthy status until -// the heartbeat ages out and the scheduler could commit to a node it cannot -// reach. -// -// What this can and cannot conclude changed with the carrier, and the -// difference is the whole point. A worker that ANSWERS is present by -// demonstration, whatever it answered. Every other outcome is this frontend -// failing to route to it, which is true of a worker that is heartbeating and -// serving another replica's requests while re-homing its tunnel; reported as -// absence it would have the scheduler demote a healthy node. So the error is -// ErrWorkerUnroutable, callers must not read it as the node having gone, and -// nothing here returns an absence sentinel at all. Task 6 replaces this with -// cluster.Registry.Presence, which is the one fact in the deployment that can -// say a worker is gone, and says it identically on every replica. -// -// backend.list rather than models.running for the reason it always was: it has -// been part of the worker protocol far longer, so it is the safer question to -// ask of a worker that may be older than this frontend. -func (a *RemoteUnloaderAdapter) PingNode(nodeID string) error { - ctx, cancel := context.WithTimeout(context.Background(), nodePingTimeout) - defer cancel() - - var reply messaging.BackendListReply - err := a.control.Call(ctx, nodeID, workerctl.PathBackendList, messaging.BackendListRequest{}, &reply) - if errors.Is(err, ErrWorkerControlUnsupported) { - // The worker answered, from a build that predates this verb. That is a - // worker that is very much there. - return nil - } - return err -} - // ListRunningModels asks a worker node which model backend processes it // currently has running. // @@ -577,11 +533,14 @@ func (a *RemoteUnloaderAdapter) StopNode(nodeID string) error { // isRequestTimeout reports whether a control RPC ended because its budget ran // out rather than because the worker said anything. // -// context.DeadlineExceeded is the one signal, and it is matchable because -// controlFailure wraps the caller's own ctx.Err(). nats.ErrTimeout is still -// accepted for as long as any verb reaches a worker over the bus. The string -// match the NATS carrier needed is deliberately NOT reproduced: a message that -// merely quotes a timeout is not one. +// context.DeadlineExceeded is the ONE signal, and it is matchable because +// controlFailure wraps the caller's own ctx.Err(). The bus sentinel it used to +// accept alongside is gone with the bus: every verb this adapter sends now +// travels over the worker's tunnel, so a nats.ErrTimeout could only arrive from +// a carrier nothing here uses. The string match that carrier needed is +// deliberately not reproduced either, in any spelling: a message that merely +// quotes a timeout is not one, and a worker error that happened to contain the +// phrase would be reported as still-installing forever. func isRequestTimeout(err error) bool { - return errors.Is(err, nats.ErrTimeout) || errors.Is(err, context.DeadlineExceeded) + return errors.Is(err, context.DeadlineExceeded) } diff --git a/core/services/nodes/unloader_ping_test.go b/core/services/nodes/unloader_ping_test.go deleted file mode 100644 index ae2f594a060f..000000000000 --- a/core/services/nodes/unloader_ping_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package nodes - -import ( - "context" - "errors" - "time" - - "github.com/nats-io/nats.go" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/mudler/LocalAI/core/services/cluster" - "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/core/services/workerctl" -) - -// The scheduler asks a worker one cheap question before committing work to it, -// and then acts on the answer by DEMOTING the node. That is only sound if the -// question is one every worker in the fleet answers. -// -// It has been the wrong question twice. It first asked models.running, which -// arrived in 4.6, so a 4.5 worker that was alive and serving never answered and -// a model pinned to it could never be placed. It then asked two NATS subjects -// the worker had stopped subscribing to when its control plane moved onto the -// tunnel, so EVERY healthy worker answered "no responders" and was marked -// unhealthy on the scheduling path. -// -// These specs pin both halves of the answer: what counts as the worker -// speaking, and what must never be read as the worker being gone. -var _ = Describe("Node liveness probe over the control plane", func() { - var ( - workers *scriptedControlWorkers - adapter *RemoteUnloaderAdapter - ) - - const nodeID = "11111111-2222-3333-4444-555555555555" - - BeforeEach(func() { - workers = newScriptedControlWorkers() - adapter = NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) - }) - - It("treats a worker that answers backend.list as alive", func() { - workers.scriptReply(controlKey(nodeID, workerctl.PathBackendList), messaging.BackendListReply{}) - - Expect(adapter.PingNode(nodeID)).To(Succeed()) - }) - - It("treats a worker too old to serve the verb as alive, since it answered", func() { - // A 404 under the control prefix is the worker saying it is older than - // this frontend. It is a deployment fact, and a worker that can state - // one is a worker that is there. - workers.scriptUnsupported(controlKey(nodeID, workerctl.PathBackendList)) - - Expect(adapter.PingNode(nodeID)).To(Succeed()) - }) - - It("reports a worker it cannot route to as unroutable, and never as absent", func() { - workers.scriptUnroutable(nodeID) - - err := adapter.PingNode(nodeID) - Expect(err).To(HaveOccurred()) - Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) - // The two things a caller must not be able to conclude from it: that - // the worker spoke, and that it is gone. - Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) - Expect(errors.Is(err, nats.ErrNoResponders)).To(BeFalse()) - }) - - It("refuses without a tunnel dialer rather than reaching for an address", func() { - plain := NewRemoteUnloaderAdapter(nil, nil, NewControlClient(nil, "tok"), time.Minute, time.Minute) - - err := plain.PingNode(nodeID) - Expect(err).To(MatchError(ErrNoWorkerDialer)) - Expect(err).To(MatchError(ErrWorkerUnroutable)) - }) -}) - -// The merge gate this task exists to clear, asserted through the SCHEDULING -// path with the real adapter rather than through the router's own double. -// -// router_liveness_test.go drives a test-owned map of dead nodes and never -// touches a transport, so it stayed green for the whole window in which every -// healthy worker read as absent. These specs run pickReachableNode against a -// real RemoteUnloaderAdapter and a worker that answers over its control plane, -// which is the only arrangement that can see the difference. -var _ = Describe("Scheduling onto a worker reached over its control plane", func() { - var ( - workers *scriptedControlWorkers - reg *fakeModelRouter - router *SmartRouter - ) - - const nodeID = "22222222-3333-4444-5555-666666666666" - - BeforeEach(func() { - workers = newScriptedControlWorkers() - reg = &fakeModelRouter{} - router = NewSmartRouter(reg, SmartRouterOptions{ - Unloader: NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), time.Minute, time.Minute), - }) - }) - - selectOnce := func(node *BackendNode) func() *BackendNode { - handed := false - return func() *BackendNode { - if handed { - return nil - } - handed = true - return node - } - } - - It("schedules onto a healthy worker instead of demoting it", func() { - node := &BackendNode{ID: nodeID, Name: "healthy", Address: "10.0.0.1:50051"} - workers.scriptReply(controlKey(nodeID, workerctl.PathBackendList), messaging.BackendListReply{}) - - Expect(router.pickReachableNode(context.Background(), selectOnce(node))).To(Equal(node)) - Expect(reg.markedUnhealthy).To(BeEmpty()) - }) - - It("schedules onto a worker this frontend could not route to, rather than demoting it", func() { - // A worker re-homing its tunnel between frontend replicas, or one whose - // owning replica is restarting, is unroutable from here while it is - // heartbeating and serving. Excluding it costs capacity; demoting it - // tells every other scheduler the same wrong thing. - node := &BackendNode{ID: nodeID, Name: "re-homing", Address: "10.0.0.2:50051"} - workers.scriptUnroutable(nodeID) - - Expect(router.pickReachableNode(context.Background(), selectOnce(node))).To(Equal(node)) - Expect(reg.markedUnhealthy).To(BeEmpty()) - }) -}) diff --git a/core/services/nodes/unloader_test.go b/core/services/nodes/unloader_test.go index 2f33ad09bfeb..971846547d32 100644 --- a/core/services/nodes/unloader_test.go +++ b/core/services/nodes/unloader_test.go @@ -599,18 +599,28 @@ var _ = Describe("RemoteUnloaderAdapter timeout handling", func() { Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) }) - It("does not read a message merely containing the words nats timeout as a timeout", func() { - // The string match the bus carrier needed would have matched a worker - // error that quoted the phrase, and would have turned a real failure - // into "still installing" forever. + It("does not read an error merely containing the words nats timeout as a timeout", func() { + // The string match the bus carrier needed would have matched any + // message quoting the phrase, and would have turned a permanent failure + // into "still installing" forever: galleryop reads that as a soft + // failure, pushes the retry out, and the operator never sees the error. + // + // The input has to be an ERROR carrying the phrase. A successful reply + // whose Error field carries it comes back with a nil error and never + // reaches the classifier at all, so a spec built on one stays green + // with the string match restored — which is exactly what the previous + // version of this spec did. workers := newScriptedControlWorkers() - workers.scriptReply(controlKey("n1", workerctl.PathBackendInstall), - messaging.BackendInstallReply{Success: false, Error: `the worker said "nats: timeout" in its log`}) + workers.scriptServerError(controlKey("n1", workerctl.PathBackendInstall), + `the worker said "nats: timeout" in its log`) adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), time.Minute, time.Minute) - reply, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "", nil) - Expect(err).ToNot(HaveOccurred()) - Expect(reply.Success).To(BeFalse()) + _, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "", nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("nats: timeout"), + "precondition: the phrase must actually reach the timeout classifier") + Expect(errors.Is(err, galleryop.ErrWorkerStillInstalling)).To(BeFalse()) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) }) }) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 3b12b08c956f..997668c1f3ad 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -277,7 +277,7 @@ The fourth exists because the other three are acted on. A relayed request crosse A refusal code the frontend does not recognise - a newer worker's vocabulary - is treated as "no route" as well, so a version skew costs a retry rather than a reaped replica. -That distinction is the whole point rather than a nicety. A scheduler told that a connected worker has gone away stops its backend and reclaims every model it is running, and the events that produce "no route" are ordinary ones: a frontend replica restarting, an ownership row a moment stale, a worker that has not dialled its tunnel yet. A worker is treated as absent only when its **heartbeat** goes stale, which is a separate mechanism with its own threshold (see `--stale-node-threshold`). +That distinction is the whole point rather than a nicety. A scheduler told that a connected worker has gone away stops its backend and reclaims every model it is running, and the events that produce "no route" are ordinary ones: a frontend replica restarting, an ownership row a moment stale, a worker that has not dialled its tunnel yet. Absence has its own two mechanisms and neither of them is a failed request: a stale **heartbeat** (see `--stale-node-threshold`), and a tunnel **departure** older than the reconnect grace (see below). #### There is no frontend-side fallback @@ -319,6 +319,10 @@ When a worker's tunnel goes, the frontend does **not** forget the worker. It rec That distinction exists because a worker loses its tunnel for entirely ordinary reasons. A frontend replica restarting during a rolling upgrade drops every tunnel it held, and each of those workers immediately re-dials the load balancer and lands on another replica. Treating that as "the worker is gone" would evict models mid-upgrade for a fleet that never actually went anywhere. +The scheduler reads exactly this before it places a cold load. A worker whose departure has outlived the grace is skipped and marked unhealthy, so every other frontend replica stops choosing it too; the log line is `Scheduled node has no tunnel and its departure outlived the reconnect grace, marking unhealthy and re-scheduling` and it names the grace it used. The demotion is a **status** change, not a deletion: the worker's `node_models` rows survive it, and a worker that comes back is marked healthy again by its next heartbeat. + +The other three answers place work as normal. **Reconnecting** (the tunnel went inside the grace) and **unknown** (no connection row at all — the worker has never dialled, or its departure has already aged out of retention) are both non-verdicts; so is a failure to read the answer, because a database hiccup that excluded workers would cost the fleet its capacity for a reason that has nothing to do with any worker. In each of those cases scheduling proceeds and the install that follows reports its own outcome. + | Flag | Env var | Default | Description | |------|---------|---------|-------------| | `--worker-reconnect-grace` | `LOCALAI_WORKER_RECONNECT_GRACE` | `90s` | How long a worker whose tunnel was lost is treated as reconnecting rather than gone. | @@ -1547,7 +1551,8 @@ Notes: - The chosen worker's tunnel was not reachable from the replica that handled the request. A node's status comes from its HTTP heartbeat, which is a separate channel: a worker that stops stays `healthy` until that heartbeat ages out, and a worker that is very much alive can be unroutable for a moment while its tunnel re-homes between frontend replicas. - It is **not** the same as the worker being gone, and nothing acts on it as if it were. A model on an unroutable worker is not reaped, its rows are left alone, and the node is not demoted: doing any of those on a lost route is how a rolling frontend restart turns into a fleet-wide eviction. - Check the worker process is running and that it has an open tunnel (`opened a tunnelled stream to a worker` in the frontend log, and the worker's own dial/reconnect lines). A worker behind a load balancer that keeps reconnecting is usually an idle-timeout or WebSocket-upgrade problem at the proxy; see the tunnel section above. -- A `nats: no responders available for request` in a modern deployment concerns only the subjects still on the bus: agent-worker jobs, MCP, and file staging. It is no longer how a serve-backend worker is reached. +- **`no route` is not `gone`, and nothing in the frontend reads it as such.** A worker is declared **gone** by one mechanism only: no live frontend replica holds its tunnel *and* its departure is older than `--worker-reconnect-grace`. That is a fact recorded in the shared database, so every replica answers it identically. "No route" is one replica failing to reach a worker right now, and it is not evidence about the worker at all. +- Older releases decided absence from `nats: no responders available for request` — one frontend's observation that nobody answered *it* within a request budget. Two replicas asking at the same moment could disagree and demote each other's workers. That signal is gone from the scheduler; if you still see the message, it concerns only the subjects that remain on the bus (agent-worker jobs and MCP), never a serve-backend worker. **A worker fills its own disk over time:** - A request that carries a file (an image, an audio clip, a video) stages that file below the worker's HTTP staging or S3 cache `ephemeral/` directory. The frontend releases each request-owned input when inference finishes, and the worker reserves capacity before accepting it. From c0b32466e54872f19a4e58b7a9eed942bfab43e0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 00:33:21 +0000 Subject: [PATCH 077/153] fix(distributed): let absence be decided by something, at all three call sites Removing "Presence: clusterRegistry" from the options literal in initDistributed left all seven suites and tests/e2e/distributed green. The predicate was right and its input was silently nil, which returns the deployment to absence being decided by nothing, with no log line and no failing request. That is the fourth finding of this exact shape in this phase. The two assignments move out of a twenty-field literal into distributedSchedulerOptions, a named function a unit spec can reach. Deleting either is now red. The health monitor takes its presence reader and grace as a required positional pair instead, so deleting those does not compile at all. requireAbsenceWiring then refuses to start a distributed frontend whose scheduler or health monitor has no source of absence, because refusing to boot is the only symptom either failure has. With a fresh heartbeat and a permanently gone tunnel there was no reaper at all. A heartbeat says the worker's supervisor is alive; it says nothing about whether anything here can reach that worker's backends, because those are reached over the tunnel. A proxy that stops upgrading WebSockets, a rotated registration credential or a reconnect loop longer than the grace left a node listed healthy forever while every request for a model already loaded on it failed "no route to that worker", and every reaper keyed on the heartbeat. The health monitor now reads presence from the same place and against the same window as the scheduler and demotes such a node. That also ends the 15s re-promotion: the demotion arm returns before the recovery arm, so the scheduler's demotion is no longer undone on the next tick, and recovery needs the tunnel back rather than just the heartbeat. The demotion is status-only. MarkOffline would DELETE the node's rows, and deleting rows on a presence read would give any future defect in that read the widest blast radius in the system for nothing the demotion does not already deliver. LRU eviction is the third path that commits work to a node, and it read only the stored status. A node full enough to be an eviction target is exactly the node the VRAM and idle selectors never offer, so pickReachableNode structurally cannot cover it. It now runs its chosen node through the same nodeMayTakeWork predicate, demotes it and evicts again rather than handing back an install that cannot land. Presence is read after the transaction and not inside it: reading it inside would hold a FOR UPDATE lock across a query needing a second pooled connection, which is how concurrent evictions deadlock a pool. Also: a router built with a presence reader and no grace now has its documented default pinned by a spec rather than only claimed by a comment; ageDeparture asserts RowsAffected, since an UPDATE matching nothing succeeds and the inside-the-grace spec returned the same verdict either way; the scheduler comment that still described the bus is corrected; the docs stop conflating heartbeat recovery with tunnel recovery and name the third reader; and an overlong rewrapped line in membership.go is folded. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/absence_wiring.go | 65 ++++++ core/application/absence_wiring_test.go | 121 ++++++++++++ core/application/distributed.go | 32 ++- core/services/cluster/membership.go | 7 +- core/services/nodes/health.go | 102 +++++++++- core/services/nodes/health_test.go | 208 +++++++++++++++++++- core/services/nodes/router.go | 59 +++++- core/services/nodes/router_liveness.go | 11 ++ core/services/nodes/router_liveness_test.go | 162 ++++++++++++++- core/services/nodes/unloader_test.go | 2 +- docs/content/features/distributed-mode.md | 16 +- 11 files changed, 749 insertions(+), 36 deletions(-) create mode 100644 core/application/absence_wiring.go create mode 100644 core/application/absence_wiring_test.go diff --git a/core/application/absence_wiring.go b/core/application/absence_wiring.go new file mode 100644 index 000000000000..b5f3df99532e --- /dev/null +++ b/core/application/absence_wiring.go @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: MIT + +package application + +import ( + "fmt" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/nodes" +) + +// distributedSchedulerOptions stamps the absence wiring onto the scheduler's +// options and returns them. +// +// Two assignments in a named function rather than two more fields in the +// twenty-field literal they used to live in. The literal cannot be reached by a +// unit spec, because the function that builds it also opens a NATS connection +// and a database; these two lines can, and they are the two lines this whole +// change comes down to. Losing them in the literal was silent and green. +// +// The grace comes from the same expression the membership loop is given +// (Membership.SetReconnectGrace), so the window a departure is measured against +// and the window a departure is RETAINED for cannot drift apart. +func distributedSchedulerOptions(cfg config.DistributedConfig, presence nodes.NodePresenceReader, opts nodes.SmartRouterOptions) nodes.SmartRouterOptions { + opts.Presence = presence + opts.ReconnectGrace = cfg.ReconnectGraceOrDefault() + return opts +} + +// requireAbsenceWiring refuses to start a distributed deployment in which +// nothing can decide that a worker has gone away. +// +// Two components read absence, from one source and against one window: the +// scheduler, which stops placing work on a departed worker, and the health +// monitor, which stops reporting one as healthy. Each reads it through a field +// assigned in a large construction literal in initDistributed. +// +// It is checked rather than assumed because losing either assignment is +// SILENT. A scheduler with no absence source places work on workers that are +// gone and demotes none; a health monitor with none reports a worker whose +// tunnel died an hour ago as healthy, forever, with every request for a model +// loaded on it failing "no route to that worker". Neither logs anything, +// neither fails a request that would not have failed anyway, and both look +// exactly like a fleet that is fine. Refusing to boot is the only symptom +// either failure has, and it is the reason this is a startup error and not a +// warning: a deployment that came up and quietly decided absence by nothing is +// the state the tunnel work exists to remove. +// +// What this guard itself rests on, stated because it is a real limit: the two +// helper functions below and above are pinned by unit specs, but the CALL to +// this one lives in initDistributed, which opens NATS and a database and so has +// no unit spec at all. Deleting the call, or writing a literal nil where +// initDistributed passes the cluster registry, compiles and leaves every suite +// in this repository green. Only tests/e2e/distributed/cluster catches it, by +// booting the real binary: the error is returned from initDistributed and +// aborts application startup, so a frontend so wired never comes up. +func requireAbsenceWiring(router *nodes.SmartRouter, health *nodes.HealthMonitor) error { + if !router.ReadsAbsence() { + return fmt.Errorf("the distributed scheduler was built with no source of worker absence: it would place work on workers that have gone away and never demote one") + } + if !health.ReadsAbsence() { + return fmt.Errorf("the node health monitor was built with no source of worker absence: it would report a worker whose tunnel is gone as healthy indefinitely") + } + return nil +} diff --git a/core/application/absence_wiring_test.go b/core/application/absence_wiring_test.go new file mode 100644 index 000000000000..5ca33122c2f6 --- /dev/null +++ b/core/application/absence_wiring_test.go @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: MIT + +package application + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/nodes" +) + +// presenceStub is any source of absence. What it answers does not matter here: +// these specs are about whether a source was wired at all, which is the one +// property that has no other symptom. +type presenceStub struct{} + +func (presenceStub) Presence(context.Context, string, time.Duration) (cluster.Presence, error) { + return cluster.PresenceConnected, nil +} + +// The guard on the two lines that connect the absence decision to production. +// +// Absence is read in exactly two places, and each reads it through one field +// assigned in a twenty-field construction literal in initDistributed. Deleting +// either assignment compiles, passes every suite in this repository, and +// returns the deployment to "absence is decided by nothing" without a log line. +// That is the failure this guard exists for, and these specs are what keep the +// guard honest: an assertion that never fails is not one. +var _ = Describe("stamping the absence wiring onto the scheduler's options", func() { + It("gives the scheduler the deployment's source of absence", func() { + reg := presenceStub{} + + opts := distributedSchedulerOptions(config.DistributedConfig{}, reg, nodes.SmartRouterOptions{}) + + Expect(opts.Presence).To(Equal(nodes.NodePresenceReader(reg))) + }) + + It("gives it the operator's reconnect grace", func() { + opts := distributedSchedulerOptions( + config.DistributedConfig{WorkerReconnectGrace: 4 * time.Minute}, presenceStub{}, nodes.SmartRouterOptions{}) + + Expect(opts.ReconnectGrace).To(Equal(4 * time.Minute)) + }) + + It("falls back to the documented default when the operator set no grace", func() { + opts := distributedSchedulerOptions(config.DistributedConfig{}, presenceStub{}, nodes.SmartRouterOptions{}) + + Expect(opts.ReconnectGrace).To(Equal(config.DefaultWorkerReconnectGrace)) + }) + + It("leaves every other option the caller built untouched", func() { + // The negative control: a stamp that rebuilt the options would drop the + // twenty fields the caller assembled, and the two assertions above + // would still pass. + opts := distributedSchedulerOptions(config.DistributedConfig{}, presenceStub{}, + nodes.SmartRouterOptions{GalleriesJSON: "[]", SharedModels: true}) + + Expect(opts.GalleriesJSON).To(Equal("[]")) + Expect(opts.SharedModels).To(BeTrue()) + }) + + It("produces a scheduler that reads absence", func() { + // The property the boot guard checks, asserted through the same call + // initDistributed makes. + router := nodes.NewSmartRouter(nil, + distributedSchedulerOptions(config.DistributedConfig{}, presenceStub{}, nodes.SmartRouterOptions{})) + + Expect(router.ReadsAbsence()).To(BeTrue()) + }) +}) + +var _ = Describe("the absence wiring a distributed deployment refuses to start without", func() { + present := func() (*nodes.SmartRouter, *nodes.HealthMonitor) { + router := nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{Presence: presenceStub{}}) + health := nodes.NewHealthMonitor(nil, nil, time.Second, time.Minute, "", false, presenceStub{}, time.Minute) + return router, health + } + + It("accepts a deployment where both readers have a source", func() { + router, health := present() + Expect(requireAbsenceWiring(router, health)).To(Succeed()) + }) + + It("refuses a scheduler built without one, and says what it would do", func() { + _, health := present() + blind := nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}) + + err := requireAbsenceWiring(blind, health) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("scheduler")) + Expect(err.Error()).To(ContainSubstring("never demote")) + }) + + It("refuses a health monitor built without one, and says what it would do", func() { + router, _ := present() + blind := nodes.NewHealthMonitor(nil, nil, time.Second, time.Minute, "", false, nil, 0) + + err := requireAbsenceWiring(router, blind) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("health monitor")) + Expect(err.Error()).To(ContainSubstring("healthy indefinitely")) + }) + + // Each reader is named separately on purpose. One guard covering "at least + // one of them" would accept a deployment that had lost the other, and the + // two failures are different: the scheduler's places work on a dead worker, + // the monitor's leaves it listed healthy while its models are unreachable. + It("names the scheduler and the health monitor as separate requirements", func() { + blindRouter := nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}) + blindHealth := nodes.NewHealthMonitor(nil, nil, time.Second, time.Minute, "", false, nil, 0) + router, health := present() + + Expect(requireAbsenceWiring(blindRouter, health)).ToNot(Succeed()) + Expect(requireAbsenceWiring(router, blindHealth)).ToNot(Succeed()) + }) +}) diff --git a/core/application/distributed.go b/core/application/distributed.go index 617196d960fe..1e1bc66c21c4 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -352,11 +352,18 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade routerGalleriesJSON = string(galleriesJSON) } + // The health monitor is the SECOND reader of absence, and it reads it from + // the same place and against the same window as the scheduler: a heartbeat + // says the worker's supervisor is alive, presence says whether anything + // here can still reach its backends, and a worker can be the first without + // being the second indefinitely. healthMon := nodes.NewHealthMonitor(registry, authDB, cfg.Distributed.HealthCheckIntervalOrDefault(), cfg.Distributed.StaleNodeThresholdOrDefault(), routerAuthToken, !cfg.Distributed.DisablePerModelHealthCheck, + clusterRegistry, + cfg.Distributed.ReconnectGraceOrDefault(), backendClients, ) @@ -542,15 +549,14 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade pinnedResolver = configLoader } modelCleanup := nodes.NewModelCleanupService(registry, remoteUnloader) - router := nodes.NewSmartRouter(registry, nodes.SmartRouterOptions{ - Unloader: remoteUnloader, - // Absence, and the only source of it the scheduler has. It is a fact - // read from the database, so every replica answers it identically; - // the bus sentinel it replaces was one frontend's observation that - // nobody answered IT within a budget, and two replicas asking at the - // same moment could disagree and demote each other's workers. - Presence: clusterRegistry, - ReconnectGrace: cfg.Distributed.ReconnectGraceOrDefault(), + // Absence is stamped on by distributedSchedulerOptions rather than written + // here. It is the only source of absence the scheduler has -- a fact read + // from the database, so every replica answers it identically, where the bus + // sentinel it replaces was one frontend's observation that nobody answered + // IT within a budget -- and a field carrying that in a literal this size is + // the easiest thing in this file to lose without a symptom. + router := nodes.NewSmartRouter(registry, distributedSchedulerOptions(cfg.Distributed, clusterRegistry, nodes.SmartRouterOptions{ + Unloader: remoteUnloader, ModelCleanup: modelCleanup, FileStager: fileStager, GalleriesJSON: routerGalleriesJSON, @@ -587,7 +593,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // Bounds the REQUEST, not the load: a caller out of budget gets 503 with // live staging progress while the job keeps running underneath. ModelLoadWait: cfg.Distributed.ModelLoadWait, - }) + })) // Wire staging-progress broadcasting so file-staging shows up on every // replica, not just the one performing the transfer. Without this, a @@ -625,6 +631,12 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade PinnedResolver: pinnedResolver, }) + // Both readers of absence, checked once, here. See requireAbsenceWiring for + // why a missing assignment has no other symptom. + if err := requireAbsenceWiring(router, healthMon); err != nil { + return nil, err + } + // Create ModelRouterAdapter to wire into ModelLoader modelAdapter := nodes.NewModelRouterAdapter(router) diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index d76388160bee..2dad1de6ae60 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -382,9 +382,10 @@ func (r *Registry) Deregister(ctx context.Context, id string) error { // The connection rows are cleared and not deleted, which is why the second // return is named for what it counts: rows this sweep CLEARED, never rows it // removed. Reading it as a delete count would make a worker that is re-dialling -// right now look like one this deployment has forgotten. A worker whose owning replica died has not gone -// anywhere: it is re-dialling the load balancer, and deleting its row would -// erase the departure that says how long ago that started. +// right now look like one this deployment has forgotten. A worker whose owning +// replica died has not gone anywhere: it is re-dialling the load balancer, and +// deleting its row would erase the departure that says how long ago that +// started. // // self is never reaped. This process may fail to heartbeat for longer than the // window (a long stall, a database blip) and still be serving: deleting its own diff --git a/core/services/nodes/health.go b/core/services/nodes/health.go index cf8b7b57fbad..f7b0bc7a969f 100644 --- a/core/services/nodes/health.go +++ b/core/services/nodes/health.go @@ -9,6 +9,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/advisorylock" + "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/xlog" "gorm.io/gorm" ) @@ -40,10 +41,16 @@ type HealthMonitor struct { autoOffline bool // mark stale nodes as offline (preserves approval status) clientFactory BackendClientFactory // creates gRPC backend clients perModelHealthCheck bool // check each model's backend process individually - missesMu sync.Mutex - misses map[modelKey]int // consecutive failed-probe counts; reset on success or model removal - cancel context.CancelFunc - cancelMu sync.Mutex + // presence and reconnectGrace are the second liveness mechanism. The + // heartbeat says the worker's supervisor is alive; presence says whether + // anything in this deployment can still reach its backends. See + // tunnelDeparted. nil disables the second mechanism entirely. + presence NodePresenceReader + reconnectGrace time.Duration + missesMu sync.Mutex + misses map[modelKey]int // consecutive failed-probe counts; reset on success or model removal + cancel context.CancelFunc + cancelMu sync.Mutex } // NewHealthMonitor creates a new HealthMonitor. @@ -54,7 +61,14 @@ type HealthMonitor struct { // every request, so per-model probes are skipped and logged rather than counted // as misses; authToken is then only the credential a working factory would have // carried. Production always passes one. -func NewHealthMonitor(registry NodeHealthStore, db *gorm.DB, checkInterval, staleThreshold time.Duration, authToken string, perModelHealthCheck bool, clientFactory ...BackendClientFactory) *HealthMonitor { +// +// presence and reconnectGrace are a REQUIRED positional pair rather than another +// optional tail, so a caller has to decide rather than inherit a nil. A nil +// reader means this deployment has nothing that can say a worker is gone, which +// is correct for a single-node install and wrong for a distributed one; a zero +// grace with a non-nil reader takes the documented default, because a zero +// window would make every departure a verdict the instant it was stamped. +func NewHealthMonitor(registry NodeHealthStore, db *gorm.DB, checkInterval, staleThreshold time.Duration, authToken string, perModelHealthCheck bool, presence NodePresenceReader, reconnectGrace time.Duration, clientFactory ...BackendClientFactory) *HealthMonitor { checkInterval = cmp.Or(checkInterval, 15*time.Second) // Heartbeat checkpointing lets last_heartbeat sit up to one checkpoint // interval behind by design, so a hardcoded 60s fallback here would mark @@ -75,10 +89,51 @@ func NewHealthMonitor(registry NodeHealthStore, db *gorm.DB, checkInterval, stal autoOffline: true, clientFactory: factory, perModelHealthCheck: perModelHealthCheck, + presence: presence, + reconnectGrace: cmp.Or(reconnectGrace, config.DefaultWorkerReconnectGrace), misses: make(map[modelKey]int), } } +// ReadsAbsence reports whether this monitor has a source for the second +// liveness mechanism. +// +// It exists to be asserted at wiring time (see core/application), and that is +// worth stating because it is the only symptom the wiring has. A monitor built +// without a presence reader does not fail, log, or behave oddly: it reports a +// worker whose tunnel died an hour ago as healthy, forever, which is exactly +// what a healthy fleet looks like. +func (hm *HealthMonitor) ReadsAbsence() bool { return hm != nil && hm.presence != nil } + +// tunnelDeparted reports whether this deployment has decided that a node's +// tunnel is gone: no live replica holds it and the departure has outlived the +// reconnect grace. +// +// Only cluster.PresenceGone answers true. PresenceReconnecting is a worker +// re-dialling right now, PresenceUnknown is a worker that has never dialled or +// whose departure aged out, and a query that FAILS is not an answer at all. +// Acting on any of those would demote a fleet for a reason that has nothing to +// do with any worker, which is the collapse this whole mechanism replaced. +// +// Backend workers only. An agent worker holds no tunnel at all, so it has no +// departure to measure and would answer PresenceUnknown anyway; the type check +// is here to save the query rather than to add a second rule. +func (hm *HealthMonitor) tunnelDeparted(ctx context.Context, node *BackendNode) bool { + if hm.presence == nil || node == nil { + return false + } + if node.NodeType != "" && node.NodeType != NodeTypeBackend { + return false + } + p, err := hm.presence.Presence(ctx, node.ID, hm.reconnectGrace) + if err != nil { + xlog.Warn("Health monitor could not read node presence; leaving the node's status alone", + "node", node.Name, "nodeID", node.ID, "error", err) + return false + } + return p == cluster.PresenceGone +} + // Start begins the health monitoring loop in a background goroutine. // If a previous instance is running, it is stopped first. func (hm *HealthMonitor) Start(ctx context.Context) { @@ -174,7 +229,42 @@ func (hm *HealthMonitor) doCheckAll(ctx context.Context) { continue } - // Heartbeat is fresh — node is alive + // The heartbeat is fresh, so the worker's supervisor is alive. That is + // NOT the same as this deployment being able to reach its backends: + // those are reached over the worker's TUNNEL, and a worker can + // heartbeat forever with no tunnel at all (a proxy that stopped + // upgrading WebSockets, a rotated registration credential, a reconnect + // loop longer than the grace). + // + // Before this check the two were conflated and the result was a node + // wedged in plain sight: listed healthy, heartbeating, with every + // request for a model already loaded on it failing "no route to that + // worker" indefinitely. Nothing reaped it, because every reaper keys on + // the heartbeat and the heartbeat was fine. + // + // Demoted and not marked offline, deliberately. MarkUnhealthy is + // status-only, and status is enough: routing and eviction both select + // on status=healthy, so the loaded rows stop being chosen and the model + // is placed somewhere reachable on the next request. MarkOffline would + // DELETE this node's rows, and deleting rows on a presence read would + // give any future defect in that read the largest blast radius in the + // system for no gain the demotion does not already deliver. + if hm.tunnelDeparted(ctx, &node) { + if node.Status != StatusUnhealthy && node.Status != StatusOffline { + xlog.Warn("Node is heartbeating but its tunnel has been gone longer than the reconnect grace; marking unhealthy", + "node", node.Name, "nodeID", node.ID, "grace", hm.reconnectGrace) + if err := hm.registry.MarkUnhealthy(ctx, node.ID); err != nil { + xlog.Error("Failed to mark a departed node unhealthy", "node", node.Name, "error", err) + } + } + // No re-promotion, and no per-model probes. The probes would dial a + // worker there is no route to, once per model per tick, and decline + // to count any of it; the re-promotion below is what used to undo + // this demotion on the very next tick. + continue + } + + // Heartbeat is fresh and the tunnel is not gone: the node is alive if node.Status == StatusUnhealthy || node.Status == StatusOffline { xlog.Info("Node recovered", "node", node.Name) if err := hm.registry.MarkHealthy(ctx, node.ID); err != nil { diff --git a/core/services/nodes/health_test.go b/core/services/nodes/health_test.go index 95f10e7dca20..089ca02162a4 100644 --- a/core/services/nodes/health_test.go +++ b/core/services/nodes/health_test.go @@ -2,6 +2,7 @@ package nodes import ( "context" + "errors" "fmt" "runtime" "time" @@ -9,6 +10,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/LocalAI/core/services/testutil" @@ -33,7 +35,7 @@ var _ = Describe("HealthMonitor", func() { // Use a 30-second stale threshold for tests. // Pass nil db to avoid advisory lock path (no distributed mode in tests). - hm = NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false) + hm = NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, nil, 0) }) makeNode := func(name, address string, vram uint64) *BackendNode { @@ -406,3 +408,207 @@ var _ = Describe("HealthMonitor (mock-based)", func() { }) }) }) + +// The wedge this check exists to end. +// +// A worker's heartbeat says its supervisor is alive. It says nothing about +// whether anything in this deployment can reach the worker's BACKENDS, because +// those are reached over the worker's tunnel. Before these specs the two were +// conflated, and a worker that heartbeated with a permanently dead tunnel (a +// proxy that stopped upgrading WebSockets, a rotated credential, a reconnect +// loop longer than the grace) stayed listed HEALTHY forever while every request +// for a model already loaded on it failed "no route to that worker". No reaper +// was scheduled for it: every reaper keys on the heartbeat, and the heartbeat +// was fine. +// +// Driven through a real cluster.Registry with the departures aged on the +// DATABASE clock, because a double that answers a Presence value cannot show +// that the window is measured where every replica agrees on it. +var _ = Describe("HealthMonitor and a worker whose tunnel is gone", func() { + var ( + ctx context.Context + db *gorm.DB + registry *NodeRegistry + clusterR *cluster.Registry + hm *HealthMonitor + ) + + const ( + grace = 60 * time.Second + instance = "inst-health" + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + ctx = context.Background() + db = testutil.SetupTestDB() + var err error + registry, err = NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + clusterR = cluster.NewRegistry(db) + Expect(clusterR.Register(ctx, instance, "10.0.0.1:8080", "v1")).To(Succeed()) + hm = NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, clusterR, grace) + }) + + // register creates a heartbeating backend worker. Its heartbeat stays fresh + // for the whole spec, which is the precondition the wedge needs: a stale + // heartbeat would take the OTHER branch and prove nothing about this one. + register := func(name string) *BackendNode { + GinkgoHelper() + node := &BackendNode{Name: name, NodeType: NodeTypeBackend, TotalVRAM: 8_000_000_000, AvailableVRAM: 8_000_000_000} + Expect(registry.Register(ctx, node, true)).To(Succeed()) + Expect(node.Status).To(Equal(StatusHealthy)) + return node + } + + statusOf := func(id string) string { + GinkgoHelper() + n, err := registry.Get(ctx, id) + Expect(err).ToNot(HaveOccurred()) + return n.Status + } + + // departTunnel gives a node a connection row, releases it, and ages the + // departure by `by` ON THE DATABASE CLOCK. A Go-side timestamp would be + // compared against the database's, which is the skew this window is written + // to be immune to. + departTunnel := func(nodeID string, by time.Duration) { + GinkgoHelper() + epoch, err := clusterR.Claim(ctx, nodeID, instance) + Expect(err).ToNot(HaveOccurred()) + Expect(clusterR.Release(ctx, nodeID, instance, epoch)).To(Succeed()) + res := db.WithContext(ctx).Exec( + `UPDATE node_connections SET disconnected_at = now() - make_interval(secs => ?) WHERE node_id = ?`, + by.Seconds(), nodeID) + Expect(res.Error).ToNot(HaveOccurred()) + Expect(res.RowsAffected).To(Equal(int64(1)), + "precondition: the departure this spec ages must exist, or the spec proves nothing") + } + + It("stops reporting a heartbeating node healthy once its tunnel has been gone past the grace", func() { + node := register("wedged-worker") + departTunnel(node.ID, grace+5*time.Second) + + hm.doCheckAll(ctx) + + Expect(statusOf(node.ID)).To(Equal(StatusUnhealthy)) + }) + + It("does not delete the node's model rows, because a demotion is enough to unwedge it", func() { + // Routing and eviction both select on status=healthy, so demoting stops + // the model being served from here and the next request places it + // somewhere reachable. Deleting rows would give any future defect in + // the presence read the largest blast radius in the system, for nothing + // the demotion does not already deliver. + node := register("wedged-with-models") + Expect(registry.SetNodeModel(ctx, node.ID, "llama", 0, "loaded", "127.0.0.1:50100", 0)).To(Succeed()) + departTunnel(node.ID, grace+5*time.Second) + + hm.doCheckAll(ctx) + + models, err := registry.GetNodeModels(ctx, node.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(models).To(HaveLen(1)) + }) + + It("does not re-promote a demoted node whose tunnel is still gone, however fresh its heartbeat", func() { + // The re-promotion branch used to fire on a fresh heartbeat alone, so + // the scheduler's demotion was undone on the next tick, every tick. Its + // cross-replica value was about one health interval. + node := register("still-gone") + departTunnel(node.ID, grace+5*time.Second) + Expect(registry.MarkUnhealthy(ctx, node.ID)).To(Succeed()) + + hm.doCheckAll(ctx) + + Expect(statusOf(node.ID)).To(Equal(StatusUnhealthy)) + }) + + It("re-promotes a demoted node once its tunnel is back", func() { + // The negative control for the spec above: without this, refusing to + // re-promote ANYTHING would satisfy it. + node := register("recovered") + departTunnel(node.ID, grace+5*time.Second) + Expect(registry.MarkUnhealthy(ctx, node.ID)).To(Succeed()) + _, err := clusterR.Claim(ctx, node.ID, instance) + Expect(err).ToNot(HaveOccurred()) + + hm.doCheckAll(ctx) + + Expect(statusOf(node.ID)).To(Equal(StatusHealthy)) + }) + + It("leaves a node whose tunnel went inside the grace alone", func() { + // A worker re-dialling the load balancer right now. Demoting it is the + // fleet-wide eviction on a rolling frontend restart. + node := register("re-homing") + departTunnel(node.ID, grace/2) + + hm.doCheckAll(ctx) + + Expect(statusOf(node.ID)).To(Equal(StatusHealthy)) + }) + + It("leaves a node that has never dialled a tunnel alone", func() { + // No connection row at all, which is also every worker in the window + // between registering and dialling. + node := register("never-dialled") + + hm.doCheckAll(ctx) + + Expect(statusOf(node.ID)).To(Equal(StatusHealthy)) + }) + + It("leaves an AGENT node alone even when its tunnel would read as gone", func() { + // Agent workers hold no tunnel and still take their one verb over the + // bus. A departure row for one is not a fact about it. + node := &BackendNode{Name: "agent-worker", NodeType: NodeTypeAgent} + Expect(registry.Register(ctx, node, true)).To(Succeed()) + departTunnel(node.ID, grace+5*time.Second) + + hm.doCheckAll(ctx) + + Expect(statusOf(node.ID)).To(Equal(StatusHealthy)) + }) + + It("leaves every node alone when it has no presence reader", func() { + // A single-node deployment has nothing that can say a worker is gone. + node := register("no-cluster-registry") + departTunnel(node.ID, grace+5*time.Second) + plain := NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, nil, 0) + + plain.doCheckAll(ctx) + + Expect(statusOf(node.ID)).To(Equal(StatusHealthy)) + Expect(plain.ReadsAbsence()).To(BeFalse()) + }) + + It("falls back to the documented grace when built with a presence reader and no grace", func() { + // Symmetric with the scheduler's default. A zero window here would make + // every departure a verdict the instant it was stamped, and every spec + // above passes an explicit grace, so nothing else reaches this. + node := register("no-grace") + stub := &stubPresence{answer: cluster.PresenceConnected} + defaulted := NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, stub, 0) + + defaulted.doCheckAll(ctx) + + Expect(stub.nodes).To(ContainElement(node.ID)) + Expect(stub.graces).To(ContainElement(config.DefaultWorkerReconnectGrace)) + }) + + It("leaves a node alone when the presence query fails", func() { + // A database hiccup must not demote the fleet. Driven with a stub, + // because a real registry cannot be made to fail on demand. + node := register("query-fails") + broken := NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, + &stubPresence{err: errors.New("connection reset")}, grace) + + broken.doCheckAll(ctx) + + Expect(statusOf(node.ID)).To(Equal(StatusHealthy)) + }) +}) diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index 68cfa00c4cb7..ef3add61644d 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -1194,11 +1194,12 @@ func (r *SmartRouter) scheduleNewModel(ctx context.Context, backendType, modelID // If freeSlotNodes is empty (everyone full), candidateNodeIDs is whatever // it was — we'll fall through to eviction below. - // Node choice is wrapped in a liveness check: a node's stored status comes - // from its HTTP heartbeat, which is a different channel from the bus that - // carries the install. A worker that has died stops answering on the bus at - // once but stays healthy in the database until its heartbeat ages out, so - // without this the scheduler could commit to a node it cannot reach. + // Node choice is wrapped in an absence check: a node's stored status comes + // from its HTTP heartbeat, which is a different channel from the tunnel + // every install and every request travels over. A worker can heartbeat with + // no tunnel at all, so without this the scheduler could commit to a node + // nothing in the deployment can reach. See nodeMayTakeWork for which of the + // four presence answers is allowed to exclude, and which three are not. // // The last selection error is kept because eviction below fires on a nil // node, and a lookup that failed is not the same answer as a cluster with @@ -2291,20 +2292,58 @@ func (r *SmartRouter) evictLRUAndFreeNodeFrom(ctx context.Context, candidateNode }) if err == nil { + node, nodeErr := r.registry.Get(ctx, lru.NodeID) + if nodeErr != nil { + return nil, fmt.Errorf("node %s not found after eviction: %w", lru.NodeID, nodeErr) + } + + // The third place work is committed to a node, and it has to read + // absence for the same reason the other two do. + // + // The query above chose on stored status, which comes from the + // heartbeat. A worker that heartbeats with a departed tunnel is + // demoted by the health monitor, but only on its next cycle, so + // between the departure ageing past the grace and that cycle this + // query still offers the node. pickReachableNode does not cover it: + // it only sees what the VRAM and idle selectors offer, and a node + // full enough to be an eviction target is exactly the node those + // selectors skip. + // + // Returning such a node hands the caller an install that cannot + // land. Demote and evict again instead: the demotion takes the node + // out of the next query, which selects on status, so the loop makes + // progress rather than re-picking it. + // + // The row this attempt deleted stays deleted, and presence is read + // after the transaction rather than inside it deliberately. Reading + // it inside would hold a FOR UPDATE lock across a query that needs a + // second pooled connection, which is how concurrent evictions + // deadlock a connection pool. The deleted row costs nothing: it + // named a backend on a worker no replica can reach, so nothing was + // serving from it to lose. + if !r.nodeMayTakeWork(ctx, node) { + xlog.Warn("Eviction target has no tunnel and its departure outlived the reconnect grace, marking unhealthy and evicting again", + "node", node.Name, "nodeID", node.ID, "model", lru.ModelName, "grace", r.reconnectGrace) + if markErr := r.registry.MarkUnhealthy(ctx, node.ID); markErr != nil { + // Without the demotion the next query hands back the same + // node, so stop rather than spin. + xlog.Warn("Failed to mark departed eviction target unhealthy", + "node", node.Name, "nodeID", node.ID, "error", markErr) + return nil, ErrEvictionBusy + } + continue + } + xlog.Info("Evicted LRU model to free capacity", "node", lru.NodeID, "model", lru.ModelName, "lastUsed", lru.LastUsed) - // Unload outside the transaction (NATS call) + // Unload outside the transaction. if r.unloader != nil { if uerr := r.unloader.UnloadModelOnNode(lru.NodeID, lru.ModelName); uerr != nil { xlog.Warn("eviction unload failed (model already removed from registry)", "error", uerr) } } - node, nodeErr := r.registry.Get(ctx, lru.NodeID) - if nodeErr != nil { - return nil, fmt.Errorf("node %s not found after eviction: %w", lru.NodeID, nodeErr) - } return node, nil } diff --git a/core/services/nodes/router_liveness.go b/core/services/nodes/router_liveness.go index 06c684623159..6065cb41fa92 100644 --- a/core/services/nodes/router_liveness.go +++ b/core/services/nodes/router_liveness.go @@ -64,6 +64,17 @@ func (r *SmartRouter) nodeMayTakeWork(ctx context.Context, node *BackendNode) bo return p != cluster.PresenceGone } +// ReadsAbsence reports whether this scheduler has a source for absence. +// +// It exists to be asserted at wiring time (see core/application), and that is +// worth stating because it is the only symptom the wiring has. A scheduler +// built without a presence reader does not fail, log, or behave oddly: it +// places work on workers that are gone and never demotes one, which is exactly +// what a healthy fleet looks like. The field it reads is one line in a +// twenty-field options literal, and losing that line silently returns the +// deployment to the state this whole change removed. +func (r *SmartRouter) ReadsAbsence() bool { return r != nil && r.presence != nil } + // pickReachableNode calls selectNode until it yields a node nodeMayTakeWork // does not exclude, and returns nil when it cannot find one. // diff --git a/core/services/nodes/router_liveness_test.go b/core/services/nodes/router_liveness_test.go index d05ddc061e39..a7ced6d30e3c 100644 --- a/core/services/nodes/router_liveness_test.go +++ b/core/services/nodes/router_liveness_test.go @@ -3,12 +3,15 @@ package nodes import ( "context" "errors" + "fmt" + "runtime" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "gorm.io/gorm" + "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/LocalAI/core/services/testutil" ) @@ -108,6 +111,18 @@ var _ = Describe("Scheduling and node presence", func() { Expect(presence.graces).To(Equal([]time.Duration{grace})) }) + It("falls back to the documented grace when a presence reader is wired without one", func() { + // A zero window would make every departure a verdict the instant it was + // stamped, so every worker that lost a tunnel a moment ago would be + // gone. Nothing else in this file constructs a router with a presence + // reader and no grace, which is exactly how a caller reaches this. + defaulted := NewSmartRouter(reg, SmartRouterOptions{Presence: presence}) + + defaulted.nodeMayTakeWork(ctx, newNode("node-1")) + + Expect(presence.graces).To(Equal([]time.Duration{config.DefaultWorkerReconnectGrace})) + }) + It("does not mark a reconnecting node unhealthy", func() { node := newNode("re-homing") presence.answer = cluster.PresenceReconnecting @@ -210,9 +225,16 @@ var _ = Describe("Scheduling against the cluster registry that answers presence" // skew the whole window is written to be immune to. ageDeparture := func(nodeID string, by time.Duration) { GinkgoHelper() - Expect(db.WithContext(ctx).Exec( + res := db.WithContext(ctx).Exec( `UPDATE node_connections SET disconnected_at = now() - make_interval(secs => ?) WHERE node_id = ?`, - by.Seconds(), nodeID).Error).To(Succeed()) + by.Seconds(), nodeID) + Expect(res.Error).ToNot(HaveOccurred()) + // An UPDATE matching nothing succeeds. Without this, the + // inside-the-grace spec below would return the same verdict whether its + // scripted input landed or not, because a departure stamped a moment + // ago is also inside the grace. + Expect(res.RowsAffected).To(Equal(int64(1)), + "precondition: the departure this spec ages must exist") } BeforeEach(func() { @@ -278,3 +300,139 @@ var _ = Describe("Scheduling against the cluster registry that answers presence" Expect(reg.markedUnhealthy).To(BeEmpty()) }) }) + +// The third place work is committed to a node. +// +// pickReachableNode covers what the VRAM and idle selectors offer, and the +// health monitor demotes a departed worker on its next cycle. Neither covers +// eviction: a node full enough to be an eviction target is exactly the node +// those selectors skip, and between a departure ageing past the grace and the +// monitor's next cycle the eviction query still offers it, because that query +// selects on the stored status and the status comes from the heartbeat. +// +// Handing such a node back is not a lost eviction, it is a scheduled install +// that cannot land, on a node the caller then reports as the failure. +var _ = Describe("Eviction and a worker whose tunnel is gone", func() { + var ( + ctx context.Context + db *gorm.DB + registry *NodeRegistry + clusterR *cluster.Registry + router *SmartRouter + ) + + const ( + grace = 60 * time.Second + instance = "inst-evict" + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + ctx = context.Background() + db = testutil.SetupTestDB() + var err error + registry, err = NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + clusterR = cluster.NewRegistry(db) + Expect(clusterR.Register(ctx, instance, "10.0.0.1:8080", "v1")).To(Succeed()) + router = NewSmartRouter(registry, SmartRouterOptions{DB: db, Presence: clusterR, ReconnectGrace: grace}) + }) + + register := func(name string) *BackendNode { + GinkgoHelper() + node := &BackendNode{Name: name, NodeType: NodeTypeBackend, Address: name + ":50051"} + Expect(registry.Register(ctx, node, true)).To(Succeed()) + fetched, err := registry.GetByName(ctx, name) + Expect(err).ToNot(HaveOccurred()) + return fetched + } + + // holdTunnel leaves the worker's tunnel claimed by the live replica. + holdTunnel := func(nodeID string) { + GinkgoHelper() + _, err := clusterR.Claim(ctx, nodeID, instance) + Expect(err).ToNot(HaveOccurred()) + } + + // departTunnel releases the worker's tunnel and ages the departure ON THE + // DATABASE CLOCK, which is where the window is measured. + departTunnel := func(nodeID string, by time.Duration) { + GinkgoHelper() + epoch, err := clusterR.Claim(ctx, nodeID, instance) + Expect(err).ToNot(HaveOccurred()) + Expect(clusterR.Release(ctx, nodeID, instance, epoch)).To(Succeed()) + res := db.WithContext(ctx).Exec( + `UPDATE node_connections SET disconnected_at = now() - make_interval(secs => ?) WHERE node_id = ?`, + by.Seconds(), nodeID) + Expect(res.Error).ToNot(HaveOccurred()) + Expect(res.RowsAffected).To(Equal(int64(1)), + "precondition: the departure this spec ages must exist, or the spec proves nothing") + } + + seeded := 0 + seedLoaded := func(node *BackendNode, model string, idleFor time.Duration) { + GinkgoHelper() + seeded++ + Expect(db.Create(&NodeModel{ + ID: fmt.Sprintf("evict-row-%d", seeded), NodeID: node.ID, ModelName: model, + WorkerLocalAddress: node.Address, State: "loaded", InFlight: 0, + LastUsed: time.Now().Add(-idleFor), UpdatedAt: time.Now(), + }).Error).To(Succeed()) + } + + statusOf := func(id string) string { + GinkgoHelper() + n, err := registry.Get(ctx, id) + Expect(err).ToNot(HaveOccurred()) + return n.Status + } + + It("does not hand back a node whose departure outlived the grace, and demotes it", func() { + gone, live := register("gone-node"), register("live-node") + departTunnel(gone.ID, grace+5*time.Second) + holdTunnel(live.ID) + // The global LRU sits on the departed node, so an eviction that read + // only the stored status would pick it and return it. + seedLoaded(gone, "lru-on-gone", 2*time.Hour) + seedLoaded(live, "newer-on-live", time.Hour) + + node, err := router.evictLRUAndFreeNodeFrom(ctx, nil) + + Expect(err).ToNot(HaveOccurred()) + Expect(node.ID).To(Equal(live.ID)) + Expect(statusOf(gone.ID)).To(Equal(StatusUnhealthy), + "the departure has to reach every other replica, not just this eviction") + Expect(statusOf(live.ID)).To(Equal(StatusHealthy)) + }) + + It("still evicts from a node whose tunnel went inside the grace", func() { + // The negative control. A worker re-dialling the load balancer is not + // absent, and excluding it would make a rolling frontend restart look + // like a cluster with no capacity left to free. + rehoming := register("rehoming-node") + departTunnel(rehoming.ID, grace/2) + seedLoaded(rehoming, "lru-on-rehoming", 2*time.Hour) + + node, err := router.evictLRUAndFreeNodeFrom(ctx, nil) + + Expect(err).ToNot(HaveOccurred()) + Expect(node.ID).To(Equal(rehoming.ID)) + Expect(statusOf(rehoming.ID)).To(Equal(StatusHealthy)) + }) + + It("gives up rather than returning a departed node when every candidate has gone", func() { + // Without the check this returned a node with no route at all, and the + // caller reported the install failure instead of the absence. + gone := register("only-node-gone") + departTunnel(gone.ID, grace+5*time.Second) + seedLoaded(gone, "lru-on-only-gone", time.Hour) + + _, err := router.evictLRUAndFreeNodeFrom(ctx, nil) + + Expect(err).To(HaveOccurred()) + Expect(statusOf(gone.ID)).To(Equal(StatusUnhealthy)) + }) +}) diff --git a/core/services/nodes/unloader_test.go b/core/services/nodes/unloader_test.go index 971846547d32..458a82da2710 100644 --- a/core/services/nodes/unloader_test.go +++ b/core/services/nodes/unloader_test.go @@ -608,7 +608,7 @@ var _ = Describe("RemoteUnloaderAdapter timeout handling", func() { // The input has to be an ERROR carrying the phrase. A successful reply // whose Error field carries it comes back with a nil error and never // reaches the classifier at all, so a spec built on one stays green - // with the string match restored — which is exactly what the previous + // with the string match restored, which is exactly what the previous // version of this spec did. workers := newScriptedControlWorkers() workers.scriptServerError(controlKey("n1", workerctl.PathBackendInstall), diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 997668c1f3ad..158e9965f23d 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -319,9 +319,19 @@ When a worker's tunnel goes, the frontend does **not** forget the worker. It rec That distinction exists because a worker loses its tunnel for entirely ordinary reasons. A frontend replica restarting during a rolling upgrade drops every tunnel it held, and each of those workers immediately re-dials the load balancer and lands on another replica. Treating that as "the worker is gone" would evict models mid-upgrade for a fleet that never actually went anywhere. -The scheduler reads exactly this before it places a cold load. A worker whose departure has outlived the grace is skipped and marked unhealthy, so every other frontend replica stops choosing it too; the log line is `Scheduled node has no tunnel and its departure outlived the reconnect grace, marking unhealthy and re-scheduling` and it names the grace it used. The demotion is a **status** change, not a deletion: the worker's `node_models` rows survive it, and a worker that comes back is marked healthy again by its next heartbeat. +Three things read this. The **scheduler** reads it before it places a cold load: a worker whose departure has outlived the grace is skipped and marked unhealthy, so every other frontend replica stops choosing it too. **LRU eviction** reads it before it hands back the node it freed capacity on, because a node full enough to be an eviction target is exactly the node the placement selectors never offer, so the scheduler's own check never sees it. The **health monitor** reads it on every cycle, which is what covers the case the heartbeat cannot see. A worker's heartbeat says its supervisor is alive; it says nothing about whether anything here can reach that worker's backends, because those are reached over the tunnel. A worker that heartbeats with a permanently dead tunnel (a proxy that stopped upgrading WebSockets, a rotated registration credential, a reconnect loop longer than the grace) is therefore marked unhealthy too, rather than staying listed healthy while every request for a model loaded on it fails "no route to that worker". -The other three answers place work as normal. **Reconnecting** (the tunnel went inside the grace) and **unknown** (no connection row at all — the worker has never dialled, or its departure has already aged out of retention) are both non-verdicts; so is a failure to read the answer, because a database hiccup that excluded workers would cost the fleet its capacity for a reason that has nothing to do with any worker. In each of those cases scheduling proceeds and the install that follows reports its own outcome. +The log lines are `Scheduled node has no tunnel and its departure outlived the reconnect grace, marking unhealthy and re-scheduling`, `Eviction target has no tunnel and its departure outlived the reconnect grace, marking unhealthy and evicting again`, and `Node is heartbeating but its tunnel has been gone longer than the reconnect grace; marking unhealthy`. Each names the grace it used. + +**The demotion is a status change, not a deletion.** The worker's `node_models` rows survive it. Status is enough to unwedge the node, because request routing and LRU eviction both select only healthy nodes, so the model stops being served from there and the next request places it somewhere reachable. Deleting rows on a presence read would give any future defect in that read the widest possible blast radius, for nothing the demotion does not already deliver. + +Status is a *trailing* signal, though: it is only as fresh as the last health cycle. That is why the two paths that are about to commit work to a node, cold-load placement and eviction, read presence directly instead of trusting the status column. Everything else reads the column. + +**A returning heartbeat does not promote a node back on its own.** Recovery needs the *tunnel* back, not just the supervisor: the health monitor re-promotes a demoted node only once presence reports it connected or reconnecting again. Before that check the two were conflated, and a heartbeating worker with a dead tunnel was promoted back to healthy on the next 15s cycle, every cycle. + +The other three answers place work as normal. **Reconnecting** (the tunnel went inside the grace) and **unknown** (no connection row at all: the worker has never dialled, or its departure has already aged out of retention) are both non-verdicts; so is a failure to read the answer, because a database hiccup that excluded workers would cost the fleet its capacity for a reason that has nothing to do with any worker. In each of those cases scheduling proceeds, the node keeps its status, and the install that follows reports its own outcome. + +A frontend refuses to start if either reader was built without a source of absence, because that failure has no other symptom: it looks exactly like a fleet that is fine. | Flag | Env var | Default | Description | |------|---------|---------|-------------| @@ -1552,7 +1562,7 @@ Notes: - It is **not** the same as the worker being gone, and nothing acts on it as if it were. A model on an unroutable worker is not reaped, its rows are left alone, and the node is not demoted: doing any of those on a lost route is how a rolling frontend restart turns into a fleet-wide eviction. - Check the worker process is running and that it has an open tunnel (`opened a tunnelled stream to a worker` in the frontend log, and the worker's own dial/reconnect lines). A worker behind a load balancer that keeps reconnecting is usually an idle-timeout or WebSocket-upgrade problem at the proxy; see the tunnel section above. - **`no route` is not `gone`, and nothing in the frontend reads it as such.** A worker is declared **gone** by one mechanism only: no live frontend replica holds its tunnel *and* its departure is older than `--worker-reconnect-grace`. That is a fact recorded in the shared database, so every replica answers it identically. "No route" is one replica failing to reach a worker right now, and it is not evidence about the worker at all. -- Older releases decided absence from `nats: no responders available for request` — one frontend's observation that nobody answered *it* within a request budget. Two replicas asking at the same moment could disagree and demote each other's workers. That signal is gone from the scheduler; if you still see the message, it concerns only the subjects that remain on the bus (agent-worker jobs and MCP), never a serve-backend worker. +- Older releases decided absence from `nats: no responders available for request`, which was one frontend's observation that nobody answered *it* within a request budget. Two replicas asking at the same moment could disagree and demote each other's workers. That signal is gone from the scheduler; if you still see the message, it concerns only the subjects that remain on the bus (agent-worker jobs and MCP), never a serve-backend worker. **A worker fills its own disk over time:** - A request that carries a file (an image, an audio clip, a video) stages that file below the worker's HTTP staging or S3 cache `ephemeral/` directory. The frontend releases each request-owned input when inference finishes, and the worker reserves capacity before accepting it. From 52c099ef0e1db83b7cc253deb4377d5df5f92a99 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 01:14:29 +0000 Subject: [PATCH 078/153] feat(distributed): take the backend worker off NATS entirely A local-ai worker no longer opens a bus connection. connectNATS and its spec are gone; Run registers once, starts its tunnel, arms /readyz on that tunnel, and heartbeats. The worker's bus credential flags (--nats-jwt, --nats-user-seed, --nats-require-auth, the three TLS flags) and Config.NatsAuthRequired go with it. --nats-url stays, accepted and ignored, so an existing worker command line still parses. /readyz was the thing most likely to wedge a tunnel-only worker: it required a live NATS link, so a worker with no bus would have reported itself unready forever. nodes.NATSReadiness becomes nodes.TunnelReadiness over a local interface{ Connected() bool }, and worker.Tunnel gains Connected(), backed by a mutex-guarded session field the loop publishes and clears. A closed-but-not-yet-cleared session reads as disconnected: the loop waits for every in-flight stream before it clears the field, and the probe must answer not-ready through that wait. The heartbeat gate is DELETED rather than re-pointed at the tunnel. The heartbeat is the worker's own answer that its process is alive; whether the frontend can reach it is a separate fact the frontend already holds and ages against LOCALAI_WORKER_RECONNECT_GRACE. Withholding the heartbeat would report an unreachable worker as an absent one on the one path with no grace, where the health monitor marks it offline and its pending backend ops are deleted behind it. heartbeatLoop is given no view of the tunnel, so a gate cannot be added back without changing its signature. Removing the NATS credential manager from this path also removes a defect it carried: its refresh loop re-registered on a timer to renew a JWT, and Register CLEARS a node's NodeModel rows. Any backend worker running on frontend-minted credentials had its replica rows deleted roughly every 18 hours. Of core/cli/workerregistry, everything survives. The manager is still used in full by core/cli/agent_worker.go, which still needs NATS: Acquire, Provider, RefreshLoop, HasCredentials and TunnelToken are all untouched. The backend worker simply calls RegisterFullWithRetry directly now. WorkerPermissions is documented as serving agent nodes, and its non-agent branch narrowed to _INBOX.> on both sides. It is NOT deleted: NATS reads an empty allow list as no restriction, so returning nil would upgrade every JWT the frontend still mints for a backend node from its own inbox to the whole account. Agent workers keep the bus everywhere: their CLI flags, their subscriptions, the agent branch of WorkerPermissions, and the compose service with its LOCALAI_NATS_URL and depends_on: nats. Also corrected two flags the Nodes page advertised that do not exist (--distributed-nats, --distributed-db), and a log line plus several comments that still named a bus the code no longer touches. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/http/react-ui/e2e/nodes-roster.spec.js | 45 ++++ core/http/react-ui/src/pages/Nodes.jsx | 8 +- core/services/nodes/worker_readiness.go | 53 ++-- core/services/nodes/worker_readiness_test.go | 38 +-- core/services/worker/addr_test.go | 19 +- core/services/worker/auth_required_test.go | 14 +- core/services/worker/config.go | 60 +++-- core/services/worker/heartbeat_test.go | 88 +++++++ core/services/worker/nats_connect.go | 33 --- core/services/worker/nats_connect_test.go | 29 --- core/services/worker/prefetch.go | 14 +- core/services/worker/supervisor.go | 6 +- core/services/worker/tunnel.go | 41 ++++ core/services/worker/tunnel_test.go | 136 ++++++++++ core/services/worker/worker.go | 246 ++++++++----------- docker-compose.distributed.yaml | 14 +- docs/content/features/distributed-mode.md | 55 ++--- docs/content/reference/cli-reference.md | 4 +- pkg/natsauth/mint_test.go | 7 +- pkg/natsauth/permissions.go | 36 ++- pkg/natsauth/permissions_coverage_test.go | 32 ++- tests/e2e/distributed/cluster/cluster.go | 6 +- tests/e2e/distributed/nats_jwt_test.go | 23 +- 23 files changed, 631 insertions(+), 376 deletions(-) create mode 100644 core/services/worker/heartbeat_test.go delete mode 100644 core/services/worker/nats_connect.go delete mode 100644 core/services/worker/nats_connect_test.go diff --git a/core/http/react-ui/e2e/nodes-roster.spec.js b/core/http/react-ui/e2e/nodes-roster.spec.js index c90587eb25fb..2ef2a1dc1d50 100644 --- a/core/http/react-ui/e2e/nodes-roster.spec.js +++ b/core/http/react-ui/e2e/nodes-roster.spec.js @@ -43,3 +43,48 @@ test.describe('Nodes fleet roster', () => { }) } }) + +test.describe('Nodes join command', () => { + // The panel emits BOTH the backend and the agent join command from one + // component, so the bus flag has to differ per tab rather than be deleted. + // Backend workers connect to no NATS server; agent workers still do. + test('omits the NATS flag for a backend worker and keeps it for an agent worker', async ({ page }) => { + await mockCluster(page, []) + await page.goto('/app/nodes') + + await page.getByRole('radio', { name: /^Backend$/ }).click() + const backendCli = page.locator('.p2p-cmd pre').first() + await expect(backendCli).toContainText('local-ai worker', { timeout: 15_000 }) + await expect(backendCli).not.toContainText('--nats-url') + const backendDocker = page.locator('.p2p-cmd pre').nth(1) + await expect(backendDocker).not.toContainText('LOCALAI_NATS_URL') + + await page.getByRole('radio', { name: /^Agent$/ }).click() + const agentCli = page.locator('.p2p-cmd pre').first() + await expect(agentCli).toContainText('local-ai agent-worker', { timeout: 15_000 }) + await expect(agentCli).toContainText('--nats-url') + const agentDocker = page.locator('.p2p-cmd pre').nth(1) + await expect(agentDocker).toContainText('LOCALAI_NATS_URL') + }) + + test('does not advertise flags the CLI does not have', async ({ page }) => { + // The "How to Enable Distributed Mode" card renders ONLY on the disabled + // state, which the page enters when /api/nodes answers 503. Mocking a + // healthy cluster here would assert absence against a card that was never + // on the page. + await page.route('**/api/nodes', r => r.fulfill({ status: 503, contentType: 'application/json', body: '{}' })) + await page.route('**/api/nodes/models', r => r.fulfill({ status: 503, contentType: 'application/json', body: '{}' })) + await page.route('**/api/nodes/scheduling', r => r.fulfill({ status: 503, contentType: 'application/json', body: '{}' })) + await page.goto('/app/nodes') + + const card = page.locator('.p2p-enable') + await expect(card).toBeVisible({ timeout: 15_000 }) + // --distributed-nats and --distributed-db were never real flags; a copied + // command carrying them fails at kong before LocalAI does anything. + await expect(card).not.toContainText('--distributed-nats') + await expect(card).not.toContainText('--distributed-db') + // And the worker step no longer tells an operator to point a backend + // worker at a bus it does not dial. + await expect(card.locator('.p2p-cmd pre').nth(1)).not.toContainText('--nats-url') + }) +}) diff --git a/core/http/react-ui/src/pages/Nodes.jsx b/core/http/react-ui/src/pages/Nodes.jsx index a3a83106f328..ec2209cef1ea 100644 --- a/core/http/react-ui/src/pages/Nodes.jsx +++ b/core/http/react-ui/src/pages/Nodes.jsx @@ -27,6 +27,8 @@ function WorkerHintCard({ addToast, nodeType = 'backend', hasWorkers }) { const { selected, setSelected, option, dev, setDev } = useImageSelector('cpu') const isAgent = nodeType === 'agent' const workerCmd = isAgent ? 'agent-worker' : 'worker' + const natsFlag = isAgent ? ' --nats-url "nats://nats:4222" \\\n' : '' + const natsEnv = isAgent ? ' -e LOCALAI_NATS_URL="nats://nats:4222" \\\n' : '' const flags = dockerFlags(option) const flagsString = flags ? `${flags} \ ` : '' @@ -39,12 +41,10 @@ function WorkerHintCard({ addToast, nodeType = 'backend', hasWorkers }) {

CLI

+${natsFlag} --registration-token "$LOCALAI_REGISTRATION_TOKEN"`} addToast={addToast} />

Docker

diff --git a/core/services/nodes/worker_readiness.go b/core/services/nodes/worker_readiness.go index a0d5b3a19490..f9e5710ba727 100644 --- a/core/services/nodes/worker_readiness.go +++ b/core/services/nodes/worker_readiness.go @@ -9,7 +9,7 @@ import ( // WorkerReadiness is the gate behind a worker's /readyz probe. // // It exists because the worker's HTTP file-transfer server is started before -// the worker has connected to NATS, and must keep serving after NATS drops. +// the worker's tunnel is up, and must keep serving after that tunnel drops. // The probe is therefore installed after the fact rather than passed as a // value, and must be safe to read from HTTP handler goroutines while the // startup goroutine is still installing it. @@ -40,36 +40,43 @@ func (r *WorkerReadiness) Check() error { return (*fn)() } -// natsConn is the slice of *messaging.Client the readiness probe needs. Kept -// as a local interface so this package does not import messaging (which would -// be an import cycle) and so tests can supply a fake. -type natsConn interface { - IsConnected() bool +// tunnelConn is the slice of *worker.Tunnel the readiness probe needs. Kept as +// a local interface so this package does not import the worker package (which +// would be an import cycle) and so tests can supply a fake. +type tunnelConn interface { + Connected() bool } -// ErrNATSDisconnected is reported by NATSReadiness when the worker has lost its -// NATS connection. -var ErrNATSDisconnected = errors.New("NATS connection is down: worker cannot receive work") +// ErrTunnelDisconnected is reported by TunnelReadiness when the worker holds no +// tunnel session. +var ErrTunnelDisconnected = errors.New("worker tunnel is down: the frontend cannot reach this worker") -// NATSReadiness builds the worker's readiness probe. +// TunnelReadiness builds the worker's readiness probe. // // A worker's real health is not "a port is open" — that is precisely the // failure mode of issue #10987, where a process that serves nothing still -// answered 200. All of a worker's actual work (backend install/start/stop -// events, inference dispatch, file-staging notifications) arrives over NATS, so -// a worker with a dead NATS link is up and useless. Registration is already -// implied by the probe being reachable at all: the file-transfer server is only -// started after the worker has successfully registered with the frontend. +// answered 200. All of a worker's actual work (backend install/start/stop, +// model lifecycle, file staging, and every inference stream) arrives over its +// tunnel to the frontend, so a worker with no tunnel session is up and +// unreachable. It binds only loopback and advertises no address, so there is no +// second way in. Registration is already implied by the probe being reachable +// at all: the file-transfer server is only started after the worker has +// successfully registered with the frontend. // -// This is deliberately something the controller cannot already see. The node -// registry's status/last_heartbeat is fed by an HTTP heartbeat to the frontend, -// a completely different network path — a worker can keep heartbeating happily -// while its NATS connection is dead, and look healthy in the registry. The -// local probe closes that gap. -func NATSReadiness(conn natsConn) func() error { +// This is deliberately something the LOCAL supervisor cannot already see. The +// node registry's status/last_heartbeat is fed by an HTTP heartbeat to the +// frontend, a completely different network path, so a worker can keep +// heartbeating happily while its tunnel is dead and look healthy in the +// registry. The local probe closes that gap. +// +// It is a readiness answer and nothing more. The frontend decides whether a +// worker is GONE from the tunnel session it holds, aged against +// LOCALAI_WORKER_RECONNECT_GRACE; a 503 here is one container's own report that +// it cannot serve right now. +func TunnelReadiness(conn tunnelConn) func() error { return func() error { - if conn == nil || !conn.IsConnected() { - return ErrNATSDisconnected + if conn == nil || !conn.Connected() { + return ErrTunnelDisconnected } return nil } diff --git a/core/services/nodes/worker_readiness_test.go b/core/services/nodes/worker_readiness_test.go index 6771291ced47..7b6f6fb8b28d 100644 --- a/core/services/nodes/worker_readiness_test.go +++ b/core/services/nodes/worker_readiness_test.go @@ -10,11 +10,11 @@ import ( . "github.com/onsi/gomega" ) -// fakeConn stands in for *messaging.Client, which cannot be constructed without -// a live NATS server. Only IsConnected() is consulted by the readiness probe. -type fakeConn struct{ connected bool } +// fakeTunnel stands in for *worker.Tunnel, which this package cannot import +// (worker imports nodes). Only Connected() is consulted by the readiness probe. +type fakeTunnel struct{ connected bool } -func (f *fakeConn) IsConnected() bool { return f.connected } +func (f fakeTunnel) Connected() bool { return f.connected } var _ = Describe("WorkerReadiness", func() { Describe("the gate itself", func() { @@ -39,16 +39,24 @@ var _ = Describe("WorkerReadiness", func() { }) }) - Describe("NATSReadiness", func() { - It("reports ready while the NATS connection is up", func() { - Expect(NATSReadiness(&fakeConn{connected: true})()).To(Succeed()) + Describe("TunnelReadiness", func() { + It("reports ready once the tunnel holds a session", func() { + Expect(TunnelReadiness(fakeTunnel{connected: true})()).To(Succeed()) }) - It("reports not-ready once the NATS connection drops", func() { + It("reports not ready while the tunnel holds no session", func() { // This is the failure mode issue #10987 is about: the process is - // up and the port is bound, but the worker can receive no work. - err := NATSReadiness(&fakeConn{connected: false})() - Expect(err).To(MatchError(ContainSubstring("NATS"))) + // up and the port is bound, but nothing can reach this worker, + // because every request the frontend makes of it arrives over the + // tunnel. + Expect(TunnelReadiness(fakeTunnel{connected: false})()).To(MatchError(ErrTunnelDisconnected)) + }) + + It("reports not ready for a nil tunnel rather than panicking", func() { + // Run installs the probe after StartTunnel, so a nil here means a + // wiring mistake. Reporting it beats taking the HTTP handler + // goroutine down with it. + Expect(TunnelReadiness(nil)()).To(MatchError(ErrTunnelDisconnected)) }) }) @@ -86,15 +94,15 @@ var _ = Describe("WorkerReadiness", func() { }) It("serves /readyz 503 once the probe reports not-ready", func() { - ready.Set(func() error { return errors.New("NATS disconnected") }) + ready.Set(func() error { return errors.New("tunnel disconnected") }) Expect(get("/readyz")).To(Equal(http.StatusServiceUnavailable)) }) It("keeps /healthz at 200 even when readiness fails", func() { // Liveness is deliberately independent of readiness: a worker whose - // NATS link is briefly down must not be killed and restarted, or a - // NATS outage turns into a restart storm across every worker. - ready.Set(func() error { return errors.New("NATS disconnected") }) + // tunnel is briefly down must not be killed and restarted, or one + // frontend restart turns into a restart storm across every worker. + ready.Set(func() error { return errors.New("tunnel disconnected") }) Expect(get("/healthz")).To(Equal(http.StatusOK)) }) }) diff --git a/core/services/worker/addr_test.go b/core/services/worker/addr_test.go index 447880653c83..10bdfc35bd13 100644 --- a/core/services/worker/addr_test.go +++ b/core/services/worker/addr_test.go @@ -93,15 +93,30 @@ var _ = Describe("Worker address resolution", func() { var _ = Describe("Worker startup validation", func() { // A Config as kong would hand it over with nothing unusual set: the tunnel - // on by its default, no auth enforcement. + // on by its default, the required frontend URL present, no auth + // enforcement. Every case below starts here and changes ONE thing, so a + // refusal it asserts is the clause it names and not an earlier one. newConfig := func() *Config { - return &Config{WorkerTunnel: true} + return &Config{WorkerTunnel: true, RegisterTo: "http://frontend:8080"} } It("accepts the default configuration", func() { Expect(newConfig().validateStartup()).To(Succeed()) }) + It("starts a backend worker with no NATS URL", func() { + // The point of this phase: a backend worker's work arrives over its + // tunnel, so a bus address is no longer part of its startup contract. + // newConfig sets none, and there is no longer a field to set. + Expect(newConfig().validateStartup()).To(Succeed()) + }) + + It("still refuses a worker with no frontend URL", func() { + cfg := newConfig() + cfg.RegisterTo = "" + Expect(cfg.validateStartup()).To(MatchError(ContainSubstring("LOCALAI_REGISTER_TO"))) + }) + It("refuses to start with the tunnel turned off", func() { // Not a warning and not a degraded mode. A worker without its tunnel // advertises nothing, binds only loopback, and has no frontend path diff --git a/core/services/worker/auth_required_test.go b/core/services/worker/auth_required_test.go index ff1deba5caaa..937a52928821 100644 --- a/core/services/worker/auth_required_test.go +++ b/core/services/worker/auth_required_test.go @@ -5,18 +5,10 @@ import ( . "github.com/onsi/gomega" ) +// The umbrella switch used to imply NATS auth as well. A backend worker no +// longer authenticates to a bus, so registration auth is all it still implies, +// and this is the only helper left to keep honest. var _ = Describe("Worker auth-required helpers", func() { - DescribeTable("NatsAuthRequired", - func(nats, umbrella, want bool) { - cfg := &Config{NatsRequireAuth: nats, DistributedRequireAuth: umbrella} - Expect(cfg.NatsAuthRequired()).To(Equal(want)) - }, - Entry("neither", false, false, false), - Entry("granular only", true, false, true), - Entry("umbrella only", false, true, true), - Entry("both", true, true, true), - ) - DescribeTable("RegistrationAuthRequired", func(reg, umbrella, want bool) { cfg := &Config{RegistrationRequireAuth: reg, DistributedRequireAuth: umbrella} diff --git a/core/services/worker/config.go b/core/services/worker/config.go index 8402a4ce19cc..b3950f1c5808 100644 --- a/core/services/worker/config.go +++ b/core/services/worker/config.go @@ -8,15 +8,11 @@ import "fmt" // which embeds Config; this package does NOT import kong and the tags are inert // here. // -// Workers are backend-agnostic — they wait for backend.install NATS events -// from the SmartRouter to install and start the required backend. -// -// NATS is required. The worker acts as a process supervisor: -// - Receives backend.install → installs backend from gallery, starts gRPC process, replies success -// - Receives backend.stop → stops the gRPC process -// - Receives stop → full shutdown (deregister + exit) -// -// Model loading (LoadModel) is always via direct gRPC — no NATS needed for that. +// Workers are backend-agnostic: they install and start whichever backend the +// frontend asks for. The worker acts as a process supervisor, and every verb +// the frontend gives it is an HTTP route on its own loopback server, served to +// the frontend through this worker's outbound tunnel (see control_routes.go and +// core/services/workerctl). A backend worker connects to no message bus. type Config struct { // Addr and ServeAddr are read for their PORT only. A worker binds nothing // on a routable interface: backend processes and the file-transfer server @@ -49,8 +45,7 @@ type Config struct { // cluster-internal path is slow (slirp/circuit-relay, CGNAT) but outbound NAT // works fine. Resolution reuses the same gallery installer the master uses, so // the on-disk /models layout is identical. Errors are non-fatal — if the gallery - // is unreachable on boot, the worker logs a warning and starts the NATS loop - // anyway; the master can still push the file on demand (existing behaviour). + // is unreachable on boot, the worker logs a warning and starts anyway; the master can still push the file on demand (existing behaviour). PrefetchModels []string `env:"LOCALAI_PREFETCH_MODELS,PREFETCH_MODELS" help:"Comma-separated gallery model IDs to download from LOCALAI_GALLERIES at worker boot (e.g. 'llama-3.2-1b-instruct,phi-3-mini-4k'). Skipped if already on disk and SHA matches." group:"server"` // HTTPAddr binds the HTTP file-transfer server. Default is loopback on @@ -64,7 +59,7 @@ type Config struct { NodeName string `env:"LOCALAI_NODE_NAME" help:"Node name for registration (defaults to hostname)" group:"registration"` RegistrationToken string `env:"LOCALAI_REGISTRATION_TOKEN" help:"Token for authenticating with the frontend" group:"registration"` RegistrationRequireAuth bool `env:"LOCALAI_REGISTRATION_REQUIRE_AUTH" default:"false" help:"Refuse to start the HTTP file-transfer server when no registration token is set (otherwise it fails open and serves read/write to models/staging/data unauthenticated)" group:"registration"` - DistributedRequireAuth bool `env:"LOCALAI_DISTRIBUTED_REQUIRE_AUTH" default:"false" help:"Umbrella switch implying both --nats-require-auth and --registration-require-auth" group:"distributed"` + DistributedRequireAuth bool `env:"LOCALAI_DISTRIBUTED_REQUIRE_AUTH" default:"false" help:"Umbrella switch implying --registration-require-auth" group:"distributed"` HeartbeatInterval string `env:"LOCALAI_HEARTBEAT_INTERVAL" default:"10s" help:"Interval between heartbeats" group:"registration"` // WorkerTunnel holds one outbound multiplexed connection to the frontend // and serves the frontend's requests over it, so the worker needs no @@ -99,14 +94,14 @@ type Config struct { // enforces it against the raw VRAM this worker reports. Empty = no cap. VRAMBudget string `env:"LOCALAI_VRAM_BUDGET" help:"Cap VRAM used for model allocation on this worker node, as a percentage (e.g. 80%) or absolute amount (e.g. 12GB)." group:"registration"` - // NATS (required) - NatsURL string `env:"LOCALAI_NATS_URL" required:"" help:"NATS server URL" group:"distributed"` - NatsJWT string `env:"LOCALAI_NATS_JWT" help:"NATS user JWT override (normally from registration nats_jwt)" group:"distributed"` - NatsUserSeed string `env:"LOCALAI_NATS_USER_SEED" help:"NATS user signing seed override (normally from registration nats_user_seed)" group:"distributed"` - NatsRequireAuth bool `env:"LOCALAI_NATS_REQUIRE_AUTH" default:"false" help:"Require NATS JWT+seed from registration or env" group:"distributed"` - NatsTLSCA string `env:"LOCALAI_NATS_TLS_CA" type:"existingfile" help:"PEM file for NATS server CA (private PKI)" group:"distributed"` - NatsTLSCert string `env:"LOCALAI_NATS_TLS_CERT" type:"existingfile" help:"Client certificate for NATS mTLS" group:"distributed"` - NatsTLSKey string `env:"LOCALAI_NATS_TLS_KEY" type:"existingfile" help:"Client private key for NATS mTLS" group:"distributed"` + // NatsURL is accepted and ignored. A backend worker no longer connects to + // NATS at all, and the credential and TLS flags that went with it are gone + // from this command. This one is kept so that an operator whose worker + // command line or unit file still carries --nats-url gets a worker that + // starts, rather than a kong parse error on an upgrade whose whole point is + // that the bus is no longer needed here. The frontend and agent workers + // still take it and still mean it. + NatsURL string `env:"LOCALAI_NATS_URL" help:"Ignored. A backend worker connects to no message bus; the frontend reaches it over its outbound tunnel. Accepted so an existing worker command line still starts." group:"distributed" hidden:""` // S3 storage for distributed file transfer StorageURL string `env:"LOCALAI_STORAGE_URL" help:"S3 endpoint URL" group:"distributed"` @@ -116,12 +111,6 @@ type Config struct { StorageSecretKey string `env:"LOCALAI_STORAGE_SECRET_KEY" help:"S3 secret key" group:"distributed"` } -// NatsAuthRequired reports whether NATS JWT credentials must be present — the -// granular flag or the umbrella (LOCALAI_DISTRIBUTED_REQUIRE_AUTH). -func (c Config) NatsAuthRequired() bool { - return c.NatsRequireAuth || c.DistributedRequireAuth -} - // RegistrationAuthRequired reports whether a registration token must be set // before the file-transfer server may start — the granular flag or the umbrella. func (c Config) RegistrationAuthRequired() bool { @@ -131,13 +120,20 @@ func (c Config) RegistrationAuthRequired() bool { // validateStartup reports a configuration this worker must refuse to boot on, // as opposed to one it can degrade under. // -// It runs before prefetch, registration and NATS, so a refusal happens while -// the worker is still invisible to the cluster. That ordering is the point of -// checking here at all: both conditions below produce a worker that would -// register, heartbeat and be scheduled onto, so discovering them later means -// discovering them as failed inferences on a node the frontend believes is -// healthy. +// It runs before prefetch and registration, so a refusal happens while the +// worker is still invisible to the cluster. That ordering is the point of +// checking here at all: these conditions produce a worker that would register, +// heartbeat and be scheduled onto, so discovering them later means discovering +// them as failed inferences on a node the frontend believes is healthy. func (c Config) validateStartup() error { + // kong marks --register-to required, so a worker started from the CLI + // cannot miss it. Checked again here because this is the fail-fast site the + // other startup refusals live at, and because RegisterTo is now load + // bearing twice over: it is where the worker registers AND the endpoint its + // tunnel dials, which is the only way anything reaches it. + if c.RegisterTo == "" { + return fmt.Errorf("no frontend URL: set LOCALAI_REGISTER_TO (or --register-to). It is where this worker registers and the endpoint its tunnel dials, and nothing can reach a worker without it") + } // The file-transfer server fails open on an empty token (see // nodes.checkBearerToken), so enforcement plus no token is a request to // serve the models directory unauthenticated. diff --git a/core/services/worker/heartbeat_test.go b/core/services/worker/heartbeat_test.go new file mode 100644 index 000000000000..5153183adb68 --- /dev/null +++ b/core/services/worker/heartbeat_test.go @@ -0,0 +1,88 @@ +package worker + +import ( + "context" + "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Worker heartbeat loop", func() { + var ( + ctx context.Context + cancel context.CancelFunc + tick chan time.Time + sent chan struct{} + done chan struct{} + ) + + BeforeEach(func() { + ctx, cancel = context.WithCancel(context.Background()) + tick = make(chan time.Time) + sent = make(chan struct{}, 8) + done = make(chan struct{}) + }) + + AfterEach(func() { + cancel() + Eventually(done, "5s").Should(BeClosed()) + }) + + // tick delivers one tick, and fails the spec rather than parking forever if + // the loop has stopped reading. A loop that returned early would otherwise + // hang the suite instead of reporting a failure. + fire := func() { + select { + case tick <- time.Now(): + case <-time.After(5 * time.Second): + Fail("the heartbeat loop stopped reading its ticker") + } + } + + run := func(send func(context.Context) error) { + go func() { + defer close(done) + heartbeatLoop(ctx, tick, send) + }() + } + + It("posts a heartbeat on every tick", func() { + run(func(context.Context) error { + sent <- struct{}{} + return nil + }) + for i := 0; i < 3; i++ { + fire() + Eventually(sent, "5s").Should(Receive(), "tick %d produced no heartbeat", i+1) + } + }) + + // The heartbeat is the worker's own answer that its process is alive, and + // the frontend reads absence from the tunnel it holds, aged against the + // reconnect grace. A loop that gave up on a failing post would let one + // frontend restart silence a worker for the rest of its life, and the + // health monitor marks a silent node offline with no grace at all. + It("keeps posting after a heartbeat fails", func() { + run(func(context.Context) error { + sent <- struct{}{} + return errors.New("frontend unreachable") + }) + for i := 0; i < 3; i++ { + fire() + Eventually(sent, "5s").Should(Receive(), "tick %d produced no heartbeat after a failure", i+1) + } + }) + + It("stops once the shutdown context is cancelled", func() { + run(func(context.Context) error { + sent <- struct{}{} + return nil + }) + fire() + Eventually(sent, "5s").Should(Receive()) + cancel() + Eventually(done, "5s").Should(BeClosed()) + }) +}) diff --git a/core/services/worker/nats_connect.go b/core/services/worker/nats_connect.go deleted file mode 100644 index 25485701d2ec..000000000000 --- a/core/services/worker/nats_connect.go +++ /dev/null @@ -1,33 +0,0 @@ -package worker - -import ( - "fmt" - - "github.com/mudler/LocalAI/core/services/messaging" -) - -// connectNATS opens a NATS client using JWT+seed from env or registration (env wins). -func connectNATS(url, envJWT, envSeed, registerJWT, registerSeed string, requireAuth bool, tls messaging.TLSFiles) (*messaging.Client, error) { - // Env credentials take precedence, but only fall back to registration when - // the env supplied neither half — otherwise a JWT set without its seed (or - // vice-versa) would be silently completed from a different source. - jwt, seed := envJWT, envSeed - if jwt == "" && seed == "" { - jwt, seed = registerJWT, registerSeed - } - // A JWT without its paired seed (or vice-versa) is a misconfiguration: refuse - // rather than silently connecting anonymously, which would look authenticated. - if (jwt == "") != (seed == "") { - return nil, fmt.Errorf("NATS JWT and seed must be provided together (got JWT set=%t, seed set=%t)", jwt != "", seed != "") - } - var opts []messaging.Option - if jwt != "" && seed != "" { - opts = append(opts, messaging.WithUserJWT(jwt, seed)) - } else if requireAuth { - return nil, fmt.Errorf("NATS JWT+seed required: set LOCALAI_NATS_JWT/LOCALAI_NATS_USER_SEED or enable frontend minting") - } - if tls.Enabled() { - opts = append(opts, messaging.WithTLS(tls)) - } - return messaging.New(url, opts...) -} diff --git a/core/services/worker/nats_connect_test.go b/core/services/worker/nats_connect_test.go deleted file mode 100644 index 8f554de4e944..000000000000 --- a/core/services/worker/nats_connect_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package worker - -import ( - "github.com/mudler/LocalAI/core/services/messaging" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("connectNATS", func() { - It("requires JWT when requireAuth is set and no credentials are provided", func() { - _, err := connectNATS("nats://127.0.0.1:4222", "", "", "", "", true, messaging.TLSFiles{}) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("NATS JWT+seed required")) - }) - - // A JWT supplied without its paired seed (or vice-versa) is an operator - // misconfiguration. Today connectNATS silently drops the unpaired credential - // and connects anonymously, so the operator believes the link is - // authenticated when it is not. It should refuse instead. - It("rejects a JWT supplied without a seed instead of connecting anonymously", func() { - client, err := connectNATS("nats://127.0.0.1:4222", "jwt-without-seed", "", "", "", false, messaging.TLSFiles{}) - if client != nil { - client.Close() - } - Expect(err).To(HaveOccurred(), - "connectNATS should reject an unpaired JWT rather than silently connecting anonymously") - }) -}) diff --git a/core/services/worker/prefetch.go b/core/services/worker/prefetch.go index 4aec36a99de8..b62effd74a7f 100644 --- a/core/services/worker/prefetch.go +++ b/core/services/worker/prefetch.go @@ -38,8 +38,8 @@ var realModelInstaller modelInstaller = func( ) error { // enforceScan=false: workers fetch from the same gallery the master already // trusts, and the master would have scanned at install time anyway. - // autoloadBackendGalleries=false: the worker installs backends on demand via - // backend.install NATS events; prefetching the backend here would race the + // autoloadBackendGalleries=false: the worker installs backends on demand when + // the frontend calls its install control route; prefetching one here would race the // supervisor's own install path and double-trigger gallery work. // requireBackendIntegrity=false: same reason — we're not installing a backend. return gallery.InstallModelFromGallery( @@ -57,13 +57,15 @@ var realModelInstaller modelInstaller = func( // prefetchModels resolves each configured gallery ID against the model gallery // and downloads the artifact into the worker's /models. It is called once at -// worker startup, BEFORE the NATS lifecycle subscription, so that the steady -// state has the file already on disk and the master never needs to stream it. +// worker startup, BEFORE the worker registers or opens its tunnel, so that the +// steady state has the file already on disk and the master never needs to +// stream it. // // Errors are intentionally non-fatal: on a fresh worker with no outbound // connectivity (or a misconfigured gallery JSON), we want the worker to still // register and serve traffic — the master will fall back to pushing files -// on-demand over NATS/HTTP, which is the pre-existing behavior. Per-model +// on-demand over the worker's file-transfer routes, which is the pre-existing +// behavior. Per-model // failures are logged at warn level and the loop continues with the next ID. // // Idempotency comes for free from pkg/downloader.URI.DownloadFileWithContext: @@ -98,7 +100,7 @@ func prefetchModels( installer = realModelInstaller } - xlog.Info("Prefetching models from gallery before entering NATS loop", "count", len(models), "models", models) + xlog.Info("Prefetching models from gallery before registering", "count", len(models), "models", models) for _, name := range models { xlog.Info("Prefetching model", "model", name) if err := installer(ctx, modelGalleries, backendGalleries, systemState, ml, name); err != nil { diff --git a/core/services/worker/supervisor.go b/core/services/worker/supervisor.go index b3f180c609f8..de273a586a08 100644 --- a/core/services/worker/supervisor.go +++ b/core/services/worker/supervisor.go @@ -177,8 +177,8 @@ type backendSupervisor struct { // reply and drops the NodeModel rows naming that address, a row still resolves // to a live listener. probeHealth verifies liveness, not identity, so a port // re-bound inside that window is dispatched to as if it were the original -// backend. The window is a NATS round-trip plus a row delete, so seconds of -// slack are ample. +// backend. The window is one control-plane round-trip plus a row delete, so +// seconds of slack are ample. // // Deliberately NOT derived from the controller's HealthCheckInterval or from // the per-model miss threshold. Tying a worker-local constant to a @@ -648,7 +648,7 @@ func (s *backendSupervisor) releaseBackendStart(key string, bp *backendProcess) // resolveProcessKeys turns a caller-supplied identifier into the set of // process map keys it refers to. PR #9583 changed s.processes to be keyed by -// `modelID#replicaIndex`, but external NATS handlers still pass the bare +// `modelID#replicaIndex`, but external callers still pass the bare // model ID — without this resolver, those lookups silently no-op'd, so // admin "Unload model" / "Delete backend" left the worker process alive. // diff --git a/core/services/worker/tunnel.go b/core/services/worker/tunnel.go index dab799cb0af7..fb22b3368594 100644 --- a/core/services/worker/tunnel.go +++ b/core/services/worker/tunnel.go @@ -142,6 +142,39 @@ type Tunnel struct { cancel context.CancelFunc done chan struct{} closeOnce sync.Once + + // mu guards session, which is the tunnel's CURRENT session or nil between + // them. It is written by the one loop goroutine and read by whatever asks + // Connected, which on a running worker is an HTTP handler goroutine + // serving /readyz. + mu sync.Mutex + session *yamux.Session +} + +// Connected reports whether the tunnel currently holds a live session. +// +// It is false between sessions and while the first dial is still in flight. +// That is a statement about REACHABILITY and nothing else: a worker whose +// tunnel is re-homing after a frontend restart reports false here and is still +// a registered, running node. Nothing may read it as the worker being gone. +func (t *Tunnel) Connected() bool { + if t == nil { + return false + } + t.mu.Lock() + sess := t.session + t.mu.Unlock() + // A session that has been closed is still the field's value until the loop + // clears it, and the gap between those two is exactly the window a probe + // must not answer 200 in. + return sess != nil && !sess.IsClosed() +} + +// setSession publishes (or clears) the session Connected reports on. +func (t *Tunnel) setSession(sess *yamux.Session) { + t.mu.Lock() + t.session = sess + t.mu.Unlock() } // StartTunnel dials the frontend and holds the tunnel until ctx is cancelled or @@ -257,6 +290,14 @@ func (t *Tunnel) connectAndServe(ctx context.Context) error { } xlog.Info("Worker tunnel established", "node", t.nodeID, "frontend", t.endpoint) + // Published before the accept loop starts. What makes the answer correct + // once the session dies is Connected's own IsClosed check, not this clear: + // the clear runs only after every in-flight stream has finished, which is a + // wait the probe must already be answering "not ready" through. The clear + // is here so a dead session is not held for the life of the reconnect. + t.setSession(sess) + defer t.setSession(nil) + // Streams are served under a context of the SESSION's, not the loop's. A // stream goroutine parked in a local dial would otherwise outlive the // session it belongs to and hold the reconnect below behind it. diff --git a/core/services/worker/tunnel_test.go b/core/services/worker/tunnel_test.go index 6afafaa20189..0930b6f72219 100644 --- a/core/services/worker/tunnel_test.go +++ b/core/services/worker/tunnel_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/nodes" ) // awaitErr runs fn on its own goroutine and reports its result on a channel. @@ -726,6 +727,141 @@ var _ = Describe("Worker tunnel client", func() { Expect(second.nodeID).To(Equal("node-1")) }) }) + + // Connected is what the worker's /readyz reports, so these specs run + // against the real WebSocket and the real yamux handshake rather than a + // flag someone sets. A double that never touches the transport cannot go + // false the way a dropped session does. + Describe("reporting whether it holds a session", func() { + It("reports connected once the frontend has accepted its dial", func() { + frontend = newFakeFrontend(false) + start(nil) + sess := session() + Expect(sess).ToNot(BeNil()) + Eventually(tunnel.Connected, "10s").Should(BeTrue()) + }) + + It("reports disconnected once the session is gone", func() { + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + // Park the reconnect so the spec observes the gap between + // sessions rather than racing the next dial. + c.sleep = func(ctx context.Context, _ time.Duration) error { + <-ctx.Done() + return ctx.Err() + } + }) + sess := session() + Eventually(tunnel.Connected, "10s").Should(BeTrue()) + + Expect(sess.Close()).To(Succeed()) + Eventually(tunnel.Connected, "10s").Should(BeFalse()) + }) + + It("reports disconnected before the first dial has landed", func() { + // The frontend is never started, so nothing can accept. A worker + // that answered ready here would announce itself the moment its + // process came up, which is exactly the 200-on-a-useless-port that + // the readiness probe exists to stop. + frontend = newFakeFrontend(false) + frontend.srv.Close() + start(func(c *TunnelConfig) { + c.sleep = func(ctx context.Context, _ time.Duration) error { + <-ctx.Done() + return ctx.Err() + } + }) + Consistently(tunnel.Connected, "500ms", "50ms").Should(BeFalse()) + }) + + It("reports disconnected while it still holds a session that has been closed", func() { + // The window this closes is real and is not the same as the one + // the loop's own clear closes. When a session dies, the loop waits + // for every stream already in flight before it returns and clears + // the field, and for the whole of that wait the tunnel still HOLDS + // a session that can carry nothing new. Reading only "the field is + // set" would answer ready for the length of that wait. + c1, c2 := net.Pipe() + DeferCleanup(func() { _ = c1.Close(); _ = c2.Close() }) + sess, err := yamux.Client(c1, nil, nil) + Expect(err).ToNot(HaveOccurred()) + + held := &Tunnel{} + held.setSession(sess) + Expect(held.Connected()).To(BeTrue()) + + Expect(sess.Close()).To(Succeed()) + Expect(held.Connected()).To(BeFalse()) + }) + + It("reports disconnected on a nil tunnel rather than panicking", func() { + var absent *Tunnel + Expect(absent.Connected()).To(BeFalse()) + }) + }) + + // The worker's /readyz is armed on the tunnel, and the arming is the kind + // of line whose loss has no symptom: WorkerReadiness fails open, so a + // worker that never armed it answers 200 forever with no session. These + // specs are what makes that line's absence visible. + Describe("arming the readiness gate", func() { + It("answers not ready until the frontend has accepted the dial", func() { + frontend = newFakeFrontend(false) + frontend.srv.Close() + readiness := &nodes.WorkerReadiness{} + var err error + tunnel, err = startTunnelAndArmReadiness(ctx, readiness, TunnelConfig{ + FrontendURL: frontend.srv.URL, + NodeID: "node-1", + Token: func() string { return "tunnel-secret" }, + sleep: func(ctx context.Context, _ time.Duration) error { + <-ctx.Done() + return ctx.Err() + }, + }) + Expect(err).ToNot(HaveOccurred()) + Consistently(readiness.Check, "500ms", "50ms").Should(MatchError(nodes.ErrTunnelDisconnected)) + }) + + It("answers ready once the tunnel holds a session, and not ready again once it goes", func() { + // Both halves, in one spec, on purpose. WorkerReadiness fails open + // when no probe is installed, so "ready once connected" passes just + // as well against a gate that was never armed at all. Only the + // return to ErrTunnelDisconnected tells those two apart. + frontend = newFakeFrontend(false) + readiness := &nodes.WorkerReadiness{} + var err error + tunnel, err = startTunnelAndArmReadiness(ctx, readiness, TunnelConfig{ + FrontendURL: frontend.srv.URL, + NodeID: "node-1", + Token: func() string { return "tunnel-secret" }, + sleep: func(ctx context.Context, _ time.Duration) error { + <-ctx.Done() + return ctx.Err() + }, + }) + Expect(err).ToNot(HaveOccurred()) + sess := session() + Eventually(readiness.Check, "10s").Should(Succeed()) + + Expect(sess.Close()).To(Succeed()) + Eventually(readiness.Check, "10s").Should(MatchError(nodes.ErrTunnelDisconnected)) + }) + + It("leaves the gate alone when the tunnel cannot start at all", func() { + // A configuration refusal is not a readiness answer: Run turns it + // into a fatal error, and a gate armed on a tunnel that does not + // exist would report on nothing. + readiness := &nodes.WorkerReadiness{} + t, err := startTunnelAndArmReadiness(ctx, readiness, TunnelConfig{ + FrontendURL: "http://frontend:8080", + Token: func() string { return "tunnel-secret" }, + }) + Expect(err).To(HaveOccurred()) + Expect(t).To(BeNil()) + Expect(readiness.Check()).To(Succeed()) + }) + }) }) // echoListenerOn is echoListener bound to a specific address, so a spec can put diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index c608faf855c0..e75a97693d1e 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -18,13 +18,11 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/storage" "github.com/mudler/LocalAI/core/services/workerctl" grpc "github.com/mudler/LocalAI/pkg/grpc" "github.com/mudler/LocalAI/pkg/model" - "github.com/mudler/LocalAI/pkg/sanitize" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/xlog" ) @@ -34,7 +32,7 @@ import ( func Run(ctx *cliContext.Context, cfg *Config) error { xlog.Info("Starting worker", "basePort", cfg.effectiveBasePort()) - // Fail fast, before prefetch, registration and NATS, on any configuration + // Fail fast, before prefetch and registration, on any configuration // that would produce a worker the cluster believes in and cannot use. See // validateStartup for what those are and why each is fatal rather than // degraded. @@ -67,7 +65,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error { } // Prefetch gallery models over the worker's outbound internet before we - // start accepting backend.install events. Non-fatal on every failure path: + // serve backend installs. Non-fatal on every failure path: // if the gallery is unreachable, an ID is unknown, or LOCALAI_GALLERIES is // malformed, the worker still starts and the master can push files on // demand (existing fallback behaviour). Placed BEFORE registration so a @@ -89,93 +87,46 @@ func Run(ctx *cliContext.Context, cfg *Config) error { defer shutdownCancel() registrationBody := cfg.registrationBody() - natsTLS := messaging.TLSFiles{CA: cfg.NatsTLSCA, Cert: cfg.NatsTLSCert, Key: cfg.NatsTLSKey} - // Resolve how to connect to NATS. Static env credentials cannot be re-minted, - // so register once and use them directly. Otherwise the credential manager - // (re)registers to obtain credentials — waiting through admin approval — and - // refreshes them before the minted JWT expires, so the connection survives - // expiry via a transparent reconnect. - var ( - nodeID string - connectNats func() (*messaging.Client, error) - // tunnelToken reads the node's CURRENT tunnel credential. It is a - // function because the frontend rotates the credential on every - // registration, so the value a reconnect must present is not - // necessarily the one this worker started with. - tunnelToken func() string - ) - if cfg.NatsJWT != "" || cfg.NatsUserSeed != "" { - res, regErr := regClient.RegisterFullWithRetry(shutdownCtx, registrationBody, 10) - if regErr != nil { - return fmt.Errorf("failed to register with frontend: %w", regErr) - } - nodeID = res.ID - // This path registers exactly once and never again, so the credential - // it holds cannot go stale by rotation from its own side. - // - // It CAN be superseded from outside: Register upserts by NAME, so a - // second worker registering under this node's name rotates the row's - // credential, and this worker then fails every tunnel dial with 401 for - // the life of the process. It logs that once per backoff and never - // recovers on its own; a restart fixes it only until the other worker - // registers again. - // - // Still deliberately not auto-re-registered, and now for a concrete - // reason rather than a deferral. Register CLEARS this node's NodeModel - // rows, on the assumption that a re-registering worker restarted with - // nothing loaded, so re-registering on a 401 would delete a live - // worker's replica rows on every retry, and under the name collision - // that produces the 401 the two workers would take turns doing it - // forever. That is a credential failure causing model reclamation, - // which is the one outcome this whole design exists to prevent. - // - // The fix belongs to whichever comes first: a re-auth path that mints a - // tunnel credential WITHOUT the rest of registration's side effects, or - // a worker identity that is not the operator-chosen name, which is what - // would make a collision detectable instead of silent. Until then the - // 401 is loud, names both causes, and the operator acts on it. - staticTunnelToken := res.TunnelToken - tunnelToken = func() string { return staticTunnelToken } - connectNats = func() (*messaging.Client, error) { - return connectNATS(cfg.NatsURL, cfg.NatsJWT, cfg.NatsUserSeed, "", "", cfg.NatsAuthRequired(), natsTLS) - } - } else { - credMgr := workerregistry.NewNATSCredentialManager( - func(ctx context.Context) (*workerregistry.RegisterResponse, error) { - return regClient.RegisterFull(ctx, registrationBody) - }, - cfg.NatsAuthRequired(), - ) - res, regErr := credMgr.Acquire(shutdownCtx) - if regErr != nil { - return fmt.Errorf("failed to register with frontend: %w", regErr) - } - nodeID = res.ID - // The manager re-registers to refresh NATS credentials, and every - // registration rotates the tunnel credential too, so this reads the - // manager rather than capturing a value. - tunnelToken = credMgr.TunnelToken - connectNats = func() (*messaging.Client, error) { - var opts []messaging.Option - if credMgr.HasCredentials() { - opts = append(opts, messaging.WithUserJWTProvider(credMgr.Provider())) - } - if natsTLS.Enabled() { - opts = append(opts, messaging.WithTLS(natsTLS)) - } - client, cerr := messaging.New(cfg.NatsURL, opts...) - if cerr == nil && credMgr.HasCredentials() { - go func() { - if err := credMgr.RefreshLoop(shutdownCtx); err != nil { - xlog.Error("NATS credential refresh permanently failed; shutting down worker", "error", err) - shutdownCancel() - } - }() - } - return client, cerr - } + // One registration, and the tunnel credential it returns is the only + // credential a backend worker holds. There is no second acquisition path: + // the bus this worker used to also authenticate against is gone from its + // startup entirely. + // + // This path registers exactly once and never again, so the credential it + // holds cannot go stale by rotation from its own side. + // + // It CAN be superseded from outside: Register upserts by NAME, so a second + // worker registering under this node's name rotates the row's credential, + // and this worker then fails every tunnel dial with 401 for the life of the + // process. It logs that once per backoff and never recovers on its own; a + // restart fixes it only until the other worker registers again. + // + // Still deliberately not auto-re-registered, and now for a concrete reason + // rather than a deferral. Register CLEARS this node's NodeModel rows, on + // the assumption that a re-registering worker restarted with nothing + // loaded, so re-registering on a 401 would delete a live worker's replica + // rows on every retry, and under the name collision that produces the 401 + // the two workers would take turns doing it forever. That is a credential + // failure causing model reclamation, which is the one outcome this whole + // design exists to prevent. It is also why the NATS credential manager, + // whose refresh loop re-registered on a timer, is no longer on this path: + // it was wiping a live worker's rows every time it renewed a JWT. + // + // The fix belongs to whichever comes first: a re-auth path that mints a + // tunnel credential WITHOUT the rest of registration's side effects, or a + // worker identity that is not the operator-chosen name, which is what would + // make a collision detectable instead of silent. Until then the 401 is + // loud, names both causes, and the operator acts on it. + res, err := regClient.RegisterFullWithRetry(shutdownCtx, registrationBody, 10) + if err != nil { + return fmt.Errorf("failed to register with frontend: %w", err) } + nodeID := res.ID + // Read through a function because StartTunnel presents the credential at + // DIAL time, not at start time; there is one value behind it today and the + // indirection is what keeps a future rotation from needing a new dial path. + tunnelToken := func() string { return res.TunnelToken } xlog.Info("Registered with frontend", "nodeID", nodeID, "frontend", cfg.RegisterTo) heartbeatInterval, err := time.ParseDuration(cfg.HeartbeatInterval) @@ -208,8 +159,8 @@ func Run(ctx *cliContext.Context, cfg *Config) error { } xlog.Info("Ephemeral staging capacity configured", "roots", ephemeralRoots, "byteLimit", byteLimit, "minFreeBytes", minFreeBytes) StartEphemeralRootsCleanup(shutdownCtx, ephemeralRoots, ephemeralCapacity, 0, 0) - // The readiness gate is created here but only armed once NATS is up and the - // backend supervisor exists, below, because the gate probes both. + // The readiness gate is created here but only armed once the tunnel and the + // backend supervisor exist, below, because the gate probes both. // Until then /readyz reports ready, which is correct: reaching this line // means the worker has already registered with the frontend, so it is // mid-startup rather than broken. @@ -288,7 +239,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error { // before this point, so there is no configuration that reaches here without // one. A guard here would be a branch nothing can take, which reads as a // supported no-tunnel mode that does not exist. - tunnel, terr := StartTunnel(shutdownCtx, TunnelConfig{ + tunnel, terr := startTunnelAndArmReadiness(shutdownCtx, readiness, TunnelConfig{ FrontendURL: cfg.RegisterTo, NodeID: nodeID, Token: tunnelToken, @@ -307,45 +258,17 @@ func Run(ctx *cliContext.Context, cfg *Config) error { } }() - // Connect to NATS - xlog.Info("Connecting to NATS", "url", sanitize.URL(cfg.NatsURL)) - natsClient, err := connectNats() - if err != nil { - nodes.ShutdownFileTransferServer(httpServer) - return fmt.Errorf("connecting to NATS: %w", err) - } - defer natsClient.Close() - - // Start heartbeat goroutine (after NATS is connected so IsConnected check works) - go func() { - ticker := time.NewTicker(heartbeatInterval) - defer ticker.Stop() - for { - select { - case <-shutdownCtx.Done(): - return - case <-ticker.C: - if !natsClient.IsConnected() { - xlog.Warn("Skipping heartbeat: NATS disconnected") - continue - } - body := cfg.heartbeatBody() - if err := regClient.Heartbeat(shutdownCtx, nodeID, body); err != nil { - xlog.Warn("Heartbeat failed", "error", err) - } - } - } - }() + ticker := time.NewTicker(heartbeatInterval) + defer ticker.Stop() + go heartbeatLoop(shutdownCtx, ticker.C, func(ctx context.Context) error { + return regClient.Heartbeat(ctx, nodeID, cfg.heartbeatBody()) + }) - // Arm the readiness gate now that the worker can actually receive work. - // NATS is already connected at this point, so a worker that is up but cut - // off from the bus reports 503 instead of a meaningless 200 (#10987). - // - // Readiness also covers the data path: a worker whose NATS link is fine - // but whose backend processes have died is up and useless, and the - // scheduler cannot tell the difference from the bus alone. + // The tunnel is the control path, while the supervisor covers the local data + // path. A worker is ready only when the frontend can reach it and each + // running backend can still be dialled locally. readiness.Set(nodes.CompositeReadiness( - nodes.NATSReadiness(natsClient), + nodes.TunnelReadiness(tunnel), nodes.BackendDataPathReadiness(supervisor, func(addr string) error { conn, err := net.DialTimeout("tcp", addr, 2*time.Second) if err != nil { @@ -355,23 +278,66 @@ func Run(ctx *cliContext.Context, cfg *Config) error { }), )) xlog.Info("Worker ready, serving its control plane over the tunnel") - // Exit on an OS signal or on an internal fatal condition (e.g. NATS - // credentials became unrenewable), so the worker restarts and re-acquires - // rather than lingering unable to serve. - var runErr error - select { - case <-sigCh: - case <-shutdownCtx.Done(): - runErr = fmt.Errorf("worker shutting down: NATS credentials unavailable") - xlog.Error("Internal shutdown requested", "error", runErr) - } + <-sigCh xlog.Info("Shutting down worker") shutdownCancel() // stop heartbeat loop immediately regClient.GracefulDeregister(nodeID) supervisor.stopAllBackends(false) nodes.ShutdownFileTransferServer(httpServer) - return runErr + return nil +} + +// heartbeatLoop posts this worker's heartbeat on every tick until ctx ends. +// +// It is given no view of the tunnel, and that absence is the point rather than +// an omission. The heartbeat is the WORKER'S OWN ANSWER that its process is +// alive; whether the frontend can REACH this worker is a separate fact, which +// the frontend reads from the tunnel session it holds and ages against +// LOCALAI_WORKER_RECONNECT_GRACE. Withholding the heartbeat while the tunnel +// re-homes would report an unreachable worker as an absent one, on the one path +// that has no grace at all: the health monitor marks a silent node offline or +// unhealthy and its pending backend ops are deleted behind it. +// +// A failed post is likewise not a reason to stop. The frontend being briefly +// unreachable is the exact moment a worker must keep trying, and a loop that +// returned here would silence a healthy worker for the rest of its life after +// one frontend restart. +func heartbeatLoop(ctx context.Context, tick <-chan time.Time, send func(context.Context) error) { + for { + select { + case <-ctx.Done(): + return + case <-tick: + if err := send(ctx); err != nil { + xlog.Warn("Heartbeat failed", "error", err) + } + } + } +} + +// startTunnelAndArmReadiness starts the worker's tunnel and points the +// readiness gate at it. +// +// One call rather than two lines at the call site, because the gate and the +// tunnel are one fact. /readyz means "the frontend can reach me", and a live +// tunnel session is the only thing that makes that true: the worker binds +// loopback, advertises no address, and every request the frontend makes of it +// arrives as a stream inside that session. Armed as a separate statement, the +// arming is a line whose loss has no symptom - the gate fails open, so the +// worker answers 200 forever with no session, which is issue #10987 back +// again and nothing else in the process would say a word. +// +// A tunnel that fails to START leaves the gate as it found it. There is no +// worker to report on: Run turns that into a fatal error before anything else +// happens. +func startTunnelAndArmReadiness(ctx context.Context, readiness *nodes.WorkerReadiness, cfg TunnelConfig) (*Tunnel, error) { + t, err := StartTunnel(ctx, cfg) + if err != nil { + return nil, err + } + readiness.Set(nodes.TunnelReadiness(t)) + return t, nil } // startWorkerHTTPServer starts the worker's loopback HTTP server with sup's diff --git a/docker-compose.distributed.yaml b/docker-compose.distributed.yaml index 937b86534e89..91e4d83fc990 100644 --- a/docker-compose.distributed.yaml +++ b/docker-compose.distributed.yaml @@ -111,9 +111,14 @@ services: # No HEALTHCHECK_ENDPOINT override is needed either: the image's healthcheck # detects worker mode and derives the port from LOCALAI_SERVE_ADDR below # (gRPC base port - 1 = 50050). It runs inside the container, so a loopback - # bind is enough for it. The worker's /readyz reports 503 when its NATS - # connection is down or a backend it serves no longer answers locally. So - # `unhealthy` means the worker cannot receive work or serve a held model. + # bind is enough for it. The worker's /readyz reports 503 while it holds no + # tunnel session or when a backend it serves no longer answers locally, so + # `unhealthy` means the frontend cannot reach this worker or its data path. + # + # No LOCALAI_NATS_URL and no dependency on the nats service: a backend + # worker connects to no bus. Everything the frontend asks of it travels the + # tunnel this container dials out to localai:8080. The frontend and the + # agent worker below still need NATS. environment: LOCALAI_SERVE_ADDR: "0.0.0.0:50051" DEBUG: "true" @@ -121,7 +126,6 @@ services: LOCALAI_NODE_NAME: "worker-1" LOCALAI_REGISTRATION_TOKEN: "changeme" # Must match frontend token LOCALAI_HEARTBEAT_INTERVAL: "10s" - LOCALAI_NATS_URL: "nats://nats:4222" GODEBUG: "netdns=go" # See note in localai service MODELS_PATH: /models volumes: @@ -129,8 +133,6 @@ services: depends_on: localai: condition: service_started - nats: - condition: service_started # --- GPU Support (NVIDIA) --- # Uncomment the following and change the image to a CUDA variant diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 158e9965f23d..56bcad7d427d 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -38,7 +38,7 @@ Each model gets its own gRPC backend process, so a single worker can serve multi ## Prerequisites - **PostgreSQL** (with pgvector extension recommended for RAG) - used for node registry, job store, auth, and shared state -- **NATS** server - used for agent-worker coordination and the frontend's own cross-replica events. Serve-backend workers no longer take any command over it: the backend and model lifecycle verbs and file staging are HTTP routes on the worker's tunnel. +- **NATS** server - used for agent-worker coordination and the frontend's own cross-replica events. **Serve-backend workers do not connect to it at all**: every verb they take, and file staging with it, is an HTTP route on the worker's tunnel. Set no `LOCALAI_NATS_URL` on a `local-ai worker`. The frontend and any `local-ai agent-worker` still need one. - All services must be on the same network (or reachable via configured URLs) ## Quick Start with Docker Compose @@ -308,7 +308,7 @@ Registering against an upgraded frontend **clears** a node's `address` and `http A worker on this release opens **no inbound listener on a routable interface**. Its backend gRPC processes and its HTTP file-transfer server all bind loopback, and the frontend reaches both through the tunnel. Concretely: -- **No inbound firewall rule, published port, Service or Ingress is needed for a worker.** A worker needs outbound access to the frontend URL (`LOCALAI_REGISTER_TO`) and to NATS (`LOCALAI_NATS_URL`), and nothing else. +- **No inbound firewall rule, published port, Service or Ingress is needed for a worker.** A serve-backend worker needs outbound access to the frontend URL (`LOCALAI_REGISTER_TO`), and nothing else - not even to NATS. An agent worker also needs outbound access to `LOCALAI_NATS_URL`. - **`LOCALAI_ADVERTISE_ADDR` and `LOCALAI_ADVERTISE_HTTP_ADDR` are gone.** There is nothing to advertise. Both are ignored if still set; remove them. - **`LOCALAI_ADDR` and `LOCALAI_SERVE_ADDR` are read for their port only.** The port is the base of the backend port range, and `port-1` is the HTTP file-transfer port. The host half names an interface nothing binds. - The node's `address` and `http_address` fields in `GET /api/nodes` are empty, and are cleared for nodes that reported them before the upgrade. @@ -424,7 +424,9 @@ A frontend replica that dies mid-load does not wedge the model: the job row carr ### NATS JWT authentication (recommended for production) -By default, NATS connections are anonymous: any client that can reach port `4222` may publish the subjects still carried on it. Serve-backend workers no longer subscribe to `nodes..backend.install` and its nine siblings - those are HTTP routes on the worker's tunnel now, see [The worker control plane](#the-worker-control-plane) - but agent workers and the frontend's own service credential still use NATS. Enable JWT auth to scope workers to their own node subjects and give the frontend a dedicated service credential. +**This section is about agent workers and the frontend.** A serve-backend worker opens no NATS connection, so none of it applies to one; its own credential is the tunnel token it gets at registration, and its control plane is authenticated by `LOCALAI_REGISTRATION_TOKEN`. + +By default, NATS connections are anonymous: any client that can reach port `4222` may publish the subjects still carried on it. Those are the agent-worker job subjects, MCP, and the frontend's own cross-replica events. `nodes..backend.install` and its nine siblings are **not** among them - they are HTTP routes on the worker's tunnel, see [The worker control plane](#the-worker-control-plane). Enable JWT auth to scope agent workers to their own subjects and give the frontend a dedicated service credential. | Flag | Env Var | Description | |------|---------|-------------| @@ -456,9 +458,9 @@ The same env vars apply to backend workers and `local-ai agent-worker`. If the s } ``` -Workers connect with that JWT and seed automatically (shown once; store securely). Override with `LOCALAI_NATS_JWT` / `LOCALAI_NATS_USER_SEED` if needed. Set `LOCALAI_NATS_REQUIRE_AUTH=true` on workers when the bus requires credentials. +Agent workers connect with that JWT and seed automatically (shown once; store securely). Override with `LOCALAI_NATS_JWT` / `LOCALAI_NATS_USER_SEED` if needed. Set `LOCALAI_NATS_REQUIRE_AUTH=true` on an agent worker when the bus requires credentials; `local-ai worker` has no such flag, because it opens no connection to require credentials for. A JWT is still minted for a serve-backend node at registration and is simply unused; it grants nothing but that connection's own reply inbox. -When `LOCALAI_NATS_REQUIRE_AUTH=true` and no static credentials are provided, a worker that registers while still **pending admin approval** keeps re-registering (with backoff) until an admin approves it and the frontend mints its JWT - it does not start unauthenticated. This retry is **bounded**: if the node is never approved (or no credentials are minted) after a large number of attempts, the worker exits non-zero so the failure is visible (a crash-looping or failed worker) rather than hanging silently. Minted worker JWTs are also **refreshed automatically** before they expire (the worker re-registers at ~75% of the JWT lifetime), so long-running workers survive past `LOCALAI_NATS_WORKER_JWT_TTL`; the NATS connection picks up the new JWT on its next reconnect. If refresh fails persistently, the worker exits (to restart and re-acquire) rather than drifting toward an expired, unrenewable JWT. Statically configured (`LOCALAI_NATS_JWT`) and service (`LOCALAI_NATS_SERVICE_JWT`) credentials are used as-is and not refreshed. +When `LOCALAI_NATS_REQUIRE_AUTH=true` and no static credentials are provided, an agent worker that registers while still **pending admin approval** keeps re-registering (with backoff) until an admin approves it and the frontend mints its JWT - it does not start unauthenticated. This retry is **bounded**: if the node is never approved (or no credentials are minted) after a large number of attempts, the worker exits non-zero so the failure is visible (a crash-looping or failed worker) rather than hanging silently. Minted worker JWTs are also **refreshed automatically** before they expire (the worker re-registers at ~75% of the JWT lifetime), so long-running workers survive past `LOCALAI_NATS_WORKER_JWT_TTL`; the NATS connection picks up the new JWT on its next reconnect. If refresh fails persistently, the worker exits (to restart and re-acquire) rather than drifting toward an expired, unrenewable JWT. Statically configured (`LOCALAI_NATS_JWT`) and service (`LOCALAI_NATS_SERVICE_JWT`) credentials are used as-is and not refreshed. Generate operator/account material with [`scripts/nats-auth-setup.sh`](https://github.com/mudler/LocalAI/blob/master/scripts/nats-auth-setup.sh) (requires [nsc](https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nsc)). Configure the NATS server with account resolver JWTs before enabling `LOCALAI_NATS_REQUIRE_AUTH`. @@ -513,7 +515,7 @@ during installation as well as the committed snapshot. {{% /notice %}} {{% notice warning %}} -The worker HTTP file transfer server is authenticated by `LOCALAI_REGISTRATION_TOKEN`. If the token is **empty**, the server **fails open** - anyone who can reach the port gets read/write access to the worker's models/staging/data directories (a remote model-poisoning / exfiltration vector), **and to the `/v1/control/` routes that install, upgrade and delete backends and stop the node**. The worker logs a loud warning at startup in this case. Always set `LOCALAI_REGISTRATION_TOKEN` in distributed mode, and set `LOCALAI_DISTRIBUTED_REQUIRE_AUTH=true` (frontend **and** workers) to make a missing token *or* missing NATS credentials a hard startup error rather than a silent fail-open. +The worker HTTP file transfer server is authenticated by `LOCALAI_REGISTRATION_TOKEN`. If the token is **empty**, the server **fails open** - anyone who can reach the port gets read/write access to the worker's models/staging/data directories (a remote model-poisoning / exfiltration vector), **and to the `/v1/control/` routes that install, upgrade and delete backends and stop the node**. The worker logs a loud warning at startup in this case. Always set `LOCALAI_REGISTRATION_TOKEN` in distributed mode, and set `LOCALAI_DISTRIBUTED_REQUIRE_AUTH=true` (frontend **and** workers) to make a missing token a hard startup error rather than a silent fail-open. On the frontend and on agent workers it also makes missing NATS credentials fatal; on a serve-backend worker it means the registration token alone, since that worker uses no bus credential. By default the server binds loopback, so "anyone who can reach the port" means a process on the worker host, and no firewall rule is required. Setting `LOCALAI_HTTP_ADDR` to a routable address opts back out of that and puts the fail-open case back on the network - if you do it, firewall the port. {{% /notice %}} @@ -559,10 +561,11 @@ Workers are started with the `worker` subcommand. Each worker is generic - it do ```bash local-ai worker \ --register-to http://frontend:8080 \ - --registration-token changeme \ - --nats-url nats://nats:4222 + --registration-token changeme ``` +There is no `--nats-url` here. A serve-backend worker connects to no message bus: it dials one outbound tunnel to `--register-to` and serves every request the frontend makes of it over that. The flag is still accepted and ignored, so an existing command line keeps working. + | Flag | Env Var | Default | Description | |------|---------|---------|-------------| | `--addr` | `LOCALAI_ADDR` | *(unset)* | Base port for backend gRPC processes. Only the port is used; nothing binds the host | @@ -575,16 +578,10 @@ local-ai worker \ | `--node-name` | `LOCALAI_NODE_NAME` | hostname | Human-readable node name | | `--registration-token` | `LOCALAI_REGISTRATION_TOKEN` | *(empty)* | Token to authenticate with the frontend | | `--registration-require-auth` | `LOCALAI_REGISTRATION_REQUIRE_AUTH` | `false` | Refuse to start the HTTP file-transfer server when no registration token is set (it would otherwise fail open) | -| `--distributed-require-auth` | `LOCALAI_DISTRIBUTED_REQUIRE_AUTH` | `false` | Umbrella switch implying both `--registration-require-auth` and `--nats-require-auth` | +| `--distributed-require-auth` | `LOCALAI_DISTRIBUTED_REQUIRE_AUTH` | `false` | Umbrella switch implying `--registration-require-auth` | | `--heartbeat-interval` | `LOCALAI_HEARTBEAT_INTERVAL` | `10s` | Interval between heartbeat pings | | `--worker-tunnel` | `LOCALAI_WORKER_TUNNEL` | `true` | Hold one outbound multiplexed tunnel to the frontend and serve its requests over it, so this worker needs no inbound port (see [Worker tunnels](#worker-tunnels)). Setting it to `false` is a **fatal startup error**, not a degraded mode: the frontend has no path that dials a worker's advertised address, so a worker without its tunnel is a worker nothing can reach. To run without tunnels, run the pre-tunnel release on both the worker and the frontend. | -| `--nats-url` | `LOCALAI_NATS_URL` | *(required)* | NATS URL. A serve-backend worker takes no command over it - backend installation and file staging are HTTP routes on its tunnel - but it must still connect for the frontend to consider it healthy. | -| `--nats-jwt` | `LOCALAI_NATS_JWT` | *(empty)* | Optional override for the `nats_jwt` returned at registration | -| `--nats-user-seed` | `LOCALAI_NATS_USER_SEED` | *(empty)* | Optional override for `nats_user_seed` from registration | -| `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | `false` | Require NATS JWT+seed (from registration or env) | -| `--nats-tls-ca` | `LOCALAI_NATS_TLS_CA` | *(empty)* | PEM file for NATS server CA | -| `--nats-tls-cert` | `LOCALAI_NATS_TLS_CERT` | *(empty)* | Client certificate for NATS mTLS | -| `--nats-tls-key` | `LOCALAI_NATS_TLS_KEY` | *(empty)* | Client private key for NATS mTLS | +| `--nats-url` | `LOCALAI_NATS_URL` | *(ignored)* | **Accepted and ignored.** A serve-backend worker opens no NATS connection. Kept so an existing worker command line still starts. | | `--backends-path` | `LOCALAI_BACKENDS_PATH` | `./backends` | Path to backend binaries | | `--models-path` | `LOCALAI_MODELS_PATH` | `./models` | Path to model files | | `--vram-budget` | `LOCALAI_VRAM_BUDGET` | *(empty)* | Cap the VRAM this node advertises for model placement, as a percentage (e.g. `80%`) or an absolute amount (e.g. `12GB`). Empty uses all detected VRAM. See [Per-node VRAM budget](#per-node-vram-budget). | @@ -607,12 +604,14 @@ The worker's HTTP server (loopback, base port - 1, default 50050) exposes two un | Endpoint | Meaning | |----------|---------| -| `/healthz` | **Liveness.** 200 whenever the process is up and serving. Deliberately independent of readiness, so a brief NATS outage does not trigger a restart storm across every worker. | -| `/readyz` | **Readiness.** 200 only when the worker is registered, its NATS connection is live, *and* every backend process it is currently serving answers a short TCP dial on its gRPC address; 503 otherwise. A worker holding no backends is ready, because idle is a healthy state, and so is one whose backends are still starting up. | +| `/healthz` | **Liveness.** 200 whenever the process is up and serving. Deliberately independent of readiness, so a frontend restart that drops every tunnel does not trigger a restart storm across every worker. | +| `/readyz` | **Readiness.** 200 only when the worker is registered, currently holds a tunnel session, *and* every backend process it is serving answers a short TCP dial on its gRPC address; 503 otherwise. A worker holding no backends is ready, because idle is a healthy state, and so is one whose backends are still starting up. | -`/readyz` reports something the frontend cannot see on its own. The node registry's `status` and `last_heartbeat` are driven by an HTTP heartbeat to the frontend, which is a different network path from NATS — a worker can keep heartbeating while its NATS link is dead, and so appear `healthy` in the registry while being unable to receive any work. The local probe closes that gap. +`/readyz` tracks the **tunnel**, because that is the only way anything reaches this worker: it binds loopback, advertises no address, and every request the frontend makes of it arrives as a stream inside that tunnel. It reports something the local supervisor cannot see on its own. The node registry's `status` and `last_heartbeat` are driven by an HTTP heartbeat to the frontend, a different network path - a worker can keep heartbeating while its tunnel is dead, and so appear `healthy` in the registry while being unreachable. The local probe closes that gap. -The same applies to the data path. A worker can hold a live NATS link while the backend processes it believes it is running have died, so it reports healthy while every load routed to it fails. `/readyz` therefore also dials the recorded gRPC address of each backend the worker is serving, and a worker whose backend port refuses connections drops out of rotation instead of absorbing work it cannot serve. +A 503 here is **this container's own report that it cannot serve right now**, and nothing else. It is not a claim that the worker is gone; the frontend decides that from the tunnel session it holds, aged against `LOCALAI_WORKER_RECONNECT_GRACE`. The worker keeps heartbeating throughout a tunnel outage for exactly that reason: withholding the heartbeat would report an unreachable worker as an absent one, on the one path that has no grace. + +The same applies to the data path. A worker can hold a live tunnel while the backend processes it believes it is running have died, so it reports healthy while every load routed to it fails. `/readyz` therefore also dials the recorded gRPC address of each backend the worker is serving, and a worker whose backend port refuses connections drops out of rotation instead of absorbing work it cannot serve. Only backends in the middle of their lifecycle are dialled. A backend that is still starting is skipped until its gRPC server has answered a health check, which can take 10 to 15 seconds on a slow node, and a backend that is stopping is skipped from the moment shutdown begins. Neither a cold start nor an ordinary shutdown makes a worker report 503, so a Kubernetes `readinessProbe` at the usual 10s period does not pull a worker out of rotation every time it loads a model. @@ -624,7 +623,6 @@ A worker needs no address configuration at all. It binds only loopback and reach ```yaml environment: - LOCALAI_NATS_URL: "nats://frontend:4222" LOCALAI_REGISTER_TO: "http://frontend:8080" LOCALAI_REGISTRATION_TOKEN: "my-secret" ``` @@ -864,7 +862,7 @@ The edit response includes these fields: - `config_revision` identifies the saved semantic configuration. - `pending_cleanup` counts old replicas that still need cleanup when the response returns. -LocalAI sends an acknowledged stop request for each exact backend process. If a worker or NATS is unreachable, LocalAI keeps the replica in the `unloading` state and retries with durable backoff. The saved edit remains successful while cleanup is pending. +LocalAI sends an acknowledged stop request for each exact backend process, over that worker's tunnel. If the worker is unreachable, LocalAI keeps the replica in the `unloading` state and retries with durable backoff. The saved edit remains successful while cleanup is pending. Workers must support the exact model-stop protocol. Upgrade all workers before you rely on revision cleanup. An older worker cannot acknowledge the request, so its stale replica remains `unloading` until cleanup succeeds or the worker re-registers. @@ -1290,13 +1288,11 @@ ds4 layer-split inference is **manual setup** in this release (Phase 1): you pla local-ai worker \ --register-to http://frontend:8080 \ --node-name worker-2 \ - --nats-url nats://nats:4222 \ --registration-token changeme local-ai worker \ --register-to http://frontend:8080 \ --node-name worker-3 \ - --nats-url nats://nats:4222 \ --registration-token changeme ``` @@ -1503,12 +1499,12 @@ Notes: |---|---|---| | **Discovery** | Automatic via libp2p token | Self-registration to frontend URL | | **State storage** | In-memory / ledger | PostgreSQL | -| **Coordination** | Gossip protocol | NATS messaging | +| **Coordination** | Gossip protocol | The worker's own tunnel for serve-backend work; NATS for agent workers and cross-replica frontend events | | **Node management** | Automatic | REST API + WebUI | | **Health monitoring** | Peer heartbeats | Centralized HealthMonitor | | **Backend management** | Manual per node | Dynamic via the worker's `backend.install` control route | | **Best for** | Ad-hoc clusters, community sharing | Production, Kubernetes, managed infrastructure | -| **Setup complexity** | Minimal (share a token) | Requires PostgreSQL + NATS | +| **Setup complexity** | Minimal (share a token) | Requires PostgreSQL on the frontend, plus NATS if you run agent workers. Serve-backend workers need neither: only an outbound route to the frontend URL. | ## Troubleshooting @@ -1518,8 +1514,9 @@ Notes: - Ensure auth is enabled on the frontend (`LOCALAI_AUTH=true`) **NATS connection errors:** +- These concern the **frontend** and **agent workers** only. A `local-ai worker` opens no NATS connection; if one is failing to join, look at its tunnel and its `--register-to` instead. - Confirm NATS is running and reachable (`nats-server --signal ldm` or check port 4222) -- Check that `--nats-url` uses the correct hostname/IP from the worker's network perspective +- Check that `--nats-url` uses the correct hostname/IP from that component's network perspective **PostgreSQL connection errors:** - Verify the connection URL format: `postgresql://user:password@host:5432/dbname?sslmode=disable` @@ -1547,7 +1544,7 @@ Notes: - Confirm that every routable replica has `state: loaded` and the same current `config_revision`. - Treat a different `effective_options_hash` as diagnostic information. Node-specific defaults can cause valid differences. - Check `cleanup_error` and `cleanup_next_retry_at` on replicas in the `unloading` state. -- Check connectivity to the worker and NATS when cleanup reports a timeout or no responder. +- Check that the worker's tunnel is up when cleanup reports a timeout or no route. - Upgrade the worker when it does not support the exact model-stop request. - Stop and restart the stale backend only as an operational recovery action. LocalAI keeps it non-routable while durable cleanup is pending. @@ -1583,7 +1580,7 @@ Notes: - The HTTP file transfer server runs on the base port - 1 (default: 50050) - All of those bind loopback, so a firewall cannot be the cause. What can is another service on the same host already holding a port in the range: move the worker's range with `LOCALAI_ADDR` (see [Worker Port Configuration](#worker-port-configuration)) or bound it with `LOCALAI_GRPC_MAX_PORT` - Verify the backend gallery configuration is correct -- The worker needs OUTBOUND network access to the gallery, to `LOCALAI_REGISTER_TO` and to `LOCALAI_NATS_URL`. It needs no inbound access at all +- The worker needs OUTBOUND network access to the gallery and to `LOCALAI_REGISTER_TO`. It needs no inbound access at all, and no access to NATS ## Routing pipeline diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md index 8cf47e74acdc..e5c75c49b656 100644 --- a/docs/content/reference/cli-reference.md +++ b/docs/content/reference/cli-reference.md @@ -222,9 +222,9 @@ LocalAI supports several subcommands beyond `run`: - `local-ai transcript` - Convert audio to text - `local-ai agent` - Run agents standalone without the full LocalAI server - `local-ai mcp-server` - Run the LocalAI admin tool surface as a stdio MCP server (controls a remote LocalAI instance over HTTP) -- `local-ai worker` - Start a worker for distributed mode (generic, backend-agnostic) +- `local-ai worker` - Start a worker for distributed mode (generic, backend-agnostic; needs only an outbound route to the frontend, no message bus) - `local-ai p2p-worker` - Run workers to distribute workload via p2p (llama.cpp-only) -- `local-ai agent-worker` - Start an agent worker for distributed mode (executes agent chats via NATS) +- `local-ai agent-worker` - Start an agent worker for distributed mode (executes agent chats via NATS, which this command still needs) - `local-ai util` - Utility commands - `local-ai explorer` - Run P2P explorer - `local-ai federated` - Run LocalAI in federated mode diff --git a/pkg/natsauth/mint_test.go b/pkg/natsauth/mint_test.go index 195f05b2c343..7c849cf16203 100644 --- a/pkg/natsauth/mint_test.go +++ b/pkg/natsauth/mint_test.go @@ -36,7 +36,12 @@ var _ = Describe("MintWorkerJWT", func() { uc, err := jwt.DecodeUserClaims(token) Expect(err).NotTo(HaveOccurred()) - Expect(uc.Permissions.Sub.Allow).To(ContainElement("nodes.550e8400-e29b-41d4-a716-446655440000.>")) + // A backend worker opens no bus connection at all, so its node subtree + // went with it. The JWT is still minted at registration and simply + // unused; asserting BOTH lists are exactly the inbox is what keeps it + // from silently becoming an unrestricted credential, since NATS reads + // an empty allow list as no restriction. + Expect(uc.Permissions.Sub.Allow).To(ConsistOf("_INBOX.>")) // The install-progress subject is gone with the carrier: progress is a // line in the install response now, so a minted worker JWT must not // still be granted a publish right for it. File staging went the same diff --git a/pkg/natsauth/permissions.go b/pkg/natsauth/permissions.go index 8e0fa985e22b..4c081c672841 100644 --- a/pkg/natsauth/permissions.go +++ b/pkg/natsauth/permissions.go @@ -9,6 +9,18 @@ func workerSubjectToken(nodeID string) string { } // WorkerPermissions returns NATS pub/sub allow lists for a registered node. +// +// It serves AGENT nodes. They are the only workers left that connect to the +// bus: an agent worker subscribes to the queue subjects listed below, while a +// backend worker connects to no bus at all, because every verb a frontend gives +// it is an HTTP route on its own server reached through its outbound tunnel +// (core/services/workerctl). +// +// The non-agent branch is therefore a grant of nothing, and it has to be +// spelled that way rather than deleted. NATS reads an EMPTY allow list as no +// restriction, so a function that returned nil here would upgrade every JWT the +// frontend still mints for a backend node from "its own inbox" to "the entire +// account". The inbox is self-scoped and reaches no cluster subject. func WorkerPermissions(nodeID, nodeType string) (pubAllow, subAllow []string) { tok := workerSubjectToken(nodeID) prefix := "nodes." + tok @@ -38,25 +50,11 @@ func WorkerPermissions(nodeID, nodeType string) (pubAllow, subAllow []string) { "_INBOX.>", } default: - // Backend worker. Every verb a frontend gives it left the bus: the - // backend and model lifecycle verbs and now file staging too are HTTP - // routes under workerctl.Prefix, served on the worker's own server and - // reached through its tunnel, so no subject is minted for them and none - // is allowed here. - // - // The subscribe wildcard stays for now. A worker subscribes to nothing - // under it on this build, but narrowing it is a change a worker - // mid-upgrade would feel, and the connection itself is what the next - // step of this removal deletes. - subAllow = []string{ - prefix + ".>", - "_INBOX.>", - } - // Nothing left to publish. backend.install.*.progress went with the - // install subject, and the file-staging replies went with theirs. - pubAllow = []string{ - "_INBOX.>", - } + // Backend worker: nothing, held open at its own inbox for the reason in + // the doc comment. The node subtree it used to subscribe on went with + // the connection itself, which this worker no longer opens. + subAllow = []string{"_INBOX.>"} + pubAllow = []string{"_INBOX.>"} } return pubAllow, subAllow } diff --git a/pkg/natsauth/permissions_coverage_test.go b/pkg/natsauth/permissions_coverage_test.go index 4a7ffa7ac748..e7ed7133212f 100644 --- a/pkg/natsauth/permissions_coverage_test.go +++ b/pkg/natsauth/permissions_coverage_test.go @@ -59,19 +59,25 @@ var _ = Describe("WorkerPermissions subject coverage", func() { Context("backend worker", func() { pub, sub := natsauth.WorkerPermissions(nodeID, "backend") - // A backend worker subscribes to no subject of its own on this build. - // Every verb a frontend gives it — the backend and model lifecycle ten, - // and now the four file-staging verbs — is an HTTP route on its - // tunnelled control plane, so there is no subject left to cover. See + // A backend worker opens no connection at all on this build. Every verb + // a frontend gives it, the backend and model lifecycle ten plus the + // four file-staging verbs, is an HTTP route on its tunnelled control + // plane, so there is no subject left to cover. See // core/services/workerctl. - // - // The subscribe wildcard is asserted rather than removed because the - // grant is still minted and a worker mid-upgrade still uses it. - It("still grants a backend worker its own node subtree to subscribe on", func() { - Expect(sub).To(ConsistOf( - "nodes."+workerSubjectTokenForTest(nodeID)+".>", - "_INBOX.>", - )) + It("no longer grants a backend worker its own node subtree to subscribe on", func() { + Expect(sub).ToNot(ContainElement("nodes."+workerSubjectTokenForTest(nodeID)+".>"), + "the node subtree went with the connection the worker no longer opens") + }) + + // The grant must be a grant of NOTHING and not an ABSENT grant: NATS + // treats an empty allow list as no restriction, so a branch that + // returned nil would silently widen every backend JWT the frontend + // still mints to the whole account. ConsistOf, not BeEmpty, is what + // tells those two apart. + It("grants a backend worker its own inbox and nothing else", func() { + Expect(sub).To(ConsistOf("_INBOX.>")) + Expect(sub).ToNot(BeEmpty(), + "an empty allow list is unrestricted in NATS, not restrictive") }) // The negative half, and it is the one that would catch a verb quietly @@ -80,6 +86,8 @@ var _ = Describe("WorkerPermissions subject coverage", func() { // exception and is not any more. It("grants a backend worker no publish rights outside its inbox", func() { Expect(pub).To(ConsistOf("_INBOX.>")) + Expect(pub).ToNot(BeEmpty(), + "an empty allow list is unrestricted in NATS, not restrictive") }) It("no longer grants a backend worker the file-staging publish subtree", func() { diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 38012204f1ad..c7d9ce363464 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -332,10 +332,14 @@ func (c *Cluster) startWorker(i int) (*Process, error) { // rest of its life: the loop posts to the URL it was given at boot and // never re-resolves it (core/cli/workerregistry/client.go), so killing a // worker's registrar orphans that worker rather than failing it over. + // + // No LOCALAI_NATS_URL: a backend worker connects to no bus, and passing + // one would make every spec here prove the tunnel-only path works while + // quietly handing the worker the thing it is supposed to do without. + // The frontends above still get it. "LOCALAI_REGISTER_TO="+c.workerFrontendURL(i), "LOCALAI_NODE_NAME="+name, "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, - "LOCALAI_NATS_URL="+c.opts.NatsURL, "DEBUG=true", ) diff --git a/tests/e2e/distributed/nats_jwt_test.go b/tests/e2e/distributed/nats_jwt_test.go index 40008c824eea..b6b2343850fa 100644 --- a/tests/e2e/distributed/nats_jwt_test.go +++ b/tests/e2e/distributed/nats_jwt_test.go @@ -18,10 +18,10 @@ var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { }) It("connects with a minted backend worker JWT and publishes on its one remaining allowed subject", func() { - // A backend worker's publish grant is `_INBOX.>` and nothing else now. + // A backend worker's whole grant is `_INBOX.>` now, on both sides. // Every verb a frontend gives it, file staging included, is an HTTP - // route on its tunnel, so the `nodes..files.>` grant went with the - // subjects. See pkg/natsauth.WorkerPermissions. + // route on its tunnel, and it no longer opens a bus connection at all; + // the JWT is minted and unused. See pkg/natsauth.WorkerPermissions. Expect(infra.NC.Publish("_INBOX.probe", map[string]string{"path": "/tmp/model"})).To(Succeed()) Expect(infra.NC.Conn().FlushTimeout(2 * time.Second)).To(Succeed()) Expect(infra.NC.Conn().LastError()).ToNot(HaveOccurred()) @@ -43,13 +43,20 @@ var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { }, "3s", "50ms").Should(HaveOccurred()) }) - It("allows backend subscribe on the node prefix", func() { + It("denies backend subscribe on the node prefix it no longer listens to", func() { + // The node subtree was granted while a backend worker still held a + // connection with nothing under it subscribed. It does not hold one at + // all now, so the grant went too; asserting the denial is what would + // catch a subject quietly coming back to the bus. wild := nodeSubjectPrefix(infra.NodeID) + ".>" sub, err := infra.NC.Subscribe(wild, func(_ []byte) {}) - Expect(err).ToNot(HaveOccurred()) - defer func() { _ = sub.Unsubscribe() }() - Expect(infra.NC.Conn().FlushTimeout(2 * time.Second)).To(Succeed()) - Expect(infra.NC.Conn().IsConnected()).To(BeTrue()) + if err == nil { + defer func() { _ = sub.Unsubscribe() }() + Eventually(func() error { + _ = infra.NC.Conn().FlushTimeout(500 * time.Millisecond) + return infra.NC.Conn().LastError() + }, "3s", "50ms").Should(HaveOccurred()) + } }) It("rejects anonymous publish on the JWT-enabled server", func() { From 1ed22228e206283852717ae53fc3ba87496965bc Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 03:03:53 +0000 Subject: [PATCH 079/153] test(distributed): prove phase 3 end to end, under real processes Five cluster specs that run the binaries an operator runs, plus the repair of eighteen specs phase 2 left red. The eighteen were router_tracking and full_flow, failing since 1cf847f29 on "reported backend installed but named no address for the process". Two contracts had changed under them: an install reply that names no worker-local address is refused rather than substituted, and a frontend with no worker dialer reaches no backend at all. Nobody noticed for a phase because phase 2 verified with --label-filter='Cluster', which excludes both suites. ServeBackendLifecycle and tunnelBackendClients state both facts once for every spec. The transport double is the part that matters. It translates a refused connect into cluster.ErrStreamTargetUnavailable, which is what a real worker answers when its backend process has died and what IsWorkerAnswer lets a reap guard act on. A bare ECONNREFUSED reaches those guards as "no route" and reaps nothing, so a double returning the raw syscall error could never fail the way production fails; putting it back reddens the stale-record spec and nothing else. The new specs cover: a backend worker with no bus URL in its /proc environ registering, being scheduled onto and serving inference; a backend install and a backend listing driven through the replica that does NOT own the worker, with the owner read through the production Owner query and re-read after; that install's progress proven to arrive before its terminal reply, made deterministic by a gallery server that holds the worker's fetch open so a reply cannot exist yet; a worker whose tunnel is genuinely gone, waited for rather than assumed, losing nothing inside the reconnect grace and re-homing after; a heartbeating worker with a permanently dead tunnel losing its healthy status while an agent worker in the same cluster keeps it; and the suite's negative control, where a control RPC to a tunnel-less worker fails naming the missing route, reaps nothing, and succeeds the moment the tunnel returns. Every scenario was attacked. The churn one was WRONG on the first attempt and only the attack found it: its hold window sat entirely inside cluster.InstanceLiveness, so a killed replica still read as a live owner throughout, presence was "connected", and the spec passed with the reconnect grace set to a nanosecond. It now blocks the tunnel before the kill and waits for the ownership row to actually empty. Attacks that redden the rest: posting at the owner, writing the install reply before the work, collapsing PresenceReconnecting into PresenceGone, removing the non-backend node-type guard, and not blocking the tunnel. Agent workers turn out to be protected twice over; no single mutation reaches them. Harness: Options.AgentWorkers and Options.ReconnectGrace, WorkerEnviron (read from /proc, because Cmd.Env is the harness agreeing with itself), NatsURL, FrontendBackendsDir, AgentWorkerName, PostJSON, and a node String() so a failing roster assertion is readable instead of several hundred bytes rendered as numbers. Budget: 20 specs at 787 to 808 seconds over three runs, up from phase 2's 591 to 612. --timeout goes to 30m so a loaded runner reports a cause rather than a spec name. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- Makefile | 14 +- tests/e2e/distributed/cluster/admin.go | 32 + tests/e2e/distributed/cluster/cluster.go | 146 +++- .../e2e/distributed/cluster_baseline_test.go | 39 + tests/e2e/distributed/cluster_control_test.go | 797 ++++++++++++++++++ tests/e2e/distributed/control_workers_test.go | 78 ++ .../distributed/distributed_full_flow_test.go | 31 +- .../e2e/distributed/nats_jwt_helpers_test.go | 2 +- tests/e2e/distributed/router_tracking_test.go | 26 +- 9 files changed, 1133 insertions(+), 32 deletions(-) create mode 100644 tests/e2e/distributed/cluster_control_test.go diff --git a/Makefile b/Makefile index debc5406ff51..354527465826 100644 --- a/Makefile +++ b/Makefile @@ -380,9 +380,21 @@ test-e2e-distributed: protogen-go # --flake-attempts is pinned to 1 rather than $(DISTRIBUTED_TEST_FLAKES), and # should stay there: this suite exists to catch nondeterministic cluster # behaviour, and a retry turns exactly that signal into a green run. +# +# Budget: 20 specs, measured at 800 to 830 seconds of Ginkgo time (13 to 14 +# minutes wall including the compile) on a fast developer box. It was 591 to 612 +# seconds before the phase 3 control-plane specs; those five added roughly 200 +# seconds, most of it in the two that wait out real windows rather than poll for +# a state change (cluster.InstanceLiveness is 30s, and a departed worker cannot +# be demoted before its reconnect grace). +# +# --timeout is 30m rather than 20m because of that. The margin is not slack: a +# Ginkgo timeout kills the suite mid-spec and reports a spec name rather than a +# cause, and 20m on a loaded CI runner was one slow health tick away from +# turning a green suite into an unreadable red one. test-e2e-cluster: protogen-go build-mock-backend @echo 'Running cluster e2e tests (label Cluster, real local-ai processes)' - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Cluster' --fail-on-empty --flake-attempts 1 --timeout=20m -v ./tests/e2e/distributed + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Cluster' --fail-on-empty --flake-attempts 1 --timeout=30m -v ./tests/e2e/distributed # vLLM multi-node DP smoke (CPU). Builds local-ai:tests and the # cpu-vllm backend from the current working tree, then drives a diff --git a/tests/e2e/distributed/cluster/admin.go b/tests/e2e/distributed/cluster/admin.go index 617bcef99086..cbecf8c37eff 100644 --- a/tests/e2e/distributed/cluster/admin.go +++ b/tests/e2e/distributed/cluster/admin.go @@ -128,6 +128,38 @@ func (c *Cluster) GetJSON(client *http.Client, frontend int, path string, out an return nil } +// PostJSON performs an authenticated POST against a frontend and decodes the +// body, reporting the status it got so a caller can act on it. +// +// It returns the status rather than requiring 200 the way GetJSON does, +// because the control-plane endpoints a spec drives through here answer 202 +// (an install was accepted and runs asynchronously) and the failure cases a +// negative control is about are statuses, not transport errors. out may be nil +// for a caller that only wants the status. +func (c *Cluster) PostJSON(client *http.Client, frontend int, path string, body any, out any) (int, error) { + base, err := c.frontendBaseURL(frontend) + if err != nil { + return 0, err + } + encoded, err := json.Marshal(body) + if err != nil { + return 0, fmt.Errorf("marshalling the body for %s: %w", path, err) + } + resp, err := client.Post(base+path, "application/json", bytes.NewReader(encoded)) + if err != nil { + return 0, fmt.Errorf("POST %s on frontend %d: %w", path, frontend, err) + } + defer func() { _ = resp.Body.Close() }() + if out == nil { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, bodyExcerptLimit)) + return resp.StatusCode, nil + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return resp.StatusCode, fmt.Errorf("decoding the %s response from frontend %d (status %d): %w", path, frontend, resp.StatusCode, err) + } + return resp.StatusCode, nil +} + // frontendBaseURL validates the index before FrontendURL indexes the slice: a // bare index panic in a helper every failover spec calls is far harder to read // than a named error. diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index c7d9ce363464..0c98421d28fb 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -15,6 +15,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" "github.com/mudler/LocalAI/pkg/httpclient" @@ -43,6 +44,35 @@ type Options struct { Frontends int Workers int + // AgentWorkers is how many `local-ai agent-worker` processes to start + // alongside the backend workers. + // + // They exist so a spec can hold the two kinds of worker side by side in one + // cluster. An agent worker still speaks NATS and holds no tunnel at all, + // which is exactly the shape the tunnel-departure rules must not act on: it + // has no node_connections row, so its presence is PresenceUnknown forever. + // A spec that asserted only on backend workers could not tell "agent + // workers are unaffected" from "nothing here looks at them". + // + // They register through the same WorkerFrontendURL hook as backend workers, + // so a spec that puts a balancer in front of the fleet gets one for its + // agent workers too. Without that, killing a replica would orphan the agent + // worker's heartbeats and it would go unhealthy for a reason that has + // nothing to do with what the spec is about. + AgentWorkers int + + // ReconnectGrace sets LOCALAI_WORKER_RECONNECT_GRACE on every frontend: how + // long a worker whose tunnel was lost is read as reconnecting rather than + // gone. Zero leaves the binary's own default (90s). + // + // It is an Option rather than a per-spec environment edit because two + // specs need it pulled in OPPOSITE directions and neither can use the + // default: one has to prove nothing is reaped inside the window and needs + // it longer than a re-home takes, and one has to prove a departed worker + // stops being reported healthy and would otherwise wait 90 seconds to say + // so. + ReconnectGrace time.Duration + // SpreadWorkerRegistrations sends worker i to frontend i%Frontends instead // of sending every worker to frontend 0. // @@ -113,7 +143,11 @@ type Cluster struct { opts Options frontends []*Process workers []*Process - baseDir string + // agentWorkers are kept apart from workers rather than appended to it. Every + // index-taking method on this type means "backend worker i", and folding the + // two together would silently renumber them for every existing spec. + agentWorkers []*Process + baseDir string } const ( @@ -185,6 +219,14 @@ func Start(opts Options) (*Cluster, error) { } c.workers = append(c.workers, p) } + for i := 0; i < opts.AgentWorkers; i++ { + p, err := c.startAgentWorker(i) + if err != nil { + c.Stop() + return nil, err + } + c.agentWorkers = append(c.agentWorkers, p) + } return c, nil } @@ -258,6 +300,9 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { "LOCALAI_AUTO_APPROVE_NODES=true", "DEBUG=true", ) + if c.opts.ReconnectGrace > 0 { + cmd.Env = append(cmd.Env, "LOCALAI_WORKER_RECONNECT_GRACE="+c.opts.ReconnectGrace.String()) + } p, err := c.spawn(name, cmd, port) if err != nil { @@ -346,6 +391,77 @@ func (c *Cluster) startWorker(i int) (*Process, error) { return c.spawn(name, cmd, grpcPort) } +// startAgentWorker starts agent worker i. +// +// It is `local-ai agent-worker`, not `local-ai worker`, and the difference is +// the whole point of having it here: an agent worker REQUIRES a NATS URL, dials +// no tunnel, and runs no backend, so it is the control for every rule this +// phase added about a worker whose tunnel is gone. It binds nothing, so there +// is no port to reserve and no readiness endpoint to wait on; a spec learns it +// is up by finding it in the roster. +func (c *Cluster) startAgentWorker(i int) (*Process, error) { + name := agentWorkerName(i) + cmd := exec.Command(c.opts.Binary, "agent-worker") + cmd.Env = append(cmd.Environ(), + // The bus, which is what makes this worker the control: a backend + // worker in this same cluster is given none. + "LOCALAI_NATS_URL="+c.opts.NatsURL, + "LOCALAI_REGISTER_TO="+c.workerFrontendURL(i), + "LOCALAI_NODE_NAME="+name, + "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, + "DEBUG=true", + ) + return c.spawn(name, cmd, 0) +} + +// WorkerEnviron is the environment of worker i's RUNNING PROCESS, read from +// /proc. +// +// Not Cmd.Env, deliberately. A spec asserting that a worker runs with no bus +// URL is asserting about the process, and Cmd.Env is the harness telling the +// spec what the harness meant to do: the two agree by construction, so a spec +// reading it proves the harness consistent with itself and nothing about the +// binary. /proc//environ is what the kernel handed the process. +// +// Linux only, which this package already is (it signals with syscall.SIGKILL +// and reserves ports by binding loopback). A platform without /proc returns the +// read error rather than falling back to Cmd.Env, so the assertion fails loudly +// instead of quietly becoming the weaker one. +func (c *Cluster) WorkerEnviron(i int) ([]string, error) { + if err := c.checkWorkerIndex(i); err != nil { + return nil, err + } + p := c.workers[i] + if p == nil || p.Cmd == nil || p.Cmd.Process == nil { + return nil, fmt.Errorf("worker %d is not running", i) + } + raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/environ", p.Cmd.Process.Pid)) + if err != nil { + return nil, fmt.Errorf("reading the environment of %s from /proc: %w", p.Name, err) + } + // NUL separated, with a trailing NUL on a non-empty environment. + entries := strings.Split(string(raw), "\x00") + out := make([]string, 0, len(entries)) + for _, e := range entries { + if e != "" { + out = append(out, e) + } + } + return out, nil +} + +// AgentWorkerName is the node name agent worker i registered under. +func (c *Cluster) AgentWorkerName(i int) string { + if i < 0 || i >= len(c.agentWorkers) { + return "" + } + return c.agentWorkers[i].Name +} + +func agentWorkerName(i int) string { + return fmt.Sprintf("agent-worker-%d", i) +} + const ( // workerPortBlockSize is how many ports one worker reserves: one for its // HTTP file-transfer server and the rest for backend processes. A spec that @@ -488,6 +604,30 @@ func (c *Cluster) RegistrationToken() string { return c.opts.RegistrationToken } +// FrontendBackendsDir is the directory frontend i installs its OWN backends +// into. +// +// It is exported for one assertion, and a filesystem one rather than an API +// one: a spec proving a node backend listing came from the WORKER has to show +// the frontend that answered does not have that backend itself, and the +// /backends endpoint cannot say so, because in distributed mode it reports the +// cluster's backends rather than this process's. +func (c *Cluster) FrontendBackendsDir(i int) (string, error) { + if err := c.checkFrontendIndex(i); err != nil { + return "", err + } + return filepath.Join(c.frontendDir(i), "backends"), nil +} + +// NatsURL is the bus this cluster's frontends were given. +// +// It is exported for one assertion: a spec proving a WORKER runs with no bus +// has to show the deployment it joined has one, or "no NATS anywhere" would +// satisfy it just as well. +func (c *Cluster) NatsURL() string { + return c.opts.NatsURL +} + // WorkerName is the node name worker i registered under. func (c *Cluster) WorkerName(i int) string { return c.workers[i].Name @@ -528,7 +668,7 @@ func (c *Cluster) Stop() { if c == nil { return } - for _, p := range append(append([]*Process{}, c.workers...), c.frontends...) { + for _, p := range append(append(append([]*Process{}, c.workers...), c.agentWorkers...), c.frontends...) { p.terminate() } if c.baseDir != "" { @@ -539,7 +679,7 @@ func (c *Cluster) Stop() { // DumpLogs writes every process log to stdout. Call from an AfterEach guarded by // CurrentSpecReport().Failed(). func (c *Cluster) DumpLogs() { - for _, p := range append(append([]*Process{}, c.frontends...), c.workers...) { + for _, p := range append(append(append([]*Process{}, c.frontends...), c.workers...), c.agentWorkers...) { if p == nil { continue } diff --git a/tests/e2e/distributed/cluster_baseline_test.go b/tests/e2e/distributed/cluster_baseline_test.go index 9ef29a976125..1b2aaae2be04 100644 --- a/tests/e2e/distributed/cluster_baseline_test.go +++ b/tests/e2e/distributed/cluster_baseline_test.go @@ -44,6 +44,11 @@ type node struct { // have dialled instead of the tunnel. Address string `json:"address"` HTTPAddress string `json:"http_address"` + // LastHeartbeat is what separates "the worker is gone" from "the worker is + // here and this deployment cannot reach it". A spec asserting the second + // has to show the first is false, and the heartbeat is the only evidence + // of that in this payload. + LastHeartbeat time.Time `json:"last_heartbeat"` // keys is what the payload actually carried, which a decoded struct cannot // tell you. Both fields above are the zero value when a worker advertises // nothing AND when the key was renamed or dropped, and the whole point of @@ -53,6 +58,17 @@ type node struct { keys map[string]json.RawMessage `json:"-"` } +// String is what a failing assertion prints for a node. +// +// Without it %+v renders keys, whose values are json.RawMessage, as slices of +// byte VALUES: one node becomes several hundred numbers and a roster of two +// buries the assertion that failed. The key set is still what advertisementOf +// reads; it is just not something a human ever needs to see. +func (n node) String() string { + return fmt.Sprintf("{name:%s status:%s id:%s lastHeartbeat:%s advertised:%q/%q}", + n.Name, n.Status, n.ID, n.LastHeartbeat.Format(time.RFC3339), n.Address, n.HTTPAddress) +} + // UnmarshalJSON decodes the fields above and keeps the raw key set beside them. func (n *node) UnmarshalJSON(data []byte) error { // A distinct type, or this method calls itself. @@ -272,6 +288,29 @@ func (p *rosterProbe) advertisementOf(name string) (string, bool) { return "", false } +// heartbeatOf is the last heartbeat the roster reported for a node, refreshed +// on every call. +// +// A zero time is returned for a node the roster does not carry, which no +// freshness assertion can accept: the spec that reads this is proving the +// worker is still alive, and a missing node must not read as a recent +// heartbeat. +func (p *rosterProbe) heartbeatOf(name string) time.Time { + var roster []node + if err := p.cluster.GetJSON(p.client, p.frontend, "/api/nodes", &roster); err != nil { + p.lastErr = err + return time.Time{} + } + p.lastErr = nil + p.lastSeen = roster + for _, n := range roster { + if n.Name == name { + return n.LastHeartbeat + } + } + return time.Time{} +} + // describe is handed to Should as the failure message. Gomega calls a // func() string description lazily, so this runs only on failure and reports // whichever of the two distinct causes actually occurred. diff --git a/tests/e2e/distributed/cluster_control_test.go b/tests/e2e/distributed/cluster_control_test.go new file mode 100644 index 000000000000..d724f6037d0b --- /dev/null +++ b/tests/e2e/distributed/cluster_control_test.go @@ -0,0 +1,797 @@ +package distributed_test + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/mudler/LocalAI/tests/e2e/distributed/cluster" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// End-to-end proof that phase 3's control plane works under real processes. +// +// Phase 3 moved fourteen control verbs off NATS and onto the tunnel the worker +// already dials, made absence a database fact rather than a bus timeout, and +// took the backend worker off the bus entirely. Every one of those changes is +// proven by unit and integration specs only; these are the first that run the +// binaries an operator runs and let the real transports fail. +// +// Read the LAST spec before trusting the others. Frontend and worker share a +// host here, so almost anything that looks like it went over the tunnel could +// have gone some other way. The last one takes the tunnel away and requires the +// control plane to become unreachable, with the refusal naming the missing +// route rather than claiming the worker is absent; without it the rest could +// pass with the tunnel doing nothing. + +const ( + // controlRPCTimeout bounds one admin control call that has to cross a + // relay and reach a worker. Generous because the install verb behind it + // copies an artifact and starts a process. + controlRPCTimeout = 3 * time.Minute + + // installJobTimeout bounds one node-scoped backend install from accepted to + // terminal. The gated spec spends most of it deliberately blocked. + installJobTimeout = "150s" + installJobPoll = "250ms" + + // gateHoldWindow is how long a spec requires an install to stay + // unfinished while its gallery fetch is blocked. + // + // It is not a tolerance. The worker is stopped inside an HTTP read that + // only this spec can complete, so a terminal reply arriving inside this + // window is a reply written before the work it reports on, which is the + // exact defect the envelope ordering exists to prevent. + gateHoldWindow = "3s" + gateHoldPoll = "250ms" + + // departedTimeout bounds the wait for a worker whose tunnel is gone past + // the grace to stop being reported healthy. It has to clear the grace the + // spec sets plus one health-monitor tick (15s by default). + departedTimeout = "90s" + departedPoll = "1s" + + // churnGrace is the reconnect grace the churn spec runs with: long enough + // that the whole scenario, including the worker's capped 30s reconnect + // backoff and the re-home after it, happens INSIDE the window. The claim + // under test is "nothing is reaped inside the grace", so the window has to + // be the thing that is generous, not the assertion. + churnGrace = 10 * time.Minute + + // churnHoldWindow is how long the churn spec requires the fleet to survive + // with its tunnel genuinely gone. It starts only after the tunnel has been + // OBSERVED unowned, and it is past the health monitor's 15s tick, so at + // least one sweep runs against a worker nothing can reach and finds nothing + // to reap. + churnHoldWindow = "20s" + churnHoldPoll = "1s" + + // churnDropTimeout bounds the wait for a killed replica's claim to stop + // reading as a live owner. + // + // It has to outlast cluster.InstanceLiveness (30s): the ownership query + // joins against instances the DATABASE still considers live, so for the + // half minute after a SIGKILL a dead replica is still reported as holding + // every tunnel it held. That window is the reason this spec waits for the + // drop instead of assuming the kill produced one, and it is what an earlier + // version of this spec got wrong: its hold window sat entirely inside the + // liveness window, so it never reached the branch it claims to test and + // stayed green with the reconnect grace set to a nanosecond. + churnDropTimeout = "90s" + churnDropPoll = "1s" + + // probeBackend is the backend a control spec installs on a worker. + // + // A name nothing else knows, and that is the point: the frontends' own + // backends directory is empty and the worker's holds only mock-backend, so + // a node backend listing that names this one can only have come from the + // worker, and only after an install that reached it. + probeBackend = "relay-probe" + + // probeGalleryName is the gallery the probe backend is served from. + probeGalleryName = "e2e-control" +) + +// nodeModel is the subset of a /api/nodes/:id/models row these specs assert on. +type nodeModel struct { + ModelName string `json:"model_name"` + State string `json:"state"` + Address string `json:"address"` +} + +// nodeBackend is the subset of a /api/nodes/:id/backends row these specs assert +// on. It is the worker's own answer to the backend.list control verb, relayed +// back through whichever replica took the request. +type nodeBackend struct { + Name string `json:"name"` +} + +// installJob is the subset of galleryop.OpStatus these specs assert on. +// +// It is read from GET /backends/jobs/:uuid and not from the UI's +// /api/backends/job/:uid, because only the former carries the per-node +// breakdown, and the per-node breakdown is where a worker's progress line +// lands. The UI endpoint drops it. +type installJob struct { + Processed bool `json:"processed"` + Message string `json:"message"` + Error string `json:"error"` + Nodes []installJobNode `json:"nodes"` +} + +type installJobNode struct { + NodeID string `json:"node_id"` + Status string `json:"status"` + Phase string `json:"phase"` + Error string `json:"error"` +} + +// nodeEntry returns the per-node row for nodeID, or nil. +func (j installJob) nodeEntry(nodeID string) *installJobNode { + for i := range j.Nodes { + if j.Nodes[i].NodeID == nodeID { + return &j.Nodes[i] + } + } + return nil +} + +// controlSession is one admin session for a whole cluster, with a budget long +// enough for a control RPC that crosses a relay. +// +// One per spec and never one per frontend, for the reason inferenceClient gives: +// the auth routes share a five-per-minute-per-IP budget and every request here +// comes from 127.0.0.1. +func controlSession(c *cluster.Cluster) *http.Client { + GinkgoHelper() + client, err := c.AdminSession(0) + Expect(err).ToNot(HaveOccurred()) + client.Timeout = controlRPCTimeout + return client +} + +// nodeBackendNames lists the backends a worker reports installed, asked at one +// specific frontend. +// +// The frontend answers this by issuing the backend.list control verb over that +// worker's tunnel, relayed through the owner when this replica is not it, so +// the returned names are the WORKER's and not this process's. +func nodeBackendNames(c *cluster.Cluster, client *http.Client, frontend int, nodeID string) ([]string, error) { + var listed []nodeBackend + if err := c.GetJSON(client, frontend, "/api/nodes/"+nodeID+"/backends", &listed); err != nil { + return nil, err + } + names := []string{} + for _, b := range listed { + names = append(names, b.Name) + } + return names, nil +} + +// nodeModelNames lists the models a frontend records as loaded on a worker. +func nodeModelNames(c *cluster.Cluster, client *http.Client, frontend int, nodeID string) ([]string, error) { + var rows []nodeModel + if err := c.GetJSON(client, frontend, "/api/nodes/"+nodeID+"/models", &rows); err != nil { + return nil, err + } + names := []string{} + for _, m := range rows { + names = append(names, m.ModelName) + } + return names, nil +} + +// gatedGallery serves one backend gallery index, and can hold the request open +// until a spec lets it go. +// +// The gate is what makes the ordering assertion deterministic rather than +// tolerant. A worker install that finishes in milliseconds gives a poller no +// window to observe progress in, and a spec that "usually" sees a tick before +// the reply is a spec that passes for timing reasons. Here the worker is +// stopped inside the gallery fetch, which happens AFTER it emits its first +// progress line and BEFORE it can produce any reply, so the two are separated +// by something the spec controls instead of by luck. +// +// fetches counts what the worker actually asked for, so a spec can prove the +// gate is on the path it thinks it is rather than assuming it. +type gatedGallery struct { + server *httptest.Server + index string + open chan struct{} + once sync.Once + fetches atomic.Int64 +} + +// newGatedGallery starts a gallery serving index. When gated, the first and +// every subsequent fetch blocks until release is called. +func newGatedGallery(index string, gated bool) *gatedGallery { + GinkgoHelper() + g := &gatedGallery{index: index, open: make(chan struct{})} + if !gated { + g.release() + } + mux := http.NewServeMux() + mux.HandleFunc("/index.yaml", func(w http.ResponseWriter, r *http.Request) { + g.fetches.Add(1) + select { + case <-g.open: + case <-r.Context().Done(): + // The worker gave up, or the spec ended. Answering nothing is + // right: writing a body here would let a spec that failed its own + // gate assertion still see a successful install. + return + } + w.Header().Set("Content-Type", "application/yaml") + _, _ = io.WriteString(w, g.index) + }) + g.server = httptest.NewServer(mux) + DeferCleanup(func() { + // Released before close so a handler still parked on the gate returns + // instead of leaking until the test binary exits. + g.release() + g.server.Close() + }) + return g +} + +func (g *gatedGallery) release() { g.once.Do(func() { close(g.open) }) } + +// URL is the index URL a worker fetches. +func (g *gatedGallery) URL() string { return g.server.URL + "/index.yaml" } + +// galleriesJSON is the backend_galleries override an install request carries. +// It is a JSON string INSIDE the request body, which is the shape +// InstallBackendOnNodeEndpoint binds. +func (g *gatedGallery) galleriesJSON() string { + return fmt.Sprintf(`[{"name":%q,"url":%q}]`, probeGalleryName, g.URL()) +} + +// probeBackendSource lays out a directory a worker can install as a backend and +// then RUN, and returns its path. +// +// It is a real backend by the two rules core/gallery enforces on a directory +// URI: a run.sh, which is the validation gate, and an executable named after +// the backend, which is what the worker's findBackend resolves and starts. The +// executable is the mock backend, so the install ends with a live gRPC process +// rather than with a process that dies and turns a relay spec into a spec about +// a broken artifact. +func probeBackendSource() string { + GinkgoHelper() + dir := filepath.Join(GinkgoT().TempDir(), probeBackend) + Expect(os.MkdirAll(dir, 0o755)).To(Succeed()) + + binary, err := os.ReadFile(mockBackendBinary()) + Expect(err).ToNot(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, probeBackend), binary, 0o755)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), + []byte("#!/bin/sh\nexec \"$(dirname \"$0\")/"+probeBackend+"\" \"$@\"\n"), 0o755)).To(Succeed()) + return dir +} + +// probeGalleryIndex is the one-entry gallery index that points at src. +func probeGalleryIndex(src string) string { + return fmt.Sprintf("- name: %s\n uri: %s\n description: e2e control-plane probe\n", probeBackend, src) +} + +// startNodeInstall posts a node-scoped backend install at one frontend and +// returns the job id it was given. +// +// The install is asynchronous by design (202 plus a job id), so this is where +// the request stops and the job polling below takes over. +func startNodeInstall(c *cluster.Cluster, client *http.Client, frontend int, nodeID, backend, galleriesJSON string) string { + GinkgoHelper() + var accepted struct { + JobID string `json:"jobID"` + } + status, err := c.PostJSON(client, frontend, "/api/nodes/"+nodeID+"/backends/install", map[string]string{ + "backend": backend, + "backend_galleries": galleriesJSON, + }, &accepted) + Expect(err).ToNot(HaveOccurred()) + Expect(status).To(Equal(http.StatusAccepted), + "frontend %d refused the install outright, so nothing was ever sent to the worker", frontend) + Expect(accepted.JobID).ToNot(BeEmpty()) + return accepted.JobID +} + +// jobProbe polls one install job at one frontend and keeps what it last saw, so +// a failing Eventually can name it. +type jobProbe struct { + cluster *cluster.Cluster + client *http.Client + frontend int + jobID string + nodeID string + + lastErr error + last installJob +} + +func newJobProbe(c *cluster.Cluster, client *http.Client, frontend int, jobID, nodeID string) *jobProbe { + return &jobProbe{cluster: c, client: client, frontend: frontend, jobID: jobID, nodeID: nodeID} +} + +// read fetches the job once, keeping the error rather than raising it. +func (p *jobProbe) read() installJob { + var job installJob + if err := p.cluster.GetJSON(p.client, p.frontend, "/backends/jobs/"+p.jobID, &job); err != nil { + p.lastErr = err + return installJob{} + } + p.lastErr = nil + p.last = job + return job +} + +// nodePhase is the phase the worker last reported for this node, or "" when the +// job carries no row for it yet. +func (p *jobProbe) nodePhase() string { + job := p.read() + if entry := job.nodeEntry(p.nodeID); entry != nil { + return entry.Phase + } + return "" +} + +// nodeStatus is the status the job last recorded for this node. +func (p *jobProbe) nodeStatus() string { + job := p.read() + if entry := job.nodeEntry(p.nodeID); entry != nil { + return entry.Status + } + return "" +} + +// processed reports whether the job has reached a terminal state. +func (p *jobProbe) processed() bool { return p.read().Processed } + +// jobError is the job's error text, empty when it has none. +func (p *jobProbe) jobError() string { return p.read().Error } + +// explain builds a lazy failure description. +// +// Gomega formats a (string, args...) description when the assertion is +// CONSTRUCTED, which for an Eventually or a Consistently is before anything has +// gone wrong: the job it quoted would be the one from before the wait. A +// func() string is called only on failure. The same trap, and the same fix, as +// rosterProbe.explain. +func (p *jobProbe) explain(format string, args ...any) func() string { + return func() string { + return fmt.Sprintf(format, args...) + ": " + p.describe() + } +} + +func (p *jobProbe) describe() string { + if p.lastErr != nil { + return fmt.Sprintf("frontend %d: the last read of job %s failed: %v", p.frontend, p.jobID, p.lastErr) + } + return fmt.Sprintf("frontend %d: job %s last read as %+v", p.frontend, p.jobID, p.last) +} + +// withReconnectGrace pins how long a lost worker tunnel is read as reconnecting +// rather than gone. +func withReconnectGrace(d time.Duration) func(*cluster.Options) { + return func(o *cluster.Options) { o.ReconnectGrace = d } +} + +// withAgentWorkers adds agent workers, which still speak NATS and hold no +// tunnel, to a cluster. +func withAgentWorkers(n int) func(*cluster.Options) { + return func(o *cluster.Options) { o.AgentWorkers = n } +} + +var _ = Describe("Control plane over the worker tunnel", Label("Distributed"), Label("Cluster"), func() { + + // Scenario 1, the headline. A backend worker that is connected to no bus at + // all registers, is scheduled onto, and serves inference. + // + // A wrong implementation leaves the worker up and inert, because nothing + // reaches it: before phase 3 every control verb travelled on NATS, so a + // worker with no NATS URL would register and heartbeat and never be given a + // backend or a model. + It("registers, schedules onto and serves a worker that is connected to no bus", func() { + c, dsn := startClusterOnFreshDB(1, 1, withMockModel("busless-model")) + client := inferenceClient(c) + + // The environment of the RUNNING process, not the one the harness + // assembled. LOCALAI_REGISTER_TO is asserted alongside so an absent + // LOCALAI_NATS_URL is a fact about the worker rather than a read that + // returned nothing: a broken read would lose both. + environ, err := c.WorkerEnviron(0) + Expect(err).ToNot(HaveOccurred()) + Expect(environ).To(ContainElement(HavePrefix("LOCALAI_REGISTER_TO=")), + "the worker's environment could not be read, so the absence below proves nothing") + for _, entry := range environ { + Expect(entry).ToNot(HavePrefix("LOCALAI_NATS_URL="), + "the worker was handed a bus URL, so this spec is not about a worker that has none") + } + // And the deployment it joined DOES have a bus, so "no NATS anywhere" + // is not what makes this pass. + Expect(c.NatsURL()).ToNot(BeEmpty()) + + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), probe.describe) + nodeID := probe.idOf(c.WorkerName(0)) + Expect(nodeID).ToNot(BeEmpty()) + + // Its tunnel is held here, and it advertises nothing. Both matter: the + // first says there is a route, the second says there is no other one. + owners := newTunnelOwners(openClusterDB(dsn)) + Eventually(func() int { return owners.ownerIndexOf(c, 1, nodeID) }, tunnelOwnershipTimeout, tunnelOwnershipPoll). + Should(Equal(0), owners.describe) + advertised, carriedKeys := probe.advertisementOf(c.WorkerName(0)) + Expect(carriedKeys).To(BeTrue(), + "the roster payload no longer carries the advertisement keys, so this spec cannot tell a worker that advertises nothing from one it cannot read") + Expect(advertised).To(BeEmpty(), + "the worker advertised %q, so a frontend could have reached it without the tunnel", advertised) + + // The inference is what drives the whole control plane: the frontend + // installs the backend on the worker, stages the model artifact to it + // and loads the model, all over the tunnel and all with no bus on the + // worker's side. + expectMockedInference(client, c.FrontendURL(0), "busless-model", + "a worker with no bus connection must still be scheduled onto and serve inference") + + // And the frontend recorded the model as loaded THERE, naming the + // process the worker started. An empty address would mean the install + // reply named none, which is refused now rather than substituted. + var rows []nodeModel + Expect(c.GetJSON(client, 0, "/api/nodes/"+nodeID+"/models", &rows)).To(Succeed()) + Expect(rows).ToNot(BeEmpty(), "no model is recorded on the worker that just served the request") + Expect(rows[0].ModelName).To(Equal("busless-model")) + Expect(rows[0].State).To(Equal("loaded")) + Expect(rows[0].Address).ToNot(BeEmpty(), + "the row names no backend process on the worker, so nothing could address it again") + }) + + // Scenario 2, the relay, and scenario 3, the ordering, in one cluster. + // + // They are one spec because a relay spec needs something to ask a worker + // that only the worker can answer, and a fresh worker's backend list is + // empty: the install is what puts something there. Splitting them would + // have cost a second cluster to assert less. + // + // A wrong implementation answers the control RPC by reaching the worker + // from the replica that took it, which works on one host and nowhere else, + // or refuses it as a worker that is not connected. A wrong STREAM + // implementation writes the terminal reply before the progress it reports + // on, or drops the progress entirely; both pass every unit spec. + It("installs a backend, streams its progress in order, and lists it back, all through the replica that does not own the worker", func() { + c, dsn := startClusterOnFreshDB(2, 1) + client := controlSession(c) + + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), probe.describe) + nodeID := probe.idOf(c.WorkerName(0)) + Expect(nodeID).ToNot(BeEmpty()) + + // Which replica owns the tunnel is READ through the production Owner + // query, not assumed. The harness sends the worker to frontend 0 by + // default, and a spec that wrote that down would keep passing after the + // default changed while quietly testing the owner path instead. + owners := newTunnelOwners(openClusterDB(dsn)) + var owner int + Eventually(func() int { + owner = owners.ownerIndexOf(c, 2, nodeID) + return owner + }, tunnelOwnershipTimeout, tunnelOwnershipPoll).Should(BeNumerically(">=", 0), owners.describe) + + nonOwner := 1 - owner + Expect(owners.ownerIndexOf(c, 2, nodeID)).ToNot(Equal(nonOwner), + "frontend %d owns the worker's tunnel, so a request to it would not be relayed and this spec would prove nothing", nonOwner) + + // Nothing is installed on the worker yet, and the request below is the + // only thing that could change that. + Expect(nodeBackendNames(c, client, nonOwner, nodeID)).ToNot(ContainElement(probeBackend), + "the worker already has %q, so a listing that names it later says nothing about this install", probeBackend) + + gallery := newGatedGallery(probeGalleryIndex(probeBackendSource()), true) + jobID := startNodeInstall(c, client, nonOwner, nodeID, probeBackend, gallery.galleriesJSON()) + job := newJobProbe(c, client, nonOwner, jobID, nodeID) + + // The worker has emitted its first progress line and is now stopped + // inside the gallery fetch. Seeing the phase here proves a progress + // envelope crossed the relay and was decoded, and it proves it BEFORE + // any reply could exist, because the worker cannot produce one until + // this spec releases the gate. + Eventually(job.nodePhase, installJobTimeout, installJobPoll).Should(Equal("resolving"), job.describe) + Eventually(gallery.fetches.Load, installJobTimeout, installJobPoll).Should(BeNumerically(">", 0), + "the worker never fetched the gated gallery, so the gate is not on the path this spec thinks it is") + + // Held, not sampled. A reply arriving in this window is a reply written + // ahead of the work it reports on. + Consistently(job.processed, gateHoldWindow, gateHoldPoll).Should(BeFalse(), + job.explain("the install reported a terminal result while the worker was still blocked fetching its gallery")) + + gallery.release() + + Eventually(job.processed, installJobTimeout, installJobPoll).Should(BeTrue(), job.describe) + Expect(job.jobError()).To(BeEmpty(), "the relayed install failed: %s", job.describe()) + Expect(job.nodeStatus()).To(Equal("success"), job.describe) + + // Nothing follows the terminal reply. A worker that appended a late + // progress line after its reply would move this row back to + // "downloading"; the guard against that is ndjsonStream.done, and this + // is the only place it is exercised over a real stream. + Consistently(job.nodeStatus, gateHoldWindow, gateHoldPoll).Should(Equal("success"), + job.explain("the node's status moved after the install's terminal reply")) + + // A SECOND control verb across the relay, and the one whose answer + // could only have come from the worker: the frontends' own backends + // directories are empty, and this backend was installed on the worker + // alone. + Eventually(func() ([]string, error) { return nodeBackendNames(c, client, nonOwner, nodeID) }, + installJobTimeout, installJobPoll).Should(ContainElement(probeBackend)) + + // And the frontend that answered does not have it itself, so it cannot + // have been reporting its own installation as the worker's. + // + // Read off the filesystem and not from GET /backends: in distributed + // mode that endpoint reports the CLUSTER's backends, so it names this + // one whether the frontend has it or not, and an assertion on it fails + // for a reason that has nothing to do with what is being proven. + backendsDir, err := c.FrontendBackendsDir(nonOwner) + Expect(err).ToNot(HaveOccurred()) + entries, err := os.ReadDir(backendsDir) + Expect(err).ToNot(HaveOccurred()) + for _, e := range entries { + Expect(e.Name()).ToNot(Equal(probeBackend), + "frontend %d installed %q locally, so its answer about the worker may be about itself", nonOwner, probeBackend) + } + + // THIS is what makes every request above a relayed one. It rules out + // the two ways they could have succeeded without a relay: a replica + // that took the tunnel for itself, and the tunnel moving to nonOwner + // mid-spec so that it served directly. Both leave the owner changed. + Expect(owners.ownerIndexOf(c, 2, nodeID)).To(Equal(owner), + "the tunnel is no longer held by frontend %d, so the requests to frontend %d were not necessarily relayed", owner, nonOwner) + }) + + // Scenario 4, absence under replica churn. The catastrophe control. + // + // A worker whose tunnel drops and returns INSIDE the grace must not be + // reaped. A wrong implementation reads a lost tunnel as a departed worker + // and evicts the models it is serving in the seconds before the re-home, + // which is the fleet-wide outage the four-valued presence answer exists to + // prevent. + It("reaps nothing when the replica holding a worker's tunnel dies and the worker re-homes inside the grace", func() { + var balancer *frontendBalancer + c, dsn := startClusterOnFreshDB(2, 1, withMockModel("churn-model"), + withBalancer(&balancer), withReconnectGrace(churnGrace)) + + client := inferenceClient(c) + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), probe.describe) + nodeID := probe.idOf(c.WorkerName(0)) + Expect(nodeID).ToNot(BeEmpty()) + + owners := newTunnelOwners(openClusterDB(dsn)) + var owner int + Eventually(func() int { + owner = owners.ownerIndexOf(c, 2, nodeID) + return owner + }, tunnelOwnershipTimeout, tunnelOwnershipPoll).Should(BeNumerically(">=", 0), owners.describe) + survivor := 1 - owner + + // A model actually loaded on the worker, or there is nothing to reap + // and the assertions below hold vacuously. + expectMockedInference(client, c.FrontendURL(owner), "churn-model", + "inference must work before the owner is killed, or nothing here is at risk") + Expect(nodeModelNames(c, client, survivor, nodeID)).To(ContainElement("churn-model")) + + // The tunnel is taken away BEFORE the owner is killed, and held away + // until the assertions below have run. + // + // Without the block the worker re-homes onto the survivor within + // seconds, and a spec that asserted over those seconds would never + // reach the state it is about: for the first half minute after a + // SIGKILL the dead replica still reads as a live owner, so presence is + // "connected" and no rule about a departed worker has anything to act + // on. Blocking makes the outage last long enough for a departure to be + // recorded, which is the only way the grace is consulted at all. + balancer.blockTunnel.Store(true) + Expect(c.KillFrontend(owner)).To(Succeed()) + Eventually(func() bool { return c.FrontendAlive(owner) }, "20s", "500ms").Should(BeFalse()) + + // Read at the SURVIVOR from here on: the probe above is bound to a dead + // process. + atSurvivor := newRosterProbe(c, client, survivor) + + // Wait for the drop to be REAL rather than assume the kill produced + // one. Until this reads empty the ownership query still reports the + // dead replica, and everything below would be asserting about a worker + // the deployment believes is connected. + Eventually(func() string { return owners.ownerOf(nodeID) }, churnDropTimeout, churnDropPoll). + Should(BeEmpty(), owners.describe) + + // THIS is the window the phase is about: the tunnel is gone, the + // departure is recorded, and the grace has not run out. Nothing may be + // reaped in it. + Consistently(func() []string { + names, err := nodeModelNames(c, client, survivor, nodeID) + if err != nil { + return nil + } + return names + }, churnHoldWindow, churnHoldPoll).Should(ContainElement("churn-model"), + "the model loaded on the worker was reaped while its tunnel was inside the reconnect grace") + Consistently(func() string { return atSurvivor.statusOf(c.WorkerName(0)) }, churnHoldWindow, churnHoldPoll). + ShouldNot(Equal("unhealthy"), + atSurvivor.explain("the worker was demoted while its tunnel was inside the reconnect grace")) + + // And it comes back: with the block lifted the tunnel re-homes onto the + // survivor and the same model serves again. Without this the assertions + // above would be satisfied by a fleet that was simply never touched. + balancer.blockTunnel.Store(false) + Eventually(func() int { return owners.ownerIndexOf(c, 2, nodeID) }, tunnelOwnershipTimeout, tunnelOwnershipPoll). + Should(Equal(survivor), owners.describe) + Eventually(atSurvivor.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), atSurvivor.describe) + Expect(atSurvivor.idOf(c.WorkerName(0))).To(Equal(nodeID), + "the worker re-registered rather than re-homing, so this proves nothing about a tunnel surviving its replica") + + eventuallyMockedInference(client, c.FrontendURL(survivor), "churn-model", + "the survivor must serve the re-homed worker") + }) + + // Scenario 5, the wedge task 6 fixed, plus the agent-worker control. + // + // A worker with a FRESH heartbeat and a tunnel that is gone past the grace + // must stop being reported healthy. Before task 6 it stayed healthy + // forever, with every request for a model on it failing "no route", because + // every reaper keyed on the heartbeat and the heartbeat was fine. + // + // The agent worker in the same cluster is the control for the other + // direction. It still speaks NATS, holds no tunnel and never will, so a + // rule that read "no tunnel" as "gone" would take the whole agent fleet + // down with it. + It("stops reporting a heartbeating worker healthy once its tunnel is gone past the grace, and leaves agent workers alone", func() { + var balancer *frontendBalancer + c, dsn := startClusterOnFreshDB(2, 1, withBalancer(&balancer), + withAgentWorkers(1), withReconnectGrace(10*time.Second)) + + client := controlSession(c) + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(And(ContainElement(c.WorkerName(0)), ContainElement(c.AgentWorkerName(0))), probe.describe) + nodeID := probe.idOf(c.WorkerName(0)) + Expect(nodeID).ToNot(BeEmpty()) + + owners := newTunnelOwners(openClusterDB(dsn)) + var owner int + Eventually(func() int { + owner = owners.ownerIndexOf(c, 2, nodeID) + return owner + }, tunnelOwnershipTimeout, tunnelOwnershipPoll).Should(BeNumerically(">=", 0), owners.describe) + survivor := 1 - owner + + // Take the tunnel away permanently: block the dial, then kill the + // replica holding the live session. The worker keeps registering and + // heartbeating through the balancer, which still proxies everything + // except the tunnel connect. + balancer.blockTunnel.Store(true) + Expect(c.KillFrontend(owner)).To(Succeed()) + Eventually(func() bool { return c.FrontendAlive(owner) }, "20s", "500ms").Should(BeFalse()) + + atSurvivor := newRosterProbe(c, client, survivor) + + // The worker really is trying and really is being refused, so what + // makes it unreachable below is the tunnel and not the worker dying. + before := balancer.tunnelDials.Load() + Eventually(balancer.tunnelDials.Load, "90s", "500ms").Should(BeNumerically(">", before), + "the worker stopped dialling its tunnel, so blocking the dial is not what keeps it away") + + // The verdict. Fresh heartbeat, no tunnel, past the grace. + Eventually(func() string { return atSurvivor.statusOf(c.WorkerName(0)) }, departedTimeout, departedPoll). + Should(Equal("unhealthy"), atSurvivor.describe) + + // The heartbeat was NOT what demoted it. Without this the assertion + // above is satisfied by a worker that simply died, which says nothing + // about tunnel departure. + Expect(atSurvivor.heartbeatOf(c.WorkerName(0))).To(BeTemporally(">", time.Now().Add(-1*time.Minute)), + "the worker's heartbeat is stale, so it was demoted for being gone rather than for having no route") + + // The agent worker, in the same cluster, under the same grace, on the + // same health monitor, is untouched. It holds no tunnel either. + Consistently(func() string { return atSurvivor.statusOf(c.AgentWorkerName(0)) }, "20s", "2s"). + Should(Equal("healthy"), + atSurvivor.explain("an agent worker was demoted by a rule about tunnels, and agent workers never hold one")) + + // And the demotion reverses when the route comes back, so it is a + // statement about the route rather than a one-way condemnation. + balancer.blockTunnel.Store(false) + Eventually(func() int { return owners.ownerIndexOf(c, 2, nodeID) }, tunnelOwnershipTimeout, tunnelOwnershipPoll). + Should(Equal(survivor), owners.describe) + Eventually(func() string { return atSurvivor.statusOf(c.WorkerName(0)) }, departedTimeout, departedPoll). + Should(Equal("healthy"), atSurvivor.describe) + }) + + // Scenario 6. THE NEGATIVE CONTROL FOR THE WHOLE SUITE. + // + // Frontend and worker share a host, so every address the control plane + // names is one the frontend could also have reached directly. If it did, + // every spec above would pass with the tunnel doing nothing. This one takes + // the tunnel away and requires the control plane to become unreachable, + // with the refusal naming the ROUTE, while registration, heartbeats and the + // roster stay exactly as they were. + // + // It cannot use LOCALAI_WORKER_TUNNEL=false: that is a fatal startup error + // after phase 2, and a worker that never started says nothing about a + // worker reachable by some other path. + It("cannot drive the control plane on a worker whose tunnel is refused, reaps nothing for it, and can as soon as it is not", func() { + var balancer *frontendBalancer + c, dsn := startClusterOnFreshDB(1, 1, + withBalancer(&balancer, func(b *frontendBalancer) { b.blockTunnel.Store(true) })) + + client := controlSession(c) + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), probe.describe) + nodeID := probe.idOf(c.WorkerName(0)) + Expect(nodeID).ToNot(BeEmpty()) + + // In every respect the worker the specs above used, except the tunnel, + // and both halves of that are asserted rather than assumed: it tried, + // and no replica holds it. + Eventually(balancer.tunnelDials.Load, "60s", "500ms").Should(BeNumerically(">", 0), + "the worker never dialled its tunnel, so blocking the dial is not what makes it unreachable below") + owners := newTunnelOwners(openClusterDB(dsn)) + Consistently(func() string { return owners.ownerOf(nodeID) }, "5s", "500ms").Should(BeEmpty(), + "a replica holds this worker's tunnel, so the blocker is not blocking") + + gallery := newGatedGallery(probeGalleryIndex(probeBackendSource()), false) + jobID := startNodeInstall(c, client, 0, nodeID, probeBackend, gallery.galleriesJSON()) + job := newJobProbe(c, client, 0, jobID, nodeID) + + Eventually(job.processed, installJobTimeout, installJobPoll).Should(BeTrue(), job.describe) + + // It fails, and it fails for the ROUTING reason. A refusal for any + // other cause (a gallery it could not read, a node it thought absent, a + // backend that would not install) would satisfy "it failed" just as + // well and would leave every spec above unproven. + // + // One substring and not a disjunction: "no route" is what + // cluster.ErrNoRoute reads as, and nothing else on this path produces + // it. In particular it is NOT what an absent worker produces, and that + // distinction is the phase's whole absence contract: a worker this + // replica cannot reach must never be reported as one that has gone. + Expect(job.jobError()).To(ContainSubstring("no route"), + "the refusal does not name the missing route, so this spec cannot tell a worker with no tunnel from an ordinary install failure: %s", job.describe()) + + // And nothing was reaped for it. The worker never stopped + // heartbeating, so a control RPC that could not be delivered must not + // have cost it its row or its health. + _, listErr := nodeBackendNames(c, client, 0, nodeID) + Expect(listErr).To(HaveOccurred(), + "the frontend answered a backend listing for a worker it has no route to, so something other than the tunnel reaches it") + Consistently(probe.healthyNames, "10s", "1s"). + Should(ContainElement(c.WorkerName(0)), + probe.explain("the worker was demoted or removed because a control RPC could not be routed to it")) + + // The control's own control: put the tunnel back, change nothing else, + // and the SAME install must now succeed. Without this the failure above + // could be any of the ordinary reasons an e2e install fails. + balancer.blockTunnel.Store(false) + Eventually(func() int { return owners.ownerIndexOf(c, 1, nodeID) }, tunnelOwnershipTimeout, tunnelOwnershipPoll). + Should(Equal(0), owners.describe) + + retryID := startNodeInstall(c, client, 0, nodeID, probeBackend, gallery.galleriesJSON()) + retry := newJobProbe(c, client, 0, retryID, nodeID) + Eventually(retry.processed, installJobTimeout, installJobPoll).Should(BeTrue(), retry.describe) + Expect(retry.jobError()).To(BeEmpty(), + "the only thing that changed is the tunnel, so the failure above was the missing tunnel and nothing else: %s", retry.describe()) + Eventually(func() ([]string, error) { return nodeBackendNames(c, client, 0, nodeID) }, + installJobTimeout, installJobPoll).Should(ContainElement(probeBackend)) + }) +}) diff --git a/tests/e2e/distributed/control_workers_test.go b/tests/e2e/distributed/control_workers_test.go index 15ef583b6088..e8d5bd2bb658 100644 --- a/tests/e2e/distributed/control_workers_test.go +++ b/tests/e2e/distributed/control_workers_test.go @@ -3,6 +3,7 @@ package distributed_test import ( "context" "encoding/json" + "fmt" "io" "net" "net/http" @@ -13,6 +14,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + clustersvc "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/workerctl" ) @@ -110,3 +113,78 @@ func (c *ControlWorkers) serve(w http.ResponseWriter, r *http.Request) { // The reply line, and it is the last thing on the body by contract. Expect(json.NewEncoder(w).Encode(workerctl.Envelope{Reply: raw})).To(Succeed()) } + +// ServeBackendLifecycle registers the three verbs every routing spec needs from +// a worker: install, the running-model list, and the backend list. +// +// The install reply NAMES the address of the backend process the fake worker +// started, and that is the half these suites used to leave out. Since phase 2 a +// worker advertises no address of its own, so this string is the only thing +// that tells the frontend WHICH process on that worker a model was loaded into, +// and installBackendOnNode refuses a success reply that omits it rather than +// substituting anything. Every one of these suites runs its mock gRPC backend +// on loopback and records where it listens in the node row it registers, so +// that row is where this reads it back from. It is the spec's own bookkeeping +// standing in for what a real worker reports about its own process; nothing in +// production reads BackendNode.Address any more. +func (c *ControlWorkers) ServeBackendLifecycle(registry *nodes.NodeRegistry) { + c.On(AnyNode, workerctl.PathBackendInstall, func(nodeID string, _ []byte) any { + node, err := registry.Get(context.Background(), nodeID) + if err != nil { + return messaging.BackendInstallReply{Success: false, Error: err.Error()} + } + return messaging.BackendInstallReply{Success: true, WorkerLocalAddress: node.Address} + }) + c.On(AnyNode, workerctl.PathModelsRunning, func(string, []byte) any { + return messaging.ModelsRunningReply{} + }) + c.On(AnyNode, workerctl.PathBackendList, func(string, []byte) any { + return messaging.BackendListReply{} + }) +} + +// workerBackendDialerFor stands in for the worker tunnel on the gRPC path. +// +// A frontend no longer dials a backend process: it opens a stream on the +// worker's tunnel and names the process by its worker-local address. These +// suites run the process on loopback, so a TCP dial to that address is the +// stand-in, and NewTunnelClientFactory below is what makes the specs go through +// a dialer at all rather than through the direct dial the default factory now +// refuses. +// +// The refusal is translated, and that is the load-bearing half. A real worker +// whose backend process has died answers the stream with +// ErrStreamTargetUnavailable, which cluster.IsWorkerAnswer reads as the WORKER +// speaking about its backend, and every reap guard in core/services/nodes acts +// only on that. A bare ECONNREFUSED from net.Dialer carries no such thing and +// reaches those guards as "no route", which reaps nothing. A double that +// reported the raw syscall error could therefore never fail the way production +// fails, and the stale-record spec in router_tracking_test.go would be +// asserting against a transport that cannot produce the condition it is about. +// That is not a hypothetical: replacing this translation with the raw error +// reddens exactly that spec and nothing else. +func workerBackendDialerFor(_ string) func(ctx context.Context, addr string) (net.Conn, error) { + return func(ctx context.Context, addr string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, "tcp", addr) + if err != nil { + return nil, fmt.Errorf("%w: %v", clustersvc.ErrStreamTargetUnavailable, err) + } + return conn, nil + } +} + +// tunnelBackendClients is the BackendClientFactory a SmartRouter needs in these +// suites. +// +// It is NewTunnelClientFactory and not a bespoke double on purpose: the default +// factory refuses every request now (see nodes.ErrNoWorkerDialer), so a spec +// that omitted this would fail at the first inference with a boot-time +// misconfiguration rather than testing anything, and a bespoke factory that +// dialled directly would put back the bypass the phase removed. +func tunnelBackendClients() nodes.BackendClientFactory { + GinkgoHelper() + factory, err := nodes.NewTunnelClientFactory("", workerBackendDialerFor) + Expect(err).ToNot(HaveOccurred()) + return factory +} diff --git a/tests/e2e/distributed/distributed_full_flow_test.go b/tests/e2e/distributed/distributed_full_flow_test.go index abf0cbb72288..b6b2ceeca1c4 100644 --- a/tests/e2e/distributed/distributed_full_flow_test.go +++ b/tests/e2e/distributed/distributed_full_flow_test.go @@ -10,7 +10,6 @@ import ( "path/filepath" "time" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/workerctl" "github.com/mudler/LocalAI/pkg/grpc/base" @@ -226,22 +225,22 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() // newTestSmartRouter creates a SmartRouter reaching a fleet of fake workers // over the tunnelled control plane, with a backend.install handler that - // always replies success for every registered node. + // replies success for every registered node and names where that node's + // backend process listens. + // + // Both halves of the post-tunnel contract are here rather than in the + // specs, because they are the same two facts in every one of them: an + // install reply that names no process address is refused + // (installBackendOnNode), and a frontend with no worker dialer reaches no + // backend at all (nodes.ErrNoWorkerDialer). newTestSmartRouter := func(reg *nodes.NodeRegistry, extraOpts ...nodes.SmartRouterOptions) *nodes.SmartRouter { workers := NewControlWorkers() - workers.On(AnyNode, workerctl.PathBackendInstall, func(string, []byte) any { - return messaging.BackendInstallReply{Success: true} - }) - workers.On(AnyNode, workerctl.PathModelsRunning, func(string, []byte) any { - return messaging.ModelsRunningReply{} - }) - workers.On(AnyNode, workerctl.PathBackendList, func(string, []byte) any { - return messaging.BackendListReply{} - }) + workers.ServeBackendLifecycle(reg) unloader := nodes.NewRemoteUnloaderAdapter(reg, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) opts := nodes.SmartRouterOptions{ - Unloader: unloader, + Unloader: unloader, + ClientFactory: tunnelBackendClients(), } if len(extraOpts) > 0 { o := extraOpts[0] @@ -332,8 +331,12 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() Expect(registry.Register(context.Background(), node2, true)).To(Succeed()) // Set both as having the model loaded - Expect(registry.SetNodeModel(context.Background(), node1.ID, "test-model", 0, "loaded", "", 0)).To(Succeed()) - Expect(registry.SetNodeModel(context.Background(), node2.ID, "test-model", 0, "loaded", "", 0)).To(Succeed()) + // The address is where that node's backend process listens, which is + // what a real install reply would have recorded on the row. A row + // carrying none names no process, so the router would re-install rather + // than route to it and this spec would be measuring the install path. + Expect(registry.SetNodeModel(context.Background(), node1.ID, "test-model", 0, "loaded", addr1, 0)).To(Succeed()) + Expect(registry.SetNodeModel(context.Background(), node2.ID, "test-model", 0, "loaded", addr2, 0)).To(Succeed()) // Set node-1 with high in-flight (5), node-2 with low in-flight (1) for range 5 { diff --git a/tests/e2e/distributed/nats_jwt_helpers_test.go b/tests/e2e/distributed/nats_jwt_helpers_test.go index 80060ef6a801..74f74355d098 100644 --- a/tests/e2e/distributed/nats_jwt_helpers_test.go +++ b/tests/e2e/distributed/nats_jwt_helpers_test.go @@ -153,4 +153,4 @@ func accountPublicKeyFromSeed(accountSeed string) string { func nodeSubjectPrefix(nodeID string) string { tok := strings.NewReplacer(".", "-", "*", "-", ">", "-", " ", "-", "\t", "-", "\n", "-").Replace(nodeID) return "nodes." + tok -} \ No newline at end of file +} diff --git a/tests/e2e/distributed/router_tracking_test.go b/tests/e2e/distributed/router_tracking_test.go index 6a4b545e0a3f..a3f92c9ef52c 100644 --- a/tests/e2e/distributed/router_tracking_test.go +++ b/tests/e2e/distributed/router_tracking_test.go @@ -4,9 +4,7 @@ import ( "context" "time" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" - "github.com/mudler/LocalAI/core/services/workerctl" "github.com/mudler/LocalAI/pkg/grpc/base" pb "github.com/mudler/LocalAI/pkg/grpc/proto" @@ -58,24 +56,21 @@ var _ = Describe("SmartRouter trackingKey", Label("Distributed"), func() { registry, err = nodes.NewNodeRegistry(db) Expect(err).ToNot(HaveOccurred()) - // Mock control plane — backend.install always replies success. + // Mock control plane. The install reply names where the backend process + // listens on that worker, which is what the frontend routes to now that + // a worker advertises no address of its own. workers := NewControlWorkers() - workers.On(AnyNode, workerctl.PathBackendInstall, func(string, []byte) any { - return messaging.BackendInstallReply{Success: true} - }) - workers.On(AnyNode, workerctl.PathModelsRunning, func(string, []byte) any { - return messaging.ModelsRunningReply{} - }) - workers.On(AnyNode, workerctl.PathBackendList, func(string, []byte) any { - return messaging.BackendListReply{} - }) + workers.ServeBackendLifecycle(registry) // Start a mock gRPC backend using the same helper as full flow tests llm := &trackingTestLLM{} grpcAddr, grpcCleanup, err = startTestGRPCServer(grpcPkg.AIModel(llm)) Expect(err).ToNot(HaveOccurred()) - // Register a node pointing to the mock backend + // Register a node whose backend process is the mock server above. The + // address is the spec's own record of where that process listens; the + // fake worker reports it back on install, and nothing in production + // reads this column any more. node := &nodes.BackendNode{ Name: "tracking-node", Address: grpcAddr, } @@ -85,6 +80,11 @@ var _ = Describe("SmartRouter trackingKey", Label("Distributed"), func() { unloader := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) router = nodes.NewSmartRouter(registry, nodes.SmartRouterOptions{ Unloader: unloader, + // Without a worker dialer the default factory refuses every + // request (nodes.ErrNoWorkerDialer), which is a boot-time + // misconfiguration rather than a routing outcome any of these + // specs is about. + ClientFactory: tunnelBackendClients(), }) }) From b35e1b4708fc39d835cbb84184b9e6ed76b7221f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 06:02:20 +0000 Subject: [PATCH 080/153] refactor(distributed): one subject matcher, shared by the carrier and the doubles Three copies of "does this filter match this subject" lived in the tree: one in testutil.FakeBus, a byte-identical second inside galleryop's own private fakeBus, and a third in pkg/natsauth with different semantics. The first two are doubles the specs publish through, and the carrier the pgbus work is about to add needs the same rule in production. Two spellings drift, and the drift reads as a peer that receives an event on one replica and not on another. messaging.SubjectMatches is now the only definition either double uses. The natsauth copy stays: it matches a NATS server allow list, so it has to implement the '>' tail wildcard this one deliberately refuses, and Task 16 deletes that package anyway. '>' is refused rather than implemented because no surviving subscription uses it, and a caller who writes one must get no messages rather than silently getting every message on the prefix. The refusal is checked BEFORE the filter == subject fast path: a verbatim port checks equality first, and then the filter "a.>" matches the literal subject "a.>", which is the contract leaking. One table row pins that ordering and it is the only row that does. messaging.ValidFilter refuses an empty filter, a '>' filter and an empty token so a subscriber is told at subscribe time instead of staying silently empty for the life of the process. FakeBus.Subscribe calls it, which is what keeps the double honest about what the carrier will do, and three new testutil specs pin that wiring: the previous state of the tree had no spec at all that failed when the double's wildcard routing was replaced by exact matching in any package the plan named. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .../galleryop/distributed_sync_test.go | 26 +------ core/services/messaging/match.go | 71 +++++++++++++++++ core/services/messaging/match_test.go | 76 +++++++++++++++++++ core/services/testutil/fakebus.go | 34 ++------- core/services/testutil/fakebus_test.go | 56 ++++++++++++++ 5 files changed, 214 insertions(+), 49 deletions(-) create mode 100644 core/services/messaging/match.go create mode 100644 core/services/messaging/match_test.go create mode 100644 core/services/testutil/fakebus_test.go diff --git a/core/services/galleryop/distributed_sync_test.go b/core/services/galleryop/distributed_sync_test.go index b1c9a31202bc..ed88ab25be37 100644 --- a/core/services/galleryop/distributed_sync_test.go +++ b/core/services/galleryop/distributed_sync_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "strings" "sync" "time" @@ -20,7 +19,8 @@ import ( // fakeBus is an in-memory MessagingClient that delivers each published // message synchronously to every registered subscriber whose subject filter -// matches, including NATS-style wildcard subjects (`*` matches one token). +// matches, using messaging.SubjectMatches so this double and the carrier agree +// on what a wildcard filter means. // // Synchronous delivery keeps the specs deterministic: the moment Publish // returns, every subscriber's handler has run, so the spec body can read @@ -37,26 +37,6 @@ type fakeBusSub struct { func newFakeBus() *fakeBus { return &fakeBus{} } -func subjectMatches(filter, subject string) bool { - if filter == subject { - return true - } - fp := strings.Split(filter, ".") - sp := strings.Split(subject, ".") - if len(fp) != len(sp) { - return false - } - for i := range fp { - if fp[i] == "*" { - continue - } - if fp[i] != sp[i] { - return false - } - } - return true -} - func (b *fakeBus) Publish(subject string, data any) error { payload, err := json.Marshal(data) if err != nil { @@ -66,7 +46,7 @@ func (b *fakeBus) Publish(subject string, data any) error { subs := append([]fakeBusSub(nil), b.subs...) b.mu.Unlock() for _, s := range subs { - if subjectMatches(s.subject, subject) { + if messaging.SubjectMatches(s.subject, subject) { s.handler(payload) } } diff --git a/core/services/messaging/match.go b/core/services/messaging/match.go new file mode 100644 index 000000000000..9acf4ebc4a73 --- /dev/null +++ b/core/services/messaging/match.go @@ -0,0 +1,71 @@ +package messaging + +import ( + "errors" + "fmt" + "strings" +) + +// ErrUnsupportedFilter is the class every ValidFilter rejection belongs to, so a +// carrier can tell "this caller asked for something we do not implement" apart +// from "the store is unhappy" without string matching. +var ErrUnsupportedFilter = errors.New("unsupported subject filter") + +// SubjectMatches reports whether a subscription filter matches a concrete +// subject, honoring the single-token `*` wildcard used by NATS. +// +// This is the single definition of subject matching for the whole tree: the +// carrier that delivers messages in production and the in-memory doubles the +// specs publish through both call it. When those were separate copies, a filter +// could match on one and not the other, and the difference read as a peer that +// received an event on one replica and missed it on another. +// +// A `>` tail wildcard is deliberately NOT implemented and makes the filter +// match nothing, so a caller who writes one receives no messages rather than +// silently receiving every message on the prefix. The refusal is checked before +// the exact-equality fast path, otherwise `a.>` would still match the literal +// subject `a.>`. +func SubjectMatches(filter, subject string) bool { + // sanitizeSubjectToken folds '>' out of every generated token, so a '>' + // anywhere in a filter can only be a hand-written tail wildcard. + if strings.Contains(filter, ">") { + return false + } + if filter == subject { + return true + } + fp := strings.Split(filter, ".") + sp := strings.Split(subject, ".") + if len(fp) != len(sp) { + return false + } + for i := range fp { + if fp[i] == "*" { + continue + } + if fp[i] != sp[i] { + return false + } + } + return true +} + +// ValidFilter reports whether a filter is one SubjectMatches can act on, so a +// subscriber is refused at subscribe time instead of staying silently empty for +// the life of the process. It rejects an empty filter, a filter containing the +// unimplemented `>` tail wildcard, and a filter with an empty token (`a..b`), +// which no subject generator can ever produce. +func ValidFilter(filter string) error { + if filter == "" { + return fmt.Errorf("%w: filter is empty", ErrUnsupportedFilter) + } + if strings.Contains(filter, ">") { + return fmt.Errorf("%w: %q uses the tail wildcard '>', which is not implemented", ErrUnsupportedFilter, filter) + } + for _, token := range strings.Split(filter, ".") { + if token == "" { + return fmt.Errorf("%w: %q has an empty token", ErrUnsupportedFilter, filter) + } + } + return nil +} diff --git a/core/services/messaging/match_test.go b/core/services/messaging/match_test.go new file mode 100644 index 000000000000..4804768cadd7 --- /dev/null +++ b/core/services/messaging/match_test.go @@ -0,0 +1,76 @@ +package messaging_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/messaging" +) + +// These rows are the shared contract between the PostgreSQL-backed carrier and +// the in-memory doubles. Both spellings of "does this filter match this +// subject" used to be copied by hand, and a drift between them reads as a peer +// that receives an event on one replica and not on another. +var _ = DescribeTable("SubjectMatches", + func(filter, subject string, expected bool) { + Expect(messaging.SubjectMatches(filter, subject)).To(Equal(expected)) + }, + Entry("an exact subject matches itself", "jobs.new", "jobs.new", true), + Entry("a different literal does not match", "jobs.new", "jobs.done", false), + Entry("matching is case sensitive", "Jobs.new", "jobs.new", false), + + Entry("two wildcards each consume one token", + "agent.*.events.*", "agent.myagent.events.user1", true), + Entry("a wildcard does not match a missing token", + "agent.*.events.*", "agent.myagent.events", false), + Entry("a wildcard does not swallow two tokens", + "agent.*.events.*", "agent.a.b.events.u", false), + Entry("a wildcard does not match a longer subject", + "agent.*.events.*", "agent.myagent.events.user1.extra", false), + + Entry("a middle wildcard matches one token", + "jobs.*.progress", "jobs.abc.progress", true), + Entry("a middle wildcard does not match two tokens", + "jobs.*.progress", "jobs.abc.def.progress", false), + Entry("a middle wildcard does not match zero tokens", + "jobs.*.progress", "jobs.progress", false), + + Entry("a bare wildcard matches exactly one token", "*", "jobs", true), + Entry("a bare wildcard does not match two tokens", "*", "jobs.new", false), + + // The sanitizer folds '.', '*' and '>' out of an id, so a model id that + // looks like several tokens still occupies exactly one, which is the only + // reason the staging wildcard can be a single '*'. + Entry("the staging wildcard matches a sanitized multi-part model id", + messaging.SubjectStagingProgressWildcard, messaging.SubjectStagingProgress("a.b.c"), true), + + // '>' is refused rather than implemented: no surviving subscription uses + // it, and a caller who writes one must get nothing rather than everything. + Entry("a tail wildcard filter matches nothing", "a.>", "a.b", false), + Entry("a tail wildcard filter matches nothing even one token deep", "a.>", "a.b.c", false), + Entry("a bare tail wildcard matches nothing", ">", "anything", false), + // This row is the one that proves the '>' refusal is checked BEFORE the + // exact-equality fast path, which is where a naive implementation leaks. + Entry("a tail wildcard filter does not even match itself", "a.>", "a.>", false), +) + +var _ = DescribeTable("ValidFilter", + func(filter string, wantErr bool) { + err := messaging.ValidFilter(filter) + if wantErr { + Expect(err).To(HaveOccurred()) + return + } + Expect(err).ToNot(HaveOccurred()) + }, + Entry("an empty filter is refused", "", true), + Entry("a tail wildcard is refused", "a.>", true), + Entry("a bare tail wildcard is refused", ">", true), + Entry("an embedded tail wildcard is refused", "a.>.b", true), + Entry("an empty middle token is refused", "a..b", true), + Entry("an empty leading token is refused", ".a", true), + Entry("an empty trailing token is refused", "a.", true), + Entry("a single-token wildcard is accepted", "a.*.b", false), + Entry("a literal filter is accepted", "a.b", false), + Entry("a bare wildcard is accepted", "*", false), +) diff --git a/core/services/testutil/fakebus.go b/core/services/testutil/fakebus.go index 7452d810fb99..f2890d7ecc29 100644 --- a/core/services/testutil/fakebus.go +++ b/core/services/testutil/fakebus.go @@ -2,7 +2,6 @@ package testutil import ( "encoding/json" - "strings" "sync" "time" @@ -11,8 +10,8 @@ import ( // FakeBus is an in-memory messaging.MessagingClient that delivers each published // message synchronously to every registered subscriber whose subject filter -// matches, including NATS-style wildcard subjects (`*` matches exactly one -// token). +// matches. Matching is messaging.SubjectMatches, the same function the real +// carrier uses, so a filter that fires here fires in production too. // // Synchronous delivery keeps specs deterministic: the moment Publish returns, // every matching subscriber's handler has already run, so the spec body can read @@ -43,28 +42,6 @@ func NewFakeBus() *FakeBus { return &FakeBus{publishCounts: map[string]int{}} } -// subjectMatches reports whether a subscription filter matches a concrete -// subject, honoring the single-token `*` wildcard used by NATS. -func subjectMatches(filter, subject string) bool { - if filter == subject { - return true - } - fp := strings.Split(filter, ".") - sp := strings.Split(subject, ".") - if len(fp) != len(sp) { - return false - } - for i := range fp { - if fp[i] == "*" { - continue - } - if fp[i] != sp[i] { - return false - } - } - return true -} - // Publish marshals data as JSON and delivers it synchronously to every matching // subscriber. func (b *FakeBus) Publish(subject string, data any) error { @@ -77,7 +54,7 @@ func (b *FakeBus) Publish(subject string, data any) error { subs := append([]fakeBusSub(nil), b.subs...) b.mu.Unlock() for _, s := range subs { - if subjectMatches(s.subject, subject) { + if messaging.SubjectMatches(s.subject, subject) { s.handler(payload) } } @@ -108,7 +85,12 @@ func (s *fakeBusSubscription) Unsubscribe() error { return nil } +// Subscribe refuses the same filters the real carrier refuses, so a spec cannot +// register a filter that would silently never fire in production. func (b *FakeBus) Subscribe(subject string, handler func([]byte)) (messaging.Subscription, error) { + if err := messaging.ValidFilter(subject); err != nil { + return nil, err + } sub := fakeBusSub{subject: subject, handler: handler} b.mu.Lock() b.subs = append(b.subs, sub) diff --git a/core/services/testutil/fakebus_test.go b/core/services/testutil/fakebus_test.go new file mode 100644 index 000000000000..977883d73e60 --- /dev/null +++ b/core/services/testutil/fakebus_test.go @@ -0,0 +1,56 @@ +package testutil_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// FakeBus stands in for the carrier in every cross-replica-sync spec in the +// tree. If it routed by its own rules, those specs would be proving a delivery +// behaviour production does not have, so these pin that it routes by the +// shared matcher and refuses the filters the carrier refuses. +var _ = Describe("FakeBus subject routing", func() { + It("delivers to a wildcard subscriber, not only to an exact one", func() { + bus := testutil.NewFakeBus() + + delivered := make(chan []byte, 4) + _, err := bus.Subscribe(messaging.SubjectStagingProgressWildcard, func(payload []byte) { + delivered <- payload + }) + Expect(err).ToNot(HaveOccurred()) + + Expect(bus.Publish(messaging.SubjectStagingProgress("a.b.c"), map[string]string{"id": "staging-payload"})).To(Succeed()) + + var payload []byte + Eventually(delivered).Should(Receive(&payload)) + Expect(string(payload)).To(ContainSubstring("staging-payload")) + }) + + It("does not deliver a subject the wildcard cannot match", func() { + bus := testutil.NewFakeBus() + + delivered := make(chan []byte, 4) + _, err := bus.Subscribe("jobs.*.progress", func(payload []byte) { + delivered <- payload + }) + Expect(err).ToNot(HaveOccurred()) + + Expect(bus.Publish("jobs.abc.def.progress", map[string]string{"id": "abc"})).To(Succeed()) + Consistently(delivered).ShouldNot(Receive()) + }) + + It("refuses a filter the carrier cannot honour instead of never firing", func() { + bus := testutil.NewFakeBus() + + sub, err := bus.Subscribe("jobs.>", func([]byte) {}) + Expect(err).To(MatchError(messaging.ErrUnsupportedFilter)) + Expect(sub).To(BeNil()) + + sub, err = bus.Subscribe("", func([]byte) {}) + Expect(err).To(MatchError(messaging.ErrUnsupportedFilter)) + Expect(sub).To(BeNil()) + }) +}) From e7d322c6cc735deef42aeec473ed92401323c4ee Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 06:28:03 +0000 Subject: [PATCH 081/153] feat(distributed): carry fan-out on PostgreSQL LISTEN/NOTIFY Distributed mode needs an operator to run a NATS cluster. This adds the carrier that replaces its fan-out half, so a deployment eventually needs PostgreSQL and its own HTTP listener and nothing else. pgbus holds one PostgreSQL session per replica, pinned for the life of the process because LISTEN registrations belong to one backend session and a pooled handle would lose them on the next checkout. Publishes go out on the pool with pg_notify. Subjects map onto a channel by their first token, from a closed set of roots. A subject outside the set is refused at publish AND at subscribe rather than mapped to a channel of its own: a channel name is capped at 63 bytes, and one LISTEN per job id would be unbounded. Refused rather than dropped, because a subject that goes nowhere and reports nothing is the class of defect this work exists to remove. PostgreSQL refuses a notify payload of 8000 bytes or more, and several subjects on this bus exceed that in normal operation: a job result carries a whole LLM output, a gallery progress event carries one entry per node. Those are written to a row and the notification carries the id. What is measured against the cap is the ENCODED notification, not the caller's payload, because the subject and the envelope travel too. The filter grammar is not respelled here. Subscribe asks messaging.ValidFilter and delivery asks messaging.SubjectMatches, which makes this the first production caller of a matcher that had only test doubles. New refuses a DSN that names a different database from the pool: that pairing publishes successfully, delivers nothing, on every replica, and reports no error anywhere. Nothing publishes on it and nothing subscribes yet. The construction is wired anyway, because the DSN has exactly one legitimate source and a setting that decides whether any broadcast is delivered should not be invented by whichever call site is migrated first. Delivery is at-most-once, like NATS core. Nothing downstream may read a message it did not receive as evidence about a node: a carrier that cannot deliver is not a worker that is gone. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 56 ++ core/application/distributed_test.go | 80 +++ core/services/messaging/interfaces.go | 35 ++ core/services/pgbus/bus.go | 629 ++++++++++++++++++++++ core/services/pgbus/bus_test.go | 340 ++++++++++++ core/services/pgbus/pgbus_suite_test.go | 15 + core/services/pgbus/spill.go | 53 ++ core/services/pgbus/spill_test.go | 200 +++++++ core/services/testutil/testdb.go | 23 +- docs/content/features/distributed-mode.md | 1 + go.mod | 2 +- 11 files changed, 1431 insertions(+), 3 deletions(-) create mode 100644 core/application/distributed_test.go create mode 100644 core/services/pgbus/bus.go create mode 100644 core/services/pgbus/bus_test.go create mode 100644 core/services/pgbus/pgbus_suite_test.go create mode 100644 core/services/pgbus/spill.go create mode 100644 core/services/pgbus/spill_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index 1e1bc66c21c4..974c6f783da4 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -21,6 +21,7 @@ import ( "github.com/mudler/LocalAI/core/services/monitoring" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/nodes/prefixcache" + "github.com/mudler/LocalAI/core/services/pgbus" "github.com/mudler/LocalAI/core/services/storage" "github.com/mudler/LocalAI/internal" "github.com/mudler/LocalAI/pkg/distributedhdr" @@ -48,6 +49,13 @@ type DistributedServices struct { Unloader *nodes.RemoteUnloaderAdapter ModelCleanup *nodes.ModelCleanupService + // Bus is the deployment's fan-out carrier, riding the auth database's + // PostgreSQL rather than a message broker. Nothing publishes on it and + // nothing subscribes yet; it is built at boot because its DSN has exactly + // one legitimate source and that has to be settled once, here, rather than + // invented by whichever call site is migrated onto it first. + Bus *pgbus.Bus + // Cluster is the replica-membership registry: which frontend replicas are // alive, at which address, and which of them holds a given worker's tunnel. Cluster *cluster.Registry @@ -117,6 +125,13 @@ func (ds *DistributedServices) Shutdown() { if ds.Nats != nil { ds.Nats.Close() } + // The broadcast carrier holds a PostgreSQL session pinned for the life + // of the process, plus the goroutine parked on it. A replica that + // leaves one behind on every restart runs the server out of + // connections, and the symptom lands on whatever connects next. + if ds.Bus != nil { + ds.Bus.Close() + } xlog.Info("Distributed services shut down") }) } @@ -206,6 +221,21 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade return nil, fmt.Errorf("distributed mode requires auth database to be initialized first") } + // The fan-out carrier, opened before anything that might want it. It is + // built here and not by its first adopter because its DSN has one + // legitimate source, and a setting that decides whether every broadcast in + // the deployment is delivered should not be settled under the time pressure + // of a migration. + bus, err := newBroadcastBus(cfg.Context, cfg, authDB) + if err != nil { + return nil, err + } + defer func() { + if !success { + bus.Close() + } + }() + registry, err := nodes.NewNodeRegistry(authDB) if err != nil { return nil, fmt.Errorf("initializing node registry: %w", err) @@ -665,9 +695,35 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade Tunnels: tunnels, WorkerDialer: workerDialer, BackendClients: backendClients, + Bus: bus, }, nil } +// newBroadcastBus opens the deployment's fan-out carrier on the auth database. +// +// The DSN is cfg.Auth.DatabaseURL and it may never be anything else. A second +// source, a flag of its own or a value read from the environment, would let the +// pinned LISTEN connection and the connection pool address two different +// databases; that carrier publishes successfully, delivers nothing, on every +// replica, and reports no error anywhere. isPostgresURL above has already +// refused a value this carrier could not use. +// +// It is a function rather than four lines inside initDistributed so that the +// equality can be pinned by a spec. initDistributed opens NATS before it +// reaches this point and so cannot be called from a unit test, which would +// leave the assignment as one line in a long function that compiles perfectly +// well when it names the wrong field. +func newBroadcastBus(ctx context.Context, cfg *config.ApplicationConfig, authDB *gorm.DB) (*pgbus.Bus, error) { + if err := pgbus.Migrate(ctx, authDB); err != nil { + return nil, fmt.Errorf("migrating the broadcast carrier: %w", err) + } + bus, err := pgbus.New(ctx, pgbus.Config{DSN: cfg.Auth.DatabaseURL, DB: authDB}) + if err != nil { + return nil, fmt.Errorf("opening the broadcast carrier: %w", err) + } + return bus, nil +} + // unadvertisedNagInterval is how often a replica that could not advertise // itself says so again. // diff --git a/core/application/distributed_test.go b/core/application/distributed_test.go new file mode 100644 index 000000000000..a822e33ff830 --- /dev/null +++ b/core/application/distributed_test.go @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MIT + +package application + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// The guard on the one setting that decides whether any broadcast in the +// deployment is ever delivered. +// +// The carrier holds a pinned LISTEN connection opened from a DSN, and publishes +// travel on a pooled handle opened from another. When those two name different +// databases every publish succeeds, every subscribe succeeds, and nothing +// arrives, on every replica, with no error anywhere. There is exactly one +// legitimate DSN, and these specs are what say so in a way that fails when it +// stops being true. +var _ = Describe("opening the deployment's broadcast carrier", func() { + It("listens on the same database URL the auth pool was built from", func() { + db, dsn := testutil.SetupTestDBWithDSN() + cfg := &config.ApplicationConfig{} + cfg.Auth.DatabaseURL = dsn + + bus, err := newBroadcastBus(context.Background(), cfg, db) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(bus.Close) + + // Equality with the field, not "is a PostgreSQL URL": the failure being + // excluded is two databases, and any DSN passes a shape check. + Expect(bus.DSN()).To(Equal(cfg.Auth.DatabaseURL)) + }) + + It("migrates the spill table, so an oversized broadcast has somewhere to go", func() { + db, dsn := testutil.SetupTestDBWithDSN() + cfg := &config.ApplicationConfig{} + cfg.Auth.DatabaseURL = dsn + + bus, err := newBroadcastBus(context.Background(), cfg, db) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(bus.Close) + + Expect(db.Migrator().HasTable(&pgbus.BusMessage{})).To(BeTrue()) + }) + + It("refuses to open a carrier whose DSN is not the pool's database", func() { + db, _ := testutil.SetupTestDBWithDSN() + _, otherDSN := testutil.SetupTestDBWithDSN() + cfg := &config.ApplicationConfig{} + cfg.Auth.DatabaseURL = otherDSN + + _, err := newBroadcastBus(context.Background(), cfg, db) + + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("shutting the distributed services down", func() { + It("closes the broadcast carrier", func() { + // A pinned PostgreSQL session and the goroutine parked on it, per + // replica restart. Nothing else in this process ever closes it, so the + // line in the shutdown closure is the whole lifecycle. + db, dsn := testutil.SetupTestDBWithDSN() + cfg := &config.ApplicationConfig{} + cfg.Auth.DatabaseURL = dsn + bus, err := newBroadcastBus(context.Background(), cfg, db) + Expect(err).ToNot(HaveOccurred()) + Expect(bus.IsConnected()).To(BeTrue()) + + (&DistributedServices{Bus: bus}).Shutdown() + + Expect(bus.IsConnected()).To(BeFalse()) + }) +}) diff --git a/core/services/messaging/interfaces.go b/core/services/messaging/interfaces.go index 863b1d66efc5..a3fdbdbbb49c 100644 --- a/core/services/messaging/interfaces.go +++ b/core/services/messaging/interfaces.go @@ -25,3 +25,38 @@ type MessagingClient interface { IsConnected() bool Close() } + +// Broadcaster is the fan-out half of the messaging surface: a publish reaches +// every subscriber on every replica. It exists so a call site can be moved onto +// the PostgreSQL carrier without waiting for the request/reply and queue-group +// halves to be retired, because both *messaging.Client and *pgbus.Bus satisfy +// it. +// +// IsConnected and Close are deliberately NOT here, and their absence is stated +// rather than left to be inferred, because a reader who knows MessagingClient +// assumes the smaller interface simply forgot them. +// +// IsConnected has no production consumer: its only non-test occurrences are its +// implementations and the MessagingClient line itself. It is asserted by specs +// and read by logging, and a carrier's consumers must not branch on it. "The +// carrier is down" is not one of the four conditions a node's state can be in, +// and no code may turn it into evidence that a worker is absent. +// +// Close DOES have real callers, and they hold a CONCRETE type rather than this +// interface, which is why the interface can omit it. Before any of them is +// narrowed from MessagingClient to Broadcaster, re-run +// +// grep -rn 'Close()' --include='*.go' . | grep -v _test +// +// because that grep is the only thing standing between "the interface does not +// need it" and a lifecycle that silently stops running. +type Broadcaster interface { + Publisher + Subscribe(subject string, handler func([]byte)) (Subscription, error) +} + +// The concrete NATS client is one of the two carriers this interface exists to +// make interchangeable. Asserted here rather than left to the first adopter, so +// a change to Client's signatures fails to compile in the package that owns the +// interface instead of in whichever call site is migrated next. +var _ Broadcaster = (*Client)(nil) diff --git a/core/services/pgbus/bus.go b/core/services/pgbus/bus.go new file mode 100644 index 000000000000..f16d664c2d2b --- /dev/null +++ b/core/services/pgbus/bus.go @@ -0,0 +1,629 @@ +// SPDX-License-Identifier: MIT + +// Package pgbus carries fan-out broadcasts between frontend replicas over the +// PostgreSQL a distributed deployment already runs, using LISTEN/NOTIFY. +// +// It exists so that a deployment needs a database and its own HTTP listener and +// nothing else. It replaces the fan-out half of the NATS surface only: +// request/reply and queue groups are not here. +// +// Delivery is at-most-once, and deliberately so. A NOTIFY reaches the sessions +// that are LISTENing when it is issued and nobody else, so a replica that is +// reconnecting misses what was published in that window, exactly as a NATS core +// subscriber does. Two consequences are load bearing: +// +// - Nothing downstream may treat a message it did not receive as evidence +// about a node. A carrier that cannot deliver is not a worker that is gone, +// and the difference between those two is the invariant this whole +// programme is built around. Absence is read from the database, never from +// silence on a bus. +// - Anything that must survive a gap belongs in a table, with the broadcast +// as a hint to go and look. +package pgbus + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/mudler/xlog" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/messaging" +) + +// channelPrefix namespaces this carrier's LISTEN channels away from anything +// else that may be using the same database. +const channelPrefix = "localai_" + +// maxNotifyPayloadBytes is PostgreSQL's own cap on a pg_notify payload, and it +// is EXCLUSIVE: async.c refuses a payload when strlen(payload) >= +// NOTIFY_PAYLOAD_MAX_LENGTH, so 7999 bytes is the largest that is accepted and +// 8000 already fails. Verified against postgres:16. +// +// What is measured against it is the ENCODED NOTIFICATION and not the caller's +// payload: the subject, the JSON escaping and the envelope's own keys all +// travel too, and a payload sized against the cap alone would ship 7999 bytes +// of data in a notification the server rejects. +const maxNotifyPayloadBytes = 8000 + +// listenPollInterval bounds how long a LISTEN or UNLISTEN waits for the +// listener goroutine to come off WaitForNotification and run it. +// +// It is a bound on REGISTRATION, never on delivery: a notification wakes +// WaitForNotification immediately. The connection survives the deadline, which +// pgx implements as a read deadline rather than a close (pgconn's peekMessage +// closes on anything except a net timeout), so this costs one syscall per +// interval and nothing else. +const listenPollInterval = 25 * time.Millisecond + +// subscribeTimeout bounds Subscribe when the listener is busy redialling a +// database that has gone away, so a caller gets an error instead of a +// goroutine that never returns. +const subscribeTimeout = 30 * time.Second + +// deliveryQueueDepth is how far one subscriber may fall behind before its +// broadcasts are dropped. +// +// Per subscriber, not per carrier. Handlers here write to SSE streams and to +// caller-owned channels, and running them on the listener goroutine would let +// one blocked writer stop delivery for every subject in the deployment. The +// drop is logged at error level: losing a broadcast loudly is recoverable, +// wedging the carrier silently is not. +const deliveryQueueDepth = 256 + +// broadcastRoots is the closed set of subject roots this carrier serves. +// A subject whose first token is not here is REFUSED at publish and at +// subscribe, rather than being mapped to a channel of its own. +// +// Refused rather than mapped, because a LISTEN channel name is capped at 63 +// bytes and a per-subject channel would mean one LISTEN per job id, per gallery +// operation and per response id, which is unbounded. Refused rather than +// silently dropped, because a new subject that goes nowhere is exactly the +// class of defect this programme exists to remove. +var broadcastRoots = map[string]struct{}{ + "jobs": {}, "agent": {}, "gallery": {}, "cache": {}, + "staging": {}, "prefixcache": {}, "responses": {}, "state": {}, +} + +// BroadcastRoots returns the roots this carrier serves, sorted. +// +// Exported so a spec can assert over the whole set rather than over a sample it +// re-spells, which is what makes "no channel can exceed PostgreSQL's identifier +// limit" a property of the set instead of a property of the examples someone +// remembered to write down. +func BroadcastRoots() []string { + roots := make([]string, 0, len(broadcastRoots)) + for root := range broadcastRoots { + roots = append(roots, root) + } + sort.Strings(roots) + return roots +} + +// ErrUnservedSubject is the class every root refusal belongs to, so a caller +// can tell "this carrier does not serve that family" apart from a database +// failure without matching on strings. +var ErrUnservedSubject = errors.New("subject root is not served by the broadcast carrier") + +// ChannelFor maps a subject onto the LISTEN channel that carries it. It is +// exported so a spec can assert the mapping directly rather than inferring it +// from delivery. +// +// This is the single spelling of the channel rule. Publish and Subscribe both +// call it, and neither may re-derive a channel name of its own: a publisher and +// a subscriber that disagree about a channel is a subject that is published +// successfully and delivered to nobody, with no error anywhere. +func ChannelFor(subject string) (string, error) { + root, _, _ := strings.Cut(subject, ".") + if root == "" { + return "", fmt.Errorf("%w: %q has no root token", ErrUnservedSubject, subject) + } + if _, ok := broadcastRoots[root]; !ok { + return "", fmt.Errorf("%w: %q (root %q); add the root to broadcastRoots if it is meant to fan out", ErrUnservedSubject, subject, root) + } + return channelPrefix + root, nil +} + +// Config configures the deployment's broadcast carrier. +type Config struct { + // DSN is the PostgreSQL connection string the LISTEN connection is opened + // from. It is separate from DB because a LISTEN connection cannot be + // borrowed from a pool: LISTEN registrations belong to one backend session, + // so a pooled handle would register them on whichever connection it handed + // out and lose them on the next one. This connection is pinned for the life + // of the Bus. + // + // Its ONE source is config.ApplicationConfig.Auth.DatabaseURL, fed by + // --auth-database-url / LOCALAI_AUTH_DATABASE_URL. It is not a flag of its + // own and it is never read from the environment: a second knob would let a + // deployment point the LISTEN connection and the pool at different + // databases, which is a carrier that publishes and never delivers with no + // error anywhere. New refuses that combination rather than trusting the + // convention. + DSN string + // DB is the pooled handle NOTIFY and the spill table are written on. + DB *gorm.DB +} + +// notification is what travels in a pg_notify payload. The keys are one byte +// each because every byte of the envelope is a byte the caller's payload cannot +// use before it has to spill. +type notification struct { + Subject string `json:"s"` + Data json.RawMessage `json:"d,omitempty"` + SpillID string `json:"i,omitempty"` +} + +// listenCmd is a LISTEN or UNLISTEN handed to the goroutine that owns the +// connection. The connection is not concurrency safe and the listener goroutine +// is parked in WaitForNotification most of the time, so registrations are +// queued to it rather than run against it. +type listenCmd struct { + sql string + done chan error +} + +// Bus is one replica's end of the broadcast carrier. +type Bus struct { + cfg Config + ctx context.Context + cancel context.CancelFunc + + // connected is a fact about this process's link to the database and about + // nothing else. No consumer may branch on it: "the carrier is down" is not + // one of the four conditions a node can be in, and turning it into absence + // evidence would let a database hiccup evict models. + connected atomic.Bool + + cmds chan listenCmd + listenerDone chan struct{} + closeOnce sync.Once + + mu sync.Mutex + nextID uint64 + subs map[string]map[uint64]*subscription +} + +// New opens the carrier: one pinned LISTEN connection, and the pooled handle +// publishes travel on. +func New(ctx context.Context, cfg Config) (*Bus, error) { + if cfg.DB == nil { + return nil, errors.New("pgbus: no database handle") + } + // The one dialect statement in this package. Everything below it is + // PostgreSQL-only (pg_notify, LISTEN, now(), make_interval), and an + // unguarded now() on SQLite reads as a missing migration rather than as a + // carrier that was never meant to run there. + if name := cfg.DB.Dialector.Name(); name != "postgres" { + return nil, fmt.Errorf("pgbus: the broadcast carrier requires PostgreSQL, got %q", name) + } + if cfg.DSN == "" { + return nil, errors.New("pgbus: no DSN for the LISTEN connection") + } + + conn, err := pgx.Connect(ctx, cfg.DSN) + if err != nil { + return nil, fmt.Errorf("pgbus: opening the LISTEN connection: %w", err) + } + if err := verifySameDatabase(ctx, conn, cfg.DB); err != nil { + _ = conn.Close(ctx) + return nil, err + } + + busCtx, cancel := context.WithCancel(context.WithoutCancel(ctx)) + b := &Bus{ + cfg: cfg, + ctx: busCtx, + cancel: cancel, + cmds: make(chan listenCmd), + listenerDone: make(chan struct{}), + subs: map[string]map[uint64]*subscription{}, + } + b.connected.Store(true) + go b.listen(conn) + go b.sweep() + return b, nil +} + +// verifySameDatabase refuses a carrier whose LISTEN connection and whose pool +// are not on the same database. +// +// That combination has no other symptom: every publish succeeds, every +// subscribe succeeds, and nothing is ever delivered, on every replica, forever. +// It is cheap to exclude here and expensive to diagnose anywhere else. +func verifySameDatabase(ctx context.Context, conn *pgx.Conn, db *gorm.DB) error { + const identity = `SELECT current_database() || '@' || coalesce(host(inet_server_addr()), 'local') || ':' || coalesce(inet_server_port(), 0)` + + var listenSide string + if err := conn.QueryRow(ctx, identity).Scan(&listenSide); err != nil { + return fmt.Errorf("pgbus: identifying the LISTEN connection's database: %w", err) + } + var poolSide string + if err := db.WithContext(ctx).Raw(identity).Row().Scan(&poolSide); err != nil { + return fmt.Errorf("pgbus: identifying the pool's database: %w", err) + } + if listenSide != poolSide { + return fmt.Errorf("pgbus: the LISTEN connection and the pool are on a different database (%s vs %s); every broadcast would be published and never delivered", listenSide, poolSide) + } + return nil +} + +// DSN returns the connection string this carrier listens on. It exists so the +// wiring can be pinned to the deployment's one database URL by equality rather +// than by shape: any DSN passes a shape check, including one that addresses a +// different database from the pool. +func (b *Bus) DSN() string { return b.cfg.DSN } + +// IsConnected reports whether the pinned LISTEN connection is up. See the +// comment on Bus.connected: it is for logging and specs, not for decisions. +func (b *Bus) IsConnected() bool { return b.connected.Load() } + +// Close stops the carrier. It is idempotent and does not wait for handlers that +// are still running: a handler that never returns must not be able to hold up +// the shutdown of the process it is in. +func (b *Bus) Close() { + b.closeOnce.Do(func() { + b.cancel() + <-b.listenerDone + b.connected.Store(false) + }) +} + +// Publish fans a message out to every subscriber of the subject on every +// replica, including this one. +func (b *Bus) Publish(subject string, data any) error { + channel, err := ChannelFor(subject) + if err != nil { + return err + } + payload, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("pgbus: encoding a broadcast on %q: %w", subject, err) + } + + encoded, err := json.Marshal(notification{Subject: subject, Data: payload}) + if err != nil { + return fmt.Errorf("pgbus: encoding the notification for %q: %w", subject, err) + } + // The one size decision in this package. Several subjects on this carrier + // exceed the cap in normal operation (a job result carries a whole LLM + // output; a gallery progress event carries one entry per node), so the + // spill is the ordinary path for them and not an error case. + if len(encoded) >= maxNotifyPayloadBytes { + id, err := b.spill(subject, payload) + if err != nil { + return err + } + encoded, err = json.Marshal(notification{Subject: subject, SpillID: id}) + if err != nil { + return fmt.Errorf("pgbus: encoding the spill notification for %q: %w", subject, err) + } + } + + // pg_notify rather than a NOTIFY statement, because the channel is a value + // here and NOTIFY would need it interpolated as an identifier. + if err := b.cfg.DB.WithContext(b.ctx).Exec("SELECT pg_notify(?, ?)", channel, string(encoded)).Error; err != nil { + return fmt.Errorf("pgbus: publishing on %q: %w", subject, err) + } + return nil +} + +// Subscribe registers a handler for every subject matching filter. The filter +// grammar is the shared one: an exact subject, or single-token `*` wildcards. +func (b *Bus) Subscribe(subject string, handler func([]byte)) (messaging.Subscription, error) { + // The shared rule, asked rather than re-spelled. A carrier that decided for + // itself which filters it accepts would drift from the matcher that decides + // which of them deliver, and the drift presents as a peer receiving an + // event on one replica and missing it on another. + if err := messaging.ValidFilter(subject); err != nil { + return nil, err + } + channel, err := ChannelFor(subject) + if err != nil { + return nil, err + } + if handler == nil { + return nil, fmt.Errorf("pgbus: no handler for %q", subject) + } + + b.mu.Lock() + b.nextID++ + sub := &subscription{ + bus: b, + channel: channel, + filter: subject, + id: b.nextID, + queue: make(chan []byte, deliveryQueueDepth), + stop: make(chan struct{}), + } + first := len(b.subs[channel]) == 0 + if first { + b.subs[channel] = map[uint64]*subscription{} + } + b.subs[channel][sub.id] = sub + b.mu.Unlock() + + go sub.run(handler) + + if first { + if err := b.command("LISTEN " + pgx.Identifier{channel}.Sanitize()); err != nil { + _ = sub.Unsubscribe() + return nil, err + } + } + return sub, nil +} + +// command hands a LISTEN or UNLISTEN to the goroutine that owns the connection +// and waits for it, so a Subscribe that has returned is a registration the +// server has already acknowledged. Returning before that would lose every +// message published in the gap. +func (b *Bus) command(sql string) error { + cmd := listenCmd{sql: sql, done: make(chan error, 1)} + timeout := time.NewTimer(subscribeTimeout) + defer timeout.Stop() + + select { + case b.cmds <- cmd: + case <-b.ctx.Done(): + return errors.New("pgbus: the carrier is closed") + case <-timeout.C: + return fmt.Errorf("pgbus: %q timed out waiting for the listen connection", sql) + } + select { + case err := <-cmd.done: + return err + case <-b.ctx.Done(): + return errors.New("pgbus: the carrier is closed") + case <-timeout.C: + return fmt.Errorf("pgbus: %q timed out on the listen connection", sql) + } +} + +// listen owns the pinned connection for the life of the Bus: it is the only +// goroutine that touches it, which is what makes a connection that is not +// concurrency safe usable from many callers. +func (b *Bus) listen(conn *pgx.Conn) { + defer close(b.listenerDone) + defer func() { + b.connected.Store(false) + // A background context, because b.ctx is already cancelled by the time + // this runs and the close still has to reach the server. + closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = conn.Close(closeCtx) + }() + + for { + if b.ctx.Err() != nil { + return + } + // Registrations first: a subscriber that has been waiting must not be + // held up behind a poll interval that has just restarted. + select { + case cmd := <-b.cmds: + _, err := conn.Exec(b.ctx, cmd.sql) + cmd.done <- err + continue + default: + } + + waitCtx, cancel := context.WithTimeout(b.ctx, listenPollInterval) + n, err := conn.WaitForNotification(waitCtx) + deadline := waitCtx.Err() != nil + cancel() + + switch { + case b.ctx.Err() != nil: + return + case deadline: + // The ordinary case: nothing arrived inside the poll window. pgx + // implements the deadline as a read deadline and keeps the + // connection, so this is not a failure. + continue + case err != nil: + xlog.Warn("Broadcast carrier lost its listen connection, redialling", "error", err) + replacement, ok := b.redial(conn) + if !ok { + return + } + conn = replacement + continue + } + b.deliver(n.Channel, n.Payload) + } +} + +// redial replaces a dead LISTEN connection and restores every registration on +// it. Without the restore the carrier comes back deaf, which looks exactly like +// a deployment where nobody is publishing any more. +func (b *Bus) redial(dead *pgx.Conn) (*pgx.Conn, bool) { + b.connected.Store(false) + closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = dead.Close(closeCtx) + cancel() + + backoff := 100 * time.Millisecond + for { + if b.ctx.Err() != nil { + return nil, false + } + conn, err := pgx.Connect(b.ctx, b.cfg.DSN) + if err == nil { + if err = b.relisten(conn); err == nil { + b.connected.Store(true) + xlog.Info("Broadcast carrier reconnected") + return conn, true + } + closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = conn.Close(closeCtx) + cancel() + } + select { + case <-b.ctx.Done(): + return nil, false + case <-time.After(backoff): + } + if backoff < 5*time.Second { + backoff *= 2 + } + } +} + +func (b *Bus) relisten(conn *pgx.Conn) error { + b.mu.Lock() + channels := make([]string, 0, len(b.subs)) + for channel, subs := range b.subs { + if len(subs) > 0 { + channels = append(channels, channel) + } + } + b.mu.Unlock() + + for _, channel := range channels { + if _, err := conn.Exec(b.ctx, "LISTEN "+pgx.Identifier{channel}.Sanitize()); err != nil { + return err + } + } + return nil +} + +// deliver resolves one notification and hands it to the subscribers whose +// filters match. +func (b *Bus) deliver(channel, payload string) { + var n notification + if err := json.Unmarshal([]byte(payload), &n); err != nil { + xlog.Error("Broadcast carrier received an undecodable notification", "channel", channel, "error", err) + return + } + + data := []byte(n.Data) + if n.SpillID != "" { + resolved, err := b.resolveSpill(n.SpillID) + if err != nil { + // A dropped broadcast, and never anything more than that. Handing + // a handler an empty message instead would let a carrier failure + // be read as a fact about the deployment. + xlog.Error("Broadcast carrier could not resolve a spilled message", "subject", n.Subject, "id", n.SpillID, "error", err) + return + } + data = resolved + } + + b.mu.Lock() + targets := make([]*subscription, 0, len(b.subs[channel])) + for _, sub := range b.subs[channel] { + // The shared matcher, in the one place delivery is decided. A second + // spelling here is the divergence the single-sourcing exists to + // prevent: the doubles the specs publish through call the same + // function. + if messaging.SubjectMatches(sub.filter, n.Subject) { + targets = append(targets, sub) + } + } + b.mu.Unlock() + + for _, sub := range targets { + sub.enqueue(n.Subject, data) + } +} + +// subscription is one handler's registration. +type subscription struct { + bus *Bus + channel string + filter string + id uint64 + + queue chan []byte + stop chan struct{} + once sync.Once +} + +func (s *subscription) run(handler func([]byte)) { + for { + select { + case data := <-s.queue: + handler(data) + case <-s.stop: + return + case <-s.bus.ctx.Done(): + return + } + } +} + +func (s *subscription) enqueue(subject string, data []byte) { + select { + case s.queue <- data: + default: + xlog.Error("Broadcast carrier dropped a message: subscriber is not keeping up", + "filter", s.filter, "subject", subject, "depth", deliveryQueueDepth) + } +} + +// Unsubscribe stops delivery to this handler and, when it was the last +// subscriber on its channel, stops listening on it. +func (s *subscription) Unsubscribe() error { + var err error + s.once.Do(func() { + s.bus.mu.Lock() + delete(s.bus.subs[s.channel], s.id) + last := len(s.bus.subs[s.channel]) == 0 + if last { + delete(s.bus.subs, s.channel) + } + s.bus.mu.Unlock() + + close(s.stop) + if last && s.bus.ctx.Err() == nil { + err = s.bus.command("UNLISTEN " + pgx.Identifier{s.channel}.Sanitize()) + } + }) + return err +} + +// spill writes a broadcast that does not fit in a notification to a row, and +// returns the id the notification carries in its place. +func (b *Bus) spill(subject string, payload []byte) (string, error) { + row := BusMessage{ID: uuid.New().String(), Subject: subject, Payload: payload} + if err := b.cfg.DB.WithContext(b.ctx).Create(&row).Error; err != nil { + return "", fmt.Errorf("pgbus: spilling a broadcast on %q: %w", subject, err) + } + return row.ID, nil +} + +func (b *Bus) resolveSpill(id string) ([]byte, error) { + var row BusMessage + if err := b.cfg.DB.WithContext(b.ctx).Where("id = ?", id).First(&row).Error; err != nil { + return nil, err + } + return row.Payload, nil +} + +// sweep retires spilled rows that every replica has had time to read. +func (b *Bus) sweep() { + ticker := time.NewTicker(spillSweepInterval) + defer ticker.Stop() + for { + select { + case <-b.ctx.Done(): + return + case <-ticker.C: + if err := SweepSpill(b.ctx, b.cfg.DB, spillRetention); err != nil && b.ctx.Err() == nil { + xlog.Warn("Broadcast carrier could not retire spilled messages", "error", err) + } + } + } +} + +var _ messaging.Broadcaster = (*Bus)(nil) diff --git a/core/services/pgbus/bus_test.go b/core/services/pgbus/bus_test.go new file mode 100644 index 000000000000..0585677ee343 --- /dev/null +++ b/core/services/pgbus/bus_test.go @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: MIT + +package pgbus_test + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// Every delivery spec here builds TWO buses on TWO connections and asserts on +// the one that did NOT publish. One bus that hears itself proves nothing about +// a carrier whose whole job is to reach the OTHER replica, and an in-memory +// double proves less than that: it cannot fail the way a LISTEN connection +// fails. +var _ = Describe("the PostgreSQL broadcast carrier", func() { + var ( + db *gorm.DB + dsn string + pub, sub *pgbus.Bus + ) + + // newBus is the only construction path in this file, so a spec cannot + // accidentally skip the migration and read a missing-table error as a + // delivery failure. + newBus := func() *pgbus.Bus { + GinkgoHelper() + b, err := pgbus.New(context.Background(), pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(b.Close) + return b + } + + BeforeEach(func() { + db, dsn = testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(context.Background(), db)).To(Succeed()) + pub = newBus() + sub = newBus() + }) + + // received registers a subscriber that forwards every delivery onto a + // channel, so specs synchronize on delivery rather than on the clock. + received := func(b *pgbus.Bus, filter string) (<-chan []byte, messaging.Subscription) { + GinkgoHelper() + out := make(chan []byte, 16) + s, err := b.Subscribe(filter, func(data []byte) { out <- data }) + Expect(err).ToNot(HaveOccurred()) + Expect(s).ToNot(BeNil()) + return out, s + } + + Describe("delivery across two connections", func() { + It("carries a message to a subscriber on the other bus", func() { + out, _ := received(sub, "jobs.abc.progress") + + Expect(pub.Publish("jobs.abc.progress", map[string]string{"phase": "downloading"})).To(Succeed()) + + Eventually(out).Should(Receive(MatchJSON(`{"phase":"downloading"}`))) + }) + + // The root set is not a formality: a subject family that maps to its own + // channel is a family a subscriber on a different channel never hears. + // M1 (Publish hardcoding one channel) and M2 (Subscribe hardcoding one) + // both present here as silence. + It("carries a message on a root other than jobs", func() { + out, _ := received(sub, "gallery.install.progress") + + Expect(pub.Publish("gallery.install.progress", map[string]int{"done": 3})).To(Succeed()) + + Eventually(out).Should(Receive(MatchJSON(`{"done":3}`))) + }) + + It("delivers to every subscriber of a subject, on the publishing bus and on the other one", func() { + // Fan-out is the whole point of this carrier: a replica that + // published must still run its own handlers, because in production + // the publisher is one of the replicas that has to react. + here, _ := received(pub, "cache.evict") + there, _ := received(sub, "cache.evict") + + Expect(pub.Publish("cache.evict", map[string]string{"model": "m"})).To(Succeed()) + + Eventually(here).Should(Receive()) + Eventually(there).Should(Receive()) + }) + }) + + Describe("which subjects reach which filters", func() { + // The expectations are COMPUTED by messaging.SubjectMatches rather than + // written out, so the carrier and the matcher cannot drift apart: if + // the carrier ever answers differently from the shared function, this + // table reddens without anyone having to notice the divergence. + // + // Ordering carries the negative half. Every case publishes the probe + // subject and then a sentinel on a subject the filter certainly + // matches; a filter that must NOT see the probe is proved by the + // sentinel arriving first, which needs no clock. + DescribeTable("agrees with messaging.SubjectMatches", + func(filter, subject string) { + out, _ := received(sub, filter) + sentinel := "jobs.sentinel.progress" + if !messaging.SubjectMatches(filter, sentinel) { + sentinel = filter + } + + Expect(pub.Publish(subject, map[string]string{"m": "probe"})).To(Succeed()) + Expect(pub.Publish(sentinel, map[string]string{"m": "sentinel"})).To(Succeed()) + + var first []byte + Eventually(out).Should(Receive(&first)) + want := `{"m":"sentinel"}` + if messaging.SubjectMatches(filter, subject) { + want = `{"m":"probe"}` + } + Expect(first).To(MatchJSON(want)) + }, + Entry("exact subject", "jobs.abc.progress", "jobs.abc.progress"), + Entry("single-token wildcard in the middle", "jobs.*.progress", "jobs.abc.progress"), + Entry("a sibling subject of the same family", "jobs.*.progress", "jobs.abc.result"), + Entry("more tokens than the filter has", "jobs.*.progress", "jobs.abc.extra.progress"), + Entry("fewer tokens than the filter has", "jobs.*.progress", "jobs.progress"), + Entry("a wildcard in the leading token", "jobs.*", "jobs.abc"), + ) + }) + + Describe("subjects this carrier refuses", func() { + It("refuses a publish whose root it does not serve, and names the subject", func() { + err := pub.Publish("weather.today", map[string]string{"sky": "blue"}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("weather.today")) + }) + + It("delivers nothing at all for a refused publish", func() { + // The refusal has to be a refusal, not an error returned beside a + // message that went somewhere anyway. + out, _ := received(sub, "jobs.abc.progress") + + Expect(pub.Publish("weather.today", map[string]string{"sky": "blue"})).ToNot(Succeed()) + Expect(pub.Publish("jobs.abc.progress", map[string]string{"m": "sentinel"})).To(Succeed()) + + var first []byte + Eventually(out).Should(Receive(&first)) + Expect(first).To(MatchJSON(`{"m":"sentinel"}`)) + }) + + It("refuses a subscribe whose root it does not serve, and registers nothing", func() { + s, err := sub.Subscribe("weather.today", func([]byte) {}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("weather.today")) + Expect(s).To(BeNil()) + }) + + It("refuses a filter the shared matcher cannot act on, with that package's error", func() { + // errors.Is rather than a string: this is what proves the carrier + // asks messaging.ValidFilter instead of re-spelling the rule. + s, err := sub.Subscribe("jobs.>", func([]byte) {}) + + Expect(errors.Is(err, messaging.ErrUnsupportedFilter)).To(BeTrue(), "got %v", err) + Expect(s).To(BeNil()) + }) + + It("refuses an empty-token filter for the same reason", func() { + s, err := sub.Subscribe("jobs..progress", func([]byte) {}) + + Expect(errors.Is(err, messaging.ErrUnsupportedFilter)).To(BeTrue(), "got %v", err) + Expect(s).To(BeNil()) + }) + }) + + Describe("unsubscribing", func() { + It("stops delivery to that handler and leaves the others alone", func() { + gone, s := received(sub, "jobs.abc.progress") + stays, _ := received(sub, "jobs.abc.progress") + + Expect(s.Unsubscribe()).To(Succeed()) + Expect(pub.Publish("jobs.abc.progress", map[string]string{"m": "after"})).To(Succeed()) + + Eventually(stays).Should(Receive(MatchJSON(`{"m":"after"}`))) + Consistently(gone).ShouldNot(Receive()) + }) + }) + + Describe("a handler that does not return", func() { + It("does not stop delivery to the other subscribers", func() { + // A carrier whose handlers run on one goroutine is a carrier one + // blocked SSE writer can wedge for the whole deployment. + block := make(chan struct{}) + DeferCleanup(func() { close(block) }) + _, _ = sub.Subscribe("jobs.abc.progress", func([]byte) { <-block }) + free, _ := received(sub, "jobs.abc.progress") + + Expect(pub.Publish("jobs.abc.progress", map[string]string{"m": "one"})).To(Succeed()) + Expect(pub.Publish("jobs.abc.progress", map[string]string{"m": "two"})).To(Succeed()) + + Eventually(free).Should(Receive(MatchJSON(`{"m":"one"}`))) + Eventually(free).Should(Receive(MatchJSON(`{"m":"two"}`))) + }) + }) + + Describe("losing the LISTEN connection", func() { + It("reconnects, restores its registrations and delivers again", func() { + // The transport failure a fake cannot produce. Terminating the + // backend sessions drops both buses' pinned connections; what must + // survive is the SUBSCRIPTION, because a carrier that comes back + // deaf looks exactly like a peer that stopped publishing. + out, _ := received(sub, "jobs.abc.progress") + + Expect(db.Exec( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid()", + ).Error).To(Succeed()) + + Eventually(sub.IsConnected, 30*time.Second, 100*time.Millisecond).Should(BeTrue()) + Eventually(func() bool { + // Retried because the publishing side's pool is recovering too, + // and because a NOTIFY that lands between the drop and the + // re-LISTEN is genuinely lost: this carrier is at-most-once. + _ = pub.Publish("jobs.abc.progress", map[string]string{"m": "after"}) + select { + case <-out: + return true + default: + return false + } + }, 30*time.Second, 200*time.Millisecond).Should(BeTrue()) + }) + }) +}) + +var _ = Describe("mapping a subject onto a LISTEN channel", func() { + It("puts a subject on the channel of its first token", func() { + Expect(pgbus.ChannelFor("jobs.abc.progress")).To(Equal("localai_jobs")) + }) + + It("refuses a subject whose root it does not serve", func() { + _, err := pgbus.ChannelFor("weather.today") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("weather.today")) + }) + + It("refuses an empty subject", func() { + _, err := pgbus.ChannelFor("") + Expect(err).To(HaveOccurred()) + }) + + It("keeps every channel it can ever return inside PostgreSQL's identifier limit", func() { + // Asserted over the whole root set rather than over a sample, so a root + // added later cannot push a channel past 63 bytes silently. Past that + // limit PostgreSQL truncates the name instead of failing, which merges + // two families onto one channel. + roots := pgbus.BroadcastRoots() + Expect(roots).ToNot(BeEmpty()) + for _, root := range roots { + channel, err := pgbus.ChannelFor(root + ".anything") + Expect(err).ToNot(HaveOccurred(), root) + Expect(len(channel)).To(BeNumerically("<=", 63), channel) + } + }) +}) + +var _ = Describe("constructing the carrier", func() { + It("refuses a non-PostgreSQL handle and names the dialect", func() { + db, err := gorm.Open(sqlite.Open(filepath.Join(GinkgoT().TempDir(), "bus.db")), &gorm.Config{}) + Expect(err).ToNot(HaveOccurred()) + + _, err = pgbus.New(context.Background(), pgbus.Config{DSN: "postgres://x/y", DB: db}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sqlite")) + }) + + It("refuses a DSN that points at a different database from the pool", func() { + // The failure this excludes has no other symptom: every publish + // succeeds, every subscribe succeeds, and nothing is ever delivered. + db, _ := testutil.SetupTestDBWithDSN() + _, otherDSN := testutil.SetupTestDBWithDSN() + + _, err := pgbus.New(context.Background(), pgbus.Config{DSN: otherDSN, DB: db}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("different database")) + }) + + It("reports itself connected while open and disconnected once closed", func() { + db, dsn := testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(context.Background(), db)).To(Succeed()) + + b, err := pgbus.New(context.Background(), pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + Expect(b.IsConnected()).To(BeTrue()) + Expect(b.DSN()).To(Equal(dsn)) + + b.Close() + Expect(b.IsConnected()).To(BeFalse()) + }) + + It("survives a second Close", func() { + db, dsn := testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(context.Background(), db)).To(Succeed()) + b, err := pgbus.New(context.Background(), pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + + b.Close() + Expect(b.Close).ToNot(Panic()) + }) + + It("refuses an empty DSN rather than opening a carrier that cannot listen", func() { + db, _ := testutil.SetupTestDBWithDSN() + + _, err := pgbus.New(context.Background(), pgbus.Config{DB: db}) + + Expect(err).To(HaveOccurred()) + }) + + It("refuses a nil handle", func() { + _, err := pgbus.New(context.Background(), pgbus.Config{DSN: "postgres://x/y"}) + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("the interface the carrier is interchangeable through", func() { + It("is satisfied by the bus", func() { + // Compile-time, not behavioural: the point is that a signature change + // on either carrier fails here rather than in whichever call site is + // migrated next. + var b messaging.Broadcaster = (*pgbus.Bus)(nil) + Expect(fmt.Sprintf("%T", b)).To(Equal("*pgbus.Bus")) + }) +}) diff --git a/core/services/pgbus/pgbus_suite_test.go b/core/services/pgbus/pgbus_suite_test.go new file mode 100644 index 000000000000..2b108a595eb2 --- /dev/null +++ b/core/services/pgbus/pgbus_suite_test.go @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT + +package pgbus_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestPgbus(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pgbus Suite") +} diff --git a/core/services/pgbus/spill.go b/core/services/pgbus/spill.go new file mode 100644 index 000000000000..b393c2419a5e --- /dev/null +++ b/core/services/pgbus/spill.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT + +package pgbus + +import ( + "context" + "time" + + "gorm.io/gorm" +) + +// spillRetention is how long a spilled broadcast stays readable after it was +// published. It is generous against the delivery it has to survive, which is +// one notification and one primary-key SELECT, and short enough that a busy +// deployment does not accumulate LLM outputs in this table. +const spillRetention = 5 * time.Minute + +// spillSweepInterval is how often each replica retires what has aged out. Every +// replica sweeps; the DELETE is idempotent and a replica that is down must not +// leave the table growing. +const spillSweepInterval = time.Minute + +// SpillSweepSQL retires spilled broadcasts older than the retention it is given. +// +// The cutoff is computed by the DATABASE and never in Go, and that is the whole +// point of holding the statement here where a spec can read it. Replicas do not +// share a clock: a replica running minutes ahead would compute a cutoff in the +// future and delete rows its peers have not read yet, and a replica running +// behind would never delete anything. now() is the one clock every replica +// agrees on, because it is the clock the rows were written by. +// +// PostgreSQL-only, like everything else in this package. New refuses a handle on +// any other dialect, which is where that is guarded. +const SpillSweepSQL = "DELETE FROM bus_messages WHERE created_at < now() - make_interval(secs => ?)" + +// BusMessage is one broadcast too large to travel in a notification. +type BusMessage struct { + ID string `gorm:"primaryKey;size:36"` + Subject string `gorm:"size:255;index"` + Payload []byte + CreatedAt time.Time `gorm:"index"` +} + +// Migrate creates the spill table. It is separate from New because a deployment +// migrates once at boot while every replica opens a carrier. +func Migrate(ctx context.Context, db *gorm.DB) error { + return db.WithContext(ctx).AutoMigrate(&BusMessage{}) +} + +// SweepSpill deletes spilled broadcasts older than retention. +func SweepSpill(ctx context.Context, db *gorm.DB, retention time.Duration) error { + return db.WithContext(ctx).Exec(SpillSweepSQL, retention.Seconds()).Error +} diff --git a/core/services/pgbus/spill_test.go b/core/services/pgbus/spill_test.go new file mode 100644 index 000000000000..16bafb2e2c9b --- /dev/null +++ b/core/services/pgbus/spill_test.go @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: MIT + +package pgbus_test + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// PostgreSQL refuses a pg_notify payload of 8000 bytes or more: async.c checks +// strlen(payload) >= NOTIFY_PAYLOAD_MAX_LENGTH, so 7999 is the largest that is +// accepted. Verified against postgres:16 rather than read off the docs, because +// the docs say "less than 8000 bytes" and the off-by-one is the whole subject of +// these specs. +// +// This literal is deliberately NOT pgbus.MaxNotifyPayloadBytes. A size spec +// that measures itself against the constant it is testing passes for every +// value of that constant, which makes it a spec about nothing. +const notifyCap = 8000 + +// encodedSize is what the carrier puts on the wire for an inline broadcast: +// the subject, the JSON escaping and the envelope's own keys all travel with +// the caller's payload. A payload sized against the cap alone would ship 7999 +// bytes of data in an 8035-byte notification and fail at the server. +func encodedSize(subject string, data any) int { + GinkgoHelper() + subjectJSON, err := json.Marshal(subject) + Expect(err).ToNot(HaveOccurred()) + dataJSON, err := json.Marshal(data) + Expect(err).ToNot(HaveOccurred()) + return len(fmt.Sprintf(`{"s":%s,"d":%s}`, subjectJSON, dataJSON)) +} + +// payloadEncodingTo builds a valid single-key JSON object whose encoded +// notification is exactly total bytes. Valid on purpose: an oversized payload +// that the decoder would reject anyway cannot prove the SIZE routed it. +func payloadEncodingTo(subject string, total int) map[string]string { + GinkgoHelper() + data := map[string]string{"k": ""} + filler := total - encodedSize(subject, data) + Expect(filler).To(BeNumerically(">", 0)) + data["k"] = strings.Repeat("a", filler) + Expect(encodedSize(subject, data)).To(Equal(total)) + return data +} + +var _ = Describe("broadcasts too large for a notification", func() { + const subject = "jobs.size.probe" + + var ( + db *gorm.DB + dsn string + pub, sub *pgbus.Bus + ) + + spilledRows := func() int64 { + GinkgoHelper() + var n int64 + Expect(db.Model(&pgbus.BusMessage{}).Where("subject = ?", subject).Count(&n).Error).To(Succeed()) + return n + } + + BeforeEach(func() { + db, dsn = testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(context.Background(), db)).To(Succeed()) + for _, b := range []**pgbus.Bus{&pub, &sub} { + built, err := pgbus.New(context.Background(), pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(built.Close) + *b = built + } + }) + + deliver := func(data any) []byte { + GinkgoHelper() + out := make(chan []byte, 4) + _, err := sub.Subscribe(subject, func(b []byte) { out <- b }) + Expect(err).ToNot(HaveOccurred()) + + Expect(pub.Publish(subject, data)).To(Succeed()) + + var got []byte + Eventually(out, 10*time.Second).Should(Receive(&got)) + return got + } + + // The three rows of the size table. Each states an ABSOLUTE size, so a + // mutation of the carrier's constant moves the behaviour without moving + // the expectation. + It("keeps a broadcast one byte under the cap in the notification itself", func() { + data := payloadEncodingTo(subject, notifyCap-1) + + got := deliver(data) + + Expect(got).To(MatchJSON(mustJSON(data))) + Expect(spilledRows()).To(BeZero(), "a notification PostgreSQL accepts must not touch the spill table") + }) + + It("spills a broadcast exactly at the cap, which PostgreSQL already refuses", func() { + data := payloadEncodingTo(subject, notifyCap) + + got := deliver(data) + + Expect(got).To(MatchJSON(mustJSON(data))) + Expect(spilledRows()).To(Equal(int64(1))) + }) + + It("carries a megabyte through the spill table byte for byte", func() { + // Absolute, not derived from the cap: this row must stay green under a + // mutated constant, which is what separates "the spill path works" from + // "the spec sized itself against the thing it is testing". + data := map[string]string{"k": strings.Repeat("a", 1<<20)} + + got := deliver(data) + + Expect(got).To(Equal(mustJSON(data))) + Expect(spilledRows()).To(Equal(int64(1))) + }) + + It("stores the caller's payload in the spill row, not the envelope", func() { + data := payloadEncodingTo(subject, notifyCap) + + _ = deliver(data) + + var row pgbus.BusMessage + Expect(db.Where("subject = ?", subject).First(&row).Error).To(Succeed()) + Expect(row.Payload).To(Equal(mustJSON(data))) + Expect(row.ID).ToNot(BeEmpty()) + Expect(row.CreatedAt).ToNot(BeZero()) + }) + + It("delivers nothing, and keeps carrying, when a spilled row cannot be found", func() { + // A notification whose row is gone is a lost message, not an empty one: + // handing a handler nil would let a consumer read a carrier failure as + // a fact about the deployment. The sentinel proves the carrier is still + // alive afterwards, which needs no clock. + out := make(chan []byte, 4) + _, err := sub.Subscribe(subject, func(b []byte) { out <- b }) + Expect(err).ToNot(HaveOccurred()) + + Expect(db.Exec("SELECT pg_notify(?, ?)", "localai_jobs", + fmt.Sprintf(`{"s":%q,"i":"no-such-row"}`, subject)).Error).To(Succeed()) + Expect(pub.Publish(subject, map[string]string{"m": "sentinel"})).To(Succeed()) + + var got []byte + Eventually(out, 10*time.Second).Should(Receive(&got)) + Expect(got).To(MatchJSON(`{"m":"sentinel"}`)) + }) +}) + +var _ = Describe("retiring spilled rows", func() { + var db *gorm.DB + + BeforeEach(func() { + db, _ = testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(context.Background(), db)).To(Succeed()) + }) + + It("leaves the cutoff to the database clock", func() { + // Pinned as a statement shape rather than as behaviour on purpose. The + // test container shares this host's clock, so a cutoff computed in Go + // and one computed by the server agree to the microsecond and no + // behavioural spec can tell them apart. The deployment they differ in + // is the one that matters: replicas whose clocks are minutes apart, + // where a Go-side cutoff reaps rows other replicas have not read yet. + Expect(pgbus.SpillSweepSQL).To(ContainSubstring("now()")) + Expect(pgbus.SpillSweepSQL).ToNot(ContainSubstring("created_at < ?")) + }) + + It("deletes rows past the retention and keeps the rest", func() { + old := pgbus.BusMessage{ID: "old", Subject: "jobs.x", Payload: []byte(`{}`)} + fresh := pgbus.BusMessage{ID: "fresh", Subject: "jobs.x", Payload: []byte(`{}`)} + Expect(db.Create(&old).Error).To(Succeed()) + Expect(db.Create(&fresh).Error).To(Succeed()) + Expect(db.Exec("UPDATE bus_messages SET created_at = now() - interval '1 hour' WHERE id = 'old'").Error).To(Succeed()) + + Expect(pgbus.SweepSpill(context.Background(), db, 5*time.Minute)).To(Succeed()) + + var ids []string + Expect(db.Model(&pgbus.BusMessage{}).Pluck("id", &ids).Error).To(Succeed()) + Expect(ids).To(ConsistOf("fresh")) + }) +}) + +func mustJSON(v any) []byte { + GinkgoHelper() + b, err := json.Marshal(v) + Expect(err).ToNot(HaveOccurred()) + return b +} diff --git a/core/services/testutil/testdb.go b/core/services/testutil/testdb.go index 4f36d52c3d31..bd732ef66862 100644 --- a/core/services/testutil/testdb.go +++ b/core/services/testutil/testdb.go @@ -113,7 +113,25 @@ func sharedPostgres() string { // SetupTestDB returns a gorm.DB on a PostgreSQL database created for the calling // spec. The database is dropped, and its connection pool closed, when the spec // ends. +// +// It is a wrapper rather than a second implementation, so a caller that also +// needs the DSN cannot end up on a database created differently from the one +// every other suite gets. func SetupTestDB() *gorm.DB { + GinkgoHelper() + db, _ := SetupTestDBWithDSN() + return db +} + +// SetupTestDBWithDSN is SetupTestDB, and additionally returns the DSN of the +// database it created. +// +// The DSN exists for callers that have to open a connection of their own rather +// than borrow one from the pool. A PostgreSQL LISTEN connection is the case +// that motivated it: LISTEN registrations belong to one backend session, so a +// pooled handle would register them on whichever connection it happened to hand +// out and lose them on the next one. +func SetupTestDBWithDSN() (*gorm.DB, string) { GinkgoHelper() if runtime.GOOS == "darwin" { Skip("testcontainers requires Docker, not available on macOS CI") @@ -135,7 +153,8 @@ func SetupTestDB() *gorm.DB { Expect(admin.Exec(fmt.Sprintf("CREATE DATABASE %q", name)).Error).To(Succeed()) }() - db, err := gorm.Open(postgres.Open(replaceDBName(dsn, name)), &gorm.Config{ + specDSN := replaceDBName(dsn, name) + db, err := gorm.Open(postgres.Open(specDSN), &gorm.Config{ Logger: logger.Default.LogMode(logger.Silent), }) Expect(err).ToNot(HaveOccurred()) @@ -162,7 +181,7 @@ func SetupTestDB() *gorm.DB { } }) - return db + return db, specDSN } // maintenanceDSN is dsn with every server-side timeout disabled as a CONNECTION diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 56bcad7d427d..8b759891374b 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -38,6 +38,7 @@ Each model gets its own gRPC backend process, so a single worker can serve multi ## Prerequisites - **PostgreSQL** (with pgvector extension recommended for RAG) - used for node registry, job store, auth, and shared state + - Each frontend replica holds **one extra PostgreSQL session** beyond its connection pool, pinned for the life of the process, and creates a `bus_messages` table. Both belong to the broadcast carrier that is replacing NATS for cross-replica fan-out. Size `max_connections` for one additional session per frontend replica. - **NATS** server - used for agent-worker coordination and the frontend's own cross-replica events. **Serve-backend workers do not connect to it at all**: every verb they take, and file staging with it, is an HTTP route on the worker's tunnel. Set no `LOCALAI_NATS_URL` on a `local-ai worker`. The frontend and any `local-ai agent-worker` still need one. - All services must be on the same network (or reachable via configured URLs) diff --git a/go.mod b/go.mod index 4833b8e3d819..be0463385be0 100644 --- a/go.mod +++ b/go.mod @@ -253,7 +253,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.9.2 // indirect + github.com/jackc/pgx/v5 v5.9.2 github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jung-kurt/gofpdf v1.16.2 // indirect From 43956a9b60a9be88d2a6f39467590870465b3fae Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 07:45:56 +0000 Subject: [PATCH 082/153] fix(distributed): order LISTEN and UNLISTEN on one lock Unsubscribe decided a channel had lost its last subscriber under one lock and issued the UNLISTEN after releasing it. A Subscribe on the same root could decide to LISTEN in that window, and the two reached the connection in the wrong order: the root ended up not listened with a live subscription on it. It does not heal, because the next Subscribe sees the registration already there and never re-LISTENs, so the whole root stays deaf on that replica until the connection drops. The decision and the statement it implies now happen under one lock, held across both, at both call sites. A second lock and not the registration lock: issuing waits on the listener goroutine, delivery takes the registration lock, and holding that across the wait deadlocks the carrier. The race is spec'd through a barrier seam rather than by racing goroutines. The natural window is microseconds wide, and a spec that waits for it to open passes by luck; the seam scripts the interleaving, so the spec decides in both directions. Resolving a spilled message moved off the listener. PostgreSQL keeps undelivered notifications in a shared, fixed-size queue, so a listener that stops draining it can block COMMIT for every publisher on the server, not only this one. The listener now only drains; one resolver goroutine reads the row back and dispatches, which also keeps a spilled message and an inline one on the same subject in the order they were published. Three wiring lines that could be deleted with the suite staying green: the sweeper's start is now pinned by a Config interval, and the two lines that carry the bus into the deployment now refuse to boot when either is missing. A subscription can also report what it dropped, so the party that missed a message is the party that can see it. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 35 ++- core/application/distributed_test.go | 30 +++ core/services/messaging/match_test.go | 8 +- core/services/pgbus/bus.go | 210 ++++++++++++++++-- core/services/pgbus/bus_test.go | 39 +++- .../pgbus/listen_ordering_internal_test.go | 178 +++++++++++++++ core/services/pgbus/spill_test.go | 68 +++++- go.mod | 2 +- 8 files changed, 528 insertions(+), 42 deletions(-) create mode 100644 core/services/pgbus/listen_ordering_internal_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index 974c6f783da4..1252199fd139 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -671,7 +671,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade modelAdapter := nodes.NewModelRouterAdapter(router) success = true - return &DistributedServices{ + ds := &DistributedServices{ Nats: natsClient, Store: store, Registry: registry, @@ -696,7 +696,38 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade WorkerDialer: workerDialer, BackendClients: backendClients, Bus: bus, - }, nil + } + // Checked once, here, on the assembled struct. See requireBroadcastCarrier. + if err := requireBroadcastCarrier(ds); err != nil { + return nil, err + } + return ds, nil +} + +// requireBroadcastCarrier refuses to hand back a distributed deployment whose +// broadcast carrier is missing. +// +// The carrier reaches the deployment over two lines: the newBroadcastBus call +// in initDistributed, and the Bus field in the twenty-three field literal +// above. Deleting either one compiles and leaves every suite in this repository +// green, and the two failures are different. Without the construction, nothing +// can ever be published between replicas. Without the assignment the carrier is +// opened and connected but Shutdown cannot see it, so every restart leaves a +// pinned PostgreSQL session and its goroutines behind until the server runs out +// of connections, and the operator sees the failure land on whatever connects +// next rather than on LocalAI. +// +// Neither line can be reddened by a spec today: initDistributed opens NATS +// before it reaches any of this, so it cannot be called from a unit test, and a +// pointer field left out of a struct literal is not a compile error. What this +// converts both omissions into is a deployment that refuses to start and names +// what is missing, which is as far as they can be pinned until initDistributed +// is testable. The guard itself is spec'd. +func requireBroadcastCarrier(ds *DistributedServices) error { + if ds == nil || ds.Bus == nil { + return fmt.Errorf("distributed mode was initialized without a broadcast carrier: nothing could be published between replicas, and the PostgreSQL session it pins could not be closed on shutdown") + } + return nil } // newBroadcastBus opens the deployment's fan-out carrier on the auth database. diff --git a/core/application/distributed_test.go b/core/application/distributed_test.go index a822e33ff830..6002a91043af 100644 --- a/core/application/distributed_test.go +++ b/core/application/distributed_test.go @@ -61,6 +61,36 @@ var _ = Describe("opening the deployment's broadcast carrier", func() { }) }) +// The partial pin on two wiring lines that cannot be reddened by a spec: the +// newBroadcastBus call, and `Bus: bus` in the returned literal. Neither is a +// compile error when deleted and initDistributed cannot be unit tested while it +// opens NATS first, so what is available is a boot refusal, and this is what +// keeps that refusal honest. +var _ = Describe("refusing a deployment with no broadcast carrier", func() { + It("accepts services that carry one", func() { + db, dsn := testutil.SetupTestDBWithDSN() + cfg := &config.ApplicationConfig{} + cfg.Auth.DatabaseURL = dsn + bus, err := newBroadcastBus(context.Background(), cfg, db) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(bus.Close) + + Expect(requireBroadcastCarrier(&DistributedServices{Bus: bus})).To(Succeed()) + }) + + It("refuses services whose carrier was never assigned, and says what it costs", func() { + err := requireBroadcastCarrier(&DistributedServices{}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("published between replicas")) + Expect(err.Error()).To(ContainSubstring("shutdown")) + }) + + It("refuses a nil deployment rather than dereferencing it", func() { + Expect(requireBroadcastCarrier(nil)).ToNot(Succeed()) + }) +}) + var _ = Describe("shutting the distributed services down", func() { It("closes the broadcast carrier", func() { // A pinned PostgreSQL session and the goroutine parked on it, per diff --git a/core/services/messaging/match_test.go b/core/services/messaging/match_test.go index 4804768cadd7..c532dadb7387 100644 --- a/core/services/messaging/match_test.go +++ b/core/services/messaging/match_test.go @@ -1,6 +1,8 @@ package messaging_test import ( + "errors" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -58,7 +60,11 @@ var _ = DescribeTable("ValidFilter", func(filter string, wantErr bool) { err := messaging.ValidFilter(filter) if wantErr { - Expect(err).To(HaveOccurred()) + // The CLASS, not merely "an error". Carriers match on + // ErrUnsupportedFilter to tell "this caller asked for something we + // do not implement" apart from "the store is unhappy", so the + // definition has to pin what the callers match on. + Expect(errors.Is(err, messaging.ErrUnsupportedFilter)).To(BeTrue(), "got %v", err) return } Expect(err).ToNot(HaveOccurred()) diff --git a/core/services/pgbus/bus.go b/core/services/pgbus/bus.go index f16d664c2d2b..3f635d0e5a72 100644 --- a/core/services/pgbus/bus.go +++ b/core/services/pgbus/bus.go @@ -80,6 +80,23 @@ const subscribeTimeout = 30 * time.Second // wedging the carrier silently is not. const deliveryQueueDepth = 256 +// notificationQueueDepth is how many notifications may be waiting to be +// resolved and dispatched before the carrier starts dropping them. +// +// It exists so that resolving a spilled broadcast, which is one SELECT, never +// happens on the goroutine that drains PostgreSQL's notification stream. That +// goroutine falling behind does not merely delay this replica: PostgreSQL holds +// undelivered notifications in a shared, fixed-size async queue, and a listener +// that stops draining it can fill that queue and block COMMIT for every +// publisher on the SERVER, LocalAI's or not. Fourteen traffic types are moving +// onto this carrier, several of which spill by construction, so this is a +// hazard the carrier has to own rather than one to leave to its adopters. +// +// Dropping locally when the resolver falls this far behind is the right trade +// against that: a lost broadcast is recoverable and loud, a stalled server is +// neither. +const notificationQueueDepth = 1024 + // broadcastRoots is the closed set of subject roots this carrier serves. // A subject whose first token is not here is REFUSED at publish and at // subscribe, rather than being mapped to a channel of its own. @@ -152,6 +169,15 @@ type Config struct { DSN string // DB is the pooled handle NOTIFY and the spill table are written on. DB *gorm.DB + // SweepInterval is how often this carrier retires spilled broadcasts that + // have aged out. Zero means spillSweepInterval. + // + // It is a field rather than a constant so that the sweeper being STARTED is + // a testable fact. SweepSpill and SpillSweepSQL can both be exercised + // directly, and neither of them proves the carrier ever calls them: a + // deleted `go b.sweep()` left the whole suite green and bus_messages + // growing forever. + SweepInterval time.Duration } // notification is what travels in a pg_notify payload. The keys are one byte @@ -185,14 +211,43 @@ type Bus struct { connected atomic.Bool cmds chan listenCmd + inbound chan inbound listenerDone chan struct{} + resolverDone chan struct{} closeOnce sync.Once + // listenMu orders a channel's refcount decision with the LISTEN or + // UNLISTEN that decision implies, as ONE step. + // + // Deciding under mu and issuing outside it is a real defect and not a + // theoretical one. A last Unsubscribe that has decided to UNLISTEN can be + // overtaken by a Subscribe that has decided to LISTEN; the two reach the + // connection in that order; the channel ends up not listened with a live + // subscription on it. It does not self-heal, because the next Subscribe on + // that root sees first == false and never re-LISTENs, so the whole root is + // silently deaf on this replica until a connection drop triggers relisten. + // + // A second lock rather than mu, because command waits on the listener + // goroutine and delivery takes mu: holding mu across that wait would + // deadlock the carrier. Neither the listener nor the resolver ever takes + // this one. + listenMu sync.Mutex + + // listenBarrier is a test seam and is nil in production. See barrier. + listenBarrier func(stage, op string) + mu sync.Mutex nextID uint64 subs map[string]map[uint64]*subscription } +// inbound is one notification as it came off the connection, before it is +// decoded, resolved and dispatched. +type inbound struct { + channel string + payload string +} + // New opens the carrier: one pinned LISTEN connection, and the pooled handle // publishes travel on. func New(ctx context.Context, cfg Config) (*Bus, error) { @@ -225,11 +280,14 @@ func New(ctx context.Context, cfg Config) (*Bus, error) { ctx: busCtx, cancel: cancel, cmds: make(chan listenCmd), + inbound: make(chan inbound, notificationQueueDepth), listenerDone: make(chan struct{}), + resolverDone: make(chan struct{}), subs: map[string]map[uint64]*subscription{}, } b.connected.Store(true) go b.listen(conn) + go b.resolve() go b.sweep() return b, nil } @@ -274,6 +332,7 @@ func (b *Bus) Close() { b.closeOnce.Do(func() { b.cancel() <-b.listenerDone + <-b.resolverDone b.connected.Store(false) }) } @@ -335,6 +394,12 @@ func (b *Bus) Subscribe(subject string, handler func([]byte)) (messaging.Subscri return nil, fmt.Errorf("pgbus: no handler for %q", subject) } + // Before the lock, so a spec can observe that this registration has + // started even when the lock is what stops it going any further. + b.barrier("enter", "LISTEN") + b.listenMu.Lock() + defer b.listenMu.Unlock() + b.mu.Lock() b.nextID++ sub := &subscription{ @@ -344,6 +409,7 @@ func (b *Bus) Subscribe(subject string, handler func([]byte)) (messaging.Subscri id: b.nextID, queue: make(chan []byte, deliveryQueueDepth), stop: make(chan struct{}), + done: make(chan struct{}), } first := len(b.subs[channel]) == 0 if first { @@ -355,18 +421,54 @@ func (b *Bus) Subscribe(subject string, handler func([]byte)) (messaging.Subscri go sub.run(handler) if first { + b.barrier("issue", "LISTEN") if err := b.command("LISTEN " + pgx.Identifier{channel}.Sanitize()); err != nil { - _ = sub.Unsubscribe() + // Not Unsubscribe: the ordering lock is already held here, and the + // connection was never listening on this channel, so there is + // nothing to UNLISTEN. + b.forget(sub) return nil, err } } return sub, nil } +// forget removes a registration without touching the channel's LISTEN state. +func (b *Bus) forget(sub *subscription) { + sub.once.Do(func() { + b.mu.Lock() + delete(b.subs[sub.channel], sub.id) + if len(b.subs[sub.channel]) == 0 { + delete(b.subs, sub.channel) + } + b.mu.Unlock() + close(sub.stop) + }) +} + +// barrier is a test seam and does nothing in production. +// +// It exists because the ordering listenMu enforces cannot be observed from +// outside this package and cannot be provoked from outside it either: the +// natural window is microseconds wide, and it was measured at zero hits in +// forty attempts while being ten out of ten once widened. A spec that waits for +// that window to open is a spec that passes by luck, which is worse than no +// spec at all for a defect that leaves a whole subject root deaf. +func (b *Bus) barrier(stage, op string) { + if b.listenBarrier != nil { + b.listenBarrier(stage, op) + } +} + // command hands a LISTEN or UNLISTEN to the goroutine that owns the connection -// and waits for it, so a Subscribe that has returned is a registration the -// server has already acknowledged. Returning before that would lose every -// message published in the gap. +// and waits for it. Returning before the server had acknowledged it would lose +// every message published in the gap. +// +// A Subscribe that has returned is therefore always a registration the server +// has acknowledged, including the case where this Subscribe issued nothing +// because the channel was already listened: listenMu means the Subscribe that +// DID issue the LISTEN had already been acknowledged before this one could see +// its registration. func (b *Bus) command(sql string) error { cmd := listenCmd{sql: sql, done: make(chan error, 1)} timeout := time.NewTimer(subscribeTimeout) @@ -439,7 +541,34 @@ func (b *Bus) listen(conn *pgx.Conn) { conn = replacement continue } - b.deliver(n.Channel, n.Payload) + b.offer(n.Channel, n.Payload) + } +} + +// offer hands a notification to the resolver. It never blocks: the listener's +// only job is to keep PostgreSQL's async queue draining. +func (b *Bus) offer(channel, payload string) { + select { + case b.inbound <- inbound{channel: channel, payload: payload}: + default: + xlog.Error("Broadcast carrier dropped a notification: the resolver is not keeping up", + "channel", channel, "depth", notificationQueueDepth) + } +} + +// resolve is where a spilled broadcast is read back and where every +// notification is dispatched. Both are off the listener on purpose, and both +// are on ONE goroutine, so a spilled message and an inline one on the same +// subject keep the order they were published in. +func (b *Bus) resolve() { + defer close(b.resolverDone) + for { + select { + case <-b.ctx.Done(): + return + case in := <-b.inbound: + b.deliver(in.channel, in.payload) + } } } @@ -500,6 +629,8 @@ func (b *Bus) relisten(conn *pgx.Conn) error { // deliver resolves one notification and hands it to the subscribers whose // filters match. func (b *Bus) deliver(channel, payload string) { + b.barrier("enter", "DELIVER") + var n notification if err := json.Unmarshal([]byte(payload), &n); err != nil { xlog.Error("Broadcast carrier received an undecodable notification", "channel", channel, "error", err) @@ -507,6 +638,14 @@ func (b *Bus) deliver(channel, payload string) { } data := []byte(n.Data) + if n.SpillID == "" && len(data) == 0 { + // Publish cannot produce this: json.Marshal never returns an empty + // encoding. It is reachable only if something other than this carrier + // notifies on a localai_ channel, and delivering nil to a handler + // would be a message that says nothing rather than no message at all. + xlog.Error("Broadcast carrier received a notification with no payload", "channel", channel, "subject", n.Subject) + return + } if n.SpillID != "" { resolved, err := b.resolveSpill(n.SpillID) if err != nil { @@ -544,12 +683,37 @@ type subscription struct { filter string id uint64 - queue chan []byte - stop chan struct{} - once sync.Once + queue chan []byte + stop chan struct{} + done chan struct{} + dropped atomic.Uint64 + once sync.Once +} + +// DropCounter is the optional interface a Subscription satisfies when it can +// report how many broadcasts it lost. +// +// It exists because the drop is otherwise invisible to the party that needs to +// know. The error log lands on the receiving replica, there is no sequence +// number and no gap signal, so "anything that must survive a gap belongs in a +// table, with the broadcast as a hint to go and look" cannot be acted on by the +// subscriber that missed the hint. A subscriber whose subject has no successor +// message, a job result rather than a progress tick, can type-assert to this +// and go read the row. +// +// It is not on messaging.Subscription: the NATS client's subscription cannot +// answer it, and widening that interface would make every existing consumer +// claim a guarantee it does not have. +type DropCounter interface { + Dropped() uint64 } +// Dropped reports how many broadcasts this subscription lost because its +// handler was too far behind. It only ever grows. +func (s *subscription) Dropped() uint64 { return s.dropped.Load() } + func (s *subscription) run(handler func([]byte)) { + defer close(s.done) for { select { case data := <-s.queue: @@ -566,8 +730,9 @@ func (s *subscription) enqueue(subject string, data []byte) { select { case s.queue <- data: default: + s.dropped.Add(1) xlog.Error("Broadcast carrier dropped a message: subscriber is not keeping up", - "filter", s.filter, "subject", subject, "depth", deliveryQueueDepth) + "filter", s.filter, "subject", subject, "depth", deliveryQueueDepth, "dropped", s.dropped.Load()) } } @@ -576,17 +741,24 @@ func (s *subscription) enqueue(subject string, data []byte) { func (s *subscription) Unsubscribe() error { var err error s.once.Do(func() { - s.bus.mu.Lock() - delete(s.bus.subs[s.channel], s.id) - last := len(s.bus.subs[s.channel]) == 0 + b := s.bus + b.barrier("enter", "UNLISTEN") + // The whole decision AND its issuance, as one step. See listenMu. + b.listenMu.Lock() + defer b.listenMu.Unlock() + + b.mu.Lock() + delete(b.subs[s.channel], s.id) + last := len(b.subs[s.channel]) == 0 if last { - delete(s.bus.subs, s.channel) + delete(b.subs, s.channel) } - s.bus.mu.Unlock() + b.mu.Unlock() close(s.stop) - if last && s.bus.ctx.Err() == nil { - err = s.bus.command("UNLISTEN " + pgx.Identifier{s.channel}.Sanitize()) + if last && b.ctx.Err() == nil { + b.barrier("issue", "UNLISTEN") + err = b.command("UNLISTEN " + pgx.Identifier{s.channel}.Sanitize()) } }) return err @@ -612,7 +784,11 @@ func (b *Bus) resolveSpill(id string) ([]byte, error) { // sweep retires spilled rows that every replica has had time to read. func (b *Bus) sweep() { - ticker := time.NewTicker(spillSweepInterval) + interval := b.cfg.SweepInterval + if interval <= 0 { + interval = spillSweepInterval + } + ticker := time.NewTicker(interval) defer ticker.Stop() for { select { diff --git a/core/services/pgbus/bus_test.go b/core/services/pgbus/bus_test.go index 0585677ee343..899536c3b064 100644 --- a/core/services/pgbus/bus_test.go +++ b/core/services/pgbus/bus_test.go @@ -5,7 +5,6 @@ package pgbus_test import ( "context" "errors" - "fmt" "path/filepath" "time" @@ -209,6 +208,34 @@ var _ = Describe("the PostgreSQL broadcast carrier", func() { }) }) + Describe("a subscriber that falls behind", func() { + It("counts what it lost, so the subscriber can go and read the row", func() { + // The drop is loud in the log of the replica that took it, and that + // is the wrong party: the subscriber is the one that has to decide + // to go and read the table instead. Task 12 moves jobs.*.result + // onto this carrier, and a result has no successor message, so an + // invisible loss there is a job whose answer silently never + // arrives. + block := make(chan struct{}) + DeferCleanup(func() { close(block) }) + s, err := sub.Subscribe("jobs.slow.consumer", func([]byte) { <-block }) + Expect(err).ToNot(HaveOccurred()) + + counter, ok := s.(pgbus.DropCounter) + Expect(ok).To(BeTrue(), "a subscription must be able to report its losses") + Expect(counter.Dropped()).To(BeZero()) + + // Comfortably past the queue depth, stated absolutely rather than + // as deliveryQueueDepth+n, so a change to the depth moves the + // behaviour and not the expectation. + for i := 0; i < 400; i++ { + Expect(pub.Publish("jobs.slow.consumer", map[string]int{"i": i})).To(Succeed()) + } + + Eventually(counter.Dropped, 30*time.Second).Should(BeNumerically(">", 0)) + }) + }) + Describe("losing the LISTEN connection", func() { It("reconnects, restores its registrations and delivers again", func() { // The transport failure a fake cannot produce. Terminating the @@ -328,13 +355,3 @@ var _ = Describe("constructing the carrier", func() { Expect(err).To(HaveOccurred()) }) }) - -var _ = Describe("the interface the carrier is interchangeable through", func() { - It("is satisfied by the bus", func() { - // Compile-time, not behavioural: the point is that a signature change - // on either carrier fails here rather than in whichever call site is - // migrated next. - var b messaging.Broadcaster = (*pgbus.Bus)(nil) - Expect(fmt.Sprintf("%T", b)).To(Equal("*pgbus.Bus")) - }) -}) diff --git a/core/services/pgbus/listen_ordering_internal_test.go b/core/services/pgbus/listen_ordering_internal_test.go new file mode 100644 index 000000000000..c7fc5657a5a7 --- /dev/null +++ b/core/services/pgbus/listen_ordering_internal_test.go @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: MIT + +package pgbus + +import ( + "context" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/testutil" +) + +// Internal on purpose. What these specs are about is the ORDER in which two +// registrations reach the connection, and that order is not observable from +// outside the package: a channel that is subscribed but not listened behaves +// exactly like a deployment where nobody is publishing. +// +// They drive that order through the barrier seam rather than by racing +// goroutines and hoping. The natural window here was measured at zero hits in +// forty attempts and ten out of ten once widened by hand, so a spec that races +// for it would pass by luck on a defect that leaves a whole subject root deaf +// until the connection drops. +var _ = Describe("ordering LISTEN and UNLISTEN on one root", func() { + var b *Bus + + newBus := func(cfg Config) *Bus { + GinkgoHelper() + built, err := New(context.Background(), cfg) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(built.Close) + return built + } + + BeforeEach(func() { + db, dsn := testutil.SetupTestDBWithDSN() + Expect(Migrate(context.Background(), db)).To(Succeed()) + b = newBus(Config{DSN: dsn, DB: db}) + }) + + It("does not let a new LISTEN overtake the UNLISTEN it races", func() { + leaving, err := b.Subscribe("jobs.race.leaving", func([]byte) {}) + Expect(err).ToNot(HaveOccurred()) + + unlistenDecided := make(chan struct{}) + release := make(chan struct{}) + arrived := make(chan struct{}) + listenIssued := make(chan struct{}, 1) + // Set after the first Subscribe, so only the racing pair is observed. + b.listenBarrier = func(stage, op string) { + switch { + case op == "UNLISTEN" && stage == "issue": + close(unlistenDecided) + <-release + case op == "LISTEN" && stage == "enter": + close(arrived) + case op == "LISTEN" && stage == "issue": + listenIssued <- struct{}{} + } + } + + unsubscribed := make(chan error, 1) + go func() { + defer GinkgoRecover() + unsubscribed <- leaving.Unsubscribe() + }() + // The last unsubscribe has decided to UNLISTEN and has not issued it. + // Eventually rather than a bare receive: a carrier that never reaches + // the decision at all must fail this spec, not hang it. + Eventually(unlistenDecided).Should(BeClosed()) + + delivered := make(chan []byte, 1) + subscribed := make(chan error, 1) + go func() { + defer GinkgoRecover() + _, err := b.Subscribe("jobs.race.arriving", func(d []byte) { delivered <- d }) + subscribed <- err + }() + // The competing subscribe has started. Waiting for this rather than for + // a duration is what makes the assertion below decide something. + Eventually(arrived).Should(BeClosed()) + + // The bite, and it is deterministic in both directions. Unserialized, + // this LISTEN is on the wire within microseconds of the line above and + // the UNLISTEN then undoes it. Serialized, it provably cannot be + // issued while the unsubscribe holds the ordering lock. + Consistently(listenIssued).ShouldNot(Receive()) + + close(release) + Eventually(unsubscribed).Should(Receive(BeNil())) + Eventually(subscribed).Should(Receive(BeNil())) + + // The consequence, stated as delivery: the root must not be deaf. + Expect(b.Publish("jobs.race.arriving", map[string]string{"m": "after"})).To(Succeed()) + Eventually(delivered).Should(Receive(MatchJSON(`{"m":"after"}`))) + }) + + It("listens once for a root and stops listening when its last subscriber leaves", func() { + // The half of Unsubscribe that the delivery spec cannot see. Without + // the registration being removed from the map, the second unsubscribe + // never decides it was the last, and the connection keeps listening on + // a root nothing is subscribed to for the life of the process. + issued := make(chan string, 4) + b.listenBarrier = func(stage, op string) { + if stage == "issue" { + issued <- op + } + } + + one, err := b.Subscribe("jobs.refcount.a", func([]byte) {}) + Expect(err).ToNot(HaveOccurred()) + two, err := b.Subscribe("jobs.refcount.b", func([]byte) {}) + Expect(err).ToNot(HaveOccurred()) + + Expect(one.Unsubscribe()).To(Succeed()) + Expect(two.Unsubscribe()).To(Succeed()) + + Expect(issued).To(HaveLen(2)) + Expect(<-issued).To(Equal("LISTEN")) + Expect(<-issued).To(Equal("UNLISTEN")) + }) + + It("waits for the resolver as well as the listener when it closes", func() { + // Close returning while the resolver is still dispatching hands the + // caller a bus whose handlers can still fire after shutdown, against a + // database handle the process is about to drop. + // + // Asserted as "Close has not returned yet" rather than as "the done + // channel is closed afterwards": the resolver exits on its own when the + // context is cancelled, so the second shape passes whether Close waits + // or not. + dispatching := make(chan struct{}, 1) + release := make(chan struct{}) + var releaseOnce sync.Once + // Registered inside the It, so it runs BEFORE the bus teardown the + // BeforeEach registered. A failing assertion below must not leave the + // resolver parked on a channel that Close is waiting for. + DeferCleanup(func() { releaseOnce.Do(func() { close(release) }) }) + b.listenBarrier = func(stage, op string) { + if op == "DELIVER" && stage == "enter" { + select { + case dispatching <- struct{}{}: + default: + } + <-release + } + } + _, err := b.Subscribe("jobs.close.pending", func([]byte) {}) + Expect(err).ToNot(HaveOccurred()) + Expect(b.Publish("jobs.close.pending", map[string]string{"m": "x"})).To(Succeed()) + Eventually(dispatching).Should(Receive()) + + closed := make(chan struct{}) + go func() { + defer GinkgoRecover() + b.Close() + close(closed) + }() + + Consistently(closed).ShouldNot(BeClosed()) + releaseOnce.Do(func() { close(release) }) + Eventually(closed).Should(BeClosed()) + }) + + It("stops the delivery goroutine of a subscription that has left", func() { + // The other half. A subscription that is deregistered but whose runner + // is never stopped leaks a goroutine per Unsubscribe, and the delivery + // spec cannot tell the two halves apart because either one alone makes + // the handler silent. + s, err := b.Subscribe("jobs.refcount.runner", func([]byte) {}) + Expect(err).ToNot(HaveOccurred()) + + Expect(s.Unsubscribe()).To(Succeed()) + + Eventually(s.(*subscription).done).Should(BeClosed()) + }) +}) diff --git a/core/services/pgbus/spill_test.go b/core/services/pgbus/spill_test.go index 16bafb2e2c9b..0545908e2b55 100644 --- a/core/services/pgbus/spill_test.go +++ b/core/services/pgbus/spill_test.go @@ -139,6 +139,25 @@ var _ = Describe("broadcasts too large for a notification", func() { Expect(row.CreatedAt).ToNot(BeZero()) }) + It("delivers nothing for a notification that carries no payload at all", func() { + // Publish cannot produce this, because json.Marshal is never empty. It + // is reachable if anything else ever notifies on a localai_ channel, + // and a handler called with nil is a message that says nothing rather + // than no message, which is exactly the confusion this carrier must + // never create. The sentinel proves it kept carrying. + out := make(chan []byte, 4) + _, err := sub.Subscribe(subject, func(b []byte) { out <- b }) + Expect(err).ToNot(HaveOccurred()) + + Expect(db.Exec("SELECT pg_notify(?, ?)", "localai_jobs", + fmt.Sprintf(`{"s":%q}`, subject)).Error).To(Succeed()) + Expect(pub.Publish(subject, map[string]string{"m": "sentinel"})).To(Succeed()) + + var got []byte + Eventually(out, 10*time.Second).Should(Receive(&got)) + Expect(got).To(MatchJSON(`{"m":"sentinel"}`)) + }) + It("delivers nothing, and keeps carrying, when a spilled row cannot be found", func() { // A notification whose row is gone is a lost message, not an empty one: // handing a handler nil would let a consumer read a carrier failure as @@ -159,13 +178,31 @@ var _ = Describe("broadcasts too large for a notification", func() { }) var _ = Describe("retiring spilled rows", func() { - var db *gorm.DB + var ( + db *gorm.DB + dsn string + ) BeforeEach(func() { - db, _ = testutil.SetupTestDBWithDSN() + db, dsn = testutil.SetupTestDBWithDSN() Expect(pgbus.Migrate(context.Background(), db)).To(Succeed()) }) + // aged writes one spilled row and backdates it on the DATABASE clock, which + // is the clock the sweep compares against. + aged := func(id string) { + GinkgoHelper() + Expect(db.Create(&pgbus.BusMessage{ID: id, Subject: "jobs.x", Payload: []byte(`{}`)}).Error).To(Succeed()) + Expect(db.Exec("UPDATE bus_messages SET created_at = now() - interval '1 hour' WHERE id = ?", id).Error).To(Succeed()) + } + + rows := func() []string { + GinkgoHelper() + var ids []string + Expect(db.Model(&pgbus.BusMessage{}).Pluck("id", &ids).Error).To(Succeed()) + return ids + } + It("leaves the cutoff to the database clock", func() { // Pinned as a statement shape rather than as behaviour on purpose. The // test container shares this host's clock, so a cutoff computed in Go @@ -177,18 +214,29 @@ var _ = Describe("retiring spilled rows", func() { Expect(pgbus.SpillSweepSQL).ToNot(ContainSubstring("created_at < ?")) }) + It("runs the sweep on its own, without anyone asking it to", func() { + // SweepSpill and SpillSweepSQL are both spec'd directly, and neither of + // them proves the carrier ever CALLS them. Deleting the sweeper's `go` + // statement left the whole suite green and bus_messages growing + // forever, which is the unpinned-wiring shape this programme exists to + // remove. The interval is a Config field so that this spec exists. + aged("swept") + Expect(db.Create(&pgbus.BusMessage{ID: "kept", Subject: "jobs.x", Payload: []byte(`{}`)}).Error).To(Succeed()) + + b, err := pgbus.New(context.Background(), pgbus.Config{DSN: dsn, DB: db, SweepInterval: 20 * time.Millisecond}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(b.Close) + + Eventually(rows, 30*time.Second).Should(ConsistOf("kept")) + }) + It("deletes rows past the retention and keeps the rest", func() { - old := pgbus.BusMessage{ID: "old", Subject: "jobs.x", Payload: []byte(`{}`)} - fresh := pgbus.BusMessage{ID: "fresh", Subject: "jobs.x", Payload: []byte(`{}`)} - Expect(db.Create(&old).Error).To(Succeed()) - Expect(db.Create(&fresh).Error).To(Succeed()) - Expect(db.Exec("UPDATE bus_messages SET created_at = now() - interval '1 hour' WHERE id = 'old'").Error).To(Succeed()) + aged("old") + Expect(db.Create(&pgbus.BusMessage{ID: "fresh", Subject: "jobs.x", Payload: []byte(`{}`)}).Error).To(Succeed()) Expect(pgbus.SweepSpill(context.Background(), db, 5*time.Minute)).To(Succeed()) - var ids []string - Expect(db.Model(&pgbus.BusMessage{}).Pluck("id", &ids).Error).To(Succeed()) - Expect(ids).To(ConsistOf("fresh")) + Expect(rows()).To(ConsistOf("fresh")) }) }) diff --git a/go.mod b/go.mod index be0463385be0..c0abdec079e2 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/gpustack/gguf-parser-go v0.25.0 github.com/hpcloud/tail v1.0.0 github.com/ipfs/go-log v1.0.5 + github.com/jackc/pgx/v5 v5.9.2 github.com/jaypipes/ghw v0.24.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/cpuid/v2 v2.3.0 @@ -253,7 +254,6 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.9.2 github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jung-kurt/gofpdf v1.16.2 // indirect From b8d514d5a3849743b976d1e0d18a658ab3a994ac Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 10:23:40 +0000 Subject: [PATCH 083/153] feat(distributed): keep the listener draining and let it come back PostgreSQL holds undelivered notifications in a queue it shares with every session on the server, and it kills a listener that stops draining. Two failure modes follow, and both are silent: a carrier that blocked on a slow resolver would lose its connection and with it every later broadcast, and a carrier that reconnected without re-registering would be connected and deaf. The receive and dispatch halves were already separate. What was missing is everything around them. The listener path moves into listener.go and gains a carrier-level Dropped() so a replica that is behind can be seen; the queue depth and the spill retention become Config fields with exported defaults; the LISTEN session gets an application_name so an operator can count listeners in pg_stat_activity and a spec can drop exactly one of them; and OnReconnect fires after the re-LISTEN, on a goroutine of its own, because a callback re-hydrates from a database and must never run on the path whose only job is to drain. That callback is reached through an optional interface assertion, so deleting its invocation compiles and every adopter silently stops converging. The spec is the only guard, and it is named in a comment at the site. The slow consumer is proved through the transport rather than a seam: an ACCESS EXCLUSIVE lock on bus_messages stalls the resolver's spill SELECT for exactly as long as the spec holds it, and the listener is shown still draining and dropping while it does. The dropped connection is a pg_terminate_backend matched on the carrier's own application name. Neither Dropped nor IsConnected is on messaging.Broadcaster, and a spec asserts that over the interface type. Both are facts about a frontend; the conditions a scheduler acts on are facts about a worker, and no consumer holding the interface can read one as the other. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/pgbus/bus.go | 294 +++-------------- core/services/pgbus/listener.go | 377 ++++++++++++++++++++++ core/services/pgbus/listener_test.go | 314 ++++++++++++++++++ core/services/pgbus/spill.go | 32 +- core/services/pgbus/spill_test.go | 37 +++ docs/content/features/distributed-mode.md | 2 + 6 files changed, 802 insertions(+), 254 deletions(-) create mode 100644 core/services/pgbus/listener.go create mode 100644 core/services/pgbus/listener_test.go diff --git a/core/services/pgbus/bus.go b/core/services/pgbus/bus.go index 3f635d0e5a72..ab843449917e 100644 --- a/core/services/pgbus/bus.go +++ b/core/services/pgbus/bus.go @@ -55,21 +55,6 @@ const channelPrefix = "localai_" // of data in a notification the server rejects. const maxNotifyPayloadBytes = 8000 -// listenPollInterval bounds how long a LISTEN or UNLISTEN waits for the -// listener goroutine to come off WaitForNotification and run it. -// -// It is a bound on REGISTRATION, never on delivery: a notification wakes -// WaitForNotification immediately. The connection survives the deadline, which -// pgx implements as a read deadline rather than a close (pgconn's peekMessage -// closes on anything except a net timeout), so this costs one syscall per -// interval and nothing else. -const listenPollInterval = 25 * time.Millisecond - -// subscribeTimeout bounds Subscribe when the listener is busy redialling a -// database that has gone away, so a caller gets an error instead of a -// goroutine that never returns. -const subscribeTimeout = 30 * time.Second - // deliveryQueueDepth is how far one subscriber may fall behind before its // broadcasts are dropped. // @@ -80,23 +65,6 @@ const subscribeTimeout = 30 * time.Second // wedging the carrier silently is not. const deliveryQueueDepth = 256 -// notificationQueueDepth is how many notifications may be waiting to be -// resolved and dispatched before the carrier starts dropping them. -// -// It exists so that resolving a spilled broadcast, which is one SELECT, never -// happens on the goroutine that drains PostgreSQL's notification stream. That -// goroutine falling behind does not merely delay this replica: PostgreSQL holds -// undelivered notifications in a shared, fixed-size async queue, and a listener -// that stops draining it can fill that queue and block COMMIT for every -// publisher on the SERVER, LocalAI's or not. Fourteen traffic types are moving -// onto this carrier, several of which spill by construction, so this is a -// hazard the carrier has to own rather than one to leave to its adopters. -// -// Dropping locally when the resolver falls this far behind is the right trade -// against that: a lost broadcast is recoverable and loud, a stalled server is -// neither. -const notificationQueueDepth = 1024 - // broadcastRoots is the closed set of subject roots this carrier serves. // A subject whose first token is not here is REFUSED at publish and at // subscribe, rather than being mapped to a channel of its own. @@ -169,14 +137,24 @@ type Config struct { DSN string // DB is the pooled handle NOTIFY and the spill table are written on. DB *gorm.DB + // Queue bounds how many notifications may wait for the resolver before this + // carrier starts dropping them. Zero means DefaultQueueDepth. + // + // A field so a deployment whose subjects spill heavily can buy itself more + // slack, and so a spec can fill the queue without publishing a thousand + // messages to do it. + Queue int + // Retention is how long a spilled row outlives its notification. Zero means + // DefaultSpillRetention. + Retention time.Duration // SweepInterval is how often this carrier retires spilled broadcasts that - // have aged out. Zero means spillSweepInterval. + // have aged out. Zero means Retention / 2. // - // It is a field rather than a constant so that the sweeper being STARTED is - // a testable fact. SweepSpill and SpillSweepSQL can both be exercised - // directly, and neither of them proves the carrier ever calls them: a - // deleted `go b.sweep()` left the whole suite green and bus_messages - // growing forever. + // It is a field rather than a derived value alone so that the purge loop + // being STARTED is a testable fact. PurgeBefore, SweepSpill and + // SpillSweepSQL can all be exercised directly, and none of them proves the + // carrier ever calls them: a deleted `go b.purge()` left the whole suite + // green and bus_messages growing forever. SweepInterval time.Duration } @@ -189,15 +167,6 @@ type notification struct { SpillID string `json:"i,omitempty"` } -// listenCmd is a LISTEN or UNLISTEN handed to the goroutine that owns the -// connection. The connection is not concurrency safe and the listener goroutine -// is parked in WaitForNotification most of the time, so registrations are -// queued to it rather than run against it. -type listenCmd struct { - sql string - done chan error -} - // Bus is one replica's end of the broadcast carrier. type Bus struct { cfg Config @@ -210,12 +179,28 @@ type Bus struct { // evidence would let a database hiccup evict models. connected atomic.Bool + // dropped counts notifications the listener discarded because the resolver + // was behind. Same rule as connected: reportable, never actionable. + dropped atomic.Uint64 + + // appName identifies this carrier's LISTEN session in pg_stat_activity, and + // retention is Config.Retention after defaulting. Both are set once in New + // and read from other goroutines afterwards, so neither may be reassigned. + appName string + retention time.Duration + cmds chan listenCmd inbound chan inbound listenerDone chan struct{} resolverDone chan struct{} closeOnce sync.Once + // cbMu guards reconnectCbs. Its own lock, because a registration arrives on + // a caller's goroutine while the listener is redialling and neither should + // wait on delivery to finish. + cbMu sync.Mutex + reconnectCbs []func() + // listenMu orders a channel's refcount decision with the LISTEN or // UNLISTEN that decision implies, as ONE step. // @@ -241,13 +226,6 @@ type Bus struct { subs map[string]map[uint64]*subscription } -// inbound is one notification as it came off the connection, before it is -// decoded, resolved and dispatched. -type inbound struct { - channel string - payload string -} - // New opens the carrier: one pinned LISTEN connection, and the pooled handle // publishes travel on. func New(ctx context.Context, cfg Config) (*Bus, error) { @@ -265,7 +243,12 @@ func New(ctx context.Context, cfg Config) (*Bus, error) { return nil, errors.New("pgbus: no DSN for the LISTEN connection") } - conn, err := pgx.Connect(ctx, cfg.DSN) + // Per carrier and not per deployment, so two replicas on one database are + // separable in pg_stat_activity and a spec can drop one listener's session + // without touching the peer it is asserting against. + appName := applicationNamePrefix + uuid.NewString() + + conn, err := dial(ctx, cfg.DSN, appName) if err != nil { return nil, fmt.Errorf("pgbus: opening the LISTEN connection: %w", err) } @@ -274,13 +257,24 @@ func New(ctx context.Context, cfg Config) (*Bus, error) { return nil, err } + queue := cfg.Queue + if queue <= 0 { + queue = DefaultQueueDepth + } + retention := cfg.Retention + if retention <= 0 { + retention = DefaultSpillRetention + } + busCtx, cancel := context.WithCancel(context.WithoutCancel(ctx)) b := &Bus{ cfg: cfg, ctx: busCtx, cancel: cancel, + appName: appName, + retention: retention, cmds: make(chan listenCmd), - inbound: make(chan inbound, notificationQueueDepth), + inbound: make(chan inbound, queue), listenerDone: make(chan struct{}), resolverDone: make(chan struct{}), subs: map[string]map[uint64]*subscription{}, @@ -288,7 +282,7 @@ func New(ctx context.Context, cfg Config) (*Bus, error) { b.connected.Store(true) go b.listen(conn) go b.resolve() - go b.sweep() + go b.purge() return b, nil } @@ -460,172 +454,6 @@ func (b *Bus) barrier(stage, op string) { } } -// command hands a LISTEN or UNLISTEN to the goroutine that owns the connection -// and waits for it. Returning before the server had acknowledged it would lose -// every message published in the gap. -// -// A Subscribe that has returned is therefore always a registration the server -// has acknowledged, including the case where this Subscribe issued nothing -// because the channel was already listened: listenMu means the Subscribe that -// DID issue the LISTEN had already been acknowledged before this one could see -// its registration. -func (b *Bus) command(sql string) error { - cmd := listenCmd{sql: sql, done: make(chan error, 1)} - timeout := time.NewTimer(subscribeTimeout) - defer timeout.Stop() - - select { - case b.cmds <- cmd: - case <-b.ctx.Done(): - return errors.New("pgbus: the carrier is closed") - case <-timeout.C: - return fmt.Errorf("pgbus: %q timed out waiting for the listen connection", sql) - } - select { - case err := <-cmd.done: - return err - case <-b.ctx.Done(): - return errors.New("pgbus: the carrier is closed") - case <-timeout.C: - return fmt.Errorf("pgbus: %q timed out on the listen connection", sql) - } -} - -// listen owns the pinned connection for the life of the Bus: it is the only -// goroutine that touches it, which is what makes a connection that is not -// concurrency safe usable from many callers. -func (b *Bus) listen(conn *pgx.Conn) { - defer close(b.listenerDone) - defer func() { - b.connected.Store(false) - // A background context, because b.ctx is already cancelled by the time - // this runs and the close still has to reach the server. - closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _ = conn.Close(closeCtx) - }() - - for { - if b.ctx.Err() != nil { - return - } - // Registrations first: a subscriber that has been waiting must not be - // held up behind a poll interval that has just restarted. - select { - case cmd := <-b.cmds: - _, err := conn.Exec(b.ctx, cmd.sql) - cmd.done <- err - continue - default: - } - - waitCtx, cancel := context.WithTimeout(b.ctx, listenPollInterval) - n, err := conn.WaitForNotification(waitCtx) - deadline := waitCtx.Err() != nil - cancel() - - switch { - case b.ctx.Err() != nil: - return - case deadline: - // The ordinary case: nothing arrived inside the poll window. pgx - // implements the deadline as a read deadline and keeps the - // connection, so this is not a failure. - continue - case err != nil: - xlog.Warn("Broadcast carrier lost its listen connection, redialling", "error", err) - replacement, ok := b.redial(conn) - if !ok { - return - } - conn = replacement - continue - } - b.offer(n.Channel, n.Payload) - } -} - -// offer hands a notification to the resolver. It never blocks: the listener's -// only job is to keep PostgreSQL's async queue draining. -func (b *Bus) offer(channel, payload string) { - select { - case b.inbound <- inbound{channel: channel, payload: payload}: - default: - xlog.Error("Broadcast carrier dropped a notification: the resolver is not keeping up", - "channel", channel, "depth", notificationQueueDepth) - } -} - -// resolve is where a spilled broadcast is read back and where every -// notification is dispatched. Both are off the listener on purpose, and both -// are on ONE goroutine, so a spilled message and an inline one on the same -// subject keep the order they were published in. -func (b *Bus) resolve() { - defer close(b.resolverDone) - for { - select { - case <-b.ctx.Done(): - return - case in := <-b.inbound: - b.deliver(in.channel, in.payload) - } - } -} - -// redial replaces a dead LISTEN connection and restores every registration on -// it. Without the restore the carrier comes back deaf, which looks exactly like -// a deployment where nobody is publishing any more. -func (b *Bus) redial(dead *pgx.Conn) (*pgx.Conn, bool) { - b.connected.Store(false) - closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - _ = dead.Close(closeCtx) - cancel() - - backoff := 100 * time.Millisecond - for { - if b.ctx.Err() != nil { - return nil, false - } - conn, err := pgx.Connect(b.ctx, b.cfg.DSN) - if err == nil { - if err = b.relisten(conn); err == nil { - b.connected.Store(true) - xlog.Info("Broadcast carrier reconnected") - return conn, true - } - closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - _ = conn.Close(closeCtx) - cancel() - } - select { - case <-b.ctx.Done(): - return nil, false - case <-time.After(backoff): - } - if backoff < 5*time.Second { - backoff *= 2 - } - } -} - -func (b *Bus) relisten(conn *pgx.Conn) error { - b.mu.Lock() - channels := make([]string, 0, len(b.subs)) - for channel, subs := range b.subs { - if len(subs) > 0 { - channels = append(channels, channel) - } - } - b.mu.Unlock() - - for _, channel := range channels { - if _, err := conn.Exec(b.ctx, "LISTEN "+pgx.Identifier{channel}.Sanitize()); err != nil { - return err - } - } - return nil -} - // deliver resolves one notification and hands it to the subscribers whose // filters match. func (b *Bus) deliver(channel, payload string) { @@ -782,24 +610,4 @@ func (b *Bus) resolveSpill(id string) ([]byte, error) { return row.Payload, nil } -// sweep retires spilled rows that every replica has had time to read. -func (b *Bus) sweep() { - interval := b.cfg.SweepInterval - if interval <= 0 { - interval = spillSweepInterval - } - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-b.ctx.Done(): - return - case <-ticker.C: - if err := SweepSpill(b.ctx, b.cfg.DB, spillRetention); err != nil && b.ctx.Err() == nil { - xlog.Warn("Broadcast carrier could not retire spilled messages", "error", err) - } - } - } -} - var _ messaging.Broadcaster = (*Bus)(nil) diff --git a/core/services/pgbus/listener.go b/core/services/pgbus/listener.go new file mode 100644 index 000000000000..25c99d28b8de --- /dev/null +++ b/core/services/pgbus/listener.go @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: MIT + +package pgbus + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "github.com/mudler/xlog" +) + +// DefaultQueueDepth bounds how many notifications may be waiting for the +// resolver before this carrier starts dropping them. +// +// A bound and not a blocking channel, deliberately. PostgreSQL holds undelivered +// notifications in a shared, server-wide async queue, and it kills a session +// whose queue the client is not draining. A listener that blocked on a slow +// resolver would therefore lose the connection and with it EVERY subsequent +// broadcast, which is unbounded loss taken on to avoid bounded loss, and it can +// block COMMIT for every other publisher on that server while it lasts. +const DefaultQueueDepth = 1024 + +// DefaultSpillRetention is how long a spilled row outlives its notification +// before the purge loop deletes it. +// +// It is generous against the delivery it has to survive, which is one +// notification and one primary-key SELECT, and short enough that a busy +// deployment does not accumulate LLM outputs in this table. +const DefaultSpillRetention = 10 * time.Minute + +// listenPollInterval bounds how long a LISTEN or UNLISTEN waits for the +// listener goroutine to come off WaitForNotification and run it. +// +// It is a bound on REGISTRATION, never on delivery: a notification wakes +// WaitForNotification immediately. The connection survives the deadline, which +// pgx implements as a read deadline rather than a close (pgconn's peekMessage +// closes on anything except a net timeout), so this costs one syscall per +// interval and nothing else. +const listenPollInterval = 25 * time.Millisecond + +// subscribeTimeout bounds Subscribe when the listener is busy redialling a +// database that has gone away, so a caller gets an error instead of a +// goroutine that never returns. +const subscribeTimeout = 30 * time.Second + +// applicationNamePrefix marks this carrier's LISTEN sessions in +// pg_stat_activity. An operator counting listeners, and a spec that has to drop +// ONE carrier's connection without touching its peer's, both need a session they +// can name rather than guess at. +const applicationNamePrefix = "localai_pgbus_" + +// listenCmd is a LISTEN or UNLISTEN handed to the goroutine that owns the +// connection. The connection is not concurrency safe and the listener goroutine +// is parked in WaitForNotification most of the time, so registrations are +// queued to it rather than run against it. +type listenCmd struct { + sql string + done chan error +} + +// inbound is one notification as it came off the connection, before it is +// decoded, resolved and dispatched. +type inbound struct { + channel string + payload string +} + +// ApplicationName is the application_name this carrier's LISTEN session reports +// in pg_stat_activity. It is unique per carrier so two Buses on one database are +// distinguishable, which is what lets an operator see how many replicas are +// listening and lets a spec terminate exactly one of them. +func (b *Bus) ApplicationName() string { return b.appName } + +// QueueDepth is the effective notification queue depth, after Config.Queue has +// been defaulted. Reported so a deployment can log what it actually got rather +// than what it thinks it configured. +func (b *Bus) QueueDepth() int { return cap(b.inbound) } + +// SpillRetention is the effective spill retention, after Config.Retention has +// been defaulted. +func (b *Bus) SpillRetention() time.Duration { return b.retention } + +// Dropped reports how many notifications this carrier discarded because the +// resolver was too far behind. It only ever grows. +// +// Dropped and IsConnected are consumed by SPECS AND LOGGING ONLY, and neither is +// on messaging.Broadcaster. No production branch may read either: "this +// replica's listener is down" and "this replica is behind" are facts about a +// FRONTEND, while the four conditions a scheduler acts on are all facts about a +// WORKER. A reaper that consulted them would be reporting a carrier failure as a +// worker's absence, which is the collapse this programme exists to prevent. +func (b *Bus) Dropped() uint64 { return b.dropped.Load() } + +// OnReconnect registers a callback invoked after the LISTEN connection has been +// re-established AND every channel has been re-listened. +// +// It is consumed through an optional interface assertion +// (interface{ OnReconnect(func()) }) in syncstate.SyncedMap.Start rather than +// through messaging.Broadcaster, so a carrier that cannot reconnect is not +// forced to claim it can. That assertion is also why nothing here fails to +// compile if the invocation is deleted: a SyncedMap whose only convergence +// mechanism is this callback would simply stop converging, in silence, so the +// spec in listener_test.go is the only guard there is. +// +// A nil callback is ignored. +func (b *Bus) OnReconnect(cb func()) { + if cb == nil { + return + } + b.cbMu.Lock() + b.reconnectCbs = append(b.reconnectCbs, cb) + b.cbMu.Unlock() +} + +// runReconnectCallbacks invokes the registered callbacks, each on a goroutine of +// its own. +// +// Detached on purpose. This runs on the goroutine that owns the LISTEN +// connection, and the callbacks are re-hydrations: a SyncedMap reads its whole +// durable source back. Running one of those inline would put a database query on +// the path whose only job is to keep PostgreSQL's async queue draining, which is +// the failure this task exists to remove, and a callback that never returned +// would leave the carrier deaf for good. +// +// The slice is copied under the lock so a callback that registers another cannot +// deadlock. +func (b *Bus) runReconnectCallbacks() { + b.cbMu.Lock() + cbs := append([]func(){}, b.reconnectCbs...) + b.cbMu.Unlock() + for _, cb := range cbs { + // Pinned by "does not run a reconnect callback on the goroutine that + // owns the connection": deleting the `go` compiles and stays green + // everywhere else. + go cb() + } +} + +// dial opens a LISTEN connection carrying this carrier's application name. +// +// The DSN is re-parsed per dial rather than a parsed config being kept, because +// pgx may record connection state on the config it is handed and reusing one +// across redials is not part of its contract. +func (b *Bus) dial(ctx context.Context) (*pgx.Conn, error) { + return dial(ctx, b.cfg.DSN, b.appName) +} + +func dial(ctx context.Context, dsn, appName string) (*pgx.Conn, error) { + connCfg, err := pgx.ParseConfig(dsn) + if err != nil { + return nil, fmt.Errorf("pgbus: parsing the LISTEN connection string: %w", err) + } + if connCfg.RuntimeParams == nil { + connCfg.RuntimeParams = map[string]string{} + } + connCfg.RuntimeParams["application_name"] = appName + return pgx.ConnectConfig(ctx, connCfg) +} + +// command hands a LISTEN or UNLISTEN to the goroutine that owns the connection +// and waits for it. Returning before the server had acknowledged it would lose +// every message published in the gap. +// +// A Subscribe that has returned is therefore always a registration the server +// has acknowledged, including the case where this Subscribe issued nothing +// because the channel was already listened: listenMu means the Subscribe that +// DID issue the LISTEN had already been acknowledged before this one could see +// its registration. +func (b *Bus) command(sql string) error { + cmd := listenCmd{sql: sql, done: make(chan error, 1)} + timeout := time.NewTimer(subscribeTimeout) + defer timeout.Stop() + + select { + case b.cmds <- cmd: + case <-b.ctx.Done(): + return errors.New("pgbus: the carrier is closed") + case <-timeout.C: + return fmt.Errorf("pgbus: %q timed out waiting for the listen connection", sql) + } + select { + case err := <-cmd.done: + return err + case <-b.ctx.Done(): + return errors.New("pgbus: the carrier is closed") + case <-timeout.C: + return fmt.Errorf("pgbus: %q timed out on the listen connection", sql) + } +} + +// listen owns the pinned connection for the life of the Bus: it is the only +// goroutine that touches it, which is what makes a connection that is not +// concurrency safe usable from many callers. +// +// A notification is copied off the connection here and NEVER processed here. The +// loop below must reach offer with nothing in between that can block on a +// database, on a handler or on the network, because whatever it waited for +// PostgreSQL would be holding notifications for this session in a queue it +// shares with every other publisher on the server. +func (b *Bus) listen(conn *pgx.Conn) { + defer close(b.listenerDone) + defer func() { + b.connected.Store(false) + // A background context, because b.ctx is already cancelled by the time + // this runs and the close still has to reach the server. + closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = conn.Close(closeCtx) + }() + + for { + if b.ctx.Err() != nil { + return + } + // Registrations first: a subscriber that has been waiting must not be + // held up behind a poll interval that has just restarted. + select { + case cmd := <-b.cmds: + _, err := conn.Exec(b.ctx, cmd.sql) + cmd.done <- err + continue + default: + } + + waitCtx, cancel := context.WithTimeout(b.ctx, listenPollInterval) + n, err := conn.WaitForNotification(waitCtx) + deadline := waitCtx.Err() != nil + cancel() + + switch { + case b.ctx.Err() != nil: + return + case deadline: + // The ordinary case: nothing arrived inside the poll window. pgx + // implements the deadline as a read deadline and keeps the + // connection, so this is not a failure. + continue + case err != nil: + xlog.Warn("Broadcast carrier lost its listen connection, redialling", "error", err) + replacement, ok := b.redial(conn) + if !ok { + return + } + conn = replacement + continue + } + b.offer(n.Channel, n.Payload) + } +} + +// offer hands a notification to the resolver. It never blocks: the listener's +// only job is to keep PostgreSQL's async queue draining. +// +// The drop is counted as well as logged. The log lands on the replica that took +// the loss, which is not always the party that needs to know, and a counter that +// only ever grows is the one thing a spec and an operator can both read. +func (b *Bus) offer(channel, payload string) { + select { + case b.inbound <- inbound{channel: channel, payload: payload}: + default: + b.dropped.Add(1) + xlog.Error("Broadcast carrier dropped a notification: the resolver is not keeping up", + "channel", channel, "depth", cap(b.inbound), "dropped", b.dropped.Load()) + } +} + +// resolve is where a spilled broadcast is read back and where every notification +// is dispatched. Both are off the listener on purpose, and both are on ONE +// goroutine, so a spilled message and an inline one on the same subject keep the +// order they were published in. +// +// This is the half that is allowed to block. A notification is processed here +// and never on the connection it arrived on, so a slow SELECT here costs this +// replica bounded loss through offer, and costs the SERVER nothing. +func (b *Bus) resolve() { + defer close(b.resolverDone) + for { + select { + case <-b.ctx.Done(): + return + case in := <-b.inbound: + b.deliver(in.channel, in.payload) + } + } +} + +// redial replaces a dead LISTEN connection and restores every registration on +// it. Without the restore the carrier comes back deaf, which looks exactly like +// a deployment where nobody is publishing any more. +func (b *Bus) redial(dead *pgx.Conn) (*pgx.Conn, bool) { + b.connected.Store(false) + closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = dead.Close(closeCtx) + cancel() + + backoff := 100 * time.Millisecond + for { + if b.ctx.Err() != nil { + return nil, false + } + conn, err := b.dial(b.ctx) + if err == nil { + if err = b.relisten(conn); err == nil { + b.connected.Store(true) + xlog.Info("Broadcast carrier reconnected") + // After the re-LISTEN and never before it. A callback + // re-hydrates from the durable source, and one that ran while + // the channels were still unregistered would converge against a + // carrier that cannot yet tell it about the next change. + b.runReconnectCallbacks() + return conn, true + } + closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = conn.Close(closeCtx) + cancel() + } + select { + case <-b.ctx.Done(): + return nil, false + case <-time.After(backoff): + } + if backoff < 5*time.Second { + backoff *= 2 + } + } +} + +// relisten re-registers every channel that still has a subscriber. +// +// A carrier that reconnected without this is CONNECTED AND DEAF, and there is no +// error and no log line anywhere that says so. +func (b *Bus) relisten(conn *pgx.Conn) error { + b.mu.Lock() + channels := make([]string, 0, len(b.subs)) + for channel, subs := range b.subs { + if len(subs) > 0 { + channels = append(channels, channel) + } + } + b.mu.Unlock() + + for _, channel := range channels { + if _, err := conn.Exec(b.ctx, "LISTEN "+pgx.Identifier{channel}.Sanitize()); err != nil { + return err + } + } + return nil +} + +// purge retires spilled rows that every replica has had time to read. +// +// Every replica purges. The DELETE is idempotent, and a deployment where only +// one replica swept would stop retiring anything the moment that replica went +// down. +func (b *Bus) purge() { + interval := b.cfg.SweepInterval + if interval <= 0 { + // Half the retention, so a row is retired within one retention of + // ageing out rather than within two. + interval = b.retention / 2 + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-b.ctx.Done(): + return + case <-ticker.C: + if _, err := b.PurgeBefore(b.ctx, b.retention); err != nil && b.ctx.Err() == nil { + xlog.Warn("Broadcast carrier could not retire spilled messages", "error", err) + } + } + } +} diff --git a/core/services/pgbus/listener_test.go b/core/services/pgbus/listener_test.go new file mode 100644 index 000000000000..027ef5c57f7c --- /dev/null +++ b/core/services/pgbus/listener_test.go @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: MIT + +package pgbus_test + +import ( + "context" + "fmt" + "reflect" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// The two failure modes NATS did not have, and both are silent. +// +// Neither is provoked with a double or a clock here. A slow consumer is a real +// SELECT held by a real table lock, and a dropped listener is a real +// pg_terminate_backend against the session's own application_name, because a +// double that never touches the transport cannot fail the way the transport +// fails. +var _ = Describe("the listener under a slow consumer and a dropped connection", func() { + var ( + db *gorm.DB + dsn string + ) + + newBus := func(cfg pgbus.Config) *pgbus.Bus { + GinkgoHelper() + cfg.DSN, cfg.DB = dsn, db + b, err := pgbus.New(context.Background(), cfg) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(b.Close) + return b + } + + received := func(b *pgbus.Bus, filter string) <-chan []byte { + GinkgoHelper() + out := make(chan []byte, 128) + s, err := b.Subscribe(filter, func(data []byte) { out <- data }) + Expect(err).ToNot(HaveOccurred()) + Expect(s).ToNot(BeNil()) + return out + } + + // terminate drops exactly this carrier's LISTEN session and nobody else's. + // Matching on the application name rather than on "every backend except + // mine" is what lets a spec assert on a peer bus that stayed up. + terminate := func(b *pgbus.Bus) { + GinkgoHelper() + var killed int64 + Expect(db.Raw( + "SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE application_name = ?", + b.ApplicationName(), + ).Scan(&killed).Error).To(Succeed()) + Expect(killed).To(BeNumerically("==", 1), "expected exactly one listener session to drop") + } + + BeforeEach(func() { + db, dsn = testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(context.Background(), db)).To(Succeed()) + }) + + Describe("a resolver that cannot keep up", func() { + It("keeps draining the connection and drops, rather than stalling on the connection", func() { + // The hazard this whole split exists for. PostgreSQL holds + // undelivered notifications in an async queue that is shared by + // every session on the SERVER, so a listener that stopped draining + // would not merely fall behind: it can block COMMIT for unrelated + // publishers, and the backend eventually kills the session, which + // costs this replica every subsequent broadcast rather than these + // few. + // + // The resolver is stalled through the transport and not through a + // seam: a spilled broadcast is resolved with a SELECT on + // bus_messages, and an ACCESS EXCLUSIVE lock held by another + // transaction stops that SELECT for exactly as long as the spec + // wants it stopped. Nothing here waits on a clock. + const queue = 8 + const sent = queue + 64 + + b := newBus(pgbus.Config{Queue: queue, Retention: time.Hour}) + out := received(b, "jobs.spill.probe") + + channel, err := pgbus.ChannelFor("jobs.spill.probe") + Expect(err).ToNot(HaveOccurred()) + + // Distinct rows, so every notification carries a distinct payload: + // PostgreSQL collapses duplicate notifications raised in one + // transaction, and a spec that relied on identical payloads would + // be counting on that not happening. + ids := make([]string, sent) + for i := range ids { + ids[i] = fmt.Sprintf("spill-%02d", i) + Expect(db.Create(&pgbus.BusMessage{ + ID: ids[i], Subject: "jobs.spill.probe", Payload: []byte(fmt.Sprintf(`{"i":%d}`, i)), + }).Error).To(Succeed()) + } + + held := db.Begin() + Expect(held.Error).ToNot(HaveOccurred()) + released := false + release := func() { + if !released { + released = true + Expect(held.Rollback().Error).ToNot(HaveOccurred()) + } + } + DeferCleanup(release) + Expect(held.Exec("LOCK TABLE bus_messages IN ACCESS EXCLUSIVE MODE").Error).To(Succeed()) + + for _, id := range ids { + Expect(db.Exec("SELECT pg_notify(?, ?)", channel, + fmt.Sprintf(`{"s":"jobs.spill.probe","i":%q}`, id)).Error).To(Succeed()) + } + + // The listener took every notification off the connection while the + // resolver could not move: the queue filled and the surplus was + // dropped. A listener that had blocked instead would leave this at + // zero forever. + Eventually(b.Dropped, 30*time.Second).Should(BeNumerically(">", 0)) + Expect(b.IsConnected()).To(BeTrue(), "dropping must not cost the connection") + + release() + + // Still a working carrier once the consumer catches up. Bounded + // loss, not a wedge. + Eventually(out, 30*time.Second).Should(Receive()) + }) + + It("reports nothing dropped on a carrier that is keeping up", func() { + // The zero the spec above is measured against. Without it a Dropped + // that returned a constant would satisfy the assertion there. + b := newBus(pgbus.Config{}) + out := received(b, "jobs.fast.consumer") + + Expect(b.Publish("jobs.fast.consumer", map[string]int{"i": 1})).To(Succeed()) + + Eventually(out).Should(Receive()) + Expect(b.Dropped()).To(BeZero()) + }) + }) + + Describe("losing the LISTEN connection", func() { + It("invokes every OnReconnect callback once the channels are listened again", func() { + // syncstate.SyncedMap syncs deltas only, so this callback is the one + // mechanism it has to converge after a gap. It reaches it through an + // optional interface assertion, which means deleting the invocation + // compiles, passes every other spec, and leaves every adopter + // silently diverging. This spec is the only guard there is. + b := newBus(pgbus.Config{}) + _ = received(b, "jobs.reconnect.probe") + + calls := make(chan struct{}, 8) + b.OnReconnect(func() { calls <- struct{}{} }) + // Not fired by the connection it was registered on: a callback that + // ran on the first connect would re-hydrate every adopter at boot + // for no reason. + Consistently(calls, 200*time.Millisecond).ShouldNot(Receive()) + + terminate(b) + + // The callback cannot fire without a lost connection, a redial and a + // completed re-LISTEN, so its arrival is what proves the whole cycle + // ran. Sampling IsConnected for the false half would be sampling: the + // window between the loss and the first redial is sub-millisecond, + // and a spec that catches it nine times in ten is worse than none. + Eventually(calls, 30*time.Second).Should(Receive()) + Expect(b.IsConnected()).To(BeTrue()) + }) + + It("delivers again on a channel it had to re-listen", func() { + // Separate from the callback assertion on purpose: a carrier can be + // reconnected, and report itself connected, and be deaf, because a + // connection without its LISTEN registrations produces no error + // anywhere. Deleting the re-LISTEN reddens this and leaves the + // callback spec above green. + b := newBus(pgbus.Config{}) + publisher := newBus(pgbus.Config{}) + out := received(b, "jobs.relisten.probe") + + terminate(b) + + Eventually(func() bool { + // Retried because a NOTIFY that lands between the loss and the + // re-LISTEN is genuinely gone: this carrier is at-most-once, and + // what is being asserted is that delivery RESUMES. + _ = publisher.Publish("jobs.relisten.probe", map[string]string{"m": "after"}) + select { + case <-out: + return true + default: + return false + } + }, 30*time.Second, 200*time.Millisecond).Should(BeTrue()) + }) + + It("does not run a reconnect callback on the goroutine that owns the connection", func() { + // A callback re-hydrates from a durable source, which is a database + // query. Running one inline would put that query on the path whose + // only job is to keep PostgreSQL's async queue draining, and a + // callback that never returned would leave this carrier deaf for + // good. Deleting the `go` compiles and is invisible everywhere else. + b := newBus(pgbus.Config{}) + publisher := newBus(pgbus.Config{}) + out := received(b, "jobs.blockedcb.probe") + + entered := make(chan struct{}, 1) + forever := make(chan struct{}) + DeferCleanup(func() { close(forever) }) + b.OnReconnect(func() { + entered <- struct{}{} + <-forever + }) + + terminate(b) + + Eventually(entered, 30*time.Second).Should(Receive()) + Eventually(func() bool { + _ = publisher.Publish("jobs.blockedcb.probe", map[string]string{"m": "after"}) + select { + case <-out: + return true + default: + return false + } + }, 30*time.Second, 200*time.Millisecond).Should(BeTrue()) + }) + }) + + Describe("naming the LISTEN session", func() { + It("gives each carrier an application_name an operator can find in pg_stat_activity", func() { + // Without it, dropping one replica's listener means dropping every + // backend on the database, and an operator asking how many replicas + // are listening has nothing to count. + b := newBus(pgbus.Config{}) + + var sessions int64 + Expect(db.Raw( + "SELECT count(*) FROM pg_stat_activity WHERE application_name = ?", b.ApplicationName(), + ).Scan(&sessions).Error).To(Succeed()) + + Expect(b.ApplicationName()).To(HavePrefix("localai_pgbus_")) + Expect(sessions).To(BeNumerically("==", 1)) + }) + + It("names two carriers on one database differently", func() { + first := newBus(pgbus.Config{}) + second := newBus(pgbus.Config{}) + + Expect(first.ApplicationName()).ToNot(Equal(second.ApplicationName())) + }) + }) + + Describe("the queue and retention settings", func() { + It("takes the exported defaults when both are zero", func() { + b := newBus(pgbus.Config{}) + + Expect(b.QueueDepth()).To(Equal(pgbus.DefaultQueueDepth)) + Expect(b.SpillRetention()).To(Equal(pgbus.DefaultSpillRetention)) + }) + + It("takes what it was configured with when they are not", func() { + b := newBus(pgbus.Config{Queue: 7, Retention: 3 * time.Minute}) + + Expect(b.QueueDepth()).To(Equal(7)) + Expect(b.SpillRetention()).To(Equal(3 * time.Minute)) + }) + + It("runs the purge loop off the retention when no interval is configured", func() { + // The interval is derived rather than configured in production, so + // nothing else proves the derivation is wired: a purge loop pinned + // to a constant would retire rows too, just not on the schedule the + // retention asks for. + Expect(db.Create(&pgbus.BusMessage{ + ID: "aged", Subject: "jobs.x", Payload: []byte(`{}`), + }).Error).To(Succeed()) + Expect(db.Exec( + "UPDATE bus_messages SET created_at = now() - interval '1 hour' WHERE id = 'aged'", + ).Error).To(Succeed()) + + newBus(pgbus.Config{Retention: 40 * time.Millisecond}) + + Eventually(func() (int64, error) { + var n int64 + err := db.Model(&pgbus.BusMessage{}).Count(&n).Error + return n, err + }, 30*time.Second).Should(BeZero()) + }) + }) + + Describe("what the carrier's own health is allowed to reach", func() { + It("keeps Dropped and IsConnected off the interface consumers hold", func() { + // The invariant, guarded structurally. "This replica is behind" and + // "this replica's listener is down" are facts about a FRONTEND; the + // conditions a scheduler acts on are all facts about a WORKER. As + // long as neither method is reachable through messaging.Broadcaster, + // no consumer can read a carrier failure as a worker's absence + // without reaching for the concrete type and saying so. + carrier := reflect.TypeOf((*messaging.Broadcaster)(nil)).Elem() + + _, hasDropped := carrier.MethodByName("Dropped") + _, hasConnected := carrier.MethodByName("IsConnected") + + Expect(hasDropped).To(BeFalse(), "a subscriber's loss must not be readable as a node's state") + Expect(hasConnected).To(BeFalse(), "a carrier's link must not be readable as a node's state") + }) + }) +}) diff --git a/core/services/pgbus/spill.go b/core/services/pgbus/spill.go index b393c2419a5e..c641203209c9 100644 --- a/core/services/pgbus/spill.go +++ b/core/services/pgbus/spill.go @@ -4,22 +4,12 @@ package pgbus import ( "context" + "fmt" "time" "gorm.io/gorm" ) -// spillRetention is how long a spilled broadcast stays readable after it was -// published. It is generous against the delivery it has to survive, which is -// one notification and one primary-key SELECT, and short enough that a busy -// deployment does not accumulate LLM outputs in this table. -const spillRetention = 5 * time.Minute - -// spillSweepInterval is how often each replica retires what has aged out. Every -// replica sweeps; the DELETE is idempotent and a replica that is down must not -// leave the table growing. -const spillSweepInterval = time.Minute - // SpillSweepSQL retires spilled broadcasts older than the retention it is given. // // The cutoff is computed by the DATABASE and never in Go, and that is the whole @@ -48,6 +38,26 @@ func Migrate(ctx context.Context, db *gorm.DB) error { } // SweepSpill deletes spilled broadcasts older than retention. +// +// For a caller that has a handle but no carrier, which is why it takes a +// *gorm.DB. The carrier's own purge loop calls PurgeBefore instead, because it +// needs the count; both run the one statement in SpillSweepSQL. func SweepSpill(ctx context.Context, db *gorm.DB, retention time.Duration) error { return db.WithContext(ctx).Exec(SpillSweepSQL, retention.Seconds()).Error } + +// PurgeBefore deletes this carrier's spilled rows older than olderThan and +// returns how many it deleted. +// +// The count is the point. SweepSpill answers "did the statement run", which a +// purge loop that is retiring nothing answers just as happily, and a spill table +// that grows without bound is a disk that fills long after the change that +// caused it. This is also what the purge loop calls, so the count an operator +// can ask for and the deletion the carrier performs are the same statement. +func (b *Bus) PurgeBefore(ctx context.Context, olderThan time.Duration) (int64, error) { + res := b.cfg.DB.WithContext(ctx).Exec(SpillSweepSQL, olderThan.Seconds()) + if res.Error != nil { + return 0, fmt.Errorf("pgbus: retiring spilled broadcasts: %w", res.Error) + } + return res.RowsAffected, nil +} diff --git a/core/services/pgbus/spill_test.go b/core/services/pgbus/spill_test.go index 0545908e2b55..801b5f6e5f3d 100644 --- a/core/services/pgbus/spill_test.go +++ b/core/services/pgbus/spill_test.go @@ -238,6 +238,43 @@ var _ = Describe("retiring spilled rows", func() { Expect(rows()).To(ConsistOf("fresh")) }) + + Describe("PurgeBefore", func() { + var b *pgbus.Bus + + BeforeEach(func() { + // A retention long enough that the carrier's own purge loop cannot + // fire inside a spec, so what these assert on is the call they make + // and not a tick that happened to land. + var err error + b, err = pgbus.New(context.Background(), pgbus.Config{DSN: dsn, DB: db, Retention: time.Hour}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(b.Close) + }) + + It("deletes a spilled row and says how many it deleted", func() { + // The count is what SweepSpill cannot answer: a purge loop retiring + // nothing and one retiring everything both look like a statement + // that ran. + aged("gone") + + deleted, err := b.PurgeBefore(context.Background(), 0) + + Expect(err).ToNot(HaveOccurred()) + Expect(deleted).To(BeNumerically("==", 1)) + Expect(rows()).To(BeEmpty()) + }) + + It("leaves a row younger than the cutoff alone", func() { + Expect(db.Create(&pgbus.BusMessage{ID: "young", Subject: "jobs.x", Payload: []byte(`{}`)}).Error).To(Succeed()) + + deleted, err := b.PurgeBefore(context.Background(), time.Hour) + + Expect(err).ToNot(HaveOccurred()) + Expect(deleted).To(BeZero()) + Expect(rows()).To(ConsistOf("young")) + }) + }) }) func mustJSON(v any) []byte { diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 8b759891374b..5cd5b83763e6 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -39,6 +39,8 @@ Each model gets its own gRPC backend process, so a single worker can serve multi - **PostgreSQL** (with pgvector extension recommended for RAG) - used for node registry, job store, auth, and shared state - Each frontend replica holds **one extra PostgreSQL session** beyond its connection pool, pinned for the life of the process, and creates a `bus_messages` table. Both belong to the broadcast carrier that is replacing NATS for cross-replica fan-out. Size `max_connections` for one additional session per frontend replica. + - That session reports an `application_name` of `localai_pgbus_`, so `SELECT count(*) FROM pg_stat_activity WHERE application_name LIKE 'localai_pgbus_%'` counts the replicas currently listening. If the carrier loses its session it redials and re-registers on its own; a broadcast published while it was down is not replayed, which is why nothing that must survive a gap is carried by a broadcast alone. + - `bus_messages` holds only broadcasts too large for a PostgreSQL notification, and every replica retires rows older than ten minutes. The table is a spill buffer, not a log: it is not a place to read past events from. - **NATS** server - used for agent-worker coordination and the frontend's own cross-replica events. **Serve-backend workers do not connect to it at all**: every verb they take, and file staging with it, is an HTTP route on the worker's tunnel. Set no `LOCALAI_NATS_URL` on a `local-ai worker`. The frontend and any `local-ai agent-worker` still need one. - All services must be on the same network (or reachable via configured URLs) From 7629804b1b8f0211bad706ebbc1cfa011eddfe2c Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 11:22:38 +0000 Subject: [PATCH 084/153] feat(distributed): give agent workers a tunnel of their own Phase 2 gated agent nodes out of tunnel credentials at the mint site. That was right while nothing dialled into an agent worker: a credential would have replaced nothing, and the gate was structural rather than a second check that could drift. It is wrong now that the frontend needs to reach an agent worker by RPC. attachTunnelToken mints for backend and agent nodes and CLEARS for anything else, through one tunnelEligible predicate rather than two conditions that can be widened separately. ConnectHandler still never reads NodeType, so an empty hash is still what refuses an ineligible node. An agent worker now starts a loopback control server behind the same bearer check a backend worker uses, and holds one tunnel whose only stream tag is http: it runs no backend processes, so the grpc tag has nothing to route to and is not offered. Its MCP tool, MCP discovery and backend.stop verbs are served from ONE implementation reached by both the bus and the tunnel, so a frontend cannot get different bytes depending on which carrier delivered. The tunnel is an ADDITION. --nats-url is still required, and agent jobs, MCP execution, MCP CI jobs and nodes..backend.stop all still travel on the bus. Absence semantics are unchanged. An agent node now has a real node_connections row whose departure ages past the grace, so the node type check in HealthMonitor.tunnelDeparted stopped being an optimisation and became the rule; its comment says so, and the spec that pins it is shown red under a mutation that deletes the check. The scheduler needed no change: every placement query already filters node_type = backend, so an agent node never reaches nodeMayTakeWork. Shared rules moved to one site each. The request bounds, the POST-only check and the unknown-path 404 live in workerctl and are called by both worker packages; the bearer check that guards every extra route is one function in core/services/nodes used by both server constructors. workerctl.AllPaths splits into BackendPaths and AgentPaths, with AllPaths as their deduped union, because a backend worker does not mount the agent verbs and asserting otherwise would fail a correct worker. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/cli/agent_worker.go | 221 +++++-- core/cli/agent_worker_test.go | 100 +++ core/http/endpoints/localai/nodes.go | 63 +- core/http/endpoints/localai/nodes_test.go | 98 ++- .../agentworker/agentworker_suite_test.go | 13 + core/services/agentworker/handlers.go | 111 ++++ core/services/agentworker/server.go | 337 +++++++++++ core/services/agentworker/server_test.go | 572 ++++++++++++++++++ core/services/messaging/subjects.go | 16 +- core/services/nodes/file_transfer_server.go | 144 +++-- core/services/nodes/health.go | 12 +- core/services/nodes/health_test.go | 7 +- core/services/nodes/interfaces.go | 9 +- .../nodes/managers_agent_node_test.go | 12 +- core/services/nodes/managers_distributed.go | 19 +- core/services/nodes/unloader.go | 20 +- core/services/nodes/unloader_test.go | 2 +- core/services/worker/control_routes.go | 70 +-- core/services/worker/control_routes_test.go | 7 +- core/services/worker/tunnel.go | 22 + core/services/worker/tunnel_test.go | 78 +++ core/services/workerctl/paths.go | 62 +- core/services/workerctl/paths_test.go | 44 +- core/services/workerctl/request.go | 85 +++ docs/content/features/distributed-mode.md | 16 +- tests/e2e/distributed/cluster/cluster.go | 23 +- tests/e2e/distributed/cluster_control_test.go | 16 +- tests/e2e/distributed/cluster_tunnel_test.go | 32 + tests/e2e/distributed/node_lifecycle_test.go | 4 +- 29 files changed, 1958 insertions(+), 257 deletions(-) create mode 100644 core/cli/agent_worker_test.go create mode 100644 core/services/agentworker/agentworker_suite_test.go create mode 100644 core/services/agentworker/handlers.go create mode 100644 core/services/agentworker/server.go create mode 100644 core/services/agentworker/server_test.go create mode 100644 core/services/workerctl/request.go diff --git a/core/cli/agent_worker.go b/core/cli/agent_worker.go index ced179e50d4e..fa49f606c7d6 100644 --- a/core/cli/agent_worker.go +++ b/core/cli/agent_worker.go @@ -16,6 +16,7 @@ import ( "github.com/mudler/LocalAI/core/config" mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp" "github.com/mudler/LocalAI/core/services/agents" + "github.com/mudler/LocalAI/core/services/agentworker" "github.com/mudler/LocalAI/core/services/jobs" mcpRemote "github.com/mudler/LocalAI/core/services/mcp" "github.com/mudler/LocalAI/core/services/messaging" @@ -31,6 +32,11 @@ import ( // receives the full agent config and skills in the NATS job payload, so it // does not need direct database access. // +// It also holds one tunnel to the frontend, so the frontend can reach its MCP +// verbs by RPC without the worker opening an inbound port. The tunnel is an +// ADDITION: --nats-url is still required, and every job, every fan-out event +// and the node backend.stop subject still travel on the bus. +// // Usage: // // localai agent-worker --nats-url nats://... --register-to http://localai:8080 @@ -170,6 +176,48 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { } defer natsClient.Close() + // The tunnel, and the loopback control plane behind it. + // + // ADDED to this worker rather than swapping anything out: every verb below + // still arrives on NATS, and will until the tasks that move them land. What + // this buys today is that the frontend can reach an agent worker by RPC at + // all, on the same carrier and with the same failure vocabulary a backend + // worker already uses, without the agent worker opening an inbound port. + // + // The credential is read through credMgr rather than captured from res, + // because every re-registration the manager performs ROTATES it and a + // captured value would lock this worker out of its own tunnel at the first + // JWT refresh. + // + // It is started AFTER registration, which is what supplies both the node + // identity the dial names and the credential it presents, and BEFORE the + // NATS subscriptions, so that a frontend that reaches this worker over the + // tunnel finds its verbs mounted rather than a 404 it would read as a + // version skew. + agentCtl, err := agentworker.Start(shutdownCtx, agentworker.Options{ + FrontendURL: cmd.RegisterTo, + NodeID: nodeID, + TunnelToken: credMgr.TunnelToken, + ControlToken: cmd.RegistrationToken, + Handlers: agentworker.Config{ + MCPTool: serveMCPToolRequest, + MCPDiscovery: serveMCPDiscoveryRequest, + // The same cleanup the nodes..backend.stop subscription below + // performs, reachable over the tunnel on the path a backend worker + // already serves. Both are live: the subject is what the frontend + // still publishes on, and it is a later task that removes it. + BackendStop: dropMCPSessionsForBackend, + }, + }) + if err != nil { + return fmt.Errorf("starting the agent worker control plane: %w", err) + } + defer func() { + if err := agentCtl.Close(); err != nil { + xlog.Warn("Closing the agent worker tunnel failed", "error", err) + } + }() + // Create event bridge for publishing results back via NATS eventBridge := agents.NewEventBridge(natsClient, nil, "agent-worker-"+nodeID) @@ -198,16 +246,14 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { // Subscribe to MCP tool execution requests (load-balanced across workers). // The frontend routes model-level MCP tool calls here via NATS request-reply. - if _, err := natsClient.QueueSubscribeReply(messaging.SubjectMCPToolExecute, messaging.QueueAgentWorkers, func(data []byte, reply func([]byte)) { - handleMCPToolRequest(data, reply) - }); err != nil { + if _, err := natsClient.QueueSubscribeReply(messaging.SubjectMCPToolExecute, messaging.QueueAgentWorkers, + replyOverNATS(messaging.SubjectMCPToolExecute, serveMCPToolRequest)); err != nil { return fmt.Errorf("subscribing to %s: %w", messaging.SubjectMCPToolExecute, err) } // Subscribe to MCP discovery requests (load-balanced across workers). - if _, err := natsClient.QueueSubscribeReply(messaging.SubjectMCPDiscovery, messaging.QueueAgentWorkers, func(data []byte, reply func([]byte)) { - handleMCPDiscoveryRequest(data, reply) - }); err != nil { + if _, err := natsClient.QueueSubscribeReply(messaging.SubjectMCPDiscovery, messaging.QueueAgentWorkers, + replyOverNATS(messaging.SubjectMCPDiscovery, serveMCPDiscoveryRequest)); err != nil { return fmt.Errorf("subscribing to %s: %w", messaging.SubjectMCPDiscovery, err) } @@ -229,12 +275,19 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { // Subscribe to backend stop events to clean up cached MCP sessions. // In the main application this is done via ml.OnModelUnload, but the agent // worker has no model loader — we listen for the NATS stop event instead. + // + // It runs BESIDE the tunnel route mounted above, not instead of it, and + // both call dropMCPSessionsForBackend. The subject is still what the + // frontend publishes on; a later task is what moves it. Two carriers, one + // implementation, so which one delivered cannot change what happened. if _, err := natsClient.Subscribe(messaging.SubjectNodeBackendStop(nodeID), func(data []byte) { - var req struct { - Backend string `json:"backend"` + var req messaging.BackendStopRequest + if err := json.Unmarshal(data, &req); err != nil { + xlog.Warn("Agent worker could not decode a backend stop event", "error", err) + return } - if json.Unmarshal(data, &req) == nil && req.Backend != "" { - mcpTools.CloseMCPSessions(req.Backend) + if err := dropMCPSessionsForBackend(context.Background(), req); err != nil { + xlog.Warn("Agent worker could not drop the MCP sessions of a stopped backend", "error", err) } }); err != nil { return fmt.Errorf("subscribing to %s: %w", messaging.SubjectNodeBackendStop(nodeID), err) @@ -263,72 +316,106 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { return runErr } -// handleMCPToolRequest handles a NATS request-reply for MCP tool execution. -// The worker creates/caches MCP sessions from the serialized config and executes the tool. -func handleMCPToolRequest(data []byte, reply func([]byte)) { +// The MCP verbs, written ONCE and served on two carriers. +// +// The bus subscription and the tunnel's control route both call the same +// serve* function and both send the same bytes, so a worker reached either way +// answers identically. Two implementations of one verb is the shape that lets a +// deployment behave differently depending on which carrier a frontend happened +// to pick, and there is no version of this migration in which that is +// acceptable: for the whole of it, both carriers are live at once. +// +// The distinction the return type carries: an MCP tool that RAN and failed is +// this worker's own answer and travels as bytes with an error field set, on a +// 200. A returned error is this worker failing to serve the verb at all, which +// becomes a non-2xx over the tunnel and nothing the frontend may act on. + +// dropMCPSessionsForBackend closes the MCP sessions this worker cached for a +// backend that is going away. +// +// It is the agent worker's whole implementation of backend.stop, and it is +// deliberately nothing like the backend worker's, which kills the process and +// recycles its port. An agent worker runs no backend processes; what it holds +// are sessions that were created against one. +// +// A backend nobody named is a no-op rather than an error. The event carries the +// name, and a request without one asks this worker to forget nothing in +// particular; failing it would put a malformed publish into the bucket the +// frontend reads as a worker that could not be reached. +func dropMCPSessionsForBackend(_ context.Context, req messaging.BackendStopRequest) error { + if req.Backend == "" { + return nil + } + mcpTools.CloseMCPSessions(req.Backend) + return nil +} + +// serveMCPToolRequest answers an MCP tool execution request. +func serveMCPToolRequest(ctx context.Context, raw json.RawMessage) (json.RawMessage, error) { + return encodeMCPReply(runMCPTool(ctx, raw)) +} + +// runMCPTool creates or reuses the named MCP sessions from the request's config +// and executes the named tool against them. +// +// Every failure inside it is an answer rather than an error, because every one +// of them is something this worker LEARNED by trying: a config it could not +// build sessions from, a discovery that failed, a tool that returned an error. +func runMCPTool(ctx context.Context, raw json.RawMessage) mcpRemote.MCPToolResponse { var req mcpRemote.MCPToolRequest - if err := json.Unmarshal(data, &req); err != nil { - sendMCPToolReply(reply, "", fmt.Sprintf("unmarshal error: %v", err)) - return + if err := json.Unmarshal(raw, &req); err != nil { + return mcpRemote.MCPToolResponse{Error: fmt.Sprintf("unmarshal error: %v", err)} } - ctx, cancel := context.WithTimeout(context.Background(), config.DefaultMCPToolTimeout) + // Bounded here rather than by the caller, so the bus path and the tunnel + // path give a stuck MCP server the same budget. + ctx, cancel := context.WithTimeout(ctx, config.DefaultMCPToolTimeout) defer cancel() - // Create/cache named MCP sessions from the provided config namedSessions, err := mcpTools.NamedSessionsFromMCPConfig(req.ModelName, req.RemoteServers, req.StdioServers, nil) if err != nil { - sendMCPToolReply(reply, "", fmt.Sprintf("session error: %v", err)) - return + return mcpRemote.MCPToolResponse{Error: fmt.Sprintf("session error: %v", err)} } // Discover tools to find the right session tools, err := mcpTools.DiscoverMCPTools(ctx, namedSessions) if err != nil { - sendMCPToolReply(reply, "", fmt.Sprintf("discovery error: %v", err)) - return + return mcpRemote.MCPToolResponse{Error: fmt.Sprintf("discovery error: %v", err)} } - // Execute the tool argsJSON, _ := json.Marshal(req.Arguments) result, err := mcpTools.ExecuteMCPToolCall(ctx, tools, req.ToolName, string(argsJSON)) if err != nil { - sendMCPToolReply(reply, "", err.Error()) - return + return mcpRemote.MCPToolResponse{Error: err.Error()} } - - sendMCPToolReply(reply, result, "") + return mcpRemote.MCPToolResponse{Result: result} } -func sendMCPToolReply(reply func([]byte), result, errMsg string) { - resp := mcpRemote.MCPToolResponse{Result: result, Error: errMsg} - data, _ := json.Marshal(resp) - reply(data) +// serveMCPDiscoveryRequest answers an MCP tool/prompt/resource discovery +// request. +func serveMCPDiscoveryRequest(ctx context.Context, raw json.RawMessage) (json.RawMessage, error) { + return encodeMCPReply(runMCPDiscovery(ctx, raw)) } -// handleMCPDiscoveryRequest handles a NATS request-reply for MCP tool/prompt/resource discovery. -func handleMCPDiscoveryRequest(data []byte, reply func([]byte)) { +// runMCPDiscovery lists the servers this worker can reach for a model, with +// their tools, prompts and resources. +func runMCPDiscovery(ctx context.Context, raw json.RawMessage) mcpRemote.MCPDiscoveryResponse { var req mcpRemote.MCPDiscoveryRequest - if err := json.Unmarshal(data, &req); err != nil { - sendMCPDiscoveryReply(reply, nil, nil, fmt.Sprintf("unmarshal error: %v", err)) - return + if err := json.Unmarshal(raw, &req); err != nil { + return mcpRemote.MCPDiscoveryResponse{Error: fmt.Sprintf("unmarshal error: %v", err)} } - ctx, cancel := context.WithTimeout(context.Background(), config.DefaultMCPDiscoveryTimeout) + ctx, cancel := context.WithTimeout(ctx, config.DefaultMCPDiscoveryTimeout) defer cancel() - // Create/cache named MCP sessions namedSessions, err := mcpTools.NamedSessionsFromMCPConfig(req.ModelName, req.RemoteServers, req.StdioServers, nil) if err != nil { - sendMCPDiscoveryReply(reply, nil, nil, fmt.Sprintf("session error: %v", err)) - return + return mcpRemote.MCPDiscoveryResponse{Error: fmt.Sprintf("session error: %v", err)} } - // List servers with their tools/prompts/resources serverInfos, err := mcpTools.ListMCPServers(ctx, namedSessions) if err != nil { - sendMCPDiscoveryReply(reply, nil, nil, fmt.Sprintf("list error: %v", err)) - return + return mcpRemote.MCPDiscoveryResponse{Error: fmt.Sprintf("list error: %v", err)} } // Also get tool function schemas for the frontend @@ -342,26 +429,48 @@ func handleMCPDiscoveryRequest(data []byte, reply func([]byte)) { }) } - // Convert server infos var servers []mcpRemote.MCPServerInfo - for _, s := range serverInfos { + for _, srv := range serverInfos { servers = append(servers, mcpRemote.MCPServerInfo{ - Name: s.Name, - Type: s.Type, - Tools: s.Tools, - Prompts: s.Prompts, - Resources: s.Resources, - Error: s.Error, + Name: srv.Name, + Type: srv.Type, + Tools: srv.Tools, + Prompts: srv.Prompts, + Resources: srv.Resources, + Error: srv.Error, }) } + return mcpRemote.MCPDiscoveryResponse{Servers: servers, Tools: toolDefs} +} - sendMCPDiscoveryReply(reply, servers, toolDefs, "") +// encodeMCPReply turns a verb's answer into the bytes both carriers send. +// +// A marshalling failure is the one thing here that is NOT an answer: this +// worker has said nothing about the request, so it is returned as an error and +// becomes a non-2xx over the tunnel rather than an empty 200. +func encodeMCPReply(resp any) (json.RawMessage, error) { + out, err := json.Marshal(resp) + if err != nil { + return nil, fmt.Errorf("encoding the reply: %w", err) + } + return out, nil } -func sendMCPDiscoveryReply(reply func([]byte), servers []mcpRemote.MCPServerInfo, tools []mcpRemote.MCPToolDef, errMsg string) { - resp := mcpRemote.MCPDiscoveryResponse{Servers: servers, Tools: tools, Error: errMsg} - data, _ := json.Marshal(resp) - reply(data) +// replyOverNATS adapts one of the serve* functions to a NATS request-reply +// subscription, so the bus carries exactly the bytes the tunnel does. +func replyOverNATS(subject string, serve func(context.Context, json.RawMessage) (json.RawMessage, error)) func([]byte, func([]byte)) { + return func(data []byte, reply func([]byte)) { + out, err := serve(context.Background(), data) + if err != nil { + // Nothing is sent. A requester on the bus reads that as a timeout, + // which is the closest the carrier has to "this worker did not + // answer"; inventing a reply body here would put a failure to serve + // into the bucket reserved for the worker's own verdict. + xlog.Error("Agent worker could not serve a bus request", "subject", subject, "error", err) + return + } + reply(out) + } } // handleMCPCIJob processes an MCP CI job on the agent worker. diff --git a/core/cli/agent_worker_test.go b/core/cli/agent_worker_test.go new file mode 100644 index 000000000000..7fad5a0b4976 --- /dev/null +++ b/core/cli/agent_worker_test.go @@ -0,0 +1,100 @@ +package cli + +import ( + "context" + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + mcpRemote "github.com/mudler/LocalAI/core/services/mcp" + "github.com/mudler/LocalAI/core/services/messaging" +) + +// The agent worker answers its MCP verbs on two carriers at once: the NATS +// subject it has always answered, and the control route on the tunnel it now +// holds. These specs pin the two properties that stops those drifting apart. +// +// The first is that there is ONE implementation. A frontend that reaches a +// worker over the bus and one that reaches the same worker over its tunnel must +// get the same bytes, because during this migration both are live and which one +// is used is not a decision anybody makes deliberately. +// +// The second is the split between an ANSWER and a FAILURE TO SERVE. A tool that +// ran and failed is the worker's own verdict and travels inside the reply; a +// verb this worker could not serve at all is a Go error, which becomes a +// non-2xx over the tunnel and silence on the bus, and which nothing may read as +// evidence about anything. +var _ = Describe("The agent worker's MCP verbs", func() { + It("answers a tool request it could not decode, rather than failing to serve it", func() { + // The decode happened on this worker and its outcome is something the + // worker LEARNED, so it belongs in the reply. Returned as an error it + // would become "no route to that worker" at the frontend. + raw, err := serveMCPToolRequest(context.Background(), json.RawMessage(`{"tool_name":`)) + Expect(err).ToNot(HaveOccurred()) + + var resp mcpRemote.MCPToolResponse + Expect(json.Unmarshal(raw, &resp)).To(Succeed()) + Expect(resp.Error).To(ContainSubstring("unmarshal error")) + Expect(resp.Result).To(BeEmpty()) + }) + + It("answers a discovery request it could not decode the same way", func() { + raw, err := serveMCPDiscoveryRequest(context.Background(), json.RawMessage(`not json`)) + Expect(err).ToNot(HaveOccurred()) + + var resp mcpRemote.MCPDiscoveryResponse + Expect(json.Unmarshal(raw, &resp)).To(Succeed()) + Expect(resp.Error).To(ContainSubstring("unmarshal error")) + }) + + It("puts on the bus exactly the bytes the tunnel route returns", func() { + // The one property that keeps the two carriers honest. A second + // implementation for the bus is how a deployment ends up behaving + // differently depending on which one a frontend happened to pick. + request := json.RawMessage(`{"tool_name":`) + overTunnel, err := serveMCPToolRequest(context.Background(), request) + Expect(err).ToNot(HaveOccurred()) + + sent := make(chan []byte, 1) + replyOverNATS("mcp.tools.execute", serveMCPToolRequest)(request, func(b []byte) { sent <- b }) + + var overBus []byte + Eventually(sent).Should(Receive(&overBus)) + Expect(string(overBus)).To(Equal(string(overTunnel))) + }) + + It("sends nothing on the bus when the verb could not be served", func() { + // A requester reads the silence as a timeout, which is the closest the + // bus has to "this worker did not answer". Inventing a reply body would + // put a failure to serve into the bucket reserved for the worker's own + // verdict, which is the collapse this whole phase exists to prevent. + sent := make(chan []byte, 1) + failing := func(context.Context, json.RawMessage) (json.RawMessage, error) { + return nil, context.DeadlineExceeded + } + + replyOverNATS("mcp.tools.execute", failing)(json.RawMessage(`{}`), func(b []byte) { sent <- b }) + + Expect(sent).ToNot(Receive()) + }) +}) + +var _ = Describe("The agent worker's backend stop", func() { + // One implementation behind both carriers, for the same reason: the bus + // subscription and the tunnel's control route both call this. + + It("treats a stop that names no backend as a no-op rather than a failure", func() { + // A malformed publish must not become a non-2xx the frontend reads as + // a worker it could not reach. + Expect(dropMCPSessionsForBackend(context.Background(), messaging.BackendStopRequest{})).To(Succeed()) + }) + + It("succeeds for a backend it holds no sessions for", func() { + // The ordinary case on a worker that never touched that backend. An + // error here would be reported as this worker failing to serve the + // verb, on every stop of every backend it does not know about. + Expect(dropMCPSessionsForBackend(context.Background(), + messaging.BackendStopRequest{Backend: "a-backend-this-worker-never-saw"})).To(Succeed()) + }) +}) diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index 6d2bd378af01..07e8aca21ca2 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -308,30 +308,39 @@ func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecr // path in core/services/worker/worker.go does), because approval alone does not // prompt a re-registration and nothing else can hand it the secret. // -// Only BACKEND nodes get one, and that is a decision rather than an oversight. -// An agent worker serves no gRPC backends and no file staging; nothing dials -// into it at all, so a tunnel replaces nothing for it and there is no client on -// the agent side that would ever open one. Minting anyway would hand out a -// working credential for a pipe nobody drives, which is surface without a -// feature, and it would contradict every comment in this change that says -// "backend workers, the ones that tunnel". +// BACKEND and AGENT nodes get one; anything else has its credential CLEARED. +// +// Both kinds of worker now dial out and hold one tunnel, and neither opens an +// inbound port. A backend worker carries its gRPC backends and its file staging +// on it; an agent worker carries only its own HTTP server, which is where its +// MCP control verbs live. The gate was once backend-only, and correctly so: +// nothing dialled into an agent worker, so a credential for it would have been +// surface with no feature behind it. That stopped being true when the frontend +// gained a reason to reach an agent worker by RPC. // // The gate lives HERE and not in ConnectHandler, which never looks at NodeType. // It does not need to, PROVIDED an ineligible node ends up with no credential // rather than merely being handed no new one, because the handler's empty-hash // branch is what does the refusing. So this CLEARS the column instead of // returning early, and the difference is not theoretical: Register upserts by -// NAME, so a backend node re-registering as an agent keeps its ID, and +// NAME, so a node re-registering under a different type keeps its ID, and // Register's struct Updates zero-skips TunnelTokenHash while writing the new -// node_type. An early return left a live credential on a row that had become an -// agent. Clearing is what makes "enforcement is structural" true. +// node_type. An early return left a live credential on a row whose type had +// changed. Clearing is what makes "enforcement is structural" true, and it is +// what the eligibility list widening rather than disappearing preserves: the +// invariant is that a node's stored hash always matches the credential its +// CURRENT type is entitled to, which for an ineligible type is none. // // It clears unconditionally rather than only when something is there, so the // invariant holds without depending on what the row happened to contain. The -// cost is one UPDATE per agent registration. +// cost is one UPDATE per registration of an ineligible node. // -// The day agent workers want a tunnel, relaxing the eligibility condition is -// the whole change, and it has to be a deliberate one. +// No node type reaches the clearing branch through RegisterNodeEndpoint today, +// which rejects any node_type that is neither backend nor agent. That is a +// reason to keep the branch rather than to drop it: a row's node_type is also +// written by older builds and will be written by future types, and the branch +// is what makes adding one a decision about eligibility rather than a silent +// grant. // // A failure to mint or to store is logged and the response goes out without the // token. Registration is what gets a worker into the cluster at all, and @@ -343,12 +352,12 @@ func attachTunnelToken(ctx context.Context, response map[string]any, registry *n if node == nil { return } - if node.NodeType != nodes.NodeTypeBackend { + if !tunnelEligible(node.NodeType) { // Cleared, not skipped. SetTunnelTokenHash writes the single column // directly rather than through a struct update, so unlike Register it // can write an empty value; see its doc. if err := registry.SetTunnelTokenHash(ctx, node.ID, ""); err != nil { - xlog.Error("Failed to clear the tunnel credential of a node that is not a backend worker", + xlog.Error("Failed to clear the tunnel credential of a node whose type holds none", "node", node.Name, "type", node.NodeType, "error", err) } return @@ -364,6 +373,18 @@ func attachTunnelToken(ctx context.Context, response map[string]any, registry *n response["tunnel_token"] = plaintext } +// tunnelEligible reports whether a node of this type holds a tunnel credential. +// +// One predicate rather than a condition written into attachTunnelToken, because +// the mint branch and the clear branch are the two halves of ONE rule and must +// not be able to disagree: written as two conditions, widening the mint without +// widening the clear leaves a node type that is granted a credential and never +// stripped of one, and widening the clear without the mint strips a node type +// on every registration it makes. Neither has a symptom until a tunnel dial. +func tunnelEligible(nodeType string) bool { + return nodeType == nodes.NodeTypeBackend || nodeType == nodes.NodeTypeAgent +} + // attachNatsJWT adds a per-node NATS user JWT to a register/approve response when minting is enabled. func attachNatsJWT(response map[string]any, node *nodes.BackendNode, natsCfg natsauth.Config) { if !natsCfg.CanMintWorkers() || node == nil || node.Status == nodes.StatusPending { @@ -723,12 +744,12 @@ func DeleteBackendOnNodeEndpoint(unloader nodes.NodeCommandSender) echo.HandlerF func ListBackendsOnNodeEndpoint(unloader nodes.NodeCommandSender, registry *nodes.NodeRegistry) echo.HandlerFunc { return func(c echo.Context) error { nodeID := c.Param("id") - // Agent-type workers don't run backends and never subscribe to the - // nodes..backend.list NATS subject, so the request would hang - // until timeout with "no responders". Their backend list is simply - // empty. Mirror the aggregate-list guard in managers_distributed.go - // (skip nodes whose NodeType is set and not "backend") so the - // single-node and cluster-wide views stay consistent. + // Agent-type workers don't run backends and mount no backend.list + // route on the tunnel they hold, so asking one can only 404. Their + // backend list is simply empty. Mirror the aggregate-list guard in + // managers_distributed.go (skip nodes whose NodeType is set and not + // "backend") so the single-node and cluster-wide views stay + // consistent. if node, err := registry.Get(c.Request().Context(), nodeID); err == nil { if node.NodeType != "" && node.NodeType != nodes.NodeTypeBackend { return c.JSON(http.StatusOK, []messaging.NodeBackendInfo{}) diff --git a/core/http/endpoints/localai/nodes_test.go b/core/http/endpoints/localai/nodes_test.go index 63f49e7c3c7e..26a3b26ed849 100644 --- a/core/http/endpoints/localai/nodes_test.go +++ b/core/http/endpoints/localai/nodes_test.go @@ -210,50 +210,98 @@ var _ = Describe("Node HTTP handlers", func() { Expect(second["tunnel_token"]).ToNot(Equal(plaintext)) }) - It("does not issue a tunnel credential to an agent node", func() { - // An agent worker serves no gRPC backends and no file staging; - // nothing dials into it, so a tunnel replaces nothing for it and no - // client on its side would open one. Minting anyway would be - // credential surface with no feature behind it. + It("issues a tunnel credential to an agent node", func() { + // An agent worker holds a tunnel too. It runs no backends and + // stages no files, so what its tunnel carries is only its own HTTP + // server, but the frontend reaches its control verbs over it and + // therefore has to be able to dial it at all. // - // Enforcement is structural rather than a second check: with no - // credential minted, the node's hash stays empty and the tunnel - // route refuses it like any other node without one. + // The gate that used to refuse this was correct while nothing + // dialled into an agent worker. Reopening it is deliberate, and the + // spec below is what keeps its other half honest. resp := register(`{"name":"agent-1","node_type":"agent"}`, "", true) Expect(resp["node_type"]).To(Equal(nodes.NodeTypeAgent)) - Expect(resp).ToNot(HaveKey("tunnel_token")) + + plaintext, _ := resp["tunnel_token"].(string) + Expect(plaintext).ToNot(BeEmpty()) node, err := registry.Get(context.Background(), resp["id"].(string)) Expect(err).ToNot(HaveOccurred()) - Expect(node.TunnelTokenHash).To(BeEmpty()) + // Stored as a hash, never as the secret, exactly as a backend + // node's is; ConnectHandler compares against this column and does + // not look at node_type at all. + Expect(node.TunnelTokenHash).To(Equal(hashOf(plaintext))) + Expect(node.TunnelTokenHash).ToNot(Equal(plaintext)) + + // Per-node, not derived from anything shared. A second agent + // registering gets a different credential. + other := register(`{"name":"agent-2","node_type":"agent"}`, "", true) + Expect(other["tunnel_token"]).ToNot(Equal(plaintext)) }) - It("clears a tunnel credential when a node stops being a backend node", func() { - // Register upserts BY NAME, so a node can change node_type in place. - // Skipping the mint on the way through leaves the credential the - // node earned as a backend sitting on a row that is now an agent: - // Register's struct Updates zero-skips the column while writing the - // new node_type, so nothing else clears it. ConnectHandler never - // looks at node_type, so that stale hash is a usable tunnel - // credential for a node type that is not supposed to hold one. + It("rewrites the hash, rather than leaving a stale one, when a node changes type", func() { + // Register upserts BY NAME, so a node can change node_type in + // place, and Register's struct Updates zero-skips TunnelTokenHash + // while writing the new node_type. The invariant is that the stored + // hash always matches the node's CURRENT credential: a node whose + // type changed must not be left holding the one it was handed + // under its old type, because that is a secret the worker still + // knows and nothing would ever retire. // // This is the same shape as the Register-upserts-by-name hazard // already carried forward: a name is not an identity. backend := register(`{"name":"shifty","address":"10.0.0.7:50051"}`, "", true) - Expect(backend["tunnel_token"]).ToNot(BeEmpty()) + // Read with a comma-ok rather than a bare assertion: a build that + // issues no credential must fail this spec on the assertion below, + // naming what it is missing, rather than panic on a nil interface. + backendToken, _ := backend["tunnel_token"].(string) + Expect(backendToken).ToNot(BeEmpty()) agent := register(`{"name":"shifty","node_type":"agent"}`, "", true) Expect(agent["id"]).To(Equal(backend["id"]), "re-registration must keep the node identity") Expect(agent["node_type"]).To(Equal(nodes.NodeTypeAgent)) - Expect(agent).ToNot(HaveKey("tunnel_token")) + agentToken, _ := agent["tunnel_token"].(string) + Expect(agentToken).ToNot(BeEmpty(), + "the node changed type and was handed no credential, so its stored hash is whatever its previous type left behind") node, err := registry.Get(context.Background(), backend["id"].(string)) Expect(err).ToNot(HaveOccurred()) - // The claim the gate makes is that an ineligible node HAS no - // credential, not merely that it was not handed a new one. Only - // then is the empty-hash refusal in ConnectHandler the enforcement. - Expect(node.TunnelTokenHash).To(BeEmpty(), - "the node kept the credential it earned as a backend, so the mint-site gate is not structural") + Expect(node.TunnelTokenHash).To(Equal(hashOf(agentToken))) + Expect(node.TunnelTokenHash).ToNot(Equal(hashOf(backendToken)), + "the node kept the credential it earned under its previous type") + }) + + It("clears the credential of a node whose type is entitled to none", func() { + // The other half of the gate, and the half that makes enforcement + // STRUCTURAL: ConnectHandler never looks at node_type, so what + // refuses an ineligible node is its empty hash. Skipping the mint + // would leave a live credential on the row. + // + // Driven through the registry rather than through + // RegisterNodeEndpoint, which rejects any node_type that is neither + // backend nor agent. That validation is exactly why the branch + // cannot be reached from the wire today, and exactly why the branch + // has to stay: a row's node_type is also written by other builds, + // and adding a type must be a decision about eligibility rather + // than a silent grant. + node := &nodes.BackendNode{Name: "shifty-future", NodeType: nodes.NodeTypeBackend} + Expect(registry.Register(context.Background(), node, true)).To(Succeed()) + response := map[string]any{} + attachTunnelToken(context.Background(), response, registry, node) + Expect(response).To(HaveKey("tunnel_token")) + stored, err := registry.Get(context.Background(), node.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(stored.TunnelTokenHash).ToNot(BeEmpty()) + + node.NodeType = "some-future-worker-kind" + response = map[string]any{} + attachTunnelToken(context.Background(), response, registry, node) + + Expect(response).ToNot(HaveKey("tunnel_token")) + stored, err = registry.Get(context.Background(), node.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(stored.TunnelTokenHash).To(BeEmpty(), + "an ineligible node kept a usable tunnel credential, so the mint-site gate is not structural") }) It("returns nats_jwt when account seed is configured", func() { diff --git a/core/services/agentworker/agentworker_suite_test.go b/core/services/agentworker/agentworker_suite_test.go new file mode 100644 index 000000000000..b0b31c7d66e1 --- /dev/null +++ b/core/services/agentworker/agentworker_suite_test.go @@ -0,0 +1,13 @@ +package agentworker_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestAgentWorker(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Agent worker control plane") +} diff --git a/core/services/agentworker/handlers.go b/core/services/agentworker/handlers.go new file mode 100644 index 000000000000..435040eddc73 --- /dev/null +++ b/core/services/agentworker/handlers.go @@ -0,0 +1,111 @@ +// Package agentworker serves an agent worker's control plane over the tunnel it +// holds to the frontend. +// +// An agent worker runs no backend processes and stages no files. What it does +// serve is MCP: it creates the sessions a frontend cannot (stdio servers under +// docker), executes tools against them, and answers discovery. Those verbs +// travelled on NATS request-reply, which meant the frontend picked a worker by +// letting the bus's queue group pick one, and neither side could say which +// worker had answered. +// +// This package is the worker half of moving them onto the same HTTP control +// plane a backend worker already serves. The frontend half, which chooses WHICH +// agent worker to ask, is not here: this package answers, it does not select. +package agentworker + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// UnaryHandler answers a verb with exactly one reply and no progress. The +// handler's bytes become the whole response body. +// +// An error returned here is this worker FAILING TO SERVE the verb, never the +// verb's own bad news. The distinction is the one the whole phase is built on: +// a body on a 200 is the WORKER'S OWN ANSWER, which a reap guard may act on, +// and everything else is a failure the frontend must not read as evidence +// about anything. An MCP tool that ran and errored is an answer, and it travels +// as bytes with an error field set, exactly as it did over the bus. +type UnaryHandler func(ctx context.Context, raw json.RawMessage) (json.RawMessage, error) + +// StreamHandler answers a verb that emits progress lines BEFORE its reply. +// +// It exists from this task even though nothing sets one yet, because the +// alternative found in review was a later task discovering that a Config of +// UnaryHandlers cannot express a verb that streams, and inventing a second +// handler shape under time pressure. +// +// pub is how a handler writes those lines. It is messaging.Publisher and not +// *agents.StreamPublisher so that this package does not import +// core/services/agents: the concrete writer is constructed by the caller that +// owns the response body, which is the server in this package, and the handler +// only ever needs Publish(subject string, data any) error. +type StreamHandler func(ctx context.Context, raw json.RawMessage, pub messaging.Publisher) (json.RawMessage, error) + +// Config is what the agent worker's control plane needs to serve its verbs. +// +// A nil field means this worker does not serve that verb, and Register mounts +// nothing for it, so the catch-all under workerctl.Prefix answers 404. That is +// the same 404 an older build gives, which is what the frontend already reads +// as "this worker does not serve that verb" (nodes.ErrWorkerControlUnsupported) +// rather than as absence. It is why a later task can add two handlers without +// any other file changing shape. +// +// workerctl.PathAgentCancel is named in the path table with no field here. It +// gets the same 404 for the same reason, and the path is fixed now so the two +// sides cannot disagree about it later. +type Config struct { + // MCPTool answers workerctl.PathMCPToolExecute. + MCPTool UnaryHandler + // MCPDiscovery answers workerctl.PathMCPDiscovery. + MCPDiscovery UnaryHandler + // BackendStop drops the MCP sessions cached for a backend that is going + // away. It answers workerctl.PathBackendStop, the SAME path a backend + // worker serves with a completely different implementation: that one kills + // the process and recycles its port. One path, two implementations, one + // caller, which is what lets the frontend stop branching on node type to + // pick a carrier. + BackendStop func(ctx context.Context, req messaging.BackendStopRequest) error + + // AgentExecute answers workerctl.PathAgentExecute and MCPCIRun answers + // workerctl.PathMCPCIRun. Both are nil in this task and both are set later, + // which is where the queue subjects they replace become claim rows. They are + // declared here so the streaming shape is decided once, with the rest of the + // control plane, rather than twice. + AgentExecute StreamHandler + MCPCIRun StreamHandler +} + +// Register mounts every agent control verb whose handler is non-nil on mux. +// +// It is the Register half of a nodes.AuthenticatedRoutes, so the mux it is +// handed is private and reachable only through that route set's bearer check. +// Nothing here does its own authentication, and nothing here may: a second +// check beside the first is the one that gets forgotten. +func (c Config) Register(mux *http.ServeMux) { + if c.MCPTool != nil { + mux.HandleFunc(workerctl.PathMCPToolExecute, serveUnary(workerctl.PathMCPToolExecute, c.MCPTool)) + } + if c.MCPDiscovery != nil { + mux.HandleFunc(workerctl.PathMCPDiscovery, serveUnary(workerctl.PathMCPDiscovery, c.MCPDiscovery)) + } + if c.BackendStop != nil { + mux.HandleFunc(workerctl.PathBackendStop, serveBackendStop(c.BackendStop)) + } + if c.AgentExecute != nil { + mux.HandleFunc(workerctl.PathAgentExecute, serveStream(workerctl.PathAgentExecute, c.AgentExecute)) + } + if c.MCPCIRun != nil { + mux.HandleFunc(workerctl.PathMCPCIRun, serveStream(workerctl.PathMCPCIRun, c.MCPCIRun)) + } + + // The catch-all, mounted unconditionally and last. A path under the control + // prefix that no verb claims is a frontend newer than this worker, or a + // verb this worker does not implement, and both are the same answer. + mux.HandleFunc(workerctl.Prefix, workerctl.WriteUnknownPath) +} diff --git a/core/services/agentworker/server.go b/core/services/agentworker/server.go new file mode 100644 index 000000000000..52031d1f2f79 --- /dev/null +++ b/core/services/agentworker/server.go @@ -0,0 +1,337 @@ +package agentworker + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "sync" + + "github.com/mudler/xlog" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/worker" + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// How a verb's failure reaches the frontend, written once. +// +// Phase 3 found this same rule written at three sites in one file and pinned at +// one, so every exit below goes through writeFailure and every verb has its own +// spec for it. The rule: a non-2xx means this worker FAILED TO SERVE the +// request, which the frontend maps onto "no route to that worker" and nobody +// may act on; a 2xx body is the WORKER'S OWN ANSWER, which a reap guard may. +// A handler's error is the first, always, and never the second. + +// writeFailure is the ONE place a verb's failure becomes a status. +func writeFailure(w http.ResponseWriter, status int, verb string, err error) { + http.Error(w, "the "+verb+" verb could not be served: "+err.Error(), status) +} + +// failToServe answers a verb this worker could not serve. 500 rather than 404, +// which the frontend reads as a verb the worker does not implement, and rather +// than any 2xx, which it reads as an answer. +func failToServe(w http.ResponseWriter, verb string, err error) { + writeFailure(w, http.StatusInternalServerError, verb, err) +} + +// refuseRequest answers a request this worker could not read. 400 because the +// fault is the caller's, and non-2xx for the same reason as above: a body this +// worker could not decode is not an answer about anything. +func refuseRequest(w http.ResponseWriter, verb string, err error) { + writeFailure(w, http.StatusBadRequest, verb, err) +} + +// serveUnary answers one unary verb: the handler's bytes, whole, on a 200. +func serveUnary(verb string, h UnaryHandler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + body, ok := workerctl.ReadRequestBody(w, r) + if !ok { + return + } + reply, err := h(r.Context(), json.RawMessage(body)) + if err != nil { + failToServe(w, verb, err) + return + } + if len(reply) == 0 { + // The shape a verb with nothing to say takes, and the same one the + // backend worker's fire-and-forget verbs take, so one frontend + // client handles both without knowing which kind of worker answered. + w.WriteHeader(http.StatusNoContent) + return + } + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(reply); err != nil { + xlog.Debug("agent worker control reply could not be written", "verb", verb, "error", err) + } + } +} + +// serveBackendStop answers backend.stop on the agent worker. +// +// It answers 204 with no body, which is exactly what a BACKEND worker answers +// on this same path. The two implementations share nothing else, and that is +// the point: the frontend issues one RPC and reads one answer without knowing +// which kind of worker it reached. +func serveBackendStop(h func(ctx context.Context, req messaging.BackendStopRequest) error) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + body, ok := workerctl.ReadRequestBody(w, r) + if !ok { + return + } + var req messaging.BackendStopRequest + if err := json.Unmarshal(body, &req); err != nil { + refuseRequest(w, workerctl.PathBackendStop, err) + return + } + // The caller's context is dropped, for the reason the backend worker's + // backend.stop drops it: this is fire-and-forget, so there is no answer + // the caller is still waiting on, and abandoning the cleanup half way + // would leave sessions cached for a backend that is gone. + if err := h(context.WithoutCancel(r.Context()), req); err != nil { + failToServe(w, workerctl.PathBackendStop, err) + return + } + w.WriteHeader(http.StatusNoContent) + } +} + +// serveStream answers one streaming verb: zero or more progress lines followed +// by exactly one reply line, NDJSON, the reply last. +// +// The status is written on the FIRST line rather than at entry, and that is +// what lets a handler that fails before publishing anything answer a non-2xx +// like every other verb here. Once a line is on the wire the status is spent: +// such a handler's failure ends the body with NO reply line, which the frontend +// reads as no answer rather than as a failed one. Both are "this worker did not +// answer", which is the only honest thing to say and the only thing nobody may +// act on. +func serveStream(verb string, h StreamHandler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + body, ok := workerctl.ReadRequestBody(w, r) + if !ok { + return + } + stream := &ndjsonStream{w: w} + reply, err := h(r.Context(), json.RawMessage(body), stream) + if err != nil { + if !stream.committed() { + failToServe(w, verb, err) + return + } + xlog.Warn("An agent worker control verb failed after it had streamed progress, so its response ends with no reply line", + "verb", verb, "error", err) + return + } + stream.writeReply(verb, reply) + } +} + +// ndjsonStream writes the Envelope lines of one streaming control response. +// +// The mutex is not optional: a handler may publish progress from a goroutine of +// its own, so without serialization a progress write can interleave with the +// terminal reply write and put a torn line on the wire. done is what enforces +// the other half of the contract: once the reply is written, a late progress +// line is dropped rather than appended after it. +type ndjsonStream struct { + mu sync.Mutex + w http.ResponseWriter + header bool + done bool +} + +// Publish writes one progress line. It is messaging.Publisher, which is what +// lets a handler written against the bus keep its shape. +// +// The subject is DISCARDED, and that is a statement about the carrier rather +// than a shortcut. On the bus a subject was how a progress message found the +// one caller waiting for it; here the response body the caller is already +// reading IS that correlation, so there is nothing left for a subject to +// select. It is kept in the signature because it is the interface the handlers +// implement. +func (n *ndjsonStream) Publish(_ string, data any) error { + raw, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("encoding a progress line: %w", err) + } + n.mu.Lock() + defer n.mu.Unlock() + if n.done { + return nil + } + n.write(workerctl.Envelope{Progress: raw}) + return nil +} + +// committed reports whether anything has been written, which is whether the +// status is still this handler's to choose. +func (n *ndjsonStream) committed() bool { + n.mu.Lock() + defer n.mu.Unlock() + return n.header +} + +// writeReply writes the single terminal reply line and closes the stream to any +// further progress. +func (n *ndjsonStream) writeReply(verb string, reply json.RawMessage) { + n.mu.Lock() + defer n.mu.Unlock() + if n.done { + return + } + if len(reply) == 0 { + // A handler that answers nothing still owes the caller a terminal line, + // or the caller reads to EOF without one and cannot tell that from a + // truncated response. + reply = json.RawMessage(`null`) + } + n.write(workerctl.Envelope{Reply: reply}) + n.done = true + xlog.Debug("agent worker control stream finished", "verb", verb) +} + +// write encodes one envelope and flushes it. Callers hold n.mu. +func (n *ndjsonStream) write(env workerctl.Envelope) { + if !n.header { + n.w.Header().Set("Content-Type", workerctl.ContentTypeStream) + // A streaming body must not be buffered into a guessed content type: + // the caller reads line by line and the first line may be minutes + // before the last. + n.w.Header().Set("X-Content-Type-Options", "nosniff") + n.w.WriteHeader(http.StatusOK) + n.header = true + } + if err := json.NewEncoder(n.w).Encode(env); err != nil { + xlog.Debug("agent worker control stream line could not be written", "error", err) + return + } + if f, ok := n.w.(http.Flusher); ok { + f.Flush() + } +} + +// Options is everything Start needs to put an agent worker on the tunnel. +type Options struct { + // FrontendURL is the value the worker registered against + // (LOCALAI_REGISTER_TO). + FrontendURL string + // NodeID is the identity registration assigned this worker. + NodeID string + // TunnelToken supplies the node's own tunnel credential. A function because + // the credential rotates on every re-registration and is read at DIAL time; + // see worker.TunnelConfig.Token. + TunnelToken func() string + // ControlToken is the bearer token guarding the loopback control server. + // It is the deployment's registration token, the same one a backend worker + // puts in front of its control plane and the same one the frontend's + // control client presents. + ControlToken string + // Handlers are the verbs this worker serves. + Handlers Config +} + +// Runtime is a running agent worker control plane: one loopback HTTP server and +// one tunnel to the frontend. +type Runtime struct { + server *http.Server + tunnel *worker.Tunnel + addr string +} + +// Addr is the loopback address the control server bound. +// +// It exists for specs and for a log line. It is deliberately NOT registered +// anywhere: nothing outside this process may learn it, because the only way in +// is meant to be the tunnel. +func (r *Runtime) Addr() string { return r.addr } + +// Connected reports whether the tunnel currently holds a live session. It is a +// statement about REACHABILITY and nothing else; nothing may read it as the +// worker being gone. +func (r *Runtime) Connected() bool { return r != nil && r.tunnel.Connected() } + +// Close stops the tunnel and the control server. It is safe on a nil receiver +// so a caller can defer it beside a failed Start. +func (r *Runtime) Close() error { + if r == nil { + return nil + } + err := r.tunnel.Close() + nodes.ShutdownFileTransferServer(r.server) + return err +} + +// Start binds the control server on loopback, mounts the agent's verbs behind +// one bearer check, and holds a tunnel to the frontend that carries them. +// +// One function rather than a run of statements in a CLI command, and that is +// the point rather than tidiness. The pieces are one fact: a control server +// nothing can reach is a worker that looks healthy and answers nothing, and a +// tunnel with no server behind it refuses every stream. Started as separate +// lines in a Run body, each is a line whose loss has no symptom and which no +// spec can reach without starting a whole worker process. +// +// The listener binds 127.0.0.1:0 and the port is never advertised: an agent +// worker needs no inbound port, which is the whole reason it is being moved off +// a bus it had to be able to dial. +// +// A failure to START the tunnel is returned, unlike a failure to CONNECT: the +// first means the frontend URL or this node's identity is unusable, and the +// second is a frontend that is down or an admin who has not approved this node +// yet, which the tunnel retries through with backoff. +func Start(ctx context.Context, opts Options) (*Runtime, error) { + lis, err := net.Listen("tcp", loopbackBind) + if err != nil { + return nil, fmt.Errorf("binding the agent worker control server: %w", err) + } + addr := lis.Addr().String() + + // Created here and armed only once the tunnel exists, below. Until then + // /readyz reports ready, which is correct: reaching this line means the + // worker has already registered with the frontend, so it is mid-startup + // rather than broken. + readiness := &nodes.WorkerReadiness{} + server, err := nodes.StartControlOnlyServer(lis, opts.ControlToken, readiness, &nodes.AuthenticatedRoutes{ + Prefix: workerctl.Prefix, + Register: opts.Handlers.Register, + }) + if err != nil { + _ = lis.Close() + return nil, fmt.Errorf("starting the agent worker control server: %w", err) + } + + tunnel, err := worker.StartTunnel(ctx, worker.TunnelConfig{ + FrontendURL: opts.FrontendURL, + NodeID: opts.NodeID, + Token: opts.TunnelToken, + // Built by worker.HTTPOnlyServices rather than inline, so the routing + // table, which is this feature's security boundary, is reachable from a + // spec without starting a worker. An agent worker runs no backend + // processes, so the grpc tag is not offered at all. + Services: worker.HTTPOnlyServices(addr), + }) + if err != nil { + nodes.ShutdownFileTransferServer(server) + return nil, fmt.Errorf("starting the agent worker tunnel: %w", err) + } + // Armed here rather than at the call site. /readyz means "the frontend can + // reach me", and a live tunnel session is the only thing that makes that + // true for a process that binds loopback and advertises nothing. As a + // separate statement elsewhere its loss has no symptom: the gate fails + // open, so the worker answers 200 forever with no session. + readiness.Set(nodes.TunnelReadiness(tunnel)) + + xlog.Info("Agent worker control plane serving over its tunnel", "node", opts.NodeID, "addr", addr) + return &Runtime{server: server, tunnel: tunnel, addr: addr}, nil +} + +// loopbackBind is where the control server binds. +// +// A constant so that "an agent worker opens no inbound port" is a fact about +// the code rather than a claim about its configuration: there is no flag, no +// environment variable and no argument that moves it. +const loopbackBind = "127.0.0.1:0" diff --git a/core/services/agentworker/server_test.go b/core/services/agentworker/server_test.go new file mode 100644 index 000000000000..d8b6367bcf1a --- /dev/null +++ b/core/services/agentworker/server_test.go @@ -0,0 +1,572 @@ +package agentworker_test + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + + "github.com/gorilla/websocket" + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/agentworker" + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// The agent worker's control plane is exercised over a REAL listener with a +// real http.Client, never against a bare ServeMux. +// +// The bearer check, the 404 for an unmounted verb and the status a failing +// handler answers with are all things the transport carries, and a double that +// calls a handler function directly cannot fail the way production fails: it +// never goes through the mount that applies the check, so a spec written that +// way stays green with the check deleted. + +const controlToken = "agent-control-token" + +// serveConfig mounts cfg the way the agent worker mounts it, on loopback behind +// the bearer check, and returns the base URL. +func serveConfig(cfg agentworker.Config) string { + GinkgoHelper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).ToNot(HaveOccurred()) + srv, err := nodes.StartControlOnlyServer(lis, controlToken, nil, &nodes.AuthenticatedRoutes{ + Prefix: workerctl.Prefix, + Register: cfg.Register, + }) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { nodes.ShutdownFileTransferServer(srv) }) + return "http://" + lis.Addr().String() +} + +// post issues one control request with the bearer token. +func post(base, path, body string) *http.Response { + GinkgoHelper() + return postWithToken(base, path, body, controlToken) +} + +func postWithToken(base, path, body, token string) *http.Response { + GinkgoHelper() + req, err := http.NewRequest(http.MethodPost, base+path, strings.NewReader(body)) + Expect(err).ToNot(HaveOccurred()) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := http.DefaultClient.Do(req) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + return resp +} + +func bodyOf(resp *http.Response) string { + GinkgoHelper() + raw, err := io.ReadAll(resp.Body) + Expect(err).ToNot(HaveOccurred()) + return string(raw) +} + +// echoHandler answers with the bytes it was given, so a spec can tell the +// handler's own answer apart from anything the server invented. +func echoHandler(reply string) agentworker.UnaryHandler { + return func(_ context.Context, raw json.RawMessage) (json.RawMessage, error) { + return json.RawMessage(reply), nil + } +} + +// failingHandler is a verb this worker could not serve. +func failingHandler(err error) agentworker.UnaryHandler { + return func(context.Context, json.RawMessage) (json.RawMessage, error) { return nil, err } +} + +// fullConfig serves every verb this task implements. +func fullConfig() agentworker.Config { + return agentworker.Config{ + MCPTool: echoHandler(`{"result":"tool-ran"}`), + MCPDiscovery: echoHandler(`{"servers":[]}`), + BackendStop: func(context.Context, messaging.BackendStopRequest) error { return nil }, + } +} + +var _ = Describe("The agent worker's control verbs", func() { + It("answers each verb with the handler's own bytes, on a 200", func() { + base := serveConfig(fullConfig()) + + tool := post(base, workerctl.PathMCPToolExecute, `{"tool_name":"x"}`) + Expect(tool.StatusCode).To(Equal(http.StatusOK)) + Expect(bodyOf(tool)).To(Equal(`{"result":"tool-ran"}`)) + + disc := post(base, workerctl.PathMCPDiscovery, `{"model_name":"m"}`) + Expect(disc.StatusCode).To(Equal(http.StatusOK)) + Expect(bodyOf(disc)).To(Equal(`{"servers":[]}`)) + }) + + It("answers backend.stop with the 204 a BACKEND worker answers on that same path", func() { + // One path, two implementations, one caller. The frontend must not have + // to know which kind of worker it reached to read the answer, which is + // what lets its carrier split for this verb die. + stopped := make(chan messaging.BackendStopRequest, 1) + cfg := fullConfig() + cfg.BackendStop = func(_ context.Context, req messaging.BackendStopRequest) error { + stopped <- req + return nil + } + base := serveConfig(cfg) + + resp := post(base, workerctl.PathBackendStop, `{"backend":"llama-cpp","force":true}`) + Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) + Expect(bodyOf(resp)).To(BeEmpty()) + var got messaging.BackendStopRequest + Eventually(stopped).Should(Receive(&got)) + // The DTO reaches the handler decoded, so the agent's implementation + // reads the same request the backend worker's does. + Expect(got.Backend).To(Equal("llama-cpp")) + Expect(got.Force).To(BeTrue()) + }) + + It("refuses a request with no bearer token", func() { + base := serveConfig(fullConfig()) + resp := postWithToken(base, workerctl.PathMCPToolExecute, `{}`, "") + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + + It("refuses a request with the wrong bearer token", func() { + // Non-empty and wrong, which a check that merely tested for the + // header's presence would let through. + base := serveConfig(fullConfig()) + resp := postWithToken(base, workerctl.PathMCPToolExecute, `{}`, "not-the-token") + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + + DescribeTable("answers a verb it does not serve with the catch-all 404", + // 404 and NOT 500. The frontend reads exactly this status as "this + // worker does not serve that verb" (nodes.ErrWorkerControlUnsupported) + // rather than as absence, which is what makes adding a handler later a + // purely additive change: an older worker and a newer one differ only + // in which paths are mounted. + func(path string) { + base := serveConfig(fullConfig()) + resp := post(base, path, `{}`) + Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) + Expect(bodyOf(resp)).To(ContainSubstring("unknown worker control path")) + }, + Entry("agent execute, whose handler is nil until it is wired", workerctl.PathAgentExecute), + Entry("mcp ci run, whose handler is nil until it is wired", workerctl.PathMCPCIRun), + Entry("agent cancel, which has no handler field at all yet", workerctl.PathAgentCancel), + Entry("a backend worker's verb, which an agent worker never serves", workerctl.PathBackendInstall), + Entry("a path no build has ever named", workerctl.Prefix+"nothing/here"), + ) + + DescribeTable("mounts nothing for a nil handler, so the catch-all answers", + // The mirror of the table above: it is the NIL FIELD that produces the + // 404, not the path being unknown to the package. + func(path string, blank func(*agentworker.Config)) { + cfg := fullConfig() + blank(&cfg) + base := serveConfig(cfg) + Expect(post(base, path, `{}`).StatusCode).To(Equal(http.StatusNotFound)) + }, + Entry("mcp tool execute", workerctl.PathMCPToolExecute, func(c *agentworker.Config) { c.MCPTool = nil }), + Entry("mcp discovery", workerctl.PathMCPDiscovery, func(c *agentworker.Config) { c.MCPDiscovery = nil }), + Entry("backend stop", workerctl.PathBackendStop, func(c *agentworker.Config) { c.BackendStop = nil }), + ) + + // A rule stated at every verb has to be pinned at every verb. The exit + // helper is one function, and that is exactly why one spec here would not + // prove the others use it. + DescribeTable("answers a handler's failure with a non-2xx and never a 200", + func(path string, breaks func(*agentworker.Config)) { + cfg := fullConfig() + breaks(&cfg) + base := serveConfig(cfg) + + resp := post(base, path, `{}`) + // The property, stated as the frontend reads it: a 2xx body is the + // WORKER'S OWN ANSWER, which a reap guard may act on, and a failure + // to serve is not an answer about anything. A 200 carrying an error + // field would put this in the wrong bucket. + Expect(resp.StatusCode).ToNot(Equal(http.StatusOK)) + Expect(resp.StatusCode).ToNot(Equal(http.StatusNoContent)) + Expect(resp.StatusCode).To(BeNumerically(">=", 400)) + // And not the 404 that means "older worker", which would send the + // frontend down a version-skew fallback for a transient failure. + Expect(resp.StatusCode).ToNot(Equal(http.StatusNotFound)) + Expect(bodyOf(resp)).To(ContainSubstring("could not be served")) + }, + Entry("mcp tool execute", workerctl.PathMCPToolExecute, func(c *agentworker.Config) { + c.MCPTool = failingHandler(errors.New("the tool plane is down")) + }), + Entry("mcp discovery", workerctl.PathMCPDiscovery, func(c *agentworker.Config) { + c.MCPDiscovery = failingHandler(errors.New("discovery is down")) + }), + Entry("backend stop", workerctl.PathBackendStop, func(c *agentworker.Config) { + c.BackendStop = func(context.Context, messaging.BackendStopRequest) error { + return errors.New("the session cache is wedged") + } + }), + Entry("agent execute, before it has published anything", workerctl.PathAgentExecute, func(c *agentworker.Config) { + c.AgentExecute = func(context.Context, json.RawMessage, messaging.Publisher) (json.RawMessage, error) { + return nil, errors.New("no agent to run") + } + }), + ) + + It("refuses a GET on a mounted verb, because a control verb is a command", func() { + base := serveConfig(fullConfig()) + req, err := http.NewRequest(http.MethodGet, base+workerctl.PathMCPToolExecute, nil) + Expect(err).ToNot(HaveOccurred()) + req.Header.Set("Authorization", "Bearer "+controlToken) + resp, err := http.DefaultClient.Do(req) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusMethodNotAllowed)) + }) + + It("refuses a body larger than the shared control cap", func() { + base := serveConfig(fullConfig()) + oversized := `{"tool_name":"` + strings.Repeat("a", workerctl.MaxRequestBytes) + `"}` + resp := post(base, workerctl.PathMCPToolExecute, oversized) + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + }) + + It("answers a body the verb could not decode with a non-2xx", func() { + base := serveConfig(fullConfig()) + resp := post(base, workerctl.PathBackendStop, `{"backend":`) + Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) + }) + + It("mounts no file-staging routes, because an agent worker stages nothing", func() { + // StartControlOnlyServer exists so that a worker with no staging + // directories cannot end up serving upload and download handlers + // against a path that resolves to its working directory. + base := serveConfig(fullConfig()) + req, err := http.NewRequest(http.MethodGet, base+"/v1/files/anything", nil) + Expect(err).ToNot(HaveOccurred()) + req.Header.Set("Authorization", "Bearer "+controlToken) + resp, err := http.DefaultClient.Do(req) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) + }) +}) + +var _ = Describe("The agent worker's streaming verbs", func() { + // Nothing sets a StreamHandler in this task. These specs exist so the shape + // is decided and pinned now: a later task adds two handlers and changes + // nothing else, and it can only do that if what a stream looks like on the + // wire is already fixed. + + // lines reads an NDJSON body into its envelopes. + lines := func(resp *http.Response) []workerctl.Envelope { + GinkgoHelper() + var out []workerctl.Envelope + sc := bufio.NewScanner(resp.Body) + for sc.Scan() { + if strings.TrimSpace(sc.Text()) == "" { + continue + } + var env workerctl.Envelope + Expect(json.Unmarshal(sc.Bytes(), &env)).To(Succeed()) + out = append(out, env) + } + Expect(sc.Err()).ToNot(HaveOccurred()) + return out + } + + It("puts every progress line before the one reply, and the reply last", func() { + cfg := fullConfig() + cfg.AgentExecute = func(_ context.Context, raw json.RawMessage, pub messaging.Publisher) (json.RawMessage, error) { + Expect(pub.Publish("agent.progress", map[string]string{"step": "one"})).To(Succeed()) + Expect(pub.Publish("agent.progress", map[string]string{"step": "two"})).To(Succeed()) + return json.RawMessage(`{"done":true}`), nil + } + base := serveConfig(cfg) + + resp := post(base, workerctl.PathAgentExecute, `{}`) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + Expect(resp.Header.Get("Content-Type")).To(Equal(workerctl.ContentTypeStream)) + + got := lines(resp) + Expect(got).To(HaveLen(3)) + Expect(string(got[0].Progress)).To(ContainSubstring(`"one"`)) + Expect(string(got[1].Progress)).To(ContainSubstring(`"two"`)) + // The contract the caller stops reading on: exactly one reply, and it + // is the last thing on the body. + Expect(got[2].Reply).ToNot(BeEmpty()) + Expect(string(got[2].Reply)).To(Equal(`{"done":true}`)) + }) + + It("drops a progress line published after the reply, rather than appending it", func() { + // Without this the reply is not the last line, and a caller that stops + // reading at the reply leaves bytes in the stream that the next read on + // a pooled connection would see. + late := make(chan error, 1) + cfg := fullConfig() + cfg.MCPCIRun = func(_ context.Context, _ json.RawMessage, pub messaging.Publisher) (json.RawMessage, error) { + go func() { + // Published from another goroutine, which is what a handler + // with a debounce timer does, and what the mutex is for. + <-late + _ = pub.Publish("job.progress", map[string]string{"step": "too-late"}) + late <- nil + }() + return json.RawMessage(`{"status":"completed"}`), nil + } + base := serveConfig(cfg) + + resp := post(base, workerctl.PathMCPCIRun, `{}`) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + got := lines(resp) + // The body ended after the reply, so the late publish had nowhere to + // go. Released only now, so the ordering is scripted rather than raced. + late <- nil + Eventually(late).Should(Receive()) + Expect(got).To(HaveLen(1)) + Expect(string(got[0].Reply)).To(Equal(`{"status":"completed"}`)) + }) + + It("ends the body with NO reply line when a handler fails after streaming", func() { + // The status is already on the wire by then, so the failure cannot be a + // non-2xx. It must also not be a reply, because a reply is the worker's + // own answer. A body that ends without one is "this worker did not + // answer", which is the only honest thing left to say. + cfg := fullConfig() + cfg.AgentExecute = func(_ context.Context, _ json.RawMessage, pub messaging.Publisher) (json.RawMessage, error) { + Expect(pub.Publish("agent.progress", map[string]string{"step": "started"})).To(Succeed()) + return nil, errors.New("the agent died mid-run") + } + base := serveConfig(cfg) + + resp := post(base, workerctl.PathAgentExecute, `{}`) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + got := lines(resp) + Expect(got).To(HaveLen(1)) + Expect(got[0].Reply).To(BeEmpty()) + Expect(string(got[0].Progress)).To(ContainSubstring("started")) + }) +}) + +// fakeFrontend is the far side of an agent worker's tunnel: the real WebSocket +// upgrade and the real yamux server handshake, so these specs exercise the +// wire rather than a mock of it. +type fakeFrontend struct { + srv *httptest.Server + sessions chan *yamux.Session +} + +func newFakeFrontend() *fakeFrontend { + f := &fakeFrontend{sessions: make(chan *yamux.Session, 8)} + upgrader := websocket.Upgrader{} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != cluster.ConnectPath { + w.WriteHeader(http.StatusNotFound) + return + } + ws, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + sess, err := yamux.Server(cluster.WebsocketConn(ws), nil, nil) + if err != nil { + _ = ws.Close() + return + } + select { + case f.sessions <- sess: + default: + _ = sess.Close() + } + })) + return f +} + +func (f *fakeFrontend) close() { + for { + select { + case sess := <-f.sessions: + _ = sess.Close() + default: + f.srv.Close() + return + } + } +} + +// awaitErr runs fn on its own goroutine and reports its result on a channel. +// +// Every blocking read below goes through it. Asserting "the worker refused this +// stream" with a read deadline is satisfied just as well by a stream the worker +// never answered at all; reading with NO deadline and requiring the channel to +// deliver inverts that. +func awaitErr(fn func() error) <-chan error { + ch := make(chan error, 1) + go func() { ch <- fn() }() + return ch +} + +var _ = Describe("An agent worker on the tunnel", func() { + var ( + ctx context.Context + cancel context.CancelFunc + frontend *fakeFrontend + rt *agentworker.Runtime + ) + + BeforeEach(func() { + ctx, cancel = context.WithCancel(context.Background()) + frontend = newFakeFrontend() + }) + + AfterEach(func() { + if rt != nil { + Expect(rt.Close()).To(Succeed()) + rt = nil + } + cancel() + frontend.close() + }) + + start := func(url string) { + GinkgoHelper() + var err error + rt, err = agentworker.Start(ctx, agentworker.Options{ + FrontendURL: url, + NodeID: "agent-node-1", + TunnelToken: func() string { return "tunnel-secret" }, + ControlToken: controlToken, + Handlers: fullConfig(), + }) + Expect(err).ToNot(HaveOccurred()) + } + + session := func() *yamux.Session { + GinkgoHelper() + var sess *yamux.Session + Eventually(frontend.sessions, "10s").Should(Receive(&sess)) + return sess + } + + It("binds only loopback, so it opens no inbound port", func() { + start(frontend.srv.URL) + host, _, err := net.SplitHostPort(rt.Addr()) + Expect(err).ToNot(HaveOccurred()) + Expect(host).To(Equal("127.0.0.1")) + }) + + It("serves its control verbs through the tunnel and nothing else", func() { + start(frontend.srv.URL) + sess := session() + + // The http tag, which is the one an agent worker offers. Driven as a + // real HTTP request over a real yamux stream, so what this proves is + // that the frontend can reach the verbs, not that a map has a key. + stream, err := sess.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagHTTP, "ignored:1")).To(Succeed()) + accepted := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + Eventually(accepted, "10s").Should(Receive(BeNil())) + + req, err := http.NewRequest(http.MethodPost, "http://agent-node-1"+workerctl.PathMCPToolExecute, + strings.NewReader(`{"tool_name":"x"}`)) + Expect(err).ToNot(HaveOccurred()) + req.Header.Set("Authorization", "Bearer "+controlToken) + Expect(req.Write(stream)).To(Succeed()) + + read := make(chan *http.Response, 1) + go func() { + resp, rerr := http.ReadResponse(bufio.NewReader(stream), req) + if rerr == nil { + read <- resp + } + close(read) + }() + var resp *http.Response + Eventually(read, "10s").Should(Receive(&resp)) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + defer func() { _ = resp.Body.Close() }() + raw, err := io.ReadAll(resp.Body) + Expect(err).ToNot(HaveOccurred()) + Expect(string(raw)).To(Equal(`{"result":"tool-ran"}`)) + }) + + It("refuses a stream tagged for gRPC, which it runs nothing to serve", func() { + start(frontend.srv.URL) + sess := session() + + stream, err := sess.OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagGRPC, "127.0.0.1:50051")).To(Succeed()) + + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + var got error + Eventually(reply, "10s").Should(Receive(&got)) + // The refusal names the tag, which tells a frontend a retry is + // pointless, rather than "unavailable", which tells it to retry + // something that can never work on this kind of worker. + Expect(got).To(MatchError(cluster.ErrStreamTagUnknown)) + Expect(got).ToNot(MatchError(cluster.ErrStreamTargetUnavailable)) + }) + + It("reports itself not ready while it holds no tunnel session", func() { + // The gate FAILS OPEN when nothing arms it, so a worker with a dead + // tunnel would answer 200 forever and nothing in the process would say + // a word. That is the whole symptom of the arming line going missing. + start("http://127.0.0.1:1") + + resp, err := http.Get("http://" + rt.Addr() + "/readyz") //nolint:noctx // loopback probe in a spec + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusServiceUnavailable)) + Expect(bodyOf(resp)).To(ContainSubstring("tunnel")) + Expect(rt.Connected()).To(BeFalse()) + }) + + It("reports itself ready once its tunnel is up", func() { + // The negative control for the spec above: a gate wired to a probe that + // always failed would satisfy it. + start(frontend.srv.URL) + session() + + Eventually(func() int { + resp, err := http.Get("http://" + rt.Addr() + "/readyz") //nolint:noctx // loopback probe in a spec + if err != nil { + return 0 + } + defer func() { _ = resp.Body.Close() }() + return resp.StatusCode + }, "10s").Should(Equal(http.StatusOK)) + }) + + It("refuses to start with no routes to serve", func() { + _, err := nodes.StartControlOnlyServer(mustListen(), controlToken, nil, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no routes")) + }) + + DescribeTable("refuses to start on a route set it cannot mount", + func(routes *nodes.AuthenticatedRoutes, want string) { + _, err := nodes.StartControlOnlyServer(mustListen(), controlToken, nil, routes) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(want)) + }, + Entry("no prefix", &nodes.AuthenticatedRoutes{Register: func(*http.ServeMux) {}}, "no prefix"), + Entry("no registrar", &nodes.AuthenticatedRoutes{Prefix: workerctl.Prefix}, "no registrar"), + ) +}) + +func mustListen() net.Listener { + GinkgoHelper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = lis.Close() }) + return lis +} diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index 3bb8a49eb57d..8f5429a3909b 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -149,11 +149,13 @@ func SubjectResponseCancel(responseID string) string { // through its tunnel, so a subject builder for any of them would be a subject // nothing publishes and nothing subscribes to. // -// ONE survives: backend.stop, and only for AGENT workers. They hold no tunnel, -// so they have no control plane to serve, and they subscribe to it to drop the -// MCP sessions cached for a backend that is going away. See -// nodes.RemoteUnloaderAdapter.stopBackend for the split, and -// core/cli/agent_worker.go for the subscriber. +// ONE survives: backend.stop, and only for AGENT workers. They now hold a +// tunnel and mount that verb on it, so what keeps this subject alive is the +// PUBLISHER: nodes.RemoteUnloaderAdapter.stopBackend still sends an agent +// node's stop here rather than over its control route. The agent worker +// subscribes to it to drop the MCP sessions cached for a backend that is going +// away; see core/cli/agent_worker.go for the subscriber, which shares one +// implementation with the control route so the two carriers cannot diverge. // // The request and reply types below are UNCHANGED and still live here: they are // the wire format of the control routes, byte for byte what the subjects @@ -285,7 +287,9 @@ type BackendStopRequest struct { // It is the one node subject left, and it is addressed only to agent nodes. A // BACKEND worker takes its stop on workerctl.PathBackendStop over its tunnel, // where it also kills the process and recycles the port; an agent worker runs -// no backend processes and only needs to hear that one went. +// no backend processes and only needs to hear that one went. An agent worker +// serves that same path over its own tunnel as well, so this subject is what +// the publisher has not moved off yet rather than the only way to reach one. func SubjectNodeBackendStop(nodeID string) string { return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".backend.stop" } diff --git a/core/services/nodes/file_transfer_server.go b/core/services/nodes/file_transfer_server.go index c68c2f05bdf8..e61bdb003beb 100644 --- a/core/services/nodes/file_transfer_server.go +++ b/core/services/nodes/file_transfer_server.go @@ -113,18 +113,9 @@ func startFileTransferServer(lis net.Listener, stagingDir, modelsDir, dataDir, t } func startFileTransferServerWithRoutes(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, capacity EphemeralCapacity, extra *AuthenticatedRoutes, logStore ...*model.BackendLogStore) (*http.Server, error) { - // Checked before anything is created. A route set that names no prefix or - // no registrar is a caller bug, and mounting nothing for it would be the - // worst possible answer: the server comes up healthy and every route the - // caller believes it registered answers 404, which through a tunnel is - // indistinguishable from a version skew. - if extra != nil { - if extra.Prefix == "" { - return nil, fmt.Errorf("extra routes were given no prefix to mount under") - } - if extra.Register == nil { - return nil, fmt.Errorf("extra routes under %q were given no registrar", extra.Prefix) - } + // Checked before anything is created; see validateAuthenticatedRoutes. + if err := validateAuthenticatedRoutes(extra); err != nil { + return nil, err } if err := os.MkdirAll(stagingDir, 0750); err != nil { return nil, fmt.Errorf("creating staging dir %s: %w", stagingDir, err) @@ -214,8 +205,101 @@ func startFileTransferServerWithRoutes(lis net.Listener, stagingDir, modelsDir, registerBackendLogHandlers(mux, token, ls) } - // Liveness/readiness probes — unauthenticated so container orchestrators - // (Docker HEALTHCHECK, k8s probes) can hit them without the bearer token. + mountProbes(mux, readiness) + mountAuthenticatedRoutes(mux, token, extra) + + return serveWorkerHTTP(lis, mux, + "stagingDir", stagingDir, "modelsDir", modelsDir, "dataDir", dataDir), nil +} + +// StartControlOnlyServer starts a worker's loopback HTTP server carrying ONLY +// the routes in extra, behind the same bearer check the file-transfer server +// applies, plus the same two probes. +// +// Separate from StartFileTransferServer rather than a flag on it: an agent +// worker stages no files, and a constructor that took empty directory arguments +// would mount the upload and download handlers against a path that resolves to +// the process working directory. The two servers share every rule they have in +// common through the helpers below rather than through a parameter that means +// "skip half of me". +// +// extra is required here, unlike on the file-transfer server where it is +// optional. A control-only server with no routes is a listener that answers +// nothing, and the failure mode is the one the route validation below already +// exists to prevent: the process comes up healthy and every verb 404s, which +// through a tunnel is indistinguishable from a version skew. +func StartControlOnlyServer(lis net.Listener, token string, readiness *WorkerReadiness, extra *AuthenticatedRoutes) (*http.Server, error) { + if extra == nil { + return nil, fmt.Errorf("a control-only worker server was given no routes to serve") + } + if err := validateAuthenticatedRoutes(extra); err != nil { + return nil, err + } + // The same warning the file-transfer server gives, for a smaller blast + // radius: an empty token makes checkBearerToken fail open, so every control + // verb this worker serves is reachable unauthenticated by anything that can + // reach this port. It binds loopback only, which is why this is a warning + // rather than a refusal. + if token == "" { + xlog.Warn("Worker control server starting WITHOUT a registration token: its control verbs are unauthenticated for anyone who can reach this port; set LOCALAI_REGISTRATION_TOKEN") + } + + mux := http.NewServeMux() + mountProbes(mux, readiness) + mountAuthenticatedRoutes(mux, token, extra) + + return serveWorkerHTTP(lis, mux, "routes", extra.Prefix), nil +} + +// validateAuthenticatedRoutes rejects a route set that names no prefix or no +// registrar. +// +// Mounting nothing for one would be the worst possible answer: the server comes +// up healthy and every route the caller believes it registered answers 404, +// which through a tunnel is indistinguishable from a version skew. +func validateAuthenticatedRoutes(extra *AuthenticatedRoutes) error { + if extra == nil { + return nil + } + if extra.Prefix == "" { + return fmt.Errorf("extra routes were given no prefix to mount under") + } + if extra.Register == nil { + return fmt.Errorf("extra routes under %q were given no registrar", extra.Prefix) + } + return nil +} + +// mountAuthenticatedRoutes puts extra behind the bearer check on its own mux. +// +// One site, and that is the point rather than a convenience. This is the check +// that stands between an unauthenticated caller and every verb a worker serves, +// and there are now two kinds of worker server mounting routes through it. A +// second copy is a second place to forget it, which would not fail any spec +// written against the first. +func mountAuthenticatedRoutes(mux *http.ServeMux, token string, extra *AuthenticatedRoutes) { + if extra == nil { + return + } + extraMux := http.NewServeMux() + extra.Register(extraMux) + mux.Handle(extra.Prefix, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !checkBearerToken(r, token) { + xlog.Debug("worker HTTP server: unauthorized request on an extra route", + "method", r.Method, "path", r.URL.Path, "remote", r.RemoteAddr) + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + extraMux.ServeHTTP(w, r) + })) +} + +// mountProbes adds the liveness and readiness probes every worker server +// carries. +// +// They are unauthenticated so container orchestrators (Docker HEALTHCHECK, k8s +// probes) can reach them without the bearer token. +func mountProbes(mux *http.ServeMux, readiness *WorkerReadiness) { probe := func(check func() error) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet && r.Method != http.MethodHead { @@ -234,27 +318,19 @@ func startFileTransferServerWithRoutes(lis net.Listener, stagingDir, modelsDir, } // Liveness: reaching this point means the listener is bound and the mux is - // serving. Deliberately independent of readiness — a worker whose NATS link - // is momentarily down must not be restarted, or a NATS blip becomes a - // cluster-wide restart storm. + // serving. Deliberately independent of readiness: a worker whose link to + // the frontend is momentarily down must not be restarted, or a blip becomes + // a cluster-wide restart storm. mux.HandleFunc("/healthz", probe(func() error { return nil })) // Readiness: "can this worker actually accept work?" See WorkerReadiness. + // A 503 here is THIS CONTAINER'S OWN REPORT that it cannot serve right now, + // never a claim that the node is gone; the frontend decides that from the + // tunnel session it holds, aged against LOCALAI_WORKER_RECONNECT_GRACE. mux.HandleFunc("/readyz", probe(readiness.Check)) +} - if extra != nil { - extraMux := http.NewServeMux() - extra.Register(extraMux) - mux.Handle(extra.Prefix, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !checkBearerToken(r, token) { - xlog.Debug("worker HTTP server: unauthorized request on an extra route", - "method", r.Method, "path", r.URL.Path, "remote", r.RemoteAddr) - http.Error(w, "unauthorized", http.StatusUnauthorized) - return - } - extraMux.ServeHTTP(w, r) - })) - } - +// serveWorkerHTTP starts serving mux on lis and returns the server. +func serveWorkerHTTP(lis net.Listener, mux *http.ServeMux, logFields ...any) *http.Server { addr := lis.Addr().String() server := &http.Server{ // Addr is informational here: Serve takes the listener, not this @@ -267,13 +343,13 @@ func startFileTransferServerWithRoutes(lis net.Listener, stagingDir, modelsDir, } go func() { - xlog.Info("HTTP file transfer server started", "addr", addr, "stagingDir", stagingDir, "modelsDir", modelsDir, "dataDir", dataDir) + xlog.Info("Worker HTTP server started", append([]any{"addr", addr}, logFields...)...) if err := server.Serve(lis); err != nil && err != http.ErrServerClosed { - xlog.Error("HTTP file transfer server error", "error", err) + xlog.Error("Worker HTTP server error", "error", err) } }() - return server, nil + return server } // handleClaimWithCapacity marks an existing ephemeral file as owned by the diff --git a/core/services/nodes/health.go b/core/services/nodes/health.go index f7b0bc7a969f..8b9061e9428e 100644 --- a/core/services/nodes/health.go +++ b/core/services/nodes/health.go @@ -115,9 +115,15 @@ func (hm *HealthMonitor) ReadsAbsence() bool { return hm != nil && hm.presence ! // Acting on any of those would demote a fleet for a reason that has nothing to // do with any worker, which is the collapse this whole mechanism replaced. // -// Backend workers only. An agent worker holds no tunnel at all, so it has no -// departure to measure and would answer PresenceUnknown anyway; the type check -// is here to save the query rather than to add a second rule. +// Backend workers only, and since agent workers hold tunnels too that check is +// now the RULE rather than an optimisation. It used to save a query: an agent +// worker dialled nothing, so it had no departure to measure and would have +// answered PresenceUnknown regardless. It now has a real node_connections row +// that really does age past the grace, and demoting an agent node for it would +// be a verdict about a route the agent's actual work does not travel on: an +// agent worker still takes its jobs and its verbs over the bus. Deleting this +// check would silently make every agent worker whose tunnel dropped go +// unhealthy, and its next heartbeat would make it healthy again. func (hm *HealthMonitor) tunnelDeparted(ctx context.Context, node *BackendNode) bool { if hm.presence == nil || node == nil { return false diff --git a/core/services/nodes/health_test.go b/core/services/nodes/health_test.go index 089ca02162a4..8ddfc8491b16 100644 --- a/core/services/nodes/health_test.go +++ b/core/services/nodes/health_test.go @@ -563,8 +563,11 @@ var _ = Describe("HealthMonitor and a worker whose tunnel is gone", func() { }) It("leaves an AGENT node alone even when its tunnel would read as gone", func() { - // Agent workers hold no tunnel and still take their one verb over the - // bus. A departure row for one is not a fact about it. + // Agent workers hold a tunnel of their own now, and still take their + // jobs and their one remaining node verb over the bus. So a departure + // row for one is real and is still not a fact about whether the agent + // worker can work. This spec is what stops a bug in the agent tunnel + // client from demoting a fleet of perfectly healthy agent workers. node := &BackendNode{Name: "agent-worker", NodeType: NodeTypeAgent} Expect(registry.Register(ctx, node, true)).To(Succeed()) departTunnel(node.ID, grace+5*time.Second) diff --git a/core/services/nodes/interfaces.go b/core/services/nodes/interfaces.go index 7231d5296510..51663514a3fe 100644 --- a/core/services/nodes/interfaces.go +++ b/core/services/nodes/interfaces.go @@ -112,10 +112,11 @@ type NodeHealthStore interface { // ModelLocator is used by RemoteUnloaderAdapter for model discovery. // // Get is here for one reason: backend.stop is the only control verb whose -// carrier depends on what KIND of worker it is addressed to, because agent -// workers hold no tunnel and still take it over the bus. The callers that come -// through NodeCommandSender carry a node id and nothing else, so the type is -// read here rather than threaded through every one of them. +// carrier depends on what KIND of worker it is addressed to, because an agent +// node's stop is still published on the bus rather than issued over the tunnel +// it now holds. The callers that come through NodeCommandSender carry a node id +// and nothing else, so the type is read here rather than threaded through every +// one of them. type ModelLocator interface { FindNodesWithModel(ctx context.Context, modelName string) ([]BackendNode, error) RemoveNodeModel(ctx context.Context, nodeID, modelName string, replicaIndex int) error diff --git a/core/services/nodes/managers_agent_node_test.go b/core/services/nodes/managers_agent_node_test.go index 5666c093d0f0..b6b18ac3c1d3 100644 --- a/core/services/nodes/managers_agent_node_test.go +++ b/core/services/nodes/managers_agent_node_test.go @@ -14,12 +14,12 @@ import ( "github.com/mudler/LocalAI/core/services/workerctl" ) -// Agent workers hold no tunnel and serve no control plane, so asking one to -// list its backends can only fail. ListBackends read that failure as a node -// that had gone away and marked it unhealthy; the node's next heartbeat marked -// it healthy again. Every poll of the backends view therefore flapped every -// agent node in the cluster, and while it was unhealthy the router would not -// schedule onto it. +// Agent workers run no backend processes and mount no backend.list route on +// the tunnel they hold, so asking one to list its backends can only fail. +// ListBackends read that failure as a node that had gone away and marked it +// unhealthy; the node's next heartbeat marked it healthy again. Every poll of +// the backends view therefore flapped every agent node in the cluster, and +// while it was unhealthy the router would not schedule onto it. var _ = Describe("Backend listing across mixed node types", func() { var ( db *gorm.DB diff --git a/core/services/nodes/managers_distributed.go b/core/services/nodes/managers_distributed.go index c488d16e19a3..11fe94b0ec41 100644 --- a/core/services/nodes/managers_distributed.go +++ b/core/services/nodes/managers_distributed.go @@ -168,9 +168,11 @@ func (d *DistributedBackendManager) enqueueAndDrainBackendOp(ctx context.Context if node.Status == StatusPending { continue } - // Backend lifecycle ops only make sense on backend-type workers. - // Agent workers hold no tunnel and serve no control plane, so - // enqueueing for them guarantees a forever-retrying row that the + // Backend lifecycle ops only make sense on backend-type workers. An + // agent worker holds a tunnel and serves a control plane, but not + // THESE verbs: it runs no backend processes, so it mounts none of the + // install/upgrade/delete routes and answers the catch-all 404 for + // them. Enqueueing for one guarantees a forever-retrying row that the // reconciler can never drain. Silently skip - they aren't consumers. if node.NodeType != "" && node.NodeType != NodeTypeBackend { continue @@ -349,11 +351,12 @@ func (d *DistributedBackendManager) ListBackends() (gallery.SystemBackends, erro if node.Status == StatusPending || node.Status == StatusOffline || node.Status == StatusDraining { continue } - // Only backend workers serve backend.list. An agent worker holds no - // tunnel, so asking one can only fail, and the failure handling used to - // read that as a node that had gone away: every poll of this view - // marked every agent node unhealthy and its next heartbeat marked it - // healthy again. The backend-op fan-out skips them for the same reason. + // Only backend workers serve backend.list. An agent worker holds a + // tunnel but runs no backend processes, so it mounts no such route and + // asking one can only 404, and the failure handling used to read that + // as a node that had gone away: every poll of this view marked every + // agent node unhealthy and its next heartbeat marked it healthy again. + // The backend-op fan-out skips them for the same reason. if node.NodeType != "" && node.NodeType != NodeTypeBackend { continue } diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index 4048b5a8cac2..30e302a63527 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -46,10 +46,11 @@ type NodeCommandSender interface { // remote nodes. // // One verb is still carried by the bus, and only for one KIND of node: -// backend.stop to an AGENT node. Agent workers hold no tunnel yet, so they have -// nothing to serve a control route on, and they subscribe to -// nodes..backend.stop to drop cached MCP sessions. Removing that publish -// would strand them; see stopBackend. +// backend.stop to an AGENT node. An agent worker now holds a tunnel and mounts +// that route on it, but this side has not been moved onto it yet, and until it +// is, the agent worker's subscription to nodes..backend.stop is what +// actually drops its cached MCP sessions. Removing that publish would strand +// them; see stopBackend. type RemoteUnloaderAdapter struct { registry ModelLocator nats messaging.MessagingClient @@ -389,11 +390,12 @@ func (a *RemoteUnloaderAdapter) nodeTypeOf(ctx context.Context, nodeID string) s // stopBackend sends one backend.stop, over the carrier that kind of worker // listens on. // -// An AGENT node keeps the bus. It holds no tunnel, so it has no control route -// to serve, and it subscribes to nodes..backend.stop to drop the MCP -// sessions cached for a backend that is going away. This is the one verb of the -// ten that is split rather than moved, and the split is the honest intermediate -// state until agent workers hold tunnels too. +// An AGENT node keeps the bus, and that is now a statement about THIS side +// rather than about the worker. The agent worker mounts backend.stop on the +// same control path a backend worker serves it on, so the tunnel could carry +// it; what has not moved is this publish. Splitting on node type is the one +// verb of the ten that is split rather than moved, and it stays split until the +// change that switches this call onto the control route retires the subject. func (a *RemoteUnloaderAdapter) stopBackend(ctx context.Context, nodeID, nodeType, backend string, force bool) error { if nodeType == NodeTypeAgent { subject := messaging.SubjectNodeBackendStop(nodeID) diff --git a/core/services/nodes/unloader_test.go b/core/services/nodes/unloader_test.go index 458a82da2710..4ad403aadc82 100644 --- a/core/services/nodes/unloader_test.go +++ b/core/services/nodes/unloader_test.go @@ -352,7 +352,7 @@ var _ = Describe("RemoteUnloaderAdapter", func() { Expect(bus.publishedSubjects()).To(BeEmpty()) }) - It("sends a backend stop to an AGENT node over the bus, because agent workers hold no tunnel", func() { + It("sends a backend stop to an AGENT node over the bus, because that publisher has not moved onto the tunnel", func() { locator.nodes = []BackendNode{{ID: "agent-1", Name: "agent", NodeType: NodeTypeAgent}} Expect(adapter.StopBackend("agent-1", "llama-backend")).To(Succeed()) diff --git a/core/services/worker/control_routes.go b/core/services/worker/control_routes.go index a31c9faff645..5480f8fb4c90 100644 --- a/core/services/worker/control_routes.go +++ b/core/services/worker/control_routes.go @@ -4,10 +4,8 @@ import ( "context" "encoding/json" "fmt" - "io" "net/http" "sync" - "unicode/utf8" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/workerctl" @@ -35,19 +33,16 @@ import ( // the bucket reserved for a broken link. Only a failure to read or route the // request is a non-2xx. -// maxControlRequestBytes bounds a control request body. -// -// The largest real body is BackendInstallRequest.BackendGalleries, a serialized -// gallery list of a few hundred kilobytes. Eight megabytes is therefore not a -// size the protocol needs: it is a defence against a body that never ends, -// arriving on a boundary this worker now serves. -const maxControlRequestBytes = 8 << 20 - -// maxEchoedPathBytes bounds how much of an unknown control path the 404 body -// repeats back. The path is caller-controlled and the answer exists to be read -// in a log line, so a caller cannot make this worker echo a request-sized -// string into one. -const maxEchoedPathBytes = 128 +// The request bounds and the unknown-path answer are workerctl's, not this +// package's. An agent worker mounts verbs under the same prefix behind the same +// bearer check, and two workers with private copies of "how big may a body be" +// and "what does an unmounted verb answer" is the rule-at-N-sites-pinned-at-one +// shape phase 3 kept finding. Aliased rather than re-typed at every use so the +// specs that reason about the cap keep naming it locally. +const ( + maxControlRequestBytes = workerctl.MaxRequestBytes + maxEchoedPathBytes = workerctl.MaxEchoedPathBytes +) // installFunc and upgradeFunc are the shapes of the two long-running verbs. // They exist as named types so the fields that override them below read as one @@ -168,7 +163,7 @@ func (s *backendSupervisor) RegisterControlRoutes(mux *http.ServeMux) { // The catch-all. A path under the control prefix that no verb claims is // a frontend newer than this worker, and the body says so, because a // bare 404 through a tunnel is indistinguishable from a proxy fault. - http.Error(w, "unknown worker control path "+truncate(r.URL.Path, maxEchoedPathBytes), http.StatusNotFound) + workerctl.WriteUnknownPath(w, r) }) } @@ -212,45 +207,14 @@ func postControlVerb(mux *http.ServeMux, path string, h controlVerb) { }) } -// readControlBody enforces the two things every control verb requires of a -// request: that it is a POST, and that its body is bounded. +// readControlBody is workerctl.ReadRequestBody under this package's own name. // -// A GET is refused rather than served because a control verb is a command, and -// a liveness probe, a link prefetch or a browser address bar must not be able -// to stop a node. +// The indirection is one line and it earns it: every verb in this file reads +// its body through this name, so the shared rule has exactly one call shape +// here rather than fifteen import-qualified ones that a future edit could +// replace individually. func readControlBody(w http.ResponseWriter, r *http.Request) ([]byte, bool) { - if r.Method != http.MethodPost { - http.Error(w, "control verbs are POST only", http.StatusMethodNotAllowed) - return nil, false - } - body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxControlRequestBytes)) - if err != nil { - // A body this worker could not READ is not an answer about any - // backend, so it must not look like one: 400 is what the frontend maps - // onto "the request was rejected", never onto "that model is gone". - http.Error(w, "reading the control request body: "+err.Error(), http.StatusBadRequest) - return nil, false - } - return body, true -} - -// truncate bounds a caller-controlled string that is about to be echoed. -// -// It cuts on a rune boundary. A byte-wise cut can split a multi-byte rune, and -// the half rune then travels as a replacement character through every log and -// UI that reads it; phase 2 shipped exactly that defect on a refusal reason and -// pinned the rule afterwards. utf8.RuneStart is the same predicate the cluster -// package uses for it, so the two are one rule rather than two hand-rolled -// copies that can drift. -func truncate(s string, max int) string { - if len(s) <= max { - return s - } - cut := max - for cut > 0 && !utf8.RuneStart(s[cut]) { - cut-- - } - return s[:cut] + "…" + return workerctl.ReadRequestBody(w, r) } // ndjsonStream writes the Envelope lines of one streaming control response. diff --git a/core/services/worker/control_routes_test.go b/core/services/worker/control_routes_test.go index c75e8548d99d..9c28cebf0920 100644 --- a/core/services/worker/control_routes_test.go +++ b/core/services/worker/control_routes_test.go @@ -515,8 +515,11 @@ var _ = Describe("the worker's HTTP server", func() { } }) - It("mounts every control verb, not just the one this spec reads", func() { - for _, p := range workerctl.AllPaths() { + It("mounts every control verb a BACKEND worker serves, not just the one this spec reads", func() { + // BackendPaths and not AllPaths. The union now includes the agent + // worker's verbs, which this worker deliberately does not mount and + // answers the catch-all 404 for. + for _, p := range workerctl.BackendPaths() { if p == workerctl.PathNodeStop { // Firing it would tear down the worker under the other specs. continue diff --git a/core/services/worker/tunnel.go b/core/services/worker/tunnel.go index fb22b3368594..d6b245d3b1a7 100644 --- a/core/services/worker/tunnel.go +++ b/core/services/worker/tunnel.go @@ -808,6 +808,28 @@ func tunnelServices(cfg *Config, httpBindAddr string) map[string]LocalService { } } +// HTTPOnlyServices is the stream-tag table for a worker that serves only its +// own HTTP server. +// +// An agent worker runs no backend processes, so the grpc tag has nothing to +// route to and is not offered. That is a security statement and not a tidiness +// one: the table IS the tunnel's boundary, and the grpc tag is the only entry +// whose target the frontend gets to influence at all. A worker that cannot +// serve it should not carry the code that would. +// +// It exists as its own function for the reason tunnelServices does: the table +// is the security boundary of the tunnel, and building it inline in a CLI Run +// left it reachable only by starting a worker, which meant it was covered by +// nothing. +// +// httpBindAddr is this worker's own listener, and fixedService ignores whatever +// the frontend names, so nothing derived from the wire reaches a dialler here. +func HTTPOnlyServices(httpBindAddr string) map[string]LocalService { + return map[string]LocalService{ + cluster.StreamTagHTTP: fixedService(loopbackAddr(httpBindAddr)), + } +} + // fixedService routes a tagged stream to one address on this worker, ignoring // whatever the frontend named. // diff --git a/core/services/worker/tunnel_test.go b/core/services/worker/tunnel_test.go index 0930b6f72219..3c68333660d0 100644 --- a/core/services/worker/tunnel_test.go +++ b/core/services/worker/tunnel_test.go @@ -279,6 +279,84 @@ var _ = Describe("Worker tunnel client", func() { }) }) + Describe("the HTTP-only routing table", func() { + // The table an AGENT worker installs. It is the security boundary of + // that worker's tunnel, so it is asserted directly and then over the + // wire, rather than only through whatever starts an agent worker. + + It("offers exactly one tag, the worker's own HTTP server", func() { + // The grpc tag is the only entry whose target the frontend gets to + // influence at all, and an agent worker runs no backend processes + // for it to reach. Offering it would be reachable surface with + // nothing behind it. + table := HTTPOnlyServices("127.0.0.1:9999") + Expect(table).To(HaveLen(1)) + Expect(table).To(HaveKey(cluster.StreamTagHTTP)) + Expect(table).ToNot(HaveKey(cluster.StreamTagGRPC)) + }) + + It("routes the http tag to the worker's own server, ignoring the address the frontend names", func() { + own := echoListener() + DeferCleanup(func() { _ = own.Close() }) + + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { c.Services = HTTPOnlyServices(own.Addr().String()) }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + // A target this worker must not dial. fixedService ignores it. + Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagHTTP, "attacker.invalid:1")).To(Succeed()) + + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + Eventually(reply, "10s").Should(Receive(BeNil())) + + _, err = stream.Write([]byte("own-server")) + Expect(err).ToNot(HaveOccurred()) + buf := make([]byte, len("own-server")) + read := awaitErr(func() error { + _, rerr := io.ReadFull(stream, buf) + return rerr + }) + Eventually(read, "10s").Should(Receive(BeNil())) + Expect(string(buf)).To(Equal("own-server")) + }) + + It("refuses a stream tagged for gRPC as an unknown tag, over the wire", func() { + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { c.Services = HTTPOnlyServices("127.0.0.1:1") }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagGRPC, "127.0.0.1:50051")).To(Succeed()) + + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + var got error + Eventually(reply, "10s").Should(Receive(&got)) + // The tag, not "unavailable". A frontend gives up on the first and + // retries the second, and retrying a tag this worker will never + // serve is a request that can never succeed. + Expect(got).To(MatchError(cluster.ErrStreamTagUnknown)) + Expect(got).ToNot(MatchError(cluster.ErrStreamTargetUnavailable)) + Expect(got).ToNot(MatchError(cluster.ErrStreamRequestInvalid)) + }) + + It("rewrites a wildcard bind onto loopback, so the dial reaches the same listener", func() { + own := echoListenerOn("127.0.0.1:0") + DeferCleanup(func() { _ = own.Close() }) + + frontend = newFakeFrontend(false) + start(func(c *TunnelConfig) { + c.Services = HTTPOnlyServices(fmt.Sprintf("0.0.0.0:%d", portOf(own))) + }) + + stream, err := session().OpenStream(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagHTTP, "")).To(Succeed()) + reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) }) + Eventually(reply, "10s").Should(Receive(BeNil())) + }) + }) + Describe("refusing a stream it cannot serve", func() { // The refusal specs all read with NO deadline, on another goroutine. // See awaitErr: a deadline would be satisfied by a stream that was diff --git a/core/services/workerctl/paths.go b/core/services/workerctl/paths.go index 4af1164d843d..7d31cd9283b2 100644 --- a/core/services/workerctl/paths.go +++ b/core/services/workerctl/paths.go @@ -42,11 +42,26 @@ const ( PathFilesListDir = "/v1/control/files/listdir" ) -// AllPaths returns every control verb's path. +// The verbs an AGENT worker serves. +// +// They live in the same package, under the same prefix and behind the same +// bearer check as the backend worker's, because a worker is a worker to the +// frontend: one control client, one path table, one set of failure meanings. +// What differs is which of them a given worker MOUNTS, which is why the two +// sets are named separately below. +const ( + PathMCPToolExecute = "/v1/control/mcp/tools/execute" + PathMCPDiscovery = "/v1/control/mcp/discovery" + PathAgentExecute = "/v1/control/agent/execute" + PathAgentCancel = "/v1/control/agent/cancel" + PathMCPCIRun = "/v1/control/mcp/ci/run" +) + +// BackendPaths returns every control verb a BACKEND worker serves. // // It exists so a spec can assert a property of the whole set rather than of a // list it re-types, which would go stale the moment a verb is added. -func AllPaths() []string { +func BackendPaths() []string { return []string{ PathBackendInstall, PathBackendUpgrade, @@ -65,6 +80,49 @@ func AllPaths() []string { } } +// AgentPaths returns every control verb an AGENT worker serves. +// +// PathBackendStop is in BOTH sets and that is the point rather than an +// oversight: one path, two implementations, one caller. A backend worker kills +// the process and recycles its port; an agent worker drops the MCP sessions it +// cached for that backend. The frontend issues the same RPC to either and does +// not branch on the node's type to pick a carrier. +// +// PathAgentCancel is named here with nothing mounting it yet. The frontend +// therefore gets the catch-all's 404, which it already reads as "this worker +// does not serve that verb" rather than as absence, and the path is fixed now +// so the two sides cannot disagree about it later. +func AgentPaths() []string { + return []string{ + PathMCPToolExecute, + PathMCPDiscovery, + PathAgentExecute, + PathAgentCancel, + PathMCPCIRun, + PathBackendStop, + } +} + +// AllPaths returns every control verb's path, each once. +// +// It is the union of the two sets above and is what package-wide properties +// (every path under the prefix, no two verbs sharing a path) are asserted +// over. A spec about what a PARTICULAR worker mounts must use BackendPaths or +// AgentPaths instead: asserting mounting over the union would require every +// worker to serve every verb, which is the opposite of what the split is for. +func AllPaths() []string { + seen := make(map[string]bool) + out := make([]string, 0, len(BackendPaths())+len(AgentPaths())) + for _, p := range append(BackendPaths(), AgentPaths()...) { + if seen[p] { + continue + } + seen[p] = true + out = append(out, p) + } + return out +} + // Envelope is one line of a streaming control response. // // Exactly one of the two is set. Zero or more Progress lines are followed by diff --git a/core/services/workerctl/paths_test.go b/core/services/workerctl/paths_test.go index 4d72124ec0c9..d912354cc10f 100644 --- a/core/services/workerctl/paths_test.go +++ b/core/services/workerctl/paths_test.go @@ -32,6 +32,11 @@ var _ = Describe("control plane paths on the wire", func() { Entry("files stage", workerctl.PathFilesStage, "/v1/control/files/stage"), Entry("files temp", workerctl.PathFilesTemp, "/v1/control/files/temp"), Entry("files listdir", workerctl.PathFilesListDir, "/v1/control/files/listdir"), + Entry("mcp tool execute", workerctl.PathMCPToolExecute, "/v1/control/mcp/tools/execute"), + Entry("mcp discovery", workerctl.PathMCPDiscovery, "/v1/control/mcp/discovery"), + Entry("agent execute", workerctl.PathAgentExecute, "/v1/control/agent/execute"), + Entry("agent cancel", workerctl.PathAgentCancel, "/v1/control/agent/cancel"), + Entry("mcp ci run", workerctl.PathMCPCIRun, "/v1/control/mcp/ci/run"), ) It("names the prefix exactly, since the worker mounts its whole control plane behind it", func() { @@ -44,13 +49,13 @@ var _ = Describe("control plane paths on the wire", func() { } }) - It("lists every verb this package names, so none can be dropped from the set", func() { + It("lists every verb a backend worker serves, so none can be dropped from the set", func() { // The claim is bounded on purpose. Go constants are not enumerable, so - // nothing here can see a NEW constant that was never added to AllPaths; + // nothing here can see a NEW constant that was never added to the set; // what this catches is an EXISTING verb going missing from it, which // matters because the prefix check above and the worker's mounting spec - // both iterate AllPaths and would silently stop covering it. - Expect(workerctl.AllPaths()).To(ConsistOf( + // both iterate these and would silently stop covering it. + Expect(workerctl.BackendPaths()).To(ConsistOf( workerctl.PathBackendInstall, workerctl.PathBackendUpgrade, workerctl.PathBackendList, @@ -68,6 +73,37 @@ var _ = Describe("control plane paths on the wire", func() { )) }) + It("lists every verb an agent worker serves", func() { + Expect(workerctl.AgentPaths()).To(ConsistOf( + workerctl.PathMCPToolExecute, + workerctl.PathMCPDiscovery, + workerctl.PathAgentExecute, + workerctl.PathAgentCancel, + workerctl.PathMCPCIRun, + workerctl.PathBackendStop, + )) + }) + + It("is the union of the two worker kinds, with the shared verb counted once", func() { + // backend.stop is in both sets, and AllPaths deduping it is what makes + // the distinctness check below a statement about the path table rather + // than about which set a verb happened to be typed into. A union that + // repeated it would fail that check for a table that is perfectly + // correct. + Expect(workerctl.AllPaths()).To(ContainElements(workerctl.BackendPaths())) + Expect(workerctl.AllPaths()).To(ContainElements(workerctl.AgentPaths())) + Expect(workerctl.AllPaths()).To(HaveLen( + len(workerctl.BackendPaths()) + len(workerctl.AgentPaths()) - 1)) + }) + + It("serves the agent worker's backend.stop on the SAME path the backend worker's is on", func() { + // One path, two implementations, one caller. The frontend's carrier + // split for backend.stop dies on this: it issues the same RPC to either + // kind of worker without branching on the node's type. + Expect(workerctl.AgentPaths()).To(ContainElement(workerctl.PathBackendStop)) + Expect(workerctl.BackendPaths()).To(ContainElement(workerctl.PathBackendStop)) + }) + It("gives each verb a distinct path", func() { seen := map[string]bool{} for _, p := range workerctl.AllPaths() { diff --git a/core/services/workerctl/request.go b/core/services/workerctl/request.go new file mode 100644 index 000000000000..d01843538275 --- /dev/null +++ b/core/services/workerctl/request.go @@ -0,0 +1,85 @@ +package workerctl + +import ( + "io" + "net/http" + "unicode/utf8" +) + +// The rules every worker control verb enforces on a request, and the one answer +// every worker gives for a path it does not serve. +// +// They live here, beside the path table both sides read, because there are now +// TWO kinds of worker mounting verbs under Prefix: a backend worker and an +// agent worker. Phase 3 found the same rule written at three sites in one file +// and pinned at one, which is how a verb ends up reading an unbounded body or +// answering a 500 where the frontend expects a 404. A second worker package +// with its own copy of these rules would be that finding again, across a +// package boundary where it is harder to see. + +// MaxRequestBytes bounds a control request body. +// +// The largest real body is BackendInstallRequest.BackendGalleries, a serialized +// gallery list of a few hundred kilobytes. Eight megabytes is therefore not a +// size the protocol needs: it is a defence against a body that never ends, +// arriving on a boundary a worker now serves. +const MaxRequestBytes = 8 << 20 + +// MaxEchoedPathBytes bounds how much of an unknown control path the 404 body +// repeats back. The path is caller-controlled and the answer exists to be read +// in a log line, so a caller cannot make a worker echo a request-sized string +// into one. +const MaxEchoedPathBytes = 128 + +// ReadRequestBody enforces the two things every control verb requires of a +// request: that it is a POST, and that its body is bounded. It writes the +// refusal itself and reports false when it did. +// +// A GET is refused rather than served because a control verb is a command, and +// a liveness probe, a link prefetch or a browser address bar must not be able +// to stop a node. +func ReadRequestBody(w http.ResponseWriter, r *http.Request) ([]byte, bool) { + if r.Method != http.MethodPost { + http.Error(w, "control verbs are POST only", http.StatusMethodNotAllowed) + return nil, false + } + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, MaxRequestBytes)) + if err != nil { + // A body a worker could not READ is not an answer about any backend, so + // it must not look like one: 400 is what the frontend maps onto "the + // request was rejected", never onto "that model is gone". + http.Error(w, "reading the control request body: "+err.Error(), http.StatusBadRequest) + return nil, false + } + return body, true +} + +// WriteUnknownPath answers a path under Prefix that this worker mounts no +// handler for. +// +// The body says what happened, because a bare 404 through a tunnel is +// indistinguishable from a proxy fault, and the frontend reads this exact +// status as "this worker does not serve that verb" +// (nodes.ErrWorkerControlUnsupported) rather than as the worker being absent. +func WriteUnknownPath(w http.ResponseWriter, r *http.Request) { + http.Error(w, "unknown worker control path "+truncate(r.URL.Path, MaxEchoedPathBytes), http.StatusNotFound) +} + +// truncate bounds a caller-controlled string that is about to be echoed. +// +// It cuts on a rune boundary. A byte-wise cut can split a multi-byte rune, and +// the half rune then travels as a replacement character through every log and +// UI that reads it; phase 2 shipped exactly that defect on a refusal reason and +// pinned the rule afterwards. utf8.RuneStart is the same predicate the cluster +// package uses for it, so the two are one rule rather than two hand-rolled +// copies that can drift. +func truncate(s string, max int) string { + if len(s) <= max { + return s + } + cut := max + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + return s[:cut] + "…" +} diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 5cd5b83763e6..1c6388b8c5e9 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -141,7 +141,11 @@ Unlike the agent worker's API key and its NATS credential, a tunnel credential * A tunnel credential does not replace `LOCALAI_REGISTRATION_TOKEN`. Without one, node registration itself is unauthenticated, so anyone who can reach the frontend can register a worker and be issued a tunnel credential for it. How far that gets them depends on auto-approve: with auto-approve on the node is healthy at once and the credential works immediately; with it off the node is pending and the credential is inert until an admin approves, so approval is the real gate. LocalAI warns about the missing token at startup. -Only **backend** nodes are issued one. An agent worker has no inbound surface for the tunnel to replace and no client for it, so minting one would widen the credential surface for nothing; its row keeps an empty tunnel credential and the tunnel route refuses it like any other node without one. +Both **backend** and **agent** nodes are issued one. Earlier releases minted a credential only for backend nodes, because nothing dialled into an agent worker; the frontend now reaches an agent worker's MCP control verbs over a tunnel of its own, so an agent worker dials one too. A node whose type is neither has its tunnel credential cleared on every registration rather than merely not renewed, so what refuses it is an empty credential and not a second check that could drift from this one. + +An agent worker's tunnel carries only the `http` tag: it runs no backend processes, so it does not offer the `grpc` tag at all. Its control server binds `127.0.0.1` on a port chosen by the kernel and advertises it nowhere, so an agent worker still opens no inbound port. + +**An agent worker still requires `--nats-url`.** The tunnel is an addition, not a replacement: agent jobs, MCP execution, MCP CI jobs and `nodes..backend.stop` all still travel on NATS. What the tunnel changes today is that the frontend *can* reach an agent worker directly, which is what later releases move those verbs onto. The tunnel lands on exactly one frontend replica, and that replica records itself as the owner of the worker's connection in the `node_connections` table. When the socket dies the claim is dropped, but the row stays behind with no owner and a `disconnected_at` stamp, so a worker that is re-dialling the load balancer can be told from one that has never connected. The row is deleted once that departure is older than ten liveness windows (five minutes). If the replica stalls long enough for its peers to reap it, it re-claims the tunnels it still holds on a live session as soon as it re-registers, skipping any whose socket has already gone. That re-claim needs the replica to have an advertised address: without one it never had an instance row to begin with, and its tunnels are usable only by the replica holding them. @@ -158,7 +162,9 @@ The worker holds the tunnel with one goroutine: it dials, serves the frontend's | Tag | Goes to | Target | |-----|---------|--------| | `grpc` | a backend process on this worker | the port; the host is discarded and only `127.0.0.1` is dialled, within the worker's own backend port range | -| `http` | the worker's own file-transfer and backend-log server | ignored; there is one such server and only the worker knows where it bound | +| `http` | the worker's own file-transfer and backend-log server (an agent worker's control server) | ignored; there is one such server and only the worker knows where it bound | + +An **agent worker** offers only the `http` row. It runs no backend processes, so the `grpc` tag has nothing to route to and a stream that names it is refused as an unknown tag. The `grpc` row is the security boundary of the tunnel, and it is worth being explicit about it. A tunnel terminates inside the worker process, so a stream arriving on it can reach anything the worker can reach; if the frontend could name the host, whoever holds the frontend end could make every worker in the fleet dial arbitrary addresses on its private network. The worker therefore builds the dial address from a constant `127.0.0.1` and a port it has validated, and the string from the wire never reaches the dialler at all. The port range is the one the worker's own allocator hands to backend processes, which by default runs to 65535; setting `LOCALAI_GRPC_MAX_PORT` narrows the allocator and this range together, and a worker with a known backend count should set it. @@ -311,7 +317,7 @@ Registering against an upgraded frontend **clears** a node's `address` and `http A worker on this release opens **no inbound listener on a routable interface**. Its backend gRPC processes and its HTTP file-transfer server all bind loopback, and the frontend reaches both through the tunnel. Concretely: -- **No inbound firewall rule, published port, Service or Ingress is needed for a worker.** A serve-backend worker needs outbound access to the frontend URL (`LOCALAI_REGISTER_TO`), and nothing else - not even to NATS. An agent worker also needs outbound access to `LOCALAI_NATS_URL`. +- **No inbound firewall rule, published port, Service or Ingress is needed for a worker.** A serve-backend worker needs outbound access to the frontend URL (`LOCALAI_REGISTER_TO`), and nothing else - not even to NATS. An agent worker binds only loopback too, and needs outbound access to both `LOCALAI_REGISTER_TO` (registration, heartbeats and now its tunnel) and `LOCALAI_NATS_URL`. - **`LOCALAI_ADVERTISE_ADDR` and `LOCALAI_ADVERTISE_HTTP_ADDR` are gone.** There is nothing to advertise. Both are ignored if still set; remove them. - **`LOCALAI_ADDR` and `LOCALAI_SERVE_ADDR` are read for their port only.** The port is the base of the backend port range, and `port-1` is the HTTP file-transfer port. The host half names an interface nothing binds. - The node's `address` and `http_address` fields in `GET /api/nodes` are empty, and are cleared for nodes that reported them before the upgrade. @@ -322,6 +328,8 @@ When a worker's tunnel goes, the frontend does **not** forget the worker. It rec That distinction exists because a worker loses its tunnel for entirely ordinary reasons. A frontend replica restarting during a rolling upgrade drops every tunnel it held, and each of those workers immediately re-dials the load balancer and lands on another replica. Treating that as "the worker is gone" would evict models mid-upgrade for a fleet that never actually went anywhere. +This applies to **backend** nodes only. An agent worker holds a tunnel and therefore has a `node_connections` row that ages exactly like a backend worker's, and nothing acts on it: an agent worker's real work travels on NATS, so a lost tunnel says nothing about whether it can run a job. The scheduler never sees agent nodes at all (every placement query selects `node_type = 'backend'`), and the health monitor skips them explicitly. + Three things read this. The **scheduler** reads it before it places a cold load: a worker whose departure has outlived the grace is skipped and marked unhealthy, so every other frontend replica stops choosing it too. **LRU eviction** reads it before it hands back the node it freed capacity on, because a node full enough to be an eviction target is exactly the node the placement selectors never offer, so the scheduler's own check never sees it. The **health monitor** reads it on every cycle, which is what covers the case the heartbeat cannot see. A worker's heartbeat says its supervisor is alive; it says nothing about whether anything here can reach that worker's backends, because those are reached over the tunnel. A worker that heartbeats with a permanently dead tunnel (a proxy that stopped upgrading WebSockets, a rotated registration credential, a reconnect loop longer than the grace) is therefore marked unhealthy too, rather than staying listed healthy while every request for a model loaded on it fails "no route to that worker". The log lines are `Scheduled node has no tunnel and its departure outlived the reconnect grace, marking unhealthy and re-scheduling`, `Eviction target has no tunnel and its departure outlived the reconnect grace, marking unhealthy and evicting again`, and `Node is heartbeating but its tunnel has been gone longer than the reconnect grace; marking unhealthy`. Each names the grace it used. @@ -427,7 +435,7 @@ A frontend replica that dies mid-load does not wedge the model: the job row carr ### NATS JWT authentication (recommended for production) -**This section is about agent workers and the frontend.** A serve-backend worker opens no NATS connection, so none of it applies to one; its own credential is the tunnel token it gets at registration, and its control plane is authenticated by `LOCALAI_REGISTRATION_TOKEN`. +**This section is about agent workers and the frontend.** A serve-backend worker opens no NATS connection, so none of it applies to one; its own credential is the tunnel token it gets at registration, and its control plane is authenticated by `LOCALAI_REGISTRATION_TOKEN`. An agent worker now has both: a NATS credential for everything still on the bus, and a tunnel token plus the same `LOCALAI_REGISTRATION_TOKEN` bearer check in front of its control server. By default, NATS connections are anonymous: any client that can reach port `4222` may publish the subjects still carried on it. Those are the agent-worker job subjects, MCP, and the frontend's own cross-replica events. `nodes..backend.install` and its nine siblings are **not** among them - they are HTTP routes on the worker's tunnel, see [The worker control plane](#the-worker-control-plane). Enable JWT auth to scope agent workers to their own subjects and give the frontend a dedicated service credential. diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 0c98421d28fb..861b47f2456a 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -48,11 +48,13 @@ type Options struct { // alongside the backend workers. // // They exist so a spec can hold the two kinds of worker side by side in one - // cluster. An agent worker still speaks NATS and holds no tunnel at all, - // which is exactly the shape the tunnel-departure rules must not act on: it - // has no node_connections row, so its presence is PresenceUnknown forever. - // A spec that asserted only on backend workers could not tell "agent - // workers are unaffected" from "nothing here looks at them". + // cluster. An agent worker still speaks NATS, and now holds a tunnel of its + // own as well, which is exactly the shape the tunnel-departure rules must + // not act on: it gets a real node_connections row whose departure really + // does age past the grace, and demoting it for that would take out a fleet + // whose actual work travels on the bus. A spec that asserted only on + // backend workers could not tell "agent workers are unaffected" from + // "nothing here looks at them". // // They register through the same WorkerFrontendURL hook as backend workers, // so a spec that puts a balancer in front of the fleet gets one for its @@ -394,11 +396,12 @@ func (c *Cluster) startWorker(i int) (*Process, error) { // startAgentWorker starts agent worker i. // // It is `local-ai agent-worker`, not `local-ai worker`, and the difference is -// the whole point of having it here: an agent worker REQUIRES a NATS URL, dials -// no tunnel, and runs no backend, so it is the control for every rule this -// phase added about a worker whose tunnel is gone. It binds nothing, so there -// is no port to reserve and no readiness endpoint to wait on; a spec learns it -// is up by finding it in the roster. +// the whole point of having it here: an agent worker REQUIRES a NATS URL and +// runs no backend, so it is the control for every rule this phase added about a +// worker whose tunnel is gone. It dials a tunnel of its own, but binds only +// loopback for the control plane behind it, so there is still no port to +// reserve and no reachable readiness endpoint to wait on; a spec learns it is +// up by finding it in the roster. func (c *Cluster) startAgentWorker(i int) (*Process, error) { name := agentWorkerName(i) cmd := exec.Command(c.opts.Binary, "agent-worker") diff --git a/tests/e2e/distributed/cluster_control_test.go b/tests/e2e/distributed/cluster_control_test.go index d724f6037d0b..8f9e472a73cc 100644 --- a/tests/e2e/distributed/cluster_control_test.go +++ b/tests/e2e/distributed/cluster_control_test.go @@ -382,8 +382,8 @@ func withReconnectGrace(d time.Duration) func(*cluster.Options) { return func(o *cluster.Options) { o.ReconnectGrace = d } } -// withAgentWorkers adds agent workers, which still speak NATS and hold no -// tunnel, to a cluster. +// withAgentWorkers adds agent workers, which still speak NATS for everything +// this phase has not moved, to a cluster. func withAgentWorkers(n int) func(*cluster.Options) { return func(o *cluster.Options) { o.AgentWorkers = n } } @@ -653,9 +653,11 @@ var _ = Describe("Control plane over the worker tunnel", Label("Distributed"), L // every reaper keyed on the heartbeat and the heartbeat was fine. // // The agent worker in the same cluster is the control for the other - // direction. It still speaks NATS, holds no tunnel and never will, so a - // rule that read "no tunnel" as "gone" would take the whole agent fleet - // down with it. + // direction. It holds a tunnel too now, and the balancer below blocks its + // dial exactly as it blocks the backend worker's, so its departure ages + // past the same grace. Its real work still travels on the bus, so a rule + // that read "no tunnel" as "gone" would take the whole agent fleet down + // with it. It("stops reporting a heartbeating worker healthy once its tunnel is gone past the grace, and leaves agent workers alone", func() { var balancer *frontendBalancer c, dsn := startClusterOnFreshDB(2, 1, withBalancer(&balancer), @@ -703,7 +705,9 @@ var _ = Describe("Control plane over the worker tunnel", Label("Distributed"), L "the worker's heartbeat is stale, so it was demoted for being gone rather than for having no route") // The agent worker, in the same cluster, under the same grace, on the - // same health monitor, is untouched. It holds no tunnel either. + // same health monitor, is untouched. Its tunnel is blocked by the same + // balancer, so this is a node whose departure really has outlived the + // grace and which must still not be demoted for it. Consistently(func() string { return atSurvivor.statusOf(c.AgentWorkerName(0)) }, "20s", "2s"). Should(Equal("healthy"), atSurvivor.explain("an agent worker was demoted by a rule about tunnels, and agent workers never hold one")) diff --git a/tests/e2e/distributed/cluster_tunnel_test.go b/tests/e2e/distributed/cluster_tunnel_test.go index 9301af5dfa69..3da9e385362f 100644 --- a/tests/e2e/distributed/cluster_tunnel_test.go +++ b/tests/e2e/distributed/cluster_tunnel_test.go @@ -383,6 +383,38 @@ var _ = Describe("Worker tunnel end to end", Label("Distributed"), Label("Cluste "a worker with no advertised address must still serve inference over its tunnel") }) + // Scenario 1b. The AGENT worker, which used to be gated out of tunnels at + // the credential mint site and now dials one like any other node. + // + // A wrong implementation is silent in both directions. If the mint gate is + // still closed, the agent worker registers, heartbeats and works exactly as + // before, with its tunnel dial refused 401 forever in a log nothing reads; + // no other spec in this suite would notice. If the absence rules were + // widened to follow, the agent worker would instead start being demoted for + // a tunnel its real work does not travel on. + It("holds a tunnel for an AGENT worker, and still does not judge it by one", func() { + c, dsn := startClusterOnFreshDB(1, 0, withAgentWorkers(1)) + client := controlSession(c) + + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.AgentWorkerName(0)), probe.describe) + nodeID := probe.idOf(c.AgentWorkerName(0)) + Expect(nodeID).ToNot(BeEmpty()) + + // The claim: a live replica holds this agent worker's tunnel. Read + // through the production Owner query, which joins against live + // instances, so a row left by a dead replica is not an owner. + owners := newTunnelOwners(openClusterDB(dsn)) + Eventually(func() int { return owners.ownerIndexOf(c, 1, nodeID) }, tunnelOwnershipTimeout, tunnelOwnershipPoll). + Should(Equal(0), owners.describe) + + // And it is still an agent worker: holding a tunnel changed nothing + // about how this deployment decides whether it is present. + Consistently(func() string { return probe.statusOf(c.AgentWorkerName(0)) }, "10s", "2s"). + Should(Equal("healthy"), probe.describe) + }) + // Scenario 2. With N replicas behind round robin this is (N-1)/N of // production traffic. A wrong implementation answers it by dialling the // worker from the replica that took the request, which works on one host diff --git a/tests/e2e/distributed/node_lifecycle_test.go b/tests/e2e/distributed/node_lifecycle_test.go index e3fdb1eefcb5..1f667f3b563a 100644 --- a/tests/e2e/distributed/node_lifecycle_test.go +++ b/tests/e2e/distributed/node_lifecycle_test.go @@ -177,7 +177,9 @@ var _ = Describe("Node Backend Lifecycle over the worker control plane", Label(" }) // The one node subject left, and it is addressed only to AGENT workers: - // they hold no tunnel and subscribe to it to drop cached MCP sessions. + // they subscribe to it to drop cached MCP sessions. They serve the same + // verb on their tunnel as well; this subject survives because the + // PUBLISHER has not moved onto that route yet. It("should keep the agent worker's backend.stop subject", func() { Expect(messaging.SubjectNodeBackendStop("node-abc")).To(Equal("nodes.node-abc.backend.stop")) }) From 7c7324b7bc6ac34f6a8ebeb7aa7999675f6002e6 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 12:04:41 +0000 Subject: [PATCH 085/153] feat(distributed): make MCP execution and discovery a selection mcp.tools.execute and mcp.discovery were the only NATS subjects that combined a queue group with a reply, and no carrier in this design provides both. They never needed one: a queue group is a way of choosing a subscriber, and choosing is a query. The frontend now lists the approved, non-draining agent nodes, asks the node_connections table in one joined statement which of those tunnels a live replica holds, prefers one this replica holds so the call skips the relay hop, and issues an ordinary control RPC on the path task 4 already mounted. A peer-held tunnel is reached through the relay. That is a choice a broker's hidden balancing could not make. The selection reads presence and nothing else. It is filtered only on node type and on the two statuses an operator controls, never on a health verdict written on another clock, because refusing a worker that is connected and answering is the same defect as picking one that is gone. An empty fleet answers ErrNoAgentWorker, which is deliberately neither ErrWorkerUnroutable nor anything cluster.IsWorkerAnswer accepts: nothing was asked of any worker, so no reap guard may act on it. A reply carrying an Error is the worker's own answer and is returned unchanged; it is never offered to a second worker, which would turn "this MCP server rejected your arguments" into "the fleet is broken" and could run a tool twice. A call that never reached a worker is retried against a different pick, at most three times, and whatever error is finally returned is returned unwrapped so its identity survives the loop. MCP prompts and resources now answer 501 in distributed mode instead of an empty 200. They are served only from sessions the frontend holds, and in distributed mode it holds none. That gap predates the removal of the bus and is not closed by it; this only stops it being silent. Agent workers keep every other subject, including nodes..backend.stop. Their minted JWT loses the two MCP subjects and keeps a non-empty allow list, because NATS reads an empty one as no restriction at all. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/agent_control_wiring.go | 41 ++ core/application/agent_control_wiring_test.go | 89 +++++ core/application/distributed.go | 18 + core/cli/agent_worker.go | 49 +-- core/cli/agent_worker_test.go | 52 +-- core/http/endpoints/anthropic/messages.go | 4 +- core/http/endpoints/localai/mcp.go | 4 +- core/http/endpoints/localai/mcp_local_only.go | 41 ++ .../endpoints/localai/mcp_local_only_test.go | 114 ++++++ core/http/endpoints/localai/mcp_prompts.go | 11 + core/http/endpoints/localai/mcp_resources.go | 10 + core/http/endpoints/localai/mcp_tools.go | 16 +- core/http/endpoints/mcp/executor.go | 45 ++- core/http/endpoints/mcp/remote_test.go | 145 +++++++ core/http/endpoints/mcp/tools.go | 96 +++-- core/http/endpoints/openai/chat.go | 4 +- .../http/endpoints/openresponses/responses.go | 4 +- core/http/routes/agent_control.go | 38 ++ .../routes/agent_control_internal_test.go | 36 ++ core/http/routes/anthropic.go | 8 +- core/http/routes/localai.go | 10 +- core/http/routes/openai.go | 10 +- core/http/routes/openresponses.go | 9 +- core/services/agentworker/handlers.go | 2 + core/services/cluster/connected_among_test.go | 173 ++++++++ core/services/cluster/ownership.go | 92 +++++ core/services/cluster/ownership_test.go | 8 + core/services/mcp/remote.go | 26 +- core/services/messaging/subjects.go | 7 - core/services/nodes/agent_control.go | 137 +++++++ core/services/nodes/agent_control_test.go | 375 ++++++++++++++++++ core/services/nodes/agent_selector.go | 171 ++++++++ core/services/nodes/agent_selector_test.go | 271 +++++++++++++ docs/content/features/distributed-mode.md | 33 +- pkg/natsauth/permissions.go | 10 +- pkg/natsauth/permissions_coverage_test.go | 24 +- tests/e2e/distributed/mcp_nats_test.go | 177 --------- tests/e2e/distributed/nats_jwt_test.go | 11 +- .../e2e/distributed/nats_queue_reply_test.go | 77 ++++ 39 files changed, 2048 insertions(+), 400 deletions(-) create mode 100644 core/application/agent_control_wiring.go create mode 100644 core/application/agent_control_wiring_test.go create mode 100644 core/http/endpoints/localai/mcp_local_only.go create mode 100644 core/http/endpoints/localai/mcp_local_only_test.go create mode 100644 core/http/endpoints/mcp/remote_test.go create mode 100644 core/http/routes/agent_control.go create mode 100644 core/http/routes/agent_control_internal_test.go create mode 100644 core/services/cluster/connected_among_test.go create mode 100644 core/services/nodes/agent_control.go create mode 100644 core/services/nodes/agent_control_test.go create mode 100644 core/services/nodes/agent_selector.go create mode 100644 core/services/nodes/agent_selector_test.go delete mode 100644 tests/e2e/distributed/mcp_nats_test.go create mode 100644 tests/e2e/distributed/nats_queue_reply_test.go diff --git a/core/application/agent_control_wiring.go b/core/application/agent_control_wiring.go new file mode 100644 index 000000000000..f991ee87505f --- /dev/null +++ b/core/application/agent_control_wiring.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT + +package application + +import ( + "fmt" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/nodes" +) + +// newAgentControl builds the frontend's agent control client: the SELECTION +// that decides which agent worker answers a verb, and the control client that +// carries the verb to it. +// +// A named function rather than two more lines in initDistributed, for the +// reason distributedSchedulerOptions is one: initDistributed opens a database +// and a bus, so no unit spec reaches it, and the argument that matters here has +// no symptom when it is wrong. An empty instance id makes every connection read +// report nothing as held by this replica, so every MCP call takes a relay hop +// through a peer even when this replica holds the worker's tunnel itself, and +// nothing anywhere says so: the calls all succeed, just through one more +// process than they need. It is refused here instead of shipped as latency. +// +// The other two refusals are the ordinary kind. A selector with no registry has +// nothing to select from and a client with no transport reaches nobody, and +// both would present as MCP being quietly unavailable in a deployment that +// looks healthy. +func newAgentControl(cfg config.DistributedConfig, registry *nodes.NodeRegistry, + conns nodes.AgentConnectionReader, control *nodes.ControlClient) (*nodes.AgentControlClient, error) { + if cfg.InstanceID == "" { + return nil, fmt.Errorf("the agent control client was built with no instance id: every MCP call would relay through a peer even for a worker whose tunnel this replica holds") + } + if registry == nil || conns == nil { + return nil, fmt.Errorf("the agent control client was built with no way to find a connected agent worker") + } + if control == nil { + return nil, fmt.Errorf("the agent control client was built with no control transport to reach an agent worker over") + } + return nodes.NewAgentControlClient(nodes.NewAgentSelector(registry, conns, cfg.InstanceID), control), nil +} diff --git a/core/application/agent_control_wiring_test.go b/core/application/agent_control_wiring_test.go new file mode 100644 index 000000000000..d8c1a02268d9 --- /dev/null +++ b/core/application/agent_control_wiring_test.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: MIT + +package application + +import ( + "context" + "runtime" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + mcpremote "github.com/mudler/LocalAI/core/services/mcp" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// recordingConnections captures the owner id the selection was made with. It is +// how this spec sees the one argument whose loss has no other symptom. +type recordingConnections struct{ owners []string } + +func (r *recordingConnections) ConnectedAmong(_ context.Context, _ []string, owner string) ([]string, []string, error) { + r.owners = append(r.owners, owner) + return nil, nil, nil +} + +// The wiring that connects MCP to the agent workers, guarded the way the +// absence wiring is and for the same reason: initDistributed opens a database +// and a bus, so no unit spec reaches the construction literal, and one of these +// arguments is silent when it is wrong. +var _ = Describe("building the frontend's agent control client", func() { + var registry *nodes.NodeRegistry + var ctx context.Context + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + ctx = context.Background() + var err error + registry, err = nodes.NewNodeRegistry(testutil.SetupTestDB()) + Expect(err).ToNot(HaveOccurred()) + }) + + // Every refusal below is given a valid value for everything except the one + // argument it is about, so no assertion can be satisfied by a guard that + // fires for the wrong reason. + It("makes the selection with THIS replica's instance id", func() { + // The silent one. With an empty id every connection read reports + // nothing as held here, so every MCP call relays through a peer even + // for a worker whose tunnel this replica holds: correct answers, one + // extra hop, and no log line anywhere. + Expect(registry.Register(ctx, &nodes.BackendNode{ + Name: "agent-1", NodeType: nodes.NodeTypeAgent, Address: "a:50051", + }, true)).To(Succeed()) + + conns := &recordingConnections{} + client, err := newAgentControl( + config.DistributedConfig{InstanceID: "replica-7"}, registry, conns, + nodes.NewControlClient(nil, "token")) + Expect(err).ToNot(HaveOccurred()) + + // Driven through a real call rather than read off a field: what has to + // be true is that the id reaches the SELECTION, not that it was stored. + _, _ = client.ExecuteMCPTool(ctx, mcpremote.MCPToolRequest{ModelName: "m"}) + Expect(conns.owners).To(ConsistOf("replica-7")) + }) + + It("refuses to build with no instance id", func() { + _, err := newAgentControl(config.DistributedConfig{}, registry, &recordingConnections{}, + nodes.NewControlClient(nil, "token")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("instance id")) + }) + + It("refuses to build with nothing to read connections through", func() { + _, err := newAgentControl(config.DistributedConfig{InstanceID: "replica-7"}, registry, nil, + nodes.NewControlClient(nil, "token")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("connected agent worker")) + }) + + It("refuses to build with no control transport", func() { + _, err := newAgentControl(config.DistributedConfig{InstanceID: "replica-7"}, registry, + &recordingConnections{}, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("control transport")) + }) +}) diff --git a/core/application/distributed.go b/core/application/distributed.go index 1252199fd139..763f69df7fbe 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -84,6 +84,13 @@ type DistributedServices struct { // WorkerDialer. Exposed so the model store built in startup.go reaches // remote models the same way every other caller does. BackendClients nodes.BackendClientFactory + // AgentControl carries the frontend's MCP verbs to whichever agent worker + // holds a tunnel this deployment can reach. It is what the chat, responses, + // messages and MCP endpoints reach an agent worker through; a nil one means + // this frontend cannot run MCP at all, which is why initDistributed refuses + // to come up without it rather than leaving the endpoints to discover it + // one request at a time. + AgentControl *nodes.AgentControlClient shutdownOnce sync.Once } @@ -450,6 +457,16 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // second client would open its own and the two would never share one. controlClient := nodes.NewControlClient(workerHTTPDialer, cfg.Distributed.RegistrationToken) + // The caller the agent worker's control plane has been waiting for. MCP + // execution and discovery used to be a NATS request onto a queue group, + // where the bus chose the worker and neither side could say which one had + // answered; they are now a query against the connection rows plus an + // ordinary control RPC over the chosen worker's tunnel. + agentControl, err := newAgentControl(cfg.Distributed, registry, clusterRegistry, controlClient) + if err != nil { + return nil, fmt.Errorf("wiring the agent control client: %w", err) + } + // Create FileStager for distributed file transfer var fileStager nodes.FileStager if cfg.Distributed.StorageURL != "" { @@ -695,6 +712,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade Tunnels: tunnels, WorkerDialer: workerDialer, BackendClients: backendClients, + AgentControl: agentControl, Bus: bus, } // Checked once, here, on the assembled struct. See requireBroadcastCarrier. diff --git a/core/cli/agent_worker.go b/core/cli/agent_worker.go index fa49f606c7d6..0b2ee70e2a2c 100644 --- a/core/cli/agent_worker.go +++ b/core/cli/agent_worker.go @@ -178,11 +178,14 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { // The tunnel, and the loopback control plane behind it. // - // ADDED to this worker rather than swapping anything out: every verb below - // still arrives on NATS, and will until the tasks that move them land. What - // this buys today is that the frontend can reach an agent worker by RPC at - // all, on the same carrier and with the same failure vocabulary a backend - // worker already uses, without the agent worker opening an inbound port. + // This is now the ONLY way MCP tool execution and discovery reach this + // worker: their queue-group subjects are gone, because a queue group was + // only ever a way of SELECTING a worker and the frontend makes that + // selection itself (nodes.AgentSelector). The remaining verbs below still + // arrive on NATS and will until the tasks that move them land. + // + // The worker opens no inbound port for any of it: it dials out and the + // control plane rides the tunnel it holds. // // The credential is read through credMgr rather than captured from res, // because every re-registration the manager performs ROTATES it and a @@ -191,9 +194,9 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { // // It is started AFTER registration, which is what supplies both the node // identity the dial names and the credential it presents, and BEFORE the - // NATS subscriptions, so that a frontend that reaches this worker over the - // tunnel finds its verbs mounted rather than a 404 it would read as a - // version skew. + // remaining NATS subscriptions, so that a frontend that reaches this worker + // over the tunnel finds its verbs mounted rather than a 404 it would read + // as a version skew. agentCtl, err := agentworker.Start(shutdownCtx, agentworker.Options{ FrontendURL: cmd.RegisterTo, NodeID: nodeID, @@ -244,19 +247,6 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { return fmt.Errorf("starting dispatcher: %w", err) } - // Subscribe to MCP tool execution requests (load-balanced across workers). - // The frontend routes model-level MCP tool calls here via NATS request-reply. - if _, err := natsClient.QueueSubscribeReply(messaging.SubjectMCPToolExecute, messaging.QueueAgentWorkers, - replyOverNATS(messaging.SubjectMCPToolExecute, serveMCPToolRequest)); err != nil { - return fmt.Errorf("subscribing to %s: %w", messaging.SubjectMCPToolExecute, err) - } - - // Subscribe to MCP discovery requests (load-balanced across workers). - if _, err := natsClient.QueueSubscribeReply(messaging.SubjectMCPDiscovery, messaging.QueueAgentWorkers, - replyOverNATS(messaging.SubjectMCPDiscovery, serveMCPDiscoveryRequest)); err != nil { - return fmt.Errorf("subscribing to %s: %w", messaging.SubjectMCPDiscovery, err) - } - // Subscribe to MCP CI job execution (load-balanced across agent workers). // In distributed mode, MCP CI jobs are routed here because the frontend // cannot create MCP sessions (e.g., stdio servers using docker). @@ -456,23 +446,6 @@ func encodeMCPReply(resp any) (json.RawMessage, error) { return out, nil } -// replyOverNATS adapts one of the serve* functions to a NATS request-reply -// subscription, so the bus carries exactly the bytes the tunnel does. -func replyOverNATS(subject string, serve func(context.Context, json.RawMessage) (json.RawMessage, error)) func([]byte, func([]byte)) { - return func(data []byte, reply func([]byte)) { - out, err := serve(context.Background(), data) - if err != nil { - // Nothing is sent. A requester on the bus reads that as a timeout, - // which is the closest the carrier has to "this worker did not - // answer"; inventing a reply body here would put a failure to serve - // into the bucket reserved for the worker's own verdict. - xlog.Error("Agent worker could not serve a bus request", "subject", subject, "error", err) - return - } - reply(out) - } -} - // handleMCPCIJob processes an MCP CI job on the agent worker. // The agent worker can create MCP sessions (has docker) and call the LocalAI API for inference. func handleMCPCIJob(shutdownCtx context.Context, data []byte, apiURL, apiToken string, natsClient messaging.MessagingClient, jobTimeout time.Duration) { diff --git a/core/cli/agent_worker_test.go b/core/cli/agent_worker_test.go index 7fad5a0b4976..1e8e1d7ec5c2 100644 --- a/core/cli/agent_worker_test.go +++ b/core/cli/agent_worker_test.go @@ -11,20 +11,16 @@ import ( "github.com/mudler/LocalAI/core/services/messaging" ) -// The agent worker answers its MCP verbs on two carriers at once: the NATS -// subject it has always answered, and the control route on the tunnel it now -// holds. These specs pin the two properties that stops those drifting apart. +// The agent worker answers its MCP verbs on ONE carrier now: the control route +// on the tunnel it holds. The queue-group subjects these used to arrive on are +// gone, because a queue group was only ever a way of SELECTING a worker, and +// the frontend now makes that selection itself (nodes.AgentSelector). // -// The first is that there is ONE implementation. A frontend that reaches a -// worker over the bus and one that reaches the same worker over its tunnel must -// get the same bytes, because during this migration both are live and which one -// is used is not a decision anybody makes deliberately. -// -// The second is the split between an ANSWER and a FAILURE TO SERVE. A tool that -// ran and failed is the worker's own verdict and travels inside the reply; a -// verb this worker could not serve at all is a Go error, which becomes a -// non-2xx over the tunnel and silence on the bus, and which nothing may read as -// evidence about anything. +// What these specs pin is the split between an ANSWER and a FAILURE TO SERVE. A +// tool that ran and failed is the worker's own verdict and travels inside the +// reply, on a 200; a verb this worker could not serve at all is a Go error, +// which becomes a non-2xx, and which nothing may read as evidence about +// anything. var _ = Describe("The agent worker's MCP verbs", func() { It("answers a tool request it could not decode, rather than failing to serve it", func() { // The decode happened on this worker and its outcome is something the @@ -48,36 +44,6 @@ var _ = Describe("The agent worker's MCP verbs", func() { Expect(resp.Error).To(ContainSubstring("unmarshal error")) }) - It("puts on the bus exactly the bytes the tunnel route returns", func() { - // The one property that keeps the two carriers honest. A second - // implementation for the bus is how a deployment ends up behaving - // differently depending on which one a frontend happened to pick. - request := json.RawMessage(`{"tool_name":`) - overTunnel, err := serveMCPToolRequest(context.Background(), request) - Expect(err).ToNot(HaveOccurred()) - - sent := make(chan []byte, 1) - replyOverNATS("mcp.tools.execute", serveMCPToolRequest)(request, func(b []byte) { sent <- b }) - - var overBus []byte - Eventually(sent).Should(Receive(&overBus)) - Expect(string(overBus)).To(Equal(string(overTunnel))) - }) - - It("sends nothing on the bus when the verb could not be served", func() { - // A requester reads the silence as a timeout, which is the closest the - // bus has to "this worker did not answer". Inventing a reply body would - // put a failure to serve into the bucket reserved for the worker's own - // verdict, which is the collapse this whole phase exists to prevent. - sent := make(chan []byte, 1) - failing := func(context.Context, json.RawMessage) (json.RawMessage, error) { - return nil, context.DeadlineExceeded - } - - replyOverNATS("mcp.tools.execute", failing)(json.RawMessage(`{}`), func(b []byte) { sent <- b }) - - Expect(sent).ToNot(Receive()) - }) }) var _ = Describe("The agent worker's backend stop", func() { diff --git a/core/http/endpoints/anthropic/messages.go b/core/http/endpoints/anthropic/messages.go index 9310cabd93e7..231444dedb80 100644 --- a/core/http/endpoints/anthropic/messages.go +++ b/core/http/endpoints/anthropic/messages.go @@ -29,7 +29,7 @@ import ( // @Param request body schema.AnthropicRequest true "query params" // @Success 200 {object} schema.AnthropicResponse "Response" // @Router /v1/messages [post] -func MessagesEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig, natsClient mcpTools.MCPNATSClient) echo.HandlerFunc { +func MessagesEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig, agentControl mcpTools.AgentControl) echo.HandlerFunc { return func(c echo.Context) error { id := uuid.New().String() @@ -70,7 +70,7 @@ func MessagesEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evalu if (len(mcpServers) > 0 || mcpPromptName != "" || len(mcpResourceURIs) > 0) && (cfg.MCP.Servers != "" || cfg.MCP.Stdio != "") { remote, stdio, mcpErr := cfg.MCP.MCPConfigFromYAML() if mcpErr == nil { - mcpExecutor = mcpTools.NewToolExecutor(c.Request().Context(), natsClient, cfg.Name, remote, stdio, mcpServers) + mcpExecutor = mcpTools.NewToolExecutor(c.Request().Context(), agentControl, cfg.Name, remote, stdio, mcpServers) // Prompt and resource injection (pre-processing step — resolves locally regardless of distributed mode) namedSessions, sessErr := mcpTools.NamedSessionsFromMCPConfig(cfg.Name, remote, stdio, mcpServers) diff --git a/core/http/endpoints/localai/mcp.go b/core/http/endpoints/localai/mcp.go index f3905442dc97..9687afac6ea1 100644 --- a/core/http/endpoints/localai/mcp.go +++ b/core/http/endpoints/localai/mcp.go @@ -57,7 +57,7 @@ type MCPErrorEvent struct { // @Param request body schema.OpenAIRequest true "query params" // @Success 200 {object} schema.OpenAIResponse "Response" // @Router /v1/mcp/chat/completions [post] -func MCPEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig, natsClient mcpTools.MCPNATSClient, compressor middleware.ChatCompressor) echo.HandlerFunc { +func MCPEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig, agentControl mcpTools.AgentControl, compressor middleware.ChatCompressor) echo.HandlerFunc { // The legacy /v1/mcp/chat/completions endpoint never opts into the // in-process LocalAI Assistant tool surface — pass nil holder so the // assistant branch in chat.go is unreachable from this code path. @@ -65,7 +65,7 @@ func MCPEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator // the per-model PII config and is kept for backward compatibility. // The request-side middleware on the main chat route handles // filtering for the standard /v1/chat/completions path. - chatHandler := openai.ChatEndpoint(cl, ml, evaluator, appConfig, natsClient, nil, compressor) + chatHandler := openai.ChatEndpoint(cl, ml, evaluator, appConfig, agentControl, nil, compressor) return func(c echo.Context) error { input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OpenAIRequest) diff --git a/core/http/endpoints/localai/mcp_local_only.go b/core/http/endpoints/localai/mcp_local_only.go new file mode 100644 index 000000000000..281b0f456ea3 --- /dev/null +++ b/core/http/endpoints/localai/mcp_local_only.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT + +package localai + +import ( + "net/http" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" +) + +// mcpLocalSessionsOnly answers the request itself when this frontend cannot +// serve it, and reports whether it did. +// +// MCP prompts and resources are served ONLY from MCP sessions this process +// holds. In distributed mode the frontend holds none: creating them is what an +// agent worker is for, because a stdio server usually means running docker, and +// nothing in this programme carries prompts or resources to one. That is a hole +// that PREDATES the removal of the message bus and is not closed by it; tools +// and discovery had a carrier and these two never did. +// +// Until this task the endpoints did not say so. A model with MCP servers +// configured answered 200 with an empty list, which reads as "this model has no +// prompts" and is a different statement from "this deployment cannot tell you". +// A client cannot act on the first and can act on the second, and an operator +// reading an empty list has no reason to look further. 501 with the reason in +// the body is the smallest honest answer, and it is deliberately NOT a 404 or +// an empty 200: the resource may well exist, on a worker this frontend has no +// verb to ask. +// +// One function for four endpoints, so the four cannot drift into four different +// stories about the same limitation. Each endpoint still has to CALL it, and +// each call is pinned by its own spec. +func mcpLocalSessionsOnly(c echo.Context, appConfig *config.ApplicationConfig, surface string) (bool, error) { + if appConfig == nil || !appConfig.Distributed.Enabled { + return false, nil + } + return true, c.JSON(http.StatusNotImplemented, map[string]string{ + "error": "MCP " + surface + " are not available in distributed mode: they are served only from MCP sessions held by this frontend, and in distributed mode the sessions live on an agent worker, which serves no " + surface + " verb", + }) +} diff --git a/core/http/endpoints/localai/mcp_local_only_test.go b/core/http/endpoints/localai/mcp_local_only_test.go new file mode 100644 index 000000000000..89c91f1162cf --- /dev/null +++ b/core/http/endpoints/localai/mcp_local_only_test.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: MIT + +package localai + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + + "github.com/labstack/echo/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" +) + +// The MCP prompts and resources endpoints work only against sessions THIS +// process holds, and in distributed mode it holds none. That is a hole older +// than the removal of the message bus: tools and discovery had a carrier to an +// agent worker and these two never did, and nothing in this programme gives +// them one. +// +// What these specs pin is that the hole is now HONEST. The old answer was 200 +// with an empty list, which reads as "this model has no prompts": a client +// cannot act on it, and an operator reading it has no reason to look further. +// +// Each endpoint is asserted separately on purpose. The refusal is one function +// but four CALLS, and a call site that lost its call would still leave the +// other three green. +var _ = Describe("MCP prompts and resources in distributed mode", func() { + distributed := &config.ApplicationConfig{} + distributed.Distributed.Enabled = true + + standalone := &config.ApplicationConfig{} + + // loader with no models at all. Every endpoint below looks a model up, and + // the refusal has to come FIRST: a 404 for an unknown model would be a + // different, and wrong, story about the same request. + emptyLoader := config.NewModelConfigLoader("") + + call := func(h echo.HandlerFunc, method, path, body string, params map[string]string) *httptest.ResponseRecorder { + GinkgoHelper() + e := echo.New() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + names := make([]string, 0, len(params)) + values := make([]string, 0, len(params)) + for k, v := range params { + names = append(names, k) + values = append(values, v) + } + c.SetParamNames(names...) + c.SetParamValues(values...) + Expect(h(c)).To(Succeed()) + return rec + } + + // assertRefused checks the STATUS and the BODY. A status-only assertion + // cannot tell a 501 with an empty body from the empty 200 this replaces, + // and the body is the whole reason the answer is worth giving. + assertRefused := func(rec *httptest.ResponseRecorder, surface string) { + GinkgoHelper() + Expect(rec.Code).To(Equal(http.StatusNotImplemented)) + var body map[string]string + Expect(json.Unmarshal(rec.Body.Bytes(), &body)).To(Succeed()) + Expect(body["error"]).To(ContainSubstring(surface)) + Expect(body["error"]).To(ContainSubstring("distributed mode")) + Expect(body["error"]).To(ContainSubstring("agent worker")) + } + + It("refuses to list prompts rather than answering with an empty list", func() { + assertRefused(call(MCPPromptsEndpoint(emptyLoader, distributed), + http.MethodGet, "/v1/mcp/prompts/m", "", map[string]string{"model": "m"}), "prompts") + }) + + It("refuses to expand a prompt", func() { + assertRefused(call(MCPGetPromptEndpoint(emptyLoader, distributed), + http.MethodPost, "/v1/mcp/prompts/m/p", `{"arguments":{}}`, + map[string]string{"model": "m", "prompt": "p"}), "prompts") + }) + + It("refuses to list resources rather than answering with an empty list", func() { + assertRefused(call(MCPResourcesEndpoint(emptyLoader, distributed), + http.MethodGet, "/v1/mcp/resources/m", "", map[string]string{"model": "m"}), "resources") + }) + + It("refuses to read a resource", func() { + assertRefused(call(MCPReadResourceEndpoint(emptyLoader, distributed), + http.MethodPost, "/v1/mcp/resources/m/read", `{"uri":"file:///x"}`, + map[string]string{"model": "m"}), "resources") + }) + + It("does not refuse in a standalone deployment, where the sessions are here", func() { + // The negative control. A guard that fired unconditionally would pass + // every assertion above and break MCP prompts for every single-binary + // deployment, which is where they actually work. + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/v1/mcp/prompts/m", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + c.SetParamNames("model") + c.SetParamValues("m") + + err := MCPPromptsEndpoint(emptyLoader, standalone)(c) + // The model does not exist, so this fails on the lookup. What matters + // is that it got PAST the guard: a 501 here would mean the guard fired + // with distributed mode off. + Expect(err).To(HaveOccurred()) + Expect(rec.Code).ToNot(Equal(http.StatusNotImplemented)) + }) +}) diff --git a/core/http/endpoints/localai/mcp_prompts.go b/core/http/endpoints/localai/mcp_prompts.go index 8f04ee6c8988..b9cc351bde05 100644 --- a/core/http/endpoints/localai/mcp_prompts.go +++ b/core/http/endpoints/localai/mcp_prompts.go @@ -17,6 +17,13 @@ func MCPPromptsEndpoint(cl *config.ModelConfigLoader, appConfig *config.Applicat return echo.ErrBadRequest } + // Before the model lookup and before the empty-config shortcut: both + // answer 200 with an empty list, which is the very answer this refuses + // to give in a deployment that cannot look. + if refused, err := mcpLocalSessionsOnly(c, appConfig, "prompts"); refused { + return err + } + cfg, exists := cl.GetModelConfig(modelName) if !exists { return fmt.Errorf("model %q not found", modelName) @@ -86,6 +93,10 @@ func MCPGetPromptEndpoint(cl *config.ModelConfigLoader, appConfig *config.Applic return echo.ErrBadRequest } + if refused, err := mcpLocalSessionsOnly(c, appConfig, "prompts"); refused { + return err + } + cfg, exists := cl.GetModelConfig(modelName) if !exists { return fmt.Errorf("model %q not found", modelName) diff --git a/core/http/endpoints/localai/mcp_resources.go b/core/http/endpoints/localai/mcp_resources.go index 0cacec3c1c0c..0683362a340b 100644 --- a/core/http/endpoints/localai/mcp_resources.go +++ b/core/http/endpoints/localai/mcp_resources.go @@ -17,6 +17,12 @@ func MCPResourcesEndpoint(cl *config.ModelConfigLoader, appConfig *config.Applic return echo.ErrBadRequest } + // See the prompts endpoint: this has to come before the empty-config + // shortcut, which answers with the empty list this refuses to invent. + if refused, err := mcpLocalSessionsOnly(c, appConfig, "resources"); refused { + return err + } + cfg, exists := cl.GetModelConfig(modelName) if !exists { return fmt.Errorf("model %q not found", modelName) @@ -73,6 +79,10 @@ func MCPReadResourceEndpoint(cl *config.ModelConfigLoader, appConfig *config.App return echo.ErrBadRequest } + if refused, err := mcpLocalSessionsOnly(c, appConfig, "resources"); refused { + return err + } + cfg, exists := cl.GetModelConfig(modelName) if !exists { return fmt.Errorf("model %q not found", modelName) diff --git a/core/http/endpoints/localai/mcp_tools.go b/core/http/endpoints/localai/mcp_tools.go index f5db27bd7003..86ca3bdc69de 100644 --- a/core/http/endpoints/localai/mcp_tools.go +++ b/core/http/endpoints/localai/mcp_tools.go @@ -13,7 +13,7 @@ import ( // MCPServersEndpoint returns the list of MCP servers and their tools for a given model. // GET /v1/mcp/servers/:model -func MCPServersEndpoint(cl *config.ModelConfigLoader, appConfig *config.ApplicationConfig, natsClient mcpTools.MCPNATSClient) echo.HandlerFunc { +func MCPServersEndpoint(cl *config.ModelConfigLoader, appConfig *config.ApplicationConfig, agentControl mcpTools.AgentControl) echo.HandlerFunc { return func(c echo.Context) error { modelName := c.Param("model") if modelName == "" { @@ -45,10 +45,10 @@ func MCPServersEndpoint(cl *config.ModelConfigLoader, appConfig *config.Applicat }) } - // In distributed mode, route discovery through NATS to an agent worker + // In distributed mode, ask an agent worker over the tunnel it holds // that can actually connect to the MCP servers. - if natsClient != nil { - resp, err := mcpTools.DiscoverMCPToolsRemote(c.Request().Context(), natsClient, cfg.Name, remote, stdio) + if agentControl != nil { + resp, err := mcpTools.DiscoverMCPToolsRemote(c.Request().Context(), agentControl, cfg.Name, remote, stdio) if err != nil { return c.JSON(http.StatusOK, map[string]any{ "model": modelName, @@ -80,7 +80,7 @@ func MCPServersEndpoint(cl *config.ModelConfigLoader, appConfig *config.Applicat // MCPServersEndpointFromMiddleware is a version that uses the middleware-resolved model config. // This allows it to use the same middleware chain as other endpoints. -func MCPServersEndpointFromMiddleware(natsClient mcpTools.MCPNATSClient) echo.HandlerFunc { +func MCPServersEndpointFromMiddleware(agentControl mcpTools.AgentControl) echo.HandlerFunc { return func(c echo.Context) error { cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) if !ok || cfg == nil { @@ -103,9 +103,9 @@ func MCPServersEndpointFromMiddleware(natsClient mcpTools.MCPNATSClient) echo.Ha }) } - // In distributed mode, route discovery through NATS to an agent worker. - if natsClient != nil { - resp, err := mcpTools.DiscoverMCPToolsRemote(c.Request().Context(), natsClient, cfg.Name, remote, stdio) + // In distributed mode, ask an agent worker over the tunnel it holds. + if agentControl != nil { + resp, err := mcpTools.DiscoverMCPToolsRemote(c.Request().Context(), agentControl, cfg.Name, remote, stdio) if err != nil { return c.JSON(http.StatusOK, map[string]any{ "model": cfg.Name, diff --git a/core/http/endpoints/mcp/executor.go b/core/http/endpoints/mcp/executor.go index 9f9b279d6a07..11dcd45746f5 100644 --- a/core/http/endpoints/mcp/executor.go +++ b/core/http/endpoints/mcp/executor.go @@ -11,7 +11,8 @@ import ( ) // ToolExecutor abstracts MCP tool discovery and execution. -// Implementations handle local (in-process sessions) vs distributed (NATS) modes. +// Implementations handle local (in-process sessions) vs distributed (an agent +// worker reached over its tunnel) modes. type ToolExecutor interface { // DiscoverTools returns the tool function schemas available from MCP servers. DiscoverTools(ctx context.Context) ([]functions.Function, error) @@ -58,28 +59,29 @@ func (e *LocalToolExecutor) HasTools() bool { return len(e.tools) > 0 } -// DistributedToolExecutor routes tool operations through NATS to agent workers. +// DistributedToolExecutor routes tool operations to agent workers over the +// tunnels they hold. type DistributedToolExecutor struct { - natsClient MCPNATSClient - modelName string - remote config.MCPGenericConfig[config.MCPRemoteServers] - stdio config.MCPGenericConfig[config.MCPSTDIOServers] - toolDefs []mcpRemote.MCPToolDef + agent AgentControl + modelName string + remote config.MCPGenericConfig[config.MCPRemoteServers] + stdio config.MCPGenericConfig[config.MCPSTDIOServers] + toolDefs []mcpRemote.MCPToolDef } -// NewDistributedToolExecutor creates a ToolExecutor that routes through NATS. -// It discovers tools immediately via a NATS request-reply to an agent worker. -func NewDistributedToolExecutor(ctx context.Context, natsClient MCPNATSClient, modelName string, +// NewDistributedToolExecutor creates a ToolExecutor that routes to agent +// workers. It discovers tools immediately with a control RPC to one of them. +func NewDistributedToolExecutor(ctx context.Context, agent AgentControl, modelName string, remote config.MCPGenericConfig[config.MCPRemoteServers], stdio config.MCPGenericConfig[config.MCPSTDIOServers], ) *DistributedToolExecutor { e := &DistributedToolExecutor{ - natsClient: natsClient, - modelName: modelName, - remote: remote, - stdio: stdio, + agent: agent, + modelName: modelName, + remote: remote, + stdio: stdio, } - resp, err := DiscoverMCPToolsRemote(ctx, natsClient, modelName, remote, stdio) + resp, err := DiscoverMCPToolsRemote(ctx, agent, modelName, remote, stdio) if err != nil { xlog.Error("Failed to discover MCP tools (distributed)", "error", err) } else if resp != nil { @@ -103,7 +105,7 @@ func (e *DistributedToolExecutor) IsTool(name string) bool { } func (e *DistributedToolExecutor) ExecuteTool(ctx context.Context, toolName, arguments string) (string, error) { - return ExecuteMCPToolCallRemote(ctx, e.natsClient, e.modelName, e.remote, e.stdio, toolName, arguments) + return ExecuteMCPToolCallRemote(ctx, e.agent, e.modelName, e.remote, e.stdio, toolName, arguments) } func (e *DistributedToolExecutor) HasTools() bool { @@ -111,15 +113,16 @@ func (e *DistributedToolExecutor) HasTools() bool { } // NewToolExecutor creates the appropriate ToolExecutor based on the current mode. -// When natsClient is non-nil, returns a DistributedToolExecutor that routes through NATS. -// When natsClient is nil, creates local sessions and returns a LocalToolExecutor. -func NewToolExecutor(ctx context.Context, natsClient MCPNATSClient, modelName string, +// When agent is non-nil, returns a DistributedToolExecutor that reaches an agent +// worker over its tunnel. When agent is nil, creates local sessions and returns +// a LocalToolExecutor. +func NewToolExecutor(ctx context.Context, agent AgentControl, modelName string, remote config.MCPGenericConfig[config.MCPRemoteServers], stdio config.MCPGenericConfig[config.MCPSTDIOServers], enabledServers []string, ) ToolExecutor { - if natsClient != nil { - return NewDistributedToolExecutor(ctx, natsClient, modelName, remote, stdio) + if agent != nil { + return NewDistributedToolExecutor(ctx, agent, modelName, remote, stdio) } sessions, err := NamedSessionsFromMCPConfig(modelName, remote, stdio, enabledServers) if err != nil || len(sessions) == 0 { diff --git a/core/http/endpoints/mcp/remote_test.go b/core/http/endpoints/mcp/remote_test.go new file mode 100644 index 000000000000..6a82b8f24577 --- /dev/null +++ b/core/http/endpoints/mcp/remote_test.go @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: MIT + +package mcp + +import ( + "context" + "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + mcpRemote "github.com/mudler/LocalAI/core/services/mcp" +) + +// recordingAgent records the context each verb was given, so a spec can read +// the budget the caller applied. +// +// The budget is invisible any other way. It travels as a context deadline, not +// as anything on the wire, so the only place it can be observed is inside the +// call; a spec that waited for it to expire would be a spec that sleeps for +// minutes, and one that read a constant would pass with the constant unused. +type recordingAgent struct { + toolDeadline time.Time + toolHasDL bool + discoDeadline time.Time + discoHasDL bool + + toolReply *mcpRemote.MCPToolResponse + discoReply *mcpRemote.MCPDiscoveryResponse + err error +} + +func (a *recordingAgent) ExecuteMCPTool(ctx context.Context, _ mcpRemote.MCPToolRequest) (*mcpRemote.MCPToolResponse, error) { + a.toolDeadline, a.toolHasDL = ctx.Deadline() + if a.err != nil { + return nil, a.err + } + return a.toolReply, nil +} + +func (a *recordingAgent) DiscoverMCPTools(ctx context.Context, _ mcpRemote.MCPDiscoveryRequest) (*mcpRemote.MCPDiscoveryResponse, error) { + a.discoDeadline, a.discoHasDL = ctx.Deadline() + if a.err != nil { + return nil, a.err + } + return a.discoReply, nil +} + +var ( + noRemote = config.MCPGenericConfig[config.MCPRemoteServers]{} + noStdio = config.MCPGenericConfig[config.MCPSTDIOServers]{} +) + +var _ = Describe("Routing MCP to an agent worker", func() { + ctx := context.Background() + + Describe("with no agent control client wired", func() { + // A frontend in distributed mode with nothing to reach an agent worker + // through must say so. The alternative shape, which this replaces, was + // a nil-pointer dereference inside a chat request. + It("names the missing client rather than dialling nothing", func() { + _, err := ExecuteMCPToolCallRemote(ctx, nil, "m", noRemote, noStdio, "weather", "{}") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("agent control client")) + }) + + It("names it for discovery too", func() { + _, err := DiscoverMCPToolsRemote(ctx, nil, "m", noRemote, noStdio) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("agent control client")) + }) + }) + + Describe("the budget it gives the call", func() { + // Two verbs, two constants, two assertions. The rule is written at both + // sites, so a single spec would leave whichever site it did not cover + // free to lose its deadline: a tool call whose worker went quiet would + // then hold the caller until the tunnel's keepalive noticed. + // + // Asserted as a WINDOW around the deadline rather than an equality, + // because the deadline is stamped from a clock this spec does not hold. + // The window is far tighter than the difference between the two + // constants, so a call given the wrong one still fails here. + const slack = 5 * time.Second + + It("bounds a tool call by the documented tool timeout", func() { + agent := &recordingAgent{toolReply: &mcpRemote.MCPToolResponse{Result: "ok"}} + start := time.Now() + + out, err := ExecuteMCPToolCallRemote(ctx, agent, "m", noRemote, noStdio, "weather", `{"city":"London"}`) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(Equal("ok")) + + Expect(agent.toolHasDL).To(BeTrue(), "the tool call was given no deadline at all") + Expect(agent.toolDeadline).To(BeTemporally("~", start.Add(config.DefaultMCPToolTimeout), slack)) + }) + + It("bounds discovery by the documented discovery timeout", func() { + agent := &recordingAgent{discoReply: &mcpRemote.MCPDiscoveryResponse{}} + start := time.Now() + + _, err := DiscoverMCPToolsRemote(ctx, agent, "m", noRemote, noStdio) + Expect(err).ToNot(HaveOccurred()) + + Expect(agent.discoHasDL).To(BeTrue(), "discovery was given no deadline at all") + Expect(agent.discoDeadline).To(BeTemporally("~", start.Add(config.DefaultMCPDiscoveryTimeout), slack)) + }) + + It("does not extend a caller's shorter deadline", func() { + // The caller's own budget wins. context.WithTimeout keeps the + // earlier of the two, and a hand-rolled deadline would not. + short, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + agent := &recordingAgent{toolReply: &mcpRemote.MCPToolResponse{Result: "ok"}} + + _, err := ExecuteMCPToolCallRemote(short, agent, "m", noRemote, noStdio, "weather", "{}") + Expect(err).ToNot(HaveOccurred()) + Expect(agent.toolDeadline).To(BeTemporally("<", time.Now().Add(config.DefaultMCPToolTimeout))) + }) + }) + + It("passes the agent's failure through with its identity intact", func() { + // The classification is the control client's, and re-deciding it here + // would be the same rule in two places. What this pins is that + // wrapping it for a human does not hide it from errors.Is. + boom := errors.New("the fleet said no") + agent := &recordingAgent{err: boom} + + _, err := ExecuteMCPToolCallRemote(ctx, agent, "m", noRemote, noStdio, "weather", "{}") + Expect(err).To(MatchError(boom)) + _, err = DiscoverMCPToolsRemote(ctx, agent, "m", noRemote, noStdio) + Expect(err).To(MatchError(boom)) + }) + + It("refuses tool arguments that are not JSON before reaching for a worker", func() { + agent := &recordingAgent{toolReply: &mcpRemote.MCPToolResponse{Result: "ok"}} + + _, err := ExecuteMCPToolCallRemote(ctx, agent, "m", noRemote, noStdio, "weather", "{not json") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid tool arguments JSON")) + Expect(agent.toolHasDL).To(BeFalse(), "no worker should have been asked") + }) +}) diff --git a/core/http/endpoints/mcp/tools.go b/core/http/endpoints/mcp/tools.go index 0b4931a0539c..f6524bf1a16f 100644 --- a/core/http/endpoints/mcp/tools.go +++ b/core/http/endpoints/mcp/tools.go @@ -16,7 +16,6 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/schema" mcpRemote "github.com/mudler/LocalAI/core/services/mcp" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/pkg/functions" "github.com/mudler/LocalAI/pkg/httpclient" @@ -100,9 +99,23 @@ var ( client = mcp.NewClient(&mcp.Implementation{Name: "LocalAI", Version: "v1.0.0"}, nil) ) -// MCPNATSClient is the interface for NATS request-reply operations needed by MCP routing. -type MCPNATSClient interface { - Request(subject string, data []byte, timeout time.Duration) ([]byte, error) +// AgentControl is the frontend's port onto the MCP verbs an agent worker serves +// over the tunnel it holds. *nodes.AgentControlClient is the only production +// implementation, and it is what decides WHICH agent worker answers. +// +// The contract a caller here relies on, and the reason these functions do no +// classification of their own: an implementation returns the worker's own Error +// field AS A GO ERROR, so a nil error means the verb succeeded. Re-reading the +// Error field at every call site would be the same rule written twice, and the +// copy that gets forgotten is the one that reports a failed tool call as an +// empty success. +// +// It replaces an interface over NATS request-reply. What that carried was a +// subject and a queue group, which between them chose a worker; the choosing is +// now a query and the carrying is an ordinary control RPC. +type AgentControl interface { + ExecuteMCPTool(ctx context.Context, req mcpRemote.MCPToolRequest) (*mcpRemote.MCPToolResponse, error) + DiscoverMCPTools(ctx context.Context, req mcpRemote.MCPDiscoveryRequest) (*mcpRemote.MCPDiscoveryResponse, error) } // MetadataKeyLocalAIAssistant is the request-metadata key the chat handler @@ -510,18 +523,28 @@ func ExecuteMCPToolCall(ctx context.Context, tools []MCPToolInfo, toolName strin return string(combined), nil } -// ExecuteMCPToolCallRemote routes an MCP tool execution request to an agent worker via NATS. -// Used in distributed mode when the frontend doesn't hold MCP sessions locally. +// ExecuteMCPToolCallRemote runs one MCP tool on an agent worker. +// +// Used in distributed mode, where the frontend holds no MCP sessions of its +// own: an agent worker is what can create them (stdio servers under docker), +// so the frontend serialises the model's MCP configuration and asks one. +// +// The budget is applied HERE, as a context deadline, and that is the whole of +// the change in where it lives. On the bus it was the request-reply timeout, +// which was the only thing bounding a worker that never answered; the control +// RPC carries no deadline of its own (see nodes.ControlClient.clientFor), so +// without this a tool call whose worker went quiet holds the caller until the +// tunnel's own keepalive notices, which is far longer than any caller expects. func ExecuteMCPToolCallRemote( ctx context.Context, - natsClient MCPNATSClient, + agent AgentControl, modelName string, remote config.MCPGenericConfig[config.MCPRemoteServers], stdio config.MCPGenericConfig[config.MCPSTDIOServers], toolName, arguments string, ) (string, error) { - if natsClient == nil { - return "", fmt.Errorf("NATS client not configured for distributed MCP") + if agent == nil { + return "", fmt.Errorf("no agent control client is configured for distributed MCP: this frontend cannot reach an agent worker to run tool %q", toolName) } var args map[string]any @@ -531,63 +554,50 @@ func ExecuteMCPToolCallRemote( } } - req := mcpRemote.MCPToolRequest{ + ctx, cancel := context.WithTimeout(ctx, config.DefaultMCPToolTimeout) + defer cancel() + + resp, err := agent.ExecuteMCPTool(ctx, mcpRemote.MCPToolRequest{ ModelName: modelName, ToolName: toolName, Arguments: args, RemoteServers: remote, StdioServers: stdio, - } - reqData, _ := json.Marshal(req) - - replyData, err := natsClient.Request(messaging.SubjectMCPToolExecute, reqData, config.DefaultMCPToolTimeout) + }) if err != nil { - return "", fmt.Errorf("NATS MCP tool request failed: %w", err) - } - - var resp mcpRemote.MCPToolResponse - if err := json.Unmarshal(replyData, &resp); err != nil { - return "", fmt.Errorf("unmarshal MCP reply: %w", err) - } - if resp.Error != "" { - return "", fmt.Errorf("remote MCP tool error: %s", resp.Error) + return "", fmt.Errorf("the MCP tool call could not be run on an agent worker: %w", err) } return resp.Result, nil } -// DiscoverMCPToolsRemote routes an MCP discovery request to an agent worker via NATS. -// Returns server info and tool function schemas from the remote worker. +// DiscoverMCPToolsRemote asks an agent worker which MCP servers and tool +// schemas a model's configuration reaches. func DiscoverMCPToolsRemote( ctx context.Context, - natsClient MCPNATSClient, + agent AgentControl, modelName string, remote config.MCPGenericConfig[config.MCPRemoteServers], stdio config.MCPGenericConfig[config.MCPSTDIOServers], ) (*mcpRemote.MCPDiscoveryResponse, error) { - if natsClient == nil { - return nil, fmt.Errorf("NATS client not configured for distributed MCP") + if agent == nil { + return nil, fmt.Errorf("no agent control client is configured for distributed MCP: this frontend cannot reach an agent worker to discover the tools of model %q", modelName) } - req := mcpRemote.MCPDiscoveryRequest{ + // Its own budget, and its own constant. Discovery opens every configured + // MCP server, which a tool call on an already-open session does not, so the + // two are not the same wait and never were. + ctx, cancel := context.WithTimeout(ctx, config.DefaultMCPDiscoveryTimeout) + defer cancel() + + resp, err := agent.DiscoverMCPTools(ctx, mcpRemote.MCPDiscoveryRequest{ ModelName: modelName, RemoteServers: remote, StdioServers: stdio, - } - reqData, _ := json.Marshal(req) - - replyData, err := natsClient.Request(messaging.SubjectMCPDiscovery, reqData, config.DefaultMCPDiscoveryTimeout) + }) if err != nil { - return nil, fmt.Errorf("NATS MCP discovery request failed: %w", err) - } - - var resp mcpRemote.MCPDiscoveryResponse - if err := json.Unmarshal(replyData, &resp); err != nil { - return nil, fmt.Errorf("unmarshal MCP discovery reply: %w", err) - } - if resp.Error != "" { - return nil, fmt.Errorf("remote MCP discovery error: %s", resp.Error) + return nil, fmt.Errorf("MCP discovery could not be run on an agent worker: %w", err) } - return &resp, nil + return resp, nil } // ListMCPServers returns server info with tool, prompt, and resource names for each session. diff --git a/core/http/endpoints/openai/chat.go b/core/http/endpoints/openai/chat.go index fbf5924eac34..f5f0d0109c7d 100644 --- a/core/http/endpoints/openai/chat.go +++ b/core/http/endpoints/openai/chat.go @@ -218,7 +218,7 @@ func applyAutoparserOverride( // @Param request body schema.OpenAIRequest true "query params" // @Success 200 {object} schema.OpenAIResponse "Response" // @Router /v1/chat/completions [post] -func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, startupOptions *config.ApplicationConfig, natsClient mcpTools.MCPNATSClient, assistantHolder *mcpTools.LocalAIAssistantHolder, compressor middleware.ChatCompressor) echo.HandlerFunc { +func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, startupOptions *config.ApplicationConfig, agentControl mcpTools.AgentControl, assistantHolder *mcpTools.LocalAIAssistantHolder, compressor middleware.ChatCompressor) echo.HandlerFunc { return func(c echo.Context) error { var textContentToReturn string id := uuid.New().String() @@ -320,7 +320,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator if (len(mcpServers) > 0 || mcpPromptName != "" || len(mcpResourceURIs) > 0) && (config.MCP.Servers != "" || config.MCP.Stdio != "") { remote, stdio, mcpErr := config.MCP.MCPConfigFromYAML() if mcpErr == nil { - mcpExecutor = mcpTools.NewToolExecutor(c.Request().Context(), natsClient, config.Name, remote, stdio, mcpServers) + mcpExecutor = mcpTools.NewToolExecutor(c.Request().Context(), agentControl, config.Name, remote, stdio, mcpServers) // Prompt and resource injection (pre-processing step — resolves locally regardless of distributed mode) namedSessions, sessErr := mcpTools.NamedSessionsFromMCPConfig(config.Name, remote, stdio, mcpServers) diff --git a/core/http/endpoints/openresponses/responses.go b/core/http/endpoints/openresponses/responses.go index 553c01558b8d..d52e79b3fe3c 100644 --- a/core/http/endpoints/openresponses/responses.go +++ b/core/http/endpoints/openresponses/responses.go @@ -31,7 +31,7 @@ import ( // @Param request body schema.OpenResponsesRequest true "Request body" // @Success 200 {object} schema.ORResponseResource "Response" // @Router /v1/responses [post] -func ResponsesEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig, natsClient mcpTools.MCPNATSClient) echo.HandlerFunc { +func ResponsesEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig, agentControl mcpTools.AgentControl) echo.HandlerFunc { return func(c echo.Context) error { createdAt := time.Now().Unix() responseID := fmt.Sprintf("resp_%s", uuid.New().String()) @@ -108,7 +108,7 @@ func ResponsesEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval if !hasMCPRequest { enabledServers = nil // backward compat: auto-activate all servers } - mcpExecutor = mcpTools.NewToolExecutor(c.Request().Context(), natsClient, cfg.Name, remote, stdio, enabledServers) + mcpExecutor = mcpTools.NewToolExecutor(c.Request().Context(), agentControl, cfg.Name, remote, stdio, enabledServers) // Prompt and resource injection (pre-processing step — resolves locally regardless of distributed mode) if hasMCPRequest { diff --git a/core/http/routes/agent_control.go b/core/http/routes/agent_control.go new file mode 100644 index 000000000000..1fc23156cb8f --- /dev/null +++ b/core/http/routes/agent_control.go @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MIT + +package routes + +import ( + "github.com/mudler/LocalAI/core/application" + mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp" +) + +// mcpAgentControl returns the port the MCP endpoints reach an agent worker +// through, or a nil interface in a deployment that has no agent workers to +// reach. +// +// One function for four route files, and it exists because of the trap rather +// than to save three lines. Every one of those endpoints decides between its +// local and its distributed path by asking whether this value is nil, and a nil +// *nodes.AgentControlClient assigned straight into the interface is NOT nil: +// the interface carries a type, so the check passes, the distributed path is +// taken, and every MCP request in a standalone deployment fails instead of +// using the in-process sessions it has. Returning the untyped nil explicitly is +// the only spelling that keeps that check meaning what its call sites think it +// means. +func mcpAgentControl(app *application.Application) mcpTools.AgentControl { + return mcpAgentControlOf(app.Distributed()) +} + +// mcpAgentControlOf is the half of the rule above that a spec can reach. +// +// It is separate because an *application.Application carries its distributed +// services in an unexported field with no way in from outside the package, so +// the typed-nil conversion could not otherwise be asserted at all, and it is +// precisely the conversion that is easy to get wrong and silent when it is. +func mcpAgentControlOf(d *application.DistributedServices) mcpTools.AgentControl { + if d == nil || d.AgentControl == nil { + return nil + } + return d.AgentControl +} diff --git a/core/http/routes/agent_control_internal_test.go b/core/http/routes/agent_control_internal_test.go new file mode 100644 index 000000000000..64fb60e2ad13 --- /dev/null +++ b/core/http/routes/agent_control_internal_test.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MIT + +package routes + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/services/nodes" +) + +// Every MCP endpoint decides between its LOCAL sessions and an agent worker by +// asking whether this value is nil. That check means what its call sites think +// it means only if the nil that reaches it is an untyped one. +var _ = Describe("the MCP endpoints' agent control port", func() { + It("is nil in a deployment with no distributed services", func() { + Expect(mcpAgentControlOf(nil)).To(BeNil()) + }) + + It("is nil, and not a non-nil interface holding a nil pointer, when nothing built the client", func() { + // The trap, stated as an assertion. A nil *nodes.AgentControlClient + // assigned straight into the interface compares NON-nil, so every MCP + // request would take the distributed path and fail, in a deployment + // that has perfectly good in-process sessions. + Expect(mcpAgentControlOf(&application.DistributedServices{})).To(BeNil()) + }) + + It("carries the client through when there is one", func() { + // The negative control: a function that returned nil unconditionally + // would pass both assertions above and disable distributed MCP + // entirely, with the local path silently taken instead. + client := nodes.NewAgentControlClient(nil, nil) + Expect(mcpAgentControlOf(&application.DistributedServices{AgentControl: client})).To(BeIdenticalTo(client)) + }) +}) diff --git a/core/http/routes/anthropic.go b/core/http/routes/anthropic.go index 124557655eb2..3c305087a7c4 100644 --- a/core/http/routes/anthropic.go +++ b/core/http/routes/anthropic.go @@ -10,7 +10,6 @@ import ( "github.com/mudler/LocalAI/core/application" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/endpoints/anthropic" - mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp" "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/services/routing/pii" @@ -25,17 +24,14 @@ func RegisterAnthropicRoutes(app *echo.Echo, application *application.Application, ) { // Anthropic Messages API endpoint - var natsClient mcpTools.MCPNATSClient - if d := application.Distributed(); d != nil { - natsClient = d.Nats - } + agentControl := mcpAgentControl(application) messagesHandler := anthropic.MessagesEndpoint( application.ModelConfigLoader(), application.ModelLoader(), application.TemplatesEvaluator(), application.ApplicationConfig(), - natsClient, + agentControl, ) messagesMiddleware := []echo.MiddlewareFunc{ diff --git a/core/http/routes/localai.go b/core/http/routes/localai.go index 1d3c7adafc43..225cf4ee91e1 100644 --- a/core/http/routes/localai.go +++ b/core/http/routes/localai.go @@ -6,7 +6,6 @@ import ( "github.com/mudler/LocalAI/core/application" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/endpoints/localai" - mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp" "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/schema" compressionservice "github.com/mudler/LocalAI/core/services/compression" @@ -466,11 +465,8 @@ func RegisterLocalAIRoutes(router *echo.Echo, compressionservice.CounterFunc(tokens.CountMessages), compressionservice.NewInferenceSummarizer(cl, ml, appConfig), ) - var mcpNATS mcpTools.MCPNATSClient - if d := app.Distributed(); d != nil { - mcpNATS = d.Nats - } - mcpStreamHandler := localai.MCPEndpoint(cl, ml, evaluator, appConfig, mcpNATS, chatCompressor) + agentControl := mcpAgentControl(app) + mcpStreamHandler := localai.MCPEndpoint(cl, ml, evaluator, appConfig, agentControl, chatCompressor) mcpStreamMiddleware := []echo.MiddlewareFunc{ requestExtractor.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_CHAT)), requestExtractor.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.OpenAIRequest) }), @@ -489,7 +485,7 @@ func RegisterLocalAIRoutes(router *echo.Echo, router.POST("/mcp/chat/completions", mcpStreamHandler, mcpStreamMiddleware...) // MCP server listing endpoint - router.GET("/v1/mcp/servers/:model", localai.MCPServersEndpoint(cl, appConfig, mcpNATS), mcpMw) + router.GET("/v1/mcp/servers/:model", localai.MCPServersEndpoint(cl, appConfig, agentControl), mcpMw) // MCP prompts endpoints router.GET("/v1/mcp/prompts/:model", localai.MCPPromptsEndpoint(cl, appConfig), mcpMw) diff --git a/core/http/routes/openai.go b/core/http/routes/openai.go index 16b74804af30..a2214f12e499 100644 --- a/core/http/routes/openai.go +++ b/core/http/routes/openai.go @@ -5,7 +5,6 @@ import ( "github.com/mudler/LocalAI/core/application" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/endpoints/localai" - mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp" "github.com/mudler/LocalAI/core/http/endpoints/openai" "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/schema" @@ -38,18 +37,15 @@ func RegisterOpenAIRoutes(app *echo.Echo, app.POST("/v1/realtime/transcription_session", openai.RealtimeTranscriptionSession(application), traceMiddleware) app.POST("/v1/realtime/calls", openai.RealtimeCalls(application), traceMiddleware) - // NATS client for distributed MCP tool routing (nil when not in distributed mode) - var natsClient mcpTools.MCPNATSClient - if d := application.Distributed(); d != nil { - natsClient = d.Nats - } + // How the MCP endpoints reach an agent worker; nil outside distributed mode. + agentControl := mcpAgentControl(application) // chat chatCompressor := compressionservice.New( compressionservice.CounterFunc(tokens.CountMessages), compressionservice.NewInferenceSummarizer(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig()), ) - chatHandler := openai.ChatEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.TemplatesEvaluator(), application.ApplicationConfig(), natsClient, application.LocalAIAssistant(), chatCompressor) + chatHandler := openai.ChatEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.TemplatesEvaluator(), application.ApplicationConfig(), agentControl, application.LocalAIAssistant(), chatCompressor) chatMiddleware := []echo.MiddlewareFunc{ nodeHeaderMiddleware, usageMiddleware, diff --git a/core/http/routes/openresponses.go b/core/http/routes/openresponses.go index 8aff6ccf13a1..18a283b82b04 100644 --- a/core/http/routes/openresponses.go +++ b/core/http/routes/openresponses.go @@ -5,7 +5,6 @@ import ( "github.com/mudler/LocalAI/core/application" "github.com/mudler/LocalAI/core/config" localai "github.com/mudler/LocalAI/core/http/endpoints/localai" - mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp" "github.com/mudler/LocalAI/core/http/endpoints/openresponses" "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/schema" @@ -16,11 +15,9 @@ func RegisterOpenResponsesRoutes(app *echo.Echo, re *middleware.RequestExtractor, application *application.Application) { - // NATS client for distributed MCP tool routing (nil when not in distributed mode) - var natsClient mcpTools.MCPNATSClient + // How the MCP endpoints reach an agent worker; nil outside distributed mode. + agentControl := mcpAgentControl(application) if d := application.Distributed(); d != nil { - natsClient = d.Nats - // Replicate response metadata across frontend replicas and subscribe to // delegated cancels. Without this a GET, a previous_response_id lookup or // a cancel that the load balancer sends to a replica other than the @@ -38,7 +35,7 @@ func RegisterOpenResponsesRoutes(app *echo.Echo, application.ModelLoader(), application.TemplatesEvaluator(), application.ApplicationConfig(), - natsClient, + agentControl, ) responsesMiddleware := []echo.MiddlewareFunc{ diff --git a/core/services/agentworker/handlers.go b/core/services/agentworker/handlers.go index 435040eddc73..b933362dfb6d 100644 --- a/core/services/agentworker/handlers.go +++ b/core/services/agentworker/handlers.go @@ -11,6 +11,8 @@ // This package is the worker half of moving them onto the same HTTP control // plane a backend worker already serves. The frontend half, which chooses WHICH // agent worker to ask, is not here: this package answers, it does not select. +// That selection is nodes.AgentSelector, and it is a query against the +// connection rows rather than anything a broker does. package agentworker import ( diff --git a/core/services/cluster/connected_among_test.go b/core/services/cluster/connected_among_test.go new file mode 100644 index 000000000000..77916d16a70d --- /dev/null +++ b/core/services/cluster/connected_among_test.go @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: MIT + +package cluster_test + +import ( + "context" + "strings" + "time" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +// ConnectedAmong is the read a SELECTION is built on: which of these nodes can +// a request reach right now, and which of those can it reach without a relay +// hop. Every spec below is about one of the two ways that answer can be wrong, +// and both are the invariant this phase is built around rather than tidiness: +// +// - naming a node NO live replica holds sends a request at a process that is +// gone, and the caller reads the failure as the worker's own answer; +// - dropping a node a live replica DOES hold refuses work to a fleet that is +// fine, which is what an absence read leaking into a routing read looks +// like. +var _ = Describe("ConnectedAmong", func() { + var ( + ctx context.Context + db *gorm.DB + reg *cluster.Registry + ) + + // live registers a replica that is heartbeating now. + live := func(id string) { + GinkgoHelper() + Expect(reg.Register(ctx, id, "10.0.0.1:8080", "v1")).To(Succeed()) + } + + // kill stops a replica heartbeating, WITHOUT stamping a departure on the + // rows it holds. That is exactly the state a replica that died leaves + // behind until a peer's membership sweep runs. + kill := func(id string) { + GinkgoHelper() + age(ctx, db, "instances", "last_seen", "id", id, cluster.InstanceLiveness+5*time.Second) + } + + BeforeEach(func() { + ctx = context.Background() + db = testutil.SetupTestDB() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg = cluster.NewRegistry(db) + }) + + It("reports a node this replica holds in BOTH lists", func() { + // heldByOwner is a subset of held and not an alternative to it. A + // caller that prefers what it owns and falls back to the rest would + // otherwise have to union the two itself, and a caller that forgot + // would never fall back at all. + live("me") + _, err := reg.Claim(ctx, "agent-1", "me") + Expect(err).ToNot(HaveOccurred()) + + held, byOwner, err := reg.ConnectedAmong(ctx, []string{"agent-1"}, "me") + Expect(err).ToNot(HaveOccurred()) + Expect(held).To(ConsistOf("agent-1")) + Expect(byOwner).To(ConsistOf("agent-1")) + }) + + It("reports a node a PEER holds as held, and not as held by this replica", func() { + live("me") + live("peer") + _, err := reg.Claim(ctx, "agent-1", "peer") + Expect(err).ToNot(HaveOccurred()) + + held, byOwner, err := reg.ConnectedAmong(ctx, []string{"agent-1"}, "me") + Expect(err).ToNot(HaveOccurred()) + Expect(held).To(ConsistOf("agent-1")) + Expect(byOwner).To(BeEmpty()) + }) + + It("drops a node whose owning replica is no longer live", func() { + // The whole reason the statement joins instances. A connection row + // outlives its owner by up to a liveness window plus a heartbeat, so + // without the join this read names a dead replica for that entire + // window and every caller acting on it dials a corpse. + live("dead") + _, err := reg.Claim(ctx, "agent-1", "dead") + Expect(err).ToNot(HaveOccurred()) + kill("dead") + + held, byOwner, err := reg.ConnectedAmong(ctx, []string{"agent-1"}, "me") + Expect(err).ToNot(HaveOccurred()) + Expect(held).To(BeEmpty()) + Expect(byOwner).To(BeEmpty()) + }) + + It("drops a node whose row records a departure, however recent", func() { + // A departed row is not a candidate for ROUTING, whatever its age. How + // old the departure is decides whether the worker has GONE, which is + // Presence's question and not this one; answering it here would put two + // windows on one fact. + live("me") + epoch, err := reg.Claim(ctx, "agent-1", "me") + Expect(err).ToNot(HaveOccurred()) + Expect(reg.Release(ctx, "agent-1", "me", epoch)).To(Succeed()) + + held, byOwner, err := reg.ConnectedAmong(ctx, []string{"agent-1"}, "me") + Expect(err).ToNot(HaveOccurred()) + Expect(held).To(BeEmpty()) + Expect(byOwner).To(BeEmpty()) + }) + + It("answers about the ids it was given and no others", func() { + live("me") + for _, id := range []string{"agent-1", "agent-2", "backend-9"} { + _, err := reg.Claim(ctx, id, "me") + Expect(err).ToNot(HaveOccurred()) + } + + held, byOwner, err := reg.ConnectedAmong(ctx, []string{"agent-1", "agent-2"}, "me") + Expect(err).ToNot(HaveOccurred()) + Expect(held).To(ConsistOf("agent-1", "agent-2")) + Expect(byOwner).To(ConsistOf("agent-1", "agent-2")) + }) + + It("answers an empty candidate set without asking the database", func() { + // `IN ()` is a syntax error in PostgreSQL, so a statement issued here + // would fail rather than answer nothing. + rec := newSQLRecorder() + recording := cluster.NewRegistry(db.Session(&gorm.Session{Logger: rec})) + + held, byOwner, err := recording.ConnectedAmong(ctx, nil, "me") + Expect(err).ToNot(HaveOccurred()) + Expect(held).To(BeEmpty()) + Expect(byOwner).To(BeEmpty()) + Expect(rec.statementCount()).To(Equal(0)) + }) + + It("answers in one joined statement measured on the database clock", func() { + live("me") + _, err := reg.Claim(ctx, "agent-1", "me") + Expect(err).ToNot(HaveOccurred()) + + rec := newSQLRecorder() + recording := cluster.NewRegistry(db.Session(&gorm.Session{Logger: rec})) + _, _, err = recording.ConnectedAmong(ctx, []string{"agent-1", "agent-2"}, "me") + Expect(err).ToNot(HaveOccurred()) + + sql := strings.ToLower(rec.only()) + // only() rules out the read-per-id shape: between two statements the + // owning replica can die, and the answer would then be assembled from + // two different snapshots of the cluster. + Expect(sql).To(ContainSubstring("join")) + Expect(sql).To(ContainSubstring("instances")) + // The liveness window is computed by the DATABASE. A Go-side cutoff + // would appear here as a bound literal and would then move with each + // replica's clock skew. The test container shares this host's clock, so + // no behavioural spec can catch that; the statement shape is the only + // place it is visible. + Expect(sql).To(ContainSubstring("make_interval")) + Expect(sql).To(ContainSubstring("now()")) + Expect(sql).ToNot(MatchRegexp(`last_seen\s*>\s*'`), + "the liveness cutoff must not be a literal timestamp from this process's clock") + // Held-ness is asked in the SQL rather than left to the join alone. An + // empty owner id matches no instance today only because no replica + // registers under one, which is an accident of who registers rather + // than a property of ownership. + Expect(strings.Join(strings.Fields(sql), " ")).To(ContainSubstring( + "node_connections.owner_instance_id <> ''")) + }) +}) diff --git a/core/services/cluster/ownership.go b/core/services/cluster/ownership.go index 1f253cd073f3..26ef64dde173 100644 --- a/core/services/cluster/ownership.go +++ b/core/services/cluster/ownership.go @@ -396,3 +396,95 @@ func (r *Registry) PurgeDepartedBefore(ctx context.Context, olderThan time.Durat } return res.RowsAffected, nil } + +// connectedAmongQuery answers, for a set of nodes at once, which of their +// tunnels a LIVE replica holds and which of those are held by one named +// replica. +// +// ONE statement rather than a read per id, for the reason Owner is one +// statement: between two reads an owning replica can die, and the answer would +// then be assembled from two different snapshots of the cluster. A selection +// built that way can name a node whose owner the second read would have +// rejected, which is the one thing this read exists to prevent. +// +// The join is what makes a DEAD owner's row invisible here. Without it a row +// outlives its owner by up to a liveness window plus a heartbeat, and every +// caller that acted on this answer would be sent at a process that is gone. +// +// Held-ness is asked in SQL and the departure is not asked at all, and that is +// the whole difference between this read and Presence. This one answers a +// ROUTING question ("can a request get there right now"), so a row nobody holds +// is simply not a candidate, whatever its age. Deciding whether a worker has +// GONE is Presence's job and needs the grace; nothing here may be read as +// absence, and nothing here reports any. +// +// The liveness window is computed by the DATABASE, like every other window in +// this package: it is compared across replicas, so a Go-side cutoff would make +// the effective window depend on each replica's clock skew. The test container +// shares this host's clock, so no behavioural spec can see the difference and +// the statement shape is pinned instead. +// +// The predicates are the package's own rather than copies, so two spellings of +// "live" or of "held" cannot drift into a node that one query calls connected +// and another calls gone. +// +// The bind order is the order the placeholders appear in the text: the owner, +// then the liveness window, then the node ids. +const connectedAmongQuery = ` +SELECT node_connections.node_id AS node_id, + (node_connections.owner_instance_id = ?) AS by_owner +FROM node_connections +JOIN instances + ON instances.id = node_connections.owner_instance_id + AND ` + instanceIsLive + ` +WHERE ` + connectionIsHeld + ` + AND node_connections.node_id IN ?` + +// ConnectedAmong returns the subset of nodeIDs whose tunnel a LIVE replica +// currently holds, and separately those held by owner. +// +// It takes ids rather than asking which nodes are agents, because this package +// is a leaf and node type lives in core/services/nodes. +// +// heldByOwner is a SUBSET of held, not an alternative to it. A caller that +// prefers the ids it owns falls back to the rest, and expressing that as two +// disjoint lists would make the fallback a union the caller had to build. +// +// The answer is a routing fact and NEVER an absence one. An id missing from +// both lists means no live replica holds its tunnel at this instant, which a +// worker reconnecting between replicas produces routinely; nothing may read it +// as the worker having gone away. See Presence for the read that can answer +// that question, and only that one. +func (r *Registry) ConnectedAmong(ctx context.Context, nodeIDs []string, owner string) (held []string, heldByOwner []string, err error) { + if len(nodeIDs) == 0 { + // Answered without a statement, because `IN ()` is a syntax error in + // PostgreSQL and gorm's expansion of an empty slice is a filter nobody + // wrote. An empty candidate set has exactly one honest answer. + return nil, nil, nil + } + // Refused rather than attempted, for the reason Owner refuses: now() and + // make_interval are PostgreSQL, so on the single-binary SQLite path this + // would fail as a missing function and read as a missing migration. It is + // deliberately not an empty answer: a deployment with no cluster has + // nothing to say about who holds a tunnel, and an empty list would be read + // as "nobody does". + if !isPostgres(r.db) { + return nil, nil, fmt.Errorf("reading which of %d nodes are connected: connection ownership requires PostgreSQL, this deployment runs on %q", len(nodeIDs), r.db.Dialector.Name()) + } + var rows []struct { + NodeID string + ByOwner bool + } + if err := r.db.WithContext(ctx).Raw(connectedAmongQuery, + owner, InstanceLiveness.Seconds(), nodeIDs, + ).Scan(&rows).Error; err != nil { + return nil, nil, fmt.Errorf("reading which of %d nodes are connected: %w", len(nodeIDs), err) + } + for _, row := range rows { + held = append(held, row.NodeID) + if row.ByOwner { + heldByOwner = append(heldByOwner, row.NodeID) + } + } + return held, heldByOwner, nil +} diff --git a/core/services/cluster/ownership_test.go b/core/services/cluster/ownership_test.go index 54195cfd86a1..d23f9d98d769 100644 --- a/core/services/cluster/ownership_test.go +++ b/core/services/cluster/ownership_test.go @@ -58,6 +58,14 @@ func (r *sqlRecorder) only() string { return r.statements[0] } +// statementCount reports how many statements were recorded, for the specs whose +// claim is that a call issued NONE. +func (r *sqlRecorder) statementCount() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.statements) +} + // writeTarget matches the table a statement writes to, anchored at the verb so // the UPDATE inside an upsert's ON CONFLICT clause cannot be mistaken for one. var writeTarget = regexp.MustCompile(`^\s*(?i:delete\s+from|update)\s+"?([a-z_]+)"?`) diff --git a/core/services/mcp/remote.go b/core/services/mcp/remote.go index 17cfc1f3685e..ae789a1abb01 100644 --- a/core/services/mcp/remote.go +++ b/core/services/mcp/remote.go @@ -5,9 +5,13 @@ import ( "github.com/mudler/LocalAI/pkg/functions" ) -// MCPToolRequest is the NATS request-reply payload for executing an MCP tool -// on an agent worker. The frontend serializes the model's MCP server config -// so the worker can create sessions and execute the tool. +// MCPToolRequest is the control-RPC payload for executing an MCP tool on an +// agent worker. The frontend serializes the model's MCP server config so the +// worker can create sessions and execute the tool. +// +// It lives in this package rather than beside either side of the call, because +// both sides need it and they are in packages that must not import each other: +// core/http/endpoints/mcp is the caller and core/cli is the worker. type MCPToolRequest struct { ModelName string `json:"model_name"` ToolName string `json:"tool_name"` @@ -16,21 +20,25 @@ type MCPToolRequest struct { StdioServers config.MCPGenericConfig[config.MCPSTDIOServers] `json:"stdio_servers"` } -// MCPToolResponse is the NATS reply for an MCP tool execution. +// MCPToolResponse is the reply to an MCP tool execution. +// +// A set Error is the WORKER'S OWN ANSWER, carried inside a successful reply +// rather than as a transport failure, which is what lets a caller tell "this +// tool rejected your arguments" from "this worker could not be reached". type MCPToolResponse struct { Result string `json:"result,omitempty"` Error string `json:"error,omitempty"` } -// MCPDiscoveryRequest is the NATS request-reply payload for discovering -// available MCP tools, prompts, and resources from a model's MCP servers. +// MCPDiscoveryRequest is the control-RPC payload for discovering available MCP +// tools, prompts, and resources from a model's MCP servers. type MCPDiscoveryRequest struct { ModelName string `json:"model_name"` RemoteServers config.MCPGenericConfig[config.MCPRemoteServers] `json:"remote_servers"` StdioServers config.MCPGenericConfig[config.MCPSTDIOServers] `json:"stdio_servers"` } -// MCPDiscoveryResponse is the NATS reply for an MCP discovery request. +// MCPDiscoveryResponse is the reply to an MCP discovery request. type MCPDiscoveryResponse struct { Servers []MCPServerInfo `json:"servers,omitempty"` Tools []MCPToolDef `json:"tools,omitempty"` // flattened tool list with functions @@ -48,8 +56,8 @@ type MCPServerInfo struct { } // MCPToolDef is a serializable tool definition (function schema) that can -// travel over NATS. Unlike MCPToolInfo which holds a live session pointer, -// this is pure data. +// travel between processes. Unlike MCPToolInfo which holds a live session +// pointer, this is pure data. type MCPToolDef struct { ServerName string `json:"server_name"` ToolName string `json:"tool_name"` diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index 8f5429a3909b..657a99f92c3c 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -47,13 +47,6 @@ func SubjectJobResult(jobID string) string { return subjectJobProgressPrefix + sanitizeSubjectToken(jobID) + ".result" } -// MCP Tool Execution (Request-Reply via NATS — load-balanced across agent workers) -const ( - SubjectMCPToolExecute = "mcp.tools.execute" - SubjectMCPDiscovery = "mcp.discovery" - QueueAgentWorkers = "agent-workers" -) - // SubjectFineTuneProgress returns the NATS subject for fine-tune progress. func SubjectFineTuneProgress(jobID string) string { return subjectFineTunePrefix + sanitizeSubjectToken(jobID) + ".progress" diff --git a/core/services/nodes/agent_control.go b/core/services/nodes/agent_control.go new file mode 100644 index 000000000000..846a5ec96ee1 --- /dev/null +++ b/core/services/nodes/agent_control.go @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: MIT + +package nodes + +import ( + "context" + "errors" + "fmt" + + mcpremote "github.com/mudler/LocalAI/core/services/mcp" + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// maxAgentPicks bounds how many agent workers one verb is offered to before it +// gives up. +// +// Three rather than "every agent in the fleet": a caller is waiting, and a +// deployment whose first three connected agents all fail to answer has a +// problem that trying a fourth does not fix. It is also not one: a worker whose +// tunnel died between the connection read and the dial is ordinary, and giving +// up on the first of those would make every re-homing agent an outage. +const maxAgentPicks = 3 + +// ErrNoAgentControl reports that this deployment has no agent control client +// wired at all. +// +// Like ErrNoAgentWorker it is neither ErrWorkerUnroutable nor anything +// cluster.IsWorkerAnswer accepts: it is a fact about this process's own wiring +// and says nothing about any worker. +var ErrNoAgentControl = errors.New("nodes: this deployment has no agent control client") + +// AgentControlClient issues the frontend's control RPCs to whichever agent +// worker the selector picks. +// +// It is the caller the agent worker's control plane has been waiting for. What +// used to be a NATS request onto a queue group is now two ordinary things: a +// SELECTION, which is a query against the connection rows, and a control RPC +// over the picked worker's own tunnel, relayed by the peer mesh when another +// replica holds it. +type AgentControlClient struct { + sel *AgentSelector + cc *ControlClient +} + +// NewAgentControlClient returns the client that carries the frontend's agent +// verbs to the workers sel picks, over cc. +func NewAgentControlClient(sel *AgentSelector, cc *ControlClient) *AgentControlClient { + return &AgentControlClient{sel: sel, cc: cc} +} + +// ExecuteMCPTool runs one MCP tool on an agent worker and returns its answer. +// +// A reply carrying an Error is the WORKER'S OWN ANSWER and comes back as an +// error naming it, never as a retry: see agentVerb. +func (a *AgentControlClient) ExecuteMCPTool(ctx context.Context, req mcpremote.MCPToolRequest) (*mcpremote.MCPToolResponse, error) { + return agentVerb(ctx, a, workerctl.PathMCPToolExecute, req, + func(r *mcpremote.MCPToolResponse) string { return r.Error }) +} + +// DiscoverMCPTools asks an agent worker which MCP servers and tools a model's +// configuration reaches. +func (a *AgentControlClient) DiscoverMCPTools(ctx context.Context, req mcpremote.MCPDiscoveryRequest) (*mcpremote.MCPDiscoveryResponse, error) { + return agentVerb(ctx, a, workerctl.PathMCPDiscovery, req, + func(r *mcpremote.MCPDiscoveryResponse) string { return r.Error }) +} + +// agentVerb is the ONE place the select-call-retry rule lives, and the one +// place the line between a retryable failure and an answer is drawn. +// +// The rule, stated as the code enforces it: +// +// - A DECODED REPLY whose Error field is set is the worker's own answer to +// this verb. It is returned as an error naming what the worker said and is +// NEVER offered to another worker: retrying it would turn "this MCP server +// rejected your arguments" into "the fleet is broken", and could run a tool +// twice. +// - Everything the RPC returns as a Go ERROR is a failure to obtain an +// answer. The request never reached a handler (a refused stream, an +// unreachable peer, a lost tunnel, a 404 from an older build) or its answer +// could not be read, so nothing ran to completion here and another worker +// may be offered the verb. +// +// This is deliberately NOT keyed on cluster.IsWorkerAnswer, and the reason is +// worth stating because the plan for this task said it should be. +// IsWorkerAnswer accepts the tunnel's stream-refusal vocabulary, which a worker +// writes BEFORE any request body reaches its control server: ErrStreamTagUnknown +// and ErrStreamTargetUnavailable both mean "I could not carry this to my own +// control plane", not "I ran your tool and here is what happened". Returning +// those unchanged without trying another agent would make one agent worker with +// a dead control server take down MCP for the whole deployment, and it is the +// case the plan's own spec list requires to be retried. +// +// The taxonomy is preserved either way: whatever error is returned is returned +// UNWRAPPED, so cluster.IsWorkerAnswer and ErrWorkerUnroutable still see +// exactly what ControlClient produced. +func agentVerb[Req any, Rep any](ctx context.Context, a *AgentControlClient, path string, req Req, + answerError func(*Rep) string) (*Rep, error) { + if a == nil || a.sel == nil || a.cc == nil { + // Guarded on the nil RECEIVER too, because the interface a caller holds + // this through is satisfied by a typed nil pointer, which is not an + // untyped nil and would otherwise panic inside a request. + return nil, fmt.Errorf("control rpc %s: %w", path, ErrNoAgentControl) + } + tried := make(map[string]bool, maxAgentPicks) + var lastErr error + for range maxAgentPicks { + nodeID, _, err := a.sel.pickConnectedExcluding(ctx, tried) + if err != nil { + if lastErr != nil && errors.Is(err, ErrNoAgentWorker) { + // Every worker that held a tunnel has now failed to answer. The + // evidence is what they said, not that the candidate list ran + // out, and reporting an empty fleet here would hide a fleet + // that is refusing. + return nil, lastErr + } + return nil, err + } + tried[nodeID] = true + + var reply Rep + if err := a.cc.Call(ctx, nodeID, path, req, &reply); err != nil { + lastErr = err + if ctx.Err() != nil { + // The caller's own budget is spent. Another pick would fail the + // same way immediately, and the error already says whose fault + // the expiry was. + return nil, lastErr + } + continue + } + if msg := answerError(&reply); msg != "" { + return nil, fmt.Errorf("agent worker %q answered %s with an error: %s", nodeID, path, msg) + } + return &reply, nil + } + return nil, lastErr +} diff --git a/core/services/nodes/agent_control_test.go b/core/services/nodes/agent_control_test.go new file mode 100644 index 000000000000..aaeaa81f11a6 --- /dev/null +++ b/core/services/nodes/agent_control_test.go @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: MIT + +package nodes_test + +import ( + "context" + "encoding/json" + "errors" + "net" + "net/http" + "net/http/httptest" + "runtime" + "sync/atomic" + + "github.com/gorilla/websocket" + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/agentworker" + "github.com/mudler/LocalAI/core/services/cluster" + mcpremote "github.com/mudler/LocalAI/core/services/mcp" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/testutil" + "github.com/mudler/LocalAI/core/services/worker" +) + +// This file drives the WHOLE path a distributed MCP call takes: a real agent +// worker holding a real tunnel, a real connection row deciding which replica +// owns it, a real relay when the owner is a peer, and the real control client +// on top. Nothing here is a double of a transport. +// +// That matters more than usual for this task. The selection it proves is a +// query against the same rows the scheduler reads absence from, and a fake +// registry that never dials would prove nothing at all about a relayed pick: +// every interesting failure (a stream a worker refuses, a tunnel a peer holds, +// a candidate whose owner is dead) exists only on the wire. + +const agentControlToken = "agent-control-token" + +// fakeFrontend is the far side of a worker's tunnel: the real WebSocket +// upgrade and the real yamux server handshake. +type fakeFrontend struct { + srv *httptest.Server + sessions chan *yamux.Session +} + +func newFakeFrontend() *fakeFrontend { + GinkgoHelper() + f := &fakeFrontend{sessions: make(chan *yamux.Session, 4)} + upgrader := websocket.Upgrader{} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != cluster.ConnectPath { + w.WriteHeader(http.StatusNotFound) + return + } + ws, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + sess, err := yamux.Server(cluster.WebsocketConn(ws), nil, nil) + if err != nil { + _ = ws.Close() + return + } + select { + case f.sessions <- sess: + default: + _ = sess.Close() + } + })) + DeferCleanup(f.srv.Close) + return f +} + +// session waits for the worker to dial in and hands back its tunnel session. +// +// Waited for on a channel rather than slept on: the dial is the worker's own +// and nothing in this process orders it against the next line of the spec. +func (f *fakeFrontend) session() *yamux.Session { + GinkgoHelper() + var sess *yamux.Session + Eventually(f.sessions, "20s").Should(Receive(&sess)) + DeferCleanup(func() { _ = sess.Close() }) + return sess +} + +// yamuxPair is the peer link between two frontend replicas, over an in-process +// pipe: the client half is what the dialling replica opens relay streams on, +// the server half is what the owning replica accepts them from. +func yamuxPair() (client, server *yamux.Session) { + GinkgoHelper() + a, b := net.Pipe() + var err error + server, err = yamux.Server(a, nil, nil) + Expect(err).ToNot(HaveOccurred()) + client, err = yamux.Client(b, nil, nil) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { + _ = client.Close() + _ = server.Close() + }) + return client, server +} + +// peerLinks is the outbound half of the peer mesh, standing in for +// cluster.PeerPool. It opens a real yamux stream onto the owning replica's +// accepted session, which is what the relay reads its request frame from. +type peerLinks struct{ sessions map[string]*yamux.Session } + +func (p *peerLinks) Open(ctx context.Context, peerID string) (net.Conn, error) { + sess, ok := p.sessions[peerID] + if !ok { + return nil, cluster.ErrPeerUnreachable + } + return sess.OpenStream(ctx) +} + +var _ = Describe("AgentControlClient", func() { + const ( + selfInstance = "replica-me" + peerInstance = "replica-peer" + toolCallReply = "Weather in London: 15C, cloudy" + ) + + var ( + ctx context.Context + cancel context.CancelFunc + db *gorm.DB + registry *nodes.NodeRegistry + clusterReg *cluster.Registry + mine *cluster.TunnelRegistry + theirs *cluster.TunnelRegistry + peers *peerLinks + control *nodes.ControlClient + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + ctx, cancel = context.WithCancel(context.Background()) + DeferCleanup(cancel) + + db = testutil.SetupTestDB() + var err error + registry, err = nodes.NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + clusterReg = cluster.NewRegistry(db) + Expect(clusterReg.Register(ctx, selfInstance, "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(clusterReg.Register(ctx, peerInstance, "10.0.0.2:8080", "v1")).To(Succeed()) + + mine = cluster.NewTunnelRegistry(clusterReg, selfInstance) + theirs = cluster.NewTunnelRegistry(clusterReg, peerInstance) + + // The owning replica's inbound half, with the relay installed, and this + // replica's outbound half pointed at it. Together they are the path a + // request takes when the worker's tunnel landed somewhere else. + relayed := cluster.NewSessionStore(cluster.NewRelay(theirs).Stream) + DeferCleanup(relayed.CloseAll) + client, server := yamuxPair() + relayed.Accept(selfInstance, server) + peers = &peerLinks{sessions: map[string]*yamux.Session{peerInstance: client}} + + dialer := cluster.NewWorkerDialer(mine, peers) + control = nodes.NewControlClient(nodes.WorkerNetDialerFor(func(nodeID string) func(context.Context, string, string) (net.Conn, error) { + return dialer.DialerFor(nodeID, cluster.StreamTagHTTP) + }), agentControlToken) + }) + + // registerAgent puts an approved agent node in the registry and returns its + // assigned id, which is the id the connection row and the tunnel both use. + registerAgent := func(name string) string { + GinkgoHelper() + Expect(registry.Register(ctx, &nodes.BackendNode{ + Name: name, NodeType: nodes.NodeTypeAgent, Address: name + ":50051", + }, true)).To(Succeed()) + node, err := registry.GetByName(ctx, name) + Expect(err).ToNot(HaveOccurred()) + return node.ID + } + + // startAgent runs a REAL agent worker serving cfg, dialling a tunnel that + // lands on holder, and returns its node id. + startAgent := func(name string, holder *cluster.TunnelRegistry, cfg agentworker.Config) string { + GinkgoHelper() + nodeID := registerAgent(name) + frontend := newFakeFrontend() + rt, err := agentworker.Start(ctx, agentworker.Options{ + FrontendURL: frontend.srv.URL, + NodeID: nodeID, + TunnelToken: func() string { return "tunnel-secret" }, + ControlToken: agentControlToken, + Handlers: cfg, + }) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = rt.Close() }) + _, err = holder.Attach(ctx, nodeID, frontend.session()) + Expect(err).ToNot(HaveOccurred()) + return nodeID + } + + // startRefusingAgent runs a worker whose tunnel is real and whose local + // service will not open, which is what a worker with a dead control server + // looks like from the frontend: the worker itself writes the refusal, and + // the frontend reads cluster.ErrStreamTargetUnavailable off the wire. + startRefusingAgent := func(name string, holder *cluster.TunnelRegistry, refusals *atomic.Int32) string { + GinkgoHelper() + nodeID := registerAgent(name) + frontend := newFakeFrontend() + tunnel, err := worker.StartTunnel(ctx, worker.TunnelConfig{ + FrontendURL: frontend.srv.URL, + NodeID: nodeID, + Token: func() string { return "tunnel-secret" }, + Services: map[string]worker.LocalService{ + cluster.StreamTagHTTP: func(context.Context, string) (net.Conn, error) { + refusals.Add(1) + return nil, errors.New("this worker's control server is not listening") + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = tunnel.Close() }) + _, err = holder.Attach(ctx, nodeID, frontend.session()) + Expect(err).ToNot(HaveOccurred()) + return nodeID + } + + // recordingTool answers every tool call with reply and counts its calls. + recordingTool := func(calls *atomic.Int32, reply mcpremote.MCPToolResponse) agentworker.UnaryHandler { + return func(_ context.Context, raw json.RawMessage) (json.RawMessage, error) { + calls.Add(1) + var req mcpremote.MCPToolRequest + Expect(json.Unmarshal(raw, &req)).To(Succeed()) + out, err := json.Marshal(reply) + Expect(err).ToNot(HaveOccurred()) + return out, nil + } + } + + agentControl := func() *nodes.AgentControlClient { + return nodes.NewAgentControlClient( + nodes.NewAgentSelector(registry, clusterReg, selfInstance), control) + } + + toolRequest := mcpremote.MCPToolRequest{ + ModelName: "test-model", + ToolName: "weather", + Arguments: map[string]any{"city": "London"}, + } + + It("carries a tool call over the tunnel of an agent THIS replica holds", func() { + var calls atomic.Int32 + startAgent("agent-local", mine, agentworker.Config{ + MCPTool: recordingTool(&calls, mcpremote.MCPToolResponse{Result: toolCallReply}), + }) + + resp, err := agentControl().ExecuteMCPTool(ctx, toolRequest) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Result).To(Equal(toolCallReply)) + Expect(calls.Load()).To(Equal(int32(1))) + }) + + It("reaches an agent whose tunnel a PEER holds, by relaying through that peer", func() { + // The hop the queue group used to hide. This replica holds nothing, so + // the only way the request reaches the worker is the connection row + // naming the peer and the peer's relay splicing the stream onto the + // tunnel it holds. + var calls atomic.Int32 + startAgent("agent-remote", theirs, agentworker.Config{ + MCPTool: recordingTool(&calls, mcpremote.MCPToolResponse{Result: toolCallReply}), + }) + Expect(mine.Held()).To(BeEmpty(), "this spec is only about the relayed path") + + resp, err := agentControl().ExecuteMCPTool(ctx, toolRequest) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Result).To(Equal(toolCallReply)) + Expect(calls.Load()).To(Equal(int32(1))) + }) + + It("answers discovery from the agent worker's own reply", func() { + startAgent("agent-disco", mine, agentworker.Config{ + MCPDiscovery: func(context.Context, json.RawMessage) (json.RawMessage, error) { + return json.Marshal(mcpremote.MCPDiscoveryResponse{ + Servers: []mcpremote.MCPServerInfo{{Name: "weather", Type: "remote", Tools: []string{"get_weather"}}}, + }) + }, + }) + + resp, err := agentControl().DiscoverMCPTools(ctx, mcpremote.MCPDiscoveryRequest{ModelName: "test-model"}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Servers).To(HaveLen(1)) + Expect(resp.Servers[0].Name).To(Equal("weather")) + }) + + It("returns a worker's own error answer and never offers the tool to a second worker", func() { + // The one thing the retry may not do. "This MCP server rejected your + // arguments" retried elsewhere becomes "the fleet is broken", and a + // tool with a side effect would run twice. + // + // Both workers answer the same refusal, so which one the random + // tie-break picks does not matter: what is asserted is that exactly ONE + // handler ran in total. + var first, second atomic.Int32 + refusal := mcpremote.MCPToolResponse{Error: "tool 'weather' is not configured"} + startAgent("agent-a", mine, agentworker.Config{MCPTool: recordingTool(&first, refusal)}) + startAgent("agent-b", mine, agentworker.Config{MCPTool: recordingTool(&second, refusal)}) + + _, err := agentControl().ExecuteMCPTool(ctx, toolRequest) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("tool 'weather' is not configured")) + Expect(first.Load()+second.Load()).To(Equal(int32(1)), + "the worker's own answer must not be offered to another worker") + }) + + It("retries against another agent when one refuses the stream, and the second answers", func() { + // Deterministic without touching the tie-break: the refusing worker is + // the one THIS replica holds, which the selector prefers, and the + // worker that answers is peer-held, which is where the retry has to + // fall back to. So the first pick always refuses, the second always + // relays, and both halves of the rule are exercised on the wire. + var refusals atomic.Int32 + var calls atomic.Int32 + startRefusingAgent("agent-dead", mine, &refusals) + startAgent("agent-alive", theirs, agentworker.Config{ + MCPTool: recordingTool(&calls, mcpremote.MCPToolResponse{Result: toolCallReply}), + }) + + resp, err := agentControl().ExecuteMCPTool(ctx, toolRequest) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Result).To(Equal(toolCallReply)) + Expect(refusals.Load()).To(Equal(int32(1)), "the first pick must have been the refusing worker") + Expect(calls.Load()).To(Equal(int32(1))) + }) + + It("gives up after three picks and returns the last refusal unchanged", func() { + // The bound, and the taxonomy. Returning the last error UNWRAPPED is + // what keeps cluster.IsWorkerAnswer able to see what the worker + // actually said; wrapping it in the unroutable umbrella here would make + // a fleet that is refusing look like a frontend with no route. + var refusals atomic.Int32 + for _, name := range []string{"agent-1", "agent-2", "agent-3", "agent-4"} { + startRefusingAgent(name, mine, &refusals) + } + + _, err := agentControl().ExecuteMCPTool(ctx, toolRequest) + Expect(err).To(MatchError(cluster.ErrStreamTargetUnavailable)) + Expect(cluster.IsWorkerAnswer(err)).To(BeTrue()) + Expect(errors.Is(err, nodes.ErrWorkerUnroutable)).To(BeFalse()) + Expect(errors.Is(err, nodes.ErrNoAgentWorker)).To(BeFalse(), + "three workers refused; that is not an empty fleet") + Expect(refusals.Load()).To(Equal(int32(3)), "exactly three picks, never a fourth") + }) + + It("reports an empty fleet as neither a route verdict nor a worker answer", func() { + registerAgent("agent-registered-but-offline") + + _, err := agentControl().ExecuteMCPTool(ctx, toolRequest) + Expect(err).To(MatchError(nodes.ErrNoAgentWorker)) + Expect(errors.Is(err, nodes.ErrWorkerUnroutable)).To(BeFalse()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + }) + + It("refuses on a nil client rather than panicking inside a request", func() { + // The interface a caller holds this through is satisfied by a typed nil + // pointer, which is not an untyped nil and passes every `!= nil` check + // a call site makes. + var missing *nodes.AgentControlClient + _, err := missing.ExecuteMCPTool(ctx, toolRequest) + Expect(err).To(MatchError(nodes.ErrNoAgentControl)) + _, err = missing.DiscoverMCPTools(ctx, mcpremote.MCPDiscoveryRequest{}) + Expect(err).To(MatchError(nodes.ErrNoAgentControl)) + }) +}) diff --git a/core/services/nodes/agent_selector.go b/core/services/nodes/agent_selector.go new file mode 100644 index 000000000000..3ac924b9ff2c --- /dev/null +++ b/core/services/nodes/agent_selector.go @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: MIT + +package nodes + +import ( + "context" + "errors" + "fmt" + "math/rand/v2" + + "github.com/mudler/LocalAI/core/services/cluster" +) + +// ErrNoAgentWorker reports that no agent worker in this deployment currently +// holds a tunnel any live replica can reach. +// +// It is a deployment fact about this moment and NOT a verdict about any worker, +// which is why it is deliberately its own sentinel: it does not wrap +// ErrWorkerUnroutable, and cluster.IsWorkerAnswer does not accept it. Nothing +// was asked of any worker and nothing was learned about one, so no reap guard +// may act on it and nothing may be marked unhealthy because of it. +// +// The umbrella matters more than it looks. ErrWorkerUnroutable is what every +// path that DELETES a node_models row matches on, and an empty agent fleet is +// not evidence about a single node. +var ErrNoAgentWorker = errors.New("nodes: no agent worker holds a tunnel to this cluster") + +// AgentConnectionReader is the narrow port onto cluster.Registry.ConnectedAmong. +// +// An interface rather than the concrete registry so a spec can drive the +// selection by deciding what is connected, without also having to stage the +// claim rows and instance heartbeats that would produce that answer. The real +// read is pinned by its own specs, against the real database, in +// core/services/cluster. +type AgentConnectionReader interface { + ConnectedAmong(ctx context.Context, nodeIDs []string, owner string) (held []string, heldByOwner []string, err error) +} + +// AgentSelector picks an agent worker to send a control RPC to. +// +// It is what replaces a NATS queue group. The queue group was never a broker +// feature this design has to reproduce: what it did was SELECT one subscriber +// out of a set, and selection is a query. Asking it here is strictly better +// than letting a broker balance, for one concrete reason: this replica can +// prefer an agent whose tunnel it holds itself and skip the relay hop +// altogether, which hidden balancing could never do. +// +// It answers a ROUTING question and never an absence one. See ErrNoAgentWorker. +type AgentSelector struct { + registry *NodeRegistry + conns AgentConnectionReader + // selfInstanceID is this replica's id, the one the connection rows record + // as an owner. An empty one is not an error the selector can report: every + // call would simply take a relay hop with nothing saying so. It is refused + // where the selector is built instead. + selfInstanceID string +} + +// NewAgentSelector returns the selector for the agent workers registry knows +// about, reading presence through conns and preferring the tunnels +// selfInstanceID holds. +func NewAgentSelector(registry *NodeRegistry, conns AgentConnectionReader, selfInstanceID string) *AgentSelector { + return &AgentSelector{registry: registry, conns: conns, selfInstanceID: selfInstanceID} +} + +// PickConnected returns the id AND the node type of an agent node whose tunnel +// a live replica holds, preferring one THIS replica owns so the call skips the +// relay. +// +// The type is returned rather than looked up again because the caller that +// re-broadcasts a resulting progress stream needs it for every line, and the +// selector has ALREADY read the node rows to build the candidate list, so it +// costs nothing here. Resolving it through the registry per line would be a +// database read per progress line and, worse, a SECOND place in the tree that +// decides what a node's type is. +// +// It is always NodeTypeAgent today. It is returned as a value rather than +// asserted as a constant because the day a backend worker takes a dispatched +// claim, the caller that must not guess is this one. +func (s *AgentSelector) PickConnected(ctx context.Context) (nodeID, nodeType string, err error) { + return s.pickConnectedExcluding(ctx, nil) +} + +// pickConnectedExcluding is PickConnected with a set of ids already tried. +// +// The exclusion exists for the retry in AgentControlClient and is not exposed: +// a retry that could re-pick the worker that just failed to answer is not a +// retry, it is the same call again, and with a small fleet it would be the +// same call most of the time. +func (s *AgentSelector) pickConnectedExcluding(ctx context.Context, tried map[string]bool) (string, string, error) { + if s == nil || s.registry == nil || s.conns == nil { + // Not an error about any worker. A deployment with no selector has + // nothing to select from, which is the same answer as an empty fleet. + return "", "", fmt.Errorf("this deployment has no agent selector: %w", ErrNoAgentWorker) + } + agents, err := s.registry.selectableAgentNodes(ctx) + if err != nil { + // Deliberately plain. A database that would not answer says nothing + // about any worker, so this must carry neither ErrWorkerUnroutable nor + // anything cluster.IsWorkerAnswer accepts. + return "", "", fmt.Errorf("listing the agent workers of this deployment: %w", err) + } + typeOf := make(map[string]string, len(agents)) + ids := make([]string, 0, len(agents)) + for _, a := range agents { + if tried[a.ID] { + continue + } + typeOf[a.ID] = a.NodeType + ids = append(ids, a.ID) + } + held, heldByOwner, err := s.conns.ConnectedAmong(ctx, ids, s.selfInstanceID) + if err != nil { + return "", "", fmt.Errorf("reading which agent workers are connected: %w", err) + } + // heldByOwner first because that call takes no relay hop. Falling back to + // the rest rather than stopping there is what keeps a replica that holds no + // agent tunnel able to reach the fleet at all. + candidates := heldByOwner + if len(candidates) == 0 { + candidates = held + } + if len(candidates) == 0 { + return "", "", fmt.Errorf("of %d agent workers registered, none is connected: %w", len(ids), ErrNoAgentWorker) + } + // Random rather than round robin. A per-replica counter is per-replica + // state that says nothing about load, and with several replicas the + // counters agree on nothing anyway. + picked := candidates[rand.IntN(len(candidates))] + nodeType := typeOf[picked] + if nodeType == "" { + // A candidate the connection read named that the node read did not is + // this selector contradicting itself. Refused rather than returned with + // an empty type: an unknown node type is denied by every allow list + // downstream, and the only symptom of that is work that silently never + // happens. + return "", "", fmt.Errorf("agent worker %q was reported connected but has no node type: %w", picked, ErrNoAgentWorker) + } + return picked, nodeType, nil +} + +// selectableAgentNodes returns the agent nodes a control RPC may be sent to. +// +// Two statuses are excluded and no more. StatusPending is a node an admin has +// not approved, which cannot hold a tunnel anyway and must never be handed +// work; StatusDraining is one that has been asked to stop taking work, which is +// the operator's decision and not a routing fact. +// +// Every other status is left in ON PURPOSE, and this is where the invariant +// bites. StatusUnhealthy and StatusOffline are written by the health monitor +// and by shutdown, on their own clocks; whether a request can reach this worker +// RIGHT NOW is what ConnectedAmong answers, and it answers it from the +// connection rows. Filtering here on a health verdict would make the selection +// refuse a worker that is connected and answering because a probe was late, +// which is an absence read leaking into a routing one. +func (r *NodeRegistry) selectableAgentNodes(ctx context.Context) ([]BackendNode, error) { + var agents []BackendNode + if err := r.db.WithContext(ctx). + Where("node_type = ? AND status NOT IN ?", NodeTypeAgent, []string{StatusPending, StatusDraining}). + Order("id"). + Find(&agents).Error; err != nil { + return nil, fmt.Errorf("listing agent nodes: %w", err) + } + return agents, nil +} + +// The real reader is the cluster registry, asserted here so a drift between +// its signature and this port fails to COMPILE. Without it the only thing that +// would notice is the wiring in core/application, which opens a database and a +// bus and so has no unit spec at all. +var _ AgentConnectionReader = (*cluster.Registry)(nil) diff --git a/core/services/nodes/agent_selector_test.go b/core/services/nodes/agent_selector_test.go new file mode 100644 index 000000000000..959762cfc14f --- /dev/null +++ b/core/services/nodes/agent_selector_test.go @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: MIT + +package nodes + +import ( + "context" + "errors" + "runtime" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// stubConnections decides what is connected, so a selection spec can state the +// cluster state it is about in one line instead of staging claim rows and +// instance heartbeats to produce it. What the real read answers for a given +// cluster state is pinned against the real database in +// core/services/cluster/connected_among_test.go, and the relayed round trip in +// agent_control_test.go drives the real registry over a real transport. +type stubConnections struct { + held []string + heldByOwner []string + err error + + // owners records the owner id each call was made with. An empty one makes + // heldByOwner always empty in production, so every call takes a relay hop + // with nothing anywhere saying so. + owners []string + // asked records the candidate list of each call. + asked [][]string +} + +func (s *stubConnections) ConnectedAmong(_ context.Context, nodeIDs []string, owner string) ([]string, []string, error) { + s.owners = append(s.owners, owner) + s.asked = append(s.asked, append([]string(nil), nodeIDs...)) + if s.err != nil { + return nil, nil, s.err + } + // The real read answers only about ids it was given, so the stub must too: + // a stub that answered about a node the selector had excluded would hide an + // exclusion that stopped working. + offered := map[string]bool{} + for _, id := range nodeIDs { + offered[id] = true + } + var held, byOwner []string + for _, id := range s.held { + if offered[id] { + held = append(held, id) + } + } + for _, id := range s.heldByOwner { + if offered[id] { + byOwner = append(byOwner, id) + } + } + return held, byOwner, nil +} + +// The selection that replaces the queue group. Two ways it can be wrong, and +// both are the invariant rather than a preference: picking a node no live +// replica holds sends a request at a process that is gone, and refusing a node +// that is merely re-homing between replicas takes MCP down for a fleet that is +// fine. +var _ = Describe("AgentSelector", func() { + var ( + ctx context.Context + db *gorm.DB + registry *NodeRegistry + conns *stubConnections + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + ctx = context.Background() + db = testutil.SetupTestDB() + var err error + registry, err = NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + conns = &stubConnections{} + }) + + register := func(name, nodeType string) string { + GinkgoHelper() + node := &BackendNode{Name: name, NodeType: nodeType, Address: name + ":50051"} + Expect(registry.Register(ctx, node, true)).To(Succeed()) + fetched, err := registry.GetByName(ctx, name) + Expect(err).ToNot(HaveOccurred()) + return fetched.ID + } + + // The repeat count every preference assertion uses. The tie-break is random + // by design, so a single call proves nothing: with two candidates a + // selector that ignored the preference entirely would pass one run in two. + const picks = 20 + + It("prefers an agent whose tunnel THIS replica holds, so the call skips the relay", func() { + mine := register("agent-mine", NodeTypeAgent) + theirs := register("agent-theirs", NodeTypeAgent) + conns.held = []string{mine, theirs} + conns.heldByOwner = []string{mine} + + sel := NewAgentSelector(registry, conns, "me") + for range picks { + id, nodeType, err := sel.PickConnected(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(id).To(Equal(mine)) + // The node type is what a later re-broadcast checks its allow list + // against, and an unknown type is DENIED there. An empty one would + // present only as progress that never reaches a browser. + Expect(nodeType).To(Equal(NodeTypeAgent)) + Expect(nodeType).ToNot(BeEmpty()) + } + Expect(conns.owners).To(HaveLen(picks)) + Expect(conns.owners[0]).To(Equal("me"), + "the connection read must be asked which tunnels THIS replica holds") + }) + + It("takes a peer-held agent when this replica holds none", func() { + theirs := register("agent-theirs", NodeTypeAgent) + conns.held = []string{theirs} + + sel := NewAgentSelector(registry, conns, "me") + for range picks { + id, nodeType, err := sel.PickConnected(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(id).To(Equal(theirs)) + Expect(nodeType).To(Equal(NodeTypeAgent)) + } + }) + + It("spreads its picks over the agents it owns rather than pinning one", func() { + // Random, not "the first one". A selector that returned candidates[0] + // would pass every assertion above and send the whole deployment's MCP + // traffic to one worker. + a := register("agent-a", NodeTypeAgent) + b := register("agent-b", NodeTypeAgent) + conns.held = []string{a, b} + conns.heldByOwner = []string{a, b} + + sel := NewAgentSelector(registry, conns, "me") + seen := map[string]int{} + // 40 draws of a fair two-way choice miss one side with probability + // 2^-39, which is far below the flake floor of anything else here. + for range 2 * picks { + id, _, err := sel.PickConnected(ctx) + Expect(err).ToNot(HaveOccurred()) + seen[id]++ + } + Expect(seen).To(HaveKey(a)) + Expect(seen).To(HaveKey(b)) + }) + + Describe("when nothing can be picked", func() { + assertNoWorker := func(id, nodeType string, err error) { + GinkgoHelper() + Expect(err).To(MatchError(ErrNoAgentWorker)) + // The taxonomy assertion, and the reason this sentinel is its own. + // ErrWorkerUnroutable is what every path that DELETES a node_models + // row matches on, and IsWorkerAnswer is what a reap guard acts on. + // An empty agent fleet is evidence about neither: nothing was asked + // of any worker. + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeFalse()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + // An id returned beside the error would be dialled by a caller that + // checked the id first, which is a caller mistake this makes + // impossible rather than one it documents. + Expect(id).To(BeEmpty()) + Expect(nodeType).To(BeEmpty()) + } + + It("refuses when registered agents exist but none is connected", func() { + register("agent-a", NodeTypeAgent) + sel := NewAgentSelector(registry, conns, "me") + assertNoWorker(sel.PickConnected(ctx)) + }) + + It("refuses when the deployment has no agent nodes at all", func() { + sel := NewAgentSelector(registry, conns, "me") + assertNoWorker(sel.PickConnected(ctx)) + }) + + It("refuses when it was built with no way to read connections", func() { + assertNoWorker(NewAgentSelector(registry, nil, "me").PickConnected(ctx)) + }) + }) + + It("never offers a BACKEND worker, which serves no MCP verb at all", func() { + backend := register("backend-1", NodeTypeBackend) + agent := register("agent-a", NodeTypeAgent) + // The connection read is told everything is connected; only the + // candidate list keeps the backend worker out, so a selector that + // listed every node would pick it and get a 404 it would read as a + // version skew. + conns.held = []string{backend, agent} + conns.heldByOwner = []string{backend, agent} + + sel := NewAgentSelector(registry, conns, "me") + for range picks { + id, _, err := sel.PickConnected(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(id).To(Equal(agent)) + } + Expect(conns.asked[0]).To(ConsistOf(agent)) + }) + + It("never offers an agent an admin has not approved", func() { + node := &BackendNode{Name: "agent-pending", NodeType: NodeTypeAgent, Address: "p:50051"} + Expect(registry.Register(ctx, node, false)).To(Succeed()) + pending, err := registry.GetByName(ctx, "agent-pending") + Expect(err).ToNot(HaveOccurred()) + Expect(pending.Status).To(Equal(StatusPending)) + conns.held = []string{pending.ID} + conns.heldByOwner = []string{pending.ID} + + sel := NewAgentSelector(registry, conns, "me") + _, _, err = sel.PickConnected(ctx) + Expect(err).To(MatchError(ErrNoAgentWorker)) + Expect(conns.asked[0]).To(BeEmpty()) + }) + + It("never offers an agent an operator has put into draining", func() { + id := register("agent-draining", NodeTypeAgent) + Expect(registry.MarkDraining(ctx, id)).To(Succeed()) + conns.held = []string{id} + conns.heldByOwner = []string{id} + + sel := NewAgentSelector(registry, conns, "me") + _, _, err := sel.PickConnected(ctx) + Expect(err).To(MatchError(ErrNoAgentWorker)) + }) + + It("still offers an agent the health monitor has marked unhealthy", func() { + // The invariant, at the one place in this task it can break subtly. + // Whether a request can reach this worker RIGHT NOW is the connection + // rows' answer; a health verdict is written on another clock, and + // letting it filter the candidates would refuse a worker that is + // connected and answering because a probe was late. + id := register("agent-a", NodeTypeAgent) + Expect(registry.MarkUnhealthy(ctx, id)).To(Succeed()) + conns.held = []string{id} + conns.heldByOwner = []string{id} + + sel := NewAgentSelector(registry, conns, "me") + picked, _, err := sel.PickConnected(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(picked).To(Equal(id)) + }) + + It("reports a connection read that failed as neither an answer nor a route verdict", func() { + register("agent-a", NodeTypeAgent) + conns.err = errors.New("the database would not answer") + + sel := NewAgentSelector(registry, conns, "me") + id, nodeType, err := sel.PickConnected(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("the database would not answer")) + // A database that would not answer says nothing about any worker. + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeFalse()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + Expect(errors.Is(err, ErrNoAgentWorker)).To(BeFalse()) + Expect(id).To(BeEmpty()) + Expect(nodeType).To(BeEmpty()) + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 1c6388b8c5e9..0bd859a22193 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -145,7 +145,7 @@ Both **backend** and **agent** nodes are issued one. Earlier releases minted a c An agent worker's tunnel carries only the `http` tag: it runs no backend processes, so it does not offer the `grpc` tag at all. Its control server binds `127.0.0.1` on a port chosen by the kernel and advertises it nowhere, so an agent worker still opens no inbound port. -**An agent worker still requires `--nats-url`.** The tunnel is an addition, not a replacement: agent jobs, MCP execution, MCP CI jobs and `nodes..backend.stop` all still travel on NATS. What the tunnel changes today is that the frontend *can* reach an agent worker directly, which is what later releases move those verbs onto. +**An agent worker still requires `--nats-url`.** The tunnel is not yet a replacement: agent jobs, MCP CI jobs and `nodes..backend.stop` still travel on NATS. MCP tool execution and MCP discovery no longer do - they are control RPCs on the tunnel, chosen by the frontend rather than by a queue group (see [MCP in Distributed Mode](#mcp-in-distributed-mode)) - and an agent worker's minted JWT no longer grants `mcp.tools.execute` or `mcp.discovery`. The tunnel lands on exactly one frontend replica, and that replica records itself as the owner of the worker's connection in the `node_connections` table. When the socket dies the claim is dropped, but the row stays behind with no owner and a `disconnected_at` stamp, so a worker that is re-dialling the load balancer can be told from one that has never connected. The row is deleted once that departure is older than ten liveness windows (five minutes). If the replica stalls long enough for its peers to reap it, it re-claims the tunnels it still holds on a live session as soon as it re-registers, skipping any whose socket has already gone. That re-claim needs the replica to have an advertised address: without one it never had an instance row to begin with, and its tunnels are usable only by the replica holding them. @@ -437,7 +437,7 @@ A frontend replica that dies mid-load does not wedge the model: the job row carr **This section is about agent workers and the frontend.** A serve-backend worker opens no NATS connection, so none of it applies to one; its own credential is the tunnel token it gets at registration, and its control plane is authenticated by `LOCALAI_REGISTRATION_TOKEN`. An agent worker now has both: a NATS credential for everything still on the bus, and a tunnel token plus the same `LOCALAI_REGISTRATION_TOKEN` bearer check in front of its control server. -By default, NATS connections are anonymous: any client that can reach port `4222` may publish the subjects still carried on it. Those are the agent-worker job subjects, MCP, and the frontend's own cross-replica events. `nodes..backend.install` and its nine siblings are **not** among them - they are HTTP routes on the worker's tunnel, see [The worker control plane](#the-worker-control-plane). Enable JWT auth to scope agent workers to their own subjects and give the frontend a dedicated service credential. +By default, NATS connections are anonymous: any client that can reach port `4222` may publish the subjects still carried on it. Those are the agent-worker job subjects and the frontend's own cross-replica events. `nodes..backend.install` and its nine siblings are **not** among them - they are HTTP routes on the worker's tunnel, see [The worker control plane](#the-worker-control-plane). Enable JWT auth to scope agent workers to their own subjects and give the frontend a dedicated service credential. | Flag | Env Var | Description | |------|---------|-------------| @@ -1111,7 +1111,7 @@ local-ai agent-worker \ Agent workers: - Execute agent chat messages dispatched via NATS - Run MCP CI jobs (with access to MCP servers via docker) -- Handle MCP tool discovery and execution requests from the frontend +- Handle MCP tool discovery and execution requests, which the frontend sends over the worker's own tunnel - Get auto-provisioned API keys during registration for calling the inference API In the docker-compose setup, the agent worker mounts the Docker socket so it can run MCP stdio servers (e.g., `docker run` commands): @@ -1125,12 +1125,31 @@ agent-worker-1: ## MCP in Distributed Mode -MCP servers configured in model configs work in distributed mode. The frontend routes MCP operations through NATS to agent workers: +MCP servers configured in model configs work in distributed mode. The frontend holds no MCP sessions of its own - creating one usually means running `docker`, which is what an agent worker is for - so it asks an agent worker instead: -- **MCP discovery** (`GET /v1/mcp/servers/:model`): routed to agent workers which create sessions and return server info -- **MCP tool execution** (during `/v1/chat/completions`): tool calls are routed to agent workers via NATS request-reply +- **MCP discovery** (`GET /v1/mcp/servers/:model`): the frontend picks a connected agent worker and asks it over that worker's tunnel; the worker creates the sessions and returns server info +- **MCP tool execution** (during `/v1/chat/completions`): the same, per tool call - **MCP CI jobs**: executed entirely on agent workers with access to docker for stdio-based MCP servers +### How a frontend picks an agent worker + +Discovery and tool execution used to be NATS request-reply onto a queue group, where the broker chose the worker and neither side could say which one had answered. They are now an ordinary control RPC plus a **selection**, because a queue group was only ever a way of choosing a subscriber, and choosing is a query: + +1. The frontend lists the approved agent nodes that are not draining. +2. It asks the `node_connections` table, in one statement joined against live replicas, which of those tunnels a **live** frontend replica currently holds. +3. It prefers one **this** replica holds, so the call skips the relay hop entirely, and otherwise takes any connected one at random. A broker's hidden balancing could not make that choice. +4. It issues the control RPC over that worker's tunnel, relayed through the owning replica when another one holds it. + +A worker that answers with an error - "no such tool", "that MCP server refused your arguments" - is the worker's own answer and is returned to you unchanged; it is never re-tried on a second worker, because that would run a tool twice. A call that never reached a worker is re-tried, against a different worker, at most three times. + +If no agent worker in the deployment currently holds a tunnel, the request fails with a message saying so. That is a statement about this moment, not about any particular worker: nothing is marked unhealthy and no model is evicted because of it. + +### MCP prompts and resources are not available in distributed mode + +`GET /v1/mcp/prompts/:model`, `POST /v1/mcp/prompts/:model/:prompt`, `GET /v1/mcp/resources/:model` and `POST /v1/mcp/resources/:model/read` are served **only** from MCP sessions held by the frontend process, and in distributed mode it holds none. There is no verb that carries prompts or resources to an agent worker. + +In distributed mode these four endpoints answer **501 Not Implemented** with the reason in the body. Earlier releases answered `200` with an empty list, which was indistinguishable from a model that genuinely has no prompts. This is a pre-existing gap rather than a consequence of moving MCP off the bus - tools and discovery had a carrier to an agent worker and these never did - and single-binary deployments are unaffected. + ## vLLM Multi-Node (Data-Parallel) A single vLLM model can span multiple GPU nodes via data parallelism: the head node serves the OpenAI API and runs the local DP ranks, follower nodes run vanilla `vllm serve --headless` and speak ZMQ directly to the head. LocalAI's role is starting the follower processes and surfacing them in the admin UI; the cross-rank tensor traffic is vLLM's own. @@ -1570,7 +1589,7 @@ Notes: - It is **not** the same as the worker being gone, and nothing acts on it as if it were. A model on an unroutable worker is not reaped, its rows are left alone, and the node is not demoted: doing any of those on a lost route is how a rolling frontend restart turns into a fleet-wide eviction. - Check the worker process is running and that it has an open tunnel (`opened a tunnelled stream to a worker` in the frontend log, and the worker's own dial/reconnect lines). A worker behind a load balancer that keeps reconnecting is usually an idle-timeout or WebSocket-upgrade problem at the proxy; see the tunnel section above. - **`no route` is not `gone`, and nothing in the frontend reads it as such.** A worker is declared **gone** by one mechanism only: no live frontend replica holds its tunnel *and* its departure is older than `--worker-reconnect-grace`. That is a fact recorded in the shared database, so every replica answers it identically. "No route" is one replica failing to reach a worker right now, and it is not evidence about the worker at all. -- Older releases decided absence from `nats: no responders available for request`, which was one frontend's observation that nobody answered *it* within a request budget. Two replicas asking at the same moment could disagree and demote each other's workers. That signal is gone from the scheduler; if you still see the message, it concerns only the subjects that remain on the bus (agent-worker jobs and MCP), never a serve-backend worker. +- Older releases decided absence from `nats: no responders available for request`, which was one frontend's observation that nobody answered *it* within a request budget. Two replicas asking at the same moment could disagree and demote each other's workers. That signal is gone from the scheduler; if you still see the message, it concerns only the subjects that remain on the bus (agent-worker jobs and MCP CI), never a serve-backend worker. **A worker fills its own disk over time:** - A request that carries a file (an image, an audio clip, a video) stages that file below the worker's HTTP staging or S3 cache `ephemeral/` directory. The frontend releases each request-owned input when inference finishes, and the worker reserves capacity before accepting it. diff --git a/pkg/natsauth/permissions.go b/pkg/natsauth/permissions.go index 4c081c672841..9145e0b3eb8d 100644 --- a/pkg/natsauth/permissions.go +++ b/pkg/natsauth/permissions.go @@ -29,6 +29,12 @@ func WorkerPermissions(nodeID, nodeType string) (pubAllow, subAllow []string) { case "agent": // Agent workers consume queue workloads; they must not handle backend.install. // Keep this list in sync with the subscriptions in core/cli/agent_worker.go. + // + // MCP tool execution and discovery are NOT here any more: they are + // control RPCs on the tunnel the worker holds, chosen by the frontend + // rather than by a queue group. Removing them narrowed this list; it + // must never be narrowed to nothing, because NATS reads an EMPTY allow + // list as no restriction at all. subAllow = []string{ "agent.execute", "agent.*.cancel", @@ -37,9 +43,7 @@ func WorkerPermissions(nodeID, nodeType string) (pubAllow, subAllow []string) { "jobs.*.cancel", "jobs.*.progress", "jobs.*.result", - "jobs.mcp-ci.new", // MCP CI jobs dispatched to agent workers - "mcp.tools.execute", - "mcp.discovery", + "jobs.mcp-ci.new", // MCP CI jobs dispatched to agent workers prefix + ".backend.stop", // stop events drive MCP session cleanup "staging.*.progress", "_INBOX.>", diff --git a/pkg/natsauth/permissions_coverage_test.go b/pkg/natsauth/permissions_coverage_test.go index e7ed7133212f..f21e51d2f285 100644 --- a/pkg/natsauth/permissions_coverage_test.go +++ b/pkg/natsauth/permissions_coverage_test.go @@ -99,15 +99,33 @@ var _ = Describe("WorkerPermissions subject coverage", func() { Context("agent worker", func() { // node_type "agent"; subjects from core/cli/agent_worker.go. pub, sub := natsauth.WorkerPermissions(nodeID, "agent") - _ = pub subscribed := []string{ messaging.SubjectAgentExecute, // dispatcher (default --agent-subject) - messaging.SubjectMCPToolExecute, // QueueSubscribeReply - messaging.SubjectMCPDiscovery, // QueueSubscribeReply messaging.SubjectMCPCIJobsNew, // QueueSubscribe — jobs.mcp-ci.new messaging.SubjectNodeBackendStop(nodeID), // Subscribe — MCP session cleanup } + + // The half that catches a narrowing going too far. NATS reads an EMPTY + // allow list as NO restriction, so a branch trimmed to nothing does not + // lock an agent worker down, it opens the whole account to it. + It("keeps the agent worker's allow lists non-empty", func() { + Expect(sub).ToNot(BeEmpty(), + "an empty allow list is unrestricted in NATS, not restrictive") + Expect(pub).ToNot(BeEmpty(), + "an empty allow list is unrestricted in NATS, not restrictive") + }) + + // MCP execution and discovery are control RPCs on the worker's tunnel + // now, chosen by the frontend rather than by a queue group. An agent + // worker subscribes to neither subject, so a JWT that still granted + // them would be granting a subscription nothing serves. + for _, subject := range []string{"mcp.tools.execute", "mcp.discovery"} { + It("no longer grants an agent worker "+subject, func() { + Expect(anyAllows(sub, subject)).To(BeFalse(), + "agent JWT sub allow-list %v still covers %s", sub, subject) + }) + } for _, subject := range subscribed { It("allows subscribing to "+subject, func() { Expect(anyAllows(sub, subject)).To(BeTrue(), diff --git a/tests/e2e/distributed/mcp_nats_test.go b/tests/e2e/distributed/mcp_nats_test.go deleted file mode 100644 index e0e868e4d29f..000000000000 --- a/tests/e2e/distributed/mcp_nats_test.go +++ /dev/null @@ -1,177 +0,0 @@ -package distributed_test - -import ( - "encoding/json" - "sync/atomic" - "time" - - "github.com/mudler/LocalAI/core/config" - mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp" - mcpRemote "github.com/mudler/LocalAI/core/services/mcp" - "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/pkg/functions" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("MCP NATS Routing", Label("Distributed"), func() { - var ( - infra *TestInfra - ) - - BeforeEach(func() { - infra = SetupNATSOnly() - }) - - Context("MCP Tool Execution via NATS", func() { - It("should execute MCP tool call via NATS request-reply", func() { - // Mock worker: subscribe to tool execute requests - sub, err := infra.NC.QueueSubscribeReply(messaging.SubjectMCPToolExecute, messaging.QueueAgentWorkers, func(data []byte, reply func([]byte)) { - var req mcpRemote.MCPToolRequest - Expect(json.Unmarshal(data, &req)).To(Succeed()) - Expect(req.ModelName).To(Equal("test-model")) - Expect(req.ToolName).To(Equal("weather")) - Expect(req.Arguments).To(HaveKeyWithValue("city", "London")) - - resp, _ := json.Marshal(mcpRemote.MCPToolResponse{ - Result: "Weather in London: 15°C, cloudy", - }) - reply(resp) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - FlushNATS(infra.NC) - - // Frontend side: pass NATS client and call remote - result, err := mcpTools.ExecuteMCPToolCallRemote( - infra.Ctx, - infra.NC, - "test-model", - config.MCPGenericConfig[config.MCPRemoteServers]{}, - config.MCPGenericConfig[config.MCPSTDIOServers]{}, - "weather", - `{"city": "London"}`, - ) - Expect(err).ToNot(HaveOccurred()) - Expect(result).To(Equal("Weather in London: 15°C, cloudy")) - }) - - It("should propagate remote MCP tool errors", func() { - sub, err := infra.NC.QueueSubscribeReply(messaging.SubjectMCPToolExecute, messaging.QueueAgentWorkers, func(data []byte, reply func([]byte)) { - resp, _ := json.Marshal(mcpRemote.MCPToolResponse{ - Error: "tool 'unknown' not found", - }) - reply(resp) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - FlushNATS(infra.NC) - - _, err = mcpTools.ExecuteMCPToolCallRemote( - infra.Ctx, - infra.NC, - "test-model", - config.MCPGenericConfig[config.MCPRemoteServers]{}, - config.MCPGenericConfig[config.MCPSTDIOServers]{}, - "unknown", - "{}", - ) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("tool 'unknown' not found")) - }) - }) - - Context("MCP Discovery via NATS", func() { - It("should discover MCP servers via NATS request-reply", func() { - sub, err := infra.NC.QueueSubscribeReply(messaging.SubjectMCPDiscovery, messaging.QueueAgentWorkers, func(data []byte, reply func([]byte)) { - var req mcpRemote.MCPDiscoveryRequest - Expect(json.Unmarshal(data, &req)).To(Succeed()) - Expect(req.ModelName).To(Equal("discovery-model")) - - resp, _ := json.Marshal(mcpRemote.MCPDiscoveryResponse{ - Servers: []mcpRemote.MCPServerInfo{ - {Name: "weather-server", Type: "remote", Tools: []string{"get_weather", "get_forecast"}}, - {Name: "db-server", Type: "stdio", Tools: []string{"query_db"}}, - }, - Tools: []mcpRemote.MCPToolDef{ - {ServerName: "weather-server", ToolName: "get_weather", Function: functions.Function{Name: "get_weather", Description: "Get weather"}}, - {ServerName: "weather-server", ToolName: "get_forecast", Function: functions.Function{Name: "get_forecast", Description: "Get forecast"}}, - {ServerName: "db-server", ToolName: "query_db", Function: functions.Function{Name: "query_db", Description: "Query database"}}, - }, - }) - reply(resp) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - FlushNATS(infra.NC) - - result, err := mcpTools.DiscoverMCPToolsRemote( - infra.Ctx, - infra.NC, - "discovery-model", - config.MCPGenericConfig[config.MCPRemoteServers]{}, - config.MCPGenericConfig[config.MCPSTDIOServers]{}, - ) - Expect(err).ToNot(HaveOccurred()) - Expect(result.Servers).To(HaveLen(2)) - Expect(result.Servers[0].Name).To(Equal("weather-server")) - Expect(result.Servers[0].Tools).To(ConsistOf("get_weather", "get_forecast")) - Expect(result.Tools).To(HaveLen(3)) - Expect(result.Tools[2].ToolName).To(Equal("query_db")) - }) - }) - - Context("QueueSubscribeReply", func() { - It("should support queue subscribe with request-reply round-trip", func() { - // Subscribe with queue group - sub, err := infra.NC.QueueSubscribeReply("test.echo", "echo-workers", func(data []byte, reply func([]byte)) { - // Echo back the request data with a prefix - reply(append([]byte("echo:"), data...)) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - FlushNATS(infra.NC) - - // Send request and wait for reply - replyData, err := infra.NC.Request("test.echo", []byte("hello"), 5*time.Second) - Expect(err).ToNot(HaveOccurred()) - Expect(string(replyData)).To(Equal("echo:hello")) - }) - - It("should load-balance requests across queue subscribers", func() { - var worker1Count, worker2Count atomic.Int32 - - sub1, _ := infra.NC.QueueSubscribeReply("test.lb", "lb-workers", func(data []byte, reply func([]byte)) { - worker1Count.Add(1) - reply([]byte("w1")) - }) - defer sub1.Unsubscribe() - - sub2, _ := infra.NC.QueueSubscribeReply("test.lb", "lb-workers", func(data []byte, reply func([]byte)) { - worker2Count.Add(1) - reply([]byte("w2")) - }) - defer sub2.Unsubscribe() - - FlushNATS(infra.NC) - - // Send multiple requests - for range 10 { - _, err := infra.NC.Request("test.lb", []byte("req"), 5*time.Second) - Expect(err).ToNot(HaveOccurred()) - } - - // Both workers should have handled some requests - total := worker1Count.Load() + worker2Count.Load() - Expect(total).To(Equal(int32(10))) - // NATS typically distributes evenly, but we just check both got work - Expect(worker1Count.Load()).To(BeNumerically(">", 0)) - Expect(worker2Count.Load()).To(BeNumerically(">", 0)) - }) - }) -}) diff --git a/tests/e2e/distributed/nats_jwt_test.go b/tests/e2e/distributed/nats_jwt_test.go index b6b2343850fa..dd0c23253f92 100644 --- a/tests/e2e/distributed/nats_jwt_test.go +++ b/tests/e2e/distributed/nats_jwt_test.go @@ -108,13 +108,10 @@ var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { Expect(err).ToNot(HaveOccurred()) DeferCleanup(nc.Close) - // Mirror core/cli/agent_worker.go exactly. - _, err = nc.QueueSubscribeReply(messaging.SubjectMCPToolExecute, messaging.QueueAgentWorkers, func([]byte, func([]byte)) {}) - Expect(err).ToNot(HaveOccurred(), "agent JWT must allow %s", messaging.SubjectMCPToolExecute) - - _, err = nc.QueueSubscribeReply(messaging.SubjectMCPDiscovery, messaging.QueueAgentWorkers, func([]byte, func([]byte)) {}) - Expect(err).ToNot(HaveOccurred(), "agent JWT must allow %s", messaging.SubjectMCPDiscovery) - + // Mirror core/cli/agent_worker.go exactly. MCP tool execution and + // discovery are absent because they are no longer bus subjects at all: + // the frontend selects an agent worker itself and reaches it with a + // control RPC over the tunnel that worker holds. _, err = nc.QueueSubscribe(messaging.SubjectMCPCIJobsNew, messaging.QueueWorkers, func([]byte) {}) Expect(err).ToNot(HaveOccurred(), "agent JWT must allow %s (MCP CI jobs)", messaging.SubjectMCPCIJobsNew) diff --git a/tests/e2e/distributed/nats_queue_reply_test.go b/tests/e2e/distributed/nats_queue_reply_test.go new file mode 100644 index 000000000000..34c0b5c2e9b7 --- /dev/null +++ b/tests/e2e/distributed/nats_queue_reply_test.go @@ -0,0 +1,77 @@ +package distributed_test + +import ( + "sync/atomic" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// What is left here is the messaging layer's own request-reply behaviour. +// +// The MCP specs that used to live beside it went with their carrier: MCP tool +// execution and discovery were the only subjects that combined a queue group +// WITH a reply, and they are now a selection plus an ordinary control RPC over +// the agent worker's tunnel. Their round trip is exercised against a real +// tunnel, a real connection row and a real relay in +// core/services/nodes/agent_control_test.go. +var _ = Describe("NATS queue request-reply", Label("Distributed"), func() { + var ( + infra *TestInfra + ) + + BeforeEach(func() { + infra = SetupNATSOnly() + }) + + Context("QueueSubscribeReply", func() { + It("should support queue subscribe with request-reply round-trip", func() { + // Subscribe with queue group + sub, err := infra.NC.QueueSubscribeReply("test.echo", "echo-workers", func(data []byte, reply func([]byte)) { + // Echo back the request data with a prefix + reply(append([]byte("echo:"), data...)) + }) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = sub.Unsubscribe() }() + + FlushNATS(infra.NC) + + // Send request and wait for reply + replyData, err := infra.NC.Request("test.echo", []byte("hello"), 5*time.Second) + Expect(err).ToNot(HaveOccurred()) + Expect(string(replyData)).To(Equal("echo:hello")) + }) + + It("should load-balance requests across queue subscribers", func() { + var worker1Count, worker2Count atomic.Int32 + + sub1, _ := infra.NC.QueueSubscribeReply("test.lb", "lb-workers", func(data []byte, reply func([]byte)) { + worker1Count.Add(1) + reply([]byte("w1")) + }) + defer func() { _ = sub1.Unsubscribe() }() + + sub2, _ := infra.NC.QueueSubscribeReply("test.lb", "lb-workers", func(data []byte, reply func([]byte)) { + worker2Count.Add(1) + reply([]byte("w2")) + }) + defer func() { _ = sub2.Unsubscribe() }() + + FlushNATS(infra.NC) + + // Send multiple requests + for range 10 { + _, err := infra.NC.Request("test.lb", []byte("req"), 5*time.Second) + Expect(err).ToNot(HaveOccurred()) + } + + // Both workers should have handled some requests + total := worker1Count.Load() + worker2Count.Load() + Expect(total).To(Equal(int32(10))) + // NATS typically distributes evenly, but we just check both got work + Expect(worker1Count.Load()).To(BeNumerically(">", 0)) + Expect(worker2Count.Load()).To(BeNumerically(">", 0)) + }) + }) +}) From 4c2d696785df78ca59552ccf6ea005cb9d82addf Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 15:32:23 +0000 Subject: [PATCH 086/153] test(distributed): name the replica endpoint the router now requires Two specs in the distributed e2e suite have been red since 1cf847f29, which stopped workers advertising an address and removed every fall-back to the node's own endpoint. After that commit a replica row must name the loopback endpoint of its own backend process: DistributedModelStore.Range refuses to list a replica whose backend process is unnamed, and SmartRouter treats an unnamed warm row as naming no process and cold-loads instead. 1cf847f29 updated the unit specs under core/services/nodes for the new contract but not tests/e2e/distributed, and the phase's closing verification ran test-e2e-cluster rather than test-e2e-distributed, so nothing reported it. The task brief named six later commits as candidates and called 671785621 known good; both are wrong, and 671785621 fails these two specs as well. The fix is the scripted input, not the guard. Both specs wrote an empty address, which is a row this release cannot produce, since installBackendOnNode refuses an install that names no address. The dedup spec is repaired rather than merely un-reddened. With an empty address its DB row was dropped by the unnamed-replica guard before Range ever consulted the seen-set, so deleting the dedup check left it green: it asserted nothing. With the endpoint named, removing that check reddens it. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/distributed_store_test.go | 12 ++++++++++-- tests/e2e/distributed/prefix_cache_routing_test.go | 8 ++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/e2e/distributed/distributed_store_test.go b/tests/e2e/distributed/distributed_store_test.go index d35a7e0439a8..8d5a096bf4d4 100644 --- a/tests/e2e/distributed/distributed_store_test.go +++ b/tests/e2e/distributed/distributed_store_test.go @@ -117,7 +117,11 @@ var _ = Describe("DistributedModelStore", Label("Distributed"), func() { Name: "range-node", Address: "range:9000", } Expect(registry.Register(context.Background(), node, true)).To(Succeed()) - Expect(registry.SetNodeModel(context.Background(), node.ID, "db-only-model", 0, "loaded", "", 0)).To(Succeed()) + // The replica address is load-bearing, not decoration. Range lists a + // remote replica by the endpoint its OWN backend process listens on; + // a row that names none is skipped, because workers advertise nothing + // and there is no node address left to fall back to. + Expect(registry.SetNodeModel(context.Background(), node.ID, "db-only-model", 0, "loaded", "127.0.0.1:59001", 0)).To(Succeed()) visited := map[string]bool{} dStore.Range(func(id string, m *model.Model) bool { @@ -132,7 +136,11 @@ var _ = Describe("DistributedModelStore", Label("Distributed"), func() { Name: "dup-node", Address: "dup:9000", } Expect(registry.Register(context.Background(), node, true)).To(Succeed()) - Expect(registry.SetNodeModel(context.Background(), node.ID, "shared-model", 0, "loaded", "", 0)).To(Succeed()) + // A named replica address is what makes this spec test deduplication + // at all: an unnamed row is dropped by Range before the seen-set is + // ever consulted, so the count would stay at one even with the + // dedup check removed. + Expect(registry.SetNodeModel(context.Background(), node.ID, "shared-model", 0, "loaded", "127.0.0.1:59002", 0)).To(Succeed()) // Also in local store localStore.Set("shared-model", model.NewModel("shared-model", "dup:9000", nil)) diff --git a/tests/e2e/distributed/prefix_cache_routing_test.go b/tests/e2e/distributed/prefix_cache_routing_test.go index e899160ea2fd..c1b465f66817 100644 --- a/tests/e2e/distributed/prefix_cache_routing_test.go +++ b/tests/e2e/distributed/prefix_cache_routing_test.go @@ -129,8 +129,12 @@ var _ = Describe("Prefix-cache aware routing", Label("Distributed"), func() { Expect(registry.Register(context.Background(), nodeY, true)).To(Succeed()) nodeXID = nodeX.ID nodeYID = nodeY.ID - Expect(registry.SetNodeModel(context.Background(), nodeXID, model, 0, "loaded", "", 0)).To(Succeed()) - Expect(registry.SetNodeModel(context.Background(), nodeYID, model, 0, "loaded", "", 0)).To(Succeed()) + // Each replica row names the endpoint its own backend process listens + // on. Without one the router cannot tell a warm replica from a missing + // one: it treats the row as naming no backend process and cold-loads, + // so nothing here would exercise prefix affinity at all. + Expect(registry.SetNodeModel(context.Background(), nodeXID, model, 0, "loaded", "127.0.0.1:59051", 0)).To(Succeed()) + Expect(registry.SetNodeModel(context.Background(), nodeYID, model, 0, "loaded", "127.0.0.1:59052", 0)).To(Succeed()) factory := &prefixStubClientFactory{client: &prefixStubBackend{healthResult: true}} router = nodes.NewSmartRouter(registry, nodes.SmartRouterOptions{ From 0d61cc179baefb3f32da12e498ae8411c6077f28 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 16:41:59 +0000 Subject: [PATCH 087/153] feat(distributed): stop a backend on one route, whatever the worker is nodes..backend.stop was the last worker-facing NATS subject, and it existed only because ONE publisher had not moved. An agent worker already mounted workerctl.PathBackendStop on the tunnel it holds, and a backend worker already took its stop there, so RemoteUnloaderAdapter branched on NodeType to pick a carrier for a verb both kinds of worker served the same way. The branch is gone, and with it nodeTypeOf and its NodeTypeBackend default, which removes one of the ten NodeType branches left to sweep. The adapter loses its messaging.MessagingClient outright rather than keeping an unused field: it now holds no publisher, so re-routing any verb back onto the bus is a change to the struct and to every caller of the constructor, and does not compile until all of them agree. messaging.SubjectNodeBackendStop and subjectNodePrefix are deleted, the agent worker's subscription with them. pkg/natsauth drops the per-node backend.stop grant from the agent SUB list. That is a narrowing of eleven entries to ten, never to nothing: NATS reads an EMPTY allow list as unrestricted, so the coverage spec asserts both that the retired subject is no longer covered and that the queue subjects an agent worker lives on still are. The e2e half proves it against a real enforcing server: one spec subscribes successfully on an agent-minted JWT, the next is refused the retired subject on a JWT minted the same way. Both halves of the old split were pinned, so both pins are re-aimed rather than deleted, and the two node types are asserted separately rather than as one parameterised case, because only two cases can show that the two used to differ. Three assertions that the adapter published nothing are deleted instead: with no publisher to hold, no change could ever redden them. The CLI's handler set moves into agentWorkerControlHandlers so a spec can stand it up and post to it. That wiring was a bare literal no spec pinned, and deleting the subscription made it the ONLY carrier for backend.stop: a dropped field would have been a 404 the frontend reads as a worker too old to serve the verb, and nothing in the repo would have noticed. Mutations: the agent branch restored off the control route reddens two specs; the backend branch restored, separately, reddens five; PathBackendDelete in place of PathBackendStop reddens nine across both node types; dropping the CLI wiring line reddens the new wiring table; re-adding the allow-list entry reddens the unit spec and the JWT e2e spec; and restoring the publisher for real does not compile. Four comments this change falsified are fixed, in core/cli, pkg/model and the distributed-mode docs, which now say both kinds of worker serve POST /v1/control/backend/stop and what each does with it. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 1 - core/cli/agent_worker.go | 62 ++++----- core/cli/agent_worker_test.go | 41 +++++- core/services/messaging/subjects.go | 41 ++---- .../messaging/subjects_upgrade_test.go | 16 --- .../nodes/managers_agent_node_test.go | 2 +- .../nodes/managers_distributed_test.go | 4 +- core/services/nodes/reconciler_test.go | 2 +- core/services/nodes/unloader.go | 79 +++-------- .../nodes/unloader_stale_rows_test.go | 2 +- core/services/nodes/unloader_test.go | 125 +++++++++--------- core/services/nodes/unloader_upgrade_test.go | 6 +- docs/content/features/distributed-mode.md | 11 +- pkg/model/connection_evicting_client.go | 4 +- pkg/model/remote_unroutable_internal_test.go | 4 +- pkg/natsauth/permissions.go | 23 ++-- pkg/natsauth/permissions_coverage_test.go | 34 +++-- .../distributed/distributed_full_flow_test.go | 4 +- tests/e2e/distributed/managers_test.go | 6 +- tests/e2e/distributed/nats_jwt_test.go | 34 ++++- tests/e2e/distributed/node_lifecycle_test.go | 56 ++++++-- tests/e2e/distributed/router_tracking_test.go | 2 +- 22 files changed, 300 insertions(+), 259 deletions(-) diff --git a/core/application/distributed.go b/core/application/distributed.go index 763f69df7fbe..201450c08ecb 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -490,7 +490,6 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // Create RemoteUnloaderAdapter — needed by SmartRouter and startup.go remoteUnloader := nodes.NewRemoteUnloaderAdapter( registry, - natsClient, controlClient, cfg.Distributed.BackendInstallTimeoutOrDefault(), cfg.Distributed.BackendUpgradeTimeoutOrDefault(), diff --git a/core/cli/agent_worker.go b/core/cli/agent_worker.go index 0b2ee70e2a2c..ac75011744ae 100644 --- a/core/cli/agent_worker.go +++ b/core/cli/agent_worker.go @@ -32,10 +32,11 @@ import ( // receives the full agent config and skills in the NATS job payload, so it // does not need direct database access. // -// It also holds one tunnel to the frontend, so the frontend can reach its MCP -// verbs by RPC without the worker opening an inbound port. The tunnel is an -// ADDITION: --nats-url is still required, and every job, every fan-out event -// and the node backend.stop subject still travel on the bus. +// It also holds one tunnel to the frontend, so the frontend can reach its +// control verbs by RPC without the worker opening an inbound port. No verb the +// frontend addresses to THIS worker travels on the bus any more. The tunnel is +// still an ADDITION rather than a replacement: --nats-url is required, because +// every job and every fan-out event does. // // Usage: // @@ -196,21 +197,14 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { // identity the dial names and the credential it presents, and BEFORE the // remaining NATS subscriptions, so that a frontend that reaches this worker // over the tunnel finds its verbs mounted rather than a 404 it would read - // as a version skew. + // as a version skew. backend.stop is among those verbs, and it is the only + // way a frontend states it now: there is no node subject left to publish on. agentCtl, err := agentworker.Start(shutdownCtx, agentworker.Options{ FrontendURL: cmd.RegisterTo, NodeID: nodeID, TunnelToken: credMgr.TunnelToken, ControlToken: cmd.RegistrationToken, - Handlers: agentworker.Config{ - MCPTool: serveMCPToolRequest, - MCPDiscovery: serveMCPDiscoveryRequest, - // The same cleanup the nodes..backend.stop subscription below - // performs, reachable over the tunnel on the path a backend worker - // already serves. Both are live: the subject is what the frontend - // still publishes on, and it is a later task that removes it. - BackendStop: dropMCPSessionsForBackend, - }, + Handlers: agentWorkerControlHandlers(), }) if err != nil { return fmt.Errorf("starting the agent worker control plane: %w", err) @@ -262,27 +256,6 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { return fmt.Errorf("subscribing to %s: %w", messaging.SubjectMCPCIJobsNew, err) } - // Subscribe to backend stop events to clean up cached MCP sessions. - // In the main application this is done via ml.OnModelUnload, but the agent - // worker has no model loader — we listen for the NATS stop event instead. - // - // It runs BESIDE the tunnel route mounted above, not instead of it, and - // both call dropMCPSessionsForBackend. The subject is still what the - // frontend publishes on; a later task is what moves it. Two carriers, one - // implementation, so which one delivered cannot change what happened. - if _, err := natsClient.Subscribe(messaging.SubjectNodeBackendStop(nodeID), func(data []byte) { - var req messaging.BackendStopRequest - if err := json.Unmarshal(data, &req); err != nil { - xlog.Warn("Agent worker could not decode a backend stop event", "error", err) - return - } - if err := dropMCPSessionsForBackend(context.Background(), req); err != nil { - xlog.Warn("Agent worker could not drop the MCP sessions of a stopped backend", "error", err) - } - }); err != nil { - return fmt.Errorf("subscribing to %s: %w", messaging.SubjectNodeBackendStop(nodeID), err) - } - xlog.Info("Agent worker ready, waiting for jobs", "subject", cmd.Subject, "queue", cmd.Queue) // Wait for an OS signal or an internal fatal condition (e.g. NATS @@ -611,3 +584,22 @@ func publishJobStatus(nc messaging.MessagingClient, jobID, status, message strin func publishJobResult(nc messaging.MessagingClient, jobID, status, result, errMsg string) { jobs.PublishJobResult(nc, jobID, status, result, errMsg) } + +// agentWorkerControlHandlers is every verb this worker serves on the tunnel it +// holds to the frontend. +// +// It is a function rather than a literal inside the start-up path so that a +// spec can stand the same set up and post to it. Each of these is the ONLY +// carrier for its verb: the queue subjects the two MCP verbs arrived on and the +// node subject backend.stop arrived on are all gone, so a field silently +// dropped here is a 404 at runtime, which the frontend reads as a worker too +// old to serve the verb. +func agentWorkerControlHandlers() agentworker.Config { + return agentworker.Config{ + MCPTool: serveMCPToolRequest, + MCPDiscovery: serveMCPDiscoveryRequest, + // Drops the MCP sessions cached for a backend that went away, on the + // path a backend worker serves by killing the process instead. + BackendStop: dropMCPSessionsForBackend, + } +} diff --git a/core/cli/agent_worker_test.go b/core/cli/agent_worker_test.go index 1e8e1d7ec5c2..680c49e1e3f4 100644 --- a/core/cli/agent_worker_test.go +++ b/core/cli/agent_worker_test.go @@ -3,12 +3,16 @@ package cli import ( "context" "encoding/json" + "net/http" + "net/http/httptest" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" mcpRemote "github.com/mudler/LocalAI/core/services/mcp" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" ) // The agent worker answers its MCP verbs on ONE carrier now: the control route @@ -47,11 +51,11 @@ var _ = Describe("The agent worker's MCP verbs", func() { }) var _ = Describe("The agent worker's backend stop", func() { - // One implementation behind both carriers, for the same reason: the bus - // subscription and the tunnel's control route both call this. + // One carrier now: the tunnel's control route. The node subject this used + // to arrive on is gone, so this implementation is reached one way only. It("treats a stop that names no backend as a no-op rather than a failure", func() { - // A malformed publish must not become a non-2xx the frontend reads as + // A malformed request must not become a non-2xx the frontend reads as // a worker it could not reach. Expect(dropMCPSessionsForBackend(context.Background(), messaging.BackendStopRequest{})).To(Succeed()) }) @@ -64,3 +68,34 @@ var _ = Describe("The agent worker's backend stop", func() { messaging.BackendStopRequest{Backend: "a-backend-this-worker-never-saw"})).To(Succeed()) }) }) + +// The wiring, pinned through the transport rather than by reading the struct. +// +// Every verb below has exactly one carrier: the control route. Dropping a field +// from agentWorkerControlHandlers mounts nothing for that path, the catch-all +// answers 404, and the frontend reads that 404 as a worker too old to serve the +// verb rather than as a wiring mistake. Nothing else in this repo would notice. +var _ = Describe("The agent worker's control-plane wiring", func() { + var base string + + BeforeEach(func() { + mux := http.NewServeMux() + agentWorkerControlHandlers().Register(mux) + srv := httptest.NewServer(mux) + DeferCleanup(srv.Close) + base = srv.URL + }) + + DescribeTable("mounts the verb an agent worker is the only server of", + func(path string) { + resp, err := http.Post(base+path, "application/json", strings.NewReader(`{}`)) //nolint:gosec,noctx // httptest server, no redirects to follow + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).ToNot(Equal(http.StatusNotFound), + "%s is not mounted; agentWorkerControlHandlers does not wire its handler", path) + }, + Entry("backend stop", workerctl.PathBackendStop), + Entry("mcp tool execute", workerctl.PathMCPToolExecute), + Entry("mcp discovery", workerctl.PathMCPDiscovery), + ) +}) diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index 657a99f92c3c..6b2985387ed8 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -136,27 +136,17 @@ func SubjectResponseCancel(responseID string) string { // Node Backend Lifecycle // -// The frontend's control plane no longer travels on NATS. The ten verbs that -// drove a worker's backend and model lifecycle are HTTP routes under +// The frontend's control plane does not travel on NATS at all. The ten verbs +// that drive a worker's backend and model lifecycle are HTTP routes under // workerctl.Prefix, served on the worker's own loopback server and reached -// through its tunnel, so a subject builder for any of them would be a subject -// nothing publishes and nothing subscribes to. -// -// ONE survives: backend.stop, and only for AGENT workers. They now hold a -// tunnel and mount that verb on it, so what keeps this subject alive is the -// PUBLISHER: nodes.RemoteUnloaderAdapter.stopBackend still sends an agent -// node's stop here rather than over its control route. The agent worker -// subscribes to it to drop the MCP sessions cached for a backend that is going -// away; see core/cli/agent_worker.go for the subscriber, which shares one -// implementation with the control route so the two carriers cannot diverge. +// through the tunnel it dials, for AGENT workers as much as for backend ones. +// There is no nodes.* subject left, and no builder for one, so a frontend +// cannot address a worker over the bus even by mistake. // // The request and reply types below are UNCHANGED and still live here: they are // the wire format of the control routes, byte for byte what the subjects // carried, so a worker and a frontend from different releases still understand // each other. -const ( - subjectNodePrefix = "nodes." -) // BackendInstallRequest is the payload for a backend.install control request. type BackendInstallRequest struct { @@ -266,27 +256,16 @@ type NodeBackendInfo struct { Digest string `json:"digest,omitempty"` } -// BackendStopRequest controls worker-side process shutdown. Force skips the -// best-effort Free RPC so a backend stuck serving a request can still be -// terminated by the watchdog. +// BackendStopRequest is the body of a backend.stop control request. A backend +// worker reads it as process shutdown and Force skips the best-effort Free RPC, +// so a backend stuck serving a request can still be terminated by the watchdog. +// An agent worker runs no backend processes and reads the same body as "that +// backend went away", closing the MCP sessions it had cached for it. type BackendStopRequest struct { Backend string `json:"backend"` Force bool `json:"force,omitempty"` } -// SubjectNodeBackendStop tells an AGENT worker that a backend is going away, so -// it can close the MCP sessions it cached for that backend. -// -// It is the one node subject left, and it is addressed only to agent nodes. A -// BACKEND worker takes its stop on workerctl.PathBackendStop over its tunnel, -// where it also kills the process and recycles the port; an agent worker runs -// no backend processes and only needs to hear that one went. An agent worker -// serves that same path over its own tunnel as well, so this subject is what -// the publisher has not moved off yet rather than the only way to reach one. -func SubjectNodeBackendStop(nodeID string) string { - return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".backend.stop" -} - type ModelStopRequest struct { ModelName string `json:"model_name"` ProcessKey string `json:"process_key"` diff --git a/core/services/messaging/subjects_upgrade_test.go b/core/services/messaging/subjects_upgrade_test.go index e1a059fac98d..df5bbebefa36 100644 --- a/core/services/messaging/subjects_upgrade_test.go +++ b/core/services/messaging/subjects_upgrade_test.go @@ -7,22 +7,6 @@ import ( "github.com/mudler/LocalAI/core/services/messaging" ) -// The surviving node subject. It is written out BY HAND and not derived from -// the builder: an agent worker built from another commit subscribes to this -// literal, and a renamed subject is silence that looks exactly like a worker -// that never started. -var _ = Describe("SubjectNodeBackendStop", func() { - It("returns the per-node stop subject an agent worker subscribes to", func() { - Expect(messaging.SubjectNodeBackendStop("abc")). - To(Equal("nodes.abc.backend.stop")) - }) - - It("sanitizes reserved NATS tokens in the node id", func() { - Expect(messaging.SubjectNodeBackendStop("a.b*c")). - To(Equal("nodes.a-b-c.backend.stop")) - }) -}) - var _ = Describe("BackendUpgradeRequest", func() { It("carries backend name, galleries JSON, and replica index", func() { req := messaging.BackendUpgradeRequest{ diff --git a/core/services/nodes/managers_agent_node_test.go b/core/services/nodes/managers_agent_node_test.go index b6b18ac3c1d3..5372d16de04a 100644 --- a/core/services/nodes/managers_agent_node_test.go +++ b/core/services/nodes/managers_agent_node_test.go @@ -40,7 +40,7 @@ var _ = Describe("Backend listing across mixed node types", func() { mc = newScriptedControlWorkers() mgr = &DistributedBackendManager{ local: stubLocalBackendManager{}, - adapter: NewRemoteUnloaderAdapter(nil, nil, mc.controlClient(), 3*time.Minute, 15*time.Minute), + adapter: NewRemoteUnloaderAdapter(nil, mc.controlClient(), 3*time.Minute, 15*time.Minute), registry: registry, } ctx = context.Background() diff --git a/core/services/nodes/managers_distributed_test.go b/core/services/nodes/managers_distributed_test.go index 68b8b642006c..ada83d15e37a 100644 --- a/core/services/nodes/managers_distributed_test.go +++ b/core/services/nodes/managers_distributed_test.go @@ -97,7 +97,7 @@ var _ = Describe("DistributedBackendManager", func() { Expect(err).ToNot(HaveOccurred()) mc = newScriptedControlWorkers() - adapter = NewRemoteUnloaderAdapter(nil, nil, mc.controlClient(), 3*time.Minute, 15*time.Minute) + adapter = NewRemoteUnloaderAdapter(nil, mc.controlClient(), 3*time.Minute, 15*time.Minute) mgr = &DistributedBackendManager{ local: stubLocalBackendManager{}, adapter: adapter, @@ -741,7 +741,7 @@ var _ = Describe("DistributedBackendManager", func() { mc.scriptHang(controlKey(n.ID, workerctl.PathBackendUpgrade)) slow := &DistributedBackendManager{ local: stubLocalBackendManager{}, - adapter: NewRemoteUnloaderAdapter(nil, nil, mc.controlClient(), time.Minute, 200*time.Millisecond), + adapter: NewRemoteUnloaderAdapter(nil, mc.controlClient(), time.Minute, 200*time.Millisecond), registry: registry, } diff --git a/core/services/nodes/reconciler_test.go b/core/services/nodes/reconciler_test.go index 64171d1859d3..03972fa0104f 100644 --- a/core/services/nodes/reconciler_test.go +++ b/core/services/nodes/reconciler_test.go @@ -903,7 +903,7 @@ var _ = Describe("ReplicaReconciler — state reconciliation", func() { Registry: registry, Scheduler: &fakeScheduler{}, DB: db, - Adapter: NewRemoteUnloaderAdapter(registry, nil, workers.controlClient(), time.Minute, time.Minute), + Adapter: NewRemoteUnloaderAdapter(registry, workers.controlClient(), time.Minute, time.Minute), }) }) diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index 30e302a63527..464d80ddd623 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -45,29 +45,25 @@ type NodeCommandSender interface { // This mirrors the local ModelLoader's startProcess()/deleteProcess() but for // remote nodes. // -// One verb is still carried by the bus, and only for one KIND of node: -// backend.stop to an AGENT node. An agent worker now holds a tunnel and mounts -// that route on it, but this side has not been moved onto it yet, and until it -// is, the agent worker's subscription to nodes..backend.stop is what -// actually drops its cached MCP sessions. Removing that publish would strand -// them; see stopBackend. +// Every verb it sends, for every kind of worker, is a control RPC. It holds no +// publisher at all, which is why re-routing any of them back onto the bus is a +// change to this struct and to every caller of the constructor rather than to +// one branch. type RemoteUnloaderAdapter struct { registry ModelLocator - nats messaging.MessagingClient control *ControlClient installTimeout time.Duration upgradeTimeout time.Duration } -// NewRemoteUnloaderAdapter creates a new adapter. control carries every verb -// except backend.stop to an agent node, which stays on nats. installTimeout and -// upgradeTimeout bound the backend.install and backend.upgrade RPCs -// respectively; use DistributedConfig.BackendInstallTimeoutOrDefault() / +// NewRemoteUnloaderAdapter creates a new adapter. control carries every verb. +// installTimeout and upgradeTimeout bound the backend.install and +// backend.upgrade RPCs respectively; use +// DistributedConfig.BackendInstallTimeoutOrDefault() / // BackendUpgradeTimeoutOrDefault() at construction. -func NewRemoteUnloaderAdapter(registry ModelLocator, nats messaging.MessagingClient, control *ControlClient, installTimeout, upgradeTimeout time.Duration) *RemoteUnloaderAdapter { +func NewRemoteUnloaderAdapter(registry ModelLocator, control *ControlClient, installTimeout, upgradeTimeout time.Duration) *RemoteUnloaderAdapter { return &RemoteUnloaderAdapter{ registry: registry, - nats: nats, control: control, installTimeout: installTimeout, upgradeTimeout: upgradeTimeout, @@ -127,8 +123,8 @@ func (a *RemoteUnloaderAdapter) StopModelReplica(ctx context.Context, nodeID str } // UnloadRemoteModel finds the node(s) hosting the given model and tells each -// to stop its backend process. The carrier is decided per node by its type, -// which is why stopBackend takes one: see stopBackend. +// to stop its backend process, over that node's own tunnel whatever kind of +// worker it is. // The worker process handles a bounded Free() followed by process termination; // forced shutdown skips Free(). // This is called by ModelLoader.deleteProcess() when process == nil (remote model). @@ -179,7 +175,7 @@ func (a *RemoteUnloaderAdapter) UnloadRemoteModelContext(ctx context.Context, mo } seenNodeIDs[node.ID] = struct{}{} xlog.Info("Sending backend.stop to node", "model", modelName, "node", node.Name, "nodeID", node.ID, "force", force) - if err := a.stopBackend(ctx, node.ID, node.NodeType, modelName, force); err != nil { + if err := a.stopBackend(ctx, node.ID, modelName, force); err != nil { xlog.Warn("Failed to send backend.stop", "node", node.Name, "error", err) unloadErr = errors.Join(unloadErr, fmt.Errorf("stopping model on node %s: %w", node.ID, err)) continue @@ -362,50 +358,19 @@ func (a *RemoteUnloaderAdapter) ListRunningModels(nodeID string) (*messaging.Mod func (a *RemoteUnloaderAdapter) StopBackend(nodeID, backend string) error { ctx, cancel := context.WithTimeout(context.Background(), backendStopTimeout) defer cancel() - return a.stopBackend(ctx, nodeID, a.nodeTypeOf(ctx, nodeID), backend, false) + return a.stopBackend(ctx, nodeID, backend, false) } -// nodeTypeOf answers which KIND of worker a node id names, so stopBackend can -// pick the carrier that node actually listens on. +// stopBackend sends one backend.stop over the node's own tunnel. // -// A lookup that fails answers NodeTypeBackend, matching the column's own -// default and the "empty means backend" reading every other node-type branch in -// this package takes. The failure directions are not symmetric: sending a -// backend node's stop over the bus is silently lost, because nothing subscribes -// to it any more, while sending an agent node's stop over the tunnel returns an -// error the caller sees. -func (a *RemoteUnloaderAdapter) nodeTypeOf(ctx context.Context, nodeID string) string { - if a.registry == nil { - return NodeTypeBackend - } - node, err := a.registry.Get(ctx, nodeID) - if err != nil || node == nil { - xlog.Debug("Could not resolve node type for a backend stop; assuming a backend worker", - "nodeID", nodeID, "error", err) - return NodeTypeBackend - } - return node.NodeType -} - -// stopBackend sends one backend.stop, over the carrier that kind of worker -// listens on. -// -// An AGENT node keeps the bus, and that is now a statement about THIS side -// rather than about the worker. The agent worker mounts backend.stop on the -// same control path a backend worker serves it on, so the tunnel could carry -// it; what has not moved is this publish. Splitting on node type is the one -// verb of the ten that is split rather than moved, and it stays split until the -// change that switches this call onto the control route retires the subject. -func (a *RemoteUnloaderAdapter) stopBackend(ctx context.Context, nodeID, nodeType, backend string, force bool) error { - if nodeType == NodeTypeAgent { - subject := messaging.SubjectNodeBackendStop(nodeID) - if backend == "" && !force { - return a.nats.Publish(subject, nil) - } - return a.nats.Publish(subject, messaging.BackendStopRequest{Backend: backend, Force: force}) - } - // An empty Backend is what the worker reads as "stop everything", the same - // meaning the bus carried as an empty payload; see decodeBackendStopRequest. +// It does not ask what KIND of worker the node is, and there is nothing left +// for the answer to change. A backend worker kills the process and recycles +// the port; an agent worker runs no backend processes and closes the MCP +// sessions it cached for that backend. Both serve the verb on the same control +// path, so the frontend states the fact and the worker decides what it means. +func (a *RemoteUnloaderAdapter) stopBackend(ctx context.Context, nodeID, backend string, force bool) error { + // An empty Backend is what the worker reads as "stop everything"; see + // decodeBackendStopRequest. var reply messaging.BackendStopReply if err := a.control.Call(ctx, nodeID, workerctl.PathBackendStop, messaging.BackendStopRequest{Backend: backend, Force: force}, &reply); err != nil { diff --git a/core/services/nodes/unloader_stale_rows_test.go b/core/services/nodes/unloader_stale_rows_test.go index 9be312917aa1..102300b2e0fc 100644 --- a/core/services/nodes/unloader_stale_rows_test.go +++ b/core/services/nodes/unloader_stale_rows_test.go @@ -28,7 +28,7 @@ var _ = Describe("RemoteUnloaderAdapter stale replica rows", func() { BeforeEach(func() { locator = &fakeModelLocator{} workers = newScriptedControlWorkers() - adapter = NewRemoteUnloaderAdapter(locator, nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) + adapter = NewRemoteUnloaderAdapter(locator, workers.controlClient(), 3*time.Minute, 15*time.Minute) }) Describe("DeleteBackend", func() { diff --git a/core/services/nodes/unloader_test.go b/core/services/nodes/unloader_test.go index 4ad403aadc82..9cddf6546395 100644 --- a/core/services/nodes/unloader_test.go +++ b/core/services/nodes/unloader_test.go @@ -75,9 +75,10 @@ func (f *fakeModelLocator) RemoveAllNodeModelReplicas(_ context.Context, nodeID, // fakeMessagingClient implements messaging.MessagingClient, recording Publish // and Request calls so we can assert on subjects and payloads. // -// Only ONE verb still reaches it: backend.stop to an agent node. Every other -// control verb travels over the tunnel, so a publish recorded here for a -// backend node is a bug, and several specs below assert exactly that. +// NO control verb reaches it any more, and the RemoteUnloaderAdapter cannot be +// handed one: it holds no publisher. It survives here for the staging-progress +// broadcasts in staging_progress_broadcast_test.go, which are cross-replica +// events rather than anything addressed to a worker. type fakeMessagingClient struct { mu sync.Mutex published []publishCall @@ -139,36 +140,43 @@ func (f *fakeMessagingClient) Request(subject string, data []byte, timeout time. func (f *fakeMessagingClient) IsConnected() bool { return true } func (f *fakeMessagingClient) Close() {} -// publishedSubjects reports what actually reached the bus. -func (f *fakeMessagingClient) publishedSubjects() []string { - f.mu.Lock() - defer f.mu.Unlock() - out := make([]string, 0, len(f.published)) - for _, c := range f.published { - out = append(out, c.Subject) - } - return out -} - type fakeSubscription struct{} func (f *fakeSubscription) Unsubscribe() error { return nil } +// stopPayloadOf decodes the backend.stop body a worker actually received on +// one (node, verb) key. Reading the payload back off the transport is what +// separates "a stop was sent" from "the RIGHT stop was sent"; the two node +// types below are distinguished only by which key they arrive on. +func stopPayloadOf(s *scriptedControlWorkers, key string) messaging.BackendStopRequest { + GinkgoHelper() + s.mu.Lock() + defer s.mu.Unlock() + for _, c := range s.calls { + if c.Subject != key { + continue + } + var payload messaging.BackendStopRequest + Expect(json.Unmarshal(c.Data, &payload)).To(Succeed()) + return payload + } + Fail("no backend.stop reached " + key) + return messaging.BackendStopRequest{} +} + // --- Tests --- var _ = Describe("RemoteUnloaderAdapter", func() { var ( locator *fakeModelLocator - bus *fakeMessagingClient workers *scriptedControlWorkers adapter *RemoteUnloaderAdapter ) BeforeEach(func() { locator = &fakeModelLocator{} - bus = &fakeMessagingClient{} workers = newScriptedControlWorkers() - adapter = NewRemoteUnloaderAdapter(locator, bus, workers.controlClient(), 3*time.Minute, 15*time.Minute) + adapter = NewRemoteUnloaderAdapter(locator, workers.controlClient(), 3*time.Minute, 15*time.Minute) }) // scriptStop lets a backend node acknowledge the tunnelled backend.stop. @@ -221,7 +229,6 @@ var _ = Describe("RemoteUnloaderAdapter", func() { locator.nodes = nil Expect(adapter.UnloadRemoteModel("my-model")).To(Succeed()) Expect(workers.callSubjects()).To(BeEmpty()) - Expect(bus.publishedSubjects()).To(BeEmpty()) }) It("stops the backend on every node holding the model, over their tunnels", func() { @@ -238,7 +245,6 @@ var _ = Describe("RemoteUnloaderAdapter", func() { controlKey("node-1", workerctl.PathBackendStop), controlKey("node-2", workerctl.PathBackendStop), })) - Expect(bus.publishedSubjects()).To(BeEmpty()) // Should have removed the model from each node in the registry. Expect(locator.removedPairs).To(HaveLen(2)) @@ -298,37 +304,37 @@ var _ = Describe("RemoteUnloaderAdapter", func() { Expect(payload).To(Equal(messaging.BackendStopRequest{Backend: "llama", Force: true})) }) - // The carrier split has TWO call sites, which is why stopBackend takes - // nodeType as a parameter. StopBackend is pinned in both directions - // below; this is the other caller, and hardcoding NodeTypeBackend here - // used to leave the whole suite green. An agent node holding a - // node_models row would then have its stop sent over a tunnel it does - // not hold, the call would fail, and the replica row would be left - // behind. - It("routes each stop by ITS node's type, not by one choice for the unload", func() { + // This unload spans BOTH kinds of worker, and the point is that it no + // longer has to care which is which. It used to: an agent node's stop + // was published and a backend node's was called, so an unload touching + // one of each exercised two carriers. A single scripted node could not + // have caught a frontend that picked one carrier for the whole unload. + It("takes the same route for every node in one unload, whatever their types", func() { locator.nodes = []BackendNode{ {ID: "agent-1", Name: "agent", NodeType: NodeTypeAgent}, {ID: "backend-1", Name: "gpu-1", NodeType: NodeTypeBackend}, } + scriptStop("agent-1") scriptStop("backend-1") Expect(adapter.UnloadRemoteModelContext(context.Background(), "llama", false)).To(Succeed()) - // The agent's stop went to the bus and only the agent's did; the - // backend's went over the tunnel and only the backend's did. - Expect(bus.publishedSubjects()).To(Equal([]string{messaging.SubjectNodeBackendStop("agent-1")})) - Expect(workers.callSubjects()).To(Equal([]string{controlKey("backend-1", workerctl.PathBackendStop)})) - // Both rows dropped, which is the negative control: a stop put on - // the carrier the other kind of worker listens on fails, and a - // failed stop keeps its row. + Expect(workers.callSubjects()).To(ConsistOf( + controlKey("agent-1", workerctl.PathBackendStop), + controlKey("backend-1", workerctl.PathBackendStop), + )) + // Both rows dropped, which is the negative control: a stop the + // worker never answered fails, and a failed stop keeps its row. Expect(locator.removedPairs).To(HaveLen(2)) }) }) - // The carrier split. It is the one verb of the ten that is decided by the - // KIND of worker, and getting it wrong is silent in both directions: a - // backend node's stop published on the bus reaches nothing, and an agent - // node's stop sent over a tunnel it does not hold reaches nothing either. + // backend.stop used to be the one verb of the ten decided by the KIND of + // worker: an agent node's went to the bus, a backend node's over the + // tunnel. Both kinds serve the same control path now, so the two node + // types are asserted SEPARATELY rather than as one parameterised case. A + // single case would show that some node gets the path; only two show that + // the two kinds no longer differ, which is the whole content of the change. Describe("StopBackend and node type", func() { It("surfaces a backend worker's stop failure", func() { locator.nodes = []BackendNode{{ID: "backend-1", Name: "gpu-1", NodeType: NodeTypeBackend}} @@ -342,37 +348,39 @@ var _ = Describe("RemoteUnloaderAdapter", func() { Expect(err).To(MatchError(ContainSubstring("process refused to die"))) }) - It("sends a backend stop to a BACKEND node over the tunnel and not over the bus", func() { + It("sends a backend stop to a BACKEND node over the tunnel, naming the backend", func() { locator.nodes = []BackendNode{{ID: "backend-1", Name: "gpu-1", NodeType: NodeTypeBackend}} scriptStop("backend-1") Expect(adapter.StopBackend("backend-1", "llama-backend")).To(Succeed()) Expect(workers.callSubjects()).To(ContainElement(controlKey("backend-1", workerctl.PathBackendStop))) - Expect(bus.publishedSubjects()).To(BeEmpty()) + Expect(stopPayloadOf(workers, controlKey("backend-1", workerctl.PathBackendStop))). + To(Equal(messaging.BackendStopRequest{Backend: "llama-backend"})) }) - It("sends a backend stop to an AGENT node over the bus, because that publisher has not moved onto the tunnel", func() { + It("sends a backend stop to an AGENT node over the tunnel too, on the same path", func() { locator.nodes = []BackendNode{{ID: "agent-1", Name: "agent", NodeType: NodeTypeAgent}} + scriptStop("agent-1") Expect(adapter.StopBackend("agent-1", "llama-backend")).To(Succeed()) - Expect(bus.publishedSubjects()).To(ContainElement(messaging.SubjectNodeBackendStop("agent-1"))) - Expect(workers.callSubjects()).ToNot(ContainElement(controlKey("agent-1", workerctl.PathBackendStop))) + Expect(workers.callSubjects()).To(ContainElement(controlKey("agent-1", workerctl.PathBackendStop))) + Expect(stopPayloadOf(workers, controlKey("agent-1", workerctl.PathBackendStop))). + To(Equal(messaging.BackendStopRequest{Backend: "llama-backend"})) }) - It("treats a node whose type cannot be read as a backend worker", func() { - // The column defaults to backend and every other node-type branch - // in this package reads an empty value the same way. The lookup - // failing must not silently move a stop onto a carrier nothing is - // listening on. + It("does not read the node's type at all before stopping a backend on it", func() { + // The carrier no longer depends on what kind of worker this is, so + // the lookup that used to choose it is gone. A registry that cannot + // answer must therefore not stop a backend from being stopped: the + // stop is issued on the node's own tunnel regardless. locator.getErr = errors.New("database is down") scriptStop("unknown-1") Expect(adapter.StopBackend("unknown-1", "llama-backend")).To(Succeed()) Expect(workers.callSubjects()).To(ContainElement(controlKey("unknown-1", workerctl.PathBackendStop))) - Expect(bus.publishedSubjects()).To(BeEmpty()) }) It("with an empty backend asks the worker to stop everything", func() { @@ -447,7 +455,6 @@ var _ = Describe("RemoteUnloaderAdapter", func() { Expect(adapter.StopNode("node-abc")).To(Succeed()) Expect(workers.callSubjects()).To(Equal([]string{controlKey("node-abc", workerctl.PathNodeStop)})) - Expect(bus.publishedSubjects()).To(BeEmpty()) }) }) @@ -508,7 +515,7 @@ var _ = Describe("RemoteUnloaderAdapter timeout configuration", func() { It("gives backend.install the configured install timeout", func() { workers := newScriptedControlWorkers() workers.scriptHang(controlKey("n1", workerctl.PathBackendInstall)) - adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), installBudget, upgradeBudget) + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), installBudget, upgradeBudget) started := time.Now() _, err := adapter.InstallBackend("n1", "llama-cpp", "", "[]", "", "", "", 0, "", nil) @@ -523,7 +530,7 @@ var _ = Describe("RemoteUnloaderAdapter timeout configuration", func() { It("gives backend.upgrade the configured upgrade timeout", func() { workers := newScriptedControlWorkers() workers.scriptHang(controlKey("n1", workerctl.PathBackendUpgrade)) - adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), installBudget, upgradeBudget) + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), installBudget, upgradeBudget) started := time.Now() _, err := adapter.UpgradeBackend("n1", "llama-cpp", "[]", "", "", "", 0, "", nil) @@ -539,7 +546,7 @@ var _ = Describe("RemoteUnloaderAdapter timeout handling", func() { It("reports a spent budget as still-installing, so the operation shows as running on the worker", func() { workers := newScriptedControlWorkers() workers.scriptTimeout("n1") - adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 100*time.Millisecond, 1*time.Second) + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), 100*time.Millisecond, 1*time.Second) _, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "", nil) Expect(err).To(HaveOccurred()) @@ -553,7 +560,7 @@ var _ = Describe("RemoteUnloaderAdapter timeout handling", func() { // other is a plain failure. Both are non-verdicts, so neither may reap. workers := newScriptedControlWorkers() workers.scriptUnroutable("n1") - adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 100*time.Millisecond, 1*time.Second) + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), 100*time.Millisecond, 1*time.Second) _, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "", nil) Expect(err).To(HaveOccurred()) @@ -576,7 +583,7 @@ var _ = Describe("RemoteUnloaderAdapter timeout handling", func() { // INSTALL but is part of an upgrade, so it must wait the upgrade // budget. Waiting the install one would satisfy a bare // still-installing assertion while carrying the wrong deadline. - adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 100*time.Millisecond, 500*time.Millisecond) + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), 100*time.Millisecond, 500*time.Millisecond) started := time.Now() _, err := adapter.installWithForceFallback("n1", "llama-cpp", "[]", "", "", "", 0, "", nil) @@ -591,7 +598,7 @@ var _ = Describe("RemoteUnloaderAdapter timeout handling", func() { workers := newScriptedControlWorkers() // The verb is not scripted, so the worker fails to SERVE it rather // than answering; that is a 5xx and lands under the no-route umbrella. - adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), time.Minute, time.Minute) + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), time.Minute, time.Minute) _, err := adapter.installWithForceFallback("n1", "llama-cpp", "[]", "", "", "", 0, "", nil) Expect(err).To(HaveOccurred()) @@ -613,7 +620,7 @@ var _ = Describe("RemoteUnloaderAdapter timeout handling", func() { workers := newScriptedControlWorkers() workers.scriptServerError(controlKey("n1", workerctl.PathBackendInstall), `the worker said "nats: timeout" in its log`) - adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), time.Minute, time.Minute) + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), time.Minute, time.Minute) _, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "", nil) Expect(err).To(HaveOccurred()) @@ -634,7 +641,7 @@ var _ = Describe("RemoteUnloaderAdapter install progress streaming", func() { {OpID: "op-abc", NodeID: "n1", Backend: "vllm", FileName: "vllm.tar.zst", Current: "500 MB", Total: "1 GB", Percentage: 50}, }) - adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), time.Second, time.Second) + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), time.Second, time.Second) var received []messaging.BackendInstallProgressEvent onProgress := func(ev messaging.BackendInstallProgressEvent) { // No lock, and that is an assertion in itself: the callback runs @@ -657,7 +664,7 @@ var _ = Describe("RemoteUnloaderAdapter install progress streaming", func() { {OpID: "", NodeID: "n1", Percentage: 42}, }) - adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), time.Second, time.Second) + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), time.Second, time.Second) reply, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "", nil) Expect(err).ToNot(HaveOccurred()) Expect(reply.Success).To(BeTrue()) diff --git a/core/services/nodes/unloader_upgrade_test.go b/core/services/nodes/unloader_upgrade_test.go index 41fe7316f16e..7e1cdba0b018 100644 --- a/core/services/nodes/unloader_upgrade_test.go +++ b/core/services/nodes/unloader_upgrade_test.go @@ -19,7 +19,7 @@ var _ = Describe("RemoteUnloaderAdapter.UpgradeBackend", func() { workers.scriptReply(controlKey(nodeID, workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: true}) - adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) reply, err := adapter.UpgradeBackend(nodeID, "llama-cpp", `[{"name":"x"}]`, "", "", "", 0, "", nil) Expect(err).ToNot(HaveOccurred()) Expect(reply.Success).To(BeTrue()) @@ -30,7 +30,7 @@ var _ = Describe("RemoteUnloaderAdapter.UpgradeBackend", func() { workers := newScriptedControlWorkers() workers.scriptUnroutable("missing-node") - adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) _, err := adapter.UpgradeBackend("missing-node", "llama-cpp", "", "", "", "", 0, "", nil) Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) // Not the worker's 404 either: nothing was asked of it, so it did not @@ -60,7 +60,7 @@ var _ = Describe("RemoteUnloaderAdapter.UpgradeBackend", func() { got = append(got, ev) } - adapter := NewRemoteUnloaderAdapter(nil, nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), 3*time.Minute, 15*time.Minute) reply, err := adapter.UpgradeBackend(nodeID, "llama-cpp", `[{"name":"x"}]`, "", "", "", 0, opID, onProgress) Expect(err).ToNot(HaveOccurred()) Expect(reply.Success).To(BeTrue()) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 0bd859a22193..c8c55fe5f4d5 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -145,7 +145,7 @@ Both **backend** and **agent** nodes are issued one. Earlier releases minted a c An agent worker's tunnel carries only the `http` tag: it runs no backend processes, so it does not offer the `grpc` tag at all. Its control server binds `127.0.0.1` on a port chosen by the kernel and advertises it nowhere, so an agent worker still opens no inbound port. -**An agent worker still requires `--nats-url`.** The tunnel is not yet a replacement: agent jobs, MCP CI jobs and `nodes..backend.stop` still travel on NATS. MCP tool execution and MCP discovery no longer do - they are control RPCs on the tunnel, chosen by the frontend rather than by a queue group (see [MCP in Distributed Mode](#mcp-in-distributed-mode)) - and an agent worker's minted JWT no longer grants `mcp.tools.execute` or `mcp.discovery`. +**An agent worker still requires `--nats-url`.** The tunnel is not yet a replacement: agent jobs and MCP CI jobs still travel on NATS. Nothing the frontend addresses to a specific agent worker does any more. MCP tool execution and MCP discovery are control RPCs on the tunnel, chosen by the frontend rather than by a queue group (see [MCP in Distributed Mode](#mcp-in-distributed-mode)), and so is the backend stop that flushes an agent worker's cached MCP sessions. There is no `nodes..*` subject left, and an agent worker's minted JWT no longer grants `mcp.tools.execute`, `mcp.discovery` or `nodes..backend.stop`. The tunnel lands on exactly one frontend replica, and that replica records itself as the owner of the worker's connection in the `node_connections` table. When the socket dies the claim is dropped, but the row stays behind with no owner and a `disconnected_at` stamp, so a worker that is re-dialling the load balancer can be told from one that has never connected. The row is deleted once that departure is older than ten liveness windows (five minutes). If the replica stalls long enough for its peers to reap it, it re-claims the tunnels it still holds on a live session as soon as it re-registers, skipping any whose socket has already gone. That re-claim needs the replica to have an advertised address: without one it never had an instance row to begin with, and its tunnels are usable only by the replica holding them. @@ -195,7 +195,14 @@ mounted only when the deployment configured an object store). The request bodies are unchanged and the reply fields keep their names and types, so nothing an operator inspects on the wire has a new shape. The one difference is that a worker now OMITS an empty reply field where the NATS handlers always emitted it, -which a client reading a missing field as the zero value cannot tell apart. Agent workers still take `nodes..backend.stop` over NATS. +which a client reading a missing field as the zero value cannot tell apart. + +`POST /v1/control/backend/stop` is served by BOTH kinds of worker, and the +frontend sends it the same way to either. A serve-backend worker kills the +backend process and recycles its port; an agent worker runs no backend +processes and closes the MCP sessions it had cached for that backend. No +`nodes..*` subject remains, so a worker never takes a control verb off the +bus whatever its type. `files/listdir` is the verb the change is most visible on. Its reply used to be sized against what the bus would carry, which put a wide model directory close to diff --git a/pkg/model/connection_evicting_client.go b/pkg/model/connection_evicting_client.go index a7176358938e..64d2af37d913 100644 --- a/pkg/model/connection_evicting_client.go +++ b/pkg/model/connection_evicting_client.go @@ -38,8 +38,8 @@ func (c *ConnectionEvictingClient) checkErr(err error) { } // The fifth site of the same shape, and the one reached during INFERENCE // rather than a health check. evict() runs ShutdownModel, which for a remote - // model sends backend.stop over NATS to every node holding it and deletes - // every replica row. In distributed mode the client underneath reaches the + // model sends a backend.stop control RPC over the tunnel of every node + // holding it and deletes every replica row. In distributed mode the client underneath reaches the // backend over the worker's tunnel, and a failure of THAT transport arrives // as the same codes.Unavailable a dead backend produces; evicting on it // stops a model that is loaded and serving, on a worker that is diff --git a/pkg/model/remote_unroutable_internal_test.go b/pkg/model/remote_unroutable_internal_test.go index eb060458b261..1d753f3f0ddd 100644 --- a/pkg/model/remote_unroutable_internal_test.go +++ b/pkg/model/remote_unroutable_internal_test.go @@ -104,8 +104,8 @@ var _ = Describe("the eviction wrapper on a remote model whose transport failed" // The FIFTH site of the same shape, found by sweeping the decorators rather // than being named. initializers.go builds this wrapper for exactly the // remote models the router produces, and its evict callback runs - // ShutdownModel, which sends backend.stop over NATS to every node holding - // the model and deletes every replica row. It fires during INFERENCE, not + // ShutdownModel, which sends a backend.stop control RPC over the tunnel of + // every node holding the model and deletes every replica row. It fires during INFERENCE, not // on a health check, so a tunnel blip mid-request was enough. failingDial := func(cause error) grpc.Backend { return grpc.NewClientWithDialer("10.0.0.1:9001", false, nil, false, "", diff --git a/pkg/natsauth/permissions.go b/pkg/natsauth/permissions.go index 9145e0b3eb8d..77c0717cba6b 100644 --- a/pkg/natsauth/permissions.go +++ b/pkg/natsauth/permissions.go @@ -21,20 +21,24 @@ func workerSubjectToken(nodeID string) string { // restriction, so a function that returned nil here would upgrade every JWT the // frontend still mints for a backend node from "its own inbox" to "the entire // account". The inbox is self-scoped and reaches no cluster subject. +// +// nodeID no longer narrows anything: no allow list below is per-node, because +// the only per-node subject an agent worker ever subscribed to was its +// backend.stop, which is a control RPC on its tunnel now. It stays in the +// signature so a future per-node grant has somewhere to come from, and because +// mint.go names the JWT user after it. func WorkerPermissions(nodeID, nodeType string) (pubAllow, subAllow []string) { - tok := workerSubjectToken(nodeID) - prefix := "nodes." + tok - switch nodeType { case "agent": // Agent workers consume queue workloads; they must not handle backend.install. // Keep this list in sync with the subscriptions in core/cli/agent_worker.go. // - // MCP tool execution and discovery are NOT here any more: they are - // control RPCs on the tunnel the worker holds, chosen by the frontend - // rather than by a queue group. Removing them narrowed this list; it - // must never be narrowed to nothing, because NATS reads an EMPTY allow - // list as no restriction at all. + // MCP tool execution and discovery are NOT here, and neither is the + // per-node backend.stop: all three are control RPCs on the tunnel the + // worker holds, addressed by the frontend rather than by a subject. + // Removing them narrowed this list; it must never be narrowed to + // nothing, because NATS reads an EMPTY allow list as no restriction at + // all, which would widen an agent JWT to the whole account. subAllow = []string{ "agent.execute", "agent.*.cancel", @@ -43,8 +47,7 @@ func WorkerPermissions(nodeID, nodeType string) (pubAllow, subAllow []string) { "jobs.*.cancel", "jobs.*.progress", "jobs.*.result", - "jobs.mcp-ci.new", // MCP CI jobs dispatched to agent workers - prefix + ".backend.stop", // stop events drive MCP session cleanup + "jobs.mcp-ci.new", // MCP CI jobs dispatched to agent workers "staging.*.progress", "_INBOX.>", } diff --git a/pkg/natsauth/permissions_coverage_test.go b/pkg/natsauth/permissions_coverage_test.go index f21e51d2f285..dbf18f049aef 100644 --- a/pkg/natsauth/permissions_coverage_test.go +++ b/pkg/natsauth/permissions_coverage_test.go @@ -101,9 +101,8 @@ var _ = Describe("WorkerPermissions subject coverage", func() { pub, sub := natsauth.WorkerPermissions(nodeID, "agent") subscribed := []string{ - messaging.SubjectAgentExecute, // dispatcher (default --agent-subject) - messaging.SubjectMCPCIJobsNew, // QueueSubscribe — jobs.mcp-ci.new - messaging.SubjectNodeBackendStop(nodeID), // Subscribe — MCP session cleanup + messaging.SubjectAgentExecute, // dispatcher (default --agent-subject) + messaging.SubjectMCPCIJobsNew, // QueueSubscribe — jobs.mcp-ci.new } // The half that catches a narrowing going too far. NATS reads an EMPTY @@ -116,16 +115,34 @@ var _ = Describe("WorkerPermissions subject coverage", func() { "an empty allow list is unrestricted in NATS, not restrictive") }) - // MCP execution and discovery are control RPCs on the worker's tunnel - // now, chosen by the frontend rather than by a queue group. An agent - // worker subscribes to neither subject, so a JWT that still granted - // them would be granting a subscription nothing serves. - for _, subject := range []string{"mcp.tools.execute", "mcp.discovery"} { + // MCP execution and discovery, and now backend.stop too, are control + // RPCs on the worker's tunnel, addressed by the frontend rather than + // by a subject. An agent worker subscribes to none of them, so a JWT + // that still granted one would be granting a subscription nothing + // serves. The backend.stop entry is spelled out by hand rather than + // built from a deleted builder, which is the only way this can still + // name the subject that used to be granted. + for _, subject := range []string{ + "mcp.tools.execute", + "mcp.discovery", + "nodes." + workerSubjectTokenForTest(nodeID) + ".backend.stop", + } { It("no longer grants an agent worker "+subject, func() { Expect(anyAllows(sub, subject)).To(BeFalse(), "agent JWT sub allow-list %v still covers %s", sub, subject) }) } + + // The other half of the narrowing, and the one the phase's own trap + // makes necessary: an allow list trimmed to nothing is UNRESTRICTED in + // NATS. Naming what must survive is what tells a narrowing from an + // escalation, so the queue workloads an agent worker still consumes are + // asserted present just above, and the list is asserted non-empty here. + It("still grants the agent worker the queue subjects it lives on", func() { + Expect(sub).To(ContainElement("agent.execute")) + Expect(sub).To(ContainElement("jobs.mcp-ci.new")) + Expect(sub).To(ContainElement("_INBOX.>")) + }) for _, subject := range subscribed { It("allows subscribing to "+subject, func() { Expect(anyAllows(sub, subject)).To(BeTrue(), @@ -149,7 +166,6 @@ var _ = Describe("Documented NATS service-user permissions", func() { messaging.SubjectPrefixCacheObserve, messaging.SubjectPrefixCacheInvalidate, messaging.SubjectPrefixCachePressure, - messaging.SubjectNodeBackendStop("node-1"), messaging.SubjectGalleryProgress("op-1"), } diff --git a/tests/e2e/distributed/distributed_full_flow_test.go b/tests/e2e/distributed/distributed_full_flow_test.go index b6b2ceeca1c4..c456c310f5cf 100644 --- a/tests/e2e/distributed/distributed_full_flow_test.go +++ b/tests/e2e/distributed/distributed_full_flow_test.go @@ -236,7 +236,7 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() newTestSmartRouter := func(reg *nodes.NodeRegistry, extraOpts ...nodes.SmartRouterOptions) *nodes.SmartRouter { workers := NewControlWorkers() workers.ServeBackendLifecycle(reg) - unloader := nodes.NewRemoteUnloaderAdapter(reg, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) + unloader := nodes.NewRemoteUnloaderAdapter(reg, workers.Client(), 3*time.Minute, 15*time.Minute) opts := nodes.SmartRouterOptions{ Unloader: unloader, @@ -397,7 +397,7 @@ var _ = Describe("Full Distributed Inference Flow", Label("Distributed"), func() }) // Create RemoteUnloaderAdapter and unload model - unloader := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) + unloader := nodes.NewRemoteUnloaderAdapter(registry, workers.Client(), 3*time.Minute, 15*time.Minute) Expect(unloader.UnloadRemoteModel("old-model")).To(Succeed()) // The worker got the stop over its tunnel, not over the bus. diff --git a/tests/e2e/distributed/managers_test.go b/tests/e2e/distributed/managers_test.go index 83080f0d72b5..f890967e8f3f 100644 --- a/tests/e2e/distributed/managers_test.go +++ b/tests/e2e/distributed/managers_test.go @@ -165,7 +165,7 @@ var _ = Describe("Model and Backend Managers", Label("Distributed"), func() { appCfg := config.NewApplicationConfig() appCfg.SystemState = ss - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, workers.Client(), 3*time.Minute, 15*time.Minute) distMgr := nodes.NewDistributedModelManager(appCfg, ml, adapter) err = distMgr.DeleteModel("big-model") @@ -226,7 +226,7 @@ var _ = Describe("Model and Backend Managers", Label("Distributed"), func() { appCfg := config.NewApplicationConfig() appCfg.SystemState = ss - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, workers.Client(), 3*time.Minute, 15*time.Minute) distMgr := nodes.NewDistributedBackendManager(appCfg, ml, adapter, registry, nil) err = distMgr.DeleteBackend("my-backend") @@ -269,7 +269,7 @@ var _ = Describe("Model and Backend Managers", Label("Distributed"), func() { appCfg := config.NewApplicationConfig() appCfg.SystemState = ss - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, workers.Client(), 3*time.Minute, 15*time.Minute) distMgr := nodes.NewDistributedBackendManager(appCfg, ml, adapter, registry, nil) // Should NOT return an error even though the backend doesn't exist locally diff --git a/tests/e2e/distributed/nats_jwt_test.go b/tests/e2e/distributed/nats_jwt_test.go index dd0c23253f92..72ea76b751ec 100644 --- a/tests/e2e/distributed/nats_jwt_test.go +++ b/tests/e2e/distributed/nats_jwt_test.go @@ -109,13 +109,37 @@ var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { DeferCleanup(nc.Close) // Mirror core/cli/agent_worker.go exactly. MCP tool execution and - // discovery are absent because they are no longer bus subjects at all: - // the frontend selects an agent worker itself and reaches it with a - // control RPC over the tunnel that worker holds. + // discovery are absent, and so is the per-node backend.stop, because + // none of the three is a bus subject any more: the frontend selects an + // agent worker itself and reaches it with a control RPC over the tunnel + // that worker holds. _, err = nc.QueueSubscribe(messaging.SubjectMCPCIJobsNew, messaging.QueueWorkers, func([]byte) {}) Expect(err).ToNot(HaveOccurred(), "agent JWT must allow %s (MCP CI jobs)", messaging.SubjectMCPCIJobsNew) - _, err = nc.Subscribe(messaging.SubjectNodeBackendStop(nodeID), func([]byte) {}) - Expect(err).ToNot(HaveOccurred(), "agent JWT must allow %s (MCP session cleanup)", messaging.SubjectNodeBackendStop(nodeID)) + _, err = nc.Subscribe(messaging.SubjectAgentExecute, func([]byte) {}) + Expect(err).ToNot(HaveOccurred(), "agent JWT must allow %s (job dispatch)", messaging.SubjectAgentExecute) + }) + + // The narrowing, proved against the enforcing server rather than against + // the allow list that feeds it. The subject is written out by hand because + // its builder is deleted; that literal is what a worker from an older + // release would still send, and this is what the server now answers it. + // + // It is a narrowing and not a lockout: the two subscriptions above are made + // on a JWT minted the same way and both succeed, so the list this trims is + // demonstrably not the empty one NATS would read as unrestricted. + It("refuses an agent-minted JWT the retired per-node backend.stop subject", func() { + const nodeID = "agent-node-stop" + cfg := natsauth.Config{AccountSeed: infra.AccountSeed, WorkerJWTTTL: time.Hour} + token, seed, err := cfg.MintWorkerJWT(nodeID, "agent") + Expect(err).ToNot(HaveOccurred()) + + nc, err := messaging.New(infra.NatsURL, messaging.WithUserJWT(token, seed)) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(nc.Close) + + _, err = nc.Subscribe("nodes."+nodeID+".backend.stop", func([]byte) {}) + Expect(err).To(HaveOccurred(), + "backend.stop is a control RPC on the worker's tunnel; the bus must not carry it") }) }) diff --git a/tests/e2e/distributed/node_lifecycle_test.go b/tests/e2e/distributed/node_lifecycle_test.go index 1f667f3b563a..4b59bd9a891e 100644 --- a/tests/e2e/distributed/node_lifecycle_test.go +++ b/tests/e2e/distributed/node_lifecycle_test.go @@ -54,7 +54,7 @@ var _ = Describe("Node Backend Lifecycle over the worker control plane", Label(" return messaging.BackendInstallReply{Success: true} }) - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, workers.Client(), 3*time.Minute, 15*time.Minute) installReply, err := adapter.InstallBackend(node.ID, "llama-cpp", "", "", "", "", "", 0, "", nil) Expect(err).ToNot(HaveOccurred()) Expect(installReply.Success).To(BeTrue()) @@ -72,7 +72,7 @@ var _ = Describe("Node Backend Lifecycle over the worker control plane", Label(" return messaging.BackendInstallReply{Success: false, Error: "backend not found"} }) - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, workers.Client(), 3*time.Minute, 15*time.Minute) installReply, err := adapter.InstallBackend(node.ID, "nonexistent", "", "", "", "", "", 0, "", nil) Expect(err).ToNot(HaveOccurred()) Expect(installReply.Success).To(BeFalse()) @@ -96,7 +96,7 @@ var _ = Describe("Node Backend Lifecycle over the worker control plane", Label(" }) // Frontend calls UnloadRemoteModel (triggered by UI "Stop" or WatchDog) - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, workers.Client(), 3*time.Minute, 15*time.Minute) Expect(adapter.UnloadRemoteModel("whisper-large")).To(Succeed()) Eventually(func() int32 { return stopReceived.Load() }, "5s").Should(Equal(int32(1))) @@ -106,6 +106,36 @@ var _ = Describe("Node Backend Lifecycle over the worker control plane", Label(" Expect(nodesWithModel).To(BeEmpty()) }) + // The same verb, on an AGENT node, over the same route. Asserted as its + // own spec rather than folded into the backend-node one above, because + // the two used to take different carriers and only two specs can show + // that they no longer do. The body is read back off the transport: a + // stop that arrived naming no backend would tell the worker to stop + // everything, which is a different instruction entirely. + It("should send backend.stop to an AGENT node over its tunnel too", func() { + node := &nodes.BackendNode{ + Name: "agent-node-1", NodeType: nodes.NodeTypeAgent, + } + Expect(registry.Register(context.Background(), node, true)).To(Succeed()) + Expect(registry.SetNodeModel(context.Background(), node.ID, "agent-hosted", 0, "loaded", "127.0.0.1:59061", 0)).To(Succeed()) + + bodies := make(chan []byte, 1) + workers := NewControlWorkers() + workers.On(node.ID, workerctl.PathBackendStop, func(_ string, body []byte) any { + bodies <- body + return nil + }) + + adapter := nodes.NewRemoteUnloaderAdapter(registry, workers.Client(), 3*time.Minute, 15*time.Minute) + Expect(adapter.UnloadRemoteModel("agent-hosted")).To(Succeed()) + + var body []byte + Eventually(bodies, "5s").Should(Receive(&body)) + var req messaging.BackendStopRequest + Expect(json.Unmarshal(body, &req)).To(Succeed()) + Expect(req).To(Equal(messaging.BackendStopRequest{Backend: "agent-hosted"})) + }) + It("should send backend.stop to all nodes hosting the model", func() { node1 := &nodes.BackendNode{Name: "n1", Address: "h1:50051"} node2 := &nodes.BackendNode{Name: "n2", Address: "h2:50051"} @@ -123,7 +153,7 @@ var _ = Describe("Node Backend Lifecycle over the worker control plane", Label(" }) } - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, workers.Client(), 3*time.Minute, 15*time.Minute) adapter.UnloadRemoteModel("shared-model") Eventually(func() int32 { return count.Load() }, "5s").Should(Equal(int32(2))) @@ -139,7 +169,7 @@ var _ = Describe("Node Backend Lifecycle over the worker control plane", Label(" // The same contract is pinned at unit level by "with no nodes // returns nil" in core/services/nodes/unloader_test.go; keep them // in step. - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, NewControlWorkers().Client(), 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, NewControlWorkers().Client(), 3*time.Minute, 15*time.Minute) Expect(adapter.UnloadRemoteModel("nonexistent-model")).To(Succeed()) }) }) @@ -158,7 +188,7 @@ var _ = Describe("Node Backend Lifecycle over the worker control plane", Label(" return nil }) - adapter := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) + adapter := nodes.NewRemoteUnloaderAdapter(registry, workers.Client(), 3*time.Minute, 15*time.Minute) Expect(adapter.StopNode(node.ID)).To(Succeed()) Eventually(func() int32 { return stopped.Load() }, "5s").Should(Equal(int32(1))) @@ -176,13 +206,13 @@ var _ = Describe("Node Backend Lifecycle over the worker control plane", Label(" Expect(workerctl.PathNodeStop).To(Equal("/v1/control/node/stop")) }) - // The one node subject left, and it is addressed only to AGENT workers: - // they subscribe to it to drop cached MCP sessions. They serve the same - // verb on their tunnel as well; this subject survives because the - // PUBLISHER has not moved onto that route yet. - It("should keep the agent worker's backend.stop subject", func() { - Expect(messaging.SubjectNodeBackendStop("node-abc")).To(Equal("nodes.node-abc.backend.stop")) - }) + // backend.stop was the last worker-facing node subject, and it was + // addressed only to AGENT workers. It has no subject any more: an agent + // node's stop is a control RPC on the path named above, the same one a + // backend node takes. There is nothing to write out by hand here, + // because the builder is deleted; what an e2e spec can show instead is + // that an AGENT node's stop reaches a worker over its tunnel, which + // "should send backend.stop to an AGENT node over its tunnel too" does. }) // Design note: LoadModel is a gRPC call through the worker's tunnel, not a diff --git a/tests/e2e/distributed/router_tracking_test.go b/tests/e2e/distributed/router_tracking_test.go index a3f92c9ef52c..cb2575ef5c73 100644 --- a/tests/e2e/distributed/router_tracking_test.go +++ b/tests/e2e/distributed/router_tracking_test.go @@ -77,7 +77,7 @@ var _ = Describe("SmartRouter trackingKey", Label("Distributed"), func() { Expect(registry.Register(context.Background(), node, true)).To(Succeed()) nodeID = node.ID - unloader := nodes.NewRemoteUnloaderAdapter(registry, infra.NC, workers.Client(), 3*time.Minute, 15*time.Minute) + unloader := nodes.NewRemoteUnloaderAdapter(registry, workers.Client(), 3*time.Minute, 15*time.Minute) router = nodes.NewSmartRouter(registry, nodes.SmartRouterOptions{ Unloader: unloader, // Without a worker dialer the default factory refuses every From 8c6c4348920788fdc0f500894ee1c840a2720d82 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 17:30:54 +0000 Subject: [PATCH 088/153] test(e2e): build the cluster binary and drop a load-sensitive budget Two things made every end-to-end result on this branch unreliable. make test-e2e-cluster did not rebuild ./local-ai. It only checked that a file by that name existed, so an edit that was never rebuilt ran the whole suite against an older build while reporting on the working tree, and a missing binary skipped every spec and exited 0 with "Test Suite Passed". The target now depends on a new e2e-binary target, which is the plain go build CI already used rather than make build, since that one also builds the React UI this suite never touches. The harness carries the other half: localAIBinary now FAILS, locally as well as under CI, when the binary is older than the newest non-test Go source in the tree, which covers a run started with LOCALAI_E2E_BINARY or by invoking ginkgo directly. Test files are excluded from that scan because they compile into the ginkgo suite and never into local-ai. The CI job drops its own build step and the env var so that one place owns the build and it happens after protogen-go. test-e2e-distributed was audited for the same hole and has none: ginkgo compiles that suite from the working tree on every run and it execs no prebuilt binary. build-mock-backend already rebuilt unconditionally. "Worker tunnel under load" bounded the worst probe inside the bulk transfer window against the worst probe under the empty-load window. A max over n samples is a biased estimator when the two n differ, and here they always do: the bulk window is by construction longer and draws several times as many chances at an unrelated scheduling outlier. Anything loading the box widens that gap, so the spec reddened on what else was running: 255ms against a 161ms budget with make lint beside it, 57ms alone. It now bounds the probe COMPLETION RATE instead, which is the statistic a serialised session actually moves and a mean over dozens of samples in both terms, so a uniform slowdown cancels in the ratio. Measured with make lint running: 0.97 direct and 0.60 relayed against a floor of 0.125. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .agents/building-and-testing.md | 3 +- .github/workflows/tests-e2e-distributed.yml | 14 +-- Makefile | 45 ++++++++- .../e2e/distributed/cluster_baseline_test.go | 99 ++++++++++++++++++- tests/e2e/distributed/cluster_tunnel_test.go | 86 +++++++++++----- 5 files changed, 212 insertions(+), 35 deletions(-) diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index eee5123ac82c..3f614d039e85 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -51,7 +51,7 @@ Rules (both gates): Two suites cover distributed mode (frontend replicas, worker nodes, PostgreSQL, NATS), split by a Ginkgo label: - `make test-e2e-distributed` runs `Distributed && !VLLMMultinode && !Cluster` over `./tests/e2e/distributed` recursively. Services are wired directly into the test binary. ~240 specs, ~75s. -- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. 6 specs, about 8m30s measured over three consecutive runs (509.1s / 509.8s / 512.3s, so 8m29s to 8m32s). +- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. It **builds `./local-ai` itself** (the `e2e-binary` target), so you do not run `make build` first and cannot forget to. 6 specs, about 8m30s measured over three consecutive runs (509.1s / 509.8s / 512.3s, so 8m29s to 8m32s). Both jobs live in `.github/workflows/tests-e2e-distributed.yml`, with `timeout-minutes: 45` each. They trigger on pull requests *and* on every push to `master`; the `paths-ignore` filter (see [.agents/ci-caching.md](ci-caching.md)) sits on the pull-request trigger only, so a master push always runs both. They are advisory only because `master` carries no branch protection, which is a repository setting and not a YAML key: `continue-on-error: true` would flip the run's *conclusion* to success and hide the failure, so it is not used. @@ -59,6 +59,7 @@ Both jobs live in `.github/workflows/tests-e2e-distributed.yml`, with `timeout-m - **Consequence for new specs:** the NATS bus is now *shared* within a Ginkgo process, so a wildcard subscriber can observe another spec's traffic. Filter assertions on an identifier your spec owns (a node ID, a job ID) instead of counting everything on `jobs.*.progress`, and verify the spec with `--randomize-all`. - **`BeforeSuite`, not `SynchronizedBeforeSuite`.** Under `ginkgo -p` each process then gets its own container pair, keeping NATS subjects isolated per process. A single shared NATS across parallel processes would let specs on different processes see each other's messages on the same subject. - **The label split.** The 8 argument-validation specs under `tests/e2e/distributed/cluster/` carry `Label("Distributed")` only, on purpose: they need no binary, no PostgreSQL and no NATS, so they belong in the fast job. That is why `test-e2e-distributed` keeps `-r` (it must reach the subpackage) and `test-e2e-cluster` deliberately does **not** (the subpackage is out of its scope). +- **The binary is built, and a stale one is refused.** `test-e2e-cluster` used to take `./local-ai` as given and only check that a file by that name existed. An edit that was never rebuilt therefore ran the entire suite against an older build while reporting on the working tree, which surfaced once as a spec failing on caller line numbers that matched no line in any source file. Two things close it. The target depends on `e2e-binary`, which is `go build -o local-ai ./cmd/local-ai` and not `make build` (that one also builds the React UI, which this suite never touches and CI stubs). And `localAIBinary` in the harness **fails**, on a developer box as well as under CI, when the binary is older than the newest non-test Go source in the tree, which covers a run started with `LOCALAI_E2E_BINARY` or by invoking ginkgo directly. `_test.go` files are excluded from that scan on purpose: they compile into the ginkgo suite and never into `local-ai`, so counting them would fire on every edit to the spec being run. Set `LOCALAI_E2E_BINARY` to opt out of the build; there is no opt-out of the staleness check, because a rebuild is the remedy. - **`--fail-on-empty` is load-bearing on both targets.** Ginkgo exits 0 when a label filter selects nothing, so without it a refactor that renames or drops `Label("Cluster")` leaves the target reporting "Test Suite Passed" having started no cluster at all. `LOCALAI_E2E_REQUIRE_BINARIES` does not cover this case: it only fires inside a spec that is actually running. - **The binary gate.** `localAIBinary()` and `mockBackendBinary()` **fail** rather than skip when `CI` is set, or when `LOCALAI_E2E_REQUIRE_BINARIES` is truthy; `LOCALAI_E2E_REQUIRE_BINARIES=0` (also `off`, `no`, `n`, `disabled`, and anything `strconv.ParseBool` reads as false) forces skipping even under CI. **Any value that parses as neither reads as ON**, not off: setting the variable to something meaningless means someone meant to turn the gate on, and reading it as false would quietly restore the silent skip the flag exists to remove. The whole polarity is deliberate, because in CI a skipped cluster spec is indistinguishable from a passing one: Ginkgo exits 0 on skips. Locally a missing binary still just skips, since `CI` is unset in an ordinary shell. - **Flake budget: no retries at all.** `--flake-attempts` is *total attempts*, not retries (ginkgo v2.29.0 `internal/group.go` sets `maxAttempts = FlakeAttempts` and loops `attempt < maxAttempts`; the flag's own usage string reads "0 - failed tests are not retried"). `DISTRIBUTED_TEST_FLAKES` defaults to **1**, so each spec runs once and a failure is a failure, and `test-e2e-cluster` pins `--flake-attempts 1` outright rather than reading the variable. The repo-wide `TEST_FLAKES=5` means up to five attempts, so up to four retries. These suites exist to surface nondeterminism, and a retry converts exactly that signal into a green run. Raise it locally when bisecting something unrelated, not in the Makefile. diff --git a/.github/workflows/tests-e2e-distributed.yml b/.github/workflows/tests-e2e-distributed.yml index 8f17e72f169a..3f952ee42843 100644 --- a/.github/workflows/tests-e2e-distributed.yml +++ b/.github/workflows/tests-e2e-distributed.yml @@ -154,11 +154,6 @@ jobs: run: | mkdir -p core/http/react-ui/dist printf 'stub\n' > core/http/react-ui/dist/index.html - - name: Build local-ai - # Not `make build`: that target pulls in the React UI build. The specs - # exec this binary directly via LOCALAI_E2E_BINARY. - run: | - PATH="$PATH:$HOME/go/bin" go build -o local-ai ./cmd/local-ai - name: Pre-pull test images # Same reasoning as the job above: pulling here keeps container-start # timing out of the spec timeouts and makes a registry outage read as a @@ -168,7 +163,14 @@ jobs: docker pull nats:2-alpine - name: Cluster E2E env: - LOCALAI_E2E_BINARY: ${{ github.workspace }}/local-ai + # No LOCALAI_E2E_BINARY and no separate build step: make test-e2e-cluster + # builds ./local-ai itself, AFTER the protogen-go it also depends on. + # Building it in a step of its own put the generated .pb.go files newer + # than the binary, which the harness's staleness check reads, correctly, + # as a binary that does not contain the tree it is about to be measured + # against. One owner for the build is also what makes a local run and + # this job exercise the same bytes. + # # Must live under the workspace so the upload step below can reach it. # The harness defaults to GinkgoT().TempDir(), which lands under # TMPDIR and would leave the artifact glob matching nothing. diff --git a/Makefile b/Makefile index 354527465826..7a114dc4a77c 100644 --- a/Makefile +++ b/Makefile @@ -359,13 +359,54 @@ DISTRIBUTED_TEST_FLAKES?=1 # --fail-on-empty because ginkgo exits 0 when a label filter matches nothing, so # without it a rename of the label would turn this target into a silent no-op # that still reports "Test Suite Passed". +# +# This target needs NO built binary and therefore has no stale-binary hole: +# ginkgo compiles the suite from the working tree on every run, and everything +# these specs exercise runs in that process against containers. The one target +# in this file that does exec a prebuilt binary is test-e2e-cluster below, and +# the reason it now BUILDS it is written there. test-e2e-distributed: protogen-go @echo 'Running distributed e2e tests (label Distributed, incl. NatsJWT)' $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode && !Cluster' --fail-on-empty --flake-attempts $(DISTRIBUTED_TEST_FLAKES) --timeout=40m -v -r ./tests/e2e/distributed +# The local-ai binary the cluster e2e execs as its frontend and worker +# processes. +# +# Deliberately not `make build`: that target also builds the embedded React UI, +# which this suite never touches, and CI stubs core/http/react-ui/dist for +# exactly that reason. What is left is the go build, which is incremental and +# costs seconds when nothing changed, so making it a prerequisite of the suite +# is cheap enough that nobody has a reason to route around it. +# +# LOCALAI_E2E_BINARY means the caller has its own binary and owns keeping it +# current, so this builds nothing. The staleness check in the harness still +# applies to it, which is the point: a binary this target did not produce is +# precisely the one nothing else vouches for. +.PHONY: e2e-binary +e2e-binary: protogen-go + @if [ -n "$$LOCALAI_E2E_BINARY" ]; then \ + echo "LOCALAI_E2E_BINARY=$$LOCALAI_E2E_BINARY is set; not building $(BINARY_NAME)"; \ + else \ + echo "Building $(BINARY_NAME) for the cluster e2e"; \ + CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o $(BINARY_NAME) ./cmd/local-ai; \ + fi + # Cluster e2e: runs local-ai as real child processes (frontend replicas + # workers) against PostgreSQL and NATS, and kills them to assert failover. -# Needs a built ./local-ai (or LOCALAI_E2E_BINARY) plus the mock backend. +# It BUILDS that binary rather than checking that a file by that name exists, +# and that is a correctness fix rather than a convenience. This target used to +# take ./local-ai as given, so an edit to core/ that was never rebuilt left the +# whole suite exercising an older build while reporting on the working tree. The +# failure mode is not theoretical and not loud: it surfaced as a spec failing on +# caller line numbers that matched no line in any source file, and it would +# otherwise have surfaced as a green run that proved nothing. build-mock-backend +# already rebuilds unconditionally, so ./local-ai was the only gap. +# +# The harness carries the other half. Building here cannot help a run started +# with LOCALAI_E2E_BINARY or by invoking ginkgo directly, so localAIBinary also +# FAILS when the binary it is about to exec is older than the newest non-test Go +# source in the tree. Together: this target makes the common path correct, and +# the harness makes every other path loud. # # The argument-validation specs in tests/e2e/distributed/cluster deliberately # stay in test-e2e-distributed above: they need no binary, no PostgreSQL and no @@ -392,7 +433,7 @@ test-e2e-distributed: protogen-go # Ginkgo timeout kills the suite mid-spec and reports a spec name rather than a # cause, and 20m on a loaded CI runner was one slow health tick away from # turning a green suite into an unreadable red one. -test-e2e-cluster: protogen-go build-mock-backend +test-e2e-cluster: protogen-go build-mock-backend e2e-binary @echo 'Running cluster e2e tests (label Cluster, real local-ai processes)' $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Cluster' --fail-on-empty --flake-attempts 1 --timeout=30m -v ./tests/e2e/distributed diff --git a/tests/e2e/distributed/cluster_baseline_test.go b/tests/e2e/distributed/cluster_baseline_test.go index 1b2aaae2be04..6f69898c23a1 100644 --- a/tests/e2e/distributed/cluster_baseline_test.go +++ b/tests/e2e/distributed/cluster_baseline_test.go @@ -130,7 +130,7 @@ func missingBinary(what, path, remedy string) { Skip(message) } -// localAIBinary resolves the built binary. +// localAIBinary resolves the built binary and refuses a stale one. func localAIBinary() string { GinkgoHelper() path := os.Getenv("LOCALAI_E2E_BINARY") @@ -139,12 +139,105 @@ func localAIBinary() string { Expect(err).ToNot(HaveOccurred()) path = filepath.Join(wd, "..", "..", "..", "local-ai") } - if _, err := os.Stat(path); err != nil { - missingBinary("local-ai binary", path, "run `make build` or set LOCALAI_E2E_BINARY") + info, err := os.Stat(path) + if err != nil { + missingBinary("local-ai binary", path, "run `make test-e2e-cluster`, which builds it, or set LOCALAI_E2E_BINARY") } + requireFreshBinary(path, info.ModTime()) return path } +// requireFreshBinary fails when the binary about to be exec'd predates the +// sources it is supposed to contain. +// +// A missing binary is a loud failure; a STALE one is a silent lie. The suite +// spawns this file as its frontends and workers, so an out-of-date build makes +// every spec a statement about code nobody can name, and the pass or fail it +// produces belongs to a commit that is not the one under test. That has already +// happened here once: a spec reported caller line numbers that matched no line +// in any source file, which was the only reason anyone noticed. +// +// It FAILS rather than skipping, and it does so on a developer box as well as +// under CI, which is the opposite polarity to missingBinary above. The +// distinction is what a wrong answer costs: a skipped spec reports nothing and +// a stale spec reports something false, so there is no environment in which +// continuing is the kinder choice. +// +// _test.go files are excluded on purpose and the exclusion is load-bearing. +// They are compiled into the ginkgo suite, never into local-ai, so counting +// them would fire on every edit to the spec being run and would train everyone +// to route around the check within a day. +func requireFreshBinary(path string, built time.Time) { + GinkgoHelper() + newest, at, err := newestGoSource() + Expect(err).ToNot(HaveOccurred(), "walking the tree for source timestamps") + if at == "" || !newest.After(built) { + return + } + Fail(fmt.Sprintf( + "%s was built at %s but %s changed at %s, so this suite would exec a binary that does not contain the tree it is about to report on; run `make test-e2e-cluster`, which builds it", + path, built.Format(time.RFC3339), at, newest.Format(time.RFC3339))) +} + +// sourceScanSkip names the directories the freshness walk does not descend +// into. Each holds either no Go the binary links, or enough files to make the +// walk cost more than the check is worth. +var sourceScanSkip = map[string]bool{ + ".git": true, + "node_modules": true, + "dist": true, + "models": true, + "backends": true, + "local-backends": true, + "vendor": true, +} + +// newestGoSource returns the newest modification time among the Go sources that +// end up in the binary, and the path that carries it. +// +// go.mod and go.sum count: a dependency bump changes neither a .go file nor the +// working tree's own code, and produces a different binary all the same. +func newestGoSource() (time.Time, string, error) { + root, err := os.Getwd() + if err != nil { + return time.Time{}, "", err + } + root = filepath.Join(root, "..", "..", "..") + var newest time.Time + var at string + err = filepath.WalkDir(root, func(p string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + // A file that vanished mid-walk is not this check's business, and + // aborting on it would turn a race in someone's editor into a + // failed suite. + return nil //nolint:nilerr // see above + } + if d.IsDir() { + if sourceScanSkip[d.Name()] { + return filepath.SkipDir + } + return nil + } + name := d.Name() + if strings.HasSuffix(name, "_test.go") { + return nil + } + if !strings.HasSuffix(name, ".go") && name != "go.mod" && name != "go.sum" { + return nil + } + info, err := d.Info() + if err != nil { + return nil //nolint:nilerr // same reasoning as the walk error above + } + if info.ModTime().After(newest) { + newest = info.ModTime() + at = p + } + return nil + }) + return newest, at, err +} + func mockBackendBinary() string { GinkgoHelper() wd, err := os.Getwd() diff --git a/tests/e2e/distributed/cluster_tunnel_test.go b/tests/e2e/distributed/cluster_tunnel_test.go index 3da9e385362f..dd77707db556 100644 --- a/tests/e2e/distributed/cluster_tunnel_test.go +++ b/tests/e2e/distributed/cluster_tunnel_test.go @@ -654,6 +654,21 @@ func percentileIndex(n int, p float64) int { // slowestOf is the worst sample, which is the statistic a blocking question // turns on: a session that stalls one request while a transfer holds it shows // up in the tail and not in the middle. +// completionRate is how many probes finished per second of the window they ran +// in. It is the statistic the head-of-line assertion is made on, because a +// serialised session shows up as work that does not complete rather than as one +// probe that finished slowly. +// +// The window is passed in rather than summed from the samples: a probe that +// never returns contributes no sample at all, and summing would then report a +// stalled session as a short, healthy window. +func completionRate(samples []time.Duration, window time.Duration) float64 { + if window <= 0 { + return 0 + } + return float64(len(samples)) / window.Seconds() +} + func slowestOf(samples []time.Duration) time.Duration { worst := time.Duration(0) for _, d := range samples { @@ -785,24 +800,44 @@ const ( // window rather than a fifth of it. holStallShare = 2 - // holStallControlFactor bounds the worst probe against the worst probe - // under the EMPTY load in the same run, which is the second half of not - // relaxing under load: a slower box raises the control and the bound with - // it, while an absolute number would simply admit more. + // holStallRateFactor bounds how far the probe COMPLETION RATE inside the + // bulk window may fall below the rate inside the empty-load window of the + // same run. It replaces a bound that compared the two windows' WORST probe, + // and the replacement is not a loosening: it is a different statistic, + // chosen because the old one could not answer the question it was asked. + // + // Why the old comparison had to go. A max over n samples is a biased + // estimator when the two n differ, and here they always differ: the bulk + // window is by construction the longer of the two, so it holds several + // times as many probes and therefore several times as many chances to draw + // an unrelated scheduling outlier. Anything that loads the box widens that + // gap rather than closing it, because the tail it samples from grows while + // the short control window keeps drawing too few samples to see the same + // tail. Measured: the worst probe was 57ms with the box quiet and 255ms + // with `make lint` running beside it, against a control-derived budget of + // 161ms that barely moved between the two. A spec that reddens on what else + // is running is not measuring this feature. // - // The empty load is the right thing to compare against and the plain - // baseline is not. Both samples then contain a cold load's contention for - // the worker, the router and the session, and the only thing that differs - // between them is 128 MiB crossing the wire. Compared against the quiet - // baseline instead, a transfer that cost nothing at all would still look - // like a regression on any box where a cold load is expensive. + // Why a rate answers it. The defect is a session that SERIALISES, and its + // signature is that almost nothing else completes while bytes are moving, + // so it shows up in how many probes finish per second and not in which one + // finished slowest. A rate is a mean over dozens of samples in both terms: + // an unrelated process slows the box, which divides both rates by much the + // same factor and cancels in the ratio, while one outlier moves a mean over + // dozens of samples by a fraction of itself. The self-scaling property the + // old bound was written for survives intact, since the control is still + // measured in the same run under the same conditions. // - // Eight, from both ends of the gap it has to sit in. Healthy runs measured - // 1.6x to 3.8x on this box under a concurrent `-race` suite, and about 2.5x - // to 3x on the reviewer's; a session that stalled a probe until the - // transfer let go would show the whole window over the same control, which - // is 14x to 43x on the same runs. - holStallControlFactor = 8 + // Eight, from both ends of the gap it has to sit in, and deliberately the + // same number the old bound carried so that a reader comparing the two + // revisions sees a changed statistic and not a raised budget. A session + // that stalled its probes until the transfer let go completes one probe or + // none in a window in which the empty-load control completes dozens, which + // is a ratio far beyond eight. Healthy runs on this box, WITH `make lint` + // running beside them, measured 0.97 direct and 0.60 relayed, so the floor + // of 0.125 sits about five times below the worse of the two; the spec + // records both numbers in its report entry on every run. + holStallRateFactor = 8 // holStallCeiling is the coarse absolute backstop under both of those, for // a transfer so slow that a quarter of its window is a latency no @@ -962,13 +997,16 @@ var _ = Describe("Worker tunnel under load", Label("Distributed"), Label("Cluste "%s: only %d probes overlapped a transfer window of %s, which is too few to say anything about head-of-line blocking", label, len(underBulk), transferWindow) - line := fmt.Sprintf("%s\n %s\n %s\n %s\n transfer window: %s of a %s load (%d MiB), empty load %s", + line := fmt.Sprintf("%s\n %s\n %s\n %s\n transfer window: %s of a %s load (%d MiB), empty load %s\n completion rate: %.1f/s under bulk vs %.1f/s under empty load (ratio %.2f, floor 1/%d)", label, summarise("baseline ", baseline), summarise("under empty load ", underTiny), summarise("under bulk load ", underBulk), transferWindow.Round(time.Millisecond), bulkElapsed.Round(time.Millisecond), - bulkArtifactSize>>20, tinyElapsed.Round(time.Millisecond)) + bulkArtifactSize>>20, tinyElapsed.Round(time.Millisecond), + completionRate(underBulk, bulkElapsed), completionRate(underTiny, tinyElapsed), + completionRate(underBulk, bulkElapsed)/completionRate(underTiny, tinyElapsed), + holStallRateFactor) report = append(report, line) GinkgoWriter.Println(line) @@ -976,11 +1014,13 @@ var _ = Describe("Worker tunnel under load", Label("Distributed"), Label("Cluste Expect(slowest).To(BeNumerically("<", transferWindow/holStallShare), "%s: a probe waited %s of the %s in which bytes were moving, which is the shape of a session that stalled the probe until the transfer let go, not of one that interleaved them", label, slowest, transferWindow) - control := slowestOf(underTiny) - Expect(control).To(BeNumerically(">", 0), "%s: the empty-load control produced no samples", label) - Expect(slowest).To(BeNumerically("<", holStallControlFactor*control), - "%s: the worst probe was %s while bytes were moving against %s under the same cold load with nothing to move, which is a stall rather than the contention a shared session costs", - label, slowest, control) + bulkRate := completionRate(underBulk, bulkElapsed) + tinyRate := completionRate(underTiny, tinyElapsed) + Expect(tinyRate).To(BeNumerically(">", 0), + "%s: the empty-load control completed no probes, so there is nothing to compare the bulk window against", label) + Expect(bulkRate).To(BeNumerically(">=", tinyRate/holStallRateFactor), + "%s: probes completed at %.1f/s while 128 MiB was crossing the session against %.1f/s under the same cold load with nothing to move, which is a session that stopped admitting work rather than one that shared it", + label, bulkRate, tinyRate) Expect(slowest).To(BeNumerically("<", holStallCeiling), "%s: a probe waited %s while the bulk transfer held the session", label, slowest) } From ae3ff21ad46380d798a7382875340a570f8dd638 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 17:31:13 +0000 Subject: [PATCH 089/153] feat(distributed): let a worker name a broadcast the frontend decides on A worker has no database and cannot NOTIFY, and it does not need to: every message it sends is produced inside a handler the frontend invoked, so there is always an open control response to write on. This adds the two ends of that, and the authorization decision that sits between them, and nothing that dispatches yet. workerctl.Envelope gains Subject, a REQUEST and not an instruction. Empty means the line is for this caller alone, which is what every pre-existing progress line is, and omitempty keeps those lines byte-identical for an older reader. It qualifies a progress line and never a reply line: a reply is the worker's verdict about the work, and there is no version of "publish my verdict for me" this control plane has to carry. nodes.MayBroadcast is the replacement for pkg/natsauth's allow list, and the inversion is the point. NATS read an EMPTY allow list as NO RESTRICTION, which is why phase 3 refused to delete the backend branch and spelled it {"_INBOX.>"}. This one reads an empty list, and an absent node type, as DENY EVERYTHING, and a table-driven spec pins that by emptying the agent entry and asserting all three of its subjects are then refused. Matching goes through messaging.SubjectMatches, the one definition in the tree, so a filter that fires here fires on the carrier. nodes.Rebroadcaster.Handle returns a bool and never an error. A refused or failed re-broadcast is logged and the RPC continues, because the RPC's outcome is the worker's verdict about the work and a publish failure says nothing about it. The return shape is asserted at compile time in the file that states the rule, so changing it to an error does not compile. ControlClient.CallStreaming's progress callback now takes the line's subject alongside its raw bytes. The client no longer decodes progress at all: what a line means is a question about the verb and whether a named broadcast may be made is a question about the node, and it knows neither. Both moved into installProgressBridge, which every one of the three streaming call sites in unloader.go goes through. A line naming a subject is dropped there rather than delivered as install progress, because backend.install and backend.upgrade are a backend worker's verbs and a backend worker is allowed no subjects. agents.StreamPublisher is the producing end, a messaging.Publisher writing NDJSON envelopes onto an in-flight control response and flushing each one, so a tick reaches the frontend while the handler is still running. Serialized, because two concurrent encodes on one http.ResponseWriter interleave bytes and tear the framing. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/agents/stream_publisher.go | 74 ++++++++ core/services/agents/stream_publisher_test.go | 166 ++++++++++++++++++ core/services/nodes/control_client.go | 22 ++- core/services/nodes/control_client_test.go | 70 +++++++- .../nodes/control_worker_fake_test.go | 20 +++ core/services/nodes/rebroadcast.go | 118 +++++++++++++ core/services/nodes/rebroadcast_test.go | 153 ++++++++++++++++ core/services/nodes/unloader.go | 46 ++++- core/services/nodes/unloader_test.go | 100 +++++++++++ .../worker/control_client_roundtrip_test.go | 14 +- core/services/workerctl/paths.go | 26 ++- core/services/workerctl/paths_test.go | 31 ++++ 12 files changed, 809 insertions(+), 31 deletions(-) create mode 100644 core/services/agents/stream_publisher.go create mode 100644 core/services/agents/stream_publisher_test.go create mode 100644 core/services/nodes/rebroadcast.go create mode 100644 core/services/nodes/rebroadcast_test.go diff --git a/core/services/agents/stream_publisher.go b/core/services/agents/stream_publisher.go new file mode 100644 index 000000000000..f3f2b6978c48 --- /dev/null +++ b/core/services/agents/stream_publisher.go @@ -0,0 +1,74 @@ +package agents + +import ( + "encoding/json" + "fmt" + "io" + "sync" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// StreamPublisher is a messaging.Publisher that writes progress lines onto an +// in-flight control response instead of onto a bus. +// +// It exists because a worker has no database and therefore cannot NOTIFY, and +// because it does not need to: every message a worker sends is produced inside +// a handler the frontend invoked, so there is always an open response to write +// on. +// +// A subject written here is a REQUEST and not a publish. Nothing on this side +// is authorised: the frontend reading the stream checks the subject against the +// allow list for the node's type and refuses what the worker has no business +// on. That refusal is silent to this end on purpose, since a worker learning +// which subjects a frontend will carry is the beginning of a worker that probes +// for them. +// +// Writes are serialized: cogito's stream callbacks run on the goroutine that is +// executing the agent, but the status callback and the tool-result callback can +// interleave with a flush, and two concurrent json.Encoder writes on one +// http.ResponseWriter interleave bytes and corrupt the NDJSON framing. +type StreamPublisher struct { + mu sync.Mutex + enc *json.Encoder + flush func() +} + +// Compile-time proof that this is the Publisher a caller can hand to anything +// that publishes. Asserted here rather than at the first adopter so a change to +// either side fails to build in the file that owns the type. +var _ messaging.Publisher = (*StreamPublisher)(nil) + +// NewStreamPublisher returns a StreamPublisher writing to w. flush may be nil, +// which is the right shape for a writer that has no buffering to push through. +func NewStreamPublisher(w io.Writer, flush func()) *StreamPublisher { + return &StreamPublisher{enc: json.NewEncoder(w), flush: flush} +} + +// Publish writes one Envelope carrying subject and the JSON encoding of data, +// then flushes, so a progress tick reaches the frontend while the handler is +// still running rather than when the response closes. +// +// data is encoded OUTSIDE the lock. Encoding is the expensive half and it +// touches nothing shared, so holding the lock across it would serialise every +// caller behind the slowest one's marshalling for no gain in framing. +func (p *StreamPublisher) Publish(subject string, data any) error { + raw, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("encoding a stream message for %q: %w", subject, err) + } + + p.mu.Lock() + defer p.mu.Unlock() + if err := p.enc.Encode(workerctl.Envelope{Subject: subject, Progress: raw}); err != nil { + return fmt.Errorf("writing a stream message for %q: %w", subject, err) + } + // The flush is inside the lock and belongs there. A flush concurrent with + // an Encode pushes a partially written line at the reader, which is the + // same torn frame the lock exists to prevent. + if p.flush != nil { + p.flush() + } + return nil +} diff --git a/core/services/agents/stream_publisher_test.go b/core/services/agents/stream_publisher_test.go new file mode 100644 index 000000000000..4932a45a471f --- /dev/null +++ b/core/services/agents/stream_publisher_test.go @@ -0,0 +1,166 @@ +package agents + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "runtime" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/workerctl" +) + +// chunkWriter is an http.ResponseWriter's concurrency contract and nothing +// else: it is NOT safe for concurrent use, and it splits every Write into +// single bytes with a scheduling point between them. +// +// Both halves are deliberate. A double that locked internally would make two +// unsynchronised Encode calls look atomic and could never fail the way a real +// response fails, which is the failure this spec exists to catch. Splitting the +// write is what turns "two goroutines might interleave" into "two goroutines +// do", so the concurrency spec reddens on the framing itself and not only under +// the race detector. +type chunkWriter struct { + buf []byte +} + +func (w *chunkWriter) Write(p []byte) (int, error) { + for _, b := range p { + w.buf = append(w.buf, b) + runtime.Gosched() + } + return len(p), nil +} + +// errWriter fails every write, for the one thing Publish promises about a +// response it cannot write on. +type errWriter struct{ err error } + +func (w errWriter) Write([]byte) (int, error) { return 0, w.err } + +// decodeLines reads every envelope out of a buffer, which is the assertion that +// fails on interleaving: a torn line does not decode, and a line that swallowed +// its neighbour does not produce two. +func decodeLines(raw []byte) ([]workerctl.Envelope, error) { + dec := json.NewDecoder(bytes.NewReader(raw)) + var out []workerctl.Envelope + for { + var env workerctl.Envelope + err := dec.Decode(&env) + if errors.Is(err, io.EOF) { + return out, nil + } + if err != nil { + return out, err + } + out = append(out, env) + } +} + +var _ = Describe("StreamPublisher", func() { + var ( + buf *bytes.Buffer + flushes int + pub *StreamPublisher + ) + + BeforeEach(func() { + buf = &bytes.Buffer{} + flushes = 0 + pub = NewStreamPublisher(buf, func() { flushes++ }) + }) + + It("writes exactly one NDJSON line whose subject and progress decode back to the inputs", func() { + Expect(pub.Publish("agent.a1.events.status", map[string]any{"state": "thinking"})).To(Succeed()) + + lines, err := decodeLines(buf.Bytes()) + Expect(err).ToNot(HaveOccurred()) + Expect(lines).To(HaveLen(1)) + Expect(lines[0].Subject).To(Equal("agent.a1.events.status")) + Expect(string(lines[0].Progress)).To(MatchJSON(`{"state":"thinking"}`)) + // A line the frontend would read as terminal ends the stream, so a + // progress publisher must never write one. + Expect(lines[0].Reply).To(BeNil()) + Expect(bytes.Count(buf.Bytes(), []byte("\n"))).To(Equal(1)) + }) + + It("flushes after every line, so a tick reaches the frontend while the handler still runs", func() { + Expect(pub.Publish("jobs.j1.progress", map[string]any{"percentage": 10})).To(Succeed()) + Expect(flushes).To(Equal(1)) + Expect(pub.Publish("jobs.j1.progress", map[string]any{"percentage": 20})).To(Succeed()) + Expect(flushes).To(Equal(2)) + }) + + It("writes a line at all when it was given no flush to call", func() { + plain := &bytes.Buffer{} + Expect(NewStreamPublisher(plain, nil).Publish("jobs.j1.result", map[string]any{"ok": true})).To(Succeed()) + lines, err := decodeLines(plain.Bytes()) + Expect(err).ToNot(HaveOccurred()) + Expect(lines).To(HaveLen(1)) + }) + + It("keeps every concurrent publish a complete, decodable line of its own", func() { + w := &chunkWriter{} + p := NewStreamPublisher(w, func() {}) + + // Four writers rather than two, and enough lines each that an + // unsynchronised run has to interleave rather than merely be allowed + // to. Measured: at two writers of twenty lines an unlocked publisher + // still produced a decodable buffer on some schedules, so the framing + // assertion below was a coin toss and only the race detector was + // reliably red. + subjects := []string{ + "agent.a1.events.status", + "agent.a2.events.status", + "jobs.j1.progress", + "jobs.j2.result", + } + const perGoroutine = 60 + var wg sync.WaitGroup + errs := make(chan error, len(subjects)*perGoroutine) + for _, subject := range subjects { + wg.Add(1) + go func(subject string) { + defer wg.Done() + for i := 0; i < perGoroutine; i++ { + if err := p.Publish(subject, map[string]any{"n": i, "subject": subject}); err != nil { + errs <- err + } + } + }(subject) + } + wg.Wait() + close(errs) + Expect(errs).ToNot(Receive()) + + lines, err := decodeLines(w.buf) + Expect(err).ToNot(HaveOccurred(), "a line was torn by an interleaved write") + Expect(lines).To(HaveLen(len(subjects) * perGoroutine)) + // Every line must still carry the pair it was written with. A decode + // that happened to succeed on spliced bytes would show up here as a + // subject that does not match its payload. + for _, line := range lines { + var payload struct { + Subject string `json:"subject"` + } + Expect(json.Unmarshal(line.Progress, &payload)).To(Succeed()) + Expect(payload.Subject).To(Equal(line.Subject)) + } + }) + + It("reports a value it cannot encode without writing anything", func() { + Expect(pub.Publish("jobs.j1.progress", make(chan int))).ToNot(Succeed()) + Expect(buf.Len()).To(BeZero()) + Expect(flushes).To(BeZero()) + }) + + It("reports a response it cannot write on, and does not flush a line it did not write", func() { + failing := NewStreamPublisher(errWriter{err: errors.New("connection reset")}, func() { flushes++ }) + Expect(failing.Publish("jobs.j1.progress", map[string]any{"percentage": 10})).ToNot(Succeed()) + Expect(flushes).To(BeZero()) + }) +}) diff --git a/core/services/nodes/control_client.go b/core/services/nodes/control_client.go index 6c943892b426..fe0d7e5f0e8f 100644 --- a/core/services/nodes/control_client.go +++ b/core/services/nodes/control_client.go @@ -11,10 +11,7 @@ import ( "sync" "time" - "github.com/mudler/xlog" - "github.com/mudler/LocalAI/core/services/cluster" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/workerctl" "github.com/mudler/LocalAI/pkg/httpclient" ) @@ -152,6 +149,14 @@ func (c *ControlClient) Call(ctx context.Context, nodeID, path string, req, repl // stream, invoking onProgress for each progress line and decoding the single // terminal reply line into reply. onProgress may be nil. // +// onProgress receives the line's SUBJECT alongside its bytes, so a caller can +// tell a private progress tick from a re-broadcast request without this client +// understanding either. An empty subject is a line for this caller alone, which +// is what every pre-existing progress line is. The client deliberately does no +// authorization and no decoding of its own: whether a named broadcast may be +// made is a policy question about the node, and what a private tick means is a +// question about the verb, and this type knows neither. +// // onProgress runs SYNCHRONOUSLY, on this goroutine. The NATS carrier ran each // progress callback on a goroutine of its own because a slow callback there // stalled the one reader thread every worker's events arrived on; here the only @@ -159,7 +164,7 @@ func (c *ControlClient) Call(ctx context.Context, nodeID, path string, req, repl // caller's business. Dropping the guard is also what makes the events arrive in // the order the worker sent them. func (c *ControlClient) CallStreaming(ctx context.Context, nodeID, path string, - req, reply any, onProgress func(messaging.BackendInstallProgressEvent)) error { + req, reply any, onProgress func(subject string, raw json.RawMessage)) error { resp, err := c.do(ctx, nodeID, path, req) if err != nil { return err @@ -188,14 +193,7 @@ func (c *ControlClient) CallStreaming(ctx context.Context, nodeID, path string, if env.Progress == nil || onProgress == nil { continue } - var ev messaging.BackendInstallProgressEvent - if err := json.Unmarshal(env.Progress, &ev); err != nil { - // Progress is transient by contract, so a line this frontend cannot - // read costs a tick and never the operation. - xlog.Debug("unreadable control progress line", "node", nodeID, "path", path, "error", err) - continue - } - onProgress(ev) + onProgress(env.Subject, env.Progress) } if reply == nil { return nil diff --git a/core/services/nodes/control_client_test.go b/core/services/nodes/control_client_test.go index d8abc1197a5d..ea19c3eff37a 100644 --- a/core/services/nodes/control_client_test.go +++ b/core/services/nodes/control_client_test.go @@ -223,13 +223,58 @@ var _ = Describe("ControlClient", func() { _ = enc.Encode(workerctl.Envelope{Progress: json.RawMessage(`{"percentage":100}`)}) _ = enc.Encode(workerctl.Envelope{Reply: json.RawMessage(`{"success":true}`)}) } - var seen []float64 + var seen []string + var subjects []string var reply messaging.BackendInstallReply err := client.CallStreaming(context.Background(), "n1", workerctl.PathBackendInstall, messaging.BackendInstallRequest{Backend: "mock", OpID: "op-1"}, &reply, - func(ev messaging.BackendInstallProgressEvent) { seen = append(seen, ev.Percentage) }) + func(subject string, raw json.RawMessage) { + subjects = append(subjects, subject) + seen = append(seen, string(raw)) + }) Expect(err).NotTo(HaveOccurred()) - Expect(seen).To(Equal([]float64{50, 100})) + Expect(seen).To(Equal([]string{`{"percentage":50}`, `{"percentage":100}`})) + // A line that named no broadcast reaches the caller with an empty + // subject, which is what every pre-existing progress line is. + Expect(subjects).To(Equal([]string{"", ""})) + Expect(reply.Success).To(BeTrue()) + }) + + It("carries the subject a worker named through to the caller, and acts on it itself not at all", func() { + // The client is not where the authorization decision lives, and it + // must not become where it lives by accident: it hands the subject + // over and keeps reading. Whether the broadcast is made is a + // question about the node's type that this type cannot answer. + handler = func(w http.ResponseWriter, _ *http.Request) { + enc := json.NewEncoder(w) + _ = enc.Encode(workerctl.Envelope{ + Subject: "agent.a1.events.status", + Progress: json.RawMessage(`{"state":"thinking"}`), + }) + _ = enc.Encode(workerctl.Envelope{ + // A subject no worker of any type is allowed. The client + // still carries it: refusing here would put the policy in + // two places. + Subject: "cache.invalidate.models", + Progress: json.RawMessage(`{"model":"m1"}`), + }) + _ = enc.Encode(workerctl.Envelope{Reply: json.RawMessage(`{"success":true}`)}) + } + type line struct { + subject string + raw string + } + var seen []line + var reply messaging.BackendInstallReply + Expect(client.CallStreaming(context.Background(), "n1", workerctl.PathBackendInstall, + struct{}{}, &reply, + func(subject string, raw json.RawMessage) { + seen = append(seen, line{subject: subject, raw: string(raw)}) + })).To(Succeed()) + Expect(seen).To(Equal([]line{ + {subject: "agent.a1.events.status", raw: `{"state":"thinking"}`}, + {subject: "cache.invalidate.models", raw: `{"model":"m1"}`}, + })) Expect(reply.Success).To(BeTrue()) }) @@ -239,11 +284,11 @@ var _ = Describe("ControlClient", func() { _ = enc.Encode(workerctl.Envelope{Reply: json.RawMessage(`{"success":true}`)}) _ = enc.Encode(workerctl.Envelope{Progress: json.RawMessage(`{"percentage":10}`)}) } - var seen []float64 + var seen []string var reply messaging.BackendInstallReply Expect(client.CallStreaming(context.Background(), "n1", workerctl.PathBackendInstall, struct{}{}, &reply, - func(ev messaging.BackendInstallProgressEvent) { seen = append(seen, ev.Percentage) })).To(Succeed()) + func(_ string, raw json.RawMessage) { seen = append(seen, string(raw)) })).To(Succeed()) Expect(seen).To(BeEmpty()) Expect(reply.Success).To(BeTrue()) }) @@ -274,19 +319,26 @@ var _ = Describe("ControlClient", func() { Expect(errors.Is(err, io.ErrUnexpectedEOF)).To(BeTrue()) }) - It("keeps going past a progress line it cannot read, since progress is transient", func() { + It("hands over a progress line it could not have decoded, because it does not decode them", func() { + // The client used to unmarshal every progress line into an install + // event and drop the ones that would not parse. It no longer knows + // what a progress line means, so a line that is not an install + // event is carried like any other and the DECODE is the caller's. + // The rule that a line the frontend cannot read costs a tick and + // never the operation now lives in installProgressBridge, and + // unloader_test.go pins it there. handler = func(w http.ResponseWriter, _ *http.Request) { enc := json.NewEncoder(w) _ = enc.Encode(workerctl.Envelope{Progress: json.RawMessage(`{"percentage":"not a number"}`)}) _ = enc.Encode(workerctl.Envelope{Progress: json.RawMessage(`{"percentage":70}`)}) _ = enc.Encode(workerctl.Envelope{Reply: json.RawMessage(`{"success":true}`)}) } - var seen []float64 + var seen []string var reply messaging.BackendInstallReply Expect(client.CallStreaming(context.Background(), "n1", workerctl.PathBackendInstall, struct{}{}, &reply, - func(ev messaging.BackendInstallProgressEvent) { seen = append(seen, ev.Percentage) })).To(Succeed()) - Expect(seen).To(Equal([]float64{70})) + func(_ string, raw json.RawMessage) { seen = append(seen, string(raw)) })).To(Succeed()) + Expect(seen).To(Equal([]string{`{"percentage":"not a number"}`, `{"percentage":70}`})) Expect(reply.Success).To(BeTrue()) }) diff --git a/core/services/nodes/control_worker_fake_test.go b/core/services/nodes/control_worker_fake_test.go index b76d59b02496..f5459e35130e 100644 --- a/core/services/nodes/control_worker_fake_test.go +++ b/core/services/nodes/control_worker_fake_test.go @@ -43,6 +43,13 @@ type scriptedControlWorkers struct { matched map[string][]matchedControlReply progress map[string][]messaging.BackendInstallProgressEvent + // rawProgress holds progress lines a spec wrote out as envelopes, for the + // two things scriptProgress cannot express: a line NAMING a subject, and a + // line whose payload is not an install-progress event at all. Both are + // things a real worker on another build can put on the wire, so the double + // has to be able to put them on the wire too. + rawProgress map[string][]workerctl.Envelope + // unreachable and expired are keyed by NODE, not by verb, because they are // failures of the ROUTE and a route belongs to a node. They are what the // dialer answers with; see scriptUnroutable and scriptTimeout. @@ -74,6 +81,7 @@ func newScriptedControlWorkers() *scriptedControlWorkers { unsupported: map[string]bool{}, matched: map[string][]matchedControlReply{}, progress: map[string][]messaging.BackendInstallProgressEvent{}, + rawProgress: map[string][]workerctl.Envelope{}, unreachable: map[string]bool{}, expired: map[string]bool{}, hangs: map[string]bool{}, @@ -131,6 +139,7 @@ func (s *scriptedControlWorkers) serve(w http.ResponseWriter, r *http.Request) { reply := s.replies[key] matchers := s.matched[key] ticks := s.progress[key] + rawTicks := s.rawProgress[key] s.mu.Unlock() if hang { @@ -182,6 +191,9 @@ func (s *scriptedControlWorkers) serve(w http.ResponseWriter, r *http.Request) { } _ = enc.Encode(workerctl.Envelope{Progress: raw}) } + for _, env := range rawTicks { + _ = enc.Encode(env) + } _ = enc.Encode(workerctl.Envelope{Reply: reply}) } @@ -270,6 +282,14 @@ func (s *scriptedControlWorkers) scriptProgress(key string, events []messaging.B s.progress[key] = events } +// scriptRawProgress queues progress lines exactly as they go on the wire, after +// any scriptProgress lines for the same verb. +func (s *scriptedControlWorkers) scriptRawProgress(key string, lines []workerctl.Envelope) { + s.mu.Lock() + defer s.mu.Unlock() + s.rawProgress[key] = lines +} + // scriptHang makes one verb on one node accept the request and never answer, // so the call ends only when the caller's budget does. // diff --git a/core/services/nodes/rebroadcast.go b/core/services/nodes/rebroadcast.go new file mode 100644 index 000000000000..10e7809c80e1 --- /dev/null +++ b/core/services/nodes/rebroadcast.go @@ -0,0 +1,118 @@ +package nodes + +import ( + "encoding/json" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/xlog" +) + +// workerBroadcastAllow lists, per node type, the subject filters a worker may +// ask a frontend to publish on its behalf. +// +// An absent node type and an empty list both DENY EVERYTHING. That is the +// opposite of the NATS allow list this replaces, where an empty list meant no +// restriction, and it is the reason deleting an entry here is safe where +// deleting a permissions branch there was a privilege escalation. Nothing may +// be added to this map without a spec that a worker of that type is refused the +// subjects NOT added. +var workerBroadcastAllow = map[string][]string{ + NodeTypeAgent: { + "jobs.*.progress", + "jobs.*.result", + "agent.*.events.*", + }, + // A backend worker asks for no broadcasts. Spelled as an empty list rather + // than omitted, so the reader sees the decision. + NodeTypeBackend: {}, +} + +// MayBroadcast reports whether a worker of nodeType may have subject published +// on its behalf. +func MayBroadcast(nodeType, subject string) bool { + return mayBroadcastIn(workerBroadcastAllow, nodeType, subject) +} + +// mayBroadcastIn is MayBroadcast against a caller-supplied table. +// +// It exists so a spec can hold the table itself constant while varying what is +// IN it, which is the only way to state "an empty list denies" as a property of +// this function rather than as a property of today's entries. Production has +// exactly one table and MayBroadcast is the only way to reach it. +func mayBroadcastIn(table map[string][]string, nodeType, subject string) bool { + // An empty subject is a line that asked for no broadcast at all, so there + // is nothing here to authorise. Refusing it is not defensive tidiness: it + // keeps a filter that happened to match the empty string from turning every + // ordinary private progress tick into a publish. + if subject == "" { + return false + } + // A missing node type indexes to nil and the loop then runs zero times, + // which is the same denial an empty list produces. One code path for both + // is deliberate: "we have never heard of this worker" and "this worker is + // allowed nothing" have the same safe answer, and giving them separate + // branches invites one of the two to acquire an exception. + for _, filter := range table[nodeType] { + if messaging.SubjectMatches(filter, subject) { + return true + } + } + return false +} + +// Rebroadcaster turns an allowed progress line into a broadcast. +// +// It holds a Broadcaster and not a MessagingClient because fan-out is all it +// needs and all it may have: a worker asking for a broadcast must not be able +// to reach a request/reply or a queue group through this path. +type Rebroadcaster struct { + bus messaging.Broadcaster +} + +// NewRebroadcaster returns a Rebroadcaster publishing on bus. +func NewRebroadcaster(bus messaging.Broadcaster) *Rebroadcaster { + return &Rebroadcaster{bus: bus} +} + +// The shape of Handle's answer is asserted here, not left to its callers. +// +// A refused or failed re-broadcast must never become an error, because the only +// errors that reach a scheduler from this direction are the worker's own answer +// and an unroutable peer, and a publish failure is neither. Changing the return +// to an error therefore has to fail to COMPILE in the file that states the +// rule, rather than fail a spec somewhere a reviewer might read as a test +// needing an update. +var _ func(string, string, json.RawMessage) bool = (*Rebroadcaster)(nil).Handle + +// Handle publishes raw on subject when the worker's type allows it, and returns +// false having published nothing when it does not. It never returns an error +// that a caller could mistake for the worker's answer: a refused or failed +// re-broadcast is logged and the RPC continues, because the RPC's outcome is +// the worker's verdict about the work and a publish failure says nothing about +// it. +// +// raw is published as it arrived. It is a json.RawMessage and every Broadcaster +// encodes what it is given with json.Marshal, which returns a RawMessage +// verbatim, so the subscriber reads the bytes the worker wrote rather than a +// re-encoding of this frontend's idea of them. +func (r *Rebroadcaster) Handle(nodeType, subject string, raw json.RawMessage) bool { + if !MayBroadcast(nodeType, subject) { + // Warn rather than Debug: this is a worker asking for something it has + // no business on, which is the shape of a compromised or mismatched + // worker and is worth seeing without turning logging up. + xlog.Warn("refusing a worker's re-broadcast request", + "nodeType", nodeType, "subject", subject) + return false + } + if r == nil || r.bus == nil { + xlog.Debug("no broadcaster to re-broadcast a worker's progress line on", + "nodeType", nodeType, "subject", subject) + return false + } + if err := r.bus.Publish(subject, raw); err != nil { + xlog.Warn("a worker's re-broadcast could not be published", + "nodeType", nodeType, "subject", subject, "error", err) + return false + } + return true +} diff --git a/core/services/nodes/rebroadcast_test.go b/core/services/nodes/rebroadcast_test.go new file mode 100644 index 000000000000..6f01c8ac1c1b --- /dev/null +++ b/core/services/nodes/rebroadcast_test.go @@ -0,0 +1,153 @@ +package nodes + +import ( + "encoding/json" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// failingBus is a Broadcaster whose Publish always fails, for the one thing +// Handle promises about a carrier that is unhappy: the failure stays here. +type failingBus struct { + testutil.FakeBus + err error +} + +func (b *failingBus) Publish(string, any) error { return b.err } + +var _ = Describe("worker re-broadcast authorization", func() { + Describe("MayBroadcast", func() { + DescribeTable("answers for the node type it was asked about", + func(nodeType, subject string, allowed bool) { + Expect(MayBroadcast(nodeType, subject)).To(Equal(allowed)) + }, + Entry("agent, job progress", NodeTypeAgent, "jobs.j1.progress", true), + Entry("agent, job result", NodeTypeAgent, "jobs.j1.result", true), + Entry("agent, agent events", NodeTypeAgent, "agent.a1.events.status", true), + // The subjects an agent worker is NOT allowed. Every entry added to + // the table above owes this list a line, which is the only thing + // keeping the allow list from growing into "everything an agent + // happened to publish once". + Entry("agent, cache invalidation", NodeTypeAgent, "cache.invalidate.models", false), + Entry("agent, node registration", NodeTypeAgent, "node.register", false), + Entry("agent, model load", NodeTypeAgent, "model.load.n1", false), + Entry("agent, a job subject one token too deep", NodeTypeAgent, "jobs.j1.progress.extra", false), + Entry("agent, a job subject one token too shallow", NodeTypeAgent, "jobs.progress", false), + // A backend worker asks for no broadcasts, including the ones an + // agent worker is allowed: the table is per node type and not a + // single global list with a type-shaped comment on it. + Entry("backend, job progress", NodeTypeBackend, "jobs.j1.progress", false), + Entry("backend, job result", NodeTypeBackend, "jobs.j1.result", false), + Entry("backend, agent events", NodeTypeBackend, "agent.a1.events.status", false), + // A node type nothing has heard of is denied rather than defaulted. + Entry("unknown type, job progress", "router", "jobs.j1.progress", false), + Entry("unknown type, agent events", "router", "agent.a1.events.status", false), + Entry("empty type, job progress", "", "jobs.j1.progress", false), + ) + + It("refuses a line that names no subject at all", func() { + // A progress line with no subject asked for no broadcast. It must + // not become one because some filter matched the empty string. + Expect(MayBroadcast(NodeTypeAgent, "")).To(BeFalse()) + Expect(MayBroadcast(NodeTypeBackend, "")).To(BeFalse()) + }) + + It("refuses the tail wildcard, which the shared matcher does not implement", func() { + // SubjectMatches is the one definition of matching in the tree and + // it makes a '>' filter match nothing. Asserting it here is what + // stops someone writing "agent.>" into the allow list and believing + // they widened it, when what they did was narrow it to nothing. + Expect(messaging.SubjectMatches("agent.>", "agent.a1.events.status")).To(BeFalse()) + }) + + DescribeTable("reads an EMPTY allow list as denying everything, which is the inversion of the NATS list this replaces", + // NATS read an empty allow list as NO RESTRICTION, which is why the + // permissions it replaced had to spell the backend worker's list as + // {"_INBOX.>"} rather than leave it empty. This table is the + // opposite, and this spec is what documents it: the assumption a + // reviewer carries over from the deleted code is exactly wrong. + func(subject string) { + emptied := map[string][]string{ + NodeTypeAgent: {}, + NodeTypeBackend: {}, + } + Expect(mayBroadcastIn(emptied, NodeTypeAgent, subject)).To(BeFalse()) + // The same subject against the real table, so the case cannot + // pass by naming something that was never allowed. + Expect(MayBroadcast(NodeTypeAgent, subject)).To(BeTrue()) + }, + Entry("job progress", "jobs.j1.progress"), + Entry("job result", "jobs.j1.result"), + Entry("agent events", "agent.a1.events.status"), + ) + + It("reads a MISSING node type the same way as an empty list", func() { + absent := map[string][]string{NodeTypeBackend: {}} + Expect(mayBroadcastIn(absent, NodeTypeAgent, "jobs.j1.progress")).To(BeFalse()) + }) + }) + + Describe("Rebroadcaster.Handle", func() { + var ( + bus *testutil.FakeBus + rb *Rebroadcaster + ) + + BeforeEach(func() { + bus = testutil.NewFakeBus() + rb = NewRebroadcaster(bus) + }) + + It("publishes an allowed subject and hands the subscriber the worker's own bytes", func() { + delivered := make(chan []byte, 1) + _, err := bus.Subscribe("agent.*.events.*", func(payload []byte) { delivered <- payload }) + Expect(err).ToNot(HaveOccurred()) + + Expect(rb.Handle(NodeTypeAgent, "agent.a1.events.status", + json.RawMessage(`{"state":"thinking"}`))).To(BeTrue()) + + Expect(bus.PublishCount("agent.a1.events.status")).To(Equal(1)) + // Verbatim, not re-encoded: a subscriber decodes the worker's DTO + // and a frontend that re-marshalled its own idea of the line would + // hand it a different shape. + Eventually(delivered).Should(Receive(MatchJSON(`{"state":"thinking"}`))) + }) + + It("refuses a subject the agent worker has no business on, and publishes nothing", func() { + Expect(rb.Handle(NodeTypeAgent, "cache.invalidate.models", + json.RawMessage(`{"model":"m1"}`))).To(BeFalse()) + Expect(bus.PublishCount("cache.invalidate.models")).To(Equal(0)) + }) + + It("refuses a BACKEND worker a subject an agent worker would be allowed", func() { + Expect(rb.Handle(NodeTypeBackend, "jobs.j1.progress", + json.RawMessage(`{"percentage":10}`))).To(BeFalse()) + Expect(bus.PublishCount("jobs.j1.progress")).To(Equal(0)) + }) + + It("refuses a node type nothing has heard of", func() { + Expect(rb.Handle("router", "jobs.j1.progress", + json.RawMessage(`{"percentage":10}`))).To(BeFalse()) + Expect(bus.PublishCount("jobs.j1.progress")).To(Equal(0)) + }) + + It("reports a carrier that could not publish as a refusal, not as anything the worker said", func() { + // Handle's return type is the whole point: there is no error here + // for a caller to confuse with the RPC's outcome. A publish that + // failed says nothing about whether the work succeeded. + rb = NewRebroadcaster(&failingBus{err: errors.New("carrier is unhappy")}) + Expect(rb.Handle(NodeTypeAgent, "jobs.j1.progress", + json.RawMessage(`{"percentage":10}`))).To(BeFalse()) + }) + + It("refuses rather than panicking when it holds no broadcaster", func() { + Expect(NewRebroadcaster(nil).Handle(NodeTypeAgent, "jobs.j1.progress", + json.RawMessage(`{"percentage":10}`))).To(BeFalse()) + }) + }) +}) diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index 464d80ddd623..bcfca1a9dfb6 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -2,6 +2,7 @@ package nodes import ( "context" + "encoding/json" "errors" "fmt" "time" @@ -190,6 +191,45 @@ func (a *RemoteUnloaderAdapter) UnloadRemoteModelContext(ctx context.Context, mo return unloadErr } +// installProgressBridge adapts an install or upgrade's progress sink to +// CallStreaming's line callback, which carries a subject the client does not +// interpret. +// +// It makes two decisions a caller of InstallBackend must not have to make. +// +// A line naming a SUBJECT is a re-broadcast request and not install progress, +// so it is dropped here rather than delivered as a tick. backend.install and +// backend.upgrade are a BACKEND worker's verbs, and MayBroadcast denies a +// backend worker every subject, so this path has nothing to publish and no +// broadcaster to publish it on. Delivering it to onProgress instead would put a +// worker's arbitrary JSON through a decode into an install-progress event and +// report whatever fell out as the state of a download. +// +// A line this frontend cannot decode costs a tick and never the operation, +// because progress is transient by contract while the reply is the worker's +// verdict. +// +// It returns nil for a nil sink so CallStreaming keeps its "no callback, no +// work" path, rather than a non-nil closure wrapping a nil function. +func installProgressBridge(nodeID, path string, onProgress func(messaging.BackendInstallProgressEvent)) func(string, json.RawMessage) { + if onProgress == nil { + return nil + } + return func(subject string, raw json.RawMessage) { + if subject != "" { + xlog.Warn("refusing a re-broadcast request on a backend control stream", + "node", nodeID, "path", path, "subject", subject) + return + } + var ev messaging.BackendInstallProgressEvent + if err := json.Unmarshal(raw, &ev); err != nil { + xlog.Debug("unreadable control progress line", "node", nodeID, "path", path, "error", err) + return + } + onProgress(ev) + } +} + // InstallBackend asks a worker node to install a backend and start its process. // Idempotent on the worker: if the (modelID, replica) process is already // running, the worker short-circuits and returns its address; if the binary @@ -228,7 +268,7 @@ func (a *RemoteUnloaderAdapter) InstallBackend( Alias: alias, ReplicaIndex: int32(replicaIndex), OpID: opID, - }, &reply, onProgress) + }, &reply, installProgressBridge(nodeID, workerctl.PathBackendInstall, onProgress)) if err != nil { if isRequestTimeout(err) { return nil, fmt.Errorf("%w (nodeID=%s backend=%s): %v", @@ -263,7 +303,7 @@ func (a *RemoteUnloaderAdapter) UpgradeBackend(nodeID, backendType, galleriesJSO Alias: alias, ReplicaIndex: int32(replicaIndex), OpID: opID, - }, &reply, onProgress) + }, &reply, installProgressBridge(nodeID, workerctl.PathBackendUpgrade, onProgress)) if err != nil { if isRequestTimeout(err) { return nil, fmt.Errorf("%w (nodeID=%s backend=%s): %v", @@ -296,7 +336,7 @@ func (a *RemoteUnloaderAdapter) installWithForceFallback(nodeID, backendType, ga ReplicaIndex: int32(replicaIndex), Force: true, OpID: opID, - }, &reply, onProgress) + }, &reply, installProgressBridge(nodeID, workerctl.PathBackendInstall, onProgress)) if err != nil { if isRequestTimeout(err) { return nil, fmt.Errorf("%w (nodeID=%s backend=%s): %v", diff --git a/core/services/nodes/unloader_test.go b/core/services/nodes/unloader_test.go index 9cddf6546395..e5f6fb7bba57 100644 --- a/core/services/nodes/unloader_test.go +++ b/core/services/nodes/unloader_test.go @@ -669,4 +669,104 @@ var _ = Describe("RemoteUnloaderAdapter install progress streaming", func() { Expect(err).ToNot(HaveOccurred()) Expect(reply.Success).To(BeTrue()) }) + + // The bridge from a control STREAM LINE to an install-progress event is one + // definition used at three call sites: InstallBackend, UpgradeBackend and + // installWithForceFallback. Each is exercised separately below, because a + // site that stopped calling it would leave the other two green. + DescribeTable("delivers a line that names no broadcast to the caller's install-progress sink, on every streaming verb", + func(path string, call func(*RemoteUnloaderAdapter, func(messaging.BackendInstallProgressEvent)) error) { + workers := newScriptedControlWorkers() + workers.scriptReply(controlKey("n1", path), messaging.BackendInstallReply{Success: true}) + workers.scriptReply(controlKey("n1", workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: true}) + workers.scriptProgress(controlKey("n1", path), []messaging.BackendInstallProgressEvent{ + {OpID: "op-abc", NodeID: "n1", Backend: "vllm", Percentage: 25}, + }) + + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), time.Second, time.Second) + var received []messaging.BackendInstallProgressEvent + Expect(call(adapter, func(ev messaging.BackendInstallProgressEvent) { + received = append(received, ev) + })).To(Succeed()) + Expect(received).To(HaveLen(1)) + Expect(received[0].Percentage).To(Equal(float64(25))) + }, + Entry("backend.install", workerctl.PathBackendInstall, + func(a *RemoteUnloaderAdapter, cb func(messaging.BackendInstallProgressEvent)) error { + _, err := a.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "op-abc", cb) + return err + }), + Entry("backend.upgrade", workerctl.PathBackendUpgrade, + func(a *RemoteUnloaderAdapter, cb func(messaging.BackendInstallProgressEvent)) error { + _, err := a.UpgradeBackend("n1", "vllm", "[]", "", "", "", 0, "op-abc", cb) + return err + }), + Entry("the legacy force-install fallback", workerctl.PathBackendInstall, + func(a *RemoteUnloaderAdapter, cb func(messaging.BackendInstallProgressEvent)) error { + _, err := a.installWithForceFallback("n1", "vllm", "[]", "", "", "", 0, "op-abc", cb) + return err + }), + ) + + DescribeTable("refuses a line that NAMES a broadcast rather than delivering it as install progress, on every streaming verb", + func(path string, call func(*RemoteUnloaderAdapter, func(messaging.BackendInstallProgressEvent)) error) { + // backend.install and backend.upgrade are a BACKEND worker's verbs, + // and MayBroadcast denies a backend worker every subject, so a line + // naming one is not something this path publishes. What it must + // also not do is deliver it: the payload is a broadcast body and + // decoding it into an install-progress event reports a percentage + // nobody sent. + workers := newScriptedControlWorkers() + workers.scriptReply(controlKey("n1", path), messaging.BackendInstallReply{Success: true}) + workers.scriptReply(controlKey("n1", workerctl.PathBackendUpgrade), messaging.BackendUpgradeReply{Success: true}) + workers.scriptRawProgress(controlKey("n1", path), []workerctl.Envelope{ + {Subject: "jobs.j1.progress", Progress: json.RawMessage(`{"percentage":99}`)}, + {Subject: "agent.a1.events.status", Progress: json.RawMessage(`{"percentage":98}`)}, + }) + + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), time.Second, time.Second) + var received []messaging.BackendInstallProgressEvent + Expect(call(adapter, func(ev messaging.BackendInstallProgressEvent) { + received = append(received, ev) + })).To(Succeed()) + // The RPC still returned the worker's reply: a refused re-broadcast + // costs a line and never the operation. + Expect(received).To(BeEmpty()) + }, + Entry("backend.install", workerctl.PathBackendInstall, + func(a *RemoteUnloaderAdapter, cb func(messaging.BackendInstallProgressEvent)) error { + _, err := a.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "op-abc", cb) + return err + }), + Entry("backend.upgrade", workerctl.PathBackendUpgrade, + func(a *RemoteUnloaderAdapter, cb func(messaging.BackendInstallProgressEvent)) error { + _, err := a.UpgradeBackend("n1", "vllm", "[]", "", "", "", 0, "op-abc", cb) + return err + }), + Entry("the legacy force-install fallback", workerctl.PathBackendInstall, + func(a *RemoteUnloaderAdapter, cb func(messaging.BackendInstallProgressEvent)) error { + _, err := a.installWithForceFallback("n1", "vllm", "[]", "", "", "", 0, "op-abc", cb) + return err + }), + ) + + It("keeps going past a progress line it cannot read, since progress is transient", func() { + // The rule the control client used to carry. It moved to the bridge + // with the decode, and it has to be pinned where it now lives: a line + // this frontend cannot read costs a tick and never the operation. + workers := newScriptedControlWorkers() + workers.scriptReply(controlKey("n1", workerctl.PathBackendInstall), messaging.BackendInstallReply{Success: true}) + workers.scriptRawProgress(controlKey("n1", workerctl.PathBackendInstall), []workerctl.Envelope{ + {Progress: json.RawMessage(`{"percentage":"not a number"}`)}, + {Progress: json.RawMessage(`{"percentage":70}`)}, + }) + + adapter := NewRemoteUnloaderAdapter(nil, workers.controlClient(), time.Second, time.Second) + var seen []float64 + reply, err := adapter.InstallBackend("n1", "vllm", "", "[]", "", "", "", 0, "op-abc", + func(ev messaging.BackendInstallProgressEvent) { seen = append(seen, ev.Percentage) }) + Expect(err).ToNot(HaveOccurred()) + Expect(seen).To(Equal([]float64{70})) + Expect(reply.Success).To(BeTrue()) + }) }) diff --git a/core/services/worker/control_client_roundtrip_test.go b/core/services/worker/control_client_roundtrip_test.go index 52046f320f1a..aae8d8498ed7 100644 --- a/core/services/worker/control_client_roundtrip_test.go +++ b/core/services/worker/control_client_roundtrip_test.go @@ -2,6 +2,7 @@ package worker import ( "context" + "encoding/json" "errors" "net" "os" @@ -96,12 +97,23 @@ var _ = Describe("the frontend's control client against the real worker", func() } var seen []float64 + var subjects []string var reply messaging.BackendInstallReply err := client.CallStreaming(context.Background(), nodeID, workerctl.PathBackendInstall, messaging.BackendInstallRequest{Backend: "mock", OpID: "op-1"}, &reply, - func(ev messaging.BackendInstallProgressEvent) { seen = append(seen, ev.Percentage) }) + func(subject string, raw json.RawMessage) { + var ev messaging.BackendInstallProgressEvent + Expect(json.Unmarshal(raw, &ev)).To(Succeed()) + subjects = append(subjects, subject) + seen = append(seen, ev.Percentage) + }) Expect(err).NotTo(HaveOccurred()) Expect(seen).To(Equal([]float64{50, 100})) + // A BACKEND worker's install progress names no broadcast, over a real + // tunnel and a real NDJSON body rather than in a double's imagination. + // It is the property MayBroadcast would refuse anyway, asserted here on + // the bytes the worker actually wrote. + Expect(subjects).To(Equal([]string{"", ""})) Expect(reply.Success).To(BeTrue()) Expect(reply.WorkerLocalAddress).To(Equal("127.0.0.1:41234")) }) diff --git a/core/services/workerctl/paths.go b/core/services/workerctl/paths.go index 7d31cd9283b2..fbc49ae12400 100644 --- a/core/services/workerctl/paths.go +++ b/core/services/workerctl/paths.go @@ -125,12 +125,12 @@ func AllPaths() []string { // Envelope is one line of a streaming control response. // -// Exactly one of the two is set. Zero or more Progress lines are followed by -// exactly ONE Reply line, and the Reply line is the last thing on the body. -// That ordering is the contract: it is what lets the frontend stop reading, and -// it is what replaces the subscribe-before-request dance the NATS carrier -// needed, since progress and reply now share one response and nothing can -// arrive before the caller is listening. +// Exactly one of Progress and Reply is set. Zero or more Progress lines are +// followed by exactly ONE Reply line, and the Reply line is the last thing on +// the body. That ordering is the contract: it is what lets the frontend stop +// reading, and it is what replaces the subscribe-before-request dance the NATS +// carrier needed, since progress and reply now share one response and nothing +// can arrive before the caller is listening. // // Progress carrying the reply's own bytes is also why the 8000-byte // notification cap that bounded the NATS progress subject has no analogue here: @@ -138,6 +138,20 @@ func AllPaths() []string { type Envelope struct { Progress json.RawMessage `json:"progress,omitempty"` Reply json.RawMessage `json:"reply,omitempty"` + + // Subject names the broadcast this progress line asks the frontend reading + // the stream to make on its behalf. Empty means the line is for this caller + // alone, which is what every pre-existing progress line is. + // + // It is a REQUEST and not an instruction: the frontend checks it against an + // allow list derived from the node's type before publishing anything. A + // worker naming a subject it has no business on is refused and logged, and + // the stream continues. + // + // It qualifies a Progress line and never a Reply line. A reply is the + // worker's verdict about the work, and there is no version of "publish my + // verdict for me" that this control plane has to carry. + Subject string `json:"subject,omitempty"` } // ContentTypeStream is the media type of a streaming control response. diff --git a/core/services/workerctl/paths_test.go b/core/services/workerctl/paths_test.go index d912354cc10f..0bfc31ea12d2 100644 --- a/core/services/workerctl/paths_test.go +++ b/core/services/workerctl/paths_test.go @@ -124,6 +124,37 @@ var _ = Describe("control plane paths on the wire", func() { Expect(string(b)).To(Equal(`{"progress":{"percentage":50}}`)) }) + It("spells the re-broadcast key on the wire as \"subject\"", func() { + // Hand-written literal, like the paths above and for the same reason: a + // worker and a frontend built from different commits read each other's + // lines, and a renamed key is a re-broadcast request that silently + // becomes a private progress tick. Deriving the expectation from the + // struct tag would pin nothing. + b, err := json.Marshal(workerctl.Envelope{ + Subject: "agent.a1.events.status", + Progress: json.RawMessage(`{"tick":1}`), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(string(b)).To(Equal(`{"progress":{"tick":1},"subject":"agent.a1.events.status"}`)) + }) + + It("omits the subject key entirely when a progress line names no broadcast", func() { + // omitempty is the backward-compatibility half: every pre-existing + // progress line has no subject, and an older frontend reading a line + // that carried an empty subject key would be reading a field it does + // not know about on every tick. + b, err := json.Marshal(workerctl.Envelope{Progress: json.RawMessage(`{"tick":1}`)}) + Expect(err).NotTo(HaveOccurred()) + Expect(string(b)).NotTo(ContainSubstring("subject")) + }) + + It("reads a subject back off the wire, which is what the frontend decides on", func() { + var env workerctl.Envelope + Expect(json.Unmarshal([]byte(`{"progress":{"tick":1},"subject":"jobs.j1.progress"}`), &env)).To(Succeed()) + Expect(env.Subject).To(Equal("jobs.j1.progress")) + Expect(string(env.Progress)).To(Equal(`{"tick":1}`)) + }) + It("names the streaming media type", func() { Expect(workerctl.ContentTypeStream).To(Equal("application/x-ndjson")) }) From dc0c50fcaba4b32f97b43b5c52d52eb2fc30447b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 20:22:34 +0000 Subject: [PATCH 090/153] feat(distributed): dispatch queued work as a claim queue The three NATS queue groups jobs.new, jobs.mcp-ci.new and agent.execute are gone. Dispatching work is now a row in a work_claims table, taken by one frontend replica with SELECT ... FOR UPDATE SKIP LOCKED and driven on an agent worker as a streaming control RPC over that worker's tunnel. Exactly-one delivery among competing consumers is a database problem, not a broker feature. An agent worker has no database, so it never claims; it executes what the claiming replica hands it. A claim must not outlive the replica that took it. The reap releases a claim whose owner is no longer a live replica in the instances table, on the database clock, and never asks how long the claim has been held. A job that legitimately runs for an hour on a heartbeating replica is left alone, while a claim whose owner stopped heartbeating becomes claimable again within one liveness window. A replica with no advertised address has no instances row at all, so it refuses to claim rather than have its work reaped out from under it mid-run. The settle rule is stated once, in settleClaim, and every exit path calls it. A transport failure releases the claim and never completes or discards it; only a decoded reply line completes it. That line is deliberately not cluster.IsWorkerAnswer, which accepts the stream refusals a worker's tunnel writes before any request body reaches its control server: completing on those would discard work that never ran. The terminal line is persisted before the claim is completed, so a store that refuses leaves the claim standing rather than leaving the job running for ever. That is the dropped-result defect fixed structurally rather than by retry. This also surfaces a pre-existing gap rather than causing one: no worker has ever served plain task jobs, and publishing them into an empty queue group left them running with no trace. Such a claim is now failed with a reason. Removes QueueWorkers, --agent-subject and --agent-queue, and narrows an agent worker's minted JWT by agent.execute and jobs.mcp-ci.new. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/agent_control_wiring_test.go | 33 +- core/application/dispatch_loop_wiring.go | 70 +++ core/application/dispatch_loop_wiring_test.go | 92 +++ core/application/distributed.go | 17 +- core/cli/agent_worker.go | 205 ++++--- core/cli/agent_worker_test.go | 47 +- core/services/agentpool/agent_pool.go | 28 +- .../agentpool/dispatch_chat_internal_test.go | 76 +++ core/services/agents/dispatcher.go | 222 +++---- core/services/agents/events.go | 47 +- core/services/agents/scheduler.go | 24 +- core/services/agents/scheduler_test.go | 97 +-- core/services/agentworker/server_test.go | 9 +- core/services/cluster/instance.go | 15 + core/services/jobs/claim.go | 276 +++++++++ core/services/jobs/claim_test.go | 502 ++++++++++++++++ core/services/jobs/dispatch_loop.go | 454 ++++++++++++++ core/services/jobs/dispatch_loop_test.go | 562 ++++++++++++++++++ core/services/jobs/dispatcher.go | 246 ++------ core/services/jobs/dispatcher_test.go | 69 ++- core/services/jobs/store.go | 7 + core/services/messaging/subjects.go | 20 +- docs/content/features/distributed-mode.md | 18 +- docs/content/reference/cli-reference.md | 2 +- pkg/natsauth/mint_test.go | 2 +- pkg/natsauth/permissions.go | 12 +- pkg/natsauth/permissions_coverage_test.go | 29 +- .../distributed/agent_native_executor_test.go | 216 +++---- tests/e2e/distributed/job_dispatch_test.go | 158 +++-- .../e2e/distributed/job_distribution_test.go | 338 ++++------- tests/e2e/distributed/mcp_ci_job_test.go | 15 +- tests/e2e/distributed/nats_jwt_test.go | 51 +- tests/e2e/distributed/sse_routes_test.go | 2 +- 33 files changed, 3012 insertions(+), 949 deletions(-) create mode 100644 core/application/dispatch_loop_wiring.go create mode 100644 core/application/dispatch_loop_wiring_test.go create mode 100644 core/services/agentpool/dispatch_chat_internal_test.go create mode 100644 core/services/jobs/claim.go create mode 100644 core/services/jobs/claim_test.go create mode 100644 core/services/jobs/dispatch_loop.go create mode 100644 core/services/jobs/dispatch_loop_test.go diff --git a/core/application/agent_control_wiring_test.go b/core/application/agent_control_wiring_test.go index d8c1a02268d9..3fb34022986c 100644 --- a/core/application/agent_control_wiring_test.go +++ b/core/application/agent_control_wiring_test.go @@ -16,14 +16,37 @@ import ( ) // recordingConnections captures the owner id the selection was made with. It is -// how this spec sees the one argument whose loss has no other symptom. -type recordingConnections struct{ owners []string } +// how these specs see the one argument whose loss has no other symptom. +// +// The channel is what lets a spec observe a selection made on ANOTHER +// goroutine, which is what the dispatch loop's own tick is. A slice read from +// the spec goroutine would be a data race, and waiting on it would be a sleep. +type recordingConnections struct { + owners []string + seen chan string +} func (r *recordingConnections) ConnectedAmong(_ context.Context, _ []string, owner string) ([]string, []string, error) { r.owners = append(r.owners, owner) + if r.seen != nil { + select { + case r.seen <- owner: + default: + } + } return nil, nil, nil } +// newRecordingConnections returns a reader whose channel is ready BEFORE any +// loop can be started against it. Creating it lazily from the spec goroutine +// would race the loop's own goroutine reading it. +func newRecordingConnections() *recordingConnections { + return &recordingConnections{seen: make(chan string, 8)} +} + +// calledBy delivers the owner id of each selection this reader answers. +func (r *recordingConnections) calledBy() chan string { return r.seen } + // The wiring that connects MCP to the agent workers, guarded the way the // absence wiring is and for the same reason: initDistributed opens a database // and a bus, so no unit spec reaches the construction literal, and one of these @@ -54,7 +77,7 @@ var _ = Describe("building the frontend's agent control client", func() { Name: "agent-1", NodeType: nodes.NodeTypeAgent, Address: "a:50051", }, true)).To(Succeed()) - conns := &recordingConnections{} + conns := newRecordingConnections() client, err := newAgentControl( config.DistributedConfig{InstanceID: "replica-7"}, registry, conns, nodes.NewControlClient(nil, "token")) @@ -67,7 +90,7 @@ var _ = Describe("building the frontend's agent control client", func() { }) It("refuses to build with no instance id", func() { - _, err := newAgentControl(config.DistributedConfig{}, registry, &recordingConnections{}, + _, err := newAgentControl(config.DistributedConfig{}, registry, newRecordingConnections(), nodes.NewControlClient(nil, "token")) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("instance id")) @@ -82,7 +105,7 @@ var _ = Describe("building the frontend's agent control client", func() { It("refuses to build with no control transport", func() { _, err := newAgentControl(config.DistributedConfig{InstanceID: "replica-7"}, registry, - &recordingConnections{}, nil) + newRecordingConnections(), nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("control transport")) }) diff --git a/core/application/dispatch_loop_wiring.go b/core/application/dispatch_loop_wiring.go new file mode 100644 index 000000000000..f797c616c798 --- /dev/null +++ b/core/application/dispatch_loop_wiring.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: MIT + +package application + +import ( + "context" + "fmt" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/jobs" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" + "gorm.io/gorm" +) + +// startJobDispatchLoop builds AND STARTS the loop that takes queued work off the +// job store and drives it on an agent worker. +// +// One function rather than a construction here and a Start somewhere else, and +// that is the point rather than tidiness. A loop that is built and never +// started is a replica that writes claim rows and takes none, so every job in +// the deployment is accepted and none is ever run, and nothing anywhere says +// so. As a separate statement in the start-up path that line's loss has no +// symptom and no spec reaches it: initDistributed opens a database and a bus. +// Fused here, the loop cannot exist without running. +// +// The rest is a named function for the reason newAgentControl is one: two of +// these arguments are silent when they are wrong. +// +// The BROADCASTER is the one worth naming. It is what re-publishes the progress +// and result lines a worker asks for, and it is checked against the allow list +// for that worker's node type. A loop built without one dispatches work +// perfectly well and every SSE stream in the deployment goes quiet: the job +// runs, the answer is persisted, and the user watching it sees nothing until +// they reload. That is a whole feature lost to a nil field, with no error +// anywhere, so it is refused here. +// +// The SELECTOR is built here rather than borrowed from newAgentControl, and +// deliberately: nodes.AgentSelector holds no per-caller state, and sharing one +// would couple the dispatch loop's lifetime to MCP's for nothing. +func startJobDispatchLoop(ctx context.Context, cfg config.DistributedConfig, db *gorm.DB, store *jobs.JobStore, + registry *nodes.NodeRegistry, conns nodes.AgentConnectionReader, + control *nodes.ControlClient, bus messaging.Broadcaster) (*jobs.DispatchLoop, error) { + if cfg.InstanceID == "" { + return nil, fmt.Errorf("the job dispatch loop was built with no instance id: its claims could not be told from ones a dead replica left") + } + if registry == nil || conns == nil { + return nil, fmt.Errorf("the job dispatch loop was built with no way to find a connected agent worker") + } + if bus == nil { + return nil, fmt.Errorf("the job dispatch loop was built with no broadcaster: every job would run with its progress and its result reaching no SSE stream in the deployment") + } + loop, err := jobs.NewDispatchLoop(jobs.DispatchConfig{ + DB: db, + Owner: cfg.InstanceID, + Selector: nodes.NewAgentSelector(registry, conns, cfg.InstanceID), + Control: control, + // The allow list lives in nodes and is keyed on the worker's node type; + // nothing here decides what a worker may broadcast on. + Broadcast: nodes.NewRebroadcaster(bus), + Store: store, + }) + if err != nil { + return nil, err + } + if err := loop.Start(ctx); err != nil { + return nil, err + } + return loop, nil +} diff --git a/core/application/dispatch_loop_wiring_test.go b/core/application/dispatch_loop_wiring_test.go new file mode 100644 index 000000000000..a42bc647f78a --- /dev/null +++ b/core/application/dispatch_loop_wiring_test.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: MIT + +package application + +import ( + "context" + "encoding/json" + "runtime" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/jobs" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// The wiring that turns queued claim rows into work on an agent worker. +// +// Guarded the way newAgentControl is and for the same reason: initDistributed +// opens a database and a bus, so no unit spec reaches the construction literal, +// and two of these arguments are silent when they are wrong. +var _ = Describe("building the job dispatch loop", func() { + var registry *nodes.NodeRegistry + var conns *recordingConnections + var ctx context.Context + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + ctx = context.Background() + var err error + registry, err = nodes.NewNodeRegistry(testutil.SetupTestDB()) + Expect(err).ToNot(HaveOccurred()) + conns = newRecordingConnections() + }) + + // The silent one. A loop with no broadcaster dispatches work perfectly + // well: the job runs, the answer is persisted, and every SSE stream in the + // deployment goes quiet, with no error anywhere. + It("refuses to build with no broadcaster to re-publish a worker's progress on", func() { + _, err := startJobDispatchLoop(ctx, config.DistributedConfig{InstanceID: "replica-7"}, + testutil.SetupTestDB(), nil, registry, conns, nodes.NewControlClient(nil, "token"), nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("broadcaster")) + }) + + It("refuses to build with no instance id", func() { + _, err := startJobDispatchLoop(ctx, config.DistributedConfig{}, + testutil.SetupTestDB(), nil, registry, conns, nodes.NewControlClient(nil, "token"), + testutil.NewFakeBus()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("instance id")) + }) + + It("refuses to build with nothing to read connections through", func() { + _, err := startJobDispatchLoop(ctx, config.DistributedConfig{InstanceID: "replica-7"}, + testutil.SetupTestDB(), nil, registry, nil, nodes.NewControlClient(nil, "token"), + testutil.NewFakeBus()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("connected agent worker")) + }) + + // Driven through the loop's OWN tick, not through a call this spec makes, + // and that is deliberate. Two things are pinned here at once: that this + // replica's id reaches the SELECTION (a loop built with the wrong id relays + // every RPC through a peer and says so nowhere), and that building the loop + // STARTED it (a loop that is never started writes claim rows and takes + // none, so every job in the deployment is accepted and never run). + It("starts on construction, and selects as THIS replica", func() { + db := testutil.SetupTestDB() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + Expect(jobs.MigrateClaims(ctx, db)).To(Succeed()) + // Registered, because a replica that is not in the instances table + // refuses to claim: its claims could not be told from a dead one's. + Expect(cluster.NewRegistry(db).Register(ctx, "replica-7", "127.0.0.1:8080", "v1")).To(Succeed()) + _, err := jobs.EnqueueClaim(ctx, db, jobs.ClaimKindAgentRun, json.RawMessage(`{}`)) + Expect(err).ToNot(HaveOccurred()) + + loop, err := startJobDispatchLoop(ctx, config.DistributedConfig{InstanceID: "replica-7"}, + db, nil, registry, conns, nodes.NewControlClient(nil, "token"), testutil.NewFakeBus()) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(loop.Stop) + + // Nothing prods it. The only thing that can make this happen is the + // loop's own goroutine. + Eventually(conns.calledBy, "20s").Should(Receive(Equal("replica-7"))) + }) +}) diff --git a/core/application/distributed.go b/core/application/distributed.go index 201450c08ecb..336153705dc4 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -91,6 +91,10 @@ type DistributedServices struct { // to come up without it rather than leaving the endpoints to discover it // one request at a time. AgentControl *nodes.AgentControlClient + // JobDispatch takes queued work off the job store and drives it on an agent + // worker over that worker's tunnel. It is what replaces the three NATS + // queue groups: dispatch is a claim, and a claim is a row and a lock. + JobDispatch *jobs.DispatchLoop shutdownOnce sync.Once } @@ -121,6 +125,9 @@ func (ds *DistributedServices) Shutdown() { if ds.Health != nil { ds.Health.Stop() } + if ds.JobDispatch != nil { + ds.JobDispatch.Stop() + } if ds.Dispatcher != nil { ds.Dispatcher.Stop() } @@ -412,7 +419,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade xlog.Info("Distributed job store initialized") // Initialize job dispatcher - dispatcher := jobs.NewDispatcher(jobStore, natsClient, authDB, cfg.Distributed.InstanceID, cfg.Distributed.JobWorkerConcurrency) + dispatcher := jobs.NewDispatcher(jobStore, natsClient, authDB, cfg.Distributed.InstanceID) // Initialize agent store agentStore, err := agents.NewAgentStore(authDB) @@ -467,6 +474,13 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade return nil, fmt.Errorf("wiring the agent control client: %w", err) } + // The consumer side of the claim queue, built and started in one act: see + // startJobDispatchLoop for why those are not two lines. + jobDispatch, err := startJobDispatchLoop(cfg.Context, cfg.Distributed, authDB, jobStore, registry, clusterRegistry, controlClient, natsClient) + if err != nil { + return nil, fmt.Errorf("wiring the job dispatch loop: %w", err) + } + // Create FileStager for distributed file transfer var fileStager nodes.FileStager if cfg.Distributed.StorageURL != "" { @@ -712,6 +726,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade WorkerDialer: workerDialer, BackendClients: backendClients, AgentControl: agentControl, + JobDispatch: jobDispatch, Bus: bus, } // Checked once, here, on the assembled struct. See requireBroadcastCarrier. diff --git a/core/cli/agent_worker.go b/core/cli/agent_worker.go index ac75011744ae..09f8c20bb2f1 100644 --- a/core/cli/agent_worker.go +++ b/core/cli/agent_worker.go @@ -27,10 +27,14 @@ import ( ) // AgentWorkerCMD starts a dedicated agent worker process for distributed mode. -// It registers with the frontend, subscribes to the NATS agent execution queue, -// and executes agent chats using cogito. The worker is a pure executor — it -// receives the full agent config and skills in the NATS job payload, so it -// does not need direct database access. +// It registers with the frontend and serves agent execution and MCP CI runs as +// STREAMING CONTROL VERBS on the tunnel it holds. The worker is a pure +// executor: it receives the full agent config and skills in the request body, +// so it does not need direct database access. +// +// It joins no queue group, and there is none left to join: a queue group only +// ever selected one consumer out of a set, the frontend makes that selection +// itself, and the work it hands over is a row it claimed on the job store. // // It also holds one tunnel to the frontend, so the frontend can reach its // control verbs by RPC without the worker opening an inbound port. No verb the @@ -55,10 +59,6 @@ type AgentWorkerCMD struct { APIURL string `env:"LOCALAI_API_URL" help:"LocalAI API URL for inference (auto-derived from RegisterTo if not set)" group:"api"` APIToken string `env:"LOCALAI_API_TOKEN" help:"API token for LocalAI inference (auto-provisioned during registration if not set)" group:"api"` - // NATS subjects - Subject string `env:"LOCALAI_AGENT_SUBJECT" default:"agent.execute" help:"NATS subject for agent execution" group:"distributed"` - Queue string `env:"LOCALAI_AGENT_QUEUE" default:"agent-workers" help:"NATS queue group name" group:"distributed"` - NatsJWT string `env:"LOCALAI_NATS_JWT" help:"NATS user JWT override (defaults to nats_jwt from registration)" group:"distributed"` NatsUserSeed string `env:"LOCALAI_NATS_USER_SEED" help:"NATS user seed override (defaults to nats_user_seed from registration)" group:"distributed"` NatsServiceJWT string `env:"LOCALAI_NATS_SERVICE_JWT" help:"Fallback NATS service JWT when registration does not mint agent JWT" group:"distributed"` @@ -177,13 +177,30 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { } defer natsClient.Close() + // The executor and the event bridge the control plane serves, built BEFORE + // the tunnel because a verb mounted with a nil handler answers a 404, which + // a frontend reads as a worker too old to serve it. + // + // No ConfigProvider and no SkillStore: config and skills arrive in the + // request body, exactly as they arrived in the job payload before, because + // an agent worker still has no database. + eventBridge := agents.NewEventBridge(natsClient, nil, "agent-worker-"+nodeID) + executor := agents.NewWorkerExecutor(eventBridge, nil, apiURL, cmd.APIToken) + + mcpCIJobTimeout, err := time.ParseDuration(cmd.MCPCIJobTimeout) + if err != nil && cmd.MCPCIJobTimeout != "" { + xlog.Warn("invalid MCP CI job timeout, using default 10m", "input", cmd.MCPCIJobTimeout, "error", err) + } + mcpCIJobTimeout = cmp.Or(mcpCIJobTimeout, config.DefaultMCPCIJobTimeout) + // The tunnel, and the loopback control plane behind it. // - // This is now the ONLY way MCP tool execution and discovery reach this - // worker: their queue-group subjects are gone, because a queue group was - // only ever a way of SELECTING a worker and the frontend makes that - // selection itself (nodes.AgentSelector). The remaining verbs below still - // arrive on NATS and will until the tasks that move them land. + // It is now the ONLY way anything the frontend addresses to THIS worker + // arrives: MCP tool execution, MCP discovery, backend.stop, agent execution + // and MCP CI runs. Every one of their subjects is gone. The two queue + // groups went last, because a queue group was only ever a way of SELECTING + // a worker: the frontend makes that selection itself + // (nodes.AgentSelector) and hands over a claim it took off the job store. // // The worker opens no inbound port for any of it: it dials out and the // control plane rides the tunnel it holds. @@ -194,17 +211,13 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { // JWT refresh. // // It is started AFTER registration, which is what supplies both the node - // identity the dial names and the credential it presents, and BEFORE the - // remaining NATS subscriptions, so that a frontend that reaches this worker - // over the tunnel finds its verbs mounted rather than a 404 it would read - // as a version skew. backend.stop is among those verbs, and it is the only - // way a frontend states it now: there is no node subject left to publish on. + // identity the dial names and the credential it presents. agentCtl, err := agentworker.Start(shutdownCtx, agentworker.Options{ FrontendURL: cmd.RegisterTo, NodeID: nodeID, TunnelToken: credMgr.TunnelToken, ControlToken: cmd.RegistrationToken, - Handlers: agentWorkerControlHandlers(), + Handlers: agentWorkerControlHandlers(executor, apiURL, cmd.APIToken, mcpCIJobTimeout), }) if err != nil { return fmt.Errorf("starting the agent worker control plane: %w", err) @@ -215,48 +228,17 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { } }() - // Create event bridge for publishing results back via NATS - eventBridge := agents.NewEventBridge(natsClient, nil, "agent-worker-"+nodeID) - - // Start cancel listener + // The cancel listener is the ONE thing still on the bus here: a cancel is a + // broadcast to every replica and every worker, because the replica holding + // the run is not the one the cancel request lands on. cancelSub, err := eventBridge.StartCancelListener() if err != nil { xlog.Warn("Failed to start cancel listener", "error", err) } else { - defer cancelSub.Unsubscribe() - } - - // Create and start the NATS dispatcher. - // No ConfigProvider or SkillStore needed — config and skills arrive in the job payload. - dispatcher := agents.NewNATSDispatcher( - natsClient, - eventBridge, - nil, // no ConfigProvider: config comes in the enriched NATS payload - apiURL, cmd.APIToken, - cmd.Subject, cmd.Queue, - 0, // no concurrency limit (CLI worker) - ) - - if err := dispatcher.Start(shutdownCtx); err != nil { - return fmt.Errorf("starting dispatcher: %w", err) + defer func() { _ = cancelSub.Unsubscribe() }() } - // Subscribe to MCP CI job execution (load-balanced across agent workers). - // In distributed mode, MCP CI jobs are routed here because the frontend - // cannot create MCP sessions (e.g., stdio servers using docker). - mcpCIJobTimeout, err := time.ParseDuration(cmd.MCPCIJobTimeout) - if err != nil && cmd.MCPCIJobTimeout != "" { - xlog.Warn("invalid MCP CI job timeout, using default 10m", "input", cmd.MCPCIJobTimeout, "error", err) - } - mcpCIJobTimeout = cmp.Or(mcpCIJobTimeout, config.DefaultMCPCIJobTimeout) - - if _, err := natsClient.QueueSubscribe(messaging.SubjectMCPCIJobsNew, messaging.QueueWorkers, func(data []byte) { - handleMCPCIJob(shutdownCtx, data, apiURL, cmd.APIToken, natsClient, mcpCIJobTimeout) - }); err != nil { - return fmt.Errorf("subscribing to %s: %w", messaging.SubjectMCPCIJobsNew, err) - } - - xlog.Info("Agent worker ready, waiting for jobs", "subject", cmd.Subject, "queue", cmd.Queue) + xlog.Info("Agent worker ready, serving agent execution and MCP CI runs on its tunnel", "node", nodeID) // Wait for an OS signal or an internal fatal condition (e.g. NATS // credentials became unrenewable), so the worker restarts and re-acquires @@ -273,7 +255,6 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { xlog.Info("Shutting down agent worker") shutdownCancel() // stop heartbeat loop immediately - dispatcher.Stop() mcpTools.CloseAllMCPSessions() regClient.GracefulDeregister(nodeID) return runErr @@ -421,44 +402,51 @@ func encodeMCPReply(resp any) (json.RawMessage, error) { // handleMCPCIJob processes an MCP CI job on the agent worker. // The agent worker can create MCP sessions (has docker) and call the LocalAI API for inference. -func handleMCPCIJob(shutdownCtx context.Context, data []byte, apiURL, apiToken string, natsClient messaging.MessagingClient, jobTimeout time.Duration) { +// +// Everything it publishes now goes onto pub, which is the response body of the +// control verb rather than the bus. The subjects are unchanged, and they are +// still what the claiming replica checks against this worker's allow list +// before re-broadcasting, so an SSE stream open on any replica still sees the +// same events on the same subjects. +// +// It returns the terminal answer rather than only publishing it. That is the +// structural half of the fix: the claiming replica persists this before it +// releases the claim, so a job that finished on a worker cannot be left +// `running` because a result message went to a subject nobody was reading. +func handleMCPCIJob(ctx context.Context, data []byte, apiURL, apiToken string, pub messaging.Publisher, jobTimeout time.Duration) jobs.ClaimReply { var evt jobs.JobEvent if err := json.Unmarshal(data, &evt); err != nil { xlog.Error("Failed to unmarshal job event", "error", err) - return + return jobs.ClaimReply{Status: "failed", Error: "unreadable job event"} } job := evt.Job task := evt.Task if job == nil || task == nil { xlog.Error("MCP CI job missing enriched data", "jobID", evt.JobID) - publishJobResult(natsClient, evt.JobID, "failed", "", "job or task data missing from NATS event") - return + return mcpCIAnswer(pub, evt.JobID, "failed", "", "job or task data missing from NATS event") } modelCfg := evt.ModelConfig if modelCfg == nil { - publishJobResult(natsClient, evt.JobID, "failed", "", "model config missing from job event") - return + return mcpCIAnswer(pub, evt.JobID, "failed", "", "model config missing from job event") } xlog.Info("Processing MCP CI job", "jobID", evt.JobID, "taskID", evt.TaskID, "model", task.Model) // Publish running status - natsClient.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ + dropTrace(pub.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ JobID: evt.JobID, Status: "running", Message: "Job started on agent worker", - }) + }), evt.JobID) // Parse MCP config if modelCfg.MCP.Servers == "" && modelCfg.MCP.Stdio == "" { - publishJobResult(natsClient, evt.JobID, "failed", "", "no MCP servers configured for model") - return + return mcpCIAnswer(pub, evt.JobID, "failed", "", "no MCP servers configured for model") } remote, stdio, err := modelCfg.MCP.MCPConfigFromYAML() if err != nil { - publishJobResult(natsClient, evt.JobID, "failed", "", fmt.Sprintf("failed to parse MCP config: %v", err)) - return + return mcpCIAnswer(pub, evt.JobID, "failed", "", fmt.Sprintf("failed to parse MCP config: %v", err)) } // Create MCP sessions locally (agent worker has docker) @@ -468,8 +456,7 @@ func handleMCPCIJob(shutdownCtx context.Context, data []byte, apiURL, apiToken s if err != nil { errMsg = fmt.Sprintf("failed to create MCP sessions: %v", err) } - publishJobResult(natsClient, evt.JobID, "failed", "", errMsg) - return + return mcpCIAnswer(pub, evt.JobID, "failed", "", errMsg) } // Build prompt from template @@ -497,11 +484,11 @@ func handleMCPCIJob(shutdownCtx context.Context, data []byte, apiURL, apiToken s llm := clients.NewLocalAILLM(task.Model, apiToken, apiURL) // Build cogito options - ctx, cancel := context.WithTimeout(shutdownCtx, jobTimeout) + ctx, cancel := context.WithTimeout(ctx, jobTimeout) defer cancel() // Update job status to running in DB - publishJobStatus(natsClient, evt.JobID, "running", "") + publishJobStatus(pub, evt.JobID, "running", "") // Buffer stream tokens and flush as complete blocks var reasoningBuf, contentBuf strings.Builder @@ -509,15 +496,15 @@ func handleMCPCIJob(shutdownCtx context.Context, data []byte, apiURL, apiToken s flushStreamBuf := func() { if reasoningBuf.Len() > 0 { - natsClient.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ + dropTrace(pub.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ JobID: evt.JobID, TraceType: "reasoning", TraceContent: reasoningBuf.String(), - }) + }), evt.JobID) reasoningBuf.Reset() } if contentBuf.Len() > 0 { - natsClient.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ + dropTrace(pub.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ JobID: evt.JobID, TraceType: "content", TraceContent: contentBuf.String(), - }) + }), evt.JobID) contentBuf.Reset() } } @@ -528,15 +515,15 @@ func handleMCPCIJob(shutdownCtx context.Context, data []byte, apiURL, apiToken s cogito.WithMCPs(sessions...), cogito.WithStatusCallback(func(status string) { flushStreamBuf() - natsClient.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ + dropTrace(pub.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ JobID: evt.JobID, TraceType: "status", TraceContent: status, - }) + }), evt.JobID) }), cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) { flushStreamBuf() - natsClient.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ + dropTrace(pub.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ JobID: evt.JobID, TraceType: "tool_result", TraceContent: fmt.Sprintf("%s: %s", t.Name, t.Result), - }) + }), evt.JobID) }), cogito.WithStreamCallback(func(ev cogito.StreamEvent) { // Flush if stream type changed (e.g., reasoning → content) @@ -550,9 +537,9 @@ func handleMCPCIJob(shutdownCtx context.Context, data []byte, apiURL, apiToken s case cogito.StreamEventContent: contentBuf.WriteString(ev.Content) case cogito.StreamEventToolCall: - natsClient.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ + dropTrace(pub.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ JobID: evt.JobID, TraceType: "tool_call", TraceContent: fmt.Sprintf("%s(%s)", ev.ToolName, ev.ToolArgs), - }) + }), evt.JobID) } }), ) @@ -565,24 +552,42 @@ func handleMCPCIJob(shutdownCtx context.Context, data []byte, apiURL, apiToken s flushStreamBuf() // flush any remaining buffered tokens if err != nil { - publishJobResult(natsClient, evt.JobID, "failed", "", fmt.Sprintf("cogito execution failed: %v", err)) - return + return mcpCIAnswer(pub, evt.JobID, "failed", "", fmt.Sprintf("cogito execution failed: %v", err)) } result := "" if msg := f.LastMessage(); msg != nil { result = msg.Content } - publishJobResult(natsClient, evt.JobID, "completed", result, "") xlog.Info("MCP CI job completed", "jobID", evt.JobID, "resultLen", len(result)) + return mcpCIAnswer(pub, evt.JobID, "completed", result, "") } -func publishJobStatus(nc messaging.MessagingClient, jobID, status, message string) { - jobs.PublishJobProgress(nc, jobID, status, message) +// dropTrace logs a progress line that could not be written, and never returns +// it. A progress line is a NOTIFICATION about a run; a failure to write one says +// nothing about the run, and returning it would make the claiming replica read +// a finished job as a verb this worker could not serve. +func dropTrace(err error, jobID string) { + if err != nil { + xlog.Debug("An MCP CI progress line could not be written", "jobID", jobID, "error", err) + } +} + +// mcpCIAnswer is the ONE place an MCP CI run's terminal state is stated. +// +// It says it TWICE and on purpose, on two carriers with different jobs. The +// publish drives the SSE streams, on the same jobs..result subject it always +// used, re-broadcast by the claiming replica after its allow-list check. The +// returned reply is the answer to the control verb, which the claiming replica +// persists BEFORE it releases the claim; that is what makes a finished job +// impossible to leave `running`, where the publish alone could reach nobody. +func mcpCIAnswer(pub messaging.Publisher, jobID, status, result, errMsg string) jobs.ClaimReply { + jobs.PublishJobResult(pub, jobID, status, result, errMsg) + return jobs.ClaimReply{JobID: jobID, Status: status, Result: result, Error: errMsg} } -func publishJobResult(nc messaging.MessagingClient, jobID, status, result, errMsg string) { - jobs.PublishJobResult(nc, jobID, status, result, errMsg) +func publishJobStatus(pub messaging.Publisher, jobID, status, message string) { + jobs.PublishJobProgress(pub, jobID, status, message) } // agentWorkerControlHandlers is every verb this worker serves on the tunnel it @@ -594,12 +599,32 @@ func publishJobResult(nc messaging.MessagingClient, jobID, status, result, errMs // node subject backend.stop arrived on are all gone, so a field silently // dropped here is a 404 at runtime, which the frontend reads as a worker too // old to serve the verb. -func agentWorkerControlHandlers() agentworker.Config { +func agentWorkerControlHandlers(executor *agents.WorkerExecutor, apiURL, apiToken string, mcpCITimeout time.Duration) agentworker.Config { return agentworker.Config{ MCPTool: serveMCPToolRequest, MCPDiscovery: serveMCPDiscoveryRequest, // Drops the MCP sessions cached for a backend that went away, on the // path a backend worker serves by killing the process instead. BackendStop: dropMCPSessionsForBackend, + // The two verbs that replace the queue groups. Both STREAM: their + // progress, their agent events and their terminal answer all travel on + // the response body the claiming replica is already reading, which is + // what lets that replica persist the terminal line before it releases + // the claim. + AgentExecute: executor.Execute, + MCPCIRun: serveMCPCIRun(apiURL, apiToken, mcpCITimeout), + } +} + +// serveMCPCIRun answers workerctl.PathMCPCIRun. +// +// The handler's ctx is the REQUEST's, not this process's shutdown context, and +// that is the point: when the claiming replica goes away the response body dies +// with it, the run stops, and the claim is reaped for another replica to take. +// Bound to this worker's shutdown context instead, the run would keep going +// with nobody reading it. +func serveMCPCIRun(apiURL, apiToken string, jobTimeout time.Duration) agentworker.StreamHandler { + return func(ctx context.Context, raw json.RawMessage, pub messaging.Publisher) (json.RawMessage, error) { + return json.Marshal(handleMCPCIJob(ctx, raw, apiURL, apiToken, pub, jobTimeout)) } } diff --git a/core/cli/agent_worker_test.go b/core/cli/agent_worker_test.go index 680c49e1e3f4..f941caf44ca0 100644 --- a/core/cli/agent_worker_test.go +++ b/core/cli/agent_worker_test.go @@ -1,17 +1,21 @@ package cli import ( + "bufio" "context" "encoding/json" "net/http" "net/http/httptest" "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/mudler/LocalAI/core/services/agents" mcpRemote "github.com/mudler/LocalAI/core/services/mcp" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/testutil" "github.com/mudler/LocalAI/core/services/workerctl" ) @@ -80,7 +84,12 @@ var _ = Describe("The agent worker's control-plane wiring", func() { BeforeEach(func() { mux := http.NewServeMux() - agentWorkerControlHandlers().Register(mux) + // Built exactly as Run builds it, from an executor and the MCP CI + // timeout, so a field this function forgets to set is a 404 here. + executor := agents.NewWorkerExecutor( + agents.NewEventBridge(testutil.NewFakeBus(), nil, "agent-worker-spec"), + nil, "http://127.0.0.1:1", "token") + agentWorkerControlHandlers(executor, "http://127.0.0.1:1", "token", time.Second).Register(mux) srv := httptest.NewServer(mux) DeferCleanup(srv.Close) base = srv.URL @@ -97,5 +106,41 @@ var _ = Describe("The agent worker's control-plane wiring", func() { Entry("backend stop", workerctl.PathBackendStop), Entry("mcp tool execute", workerctl.PathMCPToolExecute), Entry("mcp discovery", workerctl.PathMCPDiscovery), + // The two verbs that replaced the queue groups. An unwired one answers + // a 404, which is EXACTLY what an older worker answers, so nothing else + // in the tree can tell the two apart and only this spec can. + Entry("agent execute", workerctl.PathAgentExecute), + Entry("mcp ci run", workerctl.PathMCPCIRun), + ) + + DescribeTable("answers a dispatched verb as a stream, so progress and the terminal line share one body", + func(path string) { + resp, err := http.Post(base+path, "application/json", strings.NewReader(`{}`)) //nolint:gosec,noctx // httptest server, no redirects to follow + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + Expect(resp.Header.Get("Content-Type")).To(Equal(workerctl.ContentTypeStream)) + + // Exactly one reply line, and it is the last thing on the body. + // That is what the claiming replica stops reading on. + var envs []workerctl.Envelope + sc := bufio.NewScanner(resp.Body) + for sc.Scan() { + if strings.TrimSpace(sc.Text()) == "" { + continue + } + var env workerctl.Envelope + Expect(json.Unmarshal(sc.Bytes(), &env)).To(Succeed()) + envs = append(envs, env) + } + Expect(sc.Err()).ToNot(HaveOccurred()) + Expect(envs).ToNot(BeEmpty()) + Expect(envs[len(envs)-1].Reply).ToNot(BeEmpty(), "the reply line must be last") + for _, env := range envs[:len(envs)-1] { + Expect(env.Reply).To(BeEmpty(), "only the last line may be a reply") + } + }, + Entry("agent execute", workerctl.PathAgentExecute), + Entry("mcp ci run", workerctl.PathMCPCIRun), ) }) diff --git a/core/services/agentpool/agent_pool.go b/core/services/agentpool/agent_pool.go index dd9c49da8d3d..39803b0a2c15 100644 --- a/core/services/agentpool/agent_pool.go +++ b/core/services/agentpool/agent_pool.go @@ -18,6 +18,7 @@ import ( "github.com/mudler/LocalAI/core/http/auth" "github.com/mudler/LocalAI/core/services/agents" "github.com/mudler/LocalAI/core/services/distributed" + "github.com/mudler/LocalAI/core/services/jobs" "github.com/mudler/LocalAI/core/services/messaging" skillsManager "github.com/mudler/LocalAI/core/services/skills" @@ -173,8 +174,8 @@ func (s *AgentPoolService) Start(ctx context.Context) error { s.apiURL = apiURL s.apiKey = apiKey - // Distributed mode: use native executor + NATSDispatcher. - // No LocalAGI pool, no collections, no skills service — all stateless. + // Distributed mode: the frontend enqueues claims and agent workers execute + // them. No LocalAGI pool, no collections, no skills service, all stateless. if s.distributed.natsClient != nil { return s.startDistributed(ctx, apiURL, apiKey) } @@ -242,8 +243,9 @@ func (s *AgentPoolService) startDistributed(ctx context.Context, apiURL, apiKey s.users.userStorage = NewUserScopedStorage(stateDir, dataDir) // Start the background agent scheduler on the frontend. - // It needs DB access to list configs and update LastRunAt — the worker doesn't have DB. - // The advisory lock ensures only one frontend instance runs the scheduler. + // It needs DB access to list configs, update LastRunAt and write the claim + // rows, because the worker has no database. The advisory lock ensures only one + // frontend instance runs the scheduler. if s.users.authDB != nil && s.distributed.natsClient != nil && s.distributed.agentStore != nil { var schedulerOpts []agents.AgentSchedulerOpt if s.distributed.skillStore != nil { @@ -251,9 +253,7 @@ func (s *AgentPoolService) startDistributed(ctx context.Context, apiURL, apiKey } scheduler := agents.NewAgentScheduler( s.users.authDB, - s.distributed.natsClient, s.distributed.agentStore, - messaging.SubjectAgentExecute, schedulerOpts..., ) go scheduler.Start(ctx) @@ -411,7 +411,7 @@ func (s *AgentPoolService) SetAgentStore(store *agents.AgentStore) { // Agent execution in distributed mode is handled by the dedicated agent-worker process // using the NATSDispatcher from core/services/agents/dispatcher.go. -// The frontend only dispatches chat events to NATS via dispatchChat(). +// The frontend only enqueues chat claims via dispatchChat(). // --- Agent CRUD --- @@ -996,9 +996,9 @@ func (s *AgentPoolService) ChatForUser(userID, name, message string) (string, er return s.configBackend.Chat(userID, name, message) } -// dispatchChat publishes a chat event to the NATS agent execution queue. -// The event is enriched with the full agent config and resolved skills so that -// the worker does not need direct database access. +// dispatchChat writes an agent-run claim for one chat message. +// The claim payload is enriched with the full agent config and resolved skills +// so that the worker, which has no database, needs no database access. func (s *AgentPoolService) dispatchChat(userID, name, message string) (string, error) { messageID := fmt.Sprintf("%d", time.Now().UnixNano()) @@ -1040,7 +1040,11 @@ func (s *AgentPoolService) dispatchChat(userID, name, message string) (string, e Config: cfg, Skills: skills, } - if err := s.distributed.natsClient.Publish(messaging.SubjectAgentExecute, evt); err != nil { + // A claim row rather than a publish onto a queue group. A publish onto a + // group nobody had joined succeeded and the chat was simply never answered, + // with nothing anywhere recording that it had been asked for; a row that no + // dispatch loop takes is still a row. + if _, err := jobs.EnqueueClaim(context.Background(), s.users.authDB, jobs.ClaimKindAgentRun, evt); err != nil { return "", fmt.Errorf("failed to dispatch agent chat: %w", err) } return messageID, nil @@ -1165,7 +1169,7 @@ func (s *AgentPoolService) ExecuteAction(ctx context.Context, actionName string, } // loadSkillsForUser loads full skill info (name, description, content) for a user. -// Used by dispatchChat and the scheduler to enrich NATS events. +// Used by dispatchChat and the scheduler to enrich claim payloads. func (s *AgentPoolService) loadSkillsForUser(userID string) ([]agents.SkillInfo, error) { mgr, err := s.SkillManagerForUser(userID) if err != nil { diff --git a/core/services/agentpool/dispatch_chat_internal_test.go b/core/services/agentpool/dispatch_chat_internal_test.go new file mode 100644 index 000000000000..0130d1c4c7c3 --- /dev/null +++ b/core/services/agentpool/dispatch_chat_internal_test.go @@ -0,0 +1,76 @@ +package agentpool + +import ( + "context" + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/agents" + "github.com/mudler/LocalAI/core/services/jobs" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// The third producer of a claim row, pinned separately from the other two. +// +// A rule stated at three call sites and pinned at two is how the third comes to +// keep publishing onto a subject that no longer exists, or to write a row of +// the wrong kind that no dispatcher serves. The other two are +// jobs.Dispatcher.Enqueue and agents.AgentScheduler.runDueAgents. +var _ = Describe("Dispatching an agent chat in distributed mode", func() { + var ( + svc *AgentPoolService + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + db := testutil.SetupTestDB() + Expect(jobs.MigrateClaims(ctx, db)).To(Succeed()) + + store, err := agents.NewAgentStore(db) + Expect(err).ToNot(HaveOccurred()) + cfgJSON, err := json.Marshal(agents.AgentConfig{Name: "helper", Model: "m1"}) + Expect(err).ToNot(HaveOccurred()) + Expect(store.SaveConfig(&agents.AgentConfigRecord{ + UserID: "u1", Name: "helper", ConfigJSON: string(cfgJSON), Status: agents.StatusActive, + })).To(Succeed()) + + svc = &AgentPoolService{ + users: userManager{authDB: db}, + distributed: distributedBridge{agentStore: store}, + } + }) + + It("writes an agent-run claim carrying the agent's config, rather than publishing", func() { + messageID, err := svc.dispatchChat("u1", "helper", "hello") + Expect(err).ToNot(HaveOccurred()) + Expect(messageID).ToNot(BeEmpty()) + + var rows []jobs.WorkClaim + Expect(svc.users.authDB.Find(&rows).Error).To(Succeed()) + Expect(rows).To(HaveLen(1), "the chat must leave a row: a publish onto a queue group nobody joined is accepted and dropped") + Expect(rows[0].Kind).To(Equal(string(jobs.ClaimKindAgentRun))) + + var evt agents.AgentChatEvent + Expect(json.Unmarshal(rows[0].Payload, &evt)).To(Succeed()) + Expect(evt.AgentName).To(Equal("helper")) + Expect(evt.UserID).To(Equal("u1")) + Expect(evt.Message).To(Equal("hello")) + Expect(evt.MessageID).To(Equal(messageID)) + // The worker has no database, so everything it needs travels with the + // work, exactly as it did in the enriched bus payload. + Expect(evt.Config).ToNot(BeNil()) + Expect(evt.Config.Model).To(Equal("m1")) + }) + + It("refuses a chat for an agent that does not exist, rather than queueing work nothing can run", func() { + _, err := svc.dispatchChat("u1", "no-such-agent", "hello") + Expect(err).To(HaveOccurred()) + + var n int64 + Expect(svc.users.authDB.Model(&jobs.WorkClaim{}).Count(&n).Error).To(Succeed()) + Expect(n).To(BeZero()) + }) +}) diff --git a/core/services/agents/dispatcher.go b/core/services/agents/dispatcher.go index 3ed737e83f62..6f46a0346842 100644 --- a/core/services/agents/dispatcher.go +++ b/core/services/agents/dispatcher.go @@ -26,7 +26,8 @@ const ( RoleAgent = "agent" ) -// AgentChatEvent is the NATS message payload for agent chat jobs. +// AgentChatEvent is the payload of an agent-run claim, and the request body of +// the agent execution control verb that carries it to a worker. type AgentChatEvent struct { AgentName string `json:"agent_name"` UserID string `json:"user_id"` @@ -34,14 +35,16 @@ type AgentChatEvent struct { MessageID string `json:"message_id"` Role string `json:"role,omitempty"` // "user" or "system" (for periodic runs) - // Enriched payload: set by the frontend/scheduler so that the worker - // does not need direct database access. + // Enriched payload: set by the frontend/scheduler so that the worker, + // which has no database, needs no database access. Config *AgentConfig `json:"config,omitempty"` // full agent configuration Skills []SkillInfo `json:"skills,omitempty"` // resolved per-user skills } // Dispatcher routes agent chat requests to the executor. -// Two implementations: LocalDispatcher (direct goroutine) and NATSDispatcher (queue). +// The standalone implementation is LocalDispatcher (direct goroutine); in +// distributed mode the frontend writes a claim row instead (see +// agentpool.dispatchChat) and a worker runs it through WorkerExecutor. type Dispatcher interface { // Dispatch sends a chat message to an agent and returns immediately. // The response is delivered asynchronously via the configured event delivery mechanism. @@ -220,105 +223,62 @@ func (d *LocalDispatcher) buildLocalCallbacks(writer SSEWriter, messageID string } } -// --- NATS Dispatcher (distributed) --- - -// NATSDispatcher dispatches agent chats via NATS queue group. -type NATSDispatcher struct { - nats messaging.MessagingClient +// --- Worker-side agent executor (distributed) --- + +// WorkerExecutor runs agent chats on an agent worker. +// +// It used to be a NATS queue-group subscriber, which is where the name +// NATSDispatcher came from and why it had a subject and a queue. It has +// neither now: a queue group only ever SELECTED one consumer, the frontend +// makes that selection itself (nodes.AgentSelector), and what reaches this +// worker is a streaming control RPC carrying one claim. So this type no longer +// dispatches anything; it executes what it is handed and writes everything it +// produces onto the response body of that RPC. +type WorkerExecutor struct { eventBridge *EventBridge configs ConfigProvider apiURL string apiKey string - subject string - queue string - sub messaging.Subscription // stored subscription for cleanup - sem chan struct{} // concurrency limiter; nil = unlimited - wg sync.WaitGroup } -// NewNATSDispatcher creates a dispatcher that uses NATS for distribution. -// maxConcurrent limits the number of concurrent agent jobs; 0 means unlimited. -func NewNATSDispatcher(nats messaging.MessagingClient, bridge *EventBridge, configs ConfigProvider, apiURL, apiKey, subject, queue string, maxConcurrent int) *NATSDispatcher { - d := &NATSDispatcher{ - nats: nats, +// NewWorkerExecutor creates the executor an agent worker serves +// workerctl.PathAgentExecute with. +func NewWorkerExecutor(bridge *EventBridge, configs ConfigProvider, apiURL, apiKey string) *WorkerExecutor { + return &WorkerExecutor{ eventBridge: bridge, configs: configs, apiURL: apiURL, apiKey: apiKey, - subject: subject, - queue: queue, - } - if maxConcurrent > 0 { - d.sem = make(chan struct{}, maxConcurrent) - } - return d -} - -func (d *NATSDispatcher) Start(ctx context.Context) error { - sub, err := d.nats.QueueSubscribe(d.subject, d.queue, func(data []byte) { - var evt AgentChatEvent - if err := json.Unmarshal(data, &evt); err != nil { - xlog.Error("Failed to unmarshal agent chat event", "error", err) - return - } - if d.sem != nil { - select { - case d.sem <- struct{}{}: - case <-ctx.Done(): - return - } - } - d.wg.Add(1) - concurrency.SafeGo(func() { - defer d.wg.Done() - if d.sem != nil { - defer func() { <-d.sem }() - } - d.handleJob(ctx, evt) - }) - }) - if err != nil { - return fmt.Errorf("subscribing to %s: %w", d.subject, err) - } - d.sub = sub - xlog.Info("NATS agent dispatcher started", "subject", d.subject, "queue", d.queue) - return nil -} - -// Stop unsubscribes from the NATS queue, stopping message delivery. -func (d *NATSDispatcher) Stop() error { - if d.sub != nil { - err := d.sub.Unsubscribe() - d.sub = nil - d.wg.Wait() - return err } - return nil } -func (d *NATSDispatcher) Dispatch(userID, agentName, message string) (string, error) { - messageID := uuid.New().String() - - // Send user message to SSE immediately - if d.eventBridge != nil { - d.eventBridge.PublishMessage(agentName, userID, RoleUser, message, messageID+"-user") - d.eventBridge.PublishStatus(agentName, userID, "processing") - } - - evt := AgentChatEvent{ - AgentName: agentName, - UserID: userID, - Message: message, - MessageID: messageID, - Role: RoleUser, +// Execute runs one agent chat and answers the control verb. +// +// pub is where every event this run produces goes: the agent's stream events, +// its status changes, its tool results and its messages. On the control plane +// that is the response body the claiming replica is already reading, so a +// caller cannot miss an event by having subscribed too late, and the terminal +// answer below cannot be published to nobody. +// +// A returned ERROR means this worker could not serve the verb at all, and the +// claiming replica must not read it as a verdict about the work: it releases +// the claim. Everything this worker LEARNED by running the agent, including a +// failure, comes back as the reply below. +func (d *WorkerExecutor) Execute(ctx context.Context, raw json.RawMessage, pub messaging.Publisher) (json.RawMessage, error) { + var evt AgentChatEvent + if err := json.Unmarshal(raw, &evt); err != nil { + return nil, fmt.Errorf("reading an agent execution request: %w", err) } - if err := d.nats.Publish(d.subject, evt); err != nil { - return "", fmt.Errorf("failed to dispatch agent chat: %w", err) - } - return messageID, nil + bridge := d.eventBridge.WithPublisher(pub) + status, errMsg := d.handleJob(ctx, evt, bridge) + // The reply names no job: an agent run has no job row, and its output has + // already travelled as events. What it carries is the fact that this worker + // ran it to a conclusion, which is what lets the claim be completed rather + // than retried on another worker. + return json.Marshal(map[string]string{"status": status, "error": errMsg}) } -func (d *NATSDispatcher) handleJob(ctx context.Context, evt AgentChatEvent) { +func (d *WorkerExecutor) handleJob(ctx context.Context, evt AgentChatEvent, bridge *EventBridge) (status, errMsg string) { xlog.Info("Processing agent chat job", "agent", evt.AgentName, "user", evt.UserID) // Prefer config from the enriched payload (no DB needed). @@ -329,33 +289,29 @@ func (d *NATSDispatcher) handleJob(ctx context.Context, evt AgentChatEvent) { cfg, err = d.configs.GetAgentConfig(evt.UserID, evt.AgentName) if err != nil { xlog.Error("Failed to load agent config", "agent", evt.AgentName, "error", err) - if d.eventBridge != nil { - d.eventBridge.PublishStatus(evt.AgentName, evt.UserID, "error: agent config not found") - } - return + dropped(bridge.PublishStatus(evt.AgentName, evt.UserID, "error: agent config not found"), "status", evt.AgentName) + return "failed", "agent config not found" } } if cfg == nil { xlog.Error("No agent config available", "agent", evt.AgentName) - if d.eventBridge != nil { - d.eventBridge.PublishStatus(evt.AgentName, evt.UserID, "error: agent config not found") - } - return + dropped(bridge.PublishStatus(evt.AgentName, evt.UserID, "error: agent config not found"), "status", evt.AgentName) + return "failed", "agent config not found" } ctx, cancel := context.WithCancel(ctx) defer cancel() - // Register cancellation - if d.eventBridge != nil { - d.eventBridge.RegisterCancel(evt.MessageID, cancel) - defer d.eventBridge.DeregisterCancel(evt.MessageID) - } + // Register cancellation on the SHARED registry, which the bus-backed cancel + // listener also reads: a cancel arrives on the bus and has to reach an + // execution that is publishing onto a stream. + bridge.RegisterCancel(evt.MessageID, cancel) + defer bridge.DeregisterCancel(evt.MessageID) - cb := d.buildNATSCallbacks(evt) + cb := d.buildCallbacks(evt, bridge) - // Build execution options: skills come from the enriched NATS payload - // (workers have no database access). + // Build execution options: skills come from the enriched payload (workers + // have no database access). opts := ExecuteChatOpts{ UserID: evt.UserID, MessageID: evt.MessageID, @@ -364,27 +320,39 @@ func (d *NATSDispatcher) handleJob(ctx context.Context, evt AgentChatEvent) { opts.SkillProvider = &staticSkillProvider{skills: evt.Skills} } - var response string var execErr error if evt.Role == RoleSystem { // Background/autonomous run — use inner monologue template + permanent goal - response, execErr = ExecuteBackgroundRun(ctx, d.apiURL, d.apiKey, cfg, cb, opts) + _, execErr = ExecuteBackgroundRun(ctx, d.apiURL, d.apiKey, cfg, cb, opts) } else { - response, execErr = ExecuteChat(ctx, d.apiURL, d.apiKey, cfg, evt.Message, cb, opts) + _, execErr = ExecuteChat(ctx, d.apiURL, d.apiKey, cfg, evt.Message, cb, opts) } if execErr != nil { xlog.Error("Distributed agent execution failed", "agent", evt.AgentName, "error", execErr) - if d.eventBridge != nil { - d.eventBridge.PublishStatus(evt.AgentName, evt.UserID, "error") - d.eventBridge.PublishMessage(evt.AgentName, evt.UserID, RoleAgent, - fmt.Sprintf("Agent execution failed: %v", execErr), evt.MessageID+"-error") - } - return + dropped(bridge.PublishStatus(evt.AgentName, evt.UserID, "error"), "status", evt.AgentName) + dropped(bridge.PublishMessage(evt.AgentName, evt.UserID, RoleAgent, + fmt.Sprintf("Agent execution failed: %v", execErr), evt.MessageID+"-error"), "message", evt.AgentName) + // An agent that RAN and failed is this worker's own answer, not a + // failure to serve the verb: the claiming replica must complete the + // claim rather than offer the same run to another worker. + return "failed", execErr.Error() } - _ = response // already published via callbacks + // The response itself has already travelled as events. + return "completed", "" +} + +// dropped logs an agent event that could not be published, and never returns it. +// +// An event is a NOTIFICATION about a run. A failure to publish one says nothing +// about whether the run succeeded, and turning it into the verb's error would +// report a finished agent run as a failure the claiming replica must retry. +func dropped(err error, event, agentName string) { + if err != nil { + xlog.Debug("An agent event could not be published", "event", event, "agent", agentName, "error", err) + } } // staticSkillProvider provides skills from an in-memory list (from the NATS payload). @@ -396,7 +364,7 @@ func (p *staticSkillProvider) ListSkills() ([]SkillInfo, error) { return p.skills, nil } -func (d *NATSDispatcher) buildNATSCallbacks(evt AgentChatEvent) Callbacks { +func (d *WorkerExecutor) buildCallbacks(evt AgentChatEvent, bridge *EventBridge) Callbacks { // Observable tracking: build LocalAGI-compatible observable records // from cogito callbacks so the UI can render them properly. // @@ -430,7 +398,7 @@ func (d *NATSDispatcher) buildNATSCallbacks(evt AgentChatEvent) Callbacks { return Callbacks{ OnStream: func(ev cogito.StreamEvent) { - if d.eventBridge == nil { + if bridge == nil { return } data := map[string]any{"timestamp": time.Now().Format(time.RFC3339)} @@ -472,7 +440,7 @@ func (d *NATSDispatcher) buildNATSCallbacks(evt AgentChatEvent) Callbacks { default: return } - d.eventBridge.PublishStreamEvent(evt.AgentName, evt.UserID, data) + dropped(bridge.PublishStreamEvent(evt.AgentName, evt.UserID, data), "stream", evt.AgentName) }, OnReasoning: func(text string) { // Reasoning is buffered via OnStream @@ -482,13 +450,13 @@ func (d *NATSDispatcher) buildNATSCallbacks(evt AgentChatEvent) Callbacks { }, OnToolResult: func(name, result string) { // Emit tool_result stream event for real-time UI display - if d.eventBridge != nil { - d.eventBridge.PublishStreamEvent(evt.AgentName, evt.UserID, map[string]any{ + if bridge != nil { + dropped(bridge.PublishStreamEvent(evt.AgentName, evt.UserID, map[string]any{ "type": "tool_result", "tool_name": name, "tool_result": result, "timestamp": time.Now().Format(time.RFC3339), - }) + }), "tool_result", evt.AgentName) } // Persist tool result: complete the current tool observable mu.Lock() @@ -499,23 +467,23 @@ func (d *NATSDispatcher) buildNATSCallbacks(evt AgentChatEvent) Callbacks { obs.Completion = &coreTypes.Completion{ ActionResult: result, } - if d.eventBridge != nil { - d.eventBridge.PersistObservable(evt.AgentName, evt.UserID, "tool_result", obs) + if bridge != nil { + bridge.PersistObservable(evt.AgentName, evt.UserID, "tool_result", obs) } } }, OnStatus: func(status string) { - if d.eventBridge != nil { - d.eventBridge.PublishStatus(evt.AgentName, evt.UserID, status) + if bridge != nil { + dropped(bridge.PublishStatus(evt.AgentName, evt.UserID, status), "status", evt.AgentName) } }, OnMessage: func(sender, content, msgID string) { - if d.eventBridge != nil { - d.eventBridge.PublishMessage(evt.AgentName, evt.UserID, sender, content, msgID) + if bridge != nil { + dropped(bridge.PublishMessage(evt.AgentName, evt.UserID, sender, content, msgID), "message", evt.AgentName) } // On agent response, persist the root observable with completion - if sender == RoleAgent && d.eventBridge != nil { + if sender == RoleAgent && bridge != nil { rootObs.Completion = &coreTypes.Completion{ ActionResult: content, } @@ -529,7 +497,7 @@ func (d *NATSDispatcher) buildNATSCallbacks(evt AgentChatEvent) Callbacks { }, } } - d.eventBridge.PersistObservable(evt.AgentName, evt.UserID, "chat", rootObs) + bridge.PersistObservable(evt.AgentName, evt.UserID, "chat", rootObs) } }, } diff --git a/core/services/agents/events.go b/core/services/agents/events.go index b187b295ad25..4682d2a0ef22 100644 --- a/core/services/agents/events.go +++ b/core/services/agents/events.go @@ -27,7 +27,7 @@ type AgentEvent struct { Content string `json:"content,omitempty"` MessageID string `json:"message_id,omitempty"` Metadata string `json:"metadata,omitempty"` // JSON metadata - Timestamp int64 `json:"timestamp"` // Unix milliseconds (set by PublishEvent) + Timestamp int64 `json:"timestamp"` // Unix milliseconds (set by PublishEvent) } // AgentCancelEvent is the NATS message payload for cancelling agent execution. @@ -40,12 +40,23 @@ type AgentCancelEvent struct { // EventBridge bridges agent events between NATS and SSE connections. // It enables cross-instance SSE: user connects to Frontend 1, agent runs on Frontend 2. type EventBridge struct { - nats messaging.MessagingClient + nats messaging.MessagingClient + // pub is where the events this bridge produces GO, which is not always the + // bus. On an agent worker running a dispatched claim it is the response + // body of the control RPC the claiming replica is reading, so the events + // travel back on the same stream as the result rather than on a subject + // nobody may be subscribed to yet. See WithPublisher. + pub messaging.Publisher store *AgentStore instanceID string - // Cancel registry for running agent executions - cancelRegistry messaging.CancelRegistry + // Cancel registry for running agent executions. + // + // A POINTER, because WithPublisher hands out a second view of this bridge + // and both views must reach the SAME registry: the cancel listener runs on + // the bus-backed bridge and the execution it has to reach runs on the + // stream-backed one. A copied sync.Map would swallow every cancel. + cancelRegistry *messaging.CancelRegistry // Background NATS subscriptions owned by this bridge obsPersisterSub messaging.Subscription @@ -54,12 +65,32 @@ type EventBridge struct { // NewEventBridge creates a new EventBridge. func NewEventBridge(nc messaging.MessagingClient, store *AgentStore, instanceID string) *EventBridge { return &EventBridge{ - nats: nc, - store: store, - instanceID: instanceID, + nats: nc, + pub: nc, + store: store, + instanceID: instanceID, + cancelRegistry: &messaging.CancelRegistry{}, } } +// WithPublisher returns a view of this bridge whose events go to pub. +// +// Everything else is SHARED with the receiver, the cancel registry above all: a +// cancel arriving on the bus must reach an execution that is publishing onto a +// stream, and a bridge that copied the registry would register the cancel where +// nothing looks for it. +// +// A nil pub returns the receiver unchanged rather than a bridge that publishes +// nowhere, because a handler that was given no writer still has the bus. +func (b *EventBridge) WithPublisher(pub messaging.Publisher) *EventBridge { + if b == nil || pub == nil { + return b + } + view := *b + view.pub = pub + return &view +} + // PublishEvent publishes an agent event to NATS for SSE bridging. // // Timestamp is emitted in Unix milliseconds to match the local dispatcher's @@ -70,7 +101,7 @@ func NewEventBridge(nc messaging.MessagingClient, store *AgentStore, instanceID func (b *EventBridge) PublishEvent(agentName, userID string, evt AgentEvent) error { evt.Timestamp = time.Now().UnixMilli() subject := messaging.SubjectAgentEvents(agentName, userID) - return b.nats.Publish(subject, evt) + return b.pub.Publish(subject, evt) } // PersistObservable publishes an observable_update SSE event for real-time UI diff --git a/core/services/agents/scheduler.go b/core/services/agents/scheduler.go index e159d87326de..33a19caa296f 100644 --- a/core/services/agents/scheduler.go +++ b/core/services/agents/scheduler.go @@ -6,7 +6,7 @@ import ( "time" "github.com/mudler/LocalAI/core/services/advisorylock" - "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/jobs" "github.com/mudler/xlog" "gorm.io/gorm" ) @@ -18,15 +18,13 @@ type SchedulerStore interface { } // AgentScheduler periodically checks for agents with standalone_job=true -// and publishes background run events to the NATS agent execution queue. +// and enqueues a background run as a claim on the job store. // Uses a PostgreSQL advisory lock so only one instance fires the cron. // Same pattern as notetaker's runAgentScheduler and LocalAI's cronLeaderLoop. type AgentScheduler struct { db *gorm.DB - nats messaging.Publisher store SchedulerStore skillProvider SkillContentProvider // optional: loads full skill info for enriching events - subject string // NATS subject for agent execution pollInterval time.Duration // how often to check for due agents } @@ -41,12 +39,10 @@ func WithSchedulerSkillProvider(provider SkillContentProvider) AgentSchedulerOpt } // NewAgentScheduler creates a new background agent scheduler. -func NewAgentScheduler(db *gorm.DB, nats messaging.Publisher, store SchedulerStore, subject string, opts ...AgentSchedulerOpt) *AgentScheduler { +func NewAgentScheduler(db *gorm.DB, store SchedulerStore, opts ...AgentSchedulerOpt) *AgentScheduler { s := &AgentScheduler{ db: db, - nats: nats, store: store, - subject: subject, pollInterval: 15 * time.Second, } for _, opt := range opts { @@ -57,14 +53,14 @@ func NewAgentScheduler(db *gorm.DB, nats messaging.Publisher, store SchedulerSto // Start begins the scheduler loop. Blocks until ctx is cancelled. func (s *AgentScheduler) Start(ctx context.Context) { - xlog.Info("Agent scheduler started", "pollInterval", s.pollInterval, "subject", s.subject) - advisorylock.RunLeaderLoop(ctx, s.db, advisorylock.KeyAgentScheduler, s.pollInterval, s.runDueAgents) + xlog.Info("Agent scheduler started", "pollInterval", s.pollInterval) + advisorylock.RunLeaderLoop(ctx, s.db, advisorylock.KeyAgentScheduler, s.pollInterval, func() { s.runDueAgents(ctx) }) xlog.Info("Agent scheduler stopped") } // runDueAgents finds all agents with standalone_job=true that are due for a run -// and publishes background execution events to the NATS queue. -func (s *AgentScheduler) runDueAgents() { +// and enqueues a background execution claim for each. +func (s *AgentScheduler) runDueAgents(ctx context.Context) { configs, err := s.store.ListConfigs("") // all users if err != nil { xlog.Error("Agent scheduler: failed to list configs", "error", err) @@ -103,7 +99,7 @@ func (s *AgentScheduler) runDueAgents() { } } - // Publish background run event + // Enqueue the background run as a claim evt := AgentChatEvent{ AgentName: rec.Name, UserID: rec.UserID, @@ -112,8 +108,8 @@ func (s *AgentScheduler) runDueAgents() { Config: &cfg, Skills: skills, } - if err := s.nats.Publish(s.subject, evt); err != nil { - xlog.Error("Agent scheduler: failed to publish event", "agent", rec.Name, "error", err) + if _, err := jobs.EnqueueClaim(ctx, s.db, jobs.ClaimKindAgentRun, evt); err != nil { + xlog.Error("Agent scheduler: failed to enqueue a background run", "agent", rec.Name, "error", err) continue } diff --git a/core/services/agents/scheduler_test.go b/core/services/agents/scheduler_test.go index 03e81690fee1..51182498801e 100644 --- a/core/services/agents/scheduler_test.go +++ b/core/services/agents/scheduler_test.go @@ -1,30 +1,18 @@ package agents import ( + "context" "encoding/json" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/mudler/LocalAI/core/services/jobs" "github.com/mudler/LocalAI/core/services/testutil" + "gorm.io/gorm" ) -// mockPublisher records all Publish calls for assertions. -type mockPublisher struct { - calls []publishCall -} - -type publishCall struct { - subject string - data any -} - -func (m *mockPublisher) Publish(subject string, data any) error { - m.calls = append(m.calls, publishCall{subject: subject, data: data}) - return nil -} - // mockSchedulerStore implements SchedulerStore for testing. type mockSchedulerStore struct { configs []AgentConfigRecord @@ -120,19 +108,31 @@ var _ = Describe("AgentScheduler", func() { // ----------------------------------------------------------------------- Describe("runDueAgents", func() { var ( - pub *mockPublisher + db *gorm.DB + ctx context.Context mStore *mockSchedulerStore sched *AgentScheduler ) BeforeEach(func() { - db := testutil.SetupTestDB() - pub = &mockPublisher{} + db = testutil.SetupTestDB() + ctx = context.Background() + Expect(jobs.MigrateClaims(ctx, db)).To(Succeed()) mStore = &mockSchedulerStore{} - sched = NewAgentScheduler(db, pub, mStore, "agent.execute") + sched = NewAgentScheduler(db, mStore) }) - It("publishes event for a due standalone agent", func() { + // enqueued reads the claim rows the scheduler wrote, which is what it + // writes instead of a publish: a queue subject nobody had joined + // accepted a background run and dropped it. + enqueued := func() []jobs.WorkClaim { + GinkgoHelper() + var rows []jobs.WorkClaim + Expect(db.Find(&rows).Error).To(Succeed()) + return rows + } + + It("enqueues an agent-run claim for a due standalone agent", func() { past := time.Now().Add(-15 * time.Minute) cfg := AgentConfig{ StandaloneJob: true, @@ -153,13 +153,14 @@ var _ = Describe("AgentScheduler", func() { }, } - sched.runDueAgents() + sched.runDueAgents(ctx) - Expect(pub.calls).To(HaveLen(1)) - Expect(pub.calls[0].subject).To(Equal("agent.execute")) + rows := enqueued() + Expect(rows).To(HaveLen(1)) + Expect(rows[0].Kind).To(Equal(string(jobs.ClaimKindAgentRun))) - evt, ok := pub.calls[0].data.(AgentChatEvent) - Expect(ok).To(BeTrue()) + var evt AgentChatEvent + Expect(json.Unmarshal(rows[0].Payload, &evt)).To(Succeed()) Expect(evt.AgentName).To(Equal("background-agent")) Expect(evt.UserID).To(Equal("user-1")) Expect(evt.Role).To(Equal(RoleSystem)) @@ -186,9 +187,9 @@ var _ = Describe("AgentScheduler", func() { }, } - sched.runDueAgents() + sched.runDueAgents(ctx) - Expect(pub.calls).To(BeEmpty()) + Expect(enqueued()).To(BeEmpty()) }) It("skips non-standalone agents", func() { @@ -210,9 +211,9 @@ var _ = Describe("AgentScheduler", func() { }, } - sched.runDueAgents() + sched.runDueAgents(ctx) - Expect(pub.calls).To(BeEmpty()) + Expect(enqueued()).To(BeEmpty()) }) It("skips paused agents", func() { @@ -234,9 +235,9 @@ var _ = Describe("AgentScheduler", func() { }, } - sched.runDueAgents() + sched.runDueAgents(ctx) - Expect(pub.calls).To(BeEmpty()) + Expect(enqueued()).To(BeEmpty()) }) It("skips agents with invalid config JSON", func() { @@ -253,9 +254,9 @@ var _ = Describe("AgentScheduler", func() { }, } - sched.runDueAgents() + sched.runDueAgents(ctx) - Expect(pub.calls).To(BeEmpty()) + Expect(enqueued()).To(BeEmpty()) }) It("updates last run timestamp after publishing", func() { @@ -276,9 +277,9 @@ var _ = Describe("AgentScheduler", func() { }, } - sched.runDueAgents() + sched.runDueAgents(ctx) - Expect(pub.calls).To(HaveLen(1)) + Expect(enqueued()).To(HaveLen(1)) Expect(mStore.updated).To(HaveLen(1)) Expect(mStore.updated[0].userID).To(Equal("user-1")) Expect(mStore.updated[0].name).To(Equal("track-agent")) @@ -312,11 +313,12 @@ var _ = Describe("AgentScheduler", func() { } sched.skillProvider = provider - sched.runDueAgents() + sched.runDueAgents(ctx) - Expect(pub.calls).To(HaveLen(1)) - evt, ok := pub.calls[0].data.(AgentChatEvent) - Expect(ok).To(BeTrue()) + rows := enqueued() + Expect(rows).To(HaveLen(1)) + var evt AgentChatEvent + Expect(json.Unmarshal(rows[0].Payload, &evt)).To(Succeed()) Expect(evt.Skills).To(HaveLen(2)) Expect(evt.Skills[0].Name).To(Equal("search")) Expect(evt.Skills[1].Name).To(Equal("code")) @@ -349,12 +351,15 @@ var _ = Describe("AgentScheduler", func() { }, } - sched.runDueAgents() + sched.runDueAgents(ctx) - Expect(pub.calls).To(HaveLen(2)) - names := []string{ - pub.calls[0].data.(AgentChatEvent).AgentName, - pub.calls[1].data.(AgentChatEvent).AgentName, + rows := enqueued() + Expect(rows).To(HaveLen(2)) + var names []string + for _, row := range rows { + var evt AgentChatEvent + Expect(json.Unmarshal(row.Payload, &evt)).To(Succeed()) + names = append(names, evt.AgentName) } Expect(names).To(ConsistOf("agent-a", "agent-b")) }) @@ -379,9 +384,9 @@ var _ = Describe("AgentScheduler", func() { }, } - sched.runDueAgents() + sched.runDueAgents(ctx) - Expect(pub.calls).To(HaveLen(1)) + Expect(enqueued()).To(HaveLen(1)) }) }) }) diff --git a/core/services/agentworker/server_test.go b/core/services/agentworker/server_test.go index d8b6367bcf1a..e4f4f002178e 100644 --- a/core/services/agentworker/server_test.go +++ b/core/services/agentworker/server_test.go @@ -260,10 +260,11 @@ var _ = Describe("The agent worker's control verbs", func() { }) var _ = Describe("The agent worker's streaming verbs", func() { - // Nothing sets a StreamHandler in this task. These specs exist so the shape - // is decided and pinned now: a later task adds two handlers and changes - // nothing else, and it can only do that if what a stream looks like on the - // wire is already fixed. + // The shape both dispatched verbs answer with. PathAgentExecute and + // PathMCPCIRun are now set by the CLI's agentWorkerControlHandlers, and the + // claiming replica reads exactly this: zero or more progress lines followed + // by exactly one reply line, the reply last, which is what lets it persist + // the terminal state before it releases the claim. // lines reads an NDJSON body into its envelopes. lines := func(resp *http.Response) []workerctl.Envelope { diff --git a/core/services/cluster/instance.go b/core/services/cluster/instance.go index 606337a381ff..2b1e8e4b4163 100644 --- a/core/services/cluster/instance.go +++ b/core/services/cluster/instance.go @@ -107,6 +107,21 @@ func (r *Registry) Heartbeat(ctx context.Context, id string) error { // decide whether another replica is alive. const instanceIsLive = `instances.last_seen > now() - make_interval(secs => ?)` +// LiveInstanceIDsSQL selects the ids of the replicas this deployment considers +// alive. Its single bind parameter is the liveness window in seconds, exactly +// as instanceIsLive's is. +// +// It exists for the one caller that has to decide liveness INSIDE somebody +// else's statement rather than by listing rows first: the claim queue's reap, +// which releases work held by a replica that is gone and must not release work +// held by one that is merely slow. Read-then-update would put those two facts +// in different statements and let a replica die, or come back, in the gap. +// +// It is built BY CONCATENATION from instanceIsLive rather than restating it, +// so there is still exactly one spelling of "alive" in this deployment and a +// change to the predicate cannot reach one reader and miss the other. +const LiveInstanceIDsSQL = `SELECT instances.id FROM instances WHERE ` + instanceIsLive + // Live returns the instances whose LastSeen is newer than now-within. func (r *Registry) Live(ctx context.Context, within time.Duration) ([]Instance, error) { var out []Instance diff --git a/core/services/jobs/claim.go b/core/services/jobs/claim.go new file mode 100644 index 000000000000..baa7f85f8d59 --- /dev/null +++ b/core/services/jobs/claim.go @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: MIT + +package jobs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/google/uuid" + "github.com/mudler/LocalAI/core/services/advisorylock" + "github.com/mudler/LocalAI/core/services/cluster" + "gorm.io/gorm" +) + +// ClaimKind names what a claimed row asks a worker to do. It replaces the +// subject a publisher used to choose, and it is stored on the row so a claim +// that no dispatcher serves is visible rather than silent. +type ClaimKind string + +const ( + ClaimKindTask ClaimKind = "task" // was jobs.new + ClaimKindMCPCI ClaimKind = "mcp-ci" // was jobs.mcp-ci.new + ClaimKindAgentRun ClaimKind = "agent-run" // was agent.execute +) + +// WorkClaim is one unit of dispatchable work. +// +// It replaces a queue group, which was never a broker feature this design has +// to reproduce: what a queue group did was deliver one message to exactly one +// of several competing consumers, and that is a row and a lock. +// +// ClaimedBy names the frontend REPLICA holding the claim, never the worker the +// work was handed to. That is the whole difference between this table and a +// lease on a worker: a worker that dies mid-verb makes the streaming RPC fail, +// which releases the claim through the ordinary settle path; it is the +// claiming REPLICA's death that leaves a row nobody is driving, and only the +// reap can answer that. +type WorkClaim struct { + ID string `gorm:"primaryKey;size:36"` + Kind string `gorm:"size:32;index"` + Payload []byte + ClaimedBy string `gorm:"size:64;index"` + ClaimedAt *time.Time `gorm:"index"` + Attempts int + CreatedAt time.Time `gorm:"index"` +} + +// TableName pins the table this maps onto, because ReapAbandoned and ClaimNext +// name it in raw SQL and a gorm pluralisation change would leave the two +// spellings disagreeing at runtime rather than at build time. +func (WorkClaim) TableName() string { return claimsTable } + +const claimsTable = "work_claims" + +// ErrNoWork reports that no row of the requested kinds was available to claim. +// It is an ordinary emptiness and never an error about the database. +var ErrNoWork = errors.New("jobs: no unclaimed work") + +// MigrateClaims creates the claim table. +// +// Under the same advisory lock every other schema migration in this deployment +// takes, because frontends and workers start at the same time and two +// concurrent AutoMigrate calls on one table race in PostgreSQL's catalog. +func MigrateClaims(ctx context.Context, db *gorm.DB) error { + if err := advisorylock.WithLockCtx(ctx, db, advisorylock.KeySchemaMigrate, func() error { + return db.WithContext(ctx).AutoMigrate(&WorkClaim{}) + }); err != nil { + return fmt.Errorf("migrating the claim table: %w", err) + } + return nil +} + +// requirePostgres refuses an operation whose statement only PostgreSQL runs. +// +// Stated once and called from every entry point that emits such a statement. +// The refusal matters more than the syntax error it prevents: SKIP LOCKED and +// make_interval fail on SQLite with a driver-level parse error that reads like +// a missing migration, and a claim queue that cannot claim must not be +// mistaken for one whose table is not there yet. +func requirePostgres(db *gorm.DB, op string) error { + if db == nil { + return fmt.Errorf("%s: no database handle", op) + } + if name := db.Dialector.Name(); !strings.Contains(name, "postgres") { + return fmt.Errorf("%s: the claim queue requires PostgreSQL, this deployment runs on %q", op, name) + } + return nil +} + +// EnqueueClaim writes one claim row and returns its id. +// +// It replaces a publish, and unlike a publish it cannot succeed while nothing +// is listening: a row with no dispatcher is a row that is still there. +// +// created_at is stamped by the DATABASE. It is the order competing replicas +// take work in, so it has to be measured on the one clock they all share; a +// Go-side time.Now() would make the queue order depend on which replica +// happened to enqueue. +func EnqueueClaim(ctx context.Context, db *gorm.DB, kind ClaimKind, payload any) (string, error) { + if err := requirePostgres(db, "enqueueing work"); err != nil { + return "", err + } + raw, err := json.Marshal(payload) + if err != nil { + return "", fmt.Errorf("encoding the payload of a %s claim: %w", kind, err) + } + id := uuid.New().String() + if err := db.WithContext(ctx).Model(&WorkClaim{}).Create(map[string]any{ + "id": id, + "kind": string(kind), + "payload": raw, + "claimed_by": "", + "claimed_at": nil, + "attempts": 0, + "created_at": gorm.Expr("now()"), + }).Error; err != nil { + return "", fmt.Errorf("enqueueing a %s claim: %w", kind, err) + } + return id, nil +} + +// claimNextSQL takes at most one unclaimed row in ONE statement. +// +// FOR UPDATE SKIP LOCKED is the whole mechanism. Without SKIP LOCKED two +// replicas polling at the same instant do not take two rows: the second BLOCKS +// on the first's row lock and only then re-evaluates, so a slow claimant stalls +// every other replica in the deployment behind it. Without FOR UPDATE at all +// they can both read the same unclaimed row and both write it, which is the +// exactly-once property gone. +// +// One statement rather than select-then-update for the same reason: the gap +// between the two is where two claimants both see an unclaimed row. +// +// claimed_at is stamped by the database, like created_at and for the same +// reason: the reap compares it against replica liveness, which is measured on +// that clock. +const claimNextSQL = `UPDATE ` + claimsTable + ` +SET claimed_by = ?, claimed_at = now() +WHERE id = ( + SELECT id FROM ` + claimsTable + ` + WHERE claimed_at IS NULL AND kind IN ? + ORDER BY created_at, id + FOR UPDATE SKIP LOCKED + LIMIT 1 +) +RETURNING id, kind, payload, claimed_by, claimed_at, attempts, created_at` + +// ClaimNext takes at most one unclaimed row of any kind in kinds, marking it +// claimed by owner. It returns ErrNoWork when there is none. +func ClaimNext(ctx context.Context, db *gorm.DB, owner string, kinds []ClaimKind) (*WorkClaim, error) { + if err := requirePostgres(db, "claiming work"); err != nil { + return nil, err + } + if owner == "" { + // A claim nobody owns cannot be reaped when its holder dies, because + // the reap asks whether the OWNER is still live. Refused here rather + // than written, since the symptom of an unowned claim is work that is + // never retried and never completed. + return nil, errors.New("claiming work: no owner named, so the claim could never be reaped") + } + if len(kinds) == 0 { + return nil, ErrNoWork + } + wanted := make([]string, 0, len(kinds)) + for _, k := range kinds { + wanted = append(wanted, string(k)) + } + var claim WorkClaim + res := db.WithContext(ctx).Raw(claimNextSQL, owner, wanted).Scan(&claim) + if res.Error != nil { + return nil, fmt.Errorf("claiming work as %q: %w", owner, res.Error) + } + if res.RowsAffected == 0 { + return nil, ErrNoWork + } + return &claim, nil +} + +// ReleaseClaim returns a row to the pool, incrementing Attempts. +// +// It is what a TRANSPORT failure does: nothing was learned about the work, so +// it must be retried, possibly by another replica against another worker. +func ReleaseClaim(ctx context.Context, db *gorm.DB, id string) error { + if db == nil { + return errors.New("releasing a claim: no database handle") + } + res := db.WithContext(ctx).Model(&WorkClaim{}).Where("id = ?", id).Updates(map[string]any{ + "claimed_by": "", + "claimed_at": nil, + "attempts": gorm.Expr("attempts + 1"), + }) + if res.Error != nil { + return fmt.Errorf("releasing claim %q: %w", id, res.Error) + } + return nil +} + +// CompleteClaim deletes the row. +// +// It is what a WORKER'S ANSWER does, success or failure: the worker ran the +// work and said what happened, so it must not run again. +func CompleteClaim(ctx context.Context, db *gorm.DB, id string) error { + if db == nil { + return errors.New("completing a claim: no database handle") + } + if err := db.WithContext(ctx).Where("id = ?", id).Delete(&WorkClaim{}).Error; err != nil { + return fmt.Errorf("completing claim %q: %w", id, err) + } + return nil +} + +// reapAbandonedSQL releases every claim whose owner is no longer a live replica. +// +// The predicate is REPLICA ABSENCE and deliberately not claim age, and that is +// the whole of this task's invariant applied to work. A claim held for an hour +// by a replica that is heartbeating is a slow job, and stealing it would run +// the work twice; a claim held for a second by a replica that is gone is work +// nobody is driving. Age cannot tell those apart and liveness can, so the age +// term that a lease design would carry is absent on purpose: there is no window +// after which a live replica's work is taken away from it. +// +// It reuses cluster.LiveInstanceIDsSQL rather than restating what "live" means, +// so this reap and every other reader of replica absence in the deployment move +// together, and it runs as ONE statement so a replica cannot die, or come back, +// between deciding who is live and acting on it. +const reapAbandonedSQL = `UPDATE ` + claimsTable + ` +SET claimed_by = '', claimed_at = NULL, attempts = attempts + 1 +WHERE claimed_at IS NOT NULL + AND claimed_by NOT IN (` + cluster.LiveInstanceIDsSQL + `)` + +// ReapAbandoned releases the claims held by replicas that are no longer live +// within the given window, and returns how many it released. +// +// The window is the replica-liveness window, cluster.InstanceLiveness in +// production. It is NOT a timeout on the work: see reapAbandonedSQL. +func ReapAbandoned(ctx context.Context, db *gorm.DB, liveness time.Duration) (int64, error) { + if err := requirePostgres(db, "reaping abandoned claims"); err != nil { + return 0, err + } + res := db.WithContext(ctx).Exec(reapAbandonedSQL, liveness.Seconds()) + if res.Error != nil { + return 0, fmt.Errorf("reaping abandoned claims: %w", res.Error) + } + return res.RowsAffected, nil +} + +// ownerIsLiveSQL asks whether ONE named replica is currently live. +// +// The same predicate the reap uses, read the same way, because a replica that +// would not survive the reap must not be allowed to claim in the first place: +// its claim would be indistinguishable from one held by a dead replica and +// another replica would take the work away from it while it was still running. +const ownerIsLiveSQL = `SELECT count(*) FROM (` + cluster.LiveInstanceIDsSQL + ` AND instances.id = ?) AS live` + +// OwnerIsLive reports whether owner is a replica this deployment currently +// considers alive. +// +// It exists for the dispatch loop's own guard. A replica that is not in the +// instances table at all (core/application/distributed.go leaves membership +// unstarted when no peer-reachable address can be derived) answers false, which +// is the honest answer: nothing about it is observable to its peers, so nothing +// can tell its claims from abandoned ones. +func OwnerIsLive(ctx context.Context, db *gorm.DB, owner string, liveness time.Duration) (bool, error) { + if err := requirePostgres(db, "checking whether this replica is live"); err != nil { + return false, err + } + var n int64 + if err := db.WithContext(ctx).Raw(ownerIsLiveSQL, liveness.Seconds(), owner).Scan(&n).Error; err != nil { + return false, fmt.Errorf("checking whether replica %q is live: %w", owner, err) + } + return n > 0, nil +} diff --git a/core/services/jobs/claim_test.go b/core/services/jobs/claim_test.go new file mode 100644 index 000000000000..8dc4f558d6f2 --- /dev/null +++ b/core/services/jobs/claim_test.go @@ -0,0 +1,502 @@ +package jobs + +import ( + "context" + "strings" + "sync" + "time" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + gormlogger "gorm.io/gorm/logger" +) + +// claimSQLRecorder captures the statements a call under test issues, so a spec +// can pin the SHAPE of one rather than only its effect. +// +// The shape is what has to be pinned for anything measured on the database +// clock. The test container shares this host's clock, so a Go-side cutoff and +// now() agree to the microsecond and no behavioural spec can tell them apart; +// the difference only shows up in a deployment whose replicas' clocks differ, +// which is every real one. +type claimSQLRecorder struct { + gormlogger.Interface + mu sync.Mutex + statements []string + errs []error +} + +func newClaimSQLRecorder() *claimSQLRecorder { + return &claimSQLRecorder{Interface: gormlogger.Default.LogMode(gormlogger.Silent)} +} + +func (r *claimSQLRecorder) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) { + sql, rows := fc() + r.mu.Lock() + r.statements = append(r.statements, sql) + if err != nil { + r.errs = append(r.errs, err) + } + r.mu.Unlock() + // Delegated so a failing statement is still reported the way gorm would + // report it: the instrument used to prove what the SQL does must not be the + // one thing that hides it erroring. + r.Interface.Trace(ctx, begin, func() (string, int64) { return sql, rows }, err) +} + +// only returns the single recorded statement, failing the spec if the call +// under test issued anything other than exactly one. A read-then-write shape +// shows up here as two. +func (r *claimSQLRecorder) only() string { + r.mu.Lock() + defer r.mu.Unlock() + ExpectWithOffset(1, r.errs).To(BeEmpty(), "the recorded statement failed") + ExpectWithOffset(1, r.statements).To(HaveLen(1), "expected exactly one statement, got: %v", r.statements) + return r.statements[0] +} + +func (r *claimSQLRecorder) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.statements) +} + +// sqliteHandle opens an in-memory SQLite database, which is the single-binary +// dialect this deployment also ships on. +func sqliteHandle() *gorm.DB { + GinkgoHelper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) + Expect(err).ToNot(HaveOccurred()) + return db +} + +var _ = Describe("The claim queue", func() { + var ( + db *gorm.DB + ctx context.Context + ) + + BeforeEach(func() { + db = testutil.SetupTestDB() + ctx = context.Background() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + Expect(MigrateClaims(ctx, db)).To(Succeed()) + }) + + enqueue := func(kind ClaimKind, payload any) string { + GinkgoHelper() + id, err := EnqueueClaim(ctx, db, kind, payload) + Expect(err).ToNot(HaveOccurred()) + Expect(id).ToNot(BeEmpty()) + return id + } + + rowOf := func(id string) WorkClaim { + GinkgoHelper() + var got WorkClaim + Expect(db.Where("id = ?", id).First(&got).Error).To(Succeed()) + return got + } + + Describe("taking work", func() { + It("reports an empty queue distinguishably rather than as a failure", func() { + _, err := ClaimNext(ctx, db, "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).To(MatchError(ErrNoWork)) + }) + + It("takes only the kinds asked for", func() { + enqueue(ClaimKindTask, JobEvent{JobID: "j1"}) + _, err := ClaimNext(ctx, db, "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).To(MatchError(ErrNoWork)) + }) + + It("refuses to write a claim nobody owns, since nothing could ever reap it", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + _, err := ClaimNext(ctx, db, "", []ClaimKind{ClaimKindMCPCI}) + Expect(err).To(HaveOccurred()) + Expect(err).ToNot(MatchError(ErrNoWork)) + Expect(rowOf(enqueuedOnly(db)).ClaimedBy).To(BeEmpty()) + }) + + It("carries the payload through unchanged", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-payload", TaskID: "t1"}) + got, err := ClaimNext(ctx, db, "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + Expect(string(got.Payload)).To(ContainSubstring(`"job_id":"j-payload"`)) + Expect(got.Kind).To(Equal(string(ClaimKindMCPCI))) + Expect(got.ClaimedBy).To(Equal("inst-a")) + Expect(got.ClaimedAt).ToNot(BeNil()) + }) + }) + + Describe("competing claimants", func() { + // This is the exactly-once property, and it is asserted against a real + // PostgreSQL with real concurrent claimants rather than a double. A + // double cannot fail the way this fails: what makes two claimants take + // two rows is a row lock, and a fake has none. + It("hands each of eight concurrent claimants a DIFFERENT one of eight rows", func() { + const claimants = 8 + for range claimants { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j"}) + } + + var mu sync.Mutex + taken := map[string]string{} // claim id -> owner + var failures []error + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range claimants { + owner := "inst-" + string(rune('a'+i)) + wg.Add(1) + go func() { + defer GinkgoRecover() + defer wg.Done() + <-start + got, err := ClaimNext(ctx, db, owner, []ClaimKind{ClaimKindMCPCI}) + mu.Lock() + defer mu.Unlock() + if err != nil { + failures = append(failures, err) + return + } + taken[got.ID] = owner + }() + } + close(start) + done := make(chan struct{}) + go func() { defer GinkgoRecover(); wg.Wait(); close(done) }() + Eventually(done, "30s").Should(BeClosed(), "a claimant blocked instead of skipping a locked row") + + Expect(failures).To(BeEmpty()) + Expect(taken).To(HaveLen(claimants), "two claimants took the same row, so the queue is not exactly-once") + + var rows []WorkClaim + Expect(db.Find(&rows).Error).To(Succeed()) + for _, row := range rows { + Expect(row.ClaimedBy).To(Equal(taken[row.ID]), "a row's recorded owner disagrees with the claimant that was told it had won") + } + }) + + // The narrower half of the same property: with ONE row and eight + // claimants, seven must be told the queue is empty, and none may be + // handed a row a peer already holds. + It("leaves exactly one winner when eight claimants race for one row", func() { + enqueue(ClaimKindAgentRun, map[string]string{"agent": "a"}) + + var mu sync.Mutex + var winners []string + var empties int + var failures []error + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range 8 { + owner := "inst-" + string(rune('a'+i)) + wg.Add(1) + go func() { + defer GinkgoRecover() + defer wg.Done() + <-start + got, err := ClaimNext(ctx, db, owner, []ClaimKind{ClaimKindAgentRun}) + mu.Lock() + defer mu.Unlock() + switch { + case err == nil: + winners = append(winners, got.ID) + case err == ErrNoWork: + empties++ + default: + failures = append(failures, err) + } + }() + } + close(start) + done := make(chan struct{}) + go func() { defer GinkgoRecover(); wg.Wait(); close(done) }() + Eventually(done, "30s").Should(BeClosed()) + + Expect(failures).To(BeEmpty()) + Expect(winners).To(HaveLen(1), "the work was handed to more than one replica") + Expect(empties).To(Equal(7)) + }) + + // The SKIP LOCKED detector, and it is written so that dropping SKIP + // LOCKED makes the claimant BLOCK rather than merely return the same + // row. A third transaction holds the first row's lock and never + // commits, so a claimant without SKIP LOCKED waits on it for ever and + // this spec's Eventually never fires. + It("skips a row another transaction holds locked, and takes the next one instead", func() { + first := enqueue(ClaimKindMCPCI, JobEvent{JobID: "first"}) + second := enqueue(ClaimKindMCPCI, JobEvent{JobID: "second"}) + + holder := db.Begin() + Expect(holder.Error).ToNot(HaveOccurred()) + DeferCleanup(func() { holder.Rollback() }) + var lockedID string + Expect(holder.Raw(`SELECT id FROM work_claims WHERE id = ? FOR UPDATE`, first). + Scan(&lockedID).Error).To(Succeed()) + Expect(lockedID).To(Equal(first)) + + type result struct { + claim *WorkClaim + err error + } + out := make(chan result, 1) + go func() { + defer GinkgoRecover() + got, err := ClaimNext(ctx, db, "inst-b", []ClaimKind{ClaimKindMCPCI}) + out <- result{claim: got, err: err} + }() + + var got result + Eventually(out, "20s").Should(Receive(&got), + "the claimant blocked on a row another transaction holds instead of skipping it") + Expect(got.err).ToNot(HaveOccurred()) + Expect(got.claim.ID).To(Equal(second)) + }) + }) + + Describe("settling a claim", func() { + It("makes a released row claimable again and counts the attempt", func() { + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + _, err := ClaimNext(ctx, db, "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + + Expect(ReleaseClaim(ctx, db, id)).To(Succeed()) + + row := rowOf(id) + Expect(row.ClaimedAt).To(BeNil()) + Expect(row.ClaimedBy).To(BeEmpty()) + Expect(row.Attempts).To(Equal(1)) + + again, err := ClaimNext(ctx, db, "inst-b", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + Expect(again.ID).To(Equal(id)) + }) + + It("removes a completed row, so the work cannot run again", func() { + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + _, err := ClaimNext(ctx, db, "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + + Expect(CompleteClaim(ctx, db, id)).To(Succeed()) + + var n int64 + Expect(db.Model(&WorkClaim{}).Where("id = ?", id).Count(&n).Error).To(Succeed()) + Expect(n).To(BeZero()) + _, err = ClaimNext(ctx, db, "inst-b", []ClaimKind{ClaimKindMCPCI}) + Expect(err).To(MatchError(ErrNoWork)) + }) + }) + + Describe("reaping what a departed replica left", func() { + // The distinction the whole programme rests on, applied to work: a + // claim held by a replica that is GONE becomes claimable again, and a + // claim held by a replica that is merely SLOW is never taken away from + // it, however long it has held it. + It("releases a claim whose owner is no longer live", func() { + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + Expect(reg(db).Register(ctx, "inst-dead", "10.0.0.1:8080", "v1")).To(Succeed()) + _, err := ClaimNext(ctx, db, "inst-dead", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + ageInstance(db, "inst-dead", 10*time.Minute) + + released, err := ReapAbandoned(ctx, db, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(released).To(Equal(int64(1))) + + row := rowOf(id) + Expect(row.ClaimedAt).To(BeNil()) + Expect(row.Attempts).To(Equal(1)) + }) + + It("leaves a live replica's claim alone however old the claim is", func() { + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + Expect(reg(db).Register(ctx, "inst-slow", "10.0.0.2:8080", "v1")).To(Succeed()) + _, err := ClaimNext(ctx, db, "inst-slow", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + // The claim is hours old; its owner heartbeated a moment ago. Age is + // not the predicate, and a reap that used one would steal this. + ageClaim(db, id, 6*time.Hour) + + released, err := ReapAbandoned(ctx, db, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(released).To(BeZero()) + Expect(rowOf(id).ClaimedAt).ToNot(BeNil()) + Expect(rowOf(id).ClaimedBy).To(Equal("inst-slow")) + }) + + It("leaves an unclaimed row alone", func() { + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + released, err := ReapAbandoned(ctx, db, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(released).To(BeZero()) + Expect(rowOf(id).Attempts).To(BeZero()) + }) + + It("releases a claim whose owner never registered at all", func() { + // A replica with no row in the instances table is not observable to + // its peers, so its claims cannot be told from abandoned ones. That + // is why DispatchOnce refuses to claim as one; here it means such a + // row does not sit unclaimable for ever. + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + _, err := ClaimNext(ctx, db, "inst-unregistered", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + + released, err := ReapAbandoned(ctx, db, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(released).To(Equal(int64(1))) + Expect(rowOf(id).ClaimedAt).To(BeNil()) + }) + }) + + Describe("whether this replica may claim at all", func() { + It("says yes for a replica that is registered and heartbeating", func() { + Expect(reg(db).Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + live, err := OwnerIsLive(ctx, db, "inst-a", time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(live).To(BeTrue()) + }) + + It("says no for a replica whose heartbeat has aged out", func() { + Expect(reg(db).Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + ageInstance(db, "inst-a", 10*time.Minute) + live, err := OwnerIsLive(ctx, db, "inst-a", time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(live).To(BeFalse()) + }) + + It("says no for a replica that never registered", func() { + live, err := OwnerIsLive(ctx, db, "inst-ghost", time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(live).To(BeFalse()) + }) + }) + + Describe("the statements themselves", func() { + It("claims in ONE statement that skips locked rows and stamps on the database clock", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + rec := newClaimSQLRecorder() + _, err := ClaimNext(ctx, db.Session(&gorm.Session{Logger: rec}), "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + + sql := strings.ToLower(rec.only()) + Expect(sql).To(MatchRegexp(`for\s+update\s+skip\s+locked`), + "without SKIP LOCKED a second replica blocks on the first's row instead of taking another") + Expect(sql).To(ContainSubstring("now()")) + Expect(sql).ToNot(MatchRegexp(`claimed_at\s*=\s*'`), + "claimed_at must not be a literal timestamp from this process's clock") + Expect(sql).To(ContainSubstring("returning")) + }) + + It("enqueues with a created_at the database stamps, because that is the order replicas take work in", func() { + rec := newClaimSQLRecorder() + _, err := EnqueueClaim(ctx, db.Session(&gorm.Session{Logger: rec}), ClaimKindMCPCI, JobEvent{JobID: "j1"}) + Expect(err).ToNot(HaveOccurred()) + + sql := strings.ToLower(rec.only()) + Expect(sql).To(ContainSubstring("now()")) + Expect(sql).ToNot(MatchRegexp(`created_at"?\s*\)?\s*values.*'\d{4}-`), + "created_at must not be a literal timestamp from this process's clock") + }) + + It("reaps in ONE statement whose predicate is replica liveness on the database clock, not claim age", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + rec := newClaimSQLRecorder() + _, err := ReapAbandoned(ctx, db.Session(&gorm.Session{Logger: rec}), time.Minute) + Expect(err).ToNot(HaveOccurred()) + + sql := strings.ToLower(rec.only()) + // A read-then-update would show up as two statements; only() rules + // that out. The rest pins that liveness is what is being asked. + Expect(sql).To(ContainSubstring("instances")) + Expect(sql).To(ContainSubstring("last_seen")) + Expect(sql).To(ContainSubstring("now()")) + Expect(sql).To(ContainSubstring("make_interval")) + Expect(sql).ToNot(MatchRegexp(`last_seen\s*>\s*'`), + "the liveness cutoff must not be a literal timestamp from this process's clock") + Expect(sql).ToNot(MatchRegexp(`claimed_at\s*<`), + "claim age must not be a reap condition: it cannot tell a dead replica from a slow one") + }) + }) + + Describe("on a dialect that cannot run these statements", func() { + // Each entry point that emits PostgreSQL-only SQL is refused + // SEPARATELY. One shared guard stated at four call sites and pinned at + // one is how three of them come to lose it. + var lite *gorm.DB + BeforeEach(func() { lite = sqliteHandle() }) + + It("refuses to claim", func() { + _, err := ClaimNext(ctx, lite, "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).To(MatchError(ContainSubstring("requires PostgreSQL"))) + Expect(err).ToNot(MatchError(ErrNoWork)) + }) + + It("refuses to enqueue", func() { + _, err := EnqueueClaim(ctx, lite, ClaimKindMCPCI, JobEvent{JobID: "j1"}) + Expect(err).To(MatchError(ContainSubstring("requires PostgreSQL"))) + }) + + It("refuses to reap", func() { + _, err := ReapAbandoned(ctx, lite, time.Minute) + Expect(err).To(MatchError(ContainSubstring("requires PostgreSQL"))) + }) + + It("refuses to answer whether a replica is live", func() { + _, err := OwnerIsLive(ctx, lite, "inst-a", time.Minute) + Expect(err).To(MatchError(ContainSubstring("requires PostgreSQL"))) + }) + + It("runs no statement at all when it refuses, so the failure cannot be read as a missing table", func() { + rec := newClaimSQLRecorder() + guarded := lite.Session(&gorm.Session{Logger: rec}) + _, _ = ClaimNext(ctx, guarded, "inst-a", []ClaimKind{ClaimKindMCPCI}) + _, _ = EnqueueClaim(ctx, guarded, ClaimKindMCPCI, JobEvent{JobID: "j1"}) + _, _ = ReapAbandoned(ctx, guarded, time.Minute) + _, _ = OwnerIsLive(ctx, guarded, "inst-a", time.Minute) + Expect(rec.count()).To(BeZero()) + }) + }) + + It("is migrated by NewJobStore, so a deployment cannot accept jobs it has nowhere to queue", func() { + fresh := testutil.SetupTestDB() + _, err := NewJobStore(fresh) + Expect(err).ToNot(HaveOccurred()) + Expect(fresh.Migrator().HasTable(&WorkClaim{})).To(BeTrue()) + }) +}) + +// reg returns a cluster registry over db, for the specs that have to stage a +// replica's liveness. +func reg(db *gorm.DB) *cluster.Registry { return cluster.NewRegistry(db) } + +// ageInstance pushes a replica's heartbeat into the past. Written as a database +// expression rather than a Go timestamp so the staged value is on the same +// clock the predicate under test reads. +func ageInstance(db *gorm.DB, id string, by time.Duration) { + GinkgoHelper() + Expect(db.Exec(`UPDATE instances SET last_seen = now() - make_interval(secs => ?) WHERE id = ?`, + by.Seconds(), id).Error).To(Succeed()) +} + +// ageClaim pushes a claim's timestamp into the past, for the spec that proves +// age is NOT what the reap looks at. +func ageClaim(db *gorm.DB, id string, by time.Duration) { + GinkgoHelper() + Expect(db.Exec(`UPDATE work_claims SET claimed_at = now() - make_interval(secs => ?) WHERE id = ?`, + by.Seconds(), id).Error).To(Succeed()) +} + +// enqueuedOnly returns the id of the single row in the table. +func enqueuedOnly(db *gorm.DB) string { + GinkgoHelper() + var rows []WorkClaim + Expect(db.Find(&rows).Error).To(Succeed()) + Expect(rows).To(HaveLen(1)) + return rows[0].ID +} diff --git a/core/services/jobs/dispatch_loop.go b/core/services/jobs/dispatch_loop.go new file mode 100644 index 000000000000..616025cc6799 --- /dev/null +++ b/core/services/jobs/dispatch_loop.go @@ -0,0 +1,454 @@ +// SPDX-License-Identifier: MIT + +package jobs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/workerctl" + "github.com/mudler/LocalAI/pkg/concurrency" + "github.com/mudler/xlog" + "gorm.io/gorm" +) + +// The three ports below exist because neither `jobs` to `nodes` nor `nodes` to +// `jobs` is an import edge today and neither should become one for a dispatch +// loop. Each is named for what it DOES here, never for the concrete type that +// satisfies it: nodes.Rebroadcaster is a struct, and a local interface called +// Rebroadcaster in package jobs would be a second type with one name whose only +// implementation is the first, which is exactly the shape a reader +// mis-resolves. + +// AgentPicker chooses a worker to hand a claim to. Satisfied by +// *nodes.AgentSelector. +type AgentPicker interface { + PickConnected(ctx context.Context) (nodeID, nodeType string, err error) +} + +// StreamCaller issues one streaming control RPC. Satisfied by +// *nodes.ControlClient. +type StreamCaller interface { + CallStreaming(ctx context.Context, nodeID, path string, req, reply any, + onProgress func(subject string, raw json.RawMessage)) error +} + +// ProgressBroadcaster re-publishes a worker's progress line when that worker's +// node type is allowed to ask for it. Satisfied by *nodes.Rebroadcaster. +// It returns a bool and no error, deliberately: see nodes.Rebroadcaster.Handle. +type ProgressBroadcaster interface { + Handle(nodeType, subject string, raw json.RawMessage) bool +} + +// TerminalWriter records a dispatched job's terminal state. +// +// A port rather than *JobStore so the ORDER of the two writes settleClaim makes +// can be pinned: a store that refuses must leave the claim in place, and there +// is no way to make the real store refuse on demand. +type TerminalWriter interface { + UpdateJobStatus(jobID, status, result, errMsg string) error +} + +// The real writer, asserted here so a signature drift fails to COMPILE in the +// file that states the contract rather than in the wiring, which opens a +// database and therefore has no unit spec. +var _ TerminalWriter = (*JobStore)(nil) + +// ClaimReply is the terminal line a worker sends for a dispatched claim. +// +// It is the same four fields JobResultEvent carries, because it says the same +// thing; what changed is the carrier. On the bus a result was a message the +// frontend had to already be subscribed to. Here it is the last line of the +// response body the claiming replica is reading, so there is no window in which +// it can be published to nobody. +type ClaimReply struct { + JobID string `json:"job_id,omitempty"` + Status string `json:"status,omitempty"` // "completed" | "failed" | "cancelled" + Result string `json:"result,omitempty"` + Error string `json:"error,omitempty"` +} + +const ( + // DefaultDispatchInterval is how often a replica looks for claimable work. + DefaultDispatchInterval = 2 * time.Second + + // DefaultMaxConcurrent bounds how many claims one replica drives at once. + // + // It is not a tuning knob so much as the difference between a working queue + // and a serial one. A dispatched claim is driven SYNCHRONOUSLY: the RPC's + // response body is what carries the worker's progress and its terminal + // line, so the goroutine holding the claim is reading it for as long as the + // work runs, which for an MCP CI job is up to ten minutes by default. With + // one in flight, the second queued job in a deployment waits for the first + // to finish, which the queue group this replaces never did. + DefaultMaxConcurrent = 4 + + // DefaultClaimLiveness is the replica-liveness window the reap measures an + // owner's absence against. + // + // cluster.InstanceLiveness rather than a number of this package's own, + // because it is the SAME fact: a replica that its peers have written off is + // one whose claims nobody is driving. A second window here would let a + // replica be dead for the tunnel table and alive for the claim table. + DefaultClaimLiveness = cluster.InstanceLiveness +) + +// reasonNoTaskDispatcher is what a plain task job is failed with. +// +// No agent worker serves ClaimKindTask, exactly as no NATS subscriber ever did: +// the frontend published jobs.new and, in every deployment this ships in, +// nothing consumed it. That was invisible because a publish to an empty queue +// group succeeds and the job row simply stayed `running` for ever. A claim row +// makes it visible, and this makes it legible. +const reasonNoTaskDispatcher = "no worker in this deployment serves plain task jobs" + +// DispatchConfig is everything a DispatchLoop needs. +type DispatchConfig struct { + DB *gorm.DB + Owner string // the instance id + Selector AgentPicker // *nodes.AgentSelector + Control StreamCaller // *nodes.ControlClient + Broadcast ProgressBroadcaster // *nodes.Rebroadcaster + Store TerminalWriter // *jobs.JobStore + Interval time.Duration // poll interval, DefaultDispatchInterval when zero + Liveness time.Duration // replica-liveness window, DefaultClaimLiveness when zero + // MaxConcurrent is how many claims this replica drives at once, + // DefaultMaxConcurrent when zero. See that constant. + MaxConcurrent int +} + +// DispatchLoop claims work on this replica and drives it on an agent worker. +// +// The claim is made by a FRONTEND replica and never by an agent worker, because +// an agent worker has no database and so cannot claim. What the worker gets is +// a streaming control RPC, which is what puts its progress, its agent SSE +// events and its terminal result on the response body this replica is already +// reading. That is what lets the terminal line be persisted BEFORE the claim is +// released, structurally rather than by retry. +type DispatchLoop struct { + db *gorm.DB + owner string + selector AgentPicker + control StreamCaller + broadcast ProgressBroadcaster + store TerminalWriter + interval time.Duration + liveness time.Duration + + // sem bounds in-flight dispatches and wg joins them, so Stop returns only + // once every claim this replica holds has been settled. A claim left held + // by a process that has already exited is one no other replica may take + // until the liveness window has passed. + sem chan struct{} + wg sync.WaitGroup + // unregistered remembers that this replica has already said it cannot + // claim, so the refusal is one line per transition rather than one per + // poll interval for as long as the misconfiguration lasts. + unregistered atomic.Bool + cancel context.CancelFunc + done chan struct{} +} + +// NewDispatchLoop returns the dispatch loop for one replica. +// +// Every refusal below is a piece of wiring whose absence has no symptom. A loop +// with no selector claims rows and releases them for ever; one with no control +// client does the same; one with no owner writes claims that the reap cannot +// attribute. All three present as work that is accepted and never done, which +// is indistinguishable at the API from a busy deployment. +func NewDispatchLoop(cfg DispatchConfig) (*DispatchLoop, error) { + if cfg.DB == nil { + return nil, errors.New("the dispatch loop was built with no database to claim work from") + } + if cfg.Owner == "" { + return nil, errors.New("the dispatch loop was built with no instance id: its claims could not be told from ones held by a dead replica") + } + if cfg.Selector == nil { + return nil, errors.New("the dispatch loop was built with no way to pick an agent worker") + } + if cfg.Control == nil { + return nil, errors.New("the dispatch loop was built with no control transport to reach an agent worker over") + } + l := &DispatchLoop{ + db: cfg.DB, + owner: cfg.Owner, + selector: cfg.Selector, + control: cfg.Control, + broadcast: cfg.Broadcast, + store: cfg.Store, + interval: cfg.Interval, + liveness: cfg.Liveness, + done: make(chan struct{}), + } + if l.interval <= 0 { + l.interval = DefaultDispatchInterval + } + if l.liveness <= 0 { + l.liveness = DefaultClaimLiveness + } + maxConcurrent := cfg.MaxConcurrent + if maxConcurrent <= 0 { + maxConcurrent = DefaultMaxConcurrent + } + l.sem = make(chan struct{}, maxConcurrent) + return l, nil +} + +// Start begins claiming and dispatching until ctx is cancelled or Stop is +// called. +func (l *DispatchLoop) Start(ctx context.Context) error { + ctx, l.cancel = context.WithCancel(ctx) + go l.run(ctx) + xlog.Info("Job dispatch loop started", "instance", l.owner, "interval", l.interval) + return nil +} + +// Stop ends the loop and waits for every in-flight dispatch to settle. +// +// Waiting matters: a dispatch may be mid-RPC holding a claim, and returning +// before it settles would leave the claim held by a replica that is on its way +// out, which the next reap can only release after the liveness window. +func (l *DispatchLoop) Stop() { + if l == nil || l.cancel == nil { + return + } + l.cancel() + <-l.done +} + +func (l *DispatchLoop) run(ctx context.Context) { + // Ordered so the wait happens BEFORE done is closed: Stop reads done, and a + // Stop that returned while a dispatch was still in flight would leave the + // claim held by a process on its way out. + defer close(l.done) + defer l.wg.Wait() + ticker := time.NewTicker(l.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + select { + case l.sem <- struct{}{}: + default: + // Already driving as much as this replica may. Skipping the + // tick rather than queueing is deliberate: a claim taken now + // would be held by this replica with nothing driving it, and + // another replica could have taken it instead. + continue + } + l.wg.Add(1) + concurrency.SafeGo(func() { + defer l.wg.Done() + defer func() { <-l.sem }() + if err := l.DispatchOnce(ctx); err != nil && ctx.Err() == nil { + xlog.Warn("A dispatch tick failed", "instance", l.owner, "error", err) + } + }) + } + } +} + +// DispatchOnce reaps what dead replicas left, then claims and drives at most +// one unit of work. +// +// Exported because it is the seam every dispatch spec drives: a tick that a +// spec has to wait for is a spec that fails one run in ten. +func (l *DispatchLoop) DispatchOnce(ctx context.Context) error { + // This replica's OWN liveness, checked before it touches the queue at all. + // + // A replica with no row in the instances table is invisible to its peers, + // which is a supported if degraded state for serving requests but not for + // holding a claim: the reap asks whether a claim's owner is live, so a + // claim written here would be reaped out from under this replica while the + // work was still running, and the work would run twice. Refusing to claim + // is the conservative half of that choice, and it heals by itself the + // moment membership registers. + live, err := OwnerIsLive(ctx, l.db, l.owner, l.liveness) + if err != nil { + return fmt.Errorf("checking whether this replica may claim work: %w", err) + } + if !live { + // Logged on the TRANSITION rather than on every tick. The state lasts + // for as long as the misconfiguration does, and at one line per poll + // interval it would bury everything else in the deployment's logs; a + // line that scrolls the rest away is a line nobody reads. + if l.unregistered.CompareAndSwap(false, true) { + xlog.Error("This replica is not registered in the cluster, so it will not claim queued work: another replica could not tell its claims from abandoned ones", + "instance", l.owner, "knob", "LOCALAI_DISTRIBUTED_ADVERTISE_ADDR") + } + return nil + } + if l.unregistered.CompareAndSwap(true, false) { + xlog.Info("This replica is registered in the cluster again and is claiming queued work", "instance", l.owner) + } + + if released, err := ReapAbandoned(ctx, l.db, l.liveness); err != nil { + // Reported and not returned: a reap that could not run says nothing + // about the claim this tick is about to take, and giving up here would + // stop dispatch entirely over a read that will very likely succeed next + // tick. + xlog.Warn("Could not reap claims left by departed replicas", "error", err) + } else if released > 0 { + xlog.Info("Released work claimed by replicas that are no longer live", "count", released) + } + + claim, err := ClaimNext(ctx, l.db, l.owner, []ClaimKind{ClaimKindMCPCI, ClaimKindAgentRun, ClaimKindTask}) + if errors.Is(err, ErrNoWork) { + return nil + } + if err != nil { + return err + } + return l.drive(ctx, claim) +} + +// drive hands one claimed row to a worker and settles it. +func (l *DispatchLoop) drive(ctx context.Context, claim *WorkClaim) error { + path, served := verbFor(ClaimKind(claim.Kind)) + if !served { + // A verdict this DEPLOYMENT reached without asking anyone, which is + // still a verdict: no build of any worker serves this kind, so retrying + // it on another worker cannot change the answer. Settled as an answer + // for that reason, and with a reason string, because the alternative is + // the row this change exists to make visible going quiet again. + xlog.Warn("Failing a claim no worker in this deployment serves", "kind", claim.Kind, "claim", claim.ID) + return l.settleClaim(ctx, claim, &ClaimReply{ + JobID: jobIDOf(claim), + Status: "failed", + Error: reasonNoTaskDispatcher, + }, nil) + } + + nodeID, nodeType, err := l.selector.PickConnected(ctx) + if err != nil { + // No worker was reachable. Nothing was asked of anyone and nothing was + // learned, so the work goes back. + return l.settleClaim(ctx, claim, nil, fmt.Errorf("picking an agent worker for a %s claim: %w", claim.Kind, err)) + } + + var reply ClaimReply + callErr := l.control.CallStreaming(ctx, nodeID, path, json.RawMessage(claim.Payload), &reply, + func(subject string, raw json.RawMessage) { + l.rebroadcast(nodeType, subject, raw) + }) + if callErr != nil { + callErr = fmt.Errorf("driving a %s claim on agent worker %q: %w", claim.Kind, nodeID, callErr) + } + return l.settleClaim(ctx, claim, &reply, callErr) +} + +// rebroadcast forwards one progress line's broadcast REQUEST. +// +// A line with no subject is for this caller alone, which is what every +// pre-existing progress tick is, and it is dropped here rather than handed on: +// the allow list denies the empty subject anyway, so passing it would turn +// every ordinary tick into a refusal that gets logged as a worker asking for +// something it has no business on. +// +// The node type travels from the SELECTION rather than being looked up again. +// It is what the allow list is keyed on, so getting it wrong denies everything +// or, worse, allows what should not be, and there must not be a second place in +// the tree that decides what a node's type is. +func (l *DispatchLoop) rebroadcast(nodeType, subject string, raw json.RawMessage) { + if subject == "" || l.broadcast == nil { + return + } + l.broadcast.Handle(nodeType, subject, raw) +} + +// settleClaim is the ONE place the release rule is stated, and every exit path +// through drive calls it. +// +// | Outcome | Action | Why | +// | a reply line came back | persist the terminal line, then | the worker ran it and said what happened | +// | | CompleteClaim | | +// | anything else | ReleaseClaim | an unreachable peer, a lost tunnel or a refused stream is not a verdict | +// +// The line between the two rows is whether a REPLY LINE was decoded, and it is +// deliberately NOT cluster.IsWorkerAnswer, though the plan for this task said +// it should be. IsWorkerAnswer accepts the tunnel's stream-refusal vocabulary, +// which a worker writes BEFORE any request body reaches its control server: +// ErrStreamTagUnknown and ErrStreamTargetUnavailable both mean "I could not +// carry this to my own control plane", which is the opposite of "I ran your job +// and here is what happened". Completing on those would DISCARD work that never +// ran, silently, which is the collapse this whole phase exists to prevent +// pointed at work instead of at nodes. nodes.agentVerb draws the same line for +// the same reason. +// +// The order of the two writes in the first row is the point of the row. The +// terminal line is persisted BEFORE the claim is completed, so a store that +// refuses leaves the claim in place; completing first would delete the only +// record that the work is outstanding and leave the job row `running` for ever, +// which is the dropped-result defect the bus carrier had. +func (l *DispatchLoop) settleClaim(ctx context.Context, claim *WorkClaim, reply *ClaimReply, callErr error) error { + if callErr != nil { + xlog.Warn("Releasing a claim whose dispatch obtained no answer", "claim", claim.ID, "kind", claim.Kind, "error", callErr) + if err := ReleaseClaim(ctx, l.db, claim.ID); err != nil { + return fmt.Errorf("%w (and the claim could not be released: %w)", callErr, err) + } + return callErr + } + if err := l.persistTerminal(reply); err != nil { + // Left CLAIMED on purpose. The work ran, so releasing would run it + // again; the row stays, attributed to this replica, and becomes + // claimable only if this replica dies. That is a visible outstanding + // row rather than a silent loss. + return err + } + return CompleteClaim(ctx, l.db, claim.ID) +} + +// persistTerminal writes the job row's terminal state, if the reply named a job. +// +// An agent run names none: its output reaches the user as SSE events, and there +// is no job row to close. A reply that names a job but no status is still +// closed out, as failed, because the one outcome this change exists to remove +// is a job left `running` after the work has finished. +func (l *DispatchLoop) persistTerminal(reply *ClaimReply) error { + if reply == nil || reply.JobID == "" || l.store == nil { + return nil + } + status, errMsg := reply.Status, reply.Error + if status == "" { + status, errMsg = "failed", "the worker answered without naming a terminal status" + } + if err := l.store.UpdateJobStatus(reply.JobID, status, reply.Result, errMsg); err != nil { + return fmt.Errorf("persisting the terminal state of job %q: %w", reply.JobID, err) + } + return nil +} + +// verbFor maps a claim kind onto the control verb that serves it. +// +// ClaimKindTask maps to nothing, and that is a fact about this deployment +// rather than an omission here: no worker build serves it. See +// reasonNoTaskDispatcher. +func verbFor(kind ClaimKind) (string, bool) { + switch kind { + case ClaimKindMCPCI: + return workerctl.PathMCPCIRun, true + case ClaimKindAgentRun: + return workerctl.PathAgentExecute, true + default: + return "", false + } +} + +// jobIDOf reads the job id out of a claim's payload, for the kinds whose +// payload is a JobEvent. It returns "" for anything else, which the caller +// treats as "there is no job row to close". +func jobIDOf(claim *WorkClaim) string { + var evt JobEvent + if err := json.Unmarshal(claim.Payload, &evt); err != nil { + return "" + } + return evt.JobID +} diff --git a/core/services/jobs/dispatch_loop_test.go b/core/services/jobs/dispatch_loop_test.go new file mode 100644 index 000000000000..007017ebb340 --- /dev/null +++ b/core/services/jobs/dispatch_loop_test.go @@ -0,0 +1,562 @@ +package jobs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/testutil" + "github.com/mudler/LocalAI/core/services/workerctl" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +// fakePicker answers the selection the dispatch loop makes before every RPC. +// +// It holds no call counter and mutates nothing, which is what makes it safe to +// share across the concurrent dispatches the MaxConcurrent specs drive. Its +// siblings below each carry a mutex because each records what it was handed; +// this one is asked the same question by every dispatch and gives the same +// answer, so the cheaper guarantee is that there is nothing to write. +type fakePicker struct { + nodeID string + nodeType string + err error +} + +func (p *fakePicker) PickConnected(context.Context) (string, string, error) { + return p.nodeID, p.nodeType, p.err +} + +// fakeCaller stands in for *nodes.ControlClient. +// +// It is a double and it is deliberately NOT the thing that proves exactly-once: +// that is proved against a real PostgreSQL with real concurrent claimants in +// claim_test.go. What this one exercises is the outcome table, which is a +// mapping from an RPC's result onto a settle action and has no transport in it. +type fakeCaller struct { + mu sync.Mutex + paths []string + nodeIDs []string + bodies []json.RawMessage + progress []workerctl.Envelope + reply *ClaimReply + err error + // onCall fires once the RPC has been recorded, so a spec can drive a + // second dispatch from inside the first. + onCall func() +} + +func (c *fakeCaller) CallStreaming(_ context.Context, nodeID, path string, req, reply any, + onProgress func(subject string, raw json.RawMessage)) error { + c.mu.Lock() + c.paths = append(c.paths, path) + c.nodeIDs = append(c.nodeIDs, nodeID) + if raw, ok := req.(json.RawMessage); ok { + c.bodies = append(c.bodies, raw) + } + lines := append([]workerctl.Envelope(nil), c.progress...) + c.mu.Unlock() + + for _, line := range lines { + if onProgress != nil { + onProgress(line.Subject, line.Progress) + } + } + if c.onCall != nil { + c.onCall() + } + if c.err != nil { + return c.err + } + if out, ok := reply.(*ClaimReply); ok && c.reply != nil { + *out = *c.reply + } + return nil +} + +func (c *fakeCaller) callCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.paths) +} + +// recordingBroadcaster wraps the REAL nodes.Rebroadcaster, so the allow list a +// spec asserts about is the one production uses, while still recording whether +// the loop handed a line over at all. A line that is never handed over and one +// that is handed over and refused are different behaviours of this loop, and a +// double that only counted publishes could not tell them apart. +type recordingBroadcaster struct { + inner *nodes.Rebroadcaster + mu sync.Mutex + seen [][2]string // nodeType, subject +} + +func (r *recordingBroadcaster) Handle(nodeType, subject string, raw json.RawMessage) bool { + r.mu.Lock() + r.seen = append(r.seen, [2]string{nodeType, subject}) + r.mu.Unlock() + return r.inner.Handle(nodeType, subject, raw) +} + +func (r *recordingBroadcaster) handled() [][2]string { + r.mu.Lock() + defer r.mu.Unlock() + return append([][2]string(nil), r.seen...) +} + +// failingStore refuses to record a terminal line, which is the only way to +// observe the ORDER of the two writes settleClaim makes. +type failingStore struct { + mu sync.Mutex + calls []string + err error +} + +func (s *failingStore) UpdateJobStatus(jobID, status, result, errMsg string) error { + s.mu.Lock() + s.calls = append(s.calls, fmt.Sprintf("%s|%s|%s|%s", jobID, status, result, errMsg)) + s.mu.Unlock() + return s.err +} + +func (s *failingStore) recorded() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.calls...) +} + +var _ = Describe("The dispatch loop", func() { + const owner = "inst-dispatcher" + + var ( + db *gorm.DB + ctx context.Context + picker *fakePicker + caller *fakeCaller + bus *testutil.FakeBus + cast *recordingBroadcaster + store *failingStore + ) + + BeforeEach(func() { + db = testutil.SetupTestDB() + ctx = context.Background() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + Expect(MigrateClaims(ctx, db)).To(Succeed()) + // This replica is registered and heartbeating, which is what makes it + // eligible to claim at all. + Expect(cluster.NewRegistry(db).Register(ctx, owner, "10.0.0.1:8080", "v1")).To(Succeed()) + + picker = &fakePicker{nodeID: "agent-1", nodeType: nodes.NodeTypeAgent} + caller = &fakeCaller{reply: &ClaimReply{}} + bus = testutil.NewFakeBus() + cast = &recordingBroadcaster{inner: nodes.NewRebroadcaster(bus)} + store = &failingStore{} + }) + + newLoop := func() *DispatchLoop { + GinkgoHelper() + l, err := NewDispatchLoop(DispatchConfig{ + DB: db, + Owner: owner, + Selector: picker, + Control: caller, + Broadcast: cast, + Store: store, + Interval: 10 * time.Millisecond, + Liveness: time.Minute, + }) + Expect(err).ToNot(HaveOccurred()) + return l + } + + enqueue := func(kind ClaimKind, payload any) string { + GinkgoHelper() + id, err := EnqueueClaim(ctx, db, kind, payload) + Expect(err).ToNot(HaveOccurred()) + return id + } + + claimRows := func() []WorkClaim { + GinkgoHelper() + var rows []WorkClaim + Expect(db.Find(&rows).Error).To(Succeed()) + return rows + } + + Describe("wiring", func() { + DescribeTable("refuses to build without a dependency whose absence has no symptom", + func(mutate func(*DispatchConfig), want string) { + cfg := DispatchConfig{DB: db, Owner: owner, Selector: picker, Control: caller} + mutate(&cfg) + _, err := NewDispatchLoop(cfg) + Expect(err).To(MatchError(ContainSubstring(want))) + }, + Entry("no database", func(c *DispatchConfig) { c.DB = nil }, "no database"), + Entry("no instance id", func(c *DispatchConfig) { c.Owner = "" }, "no instance id"), + Entry("no selector", func(c *DispatchConfig) { c.Selector = nil }, "no way to pick an agent worker"), + Entry("no control transport", func(c *DispatchConfig) { c.Control = nil }, "no control transport"), + ) + + // A dispatched claim is driven synchronously, so a job that runs for + // minutes holds its goroutine for minutes. Without concurrency the + // second queued job in a deployment waits for the first, which the + // queue group this replaces never did. + It("drives more than one claim at a time, so a long job does not stall the queue", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-slow-1"}) + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-slow-2"}) + + // Both handlers park here until this spec releases them, so "two at + // once" is scripted rather than raced. + release := make(chan struct{}) + entered := make(chan struct{}, 4) + caller.onCall = func() { + entered <- struct{}{} + <-release + } + + loop, err := NewDispatchLoop(DispatchConfig{ + DB: db, Owner: owner, Selector: picker, Control: caller, + Broadcast: cast, Store: store, + Interval: 10 * time.Millisecond, Liveness: time.Minute, + MaxConcurrent: 2, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(loop.Start(ctx)).To(Succeed()) + DeferCleanup(func() { + close(release) + loop.Stop() + }) + + Eventually(entered, "10s").Should(Receive()) + Eventually(entered, "10s").Should(Receive(), + "the second claim waited for the first, so one slow job stalls the whole replica") + }) + + It("drives no more than its limit at once, so a replica cannot claim work it is not driving", func() { + for range 3 { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j"}) + } + release := make(chan struct{}) + var inFlight atomic.Int32 + var peak atomic.Int32 + caller.onCall = func() { + now := inFlight.Add(1) + for { + seen := peak.Load() + if now <= seen || peak.CompareAndSwap(seen, now) { + break + } + } + <-release + inFlight.Add(-1) + } + + loop, err := NewDispatchLoop(DispatchConfig{ + DB: db, Owner: owner, Selector: picker, Control: caller, + Broadcast: cast, Store: store, + Interval: time.Millisecond, Liveness: time.Minute, + MaxConcurrent: 2, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(loop.Start(ctx)).To(Succeed()) + DeferCleanup(func() { + close(release) + loop.Stop() + }) + + // Wait for the limit to be reached, then require it not to be + // exceeded: the third row stays claimable rather than being taken + // by a replica with nothing free to drive it. + Eventually(peak.Load, "10s").Should(Equal(int32(2))) + Consistently(peak.Load).Should(Equal(int32(2))) + + var unclaimed int64 + Expect(db.Model(&WorkClaim{}).Where("claimed_at IS NULL").Count(&unclaimed).Error).To(Succeed()) + Expect(unclaimed).To(Equal(int64(1))) + }) + + It("actually runs: Start drives ticks without anything else prodding it", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-start"}) + caller.reply = &ClaimReply{JobID: "j-start", Status: "completed"} + + driven := make(chan struct{}) + var once sync.Once + caller.onCall = func() { once.Do(func() { close(driven) }) } + + loop := newLoop() + Expect(loop.Start(ctx)).To(Succeed()) + DeferCleanup(loop.Stop) + + Eventually(driven, "10s").Should(BeClosed(), "Start returned without ever driving a tick") + }) + }) + + Describe("choosing the verb", func() { + It("drives an mcp-ci claim on the MCP CI run verb", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + caller.reply = &ClaimReply{JobID: "j1", Status: "completed"} + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + Expect(caller.paths).To(Equal([]string{workerctl.PathMCPCIRun})) + Expect(caller.nodeIDs).To(Equal([]string{"agent-1"})) + }) + + It("drives an agent-run claim on the agent execute verb", func() { + enqueue(ClaimKindAgentRun, map[string]string{"agent_name": "a"}) + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + Expect(caller.paths).To(Equal([]string{workerctl.PathAgentExecute})) + }) + + It("sends the claim's payload as the request body, byte for byte", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-body", TaskID: "t-body"}) + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + Expect(caller.bodies).To(HaveLen(1)) + Expect(string(caller.bodies[0])).To(ContainSubstring(`"task_id":"t-body"`)) + }) + + // The pre-existing defect this change SURFACES rather than causes: no + // agent worker has ever served plain task jobs, and a publish onto an + // empty queue group left the job `running` for ever with no trace. + It("fails a plain task claim with a reason instead of leaving it claimed", func() { + enqueue(ClaimKindTask, JobEvent{JobID: "j-plain"}) + + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + + Expect(caller.callCount()).To(BeZero(), "no verb serves this kind, so nothing may be asked of a worker") + Expect(claimRows()).To(BeEmpty(), "the claim must be settled rather than left held") + Expect(store.recorded()).To(ConsistOf( + "j-plain|failed||" + reasonNoTaskDispatcher, + )) + }) + }) + + Describe("the release rule", func() { + It("completes the claim on the worker's own answer, and persists the terminal line", func() { + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-done"}) + caller.reply = &ClaimReply{JobID: "j-done", Status: "completed", Result: "the answer"} + + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + + Expect(claimRows()).To(BeEmpty(), "claim %s survived a worker's answer", id) + Expect(store.recorded()).To(ConsistOf("j-done|completed|the answer|")) + }) + + It("completes the claim on a worker's answer that the job FAILED, because that is still an answer", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-bad"}) + caller.reply = &ClaimReply{JobID: "j-bad", Status: "failed", Error: "the tool refused"} + + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + + Expect(claimRows()).To(BeEmpty(), "a worker that ran the work and failed it must not have the work retried") + Expect(store.recorded()).To(ConsistOf("j-bad|failed||the tool refused")) + }) + + It("releases the claim on a transport failure, and counts the attempt", func() { + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-lost"}) + caller.err = errors.New("the stream ended before its reply line") + + err := newLoop().DispatchOnce(ctx) + Expect(err).To(HaveOccurred()) + + rows := claimRows() + Expect(rows).To(HaveLen(1), "claim %s was completed on a failure that learned nothing", id) + Expect(rows[0].ClaimedAt).To(BeNil()) + Expect(rows[0].ClaimedBy).To(BeEmpty()) + Expect(rows[0].Attempts).To(Equal(1)) + Expect(store.recorded()).To(BeEmpty(), "nothing was learned, so nothing may be written about the job") + }) + + // The tunnel's own refusal vocabulary. cluster.IsWorkerAnswer accepts + // these, and the plan for this task said to complete on them; they mean + // the worker could not carry the request to its own control plane, so + // the work never ran and completing would DISCARD it. + It("releases the claim on a stream refusal the worker's tunnel wrote, rather than completing it", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-refused"}) + caller.err = fmt.Errorf("opening the stream: %w", cluster.ErrStreamTargetUnavailable) + Expect(cluster.IsWorkerAnswer(caller.err)).To(BeTrue(), + "this spec is only meaningful while the sentinel is one IsWorkerAnswer accepts") + + Expect(newLoop().DispatchOnce(ctx)).To(HaveOccurred()) + + rows := claimRows() + Expect(rows).To(HaveLen(1), "work that never reached a control plane was discarded") + Expect(rows[0].ClaimedAt).To(BeNil()) + }) + + It("releases the claim when no agent worker is reachable, without asking anyone", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-nofleet"}) + picker.err = fmt.Errorf("of 0 agent workers registered, none is connected: %w", nodes.ErrNoAgentWorker) + + Expect(newLoop().DispatchOnce(ctx)).To(HaveOccurred()) + + Expect(caller.callCount()).To(BeZero()) + rows := claimRows() + Expect(rows).To(HaveLen(1)) + Expect(rows[0].ClaimedAt).To(BeNil()) + Expect(rows[0].Attempts).To(Equal(1)) + }) + + // The ordering, and the only way to see it: a store that refuses must + // leave the claim in place. Completing first would delete the only + // record that the work is outstanding. + It("persists the terminal line BEFORE completing, so a store that refuses leaves the claim standing", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-unwritable"}) + caller.reply = &ClaimReply{JobID: "j-unwritable", Status: "completed", Result: "r"} + store.err = errors.New("the database refused the update") + + Expect(newLoop().DispatchOnce(ctx)).To(HaveOccurred()) + + Expect(store.recorded()).To(HaveLen(1), "the terminal line was never attempted") + rows := claimRows() + Expect(rows).To(HaveLen(1), "the claim was completed even though its terminal line was never recorded") + Expect(rows[0].ClaimedBy).To(Equal(owner), + "the work ran, so the claim must stay attributed rather than be offered to another replica") + }) + + It("completes a claim whose reply names no job, since an agent run has no job row to close", func() { + enqueue(ClaimKindAgentRun, map[string]string{"agent_name": "a"}) + caller.reply = &ClaimReply{Status: "completed"} + + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + + Expect(claimRows()).To(BeEmpty()) + Expect(store.recorded()).To(BeEmpty()) + }) + + It("closes out a job whose worker answered without naming a status, rather than leaving it running", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-mute"}) + caller.reply = &ClaimReply{JobID: "j-mute"} + + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + + Expect(store.recorded()).To(HaveLen(1)) + Expect(store.recorded()[0]).To(HavePrefix("j-mute|failed|")) + Expect(claimRows()).To(BeEmpty()) + }) + }) + + Describe("a worker's progress lines", func() { + lineOn := func(subject string) workerctl.Envelope { + return workerctl.Envelope{Subject: subject, Progress: json.RawMessage(`{"tick":1}`)} + } + + It("re-broadcasts a line whose subject that worker's type is allowed", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + allowed := messaging.SubjectJobProgress("j1") + caller.progress = []workerctl.Envelope{lineOn(allowed)} + + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + + Expect(cast.handled()).To(Equal([][2]string{{nodes.NodeTypeAgent, allowed}})) + Expect(bus.PublishCount(allowed)).To(Equal(1)) + }) + + It("refuses a line whose subject that worker's type is not allowed", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + forbidden := messaging.SubjectCacheInvalidateModels + caller.progress = []workerctl.Envelope{lineOn(forbidden)} + + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + + Expect(cast.handled()).To(Equal([][2]string{{nodes.NodeTypeAgent, forbidden}}), + "the subject must reach the allow list rather than being dropped or rewritten here") + Expect(bus.PublishCount(forbidden)).To(BeZero()) + }) + + // A line with no subject is for the claiming replica alone, which is + // what every pre-existing progress tick is. Handing it over would make + // the allow list refuse it and log a worker asking for something it has + // no business on, once per tick. + It("does not hand a subjectless line to the broadcaster at all", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + caller.progress = []workerctl.Envelope{lineOn("")} + + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + + Expect(cast.handled()).To(BeEmpty()) + }) + + It("attributes every line to the node type the SELECTION returned", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + picker.nodeType = nodes.NodeTypeBackend // a type allowed no broadcasts at all + allowed := messaging.SubjectJobProgress("j1") + caller.progress = []workerctl.Envelope{lineOn(allowed)} + + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + + Expect(cast.handled()).To(Equal([][2]string{{nodes.NodeTypeBackend, allowed}})) + Expect(bus.PublishCount(allowed)).To(BeZero(), + "a hardcoded node type here would let a worker broadcast on subjects its own type is denied") + }) + }) + + Describe("a replica that is not observable to its peers", func() { + // The other half of the reap. A replica with no live row in the + // instances table cannot have its claims told from abandoned ones, so + // another replica would take the work away from it mid-run. + It("claims nothing while its own instance row has aged out", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + ageInstance(db, owner, 10*time.Minute) + + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + + Expect(caller.callCount()).To(BeZero()) + rows := claimRows() + Expect(rows).To(HaveLen(1)) + Expect(rows[0].ClaimedAt).To(BeNil(), "the work must stay claimable by a replica that can be reaped") + }) + + It("starts claiming again once its instance row is back", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + ageInstance(db, owner, 10*time.Minute) + loop := newLoop() + Expect(loop.DispatchOnce(ctx)).To(Succeed()) + Expect(caller.callCount()).To(BeZero()) + + Expect(cluster.NewRegistry(db).Heartbeat(ctx, owner)).To(Succeed()) + Expect(loop.DispatchOnce(ctx)).To(Succeed()) + Expect(caller.callCount()).To(Equal(1)) + }) + }) + + Describe("reaping while it dispatches", func() { + It("returns work a departed replica was holding, and takes it on the same tick", func() { + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-orphan"}) + Expect(cluster.NewRegistry(db).Register(ctx, "inst-dead", "10.0.0.9:8080", "v1")).To(Succeed()) + _, err := ClaimNext(ctx, db, "inst-dead", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + ageInstance(db, "inst-dead", 10*time.Minute) + caller.reply = &ClaimReply{JobID: "j-orphan", Status: "completed"} + + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + + Expect(caller.callCount()).To(Equal(1), "claim %s stayed held by a replica that is gone", id) + Expect(claimRows()).To(BeEmpty()) + }) + + It("does not take work a live replica is still holding", func() { + enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-busy"}) + Expect(cluster.NewRegistry(db).Register(ctx, "inst-busy", "10.0.0.8:8080", "v1")).To(Succeed()) + _, err := ClaimNext(ctx, db, "inst-busy", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + ageClaim(db, "j-busy-irrelevant", time.Hour) // no-op: proves nothing is keyed on age + + Expect(newLoop().DispatchOnce(ctx)).To(Succeed()) + + Expect(caller.callCount()).To(BeZero()) + rows := claimRows() + Expect(rows).To(HaveLen(1)) + Expect(rows[0].ClaimedBy).To(Equal("inst-busy")) + }) + }) +}) diff --git a/core/services/jobs/dispatcher.go b/core/services/jobs/dispatcher.go index a1da792ed906..4d1f9222a297 100644 --- a/core/services/jobs/dispatcher.go +++ b/core/services/jobs/dispatcher.go @@ -2,13 +2,11 @@ package jobs import ( "context" - "errors" "fmt" "time" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/advisorylock" - "github.com/mudler/LocalAI/pkg/concurrency" "github.com/mudler/LocalAI/core/services/dbutil" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/xlog" @@ -16,7 +14,8 @@ import ( "gorm.io/gorm" ) -// JobEvent is the NATS message payload for job distribution. +// JobEvent is the payload of a task or mcp-ci claim, and the request body of +// the control verb that carries it to a worker. type JobEvent struct { JobID string `json:"job_id"` TaskID string `json:"task_id"` @@ -28,7 +27,8 @@ type JobEvent struct { ModelConfig *config.ModelConfig `json:"model_config,omitempty"` // included so agent workers don't need API access for model config } -// ProgressEvent is the NATS message payload for progress updates. +// ProgressEvent is the payload of a progress broadcast, published by a worker +// as a re-broadcast REQUEST and by a frontend directly. type ProgressEvent struct { JobID string `json:"job_id"` Status string `json:"status,omitempty"` @@ -40,9 +40,12 @@ type ProgressEvent struct { TraceContent string `json:"trace_content,omitempty"` // trace payload } -// JobResultEvent is the NATS message for the final job result (terminal state). -// Published by the worker when execution finishes. The frontend subscribes and -// persists the result to PostgreSQL. +// JobResultEvent is the broadcast form of a terminal job result. +// +// It is no longer how the result REACHES the frontend: that is the reply line of +// the control verb, which the claiming replica persists before it releases the +// claim. This is the fan-out copy, so an SSE stream open on a replica that +// claimed nothing still sees the job finish. type JobResultEvent struct { JobID string `json:"job_id"` Status string `json:"status"` // "completed", "failed", "cancelled" @@ -50,18 +53,18 @@ type JobResultEvent struct { Error string `json:"error,omitempty"` } -// CancelEvent is the NATS message payload for job cancellation. +// CancelEvent is the broadcast payload for job cancellation. type CancelEvent struct { JobID string `json:"job_id"` } -// WorkerFunc is the function signature for processing a job. -// It receives the job record, task record, and a context that will be cancelled -// if the job is cancelled via NATS. -type WorkerFunc func(ctx context.Context, job *JobRecord, task *TaskRecord) error - -// Dispatcher distributes jobs across instances via NATS queue groups -// and coordinates cron execution via PostgreSQL advisory locks. +// Dispatcher enqueues jobs as claim rows, bridges the progress and result +// broadcasts a worker asks for onto the database, and coordinates cron +// execution via PostgreSQL advisory locks. +// +// It no longer CONSUMES anything. Dispatch is a claim on the job store (see +// DispatchLoop), so this type has no worker function, no queue subscription and +// no concurrency limiter: there is nothing here to limit. type Dispatcher struct { store *JobStore nats messaging.MessagingClient @@ -69,39 +72,27 @@ type Dispatcher struct { instanceID string configLoader ModelConfigLoader // optional: to enrich job events with model config - // Worker function (set by the application) - workerFn WorkerFunc - // Cancel registry (notetaker pattern) cancelRegistry messaging.CancelRegistry // NATS subscriptions - jobSub messaging.Subscription cancelSub messaging.Subscription resultSub messaging.Subscription progressSub messaging.Subscription - // Concurrency limiter; nil = unlimited - sem chan struct{} - // Lifecycle ctx context.Context cancel context.CancelFunc } // NewDispatcher creates a new distributed job Dispatcher. -// maxConcurrent limits the number of concurrent job goroutines; 0 means unlimited. -func NewDispatcher(store *JobStore, nc messaging.MessagingClient, db *gorm.DB, instanceID string, maxConcurrent int) *Dispatcher { - d := &Dispatcher{ +func NewDispatcher(store *JobStore, nc messaging.MessagingClient, db *gorm.DB, instanceID string) *Dispatcher { + return &Dispatcher{ store: store, nats: nc, db: db, instanceID: instanceID, } - if maxConcurrent > 0 { - d.sem = make(chan struct{}, maxConcurrent) - } - return d } // ModelConfigLoader loads model configurations by name. @@ -109,29 +100,17 @@ type ModelConfigLoader interface { GetModelConfig(name string) (config.ModelConfig, bool) } -// NewWorkerDispatcher creates a dispatcher that also consumes and processes jobs. -// Use this instead of NewDispatcher + SetWorkerFunc + SetModelConfigLoader when both -// the worker function and config loader are available at construction time. -func NewWorkerDispatcher(store *JobStore, nc messaging.MessagingClient, db *gorm.DB, instanceID string, maxConcurrent int, workerFn WorkerFunc, configLoader ModelConfigLoader) *Dispatcher { - d := NewDispatcher(store, nc, db, instanceID, maxConcurrent) - d.workerFn = workerFn - d.configLoader = configLoader - return d -} - -// SetWorkerFunc sets the function that processes jobs. -// Deprecated: prefer NewWorkerDispatcher when the worker function is available at construction time. -func (d *Dispatcher) SetWorkerFunc(fn WorkerFunc) { - d.workerFn = fn -} - // SetModelConfigLoader sets the model config loader for enriching job events. -// Deprecated: prefer NewWorkerDispatcher when the config loader is available at construction time. func (d *Dispatcher) SetModelConfigLoader(cl ModelConfigLoader) { d.configLoader = cl } -// Start begins listening for jobs via NATS and starts the cron leader loop. +// Start subscribes to the job broadcasts and starts the cron leader loop. +// +// It no longer listens for JOBS. Dispatch is a claim on the job store (see +// DispatchLoop); what this subscribes to is the fan-out of a worker's cancels, +// results and traces, which every replica must see because the SSE stream a +// user is watching may be open on a replica that claimed nothing. func (d *Dispatcher) Start(ctx context.Context) error { d.ctx, d.cancel = context.WithCancel(ctx) success := false @@ -141,24 +120,17 @@ func (d *Dispatcher) Start(ctx context.Context) error { } }() - // Subscribe to job queue only if a worker function is configured. - // In distributed mode, the frontend dispatcher publishes jobs but does not consume them — - // agent workers pick them up from the same NATS queue. + // No job-queue subscription. Dispatch is a claim on the job store, taken by + // a frontend replica's DispatchLoop and driven on an agent worker over that + // worker's own tunnel, so there is no subject for this dispatcher to + // consume and no queue group left to be one of. + // + // What remains here are the two BROADCAST subscriptions below. A worker's + // progress and result lines still reach every replica, because an SSE + // stream may be open on a replica that did not claim the work; they arrive + // as re-broadcasts of the lines the claiming replica read off the response + // body (see nodes.Rebroadcaster), rather than as publishes from a worker. var err error - if d.workerFn != nil { - d.jobSub, err = messaging.QueueSubscribeJSON(d.nats, messaging.SubjectJobsNew, messaging.QueueWorkers, func(evt JobEvent) { - concurrency.SafeGo(func() { - if d.sem != nil { - d.sem <- struct{}{} - defer func() { <-d.sem }() - } - d.processJob(evt) - }) - }) - if err != nil { - return fmt.Errorf("subscribing to job queue: %w", err) - } - } // Subscribe to cancel events (broadcast to all — each instance checks its registry) d.cancelSub, err = messaging.SubscribeJSON(d.nats, messaging.SubjectJobCancelWildcard, func(evt CancelEvent) { @@ -203,10 +175,6 @@ func (d *Dispatcher) Start(ctx context.Context) error { // unsubscribeAll nil-checks, unsubscribes, and nils out each NATS subscription. // Safe to call multiple times. func (d *Dispatcher) unsubscribeAll() { - if d.jobSub != nil { - d.jobSub.Unsubscribe() - d.jobSub = nil - } if d.cancelSub != nil { d.cancelSub.Unsubscribe() d.cancelSub = nil @@ -229,9 +197,17 @@ func (d *Dispatcher) Stop() { d.unsubscribeAll() } -// Enqueue publishes a job to the NATS queue for distributed processing. -// The event is enriched with the full Job and Task records so that the -// worker does not need direct database access. +// Enqueue writes ONE claim row for a job. +// +// It replaces a publish onto one of two queue subjects, and the difference is +// the point of the change: a publish to a queue group nobody is subscribed to +// succeeds and the job stays `pending` for ever with no trace, while a claim +// row nobody claims is still in the table. The kind the row carries is what the +// subject used to choose. +// +// The event is enriched with the full Job and Task records for the same reason +// it always was: an agent worker has no database access, so everything it needs +// travels with the work. func (d *Dispatcher) Enqueue(jobID, taskID, userID string) error { evt := JobEvent{ JobID: jobID, @@ -255,12 +231,20 @@ func (d *Dispatcher) Enqueue(jobID, taskID, userID string) error { } } - subject := messaging.SubjectJobsNew + kind := ClaimKindTask if evt.ModelConfig != nil && evt.ModelConfig.MCP.HasMCPServers() { - subject = messaging.SubjectMCPCIJobsNew + kind = ClaimKindMCPCI } - return d.nats.Publish(subject, evt) + // Background rather than this dispatcher's own lifecycle context, and + // deliberately. Enqueue is reachable from an HTTP handler on any goroutine, + // while d.ctx is written by Start, so reading it here would be a data race; + // and borrowing it would make a queued job's fate depend on whether the + // dispatcher happened to be shutting down. It is one INSERT. + if _, err := EnqueueClaim(context.Background(), d.db, kind, evt); err != nil { + return err + } + return nil } // Cancel publishes a cancel event to NATS (broadcast to all instances). @@ -284,116 +268,6 @@ func (d *Dispatcher) SubscribeProgress(jobID string, handler func(ProgressEvent) return messaging.SubscribeJSON(d.nats, messaging.SubjectJobProgress(jobID), handler) } -// processJob is called by the NATS queue subscriber to execute a job. -// It prefers Job+Task from the enriched NATS payload (no DB needed). -// Results are published back via NATS for the frontend to persist. -func (d *Dispatcher) processJob(evt JobEvent) { - if d.workerFn == nil { - xlog.Error("No worker function set for job dispatcher") - d.publishResult(evt.JobID, "failed", "", "no worker function configured") - return - } - - // Prefer enriched payload; fall back to DB for backward compat - job := evt.Job - if job == nil && d.store != nil { - var err error - job, err = d.store.GetJob(evt.JobID) - if err != nil { - xlog.Error("Failed to load job", "jobID", evt.JobID, "error", err) - return - } - } - if job == nil { - xlog.Error("No job data available", "jobID", evt.JobID) - return - } - - task := evt.Task - if task == nil && d.store != nil { - var err error - task, err = d.store.GetTask(job.TaskID) - if err != nil { - xlog.Error("Failed to load task for job", "jobID", evt.JobID, "taskID", job.TaskID, "error", err) - d.publishResult(evt.JobID, "failed", "", "task not found") - return - } - } - if task == nil { - xlog.Error("No task data available", "jobID", evt.JobID) - d.publishResult(evt.JobID, "failed", "", "task not found") - return - } - - // Pre-register so cancels arriving before context creation are captured - cancelled := make(chan struct{}, 1) - d.cancelRegistry.Register(evt.JobID, func() { - select { - case cancelled <- struct{}{}: - default: - } - }) - - ctx, cancelFn := context.WithCancel(d.ctx) - d.cancelRegistry.Register(evt.JobID, cancelFn) // overwrite with real cancel - - // Check if cancel arrived during the registration window - select { - case <-cancelled: - cancelFn() - default: - } - - // Check if job was cancelled in the DB before we picked it up - if d.store != nil { - if dbJob, err := d.store.GetJob(evt.JobID); err == nil && dbJob.Status == "cancelled" { - cancelFn() - } - } - - defer func() { - d.cancelRegistry.Deregister(evt.JobID) - cancelFn() - }() - - // Check if already cancelled before starting - select { - case <-ctx.Done(): - d.publishResult(evt.JobID, "cancelled", "", "") - return - default: - } - - // Mark as running - job.FrontendID = d.instanceID - d.PublishProgress(evt.JobID, "running", "Job started") - - // Execute - err := d.workerFn(ctx, job, task) - - if errors.Is(ctx.Err(), context.Canceled) { - d.publishResult(evt.JobID, "cancelled", "", "") - d.PublishProgress(evt.JobID, "cancelled", "Job cancelled") - return - } - - if err != nil { - d.publishResult(evt.JobID, "failed", "", err.Error()) - d.PublishProgress(evt.JobID, "failed", err.Error()) - return - } - - // Publish completion — result is set on the job by workerFn - d.publishResult(evt.JobID, "completed", job.Result, "") - d.PublishProgress(evt.JobID, "completed", "Job completed") -} - -// publishResult publishes the terminal job result via NATS. -// The frontend subscribes to these events and persists to DB. -func (d *Dispatcher) publishResult(jobID, status, result, errMsg string) { - PublishJobResult(d.nats, jobID, status, result, errMsg) -} - // PublishTrace publishes a trace event for a running job via NATS. // The frontend subscribes and persists traces to DB. func (d *Dispatcher) PublishTrace(jobID, traceType, traceContent string) error { @@ -454,7 +328,9 @@ func (d *Dispatcher) runDueCronTasks() { if err := d.Enqueue(job.ID, task.ID, task.UserID); err != nil { xlog.Error("Failed to enqueue cron job, marking as failed", "jobID", job.ID, "error", err) - d.store.UpdateJobStatus(job.ID, "failed", "", "NATS enqueue failed: "+err.Error()) + if uerr := d.store.UpdateJobStatus(job.ID, "failed", "", "enqueueing the job failed: "+err.Error()); uerr != nil { + xlog.Error("Failed to mark an unqueueable cron job as failed", "jobID", job.ID, "error", uerr) + } } else { xlog.Info("Cron job enqueued", "taskID", task.ID, "jobID", job.ID) } diff --git a/core/services/jobs/dispatcher_test.go b/core/services/jobs/dispatcher_test.go index 0af251fdd6a6..9628598c25ba 100644 --- a/core/services/jobs/dispatcher_test.go +++ b/core/services/jobs/dispatcher_test.go @@ -10,6 +10,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/testutil" + "gorm.io/gorm" ) // publishCall records a single Publish invocation. @@ -83,7 +84,7 @@ var _ = Describe("Dispatcher", func() { store, err = NewJobStore(db) Expect(err).ToNot(HaveOccurred()) - disp = NewDispatcher(store, nil, db, "test-instance", 0) + disp = NewDispatcher(store, nil, db, "test-instance") }) It("returns true when no previous job exists", func() { @@ -209,25 +210,32 @@ var _ = Describe("Dispatcher", func() { }) // ----------------------------------------------------------------------- - // Enqueue — test NATS subject routing via real Dispatcher.Enqueue() + // Enqueue: the kind of claim row Dispatcher.Enqueue() writes. + // + // It used to choose between two NATS subjects. The choice is the same + // choice; what it now decides is the KIND stored on the row, which is what + // the dispatch loop maps onto a control verb. Asserted against the row + // rather than a publish because a publish to a queue group nobody joined + // succeeds, and that is the failure this change exists to remove. // ----------------------------------------------------------------------- - Describe("Enqueue subject routing", func() { + Describe("Enqueue claim kind", func() { var ( store *JobStore fake *fakeMessagingClient disp *Dispatcher + db *gorm.DB ) BeforeEach(func() { - db := testutil.SetupTestDB() + db = testutil.SetupTestDB() var err error store, err = NewJobStore(db) Expect(err).ToNot(HaveOccurred()) fake = &fakeMessagingClient{} - disp = NewDispatcher(store, fake, db, "test-instance", 0) + disp = NewDispatcher(store, fake, db, "test-instance") }) - It("routes MCP jobs to SubjectMCPCIJobsNew", func() { + It("writes an mcp-ci claim for a model with MCP servers", func() { task := &TaskRecord{ UserID: "user-1", Name: "mcp-task", @@ -256,11 +264,11 @@ var _ = Describe("Dispatcher", func() { Expect(disp.Enqueue(job.ID, task.ID, "user-1")).To(Succeed()) - Expect(fake.calls).To(HaveLen(1)) - Expect(fake.calls[0].subject).To(Equal(messaging.SubjectMCPCIJobsNew)) + Expect(onlyClaim(db).Kind).To(Equal(string(ClaimKindMCPCI))) + Expect(fake.calls).To(BeEmpty(), "enqueueing must not publish: a queue subject nobody joined swallows the job") }) - It("routes non-MCP jobs to SubjectJobsNew", func() { + It("writes a plain task claim for a model without MCP servers", func() { task := &TaskRecord{ UserID: "user-1", Name: "plain-task", @@ -285,11 +293,11 @@ var _ = Describe("Dispatcher", func() { Expect(disp.Enqueue(job.ID, task.ID, "user-1")).To(Succeed()) - Expect(fake.calls).To(HaveLen(1)) - Expect(fake.calls[0].subject).To(Equal(messaging.SubjectJobsNew)) + Expect(onlyClaim(db).Kind).To(Equal(string(ClaimKindTask))) + Expect(fake.calls).To(BeEmpty(), "enqueueing must not publish: a queue subject nobody joined swallows the job") }) - It("routes to SubjectJobsNew when model config is not found", func() { + It("writes a plain task claim when the model config is not found", func() { task := &TaskRecord{ UserID: "user-1", Name: "unknown-model-task", @@ -312,28 +320,29 @@ var _ = Describe("Dispatcher", func() { Expect(disp.Enqueue(job.ID, task.ID, "user-1")).To(Succeed()) - Expect(fake.calls).To(HaveLen(1)) - Expect(fake.calls[0].subject).To(Equal(messaging.SubjectJobsNew)) + Expect(onlyClaim(db).Kind).To(Equal(string(ClaimKindTask))) + Expect(fake.calls).To(BeEmpty(), "enqueueing must not publish: a queue subject nobody joined swallows the job") }) }) // ----------------------------------------------------------------------- - // Enqueue event enrichment — verify the payload published by Enqueue() + // Enqueue event enrichment: the payload stored on the claim row // ----------------------------------------------------------------------- Describe("Enqueue event enrichment", func() { var ( store *JobStore fake *fakeMessagingClient disp *Dispatcher + db *gorm.DB ) BeforeEach(func() { - db := testutil.SetupTestDB() + db = testutil.SetupTestDB() var err error store, err = NewJobStore(db) Expect(err).ToNot(HaveOccurred()) fake = &fakeMessagingClient{} - disp = NewDispatcher(store, fake, db, "test-instance", 0) + disp = NewDispatcher(store, fake, db, "test-instance") }) It("includes full job and task records in the event", func() { @@ -368,9 +377,7 @@ var _ = Describe("Dispatcher", func() { Expect(disp.Enqueue(job.ID, task.ID, "user-1")).To(Succeed()) - Expect(fake.calls).To(HaveLen(1)) - evt, ok := fake.calls[0].data.(JobEvent) - Expect(ok).To(BeTrue(), "published data should be a JobEvent") + evt := claimEvent(onlyClaim(db)) Expect(evt.Job).ToNot(BeNil()) Expect(evt.Job.ID).To(Equal(job.ID)) Expect(evt.Task).ToNot(BeNil()) @@ -399,9 +406,7 @@ var _ = Describe("Dispatcher", func() { // No config loader — Enqueue still works, just no model config enrichment. Expect(disp.Enqueue(job.ID, task.ID, "user-1")).To(Succeed()) - Expect(fake.calls).To(HaveLen(1)) - evt, ok := fake.calls[0].data.(JobEvent) - Expect(ok).To(BeTrue()) + evt := claimEvent(onlyClaim(db)) data, err := json.Marshal(evt) Expect(err).ToNot(HaveOccurred()) @@ -413,3 +418,21 @@ var _ = Describe("Dispatcher", func() { }) }) }) + +// onlyClaim returns the single claim row Enqueue wrote, failing the spec if it +// wrote any other number. +func onlyClaim(db *gorm.DB) WorkClaim { + GinkgoHelper() + var rows []WorkClaim + Expect(db.Find(&rows).Error).To(Succeed()) + Expect(rows).To(HaveLen(1), "expected exactly one claim row") + return rows[0] +} + +// claimEvent decodes the JobEvent a claim row carries. +func claimEvent(claim WorkClaim) JobEvent { + GinkgoHelper() + var evt JobEvent + Expect(json.Unmarshal(claim.Payload, &evt)).To(Succeed()) + return evt +} diff --git a/core/services/jobs/store.go b/core/services/jobs/store.go index 2adf2da8d88c..08fa83d8f367 100644 --- a/core/services/jobs/store.go +++ b/core/services/jobs/store.go @@ -71,6 +71,13 @@ func NewJobStore(db *gorm.DB) (*JobStore, error) { }); err != nil { return nil, fmt.Errorf("migrating job tables: %w", err) } + // The claim queue's table, migrated here rather than by whoever happens to + // dispatch first. Enqueue writes a claim row on every job, so a deployment + // whose job store came up without this table accepts jobs and dispatches + // none, which is the exact failure the row was introduced to make visible. + if err := MigrateClaims(context.Background(), db); err != nil { + return nil, err + } return &JobStore{db: db}, nil } diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index 6b2985387ed8..229bba93cc9b 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -12,13 +12,19 @@ func sanitizeSubjectToken(s string) string { // NATS subject constants for the distributed architecture. // Following the notetaker pattern: . -// Job Distribution (Queue Groups — load-balanced, one consumer gets each message) -const ( - SubjectJobsNew = "jobs.new" - SubjectMCPCIJobsNew = "jobs.mcp-ci.new" - SubjectAgentExecute = "agent.execute" - QueueWorkers = "workers" -) +// Job Distribution +// +// There is no subject here and no queue group left to name. Dispatching work is +// a CLAIM on the job store: jobs.EnqueueClaim writes a row, one frontend +// replica takes it with SELECT ... FOR UPDATE SKIP LOCKED, and it drives the +// work on an agent worker over that worker's tunnel. A queue group was never a +// broker feature this design has to reproduce; what it did was deliver one +// message to exactly one of several competing consumers, which is a row and a +// lock. +// +// The difference is not only the carrier. A publish onto a queue group nobody +// had joined SUCCEEDED, so a deployment with no agent worker accepted jobs and +// silently ran none; a claim row nobody takes is still in the table. // Status Updates (Pub/Sub — all subscribers get every message, for SSE bridging) // These use parameterized subjects: e.g. SubjectAgentEvents("myagent", "user1") diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index c8c55fe5f4d5..015a79a7ace4 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -145,7 +145,23 @@ Both **backend** and **agent** nodes are issued one. Earlier releases minted a c An agent worker's tunnel carries only the `http` tag: it runs no backend processes, so it does not offer the `grpc` tag at all. Its control server binds `127.0.0.1` on a port chosen by the kernel and advertises it nowhere, so an agent worker still opens no inbound port. -**An agent worker still requires `--nats-url`.** The tunnel is not yet a replacement: agent jobs and MCP CI jobs still travel on NATS. Nothing the frontend addresses to a specific agent worker does any more. MCP tool execution and MCP discovery are control RPCs on the tunnel, chosen by the frontend rather than by a queue group (see [MCP in Distributed Mode](#mcp-in-distributed-mode)), and so is the backend stop that flushes an agent worker's cached MCP sessions. There is no `nodes..*` subject left, and an agent worker's minted JWT no longer grants `mcp.tools.execute`, `mcp.discovery` or `nodes..backend.stop`. +**An agent worker still requires `--nats-url`, and nothing the frontend sends it travels on the bus.** Every verb the frontend addresses to a specific agent worker is now a control RPC on the tunnel that worker holds: MCP tool execution, MCP discovery, the backend stop that flushes cached MCP sessions, agent execution, and MCP CI runs. What the bus still carries for an agent worker is the other direction and the broadcasts: cancellation, and the progress and result lines a worker asks the frontend to re-publish on its behalf. + +There is no `nodes..*` subject left, and an agent worker's minted JWT no longer grants `mcp.tools.execute`, `mcp.discovery`, `nodes..backend.stop`, `agent.execute` or `jobs.mcp-ci.new`. The `--agent-subject` and `--agent-queue` flags (`LOCALAI_AGENT_SUBJECT`, `LOCALAI_AGENT_QUEUE`) are gone: there is no subject for an agent worker to subscribe to and no queue group to be one of. + +### Dispatch is a claim, not a queue group + +Queued work (agent runs, MCP CI jobs, plain task jobs) is written to a `work_claims` table rather than published onto a NATS queue group. A frontend replica takes one row at a time with `SELECT ... FOR UPDATE SKIP LOCKED`, picks a connected agent worker, and drives the work as a streaming control RPC over that worker's tunnel. The worker's progress, its agent events and its terminal result all arrive on the response body the claiming replica is already reading, and that replica persists the terminal line **before** it releases the claim. + +This changes three behaviours an operator can see: + +- **A job with nowhere to run is now visible.** A publish onto a queue group nobody had joined succeeded, and the job sat `pending` for ever with no trace. A claim row that nothing takes is still in the table. +- **A plain task job (a task whose model configures no MCP servers) is now failed with a reason.** No agent worker has ever served that kind of job, and it used to be published into silence. It is now marked `failed` with `no worker in this deployment serves plain task jobs`. This surfaces a pre-existing gap rather than introducing one. +- **Dispatch is at-least-once instead of at-most-once.** A transport failure (a lost tunnel, an unreachable peer, a replica that died mid-dispatch) returns the work to the pool and increments the row's `attempts`; only the worker's own answer, success or failure, removes it. + +**A claim held by a replica that has gone becomes claimable again; a claim held by a replica that is merely slow is never taken away from it.** The reap asks whether the claim's owner is still a live replica in the `instances` table, on the database clock, and never how long the claim has been held: a job that legitimately runs for an hour on a heartbeating replica is left alone, and a claim whose owner stopped heartbeating is released on the next tick (within `30s`, the replica-liveness window). + +**A frontend replica with no advertised peer address claims no work.** Such a replica has no row in the `instances` table, so no peer can tell its claims from ones a dead replica left, and another replica would take the work away from it mid-run. It logs an error naming `LOCALAI_DISTRIBUTED_ADVERTISE_ADDR` and starts claiming as soon as it registers. This is the same configuration that already makes a replica's workers unroutable from its peers. The tunnel lands on exactly one frontend replica, and that replica records itself as the owner of the worker's connection in the `node_connections` table. When the socket dies the claim is dropped, but the row stays behind with no owner and a `disconnected_at` stamp, so a worker that is re-dialling the load balancer can be told from one that has never connected. The row is deleted once that departure is older than ten liveness windows (five minutes). If the replica stalls long enough for its peers to reap it, it re-claims the tunnels it still holds on a live session as soon as it re-registers, skipping any whose socket has already gone. That re-claim needs the replica to have an advertised address: without one it never had an instance row to begin with, and its tunnels are usable only by the replica holding them. diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md index e5c75c49b656..6d9aca0e1507 100644 --- a/docs/content/reference/cli-reference.md +++ b/docs/content/reference/cli-reference.md @@ -224,7 +224,7 @@ LocalAI supports several subcommands beyond `run`: - `local-ai mcp-server` - Run the LocalAI admin tool surface as a stdio MCP server (controls a remote LocalAI instance over HTTP) - `local-ai worker` - Start a worker for distributed mode (generic, backend-agnostic; needs only an outbound route to the frontend, no message bus) - `local-ai p2p-worker` - Run workers to distribute workload via p2p (llama.cpp-only) -- `local-ai agent-worker` - Start an agent worker for distributed mode (executes agent chats via NATS, which this command still needs) +- `local-ai agent-worker` - Start an agent worker for distributed mode (serves agent execution and MCP CI runs as control verbs on the tunnel it holds; still needs `--nats-url` for cancellation and event broadcasts) - `local-ai util` - Utility commands - `local-ai explorer` - Run P2P explorer - `local-ai federated` - Run LocalAI in federated mode diff --git a/pkg/natsauth/mint_test.go b/pkg/natsauth/mint_test.go index 7c849cf16203..f968613574c3 100644 --- a/pkg/natsauth/mint_test.go +++ b/pkg/natsauth/mint_test.go @@ -59,7 +59,7 @@ var _ = Describe("MintWorkerJWT", func() { uc, err := jwt.DecodeUserClaims(token) Expect(err).NotTo(HaveOccurred()) - Expect(uc.Permissions.Sub.Allow).To(ContainElement("agent.execute")) + Expect(uc.Permissions.Sub.Allow).To(ContainElement("agent.*.cancel")) for _, subj := range uc.Permissions.Sub.Allow { Expect(subj).NotTo(ContainSubstring("backend.install")) } diff --git a/pkg/natsauth/permissions.go b/pkg/natsauth/permissions.go index 77c0717cba6b..c64bdcd882df 100644 --- a/pkg/natsauth/permissions.go +++ b/pkg/natsauth/permissions.go @@ -30,24 +30,24 @@ func workerSubjectToken(nodeID string) string { func WorkerPermissions(nodeID, nodeType string) (pubAllow, subAllow []string) { switch nodeType { case "agent": - // Agent workers consume queue workloads; they must not handle backend.install. // Keep this list in sync with the subscriptions in core/cli/agent_worker.go. // - // MCP tool execution and discovery are NOT here, and neither is the - // per-node backend.stop: all three are control RPCs on the tunnel the - // worker holds, addressed by the frontend rather than by a subject. + // MCP tool execution, discovery, agent execution, MCP CI runs and the + // per-node backend.stop are all absent: every one of them is a control + // RPC on the tunnel the worker holds, addressed by the frontend rather + // than by a subject. The last two left when the queue groups became + // claim rows on the job store. + // // Removing them narrowed this list; it must never be narrowed to // nothing, because NATS reads an EMPTY allow list as no restriction at // all, which would widen an agent JWT to the whole account. subAllow = []string{ - "agent.execute", "agent.*.cancel", "gallery.*.cancel", "gallery.*.progress", "jobs.*.cancel", "jobs.*.progress", "jobs.*.result", - "jobs.mcp-ci.new", // MCP CI jobs dispatched to agent workers "staging.*.progress", "_INBOX.>", } diff --git a/pkg/natsauth/permissions_coverage_test.go b/pkg/natsauth/permissions_coverage_test.go index dbf18f049aef..eed0ee5b07ef 100644 --- a/pkg/natsauth/permissions_coverage_test.go +++ b/pkg/natsauth/permissions_coverage_test.go @@ -100,9 +100,12 @@ var _ = Describe("WorkerPermissions subject coverage", func() { // node_type "agent"; subjects from core/cli/agent_worker.go. pub, sub := natsauth.WorkerPermissions(nodeID, "agent") + // The subjects an agent worker still subscribes to. Both queue-group + // workloads have left this list: agent execution and MCP CI runs are + // streaming control verbs on the tunnel now, driven by a claim a + // frontend replica took off the job store. subscribed := []string{ - messaging.SubjectAgentExecute, // dispatcher (default --agent-subject) - messaging.SubjectMCPCIJobsNew, // QueueSubscribe — jobs.mcp-ci.new + messaging.SubjectAgentCancelWildcard, } // The half that catches a narrowing going too far. NATS reads an EMPTY @@ -115,17 +118,19 @@ var _ = Describe("WorkerPermissions subject coverage", func() { "an empty allow list is unrestricted in NATS, not restrictive") }) - // MCP execution and discovery, and now backend.stop too, are control - // RPCs on the worker's tunnel, addressed by the frontend rather than - // by a subject. An agent worker subscribes to none of them, so a JWT - // that still granted one would be granting a subscription nothing - // serves. The backend.stop entry is spelled out by hand rather than - // built from a deleted builder, which is the only way this can still - // name the subject that used to be granted. + // MCP execution and discovery, backend.stop, and now agent execution + // and MCP CI runs, are all control RPCs on the worker's tunnel, + // addressed by the frontend rather than by a subject. An agent worker + // subscribes to none of them, so a JWT that still granted one would be + // granting a subscription nothing serves. Every entry is spelled out by + // hand rather than built from a deleted constant, which is the only way + // this can still name the subject that used to be granted. for _, subject := range []string{ "mcp.tools.execute", "mcp.discovery", "nodes." + workerSubjectTokenForTest(nodeID) + ".backend.stop", + "agent.execute", + "jobs.mcp-ci.new", } { It("no longer grants an agent worker "+subject, func() { Expect(anyAllows(sub, subject)).To(BeFalse(), @@ -138,9 +143,9 @@ var _ = Describe("WorkerPermissions subject coverage", func() { // NATS. Naming what must survive is what tells a narrowing from an // escalation, so the queue workloads an agent worker still consumes are // asserted present just above, and the list is asserted non-empty here. - It("still grants the agent worker the queue subjects it lives on", func() { - Expect(sub).To(ContainElement("agent.execute")) - Expect(sub).To(ContainElement("jobs.mcp-ci.new")) + It("still grants the agent worker the broadcast subjects it lives on", func() { + Expect(sub).To(ContainElement("agent.*.cancel")) + Expect(sub).To(ContainElement("jobs.*.progress")) Expect(sub).To(ContainElement("_INBOX.>")) }) for _, subject := range subscribed { diff --git a/tests/e2e/distributed/agent_native_executor_test.go b/tests/e2e/distributed/agent_native_executor_test.go index b58e32141d6b..8ac0ac109aeb 100644 --- a/tests/e2e/distributed/agent_native_executor_test.go +++ b/tests/e2e/distributed/agent_native_executor_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/mudler/LocalAI/core/services/agents" + "github.com/mudler/LocalAI/core/services/jobs" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/cogito" @@ -340,40 +341,40 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f Expect(err).ToNot(HaveOccurred()) defer sub.Unsubscribe() - adapter := infra.NC - dispatcher := agents.NewNATSDispatcher(adapter, bridge, configs, "http://localhost:8080", "test-key", "agent.test.execute", "test-workers", 0) + // The executor is handed one run by the frontend replica that + // claimed it, over the control verb, and publishes everything it + // produces onto the writer that replica is reading. Here that + // writer is the bus itself, so the subscription above still sees + // the same events on the same subject. + executor := agents.NewWorkerExecutor(bridge, configs, "http://localhost:8080", "test-key") - err = dispatcher.Start(infra.Ctx) - Expect(err).ToNot(HaveOccurred()) - - // Dispatch a chat - messageID, err := dispatcher.Dispatch("user1", "test-agent", "Hello") - Expect(err).ToNot(HaveOccurred()) - Expect(messageID).ToNot(BeEmpty()) + reply, err := executor.Execute(infra.Ctx, mustAgentJSON(agents.AgentChatEvent{ + AgentName: "test-agent", + UserID: "user1", + Message: "Hello", + MessageID: "msg-test-001", + Role: "user", + }), infra.NC) + Expect(err).ToNot(HaveOccurred(), "an error here is this worker failing to SERVE the verb, not the run failing") + Expect(string(reply)).To(ContainSubstring("status")) - // Wait for events (user message + processing status should arrive immediately) + // The run reaches no real LLM, so what it produces is a status the + // frontend can show. What matters is that it travelled. Eventually(func() int { eventMu.Lock() defer eventMu.Unlock() return len(receivedEvents) - }, "5s").Should(BeNumerically(">=", 2)) + }, "5s").Should(BeNumerically(">=", 1)) - // Verify user message was published eventMu.Lock() - hasUserMsg := false - hasProcessing := false + hasStatus := false for _, evt := range receivedEvents { - if evt.EventType == "json_message" && evt.Sender == "user" { - hasUserMsg = true - } if evt.EventType == "json_message_status" { - hasProcessing = true + hasStatus = true } } eventMu.Unlock() - - Expect(hasUserMsg).To(BeTrue(), "user message should be published immediately") - Expect(hasProcessing).To(BeTrue(), "processing status should be published") + Expect(hasStatus).To(BeTrue(), "a status event should reach the subscriber") }) It("should handle cancellation via EventBridge", func() { @@ -400,10 +401,8 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f It("should execute agent chat from enriched payload without ConfigProvider", func() { bridge := agents.NewEventBridge(infra.NC, nil, "enriched-test") - // Create dispatcher with NO ConfigProvider (simulating DB-free worker) - adapter := infra.NC - dispatcher := agents.NewNATSDispatcher(adapter, bridge, nil, "http://localhost:8080", "test-key", "agent.enriched.execute", "enriched-workers", 0) - Expect(dispatcher.Start(infra.Ctx)).To(Succeed()) + // Executor with NO ConfigProvider (simulating DB-free worker) + executor := agents.NewWorkerExecutor(bridge, nil, "http://localhost:8080", "test-key") // Subscribe to events to verify processing var receivedEvents []agents.AgentEvent @@ -421,7 +420,8 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f FlushNATS(infra.NC) - // Publish an enriched AgentChatEvent with embedded Config directly to the queue + // An enriched AgentChatEvent with embedded Config, exactly the body + // the claim row carries. evt := agents.AgentChatEvent{ AgentName: "enriched-agent", UserID: "user1", @@ -434,9 +434,10 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f SystemPrompt: "Be helpful.", }, } - Expect(infra.NC.Publish("agent.enriched.execute", evt)).To(Succeed()) + _, err = executor.Execute(infra.Ctx, mustAgentJSON(evt), infra.NC) + Expect(err).ToNot(HaveOccurred()) - // The dispatcher should process this even without a ConfigProvider. + // The executor should process this even without a ConfigProvider. // It will fail at ExecuteChat (no real LLM), but it should at least // publish a processing status event before failing. Eventually(func() int { @@ -450,60 +451,9 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f eventMu.Lock() hasEvent := len(receivedEvents) > 0 eventMu.Unlock() - Expect(hasEvent).To(BeTrue(), "dispatcher should process enriched event without ConfigProvider") + Expect(hasEvent).To(BeTrue(), "executor should process enriched event without ConfigProvider") }) - It("should round-robin jobs between two dispatchers", func() { - configs := &mockConfigProvider{configs: map[string]*agents.AgentConfig{ - "rr-agent": {Name: "rr-agent", Model: "test"}, - }} - - bridge1 := agents.NewEventBridge(infra.NC, nil, "instance-1") - bridge2 := agents.NewEventBridge(infra.NC, nil, "instance-2") - - adapter := infra.NC - - var count1, count2 atomic.Int32 - - // We can't easily inject mock LLMs into NATSDispatcher since it - // uses ConfigProvider → ExecuteChat. Instead, we test that - // the NATS queue distributes messages between two subscribers. - sub1, err := infra.NC.QueueSubscribe("agent.rr.execute", "rr-workers", func(data []byte) { - count1.Add(1) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub1.Unsubscribe() - - sub2, err := infra.NC.QueueSubscribe("agent.rr.execute", "rr-workers", func(data []byte) { - count2.Add(1) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub2.Unsubscribe() - - FlushNATS(infra.NC) - - // Send 10 messages - for range 10 { - infra.NC.Publish("agent.rr.execute", agents.AgentChatEvent{ - AgentName: "rr-agent", - UserID: "user1", - Message: "hello", - }) - } - - _ = bridge1 - _ = bridge2 - _ = configs - _ = adapter - - Eventually(func() int32 { return count1.Load() + count2.Load() }, "5s").Should(Equal(int32(10))) - // Both should have received some (not all 10 to one) - Expect(count1.Load()).To(BeNumerically(">", 0)) - Expect(count2.Load()).To(BeNumerically(">", 0)) - }) - }) - - Context("AgentConfig JSON Compatibility", func() { It("should marshal/unmarshal matching LocalAGI format", func() { cfg := agents.AgentConfig{ Name: "test", @@ -657,39 +607,35 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f Expect(err).ToNot(HaveOccurred()) defer sub.Unsubscribe() - adapter := infra.NC configs := &mockConfigProvider{configs: map[string]*agents.AgentConfig{ "flow-agent": &cfg, }} + executor := agents.NewWorkerExecutor(bridge, configs, "http://localhost:8080", "test-key") - dispatcher := agents.NewNATSDispatcher(adapter, bridge, configs, "http://localhost:8080", "test-key", "agent.flow.execute", "flow-workers", 0) - Expect(dispatcher.Start(infra.Ctx)).To(Succeed()) - - // Dispatch - messageID, err := dispatcher.Dispatch("user1", "flow-agent", "Hello flow test") + _, err = executor.Execute(infra.Ctx, mustAgentJSON(agents.AgentChatEvent{ + AgentName: "flow-agent", + UserID: "user1", + Message: "Hello flow test", + MessageID: "msg-flow-001", + Role: "user", + }), infra.NC) Expect(err).ToNot(HaveOccurred()) - Expect(messageID).ToNot(BeEmpty()) - // User message + processing status should arrive immediately Eventually(func() int { eventMu.Lock() defer eventMu.Unlock() return len(receivedEvents) - }, "5s").Should(BeNumerically(">=", 2)) + }, "5s").Should(BeNumerically(">=", 1)) eventMu.Lock() - var hasUser, hasProcessing bool + var hasProcessing bool for _, evt := range receivedEvents { - if evt.EventType == "json_message" && evt.Sender == "user" && evt.Content == "Hello flow test" { - hasUser = true - } if evt.EventType == "json_message_status" { hasProcessing = true } } eventMu.Unlock() - Expect(hasUser).To(BeTrue(), "expected user message event") - Expect(hasProcessing).To(BeTrue(), "expected processing status event") + Expect(hasProcessing).To(BeTrue(), "expected a status event") }) }) @@ -739,7 +685,6 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f SystemPrompt: "You are a monitoring agent.", } - adapter := infra.NC configs := &mockConfigProvider{configs: map[string]*agents.AgentConfig{ "bg-agent": &cfg, }} @@ -758,10 +703,9 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f Expect(err).ToNot(HaveOccurred()) defer sub.Unsubscribe() - dispatcher := agents.NewNATSDispatcher(adapter, bridge, configs, "http://localhost:8080", "test-key", "agent.bg.execute", "bg-workers", 0) - Expect(dispatcher.Start(infra.Ctx)).To(Succeed()) + executor := agents.NewWorkerExecutor(bridge, configs, "http://localhost:8080", "test-key") - // Dispatch as background/system role + // A background/system role run, the shape the scheduler enqueues. evt := agents.AgentChatEvent{ AgentName: "bg-agent", UserID: "system", @@ -769,7 +713,8 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f MessageID: "bg-1", Role: "system", } - Expect(infra.NC.Publish("agent.bg.execute", evt)).To(Succeed()) + _, err = executor.Execute(infra.Ctx, mustAgentJSON(evt), infra.NC) + Expect(err).ToNot(HaveOccurred()) // Should receive at least a processing status Eventually(func() int { @@ -863,38 +808,26 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f // LastRunAt is nil — never run, so it's due immediately })).To(Succeed()) - // Subscribe to NATS to capture background run events - var receivedEvents []agents.AgentChatEvent - var eventMu sync.Mutex - sub, err := infra.NC.Subscribe("agent.sched.execute", func(data []byte) { - var evt agents.AgentChatEvent - if json.Unmarshal(data, &evt) == nil { - eventMu.Lock() - receivedEvents = append(receivedEvents, evt) - eventMu.Unlock() - } - }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - // Start scheduler with short poll interval for testing - adapter := infra.NC - scheduler := agents.NewAgentScheduler(db, adapter, store, "agent.sched.execute") + // The scheduler writes a claim row rather than publishing. A + // publish onto a queue group nobody had joined succeeded and the + // background run simply never happened, with nothing recording that + // it had been due. + Expect(jobs.MigrateClaims(infra.Ctx, db)).To(Succeed()) + scheduler := agents.NewAgentScheduler(db, store) schedCtx, schedCancel := context.WithCancel(infra.Ctx) defer schedCancel() go scheduler.Start(schedCtx) - // Wait for the scheduler to fire + var rows []jobs.WorkClaim Eventually(func() int { - eventMu.Lock() - defer eventMu.Unlock() - return len(receivedEvents) + rows = nil + Expect(db.Where("kind = ?", string(jobs.ClaimKindAgentRun)).Find(&rows).Error).To(Succeed()) + return len(rows) }, "20s").Should(BeNumerically(">=", 1)) - eventMu.Lock() - evt := receivedEvents[0] - eventMu.Unlock() + var evt agents.AgentChatEvent + Expect(json.Unmarshal(rows[0].Payload, &evt)).To(Succeed()) Expect(evt.AgentName).To(Equal("cron-agent")) Expect(evt.UserID).To(Equal("u1")) @@ -1079,14 +1012,12 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f Expect(err).ToNot(HaveOccurred()) defer sub.Unsubscribe() - adapter := infra.NC - // Point dispatcher at our mock LLM server - dispatcher := agents.NewNATSDispatcher(adapter, bridge, nil, llmURL, "test-key", "agent.e2e.execute", "e2e-workers", 0) - Expect(dispatcher.Start(infra.Ctx)).To(Succeed()) + // Point the executor at our mock LLM server + executor := agents.NewWorkerExecutor(bridge, nil, llmURL, "test-key") FlushNATS(infra.NC) - // Publish enriched event with skills + // The enriched event with skills, as the claim row carries it evt := agents.AgentChatEvent{ AgentName: "e2e-agent", UserID: "user1", @@ -1104,9 +1035,13 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f {Name: "search", Description: "Search the web", Content: "Full search skill content here"}, }, } - Expect(infra.NC.Publish("agent.e2e.execute", evt)).To(Succeed()) + reply, err := executor.Execute(infra.Ctx, mustAgentJSON(evt), infra.NC) + Expect(err).ToNot(HaveOccurred()) + // A run that reached a conclusion is this worker's own answer, and + // it travels on the response body rather than only as a publish. + Expect(string(reply)).To(ContainSubstring(`"status":"completed"`)) - // Wait for the full execution: user message + processing + agent response + completed + // Wait for the full execution: processing + agent response + completed Eventually(func() int { eventMu.Lock() defer eventMu.Unlock() @@ -1155,13 +1090,11 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f Expect(err).ToNot(HaveOccurred()) defer sub.Unsubscribe() - adapter := infra.NC - dispatcher := agents.NewNATSDispatcher(adapter, bridge, nil, llmURL, "test-key", "agent.bg-e2e.execute", "bg-e2e-workers", 0) - Expect(dispatcher.Start(infra.Ctx)).To(Succeed()) + executor := agents.NewWorkerExecutor(bridge, nil, llmURL, "test-key") FlushNATS(infra.NC) - // Publish as system role (background/autonomous run) + // A system-role run (background/autonomous) evt := agents.AgentChatEvent{ AgentName: "bg-e2e-agent", UserID: "system", @@ -1176,7 +1109,9 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f InnerMonologueTemplate: "Your goal is: {{.Goal}}. What should you do?", }, } - Expect(infra.NC.Publish("agent.bg-e2e.execute", evt)).To(Succeed()) + reply, err := executor.Execute(infra.Ctx, mustAgentJSON(evt), infra.NC) + Expect(err).ToNot(HaveOccurred()) + Expect(string(reply)).To(ContainSubstring(`"status":"completed"`)) // Wait for agent response Eventually(func() int { @@ -1332,3 +1267,12 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f }) }) }) + +// mustAgentJSON encodes one agent chat event as the control verb's request +// body, which is byte for byte what the claim row carries. +func mustAgentJSON(evt agents.AgentChatEvent) json.RawMessage { + GinkgoHelper() + raw, err := json.Marshal(evt) + Expect(err).ToNot(HaveOccurred()) + return raw +} diff --git a/tests/e2e/distributed/job_dispatch_test.go b/tests/e2e/distributed/job_dispatch_test.go index 49052e593d51..b35d93c23c28 100644 --- a/tests/e2e/distributed/job_dispatch_test.go +++ b/tests/e2e/distributed/job_dispatch_test.go @@ -2,11 +2,16 @@ package distributed_test import ( "context" + "encoding/json" "sync/atomic" + "time" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/advisorylock" + "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/LocalAI/core/services/jobs" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -36,20 +41,19 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { Expect(err).ToNot(HaveOccurred()) }) - Context("NATS job dispatch", func() { - It("should enqueue job via NATS when dispatcher is set", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "dispatch-instance", 0) - var processed atomic.Int32 - dispatcher.SetWorkerFunc(func(ctx context.Context, job *jobs.JobRecord, task *jobs.TaskRecord) error { - processed.Add(1) - store.UpdateJobStatus(job.ID, "completed", "done", "") - return nil - }) + Context("Claim-queue dispatch", func() { + // The whole path, against a real PostgreSQL: Enqueue writes a claim + // row, a frontend replica takes it with SELECT ... FOR UPDATE SKIP + // LOCKED, drives it as a streaming control RPC, and persists the + // worker's terminal line before it releases the claim. + It("enqueues a claim, drives it on a worker, and persists what the worker answered", func() { + Expect(cluster.Migrate(infra.Ctx, db)).To(Succeed()) + const owner = "dispatch-instance" + // A replica that is not registered may not claim: its claims could + // not be told from ones a dead replica left. + Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8080", "v1")).To(Succeed()) - dCtx, dCancel := context.WithCancel(infra.Ctx) - defer dCancel() - Expect(dispatcher.Start(dCtx)).To(Succeed()) - defer dispatcher.Stop() + dispatcher := jobs.NewDispatcher(store, infra.NC, db, owner) task := &jobs.TaskRecord{UserID: "u1", Name: "dispatch-task", Model: "m1", Prompt: "p1"} store.CreateTask(task) @@ -58,10 +62,63 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { Expect(dispatcher.Enqueue(job.ID, task.ID, "u1")).To(Succeed()) - Eventually(func() int32 { return processed.Load() }, "10s").Should(Equal(int32(1))) + var claimed int64 + Expect(db.Model(&jobs.WorkClaim{}).Count(&claimed).Error).To(Succeed()) + Expect(claimed).To(Equal(int64(1)), "the enqueue must leave a row, not a publish nobody may be listening for") + + worker := &scriptedWorker{reply: jobs.ClaimReply{JobID: job.ID, Status: "completed", Result: "done"}} + loop, err := jobs.NewDispatchLoop(jobs.DispatchConfig{ + DB: db, + Owner: owner, + Selector: fixedAgent{}, + Control: worker, + Store: store, + Liveness: time.Minute, + }) + Expect(err).ToNot(HaveOccurred()) + + // The claim kind is "task" here (no MCP servers on the model), and + // no worker serves it, so the loop closes the job out with a reason + // rather than leaving it running for ever. Give the row a kind a + // worker DOES serve, so this spec exercises the dispatch path. + Expect(db.Model(&jobs.WorkClaim{}).Where("kind = ?", string(jobs.ClaimKindTask)). + Update("kind", string(jobs.ClaimKindMCPCI)).Error).To(Succeed()) + Expect(loop.DispatchOnce(infra.Ctx)).To(Succeed()) + + Expect(worker.calls.Load()).To(Equal(int32(1))) updated, _ := store.GetJob(job.ID) Expect(updated.Status).To(Equal("completed")) + Expect(updated.Result).To(Equal("done")) + + Expect(db.Model(&jobs.WorkClaim{}).Count(&claimed).Error).To(Succeed()) + Expect(claimed).To(BeZero(), "an answered claim must not be able to run again") + }) + + It("leaves a plain task job failed with a reason, since no worker in this deployment serves that kind", func() { + Expect(cluster.Migrate(infra.Ctx, db)).To(Succeed()) + const owner = "plain-instance" + Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8081", "v1")).To(Succeed()) + + dispatcher := jobs.NewDispatcher(store, infra.NC, db, owner) + task := &jobs.TaskRecord{UserID: "u1", Name: "plain-task", Model: "m1", Prompt: "p1"} + Expect(store.CreateTask(task)).To(Succeed()) + job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "pending", TriggeredBy: "api"} + Expect(store.CreateJob(job)).To(Succeed()) + Expect(dispatcher.Enqueue(job.ID, task.ID, "u1")).To(Succeed()) + + worker := &scriptedWorker{} + loop, err := jobs.NewDispatchLoop(jobs.DispatchConfig{ + DB: db, Owner: owner, Selector: fixedAgent{}, Control: worker, + Store: store, Liveness: time.Minute, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(loop.DispatchOnce(infra.Ctx)).To(Succeed()) + + Expect(worker.calls.Load()).To(BeZero(), "nothing serves this kind, so nothing may be asked of a worker") + updated, _ := store.GetJob(job.ID) + Expect(updated.Status).To(Equal("failed")) + Expect(updated.Error).To(ContainSubstring("plain task jobs")) }) }) @@ -102,38 +159,33 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { }) Context("NATS job cancellation", func() { - It("should cancel running job via NATS cancel subject", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "cancel-instance", 0) - jobStarted := make(chan struct{}) - dispatcher.SetWorkerFunc(func(ctx context.Context, job *jobs.JobRecord, task *jobs.TaskRecord) error { - close(jobStarted) - <-ctx.Done() - return ctx.Err() - }) - - dCtx, dCancel := context.WithCancel(infra.Ctx) - defer dCancel() - Expect(dispatcher.Start(dCtx)).To(Succeed()) - defer dispatcher.Stop() + // Cancellation stays a BROADCAST and is not part of the claim queue: the + // replica holding a run is not the one an API cancel lands on, so the + // signal has to reach every replica and every worker. + It("broadcasts a cancel for a job on the job's own cancel subject", func() { + dispatcher := jobs.NewDispatcher(store, infra.NC, db, "cancel-instance") task := &jobs.TaskRecord{UserID: "u1", Name: "cancel-task", Model: "m1", Prompt: "p1"} store.CreateTask(task) job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "pending", TriggeredBy: "api"} store.CreateJob(job) - dispatcher.Enqueue(job.ID, task.ID, "u1") - - Eventually(jobStarted, "10s").Should(BeClosed()) + seen := make(chan string, 1) + sub, err := infra.NC.Subscribe(messaging.SubjectJobCancelWildcard, func(data []byte) { + var evt jobs.CancelEvent + if json.Unmarshal(data, &evt) == nil { + select { + case seen <- evt.JobID: + default: + } + } + }) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = sub.Unsubscribe() }() + FlushNATS(infra.NC) Expect(dispatcher.Cancel(job.ID)).To(Succeed()) - - Eventually(func() string { - j, _ := store.GetJob(job.ID) - if j == nil { - return "" - } - return j.Status - }, "10s").Should(Equal("cancelled")) + Eventually(seen, "10s").Should(Receive(Equal(job.ID))) }) }) @@ -186,3 +238,33 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { }) }) }) + +// fixedAgent is a selection that always names one connected agent worker. The +// selection itself is pinned against real connection rows in +// core/services/nodes; what this suite drives is the dispatch that follows it. +type fixedAgent struct{} + +func (fixedAgent) PickConnected(context.Context) (string, string, error) { + return "agent-node-1", nodes.NodeTypeAgent, nil +} + +// scriptedWorker answers a streaming control RPC with a scripted reply, so this +// suite exercises the claim, the persist and the settle against a real database +// without needing a worker process. +type scriptedWorker struct { + calls atomic.Int32 + reply jobs.ClaimReply + err error +} + +func (w *scriptedWorker) CallStreaming(_ context.Context, _, _ string, _, reply any, + _ func(string, json.RawMessage)) error { + w.calls.Add(1) + if w.err != nil { + return w.err + } + if out, ok := reply.(*jobs.ClaimReply); ok { + *out = w.reply + } + return nil +} diff --git a/tests/e2e/distributed/job_distribution_test.go b/tests/e2e/distributed/job_distribution_test.go index fc6c3a031a75..8c56e8730125 100644 --- a/tests/e2e/distributed/job_distribution_test.go +++ b/tests/e2e/distributed/job_distribution_test.go @@ -3,10 +3,12 @@ package distributed_test import ( "context" "encoding/json" - "sync/atomic" + "errors" + "sync" "time" "github.com/mudler/LocalAI/core/services/advisorylock" + "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/LocalAI/core/services/dbutil" "github.com/mudler/LocalAI/core/services/jobs" "github.com/mudler/LocalAI/core/services/messaging" @@ -168,110 +170,111 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { }) }) - Context("Job Distribution via NATS", func() { - It("should enqueue job via NATS and worker picks it up", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance", 0) - var processed atomic.Int32 - dispatcher.SetWorkerFunc(func(ctx context.Context, job *jobs.JobRecord, task *jobs.TaskRecord) error { - processed.Add(1) - store.UpdateJobStatus(job.ID, "completed", "done", "") - return nil - }) + Context("Job distribution through the claim queue", func() { + It("enqueues a claim and drives it on a worker, persisting what the worker answered", func() { + Expect(cluster.Migrate(infra.Ctx, db)).To(Succeed()) + const owner = "test-instance" + Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8090", "v1")).To(Succeed()) - dCtx, dCancel := context.WithCancel(infra.Ctx) - defer dCancel() - Expect(dispatcher.Start(dCtx)).To(Succeed()) - defer dispatcher.Stop() + dispatcher := jobs.NewDispatcher(store, infra.NC, db, owner) - // Create task and job task := &jobs.TaskRecord{UserID: "u1", Name: "dispatch-test", Model: "m1", Prompt: "p1"} store.CreateTask(task) job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "pending", TriggeredBy: "api"} store.CreateJob(job) - // Enqueue Expect(dispatcher.Enqueue(job.ID, task.ID, "u1")).To(Succeed()) + Expect(db.Model(&jobs.WorkClaim{}).Where("kind = ?", string(jobs.ClaimKindTask)). + Update("kind", string(jobs.ClaimKindMCPCI)).Error).To(Succeed()) - // Wait for processing - Eventually(func() int32 { return processed.Load() }, "10s").Should(Equal(int32(1))) + worker := &scriptedWorker{reply: jobs.ClaimReply{JobID: job.ID, Status: "completed", Result: "done"}} + loop, err := jobs.NewDispatchLoop(jobs.DispatchConfig{ + DB: db, Owner: owner, Selector: fixedAgent{}, Control: worker, + Store: store, Liveness: time.Minute, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(loop.DispatchOnce(infra.Ctx)).To(Succeed()) - // Verify status updated updated, _ := store.GetJob(job.ID) Expect(updated.Status).To(Equal("completed")) }) - It("should cancel running job via NATS", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance", 0) - jobStarted := make(chan struct{}) - dispatcher.SetWorkerFunc(func(ctx context.Context, job *jobs.JobRecord, task *jobs.TaskRecord) error { - close(jobStarted) - // Simulate long work — wait for cancellation - <-ctx.Done() - return ctx.Err() - }) - - dCtx, dCancel := context.WithCancel(infra.Ctx) - defer dCancel() - Expect(dispatcher.Start(dCtx)).To(Succeed()) - defer dispatcher.Stop() + It("returns a claim to the pool when the dispatch obtained no answer, rather than losing the work", func() { + Expect(cluster.Migrate(infra.Ctx, db)).To(Succeed()) + const owner = "lossy-instance" + Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8091", "v1")).To(Succeed()) - task := &jobs.TaskRecord{UserID: "u1", Name: "cancel-test", Model: "m1", Prompt: "p1"} + dispatcher := jobs.NewDispatcher(store, infra.NC, db, owner) + task := &jobs.TaskRecord{UserID: "u1", Name: "lossy-test", Model: "m1", Prompt: "p1"} store.CreateTask(task) job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "pending", TriggeredBy: "api"} store.CreateJob(job) + Expect(dispatcher.Enqueue(job.ID, task.ID, "u1")).To(Succeed()) + Expect(db.Model(&jobs.WorkClaim{}).Where("kind = ?", string(jobs.ClaimKindTask)). + Update("kind", string(jobs.ClaimKindMCPCI)).Error).To(Succeed()) - dispatcher.Enqueue(job.ID, task.ID, "u1") - - // Wait for job to start - Eventually(jobStarted, "10s").Should(BeClosed()) - - // Cancel via NATS - Expect(dispatcher.Cancel(job.ID)).To(Succeed()) + worker := &scriptedWorker{err: errors.New("the tunnel died mid-verb")} + loop, err := jobs.NewDispatchLoop(jobs.DispatchConfig{ + DB: db, Owner: owner, Selector: fixedAgent{}, Control: worker, + Store: store, Liveness: time.Minute, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(loop.DispatchOnce(infra.Ctx)).To(HaveOccurred()) - // Wait for cancellation - Eventually(func() string { - j, _ := store.GetJob(job.ID) - if j == nil { - return "" - } - return j.Status - }, "10s").Should(Equal("cancelled")) + var rows []jobs.WorkClaim + Expect(db.Find(&rows).Error).To(Succeed()) + Expect(rows).To(HaveLen(1), "a transport failure must not complete or discard the work") + Expect(rows[0].ClaimedAt).To(BeNil()) + Expect(rows[0].Attempts).To(Equal(1)) + updated, _ := store.GetJob(job.ID) + Expect(updated.Status).ToNot(Equal("completed"), "nothing was learned, so nothing may be written about the job") }) - It("should report job progress via NATS", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance", 0) - dispatcher.SetWorkerFunc(func(ctx context.Context, job *jobs.JobRecord, task *jobs.TaskRecord) error { - dispatcher.PublishProgress(job.ID, "running", "step 1") - time.Sleep(50 * time.Millisecond) - dispatcher.PublishProgress(job.ID, "running", "step 2") - store.UpdateJobStatus(job.ID, "completed", "done", "") - return nil + It("broadcasts a cancel on the job's own cancel subject", func() { + dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance") + seen := make(chan string, 1) + sub, err := infra.NC.Subscribe(messaging.SubjectJobCancelWildcard, func(data []byte) { + var evt jobs.CancelEvent + if json.Unmarshal(data, &evt) == nil { + select { + case seen <- evt.JobID: + default: + } + } }) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = sub.Unsubscribe() }() + FlushNATS(infra.NC) - dCtx, dCancel := context.WithCancel(infra.Ctx) - defer dCancel() - Expect(dispatcher.Start(dCtx)).To(Succeed()) - defer dispatcher.Stop() + Expect(dispatcher.Cancel("job-to-cancel")).To(Succeed()) + Eventually(seen, "10s").Should(Receive(Equal("job-to-cancel"))) + }) - task := &jobs.TaskRecord{UserID: "u1", Name: "progress-test", Model: "m1", Prompt: "p1"} - store.CreateTask(task) - job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "pending", TriggeredBy: "api"} - store.CreateJob(job) + It("reports job progress on the job's own progress subject", func() { + dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance") - // Subscribe to progress before enqueuing var progressEvents []jobs.ProgressEvent - sub, err := dispatcher.SubscribeProgress(job.ID, func(evt jobs.ProgressEvent) { + var mu sync.Mutex + sub, err := dispatcher.SubscribeProgress("progress-job", func(evt jobs.ProgressEvent) { + mu.Lock() progressEvents = append(progressEvents, evt) + mu.Unlock() }) Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - dispatcher.Enqueue(job.ID, task.ID, "u1") - - // Wait for completion - Eventually(func() int { return len(progressEvents) }, "10s").Should(BeNumerically(">=", 3)) + defer func() { _ = sub.Unsubscribe() }() + FlushNATS(infra.NC) - // Should have received progress events + Expect(dispatcher.PublishProgress("progress-job", "running", "step 1")).To(Succeed()) + Expect(dispatcher.PublishProgress("progress-job", "running", "step 2")).To(Succeed()) + Expect(dispatcher.PublishProgress("progress-job", "completed", "done")).To(Succeed()) + + Eventually(func() int { + mu.Lock() + defer mu.Unlock() + return len(progressEvents) + }, "10s").Should(BeNumerically(">=", 3)) + mu.Lock() + defer mu.Unlock() Expect(progressEvents[0].Status).To(Equal("running")) }) }) @@ -317,7 +320,7 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { Context("Progress Streaming (NATS → SSE bridge)", func() { It("should bridge NATS progress events", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance", 0) + dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance") dCtx, dCancel := context.WithCancel(infra.Ctx) defer dCancel() @@ -330,7 +333,7 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { events = append(events, evt) }) Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() + defer func() { _ = sub.Unsubscribe() }() FlushNATS(infra.NC) @@ -345,7 +348,7 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { }) It("should filter SSE events by job ID", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance", 0) + dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance") dCtx, dCancel := context.WithCancel(infra.Ctx) defer dCancel() @@ -374,110 +377,36 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { }) }) - Context("Enriched Job Payload (DB-free worker)", func() { - It("should enrich JobEvent with full Job and Task data", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "enrichment-test", 0) + Context("Enriched claim payload (DB-free worker)", func() { + It("stores the full Job and Task on the claim row, so the worker needs no database", func() { + dispatcher := jobs.NewDispatcher(store, infra.NC, db, "enrichment-test") - dCtx, dCancel := context.WithCancel(infra.Ctx) - defer dCancel() - Expect(dispatcher.Start(dCtx)).To(Succeed()) - defer dispatcher.Stop() - - // Create task and job task := &jobs.TaskRecord{UserID: "u1", Name: "enrich-task", Model: "m1", Prompt: "hello {{.name}}"} store.CreateTask(task) job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "pending", TriggeredBy: "api"} store.CreateJob(job) - // Capture the raw NATS event - var capturedEvt jobs.JobEvent - var captured atomic.Int32 - sub, err := infra.NC.Subscribe(messaging.SubjectJobsNew, func(data []byte) { - var evt jobs.JobEvent - if json.Unmarshal(data, &evt) == nil { - capturedEvt = evt - captured.Add(1) - } - }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - FlushNATS(infra.NC) - - // Enqueue — this should embed Job+Task in the event Expect(dispatcher.Enqueue(job.ID, task.ID, "u1")).To(Succeed()) - Eventually(func() int32 { return captured.Load() }, "5s").Should(BeNumerically(">=", 1)) - - // Verify enriched payload - Expect(capturedEvt.Job).ToNot(BeNil(), "JobEvent should contain embedded Job") - Expect(capturedEvt.Task).ToNot(BeNil(), "JobEvent should contain embedded Task") - Expect(capturedEvt.Job.ID).To(Equal(job.ID)) - Expect(capturedEvt.Task.Name).To(Equal("enrich-task")) - Expect(capturedEvt.Task.Prompt).To(Equal("hello {{.name}}")) - }) - - It("should process job from enriched payload without DB access", func() { - // Create a worker-side dispatcher with NO store (simulating DB-free worker) - workerDispatcher := jobs.NewDispatcher(nil, infra.NC, nil, "worker-no-db", 0) - - var receivedJob *jobs.JobRecord - var receivedTask *jobs.TaskRecord - processed := make(chan struct{}) - - workerDispatcher.SetWorkerFunc(func(ctx context.Context, job *jobs.JobRecord, task *jobs.TaskRecord) error { - receivedJob = job - receivedTask = task - job.Result = "processed without DB" - close(processed) - return nil - }) - - dCtx, dCancel := context.WithCancel(infra.Ctx) - defer dCancel() - Expect(workerDispatcher.Start(dCtx)).To(Succeed()) - defer workerDispatcher.Stop() - - FlushNATS(infra.NC) - - // Publish an enriched event directly (simulating what the frontend does) - evt := jobs.JobEvent{ - JobID: "test-job-123", - TaskID: "test-task-456", - UserID: "u1", - Job: &jobs.JobRecord{ - ID: "test-job-123", - TaskID: "test-task-456", - UserID: "u1", - Status: "pending", - TriggeredBy: "api", - }, - Task: &jobs.TaskRecord{ - ID: "test-task-456", - Name: "embedded-task", - Model: "test-model", - Prompt: "do something", - }, - } - Expect(infra.NC.Publish(messaging.SubjectJobsNew, evt)).To(Succeed()) - - Eventually(processed, "10s").Should(BeClosed()) - - // Verify the worker received data from the payload, not from DB - Expect(receivedJob).ToNot(BeNil()) - Expect(receivedJob.ID).To(Equal("test-job-123")) - Expect(receivedTask).ToNot(BeNil()) - Expect(receivedTask.Name).To(Equal("embedded-task")) - Expect(receivedTask.Model).To(Equal("test-model")) + var rows []jobs.WorkClaim + Expect(db.Find(&rows).Error).To(Succeed()) + Expect(rows).To(HaveLen(1)) + + var evt jobs.JobEvent + Expect(json.Unmarshal(rows[0].Payload, &evt)).To(Succeed()) + Expect(evt.Job).ToNot(BeNil(), "the claim payload should contain the embedded Job") + Expect(evt.Task).ToNot(BeNil(), "the claim payload should contain the embedded Task") + Expect(evt.Job.ID).To(Equal(job.ID)) + Expect(evt.Task.Name).To(Equal("enrich-task")) + Expect(evt.Task.Prompt).To(Equal("hello {{.name}}")) }) - It("should publish job result via NATS on completion", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "result-test", 0) - dispatcher.SetWorkerFunc(func(ctx context.Context, job *jobs.JobRecord, task *jobs.TaskRecord) error { - job.Result = "job finished successfully" - return nil - }) - + // The two subscriptions the dispatcher KEEPS. A worker's result and + // trace lines are re-broadcast by the replica that claimed the work, and + // every replica persists them, because the SSE stream a user is watching + // may be open on a replica that claimed nothing. + It("persists a result a worker's re-broadcast carried, whichever replica reads it", func() { + dispatcher := jobs.NewDispatcher(store, infra.NC, db, "result-test") dCtx, dCancel := context.WithCancel(infra.Ctx) defer dCancel() Expect(dispatcher.Start(dCtx)).To(Succeed()) @@ -485,35 +414,23 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { task := &jobs.TaskRecord{UserID: "u1", Name: "result-task", Model: "m1", Prompt: "p1"} store.CreateTask(task) - job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "pending", TriggeredBy: "api"} + job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "running", TriggeredBy: "api"} store.CreateJob(job) - - // Subscribe to result events - var resultEvt jobs.JobResultEvent - var received atomic.Int32 - sub, err := infra.NC.Subscribe(messaging.SubjectJobResult(job.ID), func(data []byte) { - json.Unmarshal(data, &resultEvt) - received.Add(1) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - FlushNATS(infra.NC) - dispatcher.Enqueue(job.ID, task.ID, "u1") - Eventually(func() int32 { return received.Load() }, "10s").Should(BeNumerically(">=", 1)) - Expect(resultEvt.JobID).To(Equal(job.ID)) - Expect(resultEvt.Status).To(Equal("completed")) - }) + jobs.PublishJobResult(infra.NC, job.ID, "completed", "job finished successfully", "") - It("should stream traces via NATS progress events", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "trace-test", 0) - dispatcher.SetWorkerFunc(func(ctx context.Context, job *jobs.JobRecord, task *jobs.TaskRecord) error { - dispatcher.PublishTrace(job.ID, "reasoning", "thinking about the problem") - dispatcher.PublishTrace(job.ID, "tool_call", "calling search tool") - return nil - }) + Eventually(func() string { + j, _ := store.GetJob(job.ID) + if j == nil { + return "" + } + return j.Status + }, "10s").Should(Equal("completed")) + }) + It("appends a trace a worker's re-broadcast carried", func() { + dispatcher := jobs.NewDispatcher(store, infra.NC, db, "trace-test") dCtx, dCancel := context.WithCancel(infra.Ctx) defer dCancel() Expect(dispatcher.Start(dCtx)).To(Succeed()) @@ -521,25 +438,24 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { task := &jobs.TaskRecord{UserID: "u1", Name: "trace-task", Model: "m1", Prompt: "p1"} store.CreateTask(task) - job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "pending", TriggeredBy: "api"} + job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "running", TriggeredBy: "api"} store.CreateJob(job) + FlushNATS(infra.NC) - // Subscribe to progress events - var traceEvents []jobs.ProgressEvent - sub, err := dispatcher.SubscribeProgress(job.ID, func(evt jobs.ProgressEvent) { - if evt.TraceType != "" { - traceEvents = append(traceEvents, evt) - } - }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - dispatcher.Enqueue(job.ID, task.ID, "u1") + Expect(dispatcher.PublishTrace(job.ID, "reasoning", "thinking about the problem")).To(Succeed()) + Expect(dispatcher.PublishTrace(job.ID, "tool_call", "calling search tool")).To(Succeed()) - Eventually(func() int { return len(traceEvents) }, "10s").Should(BeNumerically(">=", 2)) - Expect(traceEvents[0].TraceType).To(Equal("reasoning")) - Expect(traceEvents[0].TraceContent).To(Equal("thinking about the problem")) - Expect(traceEvents[1].TraceType).To(Equal("tool_call")) + Eventually(func() int { + j, _ := store.GetJob(job.ID) + if j == nil || j.TracesJSON == "" { + return 0 + } + var traces []map[string]string + if json.Unmarshal([]byte(j.TracesJSON), &traces) != nil { + return 0 + } + return len(traces) + }, "10s").Should(BeNumerically(">=", 2)) }) It("should append traces incrementally to job record", func() { diff --git a/tests/e2e/distributed/mcp_ci_job_test.go b/tests/e2e/distributed/mcp_ci_job_test.go index 1c4d8911a03d..62c696850c77 100644 --- a/tests/e2e/distributed/mcp_ci_job_test.go +++ b/tests/e2e/distributed/mcp_ci_job_test.go @@ -357,19 +357,16 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( ModelConfig: modelCfg, } - // Simulate the agent worker subscribing and processing - workerSub, err := infra.NC.QueueSubscribe(messaging.SubjectJobsNew, messaging.QueueWorkers, func(data []byte) { - // This is what the agent worker does — call handleMCPCIJob - // We import it indirectly by calling the same logic - go processMCPCIJobForTest(data, llmURL, "test-token", infra.NC) - }) + // The agent worker is handed this body by the frontend replica that + // claimed the row, on a streaming control verb. There is no queue + // group to publish onto any more, so the body goes straight to the + // worker-side logic, which is what the verb's handler calls. + raw, err := json.Marshal(evt) Expect(err).ToNot(HaveOccurred()) - defer workerSub.Unsubscribe() FlushNATS(infra.NC) - // Publish the job event - Expect(infra.NC.Publish(messaging.SubjectJobsNew, evt)).To(Succeed()) + go processMCPCIJobForTest(raw, llmURL, "test-token", infra.NC) // Wait for result Eventually(func() bool { diff --git a/tests/e2e/distributed/nats_jwt_test.go b/tests/e2e/distributed/nats_jwt_test.go index 72ea76b751ec..1683773af4a6 100644 --- a/tests/e2e/distributed/nats_jwt_test.go +++ b/tests/e2e/distributed/nats_jwt_test.go @@ -85,7 +85,12 @@ var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { claims, err := natsauth.DecodeUserClaims(token) Expect(err).ToNot(HaveOccurred()) - Expect(claims.Permissions.Sub.Allow).To(ContainElement("agent.execute")) + // agent.execute has left this list: agent execution is a streaming + // control verb on the tunnel now, driven by a claim a frontend replica + // took off the job store. What must still be here is the cancel + // broadcast, which cannot become an RPC. + Expect(claims.Permissions.Sub.Allow).To(ContainElement("agent.*.cancel")) + Expect(claims.Permissions.Sub.Allow).ToNot(ContainElement("agent.execute")) for _, subj := range claims.Permissions.Sub.Allow { Expect(subj).NotTo(ContainSubstring("backend.install")) } @@ -109,15 +114,19 @@ var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { DeferCleanup(nc.Close) // Mirror core/cli/agent_worker.go exactly. MCP tool execution and - // discovery are absent, and so is the per-node backend.stop, because - // none of the three is a bus subject any more: the frontend selects an - // agent worker itself and reaches it with a control RPC over the tunnel - // that worker holds. - _, err = nc.QueueSubscribe(messaging.SubjectMCPCIJobsNew, messaging.QueueWorkers, func([]byte) {}) - Expect(err).ToNot(HaveOccurred(), "agent JWT must allow %s (MCP CI jobs)", messaging.SubjectMCPCIJobsNew) - - _, err = nc.Subscribe(messaging.SubjectAgentExecute, func([]byte) {}) - Expect(err).ToNot(HaveOccurred(), "agent JWT must allow %s (job dispatch)", messaging.SubjectAgentExecute) + // discovery are absent, so is the per-node backend.stop, and so now are + // agent execution and MCP CI runs: none of them is a bus subject any + // more. The frontend selects an agent worker itself and reaches it with + // a control RPC over the tunnel that worker holds, and the work it + // hands over is a row it claimed on the job store. + // + // What is left is the cancel broadcast, which cannot become an RPC: the + // replica holding a run is not the one an API cancel lands on. + _, err = nc.Subscribe(messaging.SubjectAgentCancelWildcard, func([]byte) {}) + Expect(err).ToNot(HaveOccurred(), "agent JWT must allow %s (cancellation)", messaging.SubjectAgentCancelWildcard) + + _, err = nc.Subscribe(messaging.SubjectJobProgressWildcard, func([]byte) {}) + Expect(err).ToNot(HaveOccurred(), "agent JWT must allow %s (progress bridging)", messaging.SubjectJobProgressWildcard) }) // The narrowing, proved against the enforcing server rather than against @@ -142,4 +151,26 @@ var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { Expect(err).To(HaveOccurred(), "backend.stop is a control RPC on the worker's tunnel; the bus must not carry it") }) + + // The same narrowing for the two queue subjects that became claim rows. + // Written out by hand for the same reason: those literals are what a worker + // from an older release would still subscribe to, and this is what the + // server now answers it. + DescribeTable("refuses an agent-minted JWT a retired queue subject", + func(subject string) { + cfg := natsauth.Config{AccountSeed: infra.AccountSeed, WorkerJWTTTL: time.Hour} + token, seed, err := cfg.MintWorkerJWT("agent-node-queues", "agent") + Expect(err).ToNot(HaveOccurred()) + + nc, err := messaging.New(infra.NatsURL, messaging.WithUserJWT(token, seed)) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(nc.Close) + + _, err = nc.Subscribe(subject, func([]byte) {}) + Expect(err).To(HaveOccurred(), + "%s became a claim on the job store; the bus must not carry it", subject) + }, + Entry("agent execution", "agent.execute"), + Entry("mcp ci jobs", "jobs.mcp-ci.new"), + ) }) diff --git a/tests/e2e/distributed/sse_routes_test.go b/tests/e2e/distributed/sse_routes_test.go index 4cc33481404a..a936eae880a2 100644 --- a/tests/e2e/distributed/sse_routes_test.go +++ b/tests/e2e/distributed/sse_routes_test.go @@ -36,7 +36,7 @@ var _ = Describe("SSE Routes", Label("Distributed"), func() { jobStore, err := jobs.NewJobStore(db) Expect(err).ToNot(HaveOccurred()) - dispatcher := jobs.NewDispatcher(jobStore, infra.NC, db, "sse-instance", 0) + dispatcher := jobs.NewDispatcher(jobStore, infra.NC, db, "sse-instance") dCtx, dCancel := context.WithCancel(infra.Ctx) defer dCancel() From 236c3a0bb9b9243a3a83f67fba67c647cd9d0612 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 21:11:59 +0000 Subject: [PATCH 091/153] feat(distributed): give responses.metadata something to re-hydrate from The responses.metadata SyncedMap had no durable Store, so its reconnect re-hydrate replaced nothing. That was survivable while responses converged through deltas on a broker that mostly stayed up. It is not survivable on a carrier whose listener is one pinned PostgreSQL session: every response created while the subscription was down stays invisible on that replica forever, and the symptom is a 404 from one replica and a 200 from another for the same response_id. State that must survive a gap now lives in a response_metadata table, and the notification only says it changed. The map writes through on a Set and reads the table on hydrate, on reconnect and on reconcile, so the gap closes instead of becoming permanent. The row carries the whole projection as JSON rather than one column per field. A column-per-field schema would be a second definition of what a peer may act on, and the two would drift the first time syncedResponse gained a field: the map would broadcast the new field and hydrate without it, so a replica that had reconnected would serve a different response body from one that had not, with nothing failing anywhere. Only PayloadJSON is ever decoded; owner_replica and owner are indexed copies for an operator reading the table by hand. A missing row and an unreachable database are different facts. Every store and adapter method returns a driver failure as an error and never as an empty result, and syncstate replaces nothing when its source errors, so an outage leaves the map holding what it had rather than blanking it into a cluster-wide 404. Liveness is the database's clock, spelled expires_at IS NULL OR expires_at > now(), because every replica hydrating from this table must agree on which rows are live and a Go-side cutoff makes that a property of whichever process asked. The test container shares the host clock, so no behavioural spec can tell the two apart; the statement shape is pinned instead. The constructor refuses a non-PostgreSQL handle, because an unguarded now() on the single-binary path reads as a missing migration. A ticker sweeps expired rows every five minutes on each replica, and Close waits for it rather than racing it. Note that the sweep removes nothing while LOCALAI_OPEN_RESPONSES_STORE_TTL is 0, which is the default: with no TTL nothing ever expires and the table grows for the life of the deployment. The docs say so plainly. EnableDistributed takes the store positionally and last, so a call site that forgets it fails to compile rather than silently restoring the deltas-only map this change exists to replace. A nil store there is refused by name: it is reached only from the distributed branch of route registration, so it is a wiring bug and not a deployment shape. What still never leaves the owning replica is unchanged: the resume buffer and the CancelFunc. The write-through is one row per response state change, not one per generated token. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .../endpoints/openresponses/metadata_store.go | 79 +++++ .../openresponses/metadata_store_test.go | 152 +++++++++ core/http/endpoints/openresponses/store.go | 8 + core/http/endpoints/openresponses/sync.go | 101 +++++- .../http/endpoints/openresponses/sync_test.go | 126 +++++++- core/http/routes/openresponses.go | 11 +- core/services/distributed/init.go | 22 +- core/services/distributed/responses.go | 134 ++++++++ core/services/distributed/responses_test.go | 290 ++++++++++++++++++ docs/content/features/distributed-mode.md | 25 ++ tests/e2e/distributed/phase4_test.go | 28 ++ 11 files changed, 959 insertions(+), 17 deletions(-) create mode 100644 core/http/endpoints/openresponses/metadata_store.go create mode 100644 core/http/endpoints/openresponses/metadata_store_test.go create mode 100644 core/services/distributed/responses.go create mode 100644 core/services/distributed/responses_test.go diff --git a/core/http/endpoints/openresponses/metadata_store.go b/core/http/endpoints/openresponses/metadata_store.go new file mode 100644 index 000000000000..ba4833f37bfb --- /dev/null +++ b/core/http/endpoints/openresponses/metadata_store.go @@ -0,0 +1,79 @@ +package openresponses + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/mudler/LocalAI/core/services/distributed" + "github.com/mudler/LocalAI/core/services/syncstate" +) + +// responseMetadataStoreAdapter bridges the durable store onto the generic +// syncstate.Store the responses.metadata map consumes. It mirrors +// finetune.fineTuneStoreAdapter and agentpool.taskStoreAdapter, which is the +// shape every other adopter already uses. +// +// It lives in this package rather than beside the store because syncedResponse +// is unexported: the projection of a response that a peer may act on is defined +// here and nowhere else, and the durable row carries it as opaque JSON so the +// two cannot drift. +type responseMetadataStoreAdapter struct { + store *distributed.ResponseMetadataStore +} + +// compile-time assertion that the adapter satisfies the component's Store. +var _ syncstate.Store[string, *syncedResponse] = (*responseMetadataStoreAdapter)(nil) + +// List re-hydrates the map from the durable rows that are still live. +// +// A decode failure is returned rather than skipped. Skipping would drop exactly +// one response with nothing failing anywhere, which is the invisible-404 this +// whole store exists to prevent; returning the error leaves the map holding +// whatever it already had, because syncstate.hydrate replaces nothing when the +// source errors. +func (a *responseMetadataStoreAdapter) List(ctx context.Context) ([]*syncedResponse, error) { + records, err := a.store.ListUnexpired(ctx) + if err != nil { + return nil, err + } + out := make([]*syncedResponse, 0, len(records)) + for i := range records { + v := &syncedResponse{} + if err := json.Unmarshal(records[i].PayloadJSON, v); err != nil { + return nil, fmt.Errorf("decoding replicated response metadata %q: %w", records[i].ID, err) + } + out = append(out, v) + } + return out, nil +} + +// Upsert writes the whole projection as JSON, plus the columns a hydrate and a +// purge filter on. +// +// ExpiresAt is lifted out of the payload into its own column because +// ListUnexpired and PurgeExpired compare against it on the database clock. An +// adapter that left it null would make every response immortal in this table: +// the map would still expire it in memory, so nothing would look broken until +// the table had grown without bound and a restarted replica re-hydrated +// responses that died hours earlier. +func (a *responseMetadataStoreAdapter) Upsert(ctx context.Context, v *syncedResponse) error { + if v == nil { + return fmt.Errorf("replicating response metadata: nil value") + } + payload, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("encoding replicated response metadata %q: %w", v.ID, err) + } + return a.store.Upsert(ctx, &distributed.ResponseMetadataRecord{ + ID: v.ID, + OwnerReplica: v.OwnerReplica, + Owner: v.Owner, + PayloadJSON: payload, + ExpiresAt: v.ExpiresAt, + }) +} + +func (a *responseMetadataStoreAdapter) Delete(ctx context.Context, k string) error { + return a.store.Delete(ctx, k) +} diff --git a/core/http/endpoints/openresponses/metadata_store_test.go b/core/http/endpoints/openresponses/metadata_store_test.go new file mode 100644 index 000000000000..706a0fc8177a --- /dev/null +++ b/core/http/endpoints/openresponses/metadata_store_test.go @@ -0,0 +1,152 @@ +package openresponses + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/distributed" + "github.com/mudler/LocalAI/core/services/testutil" +) + +var _ = Describe("responseMetadataStoreAdapter", func() { + var ( + db *gorm.DB + adapter *responseMetadataStoreAdapter + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + db = testutil.SetupTestDB() + store, err := distributed.NewResponseMetadataStore(db) + Expect(err).ToNot(HaveOccurred()) + adapter = &responseMetadataStoreAdapter{store: store} + }) + + It("round-trips every field of the replicated projection", func() { + storedAt := time.Now().UTC().Truncate(time.Second) + expiresAt := storedAt.Add(time.Hour) + completedAt := storedAt.Unix() + + in := &syncedResponse{ + ID: "resp_roundtrip", + OwnerReplica: "replica-a", + Owner: "user-1", + Request: &schema.OpenResponsesRequest{ + Model: "test-model", + Input: "Hello", + }, + Response: &schema.ORResponseResource{ + ID: "resp_roundtrip", + Object: "response", + CreatedAt: storedAt.Unix(), + CompletedAt: &completedAt, + Status: schema.ORStatusCompleted, + Model: "test-model", + Output: []schema.ORItemField{ + {Type: "message", ID: "msg_roundtrip", Role: "assistant"}, + }, + }, + StoredAt: storedAt, + ExpiresAt: &expiresAt, + StreamEnabled: true, + IsBackground: true, + } + Expect(adapter.Upsert(ctx, in)).To(Succeed()) + + out, err := adapter.List(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(HaveLen(1)) + got := out[0] + + // Asserted field by field rather than with one Equal on the struct: a + // single Equal against a value whose Request is nil passes for an + // adapter that dropped the field, and previous_response_id chaining is + // exactly what that field feeds. + Expect(got.ID).To(Equal(in.ID)) + Expect(got.OwnerReplica).To(Equal(in.OwnerReplica)) + Expect(got.Owner).To(Equal(in.Owner)) + Expect(got.Request).ToNot(BeNil()) + Expect(got.Request.Model).To(Equal("test-model")) + Expect(got.Request.Input).To(Equal("Hello")) + Expect(got.Response).ToNot(BeNil()) + Expect(got.Response.ID).To(Equal("resp_roundtrip")) + Expect(got.Response.Status).To(Equal(schema.ORStatusCompleted)) + Expect(got.Response.Model).To(Equal("test-model")) + Expect(got.Response.CompletedAt).ToNot(BeNil()) + Expect(*got.Response.CompletedAt).To(Equal(completedAt)) + Expect(got.Response.Output).To(HaveLen(1)) + Expect(got.Response.Output[0].ID).To(Equal("msg_roundtrip")) + Expect(got.StoredAt.UTC()).To(BeTemporally("==", storedAt)) + Expect(got.ExpiresAt).ToNot(BeNil()) + Expect(got.ExpiresAt.UTC()).To(BeTemporally("==", expiresAt)) + Expect(got.StreamEnabled).To(BeTrue()) + Expect(got.IsBackground).To(BeTrue()) + }) + + It("lifts ExpiresAt into the row's own column", func() { + expiresAt := time.Now().UTC().Add(time.Hour).Truncate(time.Second) + Expect(adapter.Upsert(ctx, &syncedResponse{ID: "resp_expiry", ExpiresAt: &expiresAt})).To(Succeed()) + + // Read the column directly. The round-trip above is served entirely from + // the JSON payload and stays green for an adapter that left this column + // null, and a null here is what makes ListUnexpired and PurgeExpired + // treat the response as immortal. + var row distributed.ResponseMetadataRecord + Expect(db.First(&row, "id = ?", "resp_expiry").Error).To(Succeed()) + Expect(row.ExpiresAt).ToNot(BeNil()) + Expect(row.ExpiresAt.UTC()).To(BeTemporally("==", expiresAt)) + }) + + It("copies the owner columns out of the payload so an operator can filter on them", func() { + Expect(adapter.Upsert(ctx, &syncedResponse{ + ID: "resp_owner", OwnerReplica: "replica-b", Owner: "user-9", + })).To(Succeed()) + + var row distributed.ResponseMetadataRecord + Expect(db.First(&row, "id = ?", "resp_owner").Error).To(Succeed()) + Expect(row.OwnerReplica).To(Equal("replica-b")) + Expect(row.Owner).To(Equal("user-9")) + }) + + It("does not return a row whose expiry has passed", func() { + past := time.Now().Add(-time.Hour) + Expect(adapter.Upsert(ctx, &syncedResponse{ID: "resp_dead", ExpiresAt: &past})).To(Succeed()) + Expect(adapter.Upsert(ctx, &syncedResponse{ID: "resp_live"})).To(Succeed()) + + out, err := adapter.List(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(HaveLen(1)) + Expect(out[0].ID).To(Equal("resp_live")) + }) + + It("removes a deleted response from the durable rows", func() { + Expect(adapter.Upsert(ctx, &syncedResponse{ID: "resp_gone"})).To(Succeed()) + Expect(adapter.Delete(ctx, "resp_gone")).To(Succeed()) + + out, err := adapter.List(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(BeEmpty()) + }) + + It("reports an unreachable database as an error and never as an empty list", func() { + Expect(adapter.Upsert(ctx, &syncedResponse{ID: "resp_before_outage"})).To(Succeed()) + + sqlDB, err := db.DB() + Expect(err).ToNot(HaveOccurred()) + Expect(sqlDB.Close()).To(Succeed()) + + // "No such response" and "the database could not be reached" are + // different facts. An empty slice here would make a re-hydrate replace + // the whole map with nothing, so a transient outage would 404 every + // response on this replica. + out, err := adapter.List(ctx) + Expect(err).To(HaveOccurred()) + Expect(out).To(BeNil()) + }) +}) diff --git a/core/http/endpoints/openresponses/store.go b/core/http/endpoints/openresponses/store.go index f703e54b1985..2abee4fcb004 100644 --- a/core/http/endpoints/openresponses/store.go +++ b/core/http/endpoints/openresponses/store.go @@ -61,6 +61,14 @@ type ResponseStore struct { replicaID string lifeCtx context.Context lifeCancel context.CancelFunc + + // purgeWG tracks the durable-metadata purge sweep so Close cannot return + // while it is still issuing statements. purgeTicks, when non-nil, replaces + // that sweep's wall-clock ticker: only a spec sets it, so the sweep can be + // driven one tick at a time instead of waited out or slept through. + // Guarded by mu, and read once when the sweep starts. + purgeWG sync.WaitGroup + purgeTicks <-chan time.Time } // StreamedEvent represents a buffered SSE event for streaming resume diff --git a/core/http/endpoints/openresponses/sync.go b/core/http/endpoints/openresponses/sync.go index bd15b39cd8b6..261affbd3d85 100644 --- a/core/http/endpoints/openresponses/sync.go +++ b/core/http/endpoints/openresponses/sync.go @@ -7,6 +7,7 @@ import ( "time" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/distributed" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/syncstate" "github.com/mudler/xlog" @@ -75,12 +76,28 @@ type responseCancelEvent struct { // - a wildcard subscription on the response-cancel subject, so a cancel that // lands on the wrong replica still reaches the context.CancelFunc. // -// The SyncedMap has no durable Store: responses are ephemeral, TTL-bounded -// state that today does not survive a process restart either, so peers converge -// through deltas alone. A replica that joins later does not learn about -// responses created before it started; that is the same visibility a client had -// before this change and strictly better than the 404 it got from every peer. -func (s *ResponseStore) EnableDistributed(ctx context.Context, nats messaging.MessagingClient, replicaID string) error { +// The SyncedMap is backed by a durable Store, and that is what makes a gap in +// the carrier survivable. Both carriers deliver at most once to CONNECTED +// subscribers and neither replays: a replica whose subscription was down while a +// response was created never receives that delta, and with deltas as the only +// convergence path it would answer 404 for that response forever while its peers +// answered 200. The Store is what the reconnect re-hydrate and the periodic +// reconcile actually read, so the gap closes rather than becoming permanent. +// +// What is NOT durable is unchanged and deliberate: the resume buffer and the +// CancelFunc never leave the owning replica (see syncedResponse), so the write +// through is one row per response state change, not one per generated token. +// +// The store is required rather than optional. This is called only from the +// distributed branch of route registration, so a nil store there is a wiring bug +// and not a deployment shape; tolerating it would silently restore the +// deltas-only map this parameter exists to replace, and a non-nil interface +// wrapping a nil pointer would instead surface as a panic on a request. +func (s *ResponseStore) EnableDistributed(ctx context.Context, nats messaging.MessagingClient, + replicaID string, store *distributed.ResponseMetadataStore) error { + if store == nil { + return errors.New("enabling cross-replica Open Responses: the store parameter is nil, so a reconnecting replica would have nothing to re-hydrate from") + } if nats == nil { return nil } @@ -91,9 +108,10 @@ func (s *ResponseStore) EnableDistributed(ctx context.Context, nats messaging.Me lifeCtx, lifeCancel := context.WithCancel(context.Background()) //#nosec G118 -- cancelled in Close() synced := syncstate.New(syncstate.Config[string, *syncedResponse]{ - Name: syncStateName, - Key: func(v *syncedResponse) string { return v.ID }, - Nats: nats, + Name: syncStateName, + Key: func(v *syncedResponse) string { return v.ID }, + Nats: nats, + Store: &responseMetadataStoreAdapter{store: store}, }) if err := synced.Start(ctx); err != nil { lifeCancel() @@ -121,10 +139,70 @@ func (s *ResponseStore) EnableDistributed(ctx context.Context, nats messaging.Me s.cancelSub = sub s.mu.Unlock() + s.startMetadataPurge(lifeCtx, store) + xlog.Info("Open Responses store replicating across replicas", "replica_id", replicaID) return nil } +// DefaultResponseMetadataPurgeInterval is how often a replica sweeps rows whose +// TTL has passed out of the durable metadata table. +// +// Every replica runs the sweep; the DELETE is idempotent and rows are selected +// on the database clock, so two replicas sweeping at once cost one extra +// statement and never disagree about which rows are dead. +const DefaultResponseMetadataPurgeInterval = 5 * time.Minute + +// startMetadataPurge launches the sweep goroutine on the store's own lifetime +// context, so Close stops it. +// +// purgeTicks is the seam that makes the sweep testable without a clock: a spec +// installs its own channel and sends one tick, which is deterministic where a +// five-minute ticker and a sleep are not. +func (s *ResponseStore) startMetadataPurge(lifeCtx context.Context, store *distributed.ResponseMetadataStore) { + s.mu.Lock() + ticks := s.purgeTicks + s.mu.Unlock() + + var stop func() + if ticks == nil { + t := time.NewTicker(DefaultResponseMetadataPurgeInterval) + ticks, stop = t.C, t.Stop + } + + s.purgeWG.Add(1) + go s.purgeExpiredMetadata(lifeCtx, store, ticks, stop) +} + +// purgeExpiredMetadata sweeps expired rows until the store is closed. +// +// A failed sweep is logged and retried on the next tick rather than ending the +// loop: an unreachable database is not evidence that there is nothing to purge, +// and a loop that exited on the first transient error would leave the table +// growing for the rest of the process's life. +func (s *ResponseStore) purgeExpiredMetadata(ctx context.Context, store *distributed.ResponseMetadataStore, + ticks <-chan time.Time, stop func()) { + defer s.purgeWG.Done() + if stop != nil { + defer stop() + } + for { + select { + case <-ctx.Done(): + return + case <-ticks: + n, err := store.PurgeExpired(ctx) + if err != nil { + xlog.Warn("failed to purge expired Open Responses metadata", "error", err) + continue + } + if n > 0 { + xlog.Debug("Purged expired Open Responses metadata", "rows", n) + } + } + } +} + // Close tears down the distributed wiring. It is idempotent so a test (or a // double shutdown) can call it more than once, and is a no-op for a standalone // store. @@ -141,6 +219,11 @@ func (s *ResponseStore) Close() error { if cancel != nil { cancel() } + // Wait for the purge sweep before returning: a Close that raced its own + // goroutine would let a sweep issue a statement against a database the + // caller has already torn down, which a spec sees as a flake and a + // deployment sees as an error line during shutdown. + s.purgeWG.Wait() if sub != nil { if err := sub.Unsubscribe(); err != nil { return err diff --git a/core/http/endpoints/openresponses/sync_test.go b/core/http/endpoints/openresponses/sync_test.go index ef0a4cad1a6c..2495d1043b3e 100644 --- a/core/http/endpoints/openresponses/sync_test.go +++ b/core/http/endpoints/openresponses/sync_test.go @@ -6,9 +6,12 @@ import ( "time" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/distributed" + "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/testutil" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gorm.io/gorm" ) // These specs model the two-replica topology from issue #10993: two independent @@ -18,6 +21,8 @@ import ( var _ = Describe("ResponseStore cross-replica", func() { var ( bus *testutil.FakeBus + db *gorm.DB + store *distributed.ResponseMetadataStore replicaA *ResponseStore replicaB *ResponseStore ctx context.Context @@ -27,11 +32,18 @@ var _ = Describe("ResponseStore cross-replica", func() { ctx = context.Background() bus = testutil.NewFakeBus() + // One database, two replicas, exactly as a deployment has it: the + // durable rows are shared and the in-memory maps are not. + db = testutil.SetupTestDB() + var err error + store, err = distributed.NewResponseMetadataStore(db) + Expect(err).ToNot(HaveOccurred()) + replicaA = NewResponseStore(0) replicaB = NewResponseStore(0) - Expect(replicaA.EnableDistributed(ctx, bus, "replica-a")).To(Succeed()) - Expect(replicaB.EnableDistributed(ctx, bus, "replica-b")).To(Succeed()) + Expect(replicaA.EnableDistributed(ctx, bus, "replica-a", store)).To(Succeed()) + Expect(replicaB.EnableDistributed(ctx, bus, "replica-b", store)).To(Succeed()) }) AfterEach(func() { @@ -190,6 +202,116 @@ var _ = Describe("ResponseStore cross-replica", func() { }) }) + Describe("re-hydrating after a gap in the carrier", func() { + // Both carriers deliver at most once to CONNECTED subscribers and + // neither replays. This is the only path by which a replica learns about + // a response whose delta it never received, and it is what Task 11's + // move of this family onto pgbus depends on. + It("restores from the durable rows what a missed delta had removed from memory", func() { + const id = "resp_rehydrate" + replicaA.Store(id, &schema.OpenResponsesRequest{Model: "test-model"}, newResponse(id, schema.ORStatusCompleted)) + Expect(replicaB.Get(id)).ToNot(BeNil()) + + // Drop it from both maps' memory the way a peer delta would, which + // leaves the durable row untouched: the apply path is memory-only. + Expect(bus.Publish(messaging.SubjectSyncStateDelta(syncStateName), + map[string]any{"op": "delete", "key": id})).To(Succeed()) + _, err := replicaB.Get(id) + Expect(err).To(HaveOccurred()) + + bus.TriggerReconnect() + + stored, err := replicaB.Get(id) + Expect(err).ToNot(HaveOccurred()) + Expect(stored).ToNot(BeNil()) + Expect(stored.Response.Status).To(Equal(schema.ORStatusCompleted)) + Expect(stored.Request).ToNot(BeNil()) + }) + + It("keeps what it already had when the durable source cannot be read", func() { + const id = "resp_outage" + replicaA.Store(id, &schema.OpenResponsesRequest{Model: "test-model"}, newResponse(id, schema.ORStatusCompleted)) + Expect(replicaB.Get(id)).ToNot(BeNil()) + + sqlDB, err := db.DB() + Expect(err).ToNot(HaveOccurred()) + Expect(sqlDB.Close()).To(Succeed()) + + // An unreachable database is not the same fact as an empty table. A + // re-hydrate that could not read must change nothing, or a transient + // outage would 404 every response this replica knows about. + bus.TriggerReconnect() + + stored, err := replicaB.Get(id) + Expect(err).ToNot(HaveOccurred()) + Expect(stored).ToNot(BeNil()) + }) + }) + + Describe("purging expired durable metadata", func() { + It("sweeps rows whose TTL has passed and stops when the store is closed", func() { + ticks := make(chan time.Time) + replica := NewResponseStore(0) + replica.purgeTicks = ticks + Expect(replica.EnableDistributed(ctx, testutil.NewFakeBus(), "replica-purge", store)).To(Succeed()) + + past := time.Now().Add(-time.Hour) + Expect(store.Upsert(ctx, &distributed.ResponseMetadataRecord{ + ID: "resp_expired_row", PayloadJSON: []byte(`{"id":"resp_expired_row"}`), ExpiresAt: &past, + })).To(Succeed()) + Expect(store.Upsert(ctx, &distributed.ResponseMetadataRecord{ + ID: "resp_live_row", PayloadJSON: []byte(`{"id":"resp_live_row"}`), + })).To(Succeed()) + + // A non-blocking send that retries: if EnableDistributed never + // started the sweep there is no receiver, and this fails by name + // instead of hanging the suite on an unbuffered send. + Eventually(ticks).Should(BeSent(time.Now()), "the purge sweep must be running to receive a tick") + + Eventually(func() ([]string, error) { + recs, err := store.ListUnexpired(ctx) + if err != nil { + return nil, err + } + out := make([]string, 0, len(recs)) + for i := range recs { + out = append(out, recs[i].ID) + } + return out, nil + }).Should(ConsistOf("resp_live_row")) + + var count int64 + Eventually(func() (int64, error) { + err := db.Model(&distributed.ResponseMetadataRecord{}). + Where("id = ?", "resp_expired_row").Count(&count).Error + return count, err + }).Should(BeZero()) + + // Close waits for the sweep, so a Close that returns is proof the + // goroutine is gone. If it never exited this would hang rather than + // pass, which is why nothing here polls for a flag. + done := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(done) + Expect(replica.Close()).To(Succeed()) + }() + Eventually(done).Should(BeClosed()) + }) + }) + + Describe("wiring", func() { + It("refuses to enable replication without a durable store", func() { + // A nil store here is a wiring bug, not a deployment shape: this is + // reached only from the distributed branch of route registration. + // Tolerating it would silently restore a deltas-only map, whose gap + // is permanent. + err := NewResponseStore(0).EnableDistributed(ctx, bus, "replica-c", nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("store")) + }) + }) + Describe("standalone mode", func() { It("keeps a store with no messaging client purely local", func() { standalone := NewResponseStore(0) diff --git a/core/http/routes/openresponses.go b/core/http/routes/openresponses.go index 18a283b82b04..1d3977d28550 100644 --- a/core/http/routes/openresponses.go +++ b/core/http/routes/openresponses.go @@ -8,6 +8,7 @@ import ( "github.com/mudler/LocalAI/core/http/endpoints/openresponses" "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/distributed" "github.com/mudler/xlog" ) @@ -23,8 +24,16 @@ func RegisterOpenResponsesRoutes(app *echo.Echo, // a cancel that the load balancer sends to a replica other than the // creator 404s, and the cancel never reaches the CancelFunc (#10993). // Standalone deployments skip this entirely and stay process-local. + // + // The durable store is what a replica re-hydrates from after its + // subscription missed a delta; without it the same response_id answers + // 404 here and 200 on the peer that created it, forever. + var responseStore *distributed.ResponseMetadataStore + if d.DistStores != nil { + responseStore = d.DistStores.Responses + } if err := openresponses.GetGlobalStore().EnableDistributed( - application.ApplicationConfig().Context, d.Nats, application.InstanceID()); err != nil { + application.ApplicationConfig().Context, d.Nats, application.InstanceID(), responseStore); err != nil { xlog.Error("Failed to enable cross-replica Open Responses store", "error", err) } } diff --git a/core/services/distributed/init.go b/core/services/distributed/init.go index ac28441e24ef..42511bbf1dd1 100644 --- a/core/services/distributed/init.go +++ b/core/services/distributed/init.go @@ -13,6 +13,12 @@ type Stores struct { FineTune *FineTuneStore Quant *QuantStore Skills *SkillStore + + // Responses is the durable backing for the responses.metadata SyncedMap. It + // is what a replica re-hydrates from after its listener misses a delta; + // without it a response created during the gap is invisible on this replica + // forever and the same response_id answers 404 here and 200 on a peer. + Responses *ResponseMetadataStore } // InitStores creates and migrates all Phase 4 distributed stores. @@ -37,11 +43,17 @@ func InitStores(db *gorm.DB) (*Stores, error) { return nil, fmt.Errorf("skills store: %w", err) } - xlog.Info("Distributed stores initialized (Gallery, FineTune, Quant, Skills)") + responses, err := NewResponseMetadataStore(db) + if err != nil { + return nil, fmt.Errorf("response metadata store: %w", err) + } + + xlog.Info("Distributed stores initialized (Gallery, FineTune, Quant, Skills, Responses)") return &Stores{ - Gallery: gallery, - FineTune: ft, - Quant: quant, - Skills: skills, + Gallery: gallery, + FineTune: ft, + Quant: quant, + Skills: skills, + Responses: responses, }, nil } diff --git a/core/services/distributed/responses.go b/core/services/distributed/responses.go new file mode 100644 index 000000000000..5d93d11de9e7 --- /dev/null +++ b/core/services/distributed/responses.go @@ -0,0 +1,134 @@ +package distributed + +import ( + "context" + "fmt" + "time" + + "github.com/mudler/LocalAI/core/services/advisorylock" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// ResponseMetadataRecord is the durable row behind the responses.metadata +// SyncedMap. +// +// PayloadJSON carries the whole projection as JSON rather than one column per +// field. A column-per-field schema would be a SECOND definition of what a peer +// may act on, and the two would drift the first time the projection gains a +// field: the map would broadcast the new field and hydrate without it, so a +// replica that had reconnected would serve a different response body from one +// that had not, with nothing failing anywhere. +// +// OwnerReplica and Owner are duplicated out of the payload as indexed columns +// because they are what an operator filters on when reading this table by hand +// ("which responses did the replica that just crashed own"). No query in this +// package reads them, so they cannot drift into a second source of truth for +// what a peer acts on: only PayloadJSON is ever decoded. +type ResponseMetadataRecord struct { + ID string `gorm:"primaryKey;size:64"` + OwnerReplica string `gorm:"size:64;index"` + Owner string `gorm:"size:64;index"` + PayloadJSON []byte `gorm:"type:bytea"` + ExpiresAt *time.Time `gorm:"index"` + CreatedAt time.Time `gorm:"index"` +} + +func (ResponseMetadataRecord) TableName() string { return "response_metadata" } + +// ResponseMetadataStore is the durable half of cross-replica response metadata. +// +// It exists because a broadcast carrier is at most once to CONNECTED listeners. +// A replica whose listener was down while a response was created never sees the +// delta, and without a table to re-hydrate from it serves 404 for that response +// forever while its peers serve 200. Every method here reports a database +// failure as an error and never as an empty result, because "no such response" +// and "the database could not be reached" are different facts and a hydrate that +// confused them would blank the map on a transient outage. +type ResponseMetadataStore struct { + db *gorm.DB +} + +// NewResponseMetadataStore creates a ResponseMetadataStore and migrates its +// table. +// +// The dialect is checked rather than assumed: ListUnexpired and PurgeExpired are +// spelled with now(), and on the SQLite single-binary path an unguarded now() +// fails at query time in a way that reads as a missing migration rather than as +// a store that was never meant to run there. +// +// The migration runs under the same advisory lock NewFineTuneStore uses, because +// several replicas start at once and concurrent AutoMigrate races. +func NewResponseMetadataStore(db *gorm.DB) (*ResponseMetadataStore, error) { + if db == nil { + return nil, fmt.Errorf("response metadata store: no database handle") + } + if name := db.Dialector.Name(); name != "postgres" { + return nil, fmt.Errorf("response metadata store requires PostgreSQL, this deployment runs on %q", name) + } + if err := advisorylock.WithLockCtx(context.Background(), db, advisorylock.KeySchemaMigrate, func() error { + return db.AutoMigrate(&ResponseMetadataRecord{}) + }); err != nil { + return nil, fmt.Errorf("migrating response_metadata: %w", err) + } + return &ResponseMetadataStore{db: db}, nil +} + +// Upsert idempotently inserts or replaces one row by primary key. +// +// created_at is deliberately NOT in the update set: the row is rewritten on +// every response state change (created, stored for background execution, status +// changed, cancelled), and updating it would make the column mean "last +// touched", which is not what an operator reading the table would take it for. +func (s *ResponseMetadataStore) Upsert(ctx context.Context, rec *ResponseMetadataRecord) error { + if rec == nil || rec.ID == "" { + return fmt.Errorf("response metadata upsert: record has no id") + } + if rec.CreatedAt.IsZero() { + rec.CreatedAt = time.Now() + } + return s.db.WithContext(ctx).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "id"}}, + DoUpdates: clause.AssignmentColumns([]string{"owner_replica", "owner", "payload_json", "expires_at"}), + }).Create(rec).Error +} + +// Delete removes one row. Deleting a row that is not there is not an error: the +// map's Delete is broadcast to every replica and any of them may reap an expired +// entry, so a second delete for the same id is expected traffic. +func (s *ResponseMetadataStore) Delete(ctx context.Context, id string) error { + return s.db.WithContext(ctx).Where("id = ?", id).Delete(&ResponseMetadataRecord{}).Error +} + +// ListUnexpired returns every row whose ExpiresAt is null or still in the +// future, measured on the DATABASE clock. +// +// The clock is the database's because every replica hydrating from this table +// must agree on which rows are live, and a Go-side cutoff makes that a property +// of whichever process asked. It is spelled `expires_at IS NULL OR expires_at > +// now()` and it is dialect-guarded in the constructor, because now() on the +// SQLite single-binary path reads as a missing migration rather than as an +// error. +func (s *ResponseMetadataStore) ListUnexpired(ctx context.Context) ([]ResponseMetadataRecord, error) { + var out []ResponseMetadataRecord + if err := s.db.WithContext(ctx). + Where("expires_at IS NULL OR expires_at > now()"). + Order("created_at"). + Find(&out).Error; err != nil { + return nil, fmt.Errorf("listing unexpired response metadata: %w", err) + } + return out, nil +} + +// PurgeExpired deletes rows whose ExpiresAt has passed and returns how many. +// It is the reason a table of ephemeral state does not grow forever, and it +// runs on the same DATABASE clock as ListUnexpired. +func (s *ResponseMetadataStore) PurgeExpired(ctx context.Context) (int64, error) { + res := s.db.WithContext(ctx). + Where("expires_at IS NOT NULL AND expires_at <= now()"). + Delete(&ResponseMetadataRecord{}) + if res.Error != nil { + return 0, fmt.Errorf("purging expired response metadata: %w", res.Error) + } + return res.RowsAffected, nil +} diff --git a/core/services/distributed/responses_test.go b/core/services/distributed/responses_test.go new file mode 100644 index 000000000000..ffe77c1b0c31 --- /dev/null +++ b/core/services/distributed/responses_test.go @@ -0,0 +1,290 @@ +package distributed_test + +import ( + "context" + "regexp" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + gormlogger "gorm.io/gorm/logger" + + "github.com/mudler/LocalAI/core/services/distributed" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// responseSQLRecorder captures the statements a call under test issues, so a +// spec can pin the SHAPE of one rather than only its effect. +// +// The shape is what has to be pinned for anything measured on the database +// clock. The test container shares this host's clock, so a Go-side cutoff and +// now() agree to the microsecond and no behavioural spec can tell them apart; +// the difference only shows up in a deployment whose replicas' clocks differ, +// which is every real one. +type responseSQLRecorder struct { + gormlogger.Interface + mu sync.Mutex + statements []string + errs []error +} + +func newResponseSQLRecorder() *responseSQLRecorder { + return &responseSQLRecorder{Interface: gormlogger.Default.LogMode(gormlogger.Silent)} +} + +func (r *responseSQLRecorder) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) { + sql, rows := fc() + r.mu.Lock() + r.statements = append(r.statements, sql) + if err != nil { + r.errs = append(r.errs, err) + } + r.mu.Unlock() + // Delegated so a failing statement is still reported the way gorm would + // report it: the instrument used to prove what the SQL does must not be the + // one thing that hides it erroring. + r.Interface.Trace(ctx, begin, func() (string, int64) { return sql, rows }, err) +} + +// reset forgets everything recorded so far, so a spec can ignore the migration +// and advisory-lock traffic the constructor issues and pin only the call under +// test. +func (r *responseSQLRecorder) reset() { + r.mu.Lock() + defer r.mu.Unlock() + r.statements = nil + r.errs = nil +} + +// only returns the single recorded statement, failing the spec if the call under +// test issued anything other than exactly one. A read-then-delete shape shows up +// here as two. +func (r *responseSQLRecorder) only() string { + r.mu.Lock() + defer r.mu.Unlock() + ExpectWithOffset(1, r.errs).To(BeEmpty(), "the recorded statement failed") + ExpectWithOffset(1, r.statements).To(HaveLen(1), "expected exactly one statement, got: %v", r.statements) + return r.statements[0] +} + +var _ = Describe("ResponseMetadataStore", func() { + var ( + db *gorm.DB + store *distributed.ResponseMetadataStore + ctx context.Context + ) + + newRecord := func(id string, expiresAt *time.Time) *distributed.ResponseMetadataRecord { + return &distributed.ResponseMetadataRecord{ + ID: id, + OwnerReplica: "replica-a", + Owner: "user-1", + PayloadJSON: []byte(`{"id":"` + id + `"}`), + ExpiresAt: expiresAt, + } + } + + ids := func(recs []distributed.ResponseMetadataRecord) []string { + out := make([]string, 0, len(recs)) + for i := range recs { + out = append(out, recs[i].ID) + } + return out + } + + BeforeEach(func() { + ctx = context.Background() + db = testutil.SetupTestDB() + var err error + store, err = distributed.NewResponseMetadataStore(db) + Expect(err).ToNot(HaveOccurred()) + }) + + Describe("construction", func() { + It("migrates and is idempotent when called twice on the same handle", func() { + second, err := distributed.NewResponseMetadataStore(db) + Expect(err).ToNot(HaveOccurred()) + Expect(second).ToNot(BeNil()) + + // The second construction must have left the first store's data + // alone: a re-run AutoMigrate that dropped and recreated the table + // would pass a "no error" assertion and lose every live response. + Expect(store.Upsert(ctx, newRecord("resp_idempotent", nil))).To(Succeed()) + recs, err := second.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(ids(recs)).To(ConsistOf("resp_idempotent")) + }) + + It("refuses a SQLite handle by naming the dialect", func() { + lite, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) + Expect(err).ToNot(HaveOccurred()) + + _, err = distributed.NewResponseMetadataStore(lite) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sqlite")) + Expect(err.Error()).To(ContainSubstring("PostgreSQL")) + }) + }) + + Describe("InitStores", func() { + It("builds a working response metadata store", func() { + // Pins the wiring: without it InitStores compiles, every other store + // works, and the map that needed this one silently falls back to + // nothing to re-hydrate from. + stores, err := distributed.InitStores(db) + Expect(err).ToNot(HaveOccurred()) + Expect(stores.Responses).ToNot(BeNil()) + + Expect(stores.Responses.Upsert(ctx, newRecord("resp_from_init", nil))).To(Succeed()) + recs, err := stores.Responses.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(ids(recs)).To(ConsistOf("resp_from_init")) + }) + }) + + Describe("Upsert", func() { + It("leaves one row carrying the second payload", func() { + first := newRecord("resp_upsert", nil) + Expect(store.Upsert(ctx, first)).To(Succeed()) + + second := newRecord("resp_upsert", nil) + second.PayloadJSON = []byte(`{"id":"resp_upsert","status":"completed"}`) + second.OwnerReplica = "replica-b" + Expect(store.Upsert(ctx, second)).To(Succeed()) + + recs, err := store.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(recs).To(HaveLen(1)) + Expect(string(recs[0].PayloadJSON)).To(Equal(`{"id":"resp_upsert","status":"completed"}`)) + Expect(recs[0].OwnerReplica).To(Equal("replica-b")) + }) + + It("refuses a record with no id rather than writing an anonymous row", func() { + Expect(store.Upsert(ctx, newRecord("", nil))).ToNot(Succeed()) + }) + }) + + Describe("Delete", func() { + It("removes the row", func() { + Expect(store.Upsert(ctx, newRecord("resp_delete", nil))).To(Succeed()) + + Expect(store.Delete(ctx, "resp_delete")).To(Succeed()) + + recs, err := store.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(recs).To(BeEmpty()) + }) + }) + + Describe("ListUnexpired", func() { + It("returns rows that never expire and rows still in the future, and not one already expired", func() { + future := time.Now().Add(time.Hour) + past := time.Now().Add(-time.Hour) + + Expect(store.Upsert(ctx, newRecord("resp_forever", nil))).To(Succeed()) + Expect(store.Upsert(ctx, newRecord("resp_future", &future))).To(Succeed()) + Expect(store.Upsert(ctx, newRecord("resp_past", &past))).To(Succeed()) + + recs, err := store.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(ids(recs)).To(ConsistOf("resp_forever", "resp_future")) + }) + + It("reports an unreachable database as an error and never as an empty list", func() { + // A missing row and an unreachable database are different facts. If + // this returned (nil, nil) a hydrate would replace the whole map + // with nothing and every response on this replica would 404 for as + // long as the outage lasted. + sqlDB, err := db.DB() + Expect(err).ToNot(HaveOccurred()) + Expect(sqlDB.Close()).To(Succeed()) + + recs, err := store.ListUnexpired(ctx) + Expect(err).To(HaveOccurred()) + Expect(recs).To(BeNil()) + }) + }) + + Describe("PurgeExpired", func() { + It("deletes exactly the expired row and reports nothing left to do on a second call", func() { + future := time.Now().Add(time.Hour) + past := time.Now().Add(-time.Hour) + + Expect(store.Upsert(ctx, newRecord("resp_forever", nil))).To(Succeed()) + Expect(store.Upsert(ctx, newRecord("resp_future", &future))).To(Succeed()) + Expect(store.Upsert(ctx, newRecord("resp_past", &past))).To(Succeed()) + + n, err := store.PurgeExpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + + recs, err := store.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(ids(recs)).To(ConsistOf("resp_forever", "resp_future")) + + n, err = store.PurgeExpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(0))) + }) + + It("reports an unreachable database as an error and never as zero rows purged", func() { + sqlDB, err := db.DB() + Expect(err).ToNot(HaveOccurred()) + Expect(sqlDB.Close()).To(Succeed()) + + _, err = store.PurgeExpired(ctx) + Expect(err).To(HaveOccurred()) + }) + }) + + // The expiry cutoff is the database's clock, not the asking process's. The + // container shares this host's clock, so no behavioural spec above can tell + // a Go-side time.Now() bind parameter from now(); these pin the statement + // text instead. + Describe("statement shape", func() { + var rec *responseSQLRecorder + + BeforeEach(func() { + rec = newResponseSQLRecorder() + var err error + store, err = distributed.NewResponseMetadataStore(db.Session(&gorm.Session{Logger: rec})) + Expect(err).ToNot(HaveOccurred()) + rec.reset() + }) + + It("compares expires_at against the database clock in ListUnexpired", func() { + _, err := store.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + + sql := rec.only() + Expect(sql).To(MatchRegexp(`(?i)expires_at\s+IS\s+NULL\s+OR\s+expires_at\s*>\s*now\(\)`)) + // A bound parameter in the comparison position is a Go-side cutoff + // wearing the same behaviour; gorm's logger explains binds into the + // text, so a time literal here is exactly that defect. + Expect(sql).ToNot(MatchRegexp(`expires_at\s*>\s*['$]`)) + }) + + It("compares expires_at against the database clock in PurgeExpired", func() { + _, err := store.PurgeExpired(ctx) + Expect(err).ToNot(HaveOccurred()) + + sql := rec.only() + Expect(sql).To(MatchRegexp(`(?i)expires_at\s+IS\s+NOT\s+NULL\s+AND\s+expires_at\s*<=\s*now\(\)`)) + Expect(sql).ToNot(MatchRegexp(`expires_at\s*<=\s*['$]`)) + }) + + It("issues one statement per call, so neither reads the clock into Go first", func() { + _, err := store.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(rec.only()).ToNot(BeEmpty()) + + rec.reset() + _, err = store.PurgeExpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(regexp.MustCompile(`(?i)^\s*delete`).MatchString(rec.only())).To(BeTrue()) + }) + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 015a79a7ace4..832710ef05a6 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -123,6 +123,31 @@ The peer link is served at `/api/cluster/peer` and authenticates with `LOCALAI_R **The peer link has no per-replica credential yet.** It checks the shared registration token and takes the replica id in `?id=` on trust. Anything already holding that token - every worker holds it - can therefore open a peer link, relay through it to every worker tunnel a replica owns, by declaring another replica's id displace that replica's inbound link, and hold sessions open against the per-session receive window, which the peer-link code sizes at roughly 31 GiB of unread data per session and which on this route is also a memory budget an attacker can point at one replica. Treat `LOCALAI_REGISTRATION_TOKEN` as a cluster-wide secret with the blast radius of the whole fleet: give it its own value per deployment, do not reuse it elsewhere, and keep `/api/cluster/peer` on a network only your replicas and workers can reach. Per-replica credentials for this route are planned. {{% /notice %}} +### Open Responses across replicas + +A response created by `POST /v1/responses` is held by the replica that served the request. A round-robin load balancer sends the follow-up poll, the `previous_response_id` chain and the cancel to any replica, so that metadata is replicated to every frontend and is also written to a `response_metadata` table in PostgreSQL. + +The table is what a replica re-hydrates from. Replication is a broadcast, and a broadcast reaches only the replicas that are subscribed at that moment: a replica whose subscription was down while a response was created never receives that notification. Without the table it would answer `404` for that response forever while its peers answered `200`. With it, the replica re-reads the table when its subscription comes back and converges. + +What crosses replicas and what does not: + +| State | Replicated | Why | +|-------|-----------|-----| +| Request body, response resource, output items, status, owner, expiry | Yes, in memory and in `response_metadata` | A poll, a `previous_response_id` chain or an item lookup on any replica has to resolve | +| Cancellation | The request is, the `CancelFunc` is not | The cancel is forwarded to the owning replica, which holds the function that stops generation | +| Stream resume buffer (`starting_after`) | No | It is the full token log; replicating it would put every generated token on the bus. A resume request that lands on the wrong replica is refused with an explicit error, never with a silently truncated event list | + +**Retention.** Rows carry the expiry of the response they describe, and each replica sweeps expired rows every five minutes. The expiry comes from the Open Responses store TTL, which is **`0` (no expiration) by default**: + +```yaml +environment: + LOCALAI_OPEN_RESPONSES_STORE_TTL: "1h" +``` + +Leave it at `0` in distributed mode and nothing ever expires: `response_metadata` grows for the life of the deployment, and every replica that restarts re-hydrates every response the cluster has ever created. Set a TTL that matches how long clients are allowed to poll for a response. + +These rows carry the request body and the generated output, not just identifiers. They live in the same database as the rest of the cluster state, and the TTL above is the only thing that removes them. + ### Worker tunnels A worker can open one long-lived, multiplexed tunnel to the frontend instead of listening on a port of its own. It dials `GET /api/cluster/connect?id=`, the connection is upgraded to a WebSocket, and every subsequent request the frontend makes to that worker travels as a stream inside it. Nothing dials *into* the worker, so a worker behind NAT, in another Kubernetes cluster or on a laptop needs no inbound port and no reachable address. diff --git a/tests/e2e/distributed/phase4_test.go b/tests/e2e/distributed/phase4_test.go index 5863b64e059e..19f747ab1e35 100644 --- a/tests/e2e/distributed/phase4_test.go +++ b/tests/e2e/distributed/phase4_test.go @@ -1,6 +1,9 @@ package distributed_test import ( + "context" + "time" + "github.com/mudler/LocalAI/core/services/distributed" . "github.com/onsi/ginkgo/v2" @@ -220,4 +223,29 @@ var _ = Describe("Phase 4: MCP, Skills, Gallery, Fine-Tuning", Label("Distribute Expect(err).To(HaveOccurred()) }) }) + + Context("Open Responses Metadata", func() { + It("should keep live rows and purge expired ones on the database clock", func() { + ctx := context.Background() + Expect(stores.Responses).ToNot(BeNil()) + + past := time.Now().Add(-time.Hour) + future := time.Now().Add(time.Hour) + Expect(stores.Responses.Upsert(ctx, &distributed.ResponseMetadataRecord{ + ID: "resp_live", OwnerReplica: "replica-a", PayloadJSON: []byte(`{"id":"resp_live"}`), ExpiresAt: &future, + })).To(Succeed()) + Expect(stores.Responses.Upsert(ctx, &distributed.ResponseMetadataRecord{ + ID: "resp_dead", OwnerReplica: "replica-a", PayloadJSON: []byte(`{"id":"resp_dead"}`), ExpiresAt: &past, + })).To(Succeed()) + + live, err := stores.Responses.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(live).To(HaveLen(1)) + Expect(live[0].ID).To(Equal("resp_live")) + + purged, err := stores.Responses.PurgeExpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(purged).To(Equal(int64(1))) + }) + }) }) From a3443c31bb54b7843b9d230f865c6bdba74dcf7e Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 21:54:42 +0000 Subject: [PATCH 092/153] fix(distributed): give one tenant's agent tasks a subject of their own Every AgentJobService built its tasks SyncedMap with the name "agent.tasks", and there is one service per user. So every tenant published on and subscribed to the same subject, state.agent-tasks.delta, and SyncedMap.apply scopes nothing: a task tenant A created was written into tenant B's in-memory map on every replica, and ListTasks reads that map. Nothing repaired it short of a process restart. The subject now carries the tenant in a token of its own, state...delta. Four tokens where the unscoped builder makes three, deliberately: SubjectMatches compares token count before anything else, so a tenant's subject and the cluster-wide one cannot cross-match, and neither can two tenants. Putting the tenant inside the name token would not do that, because the sanitizer folds '.' to '-' and the only filter that could then span tenants is state.*.delta, which spans every other family too. The rule is stated once. subscribeFilters calls publishSubject rather than restating the subject, so a map cannot end up publishing scoped and subscribing unscoped, which would leak exactly as before while every publish assertion passed. The one case that decides on its own is the cluster-wide administrative view: it hydrates from every tenant's rows, so it also takes the per-tenant wildcard, or it would be stale the moment any tenant wrote. A tenant hydrates from its own rows and applies only its own deltas. PerTenant defaults to false, so finetune, quantization and the responses store keep the subject they have. The second half of the same defect was the delete. taskStoreAdapter.Delete called DeleteTask(id) and JobStore deleted by primary key with no user predicate, reachable from DELETE /api/agent/tasks/:id, which takes the id off the URL. A tenant who learned another tenant's task id destroyed that tenant's row. The user id now travels with the id and lands as a user_id predicate. Empty stays the administrative any-owner scope, the same thing an empty id already means for ListTasks and ListJobs. A foreign delete removes nothing and returns no error: not yours and not there are the same answer to the caller, and neither is a store failure. SetUserID rebuilds the tasks map for the same reason SetTaskSyncNATS does. GetJobs happens to set the user id first, nothing enforced it, and with the order reversed the map would be built with an empty tenant and put that user's tasks back on the cluster-wide subject. Both halves predate this programme; they are surfaced here rather than caused. Neither is fully closed for a deployment with the agent pool off, where the task routes are still served by one cluster-wide service that every authenticated caller shares; that is a separate gap and it is documented. testutil.FakeBus grew a real defect this was the first change to trip: Unsubscribe matched on the filter string, so with two subscribers on one filter, closing one deafened the other. Subscriptions now carry an id. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .../http/endpoints/localai/agent_jobs_test.go | 106 +++++++++++ core/services/agentpool/agent_jobs.go | 17 +- core/services/agentpool/job_persister.go | 5 +- core/services/agentpool/job_persister_db.go | 4 +- core/services/agentpool/job_persister_file.go | 4 +- core/services/agentpool/job_persister_test.go | 17 +- core/services/agentpool/task_sync_test.go | 161 ++++++++++++++++ core/services/agentpool/task_syncstore.go | 8 +- core/services/jobs/store.go | 15 +- core/services/jobs/store_test.go | 45 ++++- core/services/messaging/subjects.go | 27 +++ core/services/messaging/subjects_wire_test.go | 70 +++++++ core/services/syncstate/syncstate.go | 78 ++++++-- core/services/syncstate/syncstate_test.go | 173 ++++++++++++++++++ core/services/testutil/fakebus.go | 14 +- core/services/testutil/fakebus_test.go | 25 +++ docs/content/features/distributed-mode.md | 17 ++ 17 files changed, 758 insertions(+), 28 deletions(-) create mode 100644 core/http/endpoints/localai/agent_jobs_test.go diff --git a/core/http/endpoints/localai/agent_jobs_test.go b/core/http/endpoints/localai/agent_jobs_test.go new file mode 100644 index 000000000000..9e34eb6af387 --- /dev/null +++ b/core/http/endpoints/localai/agent_jobs_test.go @@ -0,0 +1,106 @@ +package localai + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + + "github.com/labstack/echo/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/auth" + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/agentpool" + "github.com/mudler/LocalAI/core/services/testutil" + "github.com/mudler/LocalAI/pkg/system" +) + +// DELETE /api/agent/tasks/:id takes the task id straight off the URL, so the +// only thing standing between one tenant and another tenant's task is how the +// service behind the route is scoped. +// +// The assertion is on the SURVIVING ROW rather than on the status code. Before +// the task subject carried its tenant, the handler answered 200 and destroyed +// the row; it now answers 404 because the task is not in this tenant's map. A +// status assertion would have seen nothing wrong in the first case, and reading +// the row keeps the spec honest whichever status a later change picks. +var _ = Describe("DELETE /api/agent/tasks/:id across tenants", func() { + var ( + e *echo.Echo + app *application.Application + other *agentpool.AgentJobService + cancel context.CancelFunc + taskID string + ) + + BeforeEach(func() { + tmp := GinkgoT().TempDir() + st, err := system.GetSystemState( + system.WithModelPath(filepath.Join(tmp, "models")), + system.WithBackendPath(filepath.Join(tmp, "backends")), + ) + Expect(err).ToNot(HaveOccurred()) + + var ctx context.Context + ctx, cancel = context.WithCancel(context.Background()) + app, err = application.New(config.WithContext(ctx), config.WithSystemState(st)) + Expect(err).ToNot(HaveOccurred()) + + // One carrier, two tenants. The service the route reaches belongs to + // u1; `other` is u2's, living on another replica as far as the bus is + // concerned. + bus := testutil.NewFakeBus() + + mine := app.AgentJobService() + Expect(mine).ToNot(BeNil()) + mine.SetUserID("u1") + mine.SetTaskSyncNATS(bus) + Expect(mine.LoadTasksFromFile()).To(Succeed()) + + otherDir := GinkgoT().TempDir() + otherCfg := config.NewApplicationConfig( + config.WithDynamicConfigDir(otherDir), + config.WithContext(ctx), + ) + otherCfg.SystemState = st + other = agentpool.NewAgentJobServiceWithPaths(otherCfg, nil, nil, nil, + filepath.Join(otherDir, "tasks.json"), filepath.Join(otherDir, "jobs.json")) + other.SetUserID("u2") + other.SetTaskSyncNATS(bus) + Expect(other.LoadTasksFromFile()).To(Succeed()) + + taskID, err = other.CreateTask(schema.Task{Name: "u2 only", Model: "m", Prompt: "p"}) + Expect(err).ToNot(HaveOccurred()) + + e = echo.New() + // Stand in for the auth middleware the real route runs behind: it is + // what puts the caller into the echo context, and getJobService reads + // the tenant from there. + e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + c.Set("auth_user", &auth.User{ID: "u1", Role: auth.RoleUser}) + return next(c) + } + }) + e.DELETE("/api/agent/tasks/:id", DeleteTaskEndpoint(app)) + }) + + AfterEach(func() { + cancel() + }) + + It("leaves the owning tenant's task readable", func() { + req := httptest.NewRequest(http.MethodDelete, "/api/agent/tasks/"+taskID, nil) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + + got, err := other.GetTask(taskID) + Expect(err).ToNot(HaveOccurred(), "tenant u2's task must survive a delete issued by tenant u1") + Expect(got.Name).To(Equal("u2 only")) + Expect(other.ListTasks()).To(HaveLen(1)) + }) +}) diff --git a/core/services/agentpool/agent_jobs.go b/core/services/agentpool/agent_jobs.go index 59850981a619..2ba9f9a90e81 100644 --- a/core/services/agentpool/agent_jobs.go +++ b/core/services/agentpool/agent_jobs.go @@ -96,9 +96,18 @@ func (s *AgentJobService) SetDistributedBackends(dispatcher DistributedDispatche s.dispatcher = dispatcher } -// SetUserID sets the user ID for per-user scoping of DB queries. +// SetUserID sets the user ID for per-user scoping of DB queries, and rebuilds +// the tasks map so the id lands in its subject. +// +// The rebuild is what makes the two setters order-independent. Without it the +// map keeps whichever tenant it was built with, so wiring that happened to call +// SetTaskSyncNATS first would publish this user's tasks on the CLUSTER-WIDE +// subject and every other tenant would apply them. Like SetTaskSyncNATS, this +// is only ever called before Start / hydrate, while the map is still empty, so +// rebuilding loses no state. func (s *AgentJobService) SetUserID(id string) { s.userID = id + s.buildTasksMap() } // SetDistributedJobStore sets the database-backed job store for persisting tasks/jobs. @@ -130,6 +139,12 @@ func (s *AgentJobService) buildTasksMap() { Key: func(t schema.Task) string { return t.ID }, Nats: s.taskNats, Store: &taskStoreAdapter{svc: s}, + // There is one AgentJobService per user, so this map is per-tenant and + // its deltas must not reach another tenant's copy. The empty userID is + // the cluster-wide administrative service, which hydrates from every + // tenant's rows and therefore also applies every tenant's deltas. + PerTenant: true, + Tenant: s.userID, }) } diff --git a/core/services/agentpool/job_persister.go b/core/services/agentpool/job_persister.go index 1f36f8a7bd12..6025737ae80c 100644 --- a/core/services/agentpool/job_persister.go +++ b/core/services/agentpool/job_persister.go @@ -12,7 +12,10 @@ import ( type JobPersister interface { // Write-through persistence (called after in-memory mutation) SaveTask(userID string, task schema.Task) error - DeleteTask(taskID string) error + // DeleteTask removes one task. The user id scopes the removal in DB mode; + // an empty user id is the administrative "any owner" scope, matching what + // it already means for LoadTasks and ListJobs. + DeleteTask(userID, taskID string) error SaveJob(userID string, job schema.Job) error DeleteJob(jobID string) error diff --git a/core/services/agentpool/job_persister_db.go b/core/services/agentpool/job_persister_db.go index 643b80e69eb4..739f1d605521 100644 --- a/core/services/agentpool/job_persister_db.go +++ b/core/services/agentpool/job_persister_db.go @@ -19,8 +19,8 @@ func (p *dbJobPersister) SaveTask(userID string, task schema.Task) error { return p.store.SaveTask(rec) } -func (p *dbJobPersister) DeleteTask(taskID string) error { - return p.store.DeleteTask(taskID) +func (p *dbJobPersister) DeleteTask(userID, taskID string) error { + return p.store.DeleteTask(userID, taskID) } func (p *dbJobPersister) SaveJob(userID string, job schema.Job) error { diff --git a/core/services/agentpool/job_persister_file.go b/core/services/agentpool/job_persister_file.go index b161c442be0d..ec29c9b6a023 100644 --- a/core/services/agentpool/job_persister_file.go +++ b/core/services/agentpool/job_persister_file.go @@ -41,7 +41,9 @@ func (p *fileJobPersister) SaveTask(_ string, task schema.Task) error { return p.writeTasksLocked() } -func (p *fileJobPersister) DeleteTask(taskID string) error { +// DeleteTask ignores the user id: the file persister writes one file per user, +// so the file it was constructed with already IS the scope. +func (p *fileJobPersister) DeleteTask(_ string, taskID string) error { p.mu.Lock() defer p.mu.Unlock() delete(p.taskSet, taskID) diff --git a/core/services/agentpool/job_persister_test.go b/core/services/agentpool/job_persister_test.go index 646104db626c..6aa0cfce7e18 100644 --- a/core/services/agentpool/job_persister_test.go +++ b/core/services/agentpool/job_persister_test.go @@ -53,7 +53,7 @@ var _ = Describe("JobPersister", func() { Expect(p.SaveTask("", schema.Task{ID: "t1", Name: "Keep"})).To(Succeed()) Expect(p.SaveTask("", schema.Task{ID: "t2", Name: "Delete"})).To(Succeed()) - Expect(p.DeleteTask("t2")).To(Succeed()) + Expect(p.DeleteTask("", "t2")).To(Succeed()) data, err := os.ReadFile(p.tasksFile) Expect(err).NotTo(HaveOccurred()) @@ -167,12 +167,25 @@ var _ = Describe("JobPersister", func() { task := schema.Task{ID: "t1", Name: "Gone", Model: "m", Prompt: "p"} Expect(p.SaveTask("user1", task)).To(Succeed()) - Expect(p.DeleteTask("t1")).To(Succeed()) + Expect(p.DeleteTask("user1", "t1")).To(Succeed()) _, err := p.store.GetTask("t1") Expect(err).To(HaveOccurred()) // record not found }) + It("DeleteTask leaves another user's row in the DB", func() { + // The DB persister is the one shared across tenants, so it is the + // one that has to carry the owner down to the query. + task := schema.Task{ID: "t1", Name: "Theirs", Model: "m", Prompt: "p"} + Expect(p.SaveTask("user1", task)).To(Succeed()) + + Expect(p.DeleteTask("user2", "t1")).To(Succeed()) + + rec, err := p.store.GetTask("t1") + Expect(err).NotTo(HaveOccurred(), "user1's row must survive user2's delete") + Expect(rec.UserID).To(Equal("user1")) + }) + It("SaveJob persists to PostgreSQL", func() { job := schema.Job{ID: "j1", TaskID: "t1", Status: schema.JobStatusPending} Expect(p.SaveJob("user1", job)).To(Succeed()) diff --git a/core/services/agentpool/task_sync_test.go b/core/services/agentpool/task_sync_test.go index d42a197f68de..dc65c4097c57 100644 --- a/core/services/agentpool/task_sync_test.go +++ b/core/services/agentpool/task_sync_test.go @@ -9,6 +9,7 @@ package agentpool import ( "context" + "sync" "time" . "github.com/onsi/ginkgo/v2" @@ -150,3 +151,163 @@ var _ = Describe("AgentJobService task cross-replica sync", func() { }) }) }) + +// newTaskSyncServiceForTenant builds a per-user AgentJobService the way +// UserServicesManager.GetJobs does: a user id, then the shared bus. Each +// service gets its own task file so nothing converges except over the bus. +func newTaskSyncServiceForTenant(bus messaging.MessagingClient, userID string) *AgentJobService { + tmpDir := GinkgoT().TempDir() + sysState := &system.SystemState{} + sysState.Model.ModelsPath = tmpDir + appConfig := config.NewApplicationConfig( + config.WithDynamicConfigDir(tmpDir), + config.WithContext(context.Background()), + ) + appConfig.SystemState = sysState + + svc := NewAgentJobServiceWithPaths(appConfig, nil, nil, nil, + tmpDir+"/tasks.json", tmpDir+"/jobs.json") + svc.SetUserID(userID) + svc.SetTaskSyncNATS(bus) + return svc +} + +// recordingTaskPersister records what DeleteTask was asked to delete and for +// whom, then delegates. The in-memory map is NOT the authority on ownership - +// the store is - so the argument the delete carries has to be pinned on its own +// rather than inferred from a map lookup that never happens once the subject is +// scoped. +type recordingTaskPersister struct { + JobPersister + mu sync.Mutex + calls []taskDeleteCall +} + +type taskDeleteCall struct { + userID string + taskID string +} + +func (p *recordingTaskPersister) DeleteTask(userID, taskID string) error { + p.mu.Lock() + p.calls = append(p.calls, taskDeleteCall{userID: userID, taskID: taskID}) + p.mu.Unlock() + return p.JobPersister.DeleteTask(userID, taskID) +} + +func (p *recordingTaskPersister) recorded() []taskDeleteCall { + p.mu.Lock() + defer p.mu.Unlock() + return append([]taskDeleteCall(nil), p.calls...) +} + +var _ = Describe("AgentJobService per-tenant task isolation", func() { + var ( + bus *testutil.FakeBus + u1, u2 *AgentJobService + ) + + BeforeEach(func() { + // One bus, two tenants: the topology every distributed deployment has, + // where UserServicesManager builds one AgentJobService per user id and + // every one of them talks over the same carrier. + bus = testutil.NewFakeBus() + u1 = newTaskSyncServiceForTenant(bus, "u1") + u2 = newTaskSyncServiceForTenant(bus, "u2") + Expect(u1.Start(context.Background())).To(Succeed()) + Expect(u2.Start(context.Background())).To(Succeed()) + }) + + AfterEach(func() { + Expect(u1.Stop()).To(Succeed()) + Expect(u2.Stop()).To(Succeed()) + }) + + It("does not put tenant u1's task into tenant u2's list", func() { + id, err := u1.CreateTask(schema.Task{Name: "Private", Model: "m", Prompt: "p"}) + Expect(err).NotTo(HaveOccurred()) + + Expect(u2.ListTasks()).To(BeEmpty(), "tenant u2 must not see tenant u1's task") + _, err = u2.GetTask(id) + Expect(err).To(MatchError(ErrTaskNotFound)) + + Expect(u1.ListTasks()).To(HaveLen(1), "the owning tenant still has its own task") + }) + + It("does not let tenant u1's delete remove tenant u2's task", func() { + idU2, err := u2.CreateTask(schema.Task{Name: "Theirs", Model: "m", Prompt: "p"}) + Expect(err).NotTo(HaveOccurred()) + idU1, err := u1.CreateTask(schema.Task{Name: "Mine", Model: "m", Prompt: "p"}) + Expect(err).NotTo(HaveOccurred()) + + Expect(u1.DeleteTask(idU1)).To(Succeed()) + + _, err = u2.GetTask(idU2) + Expect(err).NotTo(HaveOccurred(), "tenant u2's task must survive tenant u1's delete") + }) + + It("publishes on the tenant's own subject and not the cluster-wide one", func() { + // The wiring pin. buildTasksMap declaring PerTenant/Tenant is a struct + // literal a refactor can drop without breaking the build, so the + // subject it produces is asserted by name here. + _, err := u1.CreateTask(schema.Task{Name: "Named", Model: "m", Prompt: "p"}) + Expect(err).NotTo(HaveOccurred()) + + Expect(bus.PublishCount(messaging.SubjectSyncStateTenantDelta("agent.tasks", "u1"))).To(Equal(1)) + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("agent.tasks"))).To(Equal(0)) + }) + + It("passes the owning tenant to the persister on delete", func() { + rec := &recordingTaskPersister{JobPersister: u1.persister} + u1.persister = rec + + id, err := u1.CreateTask(schema.Task{Name: "Scoped", Model: "m", Prompt: "p"}) + Expect(err).NotTo(HaveOccurred()) + Expect(u1.DeleteTask(id)).To(Succeed()) + + Expect(rec.recorded()).To(Equal([]taskDeleteCall{{userID: "u1", taskID: id}}), + "an unscoped delete reaches every tenant's row with the same primary key") + }) +}) + +var _ = Describe("AgentJobService same-tenant replicas", func() { + It("converges two replicas of the same tenant", func() { + // The positive half. Scoping the subject must not cost the feature the + // map exists for: two frontend replicas serving the SAME user still + // have to see each other's writes. + bus := testutil.NewFakeBus() + a := newTaskSyncServiceForTenant(bus, "u1") + b := newTaskSyncServiceForTenant(bus, "u1") + Expect(a.Start(context.Background())).To(Succeed()) + Expect(b.Start(context.Background())).To(Succeed()) + defer func() { + Expect(a.Stop()).To(Succeed()) + Expect(b.Stop()).To(Succeed()) + }() + + id, err := a.CreateTask(schema.Task{Name: "Shared", Model: "m", Prompt: "p"}) + Expect(err).NotTo(HaveOccurred()) + + got, err := b.GetTask(id) + Expect(err).NotTo(HaveOccurred(), "the other replica of the same tenant must see the task") + Expect(got.Name).To(Equal("Shared")) + }) + + It("scopes the subject whichever order the setters are called in", func() { + // UserServicesManager.GetJobs happens to call SetUserID before + // SetTaskSyncNATS. Nothing enforces that order, and a map built from a + // user id that had not been set yet publishes cluster-wide - the leak, + // reintroduced by a line move. + bus := testutil.NewFakeBus() + svc := newTaskSyncService(bus) + svc.SetUserID("u1") + Expect(svc.Start(context.Background())).To(Succeed()) + defer func() { Expect(svc.Stop()).To(Succeed()) }() + + _, err := svc.CreateTask(schema.Task{Name: "Ordered", Model: "m", Prompt: "p"}) + Expect(err).NotTo(HaveOccurred()) + + Expect(bus.PublishCount(messaging.SubjectSyncStateTenantDelta("agent.tasks", "u1"))).To(Equal(1)) + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("agent.tasks"))).To(Equal(0)) + }) +}) diff --git a/core/services/agentpool/task_syncstore.go b/core/services/agentpool/task_syncstore.go index ef8f3f8ccfdc..e90fa9f99b57 100644 --- a/core/services/agentpool/task_syncstore.go +++ b/core/services/agentpool/task_syncstore.go @@ -42,6 +42,12 @@ func (a *taskStoreAdapter) Upsert(_ context.Context, task schema.Task) error { // Delete write-through removes a task locally; the SyncedMap then broadcasts the // removal to peers. +// +// The user id travels with the id, read live from the service exactly as List +// and Upsert already read it. A delete by primary key alone reaches every +// tenant's row with that key, and the in-memory map is not the authority on who +// owns a row - the store is - so the predicate belongs on the query and not on +// a lookup before it. func (a *taskStoreAdapter) Delete(_ context.Context, id string) error { - return a.svc.persister.DeleteTask(id) + return a.svc.persister.DeleteTask(a.svc.userID, id) } diff --git a/core/services/jobs/store.go b/core/services/jobs/store.go index 08fa83d8f367..655847990c6b 100644 --- a/core/services/jobs/store.go +++ b/core/services/jobs/store.go @@ -143,9 +143,18 @@ func (s *JobStore) ListTasks(userID string) ([]TaskRecord, error) { return tasks, nil } -// DeleteTask removes a task by ID. -func (s *JobStore) DeleteTask(id string) error { - return s.db.Where("id = ?", id).Delete(&TaskRecord{}).Error +// DeleteTask removes a task by ID, scoped to its owner. +// +// An empty userID is the administrative "any owner" scope, the same convention +// ListTasks and ListJobs already use. A non-empty one deletes NOTHING when the +// row belongs to somebody else and reports no error: "not yours" and "not +// there" are the same answer to the caller, and neither is a store failure. +func (s *JobStore) DeleteTask(userID, id string) error { + q := s.db.Where("id = ?", id) + if userID != "" { + q = q.Where("user_id = ?", userID) + } + return q.Delete(&TaskRecord{}).Error } // ListCronTasks returns all tasks that have a cron schedule and are enabled. diff --git a/core/services/jobs/store_test.go b/core/services/jobs/store_test.go index 11b015fb2bb9..19f0971c2c17 100644 --- a/core/services/jobs/store_test.go +++ b/core/services/jobs/store_test.go @@ -284,8 +284,10 @@ var _ = Describe("JobStore", func() { }) Describe("DeleteTask", func() { - It("removes a task and confirms it is gone", func() { - task := &TaskRecord{ + var task *TaskRecord + + BeforeEach(func() { + task = &TaskRecord{ UserID: "user-del", Name: "to-delete", Model: "gpt-4", @@ -296,10 +298,45 @@ var _ = Describe("JobStore", func() { _, err := store.GetTask(task.ID) Expect(err).ToNot(HaveOccurred()) + }) + + It("removes a task and confirms it is gone", func() { + Expect(store.DeleteTask("user-del", task.ID)).To(Succeed()) + + _, err := store.GetTask(task.ID) + Expect(err).To(HaveOccurred()) + }) - Expect(store.DeleteTask(task.ID)).To(Succeed()) + It("deletes nothing when the caller is not the owner", func() { + // "not yours" and "not there" are the same outcome for the caller + // and neither is an error: a delete by primary key with no user + // predicate lets any tenant who learns another tenant's task id + // destroy that tenant's row. + Expect(store.DeleteTask("someone-else", task.ID)).To(Succeed()) - _, err = store.GetTask(task.ID) + survivor, err := store.GetTask(task.ID) + Expect(err).ToNot(HaveOccurred(), "the owner's row must still be readable") + Expect(survivor.UserID).To(Equal("user-del")) + }) + + It("leaves the owner's other rows alone when a foreign delete is refused", func() { + second := &TaskRecord{UserID: "user-del", Name: "keep-me", Model: "gpt-4", Prompt: "test"} + Expect(store.CreateTask(second)).To(Succeed()) + + Expect(store.DeleteTask("someone-else", task.ID)).To(Succeed()) + + remaining, err := store.ListTasks("user-del") + Expect(err).ToNot(HaveOccurred()) + Expect(remaining).To(HaveLen(2)) + }) + + It("deletes for the administrative empty user id", func() { + // The empty user id already means "every user" for ListTasks and + // ListJobs in this store; the delete has to read it the same way or + // the cluster-wide agent job service can no longer clean up. + Expect(store.DeleteTask("", task.ID)).To(Succeed()) + + _, err := store.GetTask(task.ID) Expect(err).To(HaveOccurred()) }) }) diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index 229bba93cc9b..7dfc07ba2ad4 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -405,6 +405,33 @@ func SubjectSyncStateDelta(name string) string { const subjectSyncStatePrefix = "state." +// SubjectSyncStateTenantDelta returns the delta subject for ONE tenant's slice +// of a per-tenant SyncedMap: state...delta. +// +// Four tokens where the unscoped builder makes three, deliberately. A filter +// matches on token COUNT first (see SubjectMatches), so the cluster-wide map's +// three-token subject and a tenant's four-token subject can never cross-match, +// and neither can two different tenants. Putting the tenant INSIDE the name +// token would not do that: sanitizeSubjectToken replaces '.' with '-', so +// "agent.tasks."+userID collapses to one token and the only filter that could +// span tenants would be state.*.delta, which spans every other family too. +// +// An empty tenant is not routed here - the SyncedMap sends that case to the +// cluster-wide subject - and would mint an empty token, which ValidFilter +// refuses at subscribe time rather than leaving a subscription that never +// fires. +func SubjectSyncStateTenantDelta(name, tenant string) string { + return subjectSyncStatePrefix + sanitizeSubjectToken(name) + "." + sanitizeSubjectToken(tenant) + ".delta" +} + +// SubjectSyncStateTenantWildcard returns the filter matching EVERY tenant's +// deltas for name: state..*.delta. Only the cluster-wide administrative +// view subscribes to it; a tenant map that used it would be back to reading +// every other tenant's writes. +func SubjectSyncStateTenantWildcard(name string) string { + return subjectSyncStatePrefix + sanitizeSubjectToken(name) + ".*.delta" +} + // Prefix-Cache Routing Sync (Pub/Sub - broadcast to all frontends) // // Frontends share prefix-cache observations so a request routed to any replica diff --git a/core/services/messaging/subjects_wire_test.go b/core/services/messaging/subjects_wire_test.go index 2c90254e8ff5..33a69aa17995 100644 --- a/core/services/messaging/subjects_wire_test.go +++ b/core/services/messaging/subjects_wire_test.go @@ -44,3 +44,73 @@ var _ = Describe("backend.install reply wire format", func() { Expect(string(out)).ToNot(ContainSubstring("address")) }) }) + +// The per-tenant SyncedMap subject is a data-boundary, not a naming +// convenience: one subject shared by N tenants means tenant A's delta is +// applied into tenant B's in-memory map on every replica. These pins hold the +// SHAPE that makes cross-matching impossible, because the shape is the whole +// mechanism. +var _ = Describe("per-tenant SyncedMap subjects", func() { + It("puts the tenant in its own token", func() { + Expect(SubjectSyncStateTenantDelta("agent.tasks", "u1")).To(Equal("state.agent-tasks.u1.delta")) + }) + + It("builds the every-tenant filter from the same name", func() { + Expect(SubjectSyncStateTenantWildcard("agent.tasks")).To(Equal("state.agent-tasks.*.delta")) + }) + + It("sanitizes the tenant so an id cannot inject a token", func() { + // A tenant id is user-controlled in the sense that it comes from the + // auth store rather than from this package. If it were interpolated + // raw, an id containing '.' would mint extra tokens and an id + // containing '*' would mint a wildcard that spans every other tenant. + Expect(SubjectSyncStateTenantDelta("agent.tasks", "a.b*c>d")).To(Equal("state.agent-tasks.a-b-c-d.delta")) + }) + + It("cannot cross-match the cluster-wide subject", func() { + // SubjectMatches compares token COUNT first, so three tokens and four + // tokens are unrelated no matter what the tokens say. This is the + // assertion the four-token choice exists for. + Expect(SubjectMatches(SubjectSyncStateTenantWildcard("agent.tasks"), SubjectSyncStateDelta("agent.tasks"))).To(BeFalse(), + "the every-tenant filter must not swallow the cluster-wide subject") + Expect(SubjectMatches(SubjectSyncStateDelta("agent.tasks"), SubjectSyncStateTenantDelta("agent.tasks", "u1"))).To(BeFalse(), + "the cluster-wide subject must not double as a tenant filter") + }) + + It("matches every tenant through the wildcard", func() { + // The complement of the token-count assertion, and the half the + // cluster-wide administrative view depends on: four tokens are only + // the right shape if the wildcard actually reaches them. + Expect(SubjectMatches( + SubjectSyncStateTenantWildcard("agent.tasks"), + SubjectSyncStateTenantDelta("agent.tasks", "u1"), + )).To(BeTrue()) + Expect(SubjectMatches( + SubjectSyncStateTenantWildcard("agent.tasks"), + SubjectSyncStateTenantDelta("agent.tasks", "u2"), + )).To(BeTrue()) + }) + + It("cannot cross-match another tenant", func() { + Expect(SubjectMatches( + SubjectSyncStateTenantDelta("agent.tasks", "u1"), + SubjectSyncStateTenantDelta("agent.tasks", "u2"), + )).To(BeFalse(), "tenant u1's filter must never match tenant u2's subject") + }) + + It("cannot cross-match a different state family", func() { + Expect(SubjectMatches( + SubjectSyncStateTenantWildcard("agent.tasks"), + SubjectSyncStateTenantDelta("finetune.jobs", "u1"), + )).To(BeFalse()) + }) + + It("refuses an empty tenant at subscribe time rather than silently spanning", func() { + // Nothing in the tree calls the builder with an empty tenant - the + // SyncedMap routes the empty case to the cluster-wide subject instead. + // Pin what happens anyway, because the failure mode of a subject with + // an empty token is a subscription that never fires, and a caller must + // meet that as a refusal rather than as silence. + Expect(ValidFilter(SubjectSyncStateTenantDelta("agent.tasks", ""))).To(MatchError(ErrUnsupportedFilter)) + }) +}) diff --git a/core/services/syncstate/syncstate.go b/core/services/syncstate/syncstate.go index 809177d40fe7..8e70138d2bd0 100644 --- a/core/services/syncstate/syncstate.go +++ b/core/services/syncstate/syncstate.go @@ -43,6 +43,24 @@ type Config[K comparable, V any] struct { Loader func(ctx context.Context) ([]V, error) // source when there is no Store (e.g. disk reload) OnApply func(op string, k K, v V) // optional hook after an applied change (e.g. ShutdownModel) Reconcile time.Duration // optional periodic re-hydrate; 0 = off + + // PerTenant declares that this map is instantiated once per tenant, so its + // deltas must not reach another tenant's copy. A map with PerTenant false + // keeps exactly the subject it has today, which is why the finetune, quant + // and responses adopters need no change. + PerTenant bool + + // Tenant scopes this map when PerTenant is set. Non-empty publishes and + // subscribes on that tenant's subject ALONE. + // + // Empty with PerTenant set is the CLUSTER-WIDE view: it publishes on the + // unscoped subject, and it subscribes on that subject AND on the per-tenant + // wildcard. That asymmetry is not an oversight. This map hydrates from a + // Store that returns every tenant's rows, so a view that hydrates across + // tenants must apply deltas across tenants or it is stale the moment any + // tenant writes. A tenant map hydrates from its own rows and must apply + // only its own deltas. + Tenant string } // delta is the JSON wire envelope broadcast on every local mutation. Value is @@ -66,7 +84,9 @@ type SyncedMap[K comparable, V any] struct { mu sync.RWMutex data map[K]V - sub Subscription + // subs holds every filter this map applies deltas from. Only the + // cluster-wide view of a per-tenant map has more than one. + subs []Subscription // lifeCtx outlives Start's argument: a reconnect callback or reconcile tick // can fire long after Start returns, so they must not be tied to a ctx the @@ -84,10 +104,33 @@ func New[K comparable, V any](cfg Config[K, V]) *SyncedMap[K, V] { return &SyncedMap[K, V]{cfg: cfg, data: make(map[K]V)} } -func (m *SyncedMap[K, V]) subject() string { +// publishSubject is the subject a local mutation broadcasts on, and the SINGLE +// definition of "the subject this map's tenant owns". subscribeFilters reads it +// rather than restating the rule, so a per-tenant map cannot end up publishing +// on one subject and subscribing on another - the split that would leak exactly +// as before while every publish assertion still passed. +func (m *SyncedMap[K, V]) publishSubject() string { + if m.cfg.PerTenant && m.cfg.Tenant != "" { + return messaging.SubjectSyncStateTenantDelta(m.cfg.Name, m.cfg.Tenant) + } return messaging.SubjectSyncStateDelta(m.cfg.Name) } +// subscribeFilters is the filter or filters this map applies deltas from. +// +// Every map subscribes to what it publishes on. The cluster-wide view of a +// per-tenant map additionally takes the tenant wildcard, because it hydrates +// from every tenant's rows and would otherwise be stale the moment any tenant +// wrote. No other case gets a second filter: a tenant that took the wildcard +// would read every other tenant's writes, which is the leak this exists to +// close. +func (m *SyncedMap[K, V]) subscribeFilters() []string { + if m.cfg.PerTenant && m.cfg.Tenant == "" { + return []string{m.publishSubject(), messaging.SubjectSyncStateTenantWildcard(m.cfg.Name)} + } + return []string{m.publishSubject()} +} + // Start hydrates from the source, subscribes for peer deltas, registers a // reconnect re-hydrate (when the client supports it), and starts the optional // reconcile ticker. @@ -102,11 +145,13 @@ func (m *SyncedMap[K, V]) Start(ctx context.Context) error { m.lifeCtx, m.cancel = context.WithCancel(context.Background()) // #nosec G118 -- cancel is invoked in Close() if m.cfg.Nats != nil { - sub, err := messaging.SubscribeJSON(m.cfg.Nats, m.subject(), m.apply) - if err != nil { - return err + for _, filter := range m.subscribeFilters() { + sub, err := messaging.SubscribeJSON(m.cfg.Nats, filter, m.apply) + if err != nil { + return err + } + m.subs = append(m.subs, sub) } - m.sub = sub // nats.go transparently resubscribes on reconnect, but it cannot know we // kept derived in-memory state that may have drifted while the link was @@ -129,16 +174,27 @@ func (m *SyncedMap[K, V]) Start(ctx context.Context) error { return nil } -// Close unsubscribes and stops the reconcile ticker. +// Close unsubscribes every filter and stops the reconcile ticker. It keeps +// going after a failure and returns the first error, so one unsubscribe that +// fails cannot strand the others: a live handler on a closed map keeps writing +// into memory nobody reads, and for the cluster-wide view that handler is the +// one carrying other tenants' rows. func (m *SyncedMap[K, V]) Close() error { if m.cancel != nil { m.cancel() } m.wg.Wait() - if m.sub != nil { - return m.sub.Unsubscribe() + var firstErr error + for _, sub := range m.subs { + if sub == nil { + continue + } + if err := sub.Unsubscribe(); err != nil && firstErr == nil { + firstErr = err + } } - return nil + m.subs = nil + return firstErr } // Set updates the value locally, writes through the Store, then broadcasts. @@ -210,7 +266,7 @@ func (m *SyncedMap[K, V]) publish(op string, k K, v V) { if m.cfg.Nats == nil { return } - if err := m.cfg.Nats.Publish(m.subject(), delta[K, V]{Op: op, Key: k, Value: v}); err != nil { + if err := m.cfg.Nats.Publish(m.publishSubject(), delta[K, V]{Op: op, Key: k, Value: v}); err != nil { xlog.Warn("syncstate: failed to broadcast delta", "name", m.cfg.Name, "op", op, "error", err) } } diff --git a/core/services/syncstate/syncstate_test.go b/core/services/syncstate/syncstate_test.go index 1e31db41b55a..47116a1b4804 100644 --- a/core/services/syncstate/syncstate_test.go +++ b/core/services/syncstate/syncstate_test.go @@ -289,3 +289,176 @@ var _ = Describe("SyncedMap", func() { }) }) }) + +// Per-tenant maps. +// +// A SyncedMap instantiated once per tenant used to publish and subscribe on the +// SAME subject as every other tenant's copy, so a delta was applied into every +// other tenant's in-memory map. These specs assert the NEGATIVE case - tenant A +// must not receive tenant B's traffic - at least as hard as the positive one, +// because the negative case is the data-exposure bug and the positive case is +// only the feature. +var _ = Describe("SyncedMap per-tenant subjects", func() { + ctx := context.Background() + + newTenantMap := func(bus *testutil.FakeBus, tenant string) *syncstate.SyncedMap[string, *job] { + m := syncstate.New(syncstate.Config[string, *job]{ + Name: stateName, + Key: jobKey, + Nats: bus, + PerTenant: true, + Tenant: tenant, + }) + Expect(m.Start(ctx)).To(Succeed()) + return m + } + + Describe("three maps on one bus", func() { + var ( + bus *testutil.FakeBus + cluster, u1, u2 *syncstate.SyncedMap[string, *job] + ) + + BeforeEach(func() { + bus = testutil.NewFakeBus() + // Tenant "" is the cluster-wide administrative view: it hydrates + // from every tenant's rows, so it must also apply every tenant's + // deltas or it is stale the moment any tenant writes. + cluster = newTenantMap(bus, "") + u1 = newTenantMap(bus, "u1") + u2 = newTenantMap(bus, "u2") + }) + + AfterEach(func() { + Expect(u2.Close()).To(Succeed()) + Expect(u1.Close()).To(Succeed()) + Expect(cluster.Close()).To(Succeed()) + }) + + It("keeps a Set on u1 out of u2 while the cluster-wide view sees it", func() { + Expect(u1.Set(ctx, &job{ID: "j-u1", Status: "running"})).To(Succeed()) + + // Assert on Get, not on a publish counter: a counter is satisfied + // by a map that received the delta and applied it under a key the + // spec never looks up. + _, leaked := u2.Get("j-u1") + Expect(leaked).To(BeFalse(), "tenant u2 must never receive tenant u1's delta") + + _, mine := u1.Get("j-u1") + Expect(mine).To(BeTrue(), "the originating tenant keeps its own write") + + _, admin := cluster.Get("j-u1") + Expect(admin).To(BeTrue(), "the cluster-wide view must apply every tenant's delta") + }) + + It("keeps a Set on u2 out of u1 while the cluster-wide view sees it", func() { + // The mirror direction, spelled out rather than assumed: a filter + // built from the wrong tenant would pass one direction only. + Expect(u2.Set(ctx, &job{ID: "j-u2", Status: "running"})).To(Succeed()) + + _, leaked := u1.Get("j-u2") + Expect(leaked).To(BeFalse(), "tenant u1 must never receive tenant u2's delta") + + _, admin := cluster.Get("j-u2") + Expect(admin).To(BeTrue(), "the cluster-wide view must apply every tenant's delta") + }) + + It("keeps a Delete on u1 out of u2", func() { + // A delete carries only op+key, so a leaked delete removes a row + // from a map that never held the create - the same boundary, the + // destructive direction. + Expect(u2.Set(ctx, &job{ID: "shared-id", Status: "u2s"})).To(Succeed()) + Expect(u1.Set(ctx, &job{ID: "shared-id", Status: "u1s"})).To(Succeed()) + + Expect(u1.Delete(ctx, "shared-id")).To(Succeed()) + + _, survives := u2.Get("shared-id") + Expect(survives).To(BeTrue(), "tenant u1 deleting its own key must not erase tenant u2's") + }) + + It("keeps a Set on the cluster-wide map out of both tenant maps", func() { + Expect(cluster.Set(ctx, &job{ID: "j-admin", Status: "running"})).To(Succeed()) + + _, in1 := u1.Get("j-admin") + Expect(in1).To(BeFalse()) + _, in2 := u2.Get("j-admin") + Expect(in2).To(BeFalse()) + }) + + It("publishes a tenant's mutation on that tenant's subject alone", func() { + // The publish half of the rule, asserted by subject name so a map + // that published unscoped but subscribed scoped - which would look + // like a dead bus rather than a leak - fails here by name. + Expect(u1.Set(ctx, &job{ID: "j-u1", Status: "running"})).To(Succeed()) + + Expect(bus.PublishCount(messaging.SubjectSyncStateTenantDelta(stateName, "u1"))).To(Equal(1)) + Expect(bus.PublishCount(deltaSubject())).To(Equal(0), + "a tenant must not put anything on the cluster-wide subject") + Expect(bus.PublishCount(messaging.SubjectSyncStateTenantDelta(stateName, "u2"))).To(Equal(0)) + }) + + It("publishes the cluster-wide map's mutation on the unscoped subject", func() { + Expect(cluster.Set(ctx, &job{ID: "j-admin", Status: "running"})).To(Succeed()) + + Expect(bus.PublishCount(deltaSubject())).To(Equal(1)) + }) + }) + + Describe("Close on the cluster-wide map", func() { + It("drops BOTH of its subscriptions", func() { + // The cluster-wide view is the only map holding two subscriptions, + // so it is the only one where a Close that unsubscribes the first + // and returns leaves a live handler writing into a closed map. + bus := testutil.NewFakeBus() + cluster := newTenantMap(bus, "") + peerCluster := newTenantMap(bus, "") + u1 := newTenantMap(bus, "u1") + defer func() { + Expect(peerCluster.Close()).To(Succeed()) + Expect(u1.Close()).To(Succeed()) + }() + + Expect(cluster.Close()).To(Succeed()) + + // The tenant-wildcard subscription. + Expect(u1.Set(ctx, &job{ID: "after-close-tenant", Status: "x"})).To(Succeed()) + _, got := cluster.Get("after-close-tenant") + Expect(got).To(BeFalse(), "the tenant-wildcard subscription must be gone after Close") + + // The unscoped subscription. + Expect(peerCluster.Set(ctx, &job{ID: "after-close-unscoped", Status: "x"})).To(Succeed()) + _, got = cluster.Get("after-close-unscoped") + Expect(got).To(BeFalse(), "the unscoped subscription must be gone after Close") + }) + }) + + Describe("PerTenant false", func() { + It("keeps exactly the subject it has today", func() { + // finetune.jobs, quantization and the responses store are unscoped + // adopters. Pin that this change moved none of them. + bus := testutil.NewFakeBus() + a := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus}) + b := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus}) + Expect(a.Start(ctx)).To(Succeed()) + Expect(b.Start(ctx)).To(Succeed()) + tenant := newTenantMap(bus, "u1") + defer func() { + Expect(a.Close()).To(Succeed()) + Expect(b.Close()).To(Succeed()) + Expect(tenant.Close()).To(Succeed()) + }() + + Expect(a.Set(ctx, &job{ID: "unscoped", Status: "running"})).To(Succeed()) + + Expect(bus.PublishCount(deltaSubject())).To(Equal(1)) + _, peer := b.Get("unscoped") + Expect(peer).To(BeTrue(), "unscoped adopters must keep converging with each other") + _, crossed := tenant.Get("unscoped") + Expect(crossed).To(BeFalse(), "an unscoped map must not reach a tenant map") + + Expect(tenant.Set(ctx, &job{ID: "scoped", Status: "running"})).To(Succeed()) + _, back := a.Get("scoped") + Expect(back).To(BeFalse(), "a tenant map must not reach an unscoped map") + }) + }) +}) diff --git a/core/services/testutil/fakebus.go b/core/services/testutil/fakebus.go index f2890d7ecc29..d178fc3ee166 100644 --- a/core/services/testutil/fakebus.go +++ b/core/services/testutil/fakebus.go @@ -22,6 +22,11 @@ import ( type FakeBus struct { mu sync.Mutex subs []fakeBusSub + // nextSubID names each subscription uniquely. Unsubscribe used to match on + // the filter string, which silently removed a DIFFERENT subscriber's entry + // whenever two subscribers shared one filter - and two subscribers sharing + // one filter is exactly the topology every cross-replica spec builds. + nextSubID int64 // publishCounts records how many messages were published per subject, so a // spec can assert the echo-loop guard (an applied delta must not re-publish). publishCounts map[string]int @@ -33,6 +38,7 @@ type FakeBus struct { } type fakeBusSub struct { + id int64 subject string handler func([]byte) } @@ -73,11 +79,14 @@ type fakeBusSubscription struct { subRef fakeBusSub } +// Unsubscribe removes THIS subscription and no other. Identity is the id +// minted at Subscribe time, not the filter: a map that subscribes to the same +// filter as a peer must not be able to deafen the peer by closing itself. func (s *fakeBusSubscription) Unsubscribe() error { s.bus.mu.Lock() defer s.bus.mu.Unlock() for i, candidate := range s.bus.subs { - if candidate.subject == s.subRef.subject { + if candidate.id == s.subRef.id { s.bus.subs = append(s.bus.subs[:i], s.bus.subs[i+1:]...) return nil } @@ -91,8 +100,9 @@ func (b *FakeBus) Subscribe(subject string, handler func([]byte)) (messaging.Sub if err := messaging.ValidFilter(subject); err != nil { return nil, err } - sub := fakeBusSub{subject: subject, handler: handler} b.mu.Lock() + b.nextSubID++ + sub := fakeBusSub{id: b.nextSubID, subject: subject, handler: handler} b.subs = append(b.subs, sub) b.mu.Unlock() return &fakeBusSubscription{bus: b, subRef: sub}, nil diff --git a/core/services/testutil/fakebus_test.go b/core/services/testutil/fakebus_test.go index 977883d73e60..ef7c3a3c19cd 100644 --- a/core/services/testutil/fakebus_test.go +++ b/core/services/testutil/fakebus_test.go @@ -54,3 +54,28 @@ var _ = Describe("FakeBus subject routing", func() { Expect(sub).To(BeNil()) }) }) + +var _ = Describe("FakeBus subscription identity", func() { + It("unsubscribing one of two subscribers on the same filter leaves the other receiving", func() { + // Two SyncedMaps in one process legitimately share a filter (the + // cluster-wide view and any other unscoped map on the same name). If + // Unsubscribe removed the first entry matching the FILTER, closing one + // map would silently deafen the other, and the resulting spec failure + // would read as a broken carrier rather than as a broken double. + bus := testutil.NewFakeBus() + + firstSeen := 0 + first, err := bus.Subscribe("state.jobs.delta", func([]byte) { firstSeen++ }) + Expect(err).ToNot(HaveOccurred()) + + secondSeen := 0 + _, err = bus.Subscribe("state.jobs.delta", func([]byte) { secondSeen++ }) + Expect(err).ToNot(HaveOccurred()) + + Expect(first.Unsubscribe()).To(Succeed()) + Expect(bus.Publish("state.jobs.delta", map[string]string{"op": "set"})).To(Succeed()) + + Expect(firstSeen).To(Equal(0), "the unsubscribed handler must not fire") + Expect(secondSeen).To(Equal(1), "the surviving subscriber must still receive") + }) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 832710ef05a6..bc3c629ce2b4 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -148,6 +148,23 @@ Leave it at `0` in distributed mode and nothing ever expires: `response_metadata These rows carry the request body and the generated output, not just identifiers. They live in the same database as the rest of the cluster state, and the TTL above is the only thing that removes them. +### Agent tasks are scoped to their tenant + +Every frontend replica keeps agent task definitions in memory so that `GET /api/agent/tasks` answers from any replica. That in-memory copy is kept current by a broadcast on the cluster bus, and the broadcast carries the owning user in the subject: + +| Map | Subject it publishes on | Subjects it applies | +|-----|------------------------|---------------------| +| One user's tasks | `state.agent-tasks..delta` | that subject alone | +| The administrative, cluster-wide view | `state.agent-tasks.delta` | `state.agent-tasks.delta` and `state.agent-tasks.*.delta` | + +The user id is its own subject token, so one user's subject can never match another user's. A user's agent tasks are therefore visible only to that user and to the administrative view, which is the same scope the `agent_tasks` table already applies to reads. + +`DELETE /api/agent/tasks/{id}` is scoped the same way. The delete carries the calling user down to the database, so a request naming a task id that belongs to another user removes nothing and answers `404`. The administrative view keeps the unscoped delete, matching how an empty user id already means "every user" for the task and job listings. + +Deployments that ran a release before this scoping existed may have in-memory copies of other users' tasks on their replicas. Nothing is written to the database by that, and a restart of the frontend clears it. + +Per-user scoping needs the agent pool running, because that is what creates the per-user services. With `LOCALAI_DISABLE_AGENTS=true`, the agent task routes are still served, and they are served by one cluster-wide service that every authenticated caller shares. + ### Worker tunnels A worker can open one long-lived, multiplexed tunnel to the frontend instead of listening on a port of its own. It dials `GET /api/cluster/connect?id=`, the connection is upgraded to a WebSocket, and every subsequent request the frontend makes to that worker travels as a stream inside it. Nothing dials *into* the worker, so a worker behind NAT, in another Kubernetes cluster or on a laptop needs no inbound port and no reachable address. From 27e2ead7188f96a2153ee4ee908caba406963b2f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 3 Sep 2026 22:50:54 +0000 Subject: [PATCH 093/153] feat(distributed): carry the state.*.delta families on PostgreSQL syncstate.Config held one carrier field typed as the NATS client, so a pgbus.Bus could not be handed to a SyncedMap at all: it satisfies messaging.Broadcaster and not MessagingClient. The durable re-hydration path built for the responses map therefore had a NATS-only consumer and nothing in the build said so. The field becomes Bus messaging.Broadcaster, SubscribeJSON moves to its own file and relaxes its parameter to Broadcaster, and the four adopters fan out over PostgreSQL LISTEN/NOTIFY: fine-tune jobs, quantization jobs, agent tasks with their per-tenant children, and Open Responses metadata. A new spec proves it on a real database, over two Bus instances on two pinned listener connections: a Set and a Delete carry, a payload past the 8000-byte notification cap comes back byte identical through the spill row, two families sharing one LISTEN channel stay separate, and a terminated listener re-hydrates a row written while it was gone. The five sites that each chose a carrier for an adopter are collapsed into one DistributedServices.Broadcast() accessor. Five field reads were five chances to leave one family on NATS with nothing failing, because messaging.Client satisfies Broadcaster too. The accessor also refuses to hand out a nil pgbus.Bus wrapped in a non-nil interface, which every adopter would read as "broadcast" and dereference on the first Set. SetTaskSyncNATS and SetJobSyncNATS are renamed to SetTaskSyncBus and SetJobSyncBus so a missed wiring site fails to compile. The response metadata table gains a retention of its own, defaulting to 24 hours. It inherited the Open Responses store TTL, which defaults to 0 meaning no expiration. Zero is defensible for a map that dies with the process and is not for a table: the table grew for the life of the deployment and a restarting replica re-hydrated every response the cluster had ever created. A row that names its own expiry is still judged on that column alone, and "this row is dead" now has one SQL spelling that PurgeExpired deletes by and ListUnexpired is the negation of, so a hydrate cannot resurrect what a sweep has already retired. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/agent_jobs.go | 7 +- core/application/application.go | 2 +- core/application/distributed.go | 27 ++ core/application/distributed_test.go | 63 ++++ core/application/startup.go | 4 +- core/http/app.go | 27 +- .../http/endpoints/localai/agent_jobs_test.go | 4 +- .../endpoints/openresponses/metadata_store.go | 13 +- core/http/endpoints/openresponses/store.go | 2 +- core/http/endpoints/openresponses/sync.go | 23 +- .../http/endpoints/openresponses/sync_test.go | 27 ++ core/http/routes/openresponses.go | 2 +- core/services/agentpool/agent_jobs.go | 45 +-- core/services/agentpool/task_sync_test.go | 93 +++++- core/services/agentpool/user_services.go | 25 +- core/services/distributed/responses.go | 87 +++++- core/services/distributed/responses_test.go | 122 +++++++- core/services/finetune/service.go | 17 +- core/services/finetune/service_test.go | 32 +- core/services/messaging/client.go | 13 - core/services/messaging/json.go | 26 ++ core/services/quantization/service.go | 17 +- core/services/quantization/service_test.go | 26 +- core/services/syncstate/syncstate.go | 57 +++- core/services/syncstate/syncstate_pg_test.go | 293 ++++++++++++++++++ core/services/syncstate/syncstate_test.go | 57 +++- core/services/testutil/fakebus.go | 20 ++ docs/content/features/distributed-mode.md | 43 ++- .../distributed/syncstate_distributed_test.go | 43 ++- 29 files changed, 1051 insertions(+), 166 deletions(-) create mode 100644 core/services/messaging/json.go create mode 100644 core/services/syncstate/syncstate_pg_test.go diff --git a/core/application/agent_jobs.go b/core/application/agent_jobs.go index f380b075002a..f8afe8c02efe 100644 --- a/core/application/agent_jobs.go +++ b/core/application/agent_jobs.go @@ -37,8 +37,11 @@ func (a *Application) RestartAgentJobService() error { if d.JobStore != nil { agentJobService.SetDistributedJobStore(d.JobStore) } - // Keep agent tasks consistent across replicas (same client the dispatcher uses). - agentJobService.SetTaskSyncNATS(d.Nats) + // Keep agent tasks consistent across replicas, on the deployment's + // broadcast carrier. This is the restart path and it is a second site + // for the same rule: a fix applied only in startup.go leaves every + // service the settings UI restarts on whatever carrier it picked here. + agentJobService.SetTaskSyncBus(d.Broadcast()) } // Start the service diff --git a/core/application/application.go b/core/application/application.go index b49851d4780f..6a4f99f32a7a 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -680,7 +680,7 @@ func (a *Application) StartAgentPool() { } // Keep per-user agent tasks consistent across replicas (nil in standalone). if d := a.Distributed(); d != nil { - usm.SetJobSyncNATS(d.Nats) + usm.SetJobSyncBus(d.Broadcast()) } aps.SetUserServicesManager(usm) diff --git a/core/application/distributed.go b/core/application/distributed.go index 336153705dc4..e803e1768b58 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -762,6 +762,33 @@ func requireBroadcastCarrier(ds *DistributedServices) error { return nil } +// Broadcast is the ONE place a wiring site gets the deployment's fan-out +// carrier, and it exists so that "this family travels on the broadcast carrier +// and not on NATS" is decided once instead of at every adopter. +// +// It was five sites before this: the fine-tune service, the quantization +// service, the agent-task setter (twice, on two startup paths), the per-user +// services manager and the Open Responses store. Every one of them takes a +// messaging.Broadcaster, and *messaging.Client satisfies that interface too, so +// a site left holding ds.Nats compiles, starts, publishes and is delivered - +// onto a carrier the deployment is being taken off. Nothing would fail until +// NATS went away. Collapsing the choice to one function makes it a fact a spec +// can pin, which five scattered field reads were not. +// +// The return is the interface and not *pgbus.Bus on purpose: handing a nil +// *pgbus.Bus to an adopter would produce a non-nil interface wrapping a nil +// pointer, and every adopter reads a nil carrier as "standalone, do not +// broadcast". A typed nil would instead panic on the first Set. initDistributed +// already refuses to return a deployment with no carrier (see +// requireBroadcastCarrier), so the nil branch here is belt and braces for a +// zero-valued struct in a test. +func (ds *DistributedServices) Broadcast() messaging.Broadcaster { + if ds == nil || ds.Bus == nil { + return nil + } + return ds.Bus +} + // newBroadcastBus opens the deployment's fan-out carrier on the auth database. // // The DSN is cfg.Auth.DatabaseURL and it may never be anything else. A second diff --git a/core/application/distributed_test.go b/core/application/distributed_test.go index 6002a91043af..edcf7115a534 100644 --- a/core/application/distributed_test.go +++ b/core/application/distributed_test.go @@ -9,7 +9,9 @@ import ( . "github.com/onsi/gomega" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/syncstate" "github.com/mudler/LocalAI/core/services/testutil" ) @@ -108,3 +110,64 @@ var _ = Describe("shutting the distributed services down", func() { Expect(bus.IsConnected()).To(BeFalse()) }) }) + +// The one place the four state.*.delta families are told which carrier they +// travel on. +// +// It was five field reads before this: the fine-tune service, the quantization +// service, the agent-task setter on two startup paths, the per-user services +// manager and the Open Responses store. Every one of them takes a +// messaging.Broadcaster, which *messaging.Client satisfies too, so a site left +// holding ds.Nats compiled, started, published and was delivered onto the +// carrier the deployment is being taken off, and nothing failed until NATS did. +// Collapsing the choice into one function is what makes it a fact these specs +// can hold. +var _ = Describe("handing the broadcast carrier to its adopters", func() { + It("returns the carrier the deployment opened and never the NATS client", func() { + db, dsn := testutil.SetupTestDBWithDSN() + cfg := &config.ApplicationConfig{} + cfg.Auth.DatabaseURL = dsn + bus, err := newBroadcastBus(context.Background(), cfg, db) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(bus.Close) + + // A NATS client is present on the struct, exactly as it is in a real + // deployment for as long as the request/reply and queue halves survive. + // Identity, not "is a Broadcaster": both fields satisfy that interface. + ds := &DistributedServices{Nats: &messaging.Client{}, Bus: bus} + + Expect(ds.Broadcast()).To(BeIdenticalTo(messaging.Broadcaster(bus))) + }) + + It("returns an interface that reads as absent, not a typed nil, when there is no carrier", func() { + // Every adopter branches on `bus == nil` to mean standalone. A nil + // *pgbus.Bus placed in an interface is NOT nil, so that branch would be + // skipped and the first Set would panic on a request rather than at + // boot. + // + // Compared with == and not with BeNil(). Gomega's BeNil reports a nil + // POINTER inside an interface as nil, so it passes on exactly the value + // this spec exists to reject; the first draft of this spec did, and the + // mutation that removed the guard stayed green. + var ds *DistributedServices + Expect(ds.Broadcast() == nil).To(BeTrue(), "a nil deployment must yield an interface that is itself nil") + Expect((&DistributedServices{}).Broadcast() == nil).To(BeTrue(), + "a deployment with no carrier must yield an interface that is itself nil, not one wrapping a nil *pgbus.Bus") + }) + + It("gives an adopter a carrier-less map rather than one that panics on the first write", func() { + // The consequence, driven through the component every adopter builds. + // A typed nil satisfies `!= nil`, so Start subscribes on it and Set + // publishes on it, and both dereference a nil *pgbus.Bus on a request + // path rather than at boot. + m := syncstate.New(syncstate.Config[string, string]{ + Name: "test.jobs", + Key: func(v string) string { return v }, + Bus: (&DistributedServices{}).Broadcast(), + }) + Expect(m.Start(context.Background())).To(Succeed()) + DeferCleanup(func() { Expect(m.Close()).To(Succeed()) }) + + Expect(func() { Expect(m.Set(context.Background(), "v")).To(Succeed()) }).ToNot(Panic()) + }) +}) diff --git a/core/application/startup.go b/core/application/startup.go index c790a1403617..b985d7e55f0d 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -328,8 +328,8 @@ func New(opts ...config.AppOption) (*Application, error) { application.agentJobService.SetDistributedBackends(distSvc.Dispatcher) application.agentJobService.SetDistributedJobStore(distSvc.JobStore) // Keep agent tasks consistent across replicas (jobs already sync via the - // dispatcher + DB read-through). Same NATS client the dispatcher uses. - application.agentJobService.SetTaskSyncNATS(distSvc.Nats) + // dispatcher + DB read-through), on the deployment's broadcast carrier. + application.agentJobService.SetTaskSyncBus(distSvc.Broadcast()) } // Wire skill store into AgentPoolService (wired at pool start time via closure) // The actual wiring happens in StartAgentPool since the pool doesn't exist yet. diff --git a/core/http/app.go b/core/http/app.go index d1fa8ced93a2..54ba3b205cfc 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -497,13 +497,15 @@ func API(application *application.Application) (*echo.Echo, error) { routes.RegisterAgentPoolRoutes(e, application, agentsMw, skillsMw, collectionsMw) // Fine-tuning routes fineTuningMw := auth.RequireFeature(application.AuthDB(), auth.FeatureFineTuning) - // In distributed mode pass the shared NATS client + PostgreSQL store so - // fine-tune jobs stay consistent across replicas (the SyncedMap broadcasts - // mutations and hydrates from the DB); standalone passes nil for both. - var ftNats messaging.MessagingClient + // In distributed mode pass the deployment's broadcast carrier + PostgreSQL + // store so fine-tune jobs stay consistent across replicas (the SyncedMap + // broadcasts mutations and hydrates from the DB); standalone passes nil for + // both. The carrier comes from Broadcast() and never from a field read here: + // see the comment on that method for why the choice is made in one place. + var ftBus messaging.Broadcaster var ftStore *distributed.FineTuneStore if d := application.Distributed(); d != nil { - ftNats = d.Nats + ftBus = d.Broadcast() if d.DistStores != nil && d.DistStores.FineTune != nil { ftStore = d.DistStores.FineTune } @@ -512,20 +514,21 @@ func API(application *application.Application) (*echo.Echo, error) { application.ApplicationConfig(), application.ModelLoader(), application.ModelConfigLoader(), - ftNats, + ftBus, ftStore, ) routes.RegisterFineTuningRoutes(e, ftService, application.ApplicationConfig(), application, fineTuningMw) // Quantization routes quantizationMw := auth.RequireFeature(application.AuthDB(), auth.FeatureQuantization) - // In distributed mode pass the shared NATS client + PostgreSQL store so - // quantization jobs stay consistent across replicas (the SyncedMap broadcasts - // mutations and hydrates from the DB); standalone passes nil for both. - var quantNats messaging.MessagingClient + // In distributed mode pass the deployment's broadcast carrier + PostgreSQL + // store so quantization jobs stay consistent across replicas (the SyncedMap + // broadcasts mutations and hydrates from the DB); standalone passes nil for + // both. Same rule and same single source as the fine-tune wiring above. + var quantBus messaging.Broadcaster var quantStore *distributed.QuantStore if d := application.Distributed(); d != nil { - quantNats = d.Nats + quantBus = d.Broadcast() if d.DistStores != nil && d.DistStores.Quant != nil { quantStore = d.DistStores.Quant } @@ -534,7 +537,7 @@ func API(application *application.Application) (*echo.Echo, error) { application.ApplicationConfig(), application.ModelLoader(), application.ModelConfigLoader(), - quantNats, + quantBus, quantStore, ) routes.RegisterQuantizationRoutes(e, qService, application.ApplicationConfig(), application, quantizationMw) diff --git a/core/http/endpoints/localai/agent_jobs_test.go b/core/http/endpoints/localai/agent_jobs_test.go index 9e34eb6af387..122bfae14dd3 100644 --- a/core/http/endpoints/localai/agent_jobs_test.go +++ b/core/http/endpoints/localai/agent_jobs_test.go @@ -58,7 +58,7 @@ var _ = Describe("DELETE /api/agent/tasks/:id across tenants", func() { mine := app.AgentJobService() Expect(mine).ToNot(BeNil()) mine.SetUserID("u1") - mine.SetTaskSyncNATS(bus) + mine.SetTaskSyncBus(bus) Expect(mine.LoadTasksFromFile()).To(Succeed()) otherDir := GinkgoT().TempDir() @@ -70,7 +70,7 @@ var _ = Describe("DELETE /api/agent/tasks/:id across tenants", func() { other = agentpool.NewAgentJobServiceWithPaths(otherCfg, nil, nil, nil, filepath.Join(otherDir, "tasks.json"), filepath.Join(otherDir, "jobs.json")) other.SetUserID("u2") - other.SetTaskSyncNATS(bus) + other.SetTaskSyncBus(bus) Expect(other.LoadTasksFromFile()).To(Succeed()) taskID, err = other.CreateTask(schema.Task{Name: "u2 only", Model: "m", Prompt: "p"}) diff --git a/core/http/endpoints/openresponses/metadata_store.go b/core/http/endpoints/openresponses/metadata_store.go index ba4833f37bfb..53afc51da9f5 100644 --- a/core/http/endpoints/openresponses/metadata_store.go +++ b/core/http/endpoints/openresponses/metadata_store.go @@ -52,11 +52,14 @@ func (a *responseMetadataStoreAdapter) List(ctx context.Context) ([]*syncedRespo // purge filter on. // // ExpiresAt is lifted out of the payload into its own column because -// ListUnexpired and PurgeExpired compare against it on the database clock. An -// adapter that left it null would make every response immortal in this table: -// the map would still expire it in memory, so nothing would look broken until -// the table had grown without bound and a restarted replica re-hydrated -// responses that died hours earlier. +// ListUnexpired and PurgeExpired compare against it on the database clock. +// +// It is null whenever the deployment runs the default Open Responses TTL of 0, +// which is the ordinary case and not an error: the store then falls back to +// distributed.DefaultResponseMetadataRetention, so the row is still swept and +// still drops out of a hydrate. What the column buys is the other direction. A +// deployment that DOES configure a TTL gets that TTL honoured here, rather than +// having its responses outlive the map they mirror or die before it. func (a *responseMetadataStoreAdapter) Upsert(ctx context.Context, v *syncedResponse) error { if v == nil { return fmt.Errorf("replicating response metadata: nil value") diff --git a/core/http/endpoints/openresponses/store.go b/core/http/endpoints/openresponses/store.go index 2abee4fcb004..cbcd8abe33d9 100644 --- a/core/http/endpoints/openresponses/store.go +++ b/core/http/endpoints/openresponses/store.go @@ -56,7 +56,7 @@ type ResponseStore struct { // (see sync.go), which is how a standalone deployment keeps exactly the // previous process-local behaviour. Guarded by mu. synced *syncstate.SyncedMap[string, *syncedResponse] - nats messaging.MessagingClient + bus messaging.Broadcaster cancelSub messaging.Subscription replicaID string lifeCtx context.Context diff --git a/core/http/endpoints/openresponses/sync.go b/core/http/endpoints/openresponses/sync.go index 261affbd3d85..09b237e21e6f 100644 --- a/core/http/endpoints/openresponses/sync.go +++ b/core/http/endpoints/openresponses/sync.go @@ -14,7 +14,8 @@ import ( ) // syncStateName is the syncstate namespace for replicated response metadata. -// It becomes the NATS subject "state.responses.metadata.delta". +// It becomes the broadcast subject "state.responses-metadata.delta" (the '.' in +// the name is sanitized to '-', so the subject keeps three tokens). const syncStateName = "responses.metadata" // ErrResponseNotLocal is returned by the stream-resume accessors when the @@ -93,12 +94,12 @@ type responseCancelEvent struct { // and not a deployment shape; tolerating it would silently restore the // deltas-only map this parameter exists to replace, and a non-nil interface // wrapping a nil pointer would instead surface as a panic on a request. -func (s *ResponseStore) EnableDistributed(ctx context.Context, nats messaging.MessagingClient, +func (s *ResponseStore) EnableDistributed(ctx context.Context, bus messaging.Broadcaster, replicaID string, store *distributed.ResponseMetadataStore) error { if store == nil { return errors.New("enabling cross-replica Open Responses: the store parameter is nil, so a reconnecting replica would have nothing to re-hydrate from") } - if nats == nil { + if bus == nil { return nil } @@ -110,7 +111,7 @@ func (s *ResponseStore) EnableDistributed(ctx context.Context, nats messaging.Me synced := syncstate.New(syncstate.Config[string, *syncedResponse]{ Name: syncStateName, Key: func(v *syncedResponse) string { return v.ID }, - Nats: nats, + Bus: bus, Store: &responseMetadataStoreAdapter{store: store}, }) if err := synced.Start(ctx); err != nil { @@ -122,12 +123,12 @@ func (s *ResponseStore) EnableDistributed(ctx context.Context, nats messaging.Me // the store, so everything it reads has to be in place first. s.mu.Lock() s.replicaID = replicaID - s.nats = nats + s.bus = bus s.lifeCtx, s.lifeCancel = lifeCtx, lifeCancel s.synced = synced s.mu.Unlock() - sub, err := messaging.SubscribeJSON(nats, messaging.SubjectResponseCancelWildcard, s.applyRemoteCancel) + sub, err := messaging.SubscribeJSON(bus, messaging.SubjectResponseCancelWildcard, s.applyRemoteCancel) if err != nil { if cerr := s.Close(); cerr != nil { xlog.Warn("failed to tear down response metadata sync after subscribe error", "error", cerr) @@ -245,14 +246,14 @@ func (s *ResponseStore) syncMap() *syncstate.SyncedMap[string, *syncedResponse] // distributed returns the replication handles as a consistent snapshot. Every // path that broadcasts reads them through here so a concurrent Close cannot be // observed half-applied. A nil map means standalone mode. -func (s *ResponseStore) distributed() (*syncstate.SyncedMap[string, *syncedResponse], context.Context, messaging.MessagingClient, string) { +func (s *ResponseStore) distributed() (*syncstate.SyncedMap[string, *syncedResponse], context.Context, messaging.Broadcaster, string) { s.mu.RLock() defer s.mu.RUnlock() ctx := s.lifeCtx if ctx == nil { ctx = context.Background() } - return s.synced, ctx, s.nats, s.replicaID + return s.synced, ctx, s.bus, s.replicaID } // replicaIdentity returns this process's replica ID (empty in standalone mode). @@ -373,9 +374,9 @@ func (s *ResponseStore) delegateCancel(v *syncedResponse) (*schema.ORResponseRes return v.Response, nil } - m, ctx, nats, replicaID := s.distributed() - if nats != nil { - if err := nats.Publish(messaging.SubjectResponseCancel(v.ID), + m, ctx, bus, replicaID := s.distributed() + if bus != nil { + if err := bus.Publish(messaging.SubjectResponseCancel(v.ID), responseCancelEvent{ResponseID: v.ID, Origin: replicaID}); err != nil { xlog.Warn("failed to broadcast Open Responses cancel", "response_id", v.ID, "error", err) } diff --git a/core/http/endpoints/openresponses/sync_test.go b/core/http/endpoints/openresponses/sync_test.go index 2495d1043b3e..608a798c0381 100644 --- a/core/http/endpoints/openresponses/sync_test.go +++ b/core/http/endpoints/openresponses/sync_test.go @@ -301,6 +301,33 @@ var _ = Describe("ResponseStore cross-replica", func() { }) Describe("wiring", func() { + It("publishes response metadata on the carrier it was handed, on the responses family's subject", func() { + // S5 in the wiring table. EnableDistributed takes a + // messaging.Broadcaster, and route registration hands it the + // deployment's broadcast carrier; the subject is asserted by name + // because every state.* family shares one LISTEN channel and the + // subject is the only thing that separates them. + const id = "resp_subject" + replicaA.Store(id, &schema.OpenResponsesRequest{Model: "test-model"}, newResponse(id, schema.ORStatusCompleted)) + + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta(syncStateName))).To(BeNumerically(">=", 1)) + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("finetune.jobs"))).To(Equal(0)) + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("agent.tasks"))).To(Equal(0)) + }) + + It("subscribes for delegated cancels on that same carrier", func() { + // The second leg EnableDistributed wires. It rides the same carrier + // as the metadata map, and a cancel that lands on the wrong replica + // reaches nothing without it. + solo := NewResponseStore(0) + own := testutil.NewFakeBus() + Expect(solo.EnableDistributed(ctx, own, "replica-solo", store)).To(Succeed()) + DeferCleanup(func() { Expect(solo.Close()).To(Succeed()) }) + + Expect(own.Subscribers()).To(BeNumerically(">=", 2), + "one subscription for the metadata map and one for the cancel wildcard") + }) + It("refuses to enable replication without a durable store", func() { // A nil store here is a wiring bug, not a deployment shape: this is // reached only from the distributed branch of route registration. diff --git a/core/http/routes/openresponses.go b/core/http/routes/openresponses.go index 1d3977d28550..4cf243864d82 100644 --- a/core/http/routes/openresponses.go +++ b/core/http/routes/openresponses.go @@ -33,7 +33,7 @@ func RegisterOpenResponsesRoutes(app *echo.Echo, responseStore = d.DistStores.Responses } if err := openresponses.GetGlobalStore().EnableDistributed( - application.ApplicationConfig().Context, d.Nats, application.InstanceID(), responseStore); err != nil { + application.ApplicationConfig().Context, d.Broadcast(), application.InstanceID(), responseStore); err != nil { xlog.Error("Failed to enable cross-replica Open Responses store", "error", err) } } diff --git a/core/services/agentpool/agent_jobs.go b/core/services/agentpool/agent_jobs.go index 2ba9f9a90e81..8c310a75e479 100644 --- a/core/services/agentpool/agent_jobs.go +++ b/core/services/agentpool/agent_jobs.go @@ -46,15 +46,17 @@ type AgentJobService struct { evaluator *templates.Evaluator // tasks is the cross-replica task store: an in-memory map kept consistent - // across replicas via NATS, with read-through to the configured persister + // across replicas over the deployment's fan-out carrier, with read-through + // to the configured persister // (file in standalone, PostgreSQL in distributed). Unlike jobs - which already // converge via the dispatcher + DB read-through - tasks previously read // in-memory only, so ListTasks went stale on non-originating replicas. tasks *syncstate.SyncedMap[string, schema.Task] - // taskNats is the distributed NATS client backing the tasks SyncedMap. It is - // not available at construction time, so it is injected via SetTaskSyncNATS - // during distributed wiring; nil keeps tasks in-memory-only (standalone). - taskNats messaging.MessagingClient + // taskBus is the deployment's broadcast carrier backing the tasks SyncedMap. + // It is not available at construction time, so it is injected via + // SetTaskSyncBus during distributed wiring; nil keeps tasks in-memory-only + // (standalone). + taskBus messaging.Broadcaster // Storage (in-memory primary, persister for secondary persistence) jobs *xsync.SyncedMap[string, schema.Job] @@ -101,8 +103,8 @@ func (s *AgentJobService) SetDistributedBackends(dispatcher DistributedDispatche // // The rebuild is what makes the two setters order-independent. Without it the // map keeps whichever tenant it was built with, so wiring that happened to call -// SetTaskSyncNATS first would publish this user's tasks on the CLUSTER-WIDE -// subject and every other tenant would apply them. Like SetTaskSyncNATS, this +// SetTaskSyncBus first would publish this user's tasks on the CLUSTER-WIDE +// subject and every other tenant would apply them. Like SetTaskSyncBus, this // is only ever called before Start / hydrate, while the map is still empty, so // rebuilding loses no state. func (s *AgentJobService) SetUserID(id string) { @@ -117,27 +119,32 @@ func (s *AgentJobService) SetDistributedJobStore(store *jobs.JobStore) { s.persister = &dbJobPersister{store: store} } -// SetTaskSyncNATS wires the distributed NATS client used to keep agent *tasks* -// consistent across replicas (jobs already converge via the dispatcher + DB -// read-through, so they are left untouched). The client is not available when the -// service is constructed, so it is injected here during distributed wiring and the -// tasks SyncedMap is rebuilt to pick it up. It is always called before Start / +// SetTaskSyncBus wires the deployment's broadcast carrier used to keep agent +// *tasks* consistent across replicas (jobs already converge via the dispatcher + +// DB read-through, so they are left untouched). The carrier is not available when +// the service is constructed, so it is injected here during distributed wiring and +// the tasks SyncedMap is rebuilt to pick it up. It is always called before Start / // hydrate, while the map is still empty, so rebuilding loses no state. Passing nil // (standalone) keeps the map in-memory-only with no broadcast. -func (s *AgentJobService) SetTaskSyncNATS(nats messaging.MessagingClient) { - s.taskNats = nats +// +// The parameter is messaging.Broadcaster, so in distributed mode this family +// travels on PostgreSQL LISTEN/NOTIFY. The name says Bus and not NATS because +// the two are no longer the same thing and a stale name here would be the only +// documentation a wiring site reads. +func (s *AgentJobService) SetTaskSyncBus(bus messaging.Broadcaster) { + s.taskBus = bus s.buildTasksMap() } // buildTasksMap (re)constructs the cross-replica tasks SyncedMap from the current -// taskNats. The Store adapter reads s.persister/s.userID live, so a persister swap -// (SetDistributedJobStore) needs no rebuild; only the NATS client, fixed at -// New-time, forces one - hence SetTaskSyncNATS calls this. +// taskBus. The Store adapter reads s.persister/s.userID live, so a persister swap +// (SetDistributedJobStore) needs no rebuild; only the carrier, fixed at map-build +// time, forces one - hence SetTaskSyncBus calls this. func (s *AgentJobService) buildTasksMap() { s.tasks = syncstate.New(syncstate.Config[string, schema.Task]{ Name: "agent.tasks", Key: func(t schema.Task) string { return t.ID }, - Nats: s.taskNats, + Bus: s.taskBus, Store: &taskStoreAdapter{svc: s}, // There is one AgentJobService per user, so this map is per-tenant and // its deltas must not reach another tenant's copy. The empty userID is @@ -276,7 +283,7 @@ func NewAgentJobServiceWithPaths( cronEntries: xsync.NewSyncedMap[string, cron.EntryID](), retentionDays: retentionDays, } - // Build the cross-replica tasks map standalone (nil NATS); SetTaskSyncNATS + // Build the cross-replica tasks map standalone (nil carrier); SetTaskSyncBus // rebuilds it with the distributed client once that is available, before Start. s.buildTasksMap() return s diff --git a/core/services/agentpool/task_sync_test.go b/core/services/agentpool/task_sync_test.go index dc65c4097c57..15bb4c92fb8b 100644 --- a/core/services/agentpool/task_sync_test.go +++ b/core/services/agentpool/task_sync_test.go @@ -26,7 +26,7 @@ import ( // newTaskSyncService builds an AgentJobService wired to the given bus and a // throwaway data dir (so the file persister has somewhere to write). Model/config // loaders are nil because the task sync paths under test never touch them. -func newTaskSyncService(bus messaging.MessagingClient) *AgentJobService { +func newTaskSyncService(bus messaging.Broadcaster) *AgentJobService { tmpDir := GinkgoT().TempDir() sysState := &system.SystemState{} sysState.Model.ModelsPath = tmpDir @@ -40,7 +40,7 @@ func newTaskSyncService(bus messaging.MessagingClient) *AgentJobService { // Distinct per-replica files so the file persister write-through never // crosses replicas: convergence here must be proven via the bus alone. tmpDir+"/tasks.json", tmpDir+"/jobs.json") - svc.SetTaskSyncNATS(bus) + svc.SetTaskSyncBus(bus) return svc } @@ -168,7 +168,7 @@ func newTaskSyncServiceForTenant(bus messaging.MessagingClient, userID string) * svc := NewAgentJobServiceWithPaths(appConfig, nil, nil, nil, tmpDir+"/tasks.json", tmpDir+"/jobs.json") svc.SetUserID(userID) - svc.SetTaskSyncNATS(bus) + svc.SetTaskSyncBus(bus) return svc } @@ -293,9 +293,41 @@ var _ = Describe("AgentJobService same-tenant replicas", func() { Expect(got.Name).To(Equal("Shared")) }) + It("publishes the cluster-wide view on the carrier SetTaskSyncBus was handed", func() { + // S3 in the wiring table. Two startup paths call this one setter + // (startup.go and the settings-driven restart in agent_jobs.go), so the + // service must broadcast on whatever it was given and on the agent-task + // family's own subject, never on another family's. + bus := testutil.NewFakeBus() + svc := newTaskSyncService(bus) + Expect(svc.Start(context.Background())).To(Succeed()) + defer func() { Expect(svc.Stop()).To(Succeed()) }() + + _, err := svc.CreateTask(schema.Task{Name: "Cluster", Model: "m", Prompt: "p"}) + Expect(err).NotTo(HaveOccurred()) + + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("agent.tasks"))).To(Equal(1)) + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("finetune.jobs"))).To(Equal(0)) + }) + + It("broadcasts nothing when it is handed no carrier at all", func() { + // The standalone half of S3: a single-binary deployment passes nil and + // must get a map that still works and never reaches for a carrier. + bus := testutil.NewFakeBus() + svc := newTaskSyncService(nil) + Expect(svc.Start(context.Background())).To(Succeed()) + defer func() { Expect(svc.Stop()).To(Succeed()) }() + + id, err := svc.CreateTask(schema.Task{Name: "Solo", Model: "m", Prompt: "p"}) + Expect(err).NotTo(HaveOccurred()) + _, err = svc.GetTask(id) + Expect(err).NotTo(HaveOccurred(), "a carrier-less service must still serve its own reads") + Expect(bus.Subscribers()).To(Equal(0)) + }) + It("scopes the subject whichever order the setters are called in", func() { // UserServicesManager.GetJobs happens to call SetUserID before - // SetTaskSyncNATS. Nothing enforces that order, and a map built from a + // SetTaskSyncBus. Nothing enforces that order, and a map built from a // user id that had not been set yet publishes cluster-wide - the leak, // reintroduced by a line move. bus := testutil.NewFakeBus() @@ -311,3 +343,56 @@ var _ = Describe("AgentJobService same-tenant replicas", func() { Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("agent.tasks"))).To(Equal(0)) }) }) + +// The per-user manager is the wiring site that fails silently. +// +// S4 in the table, and it is the reason the table has five rows rather than +// four: the global AgentJobService and every per-user one are different +// objects, so a carrier handed only to the global service leaves every tenant's +// tasks unreplicated with nothing failing anywhere. The manager stores the +// carrier once and every service it builds afterwards inherits it. +var _ = Describe("UserServicesManager task carrier propagation", func() { + newManager := func() *UserServicesManager { + GinkgoHelper() + tmpDir := GinkgoT().TempDir() + sysState := &system.SystemState{} + sysState.Model.ModelsPath = tmpDir + appConfig := config.NewApplicationConfig( + config.WithDynamicConfigDir(tmpDir), + config.WithContext(context.Background()), + ) + appConfig.SystemState = sysState + return NewUserServicesManager(NewUserScopedStorage(tmpDir, tmpDir), appConfig, nil, nil, nil) + } + + It("hands the carrier it was given to each per-user service, on that tenant's subject", func() { + bus := testutil.NewFakeBus() + m := newManager() + m.SetJobSyncBus(bus) + + svc, err := m.GetJobs("u1") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(svc.Stop()).To(Succeed()) }) + + _, err = svc.CreateTask(schema.Task{Name: "Tenant", Model: "m", Prompt: "p"}) + Expect(err).NotTo(HaveOccurred()) + + Expect(bus.PublishCount(messaging.SubjectSyncStateTenantDelta("agent.tasks", "u1"))).To(Equal(1), + "a per-user service that inherited no carrier leaves that tenant unreplicated, silently") + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("agent.tasks"))).To(Equal(0), + "and one that inherited it must not publish a tenant's tasks cluster-wide") + }) + + It("leaves per-user services carrier-less when the manager was given none", func() { + bus := testutil.NewFakeBus() + m := newManager() + + svc, err := m.GetJobs("u2") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(svc.Stop()).To(Succeed()) }) + + _, err = svc.CreateTask(schema.Task{Name: "Tenant", Model: "m", Prompt: "p"}) + Expect(err).NotTo(HaveOccurred()) + Expect(bus.Subscribers()).To(Equal(0)) + }) +}) diff --git a/core/services/agentpool/user_services.go b/core/services/agentpool/user_services.go index 56d19e0fc8ea..c0cbcfef7b40 100644 --- a/core/services/agentpool/user_services.go +++ b/core/services/agentpool/user_services.go @@ -29,9 +29,9 @@ type UserServicesManager struct { // Shared distributed backends (set once, inherited by per-user job services) jobDispatcher DistributedDispatcher jobDBStore *jobs.JobStore - // jobNats keeps per-user agent tasks consistent across replicas (nil in + // jobBus keeps per-user agent tasks consistent across replicas (nil in // standalone). Inherited by each per-user AgentJobService. - jobNats messaging.MessagingClient + jobBus messaging.Broadcaster } // NewUserServicesManager creates a new UserServicesManager. @@ -166,10 +166,14 @@ func (m *UserServicesManager) GetJobs(userID string) (*AgentJobService, error) { if m.jobDispatcher != nil { svc.SetDistributedBackends(m.jobDispatcher) } - // Inherit the NATS client so per-user tasks broadcast across replicas. Must be - // set before the hydrate below (LoadFromDB / LoadTasksFromFile) so the tasks - // SyncedMap is rebuilt with the client while it is still empty. - svc.SetTaskSyncNATS(m.jobNats) + // Inherit the broadcast carrier so per-user tasks fan out across replicas. + // Must be set before the hydrate below (LoadFromDB / LoadTasksFromFile) so the + // tasks SyncedMap is rebuilt with the carrier while it is still empty. + // + // This is a second wiring site for the same rule, and it is the one that is + // invisible: fixing the global service alone leaves every tenant's map on + // whatever carrier this manager was handed, with nothing failing. + svc.SetTaskSyncBus(m.jobBus) if m.jobDBStore != nil { svc.SetDistributedJobStore(m.jobDBStore) // Load tasks/jobs from DB immediately (per-user services skip Start()) @@ -197,10 +201,11 @@ func (m *UserServicesManager) SetJobDBStore(s *jobs.JobStore) { m.jobDBStore = s } -// SetJobSyncNATS sets the NATS client used to keep per-user agent tasks consistent -// across replicas. -func (m *UserServicesManager) SetJobSyncNATS(nats messaging.MessagingClient) { - m.jobNats = nats +// SetJobSyncBus sets the broadcast carrier used to keep per-user agent tasks +// consistent across replicas. Every per-user service built afterwards inherits +// it; see the call in the builder above. +func (m *UserServicesManager) SetJobSyncBus(bus messaging.Broadcaster) { + m.jobBus = bus } // ListAllUserIDs returns all user IDs that have scoped data directories. diff --git a/core/services/distributed/responses.go b/core/services/distributed/responses.go index 5d93d11de9e7..ede01803b4d6 100644 --- a/core/services/distributed/responses.go +++ b/core/services/distributed/responses.go @@ -47,8 +47,52 @@ func (ResponseMetadataRecord) TableName() string { return "response_metadata" } // confused them would blank the map on a transient outage. type ResponseMetadataStore struct { db *gorm.DB + + // retention bounds how long a row that carries NO expiry of its own stays + // in this table. See DefaultResponseMetadataRetention. + retention time.Duration } +// DefaultResponseMetadataRetention is how long a response row with no expiry of +// its own is kept. +// +// It is INDEPENDENT of the Open Responses store TTL, and that is the whole +// point. That TTL defaults to 0, documented as "no expiration", and 0 there is +// defensible: the in-memory map it governs dies with the process, so unbounded +// means "bounded by this process's lifetime and by memory an operator can see". +// A TABLE has neither bound. Inheriting that 0 here made every replicated +// response immortal on disk, so the table grew for the life of the deployment +// and a restarting replica re-hydrated its map with every response the cluster +// had ever created. Neither is a condition anything reports. +// +// So the durable projection gets a retention of its own. It is a floor on +// CROSS-REPLICA visibility and never on the response: the replica that owns a +// response keeps it in memory for exactly as long as the configured TTL says, +// and an operator who sets a TTL longer than this gets that TTL honoured, +// because a row with an expires_at is judged on that column alone. +// +// Twenty-four hours is chosen against what the row is for. It exists so a peer +// replica, or a restarted one, can answer a GET or resolve a +// previous_response_id it did not create. That is a lookup a client makes +// within minutes of the response, not a day later, and the row is metadata +// rather than the generation itself. +const DefaultResponseMetadataRetention = 24 * time.Hour + +// deadResponseMetadata is the ONE definition of "this row is no longer live", +// evaluated on the DATABASE clock so every replica agrees. +// +// PurgeExpired deletes exactly the rows it matches and ListUnexpired returns +// exactly the rows it does not, spelled as NOT of this same string. Two +// separate spellings would let a hydrate resurrect a row a sweep had already +// decided was dead, or leave a row that is invisible to every reader sitting in +// the table forever, and neither reports anything. +// +// The bind parameter is the retention in seconds, and it is the only one: +// make_interval(secs => ?) turns it into an interval the SERVER subtracts from +// its own now(), so no process's clock enters the comparison. +const deadResponseMetadata = `((expires_at IS NOT NULL AND expires_at <= now()) OR ` + + `(expires_at IS NULL AND created_at <= now() - make_interval(secs => ?)))` + // NewResponseMetadataStore creates a ResponseMetadataStore and migrates its // table. // @@ -60,9 +104,23 @@ type ResponseMetadataStore struct { // The migration runs under the same advisory lock NewFineTuneStore uses, because // several replicas start at once and concurrent AutoMigrate races. func NewResponseMetadataStore(db *gorm.DB) (*ResponseMetadataStore, error) { + return NewResponseMetadataStoreWithRetention(db, DefaultResponseMetadataRetention) +} + +// NewResponseMetadataStoreWithRetention is NewResponseMetadataStore with an +// explicit bound on rows that carry no expiry of their own. +// +// A non-positive retention is refused rather than taken to mean "keep +// everything": unbounded is the shape this store had, it grows a table nothing +// ever sweeps, and a deployment that wants a different bound should say which +// one rather than switch the bound off. +func NewResponseMetadataStoreWithRetention(db *gorm.DB, retention time.Duration) (*ResponseMetadataStore, error) { if db == nil { return nil, fmt.Errorf("response metadata store: no database handle") } + if retention <= 0 { + return nil, fmt.Errorf("response metadata store: retention must be positive, got %s; a row with no expiry of its own would never be swept and the table would grow for the life of the deployment", retention) + } if name := db.Dialector.Name(); name != "postgres" { return nil, fmt.Errorf("response metadata store requires PostgreSQL, this deployment runs on %q", name) } @@ -71,9 +129,13 @@ func NewResponseMetadataStore(db *gorm.DB) (*ResponseMetadataStore, error) { }); err != nil { return nil, fmt.Errorf("migrating response_metadata: %w", err) } - return &ResponseMetadataStore{db: db}, nil + return &ResponseMetadataStore{db: db, retention: retention}, nil } +// Retention is the effective bound on rows with no expiry of their own. +// Reported so a deployment can log what it actually got. +func (s *ResponseMetadataStore) Retention() time.Duration { return s.retention } + // Upsert idempotently inserts or replaces one row by primary key. // // created_at is deliberately NOT in the update set: the row is rewritten on @@ -100,19 +162,20 @@ func (s *ResponseMetadataStore) Delete(ctx context.Context, id string) error { return s.db.WithContext(ctx).Where("id = ?", id).Delete(&ResponseMetadataRecord{}).Error } -// ListUnexpired returns every row whose ExpiresAt is null or still in the -// future, measured on the DATABASE clock. +// ListUnexpired returns every row that is still live: its ExpiresAt is in the +// future, or it has none and is younger than the retention. Both legs are +// measured on the DATABASE clock. // // The clock is the database's because every replica hydrating from this table // must agree on which rows are live, and a Go-side cutoff makes that a property -// of whichever process asked. It is spelled `expires_at IS NULL OR expires_at > -// now()` and it is dialect-guarded in the constructor, because now() on the -// SQLite single-binary path reads as a missing migration rather than as an -// error. +// of whichever process asked. The predicate is deadResponseMetadata negated, so +// this and PurgeExpired cannot disagree about a row; it is dialect-guarded in +// the constructor, because now() and make_interval on the SQLite single-binary +// path read as a missing migration rather than as an error. func (s *ResponseMetadataStore) ListUnexpired(ctx context.Context) ([]ResponseMetadataRecord, error) { var out []ResponseMetadataRecord if err := s.db.WithContext(ctx). - Where("expires_at IS NULL OR expires_at > now()"). + Where("NOT "+deadResponseMetadata, s.retention.Seconds()). Order("created_at"). Find(&out).Error; err != nil { return nil, fmt.Errorf("listing unexpired response metadata: %w", err) @@ -120,12 +183,12 @@ func (s *ResponseMetadataStore) ListUnexpired(ctx context.Context) ([]ResponseMe return out, nil } -// PurgeExpired deletes rows whose ExpiresAt has passed and returns how many. -// It is the reason a table of ephemeral state does not grow forever, and it -// runs on the same DATABASE clock as ListUnexpired. +// PurgeExpired deletes every row ListUnexpired would refuse to return and +// reports how many. It is the reason a table of ephemeral state does not grow +// forever, and it runs on the same DATABASE clock and the same predicate. func (s *ResponseMetadataStore) PurgeExpired(ctx context.Context) (int64, error) { res := s.db.WithContext(ctx). - Where("expires_at IS NOT NULL AND expires_at <= now()"). + Where(deadResponseMetadata, s.retention.Seconds()). Delete(&ResponseMetadataRecord{}) if res.Error != nil { return 0, fmt.Errorf("purging expired response metadata: %w", res.Error) diff --git a/core/services/distributed/responses_test.go b/core/services/distributed/responses_test.go index ffe77c1b0c31..f529959ae6d7 100644 --- a/core/services/distributed/responses_test.go +++ b/core/services/distributed/responses_test.go @@ -240,6 +240,89 @@ var _ = Describe("ResponseMetadataStore", func() { }) }) + // The bound on rows that carry no expiry of their own. + // + // The Open Responses store TTL defaults to 0, documented as "no + // expiration", and every response then reaches this table with a null + // expires_at. Zero is defensible for the in-memory map it governs, which + // dies with the process; a table has no such bound, so it grew for the life + // of the deployment and a restarting replica re-hydrated its map with every + // response the cluster had ever created. Neither is a condition anything + // reports, which is why it needed a default of its own rather than a note. + Describe("retention for rows with no expiry of their own", func() { + aged := func(id string, age time.Duration, expiresAt *time.Time) *distributed.ResponseMetadataRecord { + r := newRecord(id, expiresAt) + r.CreatedAt = time.Now().Add(-age) + return r + } + + It("stops listing and then sweeps an untagged row older than the retention", func() { + Expect(store.Upsert(ctx, aged("resp_stale", distributed.DefaultResponseMetadataRetention+time.Hour, nil))).To(Succeed()) + Expect(store.Upsert(ctx, aged("resp_recent", time.Minute, nil))).To(Succeed()) + + recs, err := store.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(ids(recs)).To(ConsistOf("resp_recent"), + "a row past the retention must not re-hydrate a replica's map") + + n, err := store.PurgeExpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(1)), "and the sweep must actually retire it, or the table still grows forever") + + recs, err = store.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(ids(recs)).To(ConsistOf("resp_recent")) + }) + + It("honours a row's own expiry over the retention, in both directions", func() { + // The retention is a floor on rows that named no expiry, never a + // ceiling on rows that did. A deployment that configured a longer + // TTL must get it, and one that configured a shorter one must not + // have its responses kept alive by this default. + far := time.Now().Add(distributed.DefaultResponseMetadataRetention * 10) + past := time.Now().Add(-time.Minute) + + Expect(store.Upsert(ctx, aged("resp_long_ttl", distributed.DefaultResponseMetadataRetention+time.Hour, &far))).To(Succeed()) + Expect(store.Upsert(ctx, aged("resp_short_ttl", time.Minute, &past))).To(Succeed()) + + recs, err := store.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(ids(recs)).To(ConsistOf("resp_long_ttl")) + + n, err := store.PurgeExpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + }) + + It("takes an explicit retention and reports the one it is using", func() { + short, err := distributed.NewResponseMetadataStoreWithRetention(db, time.Hour) + Expect(err).ToNot(HaveOccurred()) + Expect(short.Retention()).To(Equal(time.Hour)) + Expect(store.Retention()).To(Equal(distributed.DefaultResponseMetadataRetention)) + + Expect(short.Upsert(ctx, aged("resp_two_hours", 2*time.Hour, nil))).To(Succeed()) + + recs, err := short.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(recs).To(BeEmpty(), "the configured retention, not the default, decides") + + // The same row is still live to a store with the default retention, + // which is what proves the bound is the store's and not the row's. + recs, err = store.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(ids(recs)).To(ConsistOf("resp_two_hours")) + }) + + It("refuses a non-positive retention rather than restoring an unbounded table", func() { + _, err := distributed.NewResponseMetadataStoreWithRetention(db, 0) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("grow")) + + _, err = distributed.NewResponseMetadataStoreWithRetention(db, -time.Hour) + Expect(err).To(HaveOccurred()) + }) + }) + // The expiry cutoff is the database's clock, not the asking process's. The // container shares this host's clock, so no behavioural spec above can tell // a Go-side time.Now() bind parameter from now(); these pin the statement @@ -260,11 +343,11 @@ var _ = Describe("ResponseMetadataStore", func() { Expect(err).ToNot(HaveOccurred()) sql := rec.only() - Expect(sql).To(MatchRegexp(`(?i)expires_at\s+IS\s+NULL\s+OR\s+expires_at\s*>\s*now\(\)`)) + Expect(sql).To(MatchRegexp(`(?i)expires_at\s+IS\s+NOT\s+NULL\s+AND\s+expires_at\s*<=\s*now\(\)`)) // A bound parameter in the comparison position is a Go-side cutoff // wearing the same behaviour; gorm's logger explains binds into the // text, so a time literal here is exactly that defect. - Expect(sql).ToNot(MatchRegexp(`expires_at\s*>\s*['$]`)) + Expect(sql).ToNot(MatchRegexp(`expires_at\s*<=\s*['$]`)) }) It("compares expires_at against the database clock in PurgeExpired", func() { @@ -276,6 +359,41 @@ var _ = Describe("ResponseMetadataStore", func() { Expect(sql).ToNot(MatchRegexp(`expires_at\s*<=\s*['$]`)) }) + It("ages an untagged row out against the database clock too, never a Go-side cutoff", func() { + // The retention leg is subtracted from the SERVER's now() with + // make_interval, so the only bind is a number of seconds. A + // timestamp literal here would be this process's clock deciding + // which rows the whole deployment can still see. + _, err := store.PurgeExpired(ctx) + Expect(err).ToNot(HaveOccurred()) + + sql := rec.only() + Expect(sql).To(MatchRegexp(`(?i)created_at\s*<=\s*now\(\)\s*-\s*make_interval\(secs\s*=>`)) + Expect(sql).ToNot(MatchRegexp(`created_at\s*<=\s*['$]`)) + }) + + It("selects and deletes on ONE predicate, so a hydrate cannot resurrect what a sweep killed", func() { + // Two spellings of "this row is dead" drift, and the drift is + // silent both ways: a row invisible to every reader that nothing + // deletes, or a row a sweep removed that a hydrate had just served. + _, err := store.ListUnexpired(ctx) + Expect(err).ToNot(HaveOccurred()) + listSQL := rec.only() + + rec.reset() + _, err = store.PurgeExpired(ctx) + Expect(err).ToNot(HaveOccurred()) + purgeSQL := rec.only() + + predicate := regexp.MustCompile(`(?is)\(\(expires_at.*?make_interval\(secs\s*=>[^)]*\)\)\)`) + listPredicate := predicate.FindString(listSQL) + Expect(listPredicate).ToNot(BeEmpty(), "ListUnexpired must carry the shared predicate") + Expect(purgeSQL).To(ContainSubstring(listPredicate), + "PurgeExpired must delete exactly what ListUnexpired refuses to return") + Expect(listSQL).To(ContainSubstring("NOT "+listPredicate), + "and ListUnexpired must be its negation rather than a second spelling") + }) + It("issues one statement per call, so neither reads the clock into Go first", func() { _, err := store.ListUnexpired(ctx) Expect(err).ToNot(HaveOccurred()) diff --git a/core/services/finetune/service.go b/core/services/finetune/service.go index 3e2431df2798..8e3978a00a9a 100644 --- a/core/services/finetune/service.go +++ b/core/services/finetune/service.go @@ -40,19 +40,24 @@ type FineTuneService struct { mu sync.Mutex // jobs is the cross-replica job store: an in-memory map kept consistent across - // replicas via NATS, optionally read-through to PostgreSQL in distributed mode. + // replicas over the deployment's fan-out carrier, optionally read-through to + // PostgreSQL in distributed mode. jobs *syncstate.SyncedMap[string, *schema.FineTuneJob] } // NewFineTuneService creates a new FineTuneService. In distributed mode pass the -// shared NATS client and PostgreSQL store so jobs stay consistent across -// replicas; pass nil for both in standalone mode, where the disk Loader hydrates -// the map and there is nothing to broadcast. +// deployment's broadcast carrier and PostgreSQL store so jobs stay consistent +// across replicas; pass nil for both in standalone mode, where the disk Loader +// hydrates the map and there is nothing to broadcast. +// +// bus is messaging.Broadcaster and not the NATS client: this state.*.delta +// family travels on whatever the deployment's fan-out carrier is, and in +// distributed mode that is PostgreSQL LISTEN/NOTIFY. func NewFineTuneService( appConfig *config.ApplicationConfig, modelLoader *model.ModelLoader, configLoader *config.ModelConfigLoader, - nats messaging.MessagingClient, + bus messaging.Broadcaster, store *distributed.FineTuneStore, ) *FineTuneService { s := &FineTuneService{ @@ -72,7 +77,7 @@ func NewFineTuneService( s.jobs = syncstate.New(syncstate.Config[string, *schema.FineTuneJob]{ Name: "finetune.jobs", Key: func(j *schema.FineTuneJob) string { return j.ID }, - Nats: nats, + Bus: bus, Store: syncStore, Loader: s.loadJobsFromDisk, // ignored when Store is set (distributed mode) }) diff --git a/core/services/finetune/service_test.go b/core/services/finetune/service_test.go index dc7c53290b78..5c40daabb624 100644 --- a/core/services/finetune/service_test.go +++ b/core/services/finetune/service_test.go @@ -14,13 +14,14 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/services/distributed" + "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/testutil" ) // newTestService builds a standalone FineTuneService wired to the given bus. The // model/config loaders are nil because the read/sync paths under test never touch // them; the data dir is a throwaway temp dir so the disk Loader finds nothing. -func newTestService(bus *testutil.FakeBus) *FineTuneService { +func newTestService(bus messaging.Broadcaster) *FineTuneService { appConfig := &config.ApplicationConfig{ Context: context.Background(), DataPath: GinkgoT().TempDir(), @@ -77,6 +78,35 @@ var _ = Describe("FineTuneService", func() { Expect(err).To(HaveOccurred(), "a delete on A must remove the job from B") }) + It("publishes on the carrier it was handed, and on the fine-tune family's own subject", func() { + // S1 in the wiring table: this service takes a + // messaging.Broadcaster and must broadcast on THAT and on nothing + // else. The subject is asserted by name rather than inferred from + // delivery, because a service that published on the wrong family + // would still reach a peer of its own kind here while colliding + // with quantization in production. + job := &schema.FineTuneJob{ID: "job-subject", UserID: "user-1", Status: "queued", CreatedAt: "2026-06-27T10:00:00Z"} + Expect(a.jobs.Set(ctx, job)).To(Succeed()) + + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("finetune.jobs"))).To(Equal(1)) + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("quant.jobs"))).To(Equal(0)) + }) + + It("broadcasts nothing at all when it is built with no carrier", func() { + // Standalone is the other half of the same wiring rule: a nil + // carrier must be a strict no-op rather than a panic on the first + // job a single-binary user starts. + solo := newTestService(nil) + DeferCleanup(func() { Expect(solo.Close()).To(Succeed()) }) + + Expect(solo.jobs.Set(ctx, &schema.FineTuneJob{ID: "solo", UserID: "u", CreatedAt: "2026-06-27T10:00:00Z"})).To(Succeed()) + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("finetune.jobs"))).To(Equal(0)) + + got, err := solo.GetJob("u", "solo") + Expect(err).ToNot(HaveOccurred(), "a carrier-less service must still serve its own reads") + Expect(got.ID).To(Equal("solo")) + }) + It("propagates a status update from A to B", func() { job := &schema.FineTuneJob{ID: "job-3", UserID: "user-1", Status: "training", CreatedAt: "2026-06-27T10:00:00Z"} Expect(a.jobs.Set(ctx, job)).To(Succeed()) diff --git a/core/services/messaging/client.go b/core/services/messaging/client.go index e01c7d9cabe8..7450d8b17651 100644 --- a/core/services/messaging/client.go +++ b/core/services/messaging/client.go @@ -263,19 +263,6 @@ func (c *Client) QueueSubscribeReply(subject, queue string, handler func(data [] }) } -// SubscribeJSON creates a subscription that automatically unmarshals JSON messages. -// Invalid JSON messages are logged and skipped. -func SubscribeJSON[T any](c MessagingClient, subject string, handler func(T)) (Subscription, error) { - return c.Subscribe(subject, func(data []byte) { - var evt T - if err := json.Unmarshal(data, &evt); err != nil { - xlog.Warn("Failed to unmarshal NATS message", "subject", subject, "error", err) - return - } - handler(evt) - }) -} - // QueueSubscribeJSON creates a queue subscription that automatically unmarshals JSON messages. // Invalid JSON messages are logged and skipped. func QueueSubscribeJSON[T any](c MessagingClient, subject, queue string, handler func(T)) (Subscription, error) { diff --git a/core/services/messaging/json.go b/core/services/messaging/json.go new file mode 100644 index 000000000000..ccc27b5319e8 --- /dev/null +++ b/core/services/messaging/json.go @@ -0,0 +1,26 @@ +package messaging + +import ( + "encoding/json" + + "github.com/mudler/xlog" +) + +// SubscribeJSON creates a subscription that automatically unmarshals JSON +// messages. Invalid JSON messages are logged and skipped. +// +// Its parameter is Broadcaster and not MessagingClient, so a call site can be +// moved onto the PostgreSQL carrier without also being rewritten. It lives in +// its own file because client.go is deleted once the request/reply and queue +// halves are retired, and a generic helper with this many call sites must not +// go with it. +func SubscribeJSON[T any](c Broadcaster, subject string, handler func(T)) (Subscription, error) { + return c.Subscribe(subject, func(data []byte) { + var evt T + if err := json.Unmarshal(data, &evt); err != nil { + xlog.Warn("Failed to unmarshal a broadcast message", "subject", subject, "error", err) + return + } + handler(evt) + }) +} diff --git a/core/services/quantization/service.go b/core/services/quantization/service.go index a6a6eefcbc09..fc5c4f4c5368 100644 --- a/core/services/quantization/service.go +++ b/core/services/quantization/service.go @@ -40,7 +40,8 @@ type QuantizationService struct { mu sync.Mutex // jobs is the cross-replica job store: an in-memory map kept consistent across - // replicas via NATS, optionally read-through to PostgreSQL in distributed mode. + // replicas over the deployment's fan-out carrier, optionally read-through to + // PostgreSQL in distributed mode. jobs *syncstate.SyncedMap[string, *schema.QuantizationJob] // progressMu guards progressSubs. @@ -67,14 +68,18 @@ func isTerminalStatus(status string) bool { } // NewQuantizationService creates a new QuantizationService. In distributed mode -// pass the shared NATS client and PostgreSQL store so jobs stay consistent across -// replicas; pass nil for both in standalone mode, where the disk Loader hydrates -// the map and there is nothing to broadcast. +// pass the deployment's broadcast carrier and PostgreSQL store so jobs stay +// consistent across replicas; pass nil for both in standalone mode, where the +// disk Loader hydrates the map and there is nothing to broadcast. +// +// bus is messaging.Broadcaster and not the NATS client: this state.*.delta +// family travels on whatever the deployment's fan-out carrier is, and in +// distributed mode that is PostgreSQL LISTEN/NOTIFY. func NewQuantizationService( appConfig *config.ApplicationConfig, modelLoader *model.ModelLoader, configLoader *config.ModelConfigLoader, - nats messaging.MessagingClient, + bus messaging.Broadcaster, store *distributed.QuantStore, ) *QuantizationService { s := &QuantizationService{ @@ -95,7 +100,7 @@ func NewQuantizationService( s.jobs = syncstate.New(syncstate.Config[string, *schema.QuantizationJob]{ Name: "quant.jobs", Key: func(j *schema.QuantizationJob) string { return j.ID }, - Nats: nats, + Bus: bus, Store: syncStore, Loader: s.loadJobsFromDisk, // ignored when Store is set (distributed mode) }) diff --git a/core/services/quantization/service_test.go b/core/services/quantization/service_test.go index ae862ffcaa85..4b0c804d0476 100644 --- a/core/services/quantization/service_test.go +++ b/core/services/quantization/service_test.go @@ -18,6 +18,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/services/distributed" + "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/testutil" pb "github.com/mudler/LocalAI/pkg/grpc/proto" ) @@ -26,7 +27,7 @@ import ( // The model/config loaders are nil because the read/sync paths under test never // touch them; the data dir is a throwaway temp dir so the disk Loader finds // nothing. -func newTestService(bus *testutil.FakeBus) *QuantizationService { +func newTestService(bus messaging.Broadcaster) *QuantizationService { appConfig := &config.ApplicationConfig{ Context: context.Background(), DataPath: GinkgoT().TempDir(), @@ -83,6 +84,29 @@ var _ = Describe("QuantizationService", func() { Expect(err).To(HaveOccurred(), "a delete on A must remove the job from B") }) + It("publishes on the carrier it was handed, and on the quantization family's own subject", func() { + // S2 in the wiring table. The negative half is the one that matters: + // finetune and quant land on the SAME LISTEN channel in distributed + // mode, so the subject is all that separates them. + job := &schema.QuantizationJob{ID: "job-subject", UserID: "user-1", Status: "queued", CreatedAt: "2026-06-27T10:00:00Z"} + Expect(a.jobs.Set(ctx, job)).To(Succeed()) + + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("quant.jobs"))).To(Equal(1)) + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("finetune.jobs"))).To(Equal(0)) + }) + + It("broadcasts nothing at all when it is built with no carrier", func() { + solo := newTestService(nil) + DeferCleanup(func() { Expect(solo.Close()).To(Succeed()) }) + + Expect(solo.jobs.Set(ctx, &schema.QuantizationJob{ID: "solo", UserID: "u", CreatedAt: "2026-06-27T10:00:00Z"})).To(Succeed()) + Expect(bus.PublishCount(messaging.SubjectSyncStateDelta("quant.jobs"))).To(Equal(0)) + + got, err := solo.GetJob("u", "solo") + Expect(err).ToNot(HaveOccurred(), "a carrier-less service must still serve its own reads") + Expect(got.ID).To(Equal("solo")) + }) + It("propagates a status update from A to B", func() { job := &schema.QuantizationJob{ID: "job-3", UserID: "user-1", Status: "quantizing", CreatedAt: "2026-06-27T10:00:00Z"} Expect(a.jobs.Set(ctx, job)).To(Succeed()) diff --git a/core/services/syncstate/syncstate.go b/core/services/syncstate/syncstate.go index 8e70138d2bd0..7373c2f05842 100644 --- a/core/services/syncstate/syncstate.go +++ b/core/services/syncstate/syncstate.go @@ -5,9 +5,18 @@ // that is surfaced to the HTTP/UI API; without cross-replica sync a poll that // lands on a replica which did not originate a change sees stale or missing data. // SyncedMap collapses the three legs each feature otherwise hand-wires - an -// in-memory map, a NATS broadcast/apply path, and optional durable read-through - -// into one well-tested component so cross-replica consistency is a configuration -// choice rather than a bespoke re-implementation. +// in-memory map, a broadcast/apply path over the deployment's fan-out carrier, +// and optional durable read-through - into one well-tested component so +// cross-replica consistency is a configuration choice rather than a bespoke +// re-implementation. +// +// The carrier is messaging.Broadcaster and nothing narrower, so a deployment +// can carry these deltas on PostgreSQL LISTEN/NOTIFY. That is not a detail of +// the transport: LISTEN/NOTIFY is at most once to CONNECTED listeners and never +// replays, so a map whose only convergence path were the deltas would answer +// from state it can never repair. Store (or Loader) is what hydrate, the +// reconnect callback and the reconcile ticker read, and it is the reason a +// dropped delta is a gap that closes rather than a value that was never set. package syncstate import ( @@ -36,9 +45,20 @@ type Store[K comparable, V any] interface { // Config configures a SyncedMap. type Config[K comparable, V any] struct { - Name string // subject namespace, e.g. "finetune.jobs" - Key func(V) K // extract the key from a value - Nats messaging.MessagingClient // nil => standalone: in-memory only, no broadcast/subscribe + Name string // subject namespace, e.g. "finetune.jobs" + Key func(V) K // extract the key from a value + + // Bus is the fan-out carrier. nil => standalone: in-memory only, no + // broadcast and no subscribe. + // + // It is messaging.Broadcaster rather than messaging.MessagingClient because + // this component only ever publishes and subscribes: request/reply and + // queue groups are not part of what a replicated map needs, and demanding + // them would rule out every carrier that does not have them. The field is + // not called Nats because a field of that name holding a PostgreSQL carrier + // is a comment that claims more than the code does. + Bus messaging.Broadcaster + Store Store[K, V] // optional read-through persistence Loader func(ctx context.Context) ([]V, error) // source when there is no Store (e.g. disk reload) OnApply func(op string, k K, v V) // optional hook after an applied change (e.g. ShutdownModel) @@ -144,21 +164,24 @@ func (m *SyncedMap[K, V]) Start(ctx context.Context) error { // goroutines, so it cannot be cancelled or deferred within this scope. m.lifeCtx, m.cancel = context.WithCancel(context.Background()) // #nosec G118 -- cancel is invoked in Close() - if m.cfg.Nats != nil { + if m.cfg.Bus != nil { for _, filter := range m.subscribeFilters() { - sub, err := messaging.SubscribeJSON(m.cfg.Nats, filter, m.apply) + sub, err := messaging.SubscribeJSON(m.cfg.Bus, filter, m.apply) if err != nil { return err } m.subs = append(m.subs, sub) } - // nats.go transparently resubscribes on reconnect, but it cannot know we - // kept derived in-memory state that may have drifted while the link was - // down, so re-hydrate from the durable source. Detected via an optional - // interface so MessagingClient itself stays minimal; standalone/test - // clients without the method simply fall back to the reconcile ticker. - if r, ok := m.cfg.Nats.(interface{ OnReconnect(func()) }); ok { + // A carrier that reconnects restores its own registrations, but it + // cannot know we kept derived in-memory state that drifted while the + // link was down: every delta published in that window was delivered to + // the replicas that were connected and to nobody else, and neither + // carrier replays. Re-hydrating from the durable source is what turns + // that gap into a delay instead of a permanently wrong map. Detected + // via an optional interface so Broadcaster itself stays minimal; + // carriers without the method fall back to the reconcile ticker. + if r, ok := m.cfg.Bus.(interface{ OnReconnect(func()) }); ok { r.OnReconnect(func() { if err := m.hydrate(m.lifeCtx); err != nil { xlog.Warn("syncstate: reconnect re-hydrate failed", "name", m.cfg.Name, "error", err) @@ -261,12 +284,12 @@ func (m *SyncedMap[K, V]) Snapshot() map[K]V { return out } -// publish broadcasts a delta. Standalone (nil Nats) is a strict no-op. +// publish broadcasts a delta. Standalone (nil Bus) is a strict no-op. func (m *SyncedMap[K, V]) publish(op string, k K, v V) { - if m.cfg.Nats == nil { + if m.cfg.Bus == nil { return } - if err := m.cfg.Nats.Publish(m.publishSubject(), delta[K, V]{Op: op, Key: k, Value: v}); err != nil { + if err := m.cfg.Bus.Publish(m.publishSubject(), delta[K, V]{Op: op, Key: k, Value: v}); err != nil { xlog.Warn("syncstate: failed to broadcast delta", "name", m.cfg.Name, "op", op, "error", err) } } diff --git a/core/services/syncstate/syncstate_pg_test.go b/core/services/syncstate/syncstate_pg_test.go new file mode 100644 index 000000000000..4e23b6286284 --- /dev/null +++ b/core/services/syncstate/syncstate_pg_test.go @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: MIT + +package syncstate_test + +import ( + "context" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/distributed" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/syncstate" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// ftAdapter is the same bridge the fine-tune service builds, spelled here so +// this file exercises the component over the REAL durable source rather than +// over a map that cannot fail the way a table fails. +type ftAdapter struct{ s *distributed.FineTuneStore } + +func (a ftAdapter) List(_ context.Context) ([]*distributed.FineTuneJobRecord, error) { + recs, err := a.s.ListAll() + if err != nil { + return nil, err + } + out := make([]*distributed.FineTuneJobRecord, len(recs)) + for i := range recs { + r := recs[i] + out[i] = &r + } + return out, nil +} + +func (a ftAdapter) Upsert(_ context.Context, r *distributed.FineTuneJobRecord) error { + return a.s.Upsert(r) +} + +func (a ftAdapter) Delete(_ context.Context, k string) error { return a.s.Delete(k) } + +// This file is the carriage proof, and it is on a real PostgreSQL for a reason +// the doubles cannot cover. +// +// A FakeBus delivers synchronously, in process, with no payload limit and no +// connection to lose. The three things this component's correctness actually +// rests on in distributed mode are exactly the three it cannot express: a +// notification bigger than PostgreSQL's 8000-byte payload cap has to spill to a +// row and come back byte identical; two maps in different families share ONE +// LISTEN channel, so separation is a filter decision and not a channel +// decision; and a listener whose session is terminated has to come back and +// re-hydrate, because every delta published while it was gone reached the +// replicas that were connected and nobody else. +// +// That last one is the invariant. A delta that was dropped or never delivered +// must not be able to read as a state that was never set: the notification says +// only that something changed, and the table is what says what it changed to. +var _ = Describe("SyncedMap on the PostgreSQL broadcast carrier", func() { + var ( + db *gorm.DB + dsn string + ) + + BeforeEach(func() { + db, dsn = testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(context.Background(), db)).To(Succeed()) + }) + + // newBus is one replica's end of the carrier: its own pinned LISTEN + // connection, so a spec can drop one replica's session without touching the + // peer it is asserting against. + newBus := func() *pgbus.Bus { + GinkgoHelper() + b, err := pgbus.New(context.Background(), pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(b.Close) + return b + } + + // terminate drops exactly this carrier's LISTEN session, matched on the + // application name it reports in pg_stat_activity, so the peer bus in the + // same spec stays up. + terminate := func(b *pgbus.Bus) { + GinkgoHelper() + var killed int64 + Expect(db.Raw( + "SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE application_name = ?", + b.ApplicationName(), + ).Scan(&killed).Error).To(Succeed()) + Expect(killed).To(BeNumerically("==", 1), "expected exactly one listener session to drop") + } + + Describe("two replicas over one shared store", func() { + var ( + ftStore *distributed.FineTuneStore + busA *pgbus.Bus + busB *pgbus.Bus + a, b *syncstate.SyncedMap[string, *distributed.FineTuneJobRecord] + ) + + newMap := func(bus *pgbus.Bus) *syncstate.SyncedMap[string, *distributed.FineTuneJobRecord] { + GinkgoHelper() + m := syncstate.New(syncstate.Config[string, *distributed.FineTuneJobRecord]{ + Name: "finetune.jobs", + Key: func(r *distributed.FineTuneJobRecord) string { return r.ID }, + // The carrier goes in as messaging.Broadcaster. Before this + // change the field was typed as the NATS client, so this line + // did not compile and the durable re-hydrate leg below had no + // consumer that could reach it. + Bus: bus, + Store: ftAdapter{s: ftStore}, + }) + Expect(m.Start(context.Background())).To(Succeed()) + DeferCleanup(func() { Expect(m.Close()).To(Succeed()) }) + return m + } + + rec := func(id, status, message string) *distributed.FineTuneJobRecord { + return &distributed.FineTuneJobRecord{ + ID: id, UserID: "u1", Model: "m", Backend: "bk", + TrainingType: "lora", TrainingMethod: "sft", Status: status, Message: message, + } + } + + BeforeEach(func() { + var err error + ftStore, err = distributed.NewFineTuneStore(db) + Expect(err).ToNot(HaveOccurred()) + + busA, busB = newBus(), newBus() + a, b = newMap(busA), newMap(busB) + }) + + It("carries a Set from one replica to the other", func() { + Expect(a.Set(context.Background(), rec("job-1", "queued", ""))).To(Succeed()) + + Eventually(func() string { + if r, ok := b.Get("job-1"); ok { + return r.Status + } + return "" + }, 30*time.Second, 50*time.Millisecond).Should(Equal("queued")) + }) + + It("carries a Delete from one replica to the other", func() { + Expect(a.Set(context.Background(), rec("job-2", "queued", ""))).To(Succeed()) + Eventually(func() bool { _, ok := b.Get("job-2"); return ok }, 30*time.Second, 50*time.Millisecond). + Should(BeTrue()) + + Expect(a.Delete(context.Background(), "job-2")).To(Succeed()) + Eventually(func() bool { _, ok := b.Get("job-2"); return ok }, 30*time.Second, 50*time.Millisecond). + Should(BeFalse(), "a delete that does not carry leaves a job the peer can never clear") + }) + + It("carries a delta too large for one notification, byte identical", func() { + // Comfortably over the carrier's 8000-byte notification cap, so + // this delta can only arrive through the spill row. A truncated or + // re-encoded payload here would surface as a job whose message the + // peer renders differently from the replica that owns it, with + // nothing failing. + big := strings.Repeat("training log line; ", 1200) + Expect(len(big)).To(BeNumerically(">", 8000)) + + Expect(a.Set(context.Background(), rec("job-3", "training", big))).To(Succeed()) + + Eventually(func() int { + if r, ok := b.Get("job-3"); ok { + return len(r.Message) + } + return 0 + }, 30*time.Second, 50*time.Millisecond).Should(Equal(len(big))) + + got, ok := b.Get("job-3") + Expect(ok).To(BeTrue()) + Expect(got.Message).To(Equal(big), "the spilled payload must come back exactly as it was published") + }) + + It("re-hydrates what it missed once its dropped listener reconnects", func() { + // The whole reason the durable store exists. The row is written + // straight to the shared table and never broadcast, which is what a + // peer's delta amounts to for a replica whose session was gone when + // it was published: at most once, no replay, no gap signal. If the + // reconnect callback is not registered, or the map converges on + // deltas alone, this replica answers as though the job had never + // been created - for good. + // + // The row is written BEFORE the session is dropped, and never + // broadcast, so this spec cannot pass by racing a reconnect against + // a NOTIFY: there is no NOTIFY, and the only path into b is a + // re-hydrate that the reconnect triggers. + Expect(ftStore.Upsert(rec("job-4", "completed", ""))).To(Succeed()) + _, present := b.Get("job-4") + Expect(present).To(BeFalse(), "a row written straight to the table reaches no map until something re-reads it") + + terminate(busB) + Eventually(busB.IsConnected, 60*time.Second, 100*time.Millisecond).Should(BeTrue()) + + Eventually(func() string { + if r, ok := b.Get("job-4"); ok { + return r.Status + } + return "" + }, 60*time.Second, 100*time.Millisecond). + Should(Equal("completed"), "a reconnected replica must re-read the durable source, not wait for a delta that is gone") + }) + + It("keeps carrying deltas after the reconnect", func() { + // The half the re-hydrate does not cover: a carrier that came back + // deaf re-hydrates once and then looks exactly like a deployment + // where nobody is publishing. + terminate(busB) + Eventually(busB.IsConnected, 60*time.Second, 100*time.Millisecond).Should(BeTrue()) + + Eventually(func() bool { + // Retried because a NOTIFY issued between the drop and the + // re-LISTEN is genuinely lost: this carrier is at most once. + Expect(a.Set(context.Background(), rec("job-5", "queued", ""))).To(Succeed()) + _, ok := b.Get("job-5") + return ok + }, 60*time.Second, 200*time.Millisecond).Should(BeTrue()) + }) + }) + + Describe("two families sharing one LISTEN channel", func() { + // Every state.* subject maps onto the single localai_state channel, so + // a subscriber hears its peers' families too and separation is decided + // by the filter alone. A carrier that delivered to every subscriber on + // the channel would put quantization jobs into the fine-tune map, and + // the fake bus cannot express the hazard because it has no channels. + newNamed := func(bus *pgbus.Bus, name string) *syncstate.SyncedMap[string, *job] { + GinkgoHelper() + m := syncstate.New(syncstate.Config[string, *job]{Name: name, Key: jobKey, Bus: bus}) + Expect(m.Start(context.Background())).To(Succeed()) + DeferCleanup(func() { Expect(m.Close()).To(Succeed()) }) + return m + } + + It("delivers a quantization delta to the quantization map and not to the fine-tune map", func() { + busA, busB := newBus(), newBus() + + quantA := newNamed(busA, "quant.jobs") + quantB := newNamed(busB, "quant.jobs") + ftB := newNamed(busB, "finetune.jobs") + + Expect(quantA.Set(context.Background(), &job{ID: "q-1", Status: "running"})).To(Succeed()) + + // The positive leg first, so the negative one below is asserted + // AFTER delivery has demonstrably happened rather than before it + // could have. + Eventually(func() bool { _, ok := quantB.Get("q-1"); return ok }, 30*time.Second, 50*time.Millisecond). + Should(BeTrue()) + + Expect(ftB.List()).To(BeEmpty(), "a fine-tune map must not apply a quantization delta off the shared channel") + }) + }) + + Describe("per-tenant subjects on the real carrier", func() { + // Task 10 pinned this against a fake bus. Repeated here because this is + // the commit where those subjects actually become LISTEN traffic, and + // because both the tenant subject and the cluster-wide one land on the + // same channel: if tenancy were a channel decision it would be right by + // accident, and it is not. + newTenant := func(bus *pgbus.Bus, tenant string) *syncstate.SyncedMap[string, *job] { + GinkgoHelper() + m := syncstate.New(syncstate.Config[string, *job]{ + Name: "agent.tasks", Key: jobKey, Bus: bus, PerTenant: true, Tenant: tenant, + }) + Expect(m.Start(context.Background())).To(Succeed()) + DeferCleanup(func() { Expect(m.Close()).To(Succeed()) }) + return m + } + + It("reaches the same tenant and the cluster-wide view, and no other tenant", func() { + busA, busB := newBus(), newBus() + + u1A := newTenant(busA, "u1") + u1B := newTenant(busB, "u1") + u2B := newTenant(busB, "u2") + clusterB := newTenant(busB, "") + + Expect(u1A.Set(context.Background(), &job{ID: "t-1", Status: "running"})).To(Succeed()) + + Eventually(func() bool { _, ok := u1B.Get("t-1"); return ok }, 30*time.Second, 50*time.Millisecond). + Should(BeTrue(), "the same tenant on another replica must see it") + Eventually(func() bool { _, ok := clusterB.Get("t-1"); return ok }, 30*time.Second, 50*time.Millisecond). + Should(BeTrue(), "the cluster-wide view hydrates across tenants, so it must apply across tenants") + + Expect(u2B.List()).To(BeEmpty(), "another tenant must never see this tenant's task") + }) + }) +}) diff --git a/core/services/syncstate/syncstate_test.go b/core/services/syncstate/syncstate_test.go index 47116a1b4804..d5339135ce30 100644 --- a/core/services/syncstate/syncstate_test.go +++ b/core/services/syncstate/syncstate_test.go @@ -23,6 +23,12 @@ func jobKey(j *job) string { return j.ID } const stateName = "test.jobs" +// The retype this whole change rests on. Config.Bus is messaging.Broadcaster, +// so anything a spec or a deployment hands it must satisfy that and nothing +// wider; a fake that quietly needed MessagingClient would mean the component +// still could not be handed the PostgreSQL carrier. +var _ messaging.Broadcaster = (*testutil.FakeBus)(nil) + func deltaSubject() string { return messaging.SubjectSyncStateDelta(stateName) } // fakeStore is an in-memory Store that records call counts so specs can assert @@ -96,8 +102,8 @@ var _ = Describe("SyncedMap", func() { BeforeEach(func() { bus = testutil.NewFakeBus() - a = syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus}) - b = syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus}) + a = syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Bus: bus}) + b = syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Bus: bus}) Expect(a.Start(ctx)).To(Succeed()) Expect(b.Start(ctx)).To(Succeed()) }) @@ -157,8 +163,8 @@ var _ = Describe("SyncedMap", func() { Describe("echo-loop guard", func() { It("applies its own broadcast once and does not re-publish", func() { bus := testutil.NewFakeBus() - a := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus}) - b := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus}) + a := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Bus: bus}) + b := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Bus: bus}) Expect(a.Start(ctx)).To(Succeed()) Expect(b.Start(ctx)).To(Succeed()) defer func() { @@ -183,8 +189,8 @@ var _ = Describe("SyncedMap", func() { bus := testutil.NewFakeBus() storeA := newFakeStore() storeB := newFakeStore() - a := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus, Store: storeA}) - b := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus, Store: storeB}) + a := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Bus: bus, Store: storeA}) + b := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Bus: bus, Store: storeB}) Expect(a.Start(ctx)).To(Succeed()) Expect(b.Start(ctx)).To(Succeed()) defer func() { @@ -215,9 +221,9 @@ var _ = Describe("SyncedMap", func() { ops []string keys []string ) - a := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus}) + a := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Bus: bus}) b := syncstate.New(syncstate.Config[string, *job]{ - Name: stateName, Key: jobKey, Nats: bus, + Name: stateName, Key: jobKey, Bus: bus, OnApply: func(op string, k string, _ *job) { mu.Lock() ops = append(ops, op) @@ -242,7 +248,32 @@ var _ = Describe("SyncedMap", func() { }) }) - Describe("standalone (nil Nats)", func() { + Describe("standalone (nil Bus)", func() { + It("registers nothing and broadcasts nothing, and still serves reads from hydrate", func() { + // The bus exists in this spec and is deliberately NOT handed over. + // Zero publishes and zero subscribers is the assertion: "nothing was + // delivered" cannot tell a strict no-op apart from a subscription + // nobody happened to publish to, and a component that reached for a + // carrier of its own would pass that weaker check. + bus := testutil.NewFakeBus() + store := newFakeStore(&job{ID: "seeded", Status: "completed"}) + + m := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Store: store}) + Expect(m.Start(ctx)).To(Succeed()) + defer func() { Expect(m.Close()).To(Succeed()) }() + + Expect(bus.Subscribers()).To(Equal(0), "a standalone map must register no subscription anywhere") + + got, ok := m.Get("seeded") + Expect(ok).To(BeTrue(), "hydrate must serve reads with no carrier at all") + Expect(got.Status).To(Equal("completed")) + Expect(m.List()).To(HaveLen(1)) + + Expect(m.Set(ctx, &job{ID: "local", Status: "running"})).To(Succeed()) + Expect(m.Delete(ctx, "seeded")).To(Succeed()) + Expect(bus.PublishCount(deltaSubject())).To(Equal(0), "a standalone map must not broadcast") + }) + It("works in-memory with no panic and nothing to broadcast", func() { m := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey}) Expect(m.Start(ctx)).To(Succeed()) @@ -268,7 +299,7 @@ var _ = Describe("SyncedMap", func() { It("re-reads the source when the messaging client reconnects", func() { bus := testutil.NewFakeBus() store := newFakeStore(&job{ID: "init", Status: "running"}) - m := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus, Store: store}) + m := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Bus: bus, Store: store}) Expect(m.Start(ctx)).To(Succeed()) defer func() { Expect(m.Close()).To(Succeed()) }() @@ -305,7 +336,7 @@ var _ = Describe("SyncedMap per-tenant subjects", func() { m := syncstate.New(syncstate.Config[string, *job]{ Name: stateName, Key: jobKey, - Nats: bus, + Bus: bus, PerTenant: true, Tenant: tenant, }) @@ -437,8 +468,8 @@ var _ = Describe("SyncedMap per-tenant subjects", func() { // finetune.jobs, quantization and the responses store are unscoped // adopters. Pin that this change moved none of them. bus := testutil.NewFakeBus() - a := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus}) - b := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus}) + a := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Bus: bus}) + b := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Bus: bus}) Expect(a.Start(ctx)).To(Succeed()) Expect(b.Start(ctx)).To(Succeed()) tenant := newTenantMap(bus, "u1") diff --git a/core/services/testutil/fakebus.go b/core/services/testutil/fakebus.go index d178fc3ee166..1607a5e9c578 100644 --- a/core/services/testutil/fakebus.go +++ b/core/services/testutil/fakebus.go @@ -43,6 +43,15 @@ type fakeBusSub struct { handler func([]byte) } +// The two interfaces this double stands in for. Broadcaster is asserted here +// and not left to the first adopter: every cross-replica map now takes a +// messaging.Broadcaster, and a fake that drifted out of that interface would +// break each adopter's suite in turn rather than the package that owns it. +var ( + _ messaging.MessagingClient = (*FakeBus)(nil) + _ messaging.Broadcaster = (*FakeBus)(nil) +) + // NewFakeBus returns a ready-to-use in-memory bus. func NewFakeBus() *FakeBus { return &FakeBus{publishCounts: map[string]int{}} @@ -74,6 +83,17 @@ func (b *FakeBus) PublishCount(subject string) int { return b.publishCounts[subject] } +// Subscribers reports how many live subscriptions this bus is carrying. +// +// It exists so a spec can assert the NEGATIVE: a component configured +// standalone must register nothing at all, and "nothing was delivered" cannot +// tell that apart from "a subscription exists and nobody published". +func (b *FakeBus) Subscribers() int { + b.mu.Lock() + defer b.mu.Unlock() + return len(b.subs) +} + type fakeBusSubscription struct { bus *FakeBus subRef fakeBusSub diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index bc3c629ce2b4..8039857f11b4 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -38,7 +38,7 @@ Each model gets its own gRPC backend process, so a single worker can serve multi ## Prerequisites - **PostgreSQL** (with pgvector extension recommended for RAG) - used for node registry, job store, auth, and shared state - - Each frontend replica holds **one extra PostgreSQL session** beyond its connection pool, pinned for the life of the process, and creates a `bus_messages` table. Both belong to the broadcast carrier that is replacing NATS for cross-replica fan-out. Size `max_connections` for one additional session per frontend replica. + - Each frontend replica holds **one extra PostgreSQL session** beyond its connection pool, pinned for the life of the process, and creates a `bus_messages` table. Both belong to the broadcast carrier that is replacing NATS for cross-replica fan-out; it already carries the four `state.*.delta` families (see [Cross-replica in-memory state](#cross-replica-in-memory-state)). Size `max_connections` for one additional session per frontend replica. - That session reports an `application_name` of `localai_pgbus_`, so `SELECT count(*) FROM pg_stat_activity WHERE application_name LIKE 'localai_pgbus_%'` counts the replicas currently listening. If the carrier loses its session it redials and re-registers on its own; a broadcast published while it was down is not replayed, which is why nothing that must survive a gap is carried by a broadcast alone. - `bus_messages` holds only broadcasts too large for a PostgreSQL notification, and every replica retires rows older than ten minutes. The table is a spill buffer, not a log: it is not a place to read past events from. - **NATS** server - used for agent-worker coordination and the frontend's own cross-replica events. **Serve-backend workers do not connect to it at all**: every verb they take, and file staging with it, is an HTTP route on the worker's tunnel. Set no `LOCALAI_NATS_URL` on a `local-ai worker`. The frontend and any `local-ai agent-worker` still need one. @@ -123,6 +123,25 @@ The peer link is served at `/api/cluster/peer` and authenticates with `LOCALAI_R **The peer link has no per-replica credential yet.** It checks the shared registration token and takes the replica id in `?id=` on trust. Anything already holding that token - every worker holds it - can therefore open a peer link, relay through it to every worker tunnel a replica owns, by declaring another replica's id displace that replica's inbound link, and hold sessions open against the per-session receive window, which the peer-link code sizes at roughly 31 GiB of unread data per session and which on this route is also a memory budget an attacker can point at one replica. Treat `LOCALAI_REGISTRATION_TOKEN` as a cluster-wide secret with the blast radius of the whole fleet: give it its own value per deployment, do not reuse it elsewhere, and keep `/api/cluster/peer` on a network only your replicas and workers can reach. Per-replica credentials for this route are planned. {{% /notice %}} +### Cross-replica in-memory state + +Several features keep state in a frontend's process memory and surface it over the API: fine-tune jobs, quantization jobs, agent tasks and Open Responses metadata. A round-robin load balancer sends a follow-up request to any replica, so each of those maps is kept current on every replica by a broadcast. + +**Those four families travel on PostgreSQL, not on NATS.** Each mutation is a `NOTIFY` on the database the deployment already runs, and each replica holds one `LISTEN` session for it. There is nothing to configure: the carrier uses the same database URL as `--auth-database-url` / `LOCALAI_AUTH_DATABASE_URL`. + +| Map | Subject | +|-----|---------| +| Fine-tune jobs | `state.finetune-jobs.delta` | +| Quantization jobs | `state.quant-jobs.delta` | +| Agent tasks | `state.agent-tasks.delta` and `state.agent-tasks..delta` | +| Open Responses metadata | `state.responses-metadata.delta` | + +Two carrier details are visible to an operator. + +**The 8000-byte notification cap.** PostgreSQL refuses a `pg_notify` payload of 8000 bytes or more, and that limit is measured against the whole encoded notification, not just the value being replicated. A broadcast that does not fit is written to the `bus_messages` table and the notification carries the row id instead; the receiving replica reads the row and delivers the original bytes. This is an ordinary path and not an error: a fine-tune job carrying a long training message spills every time. Rows are retired ten minutes after they are written, by every replica, so `bus_messages` is a spill buffer and never a log of past events. + +**A broadcast is at most once, and is never replayed.** A `NOTIFY` reaches the sessions that are listening when it is issued and nobody else. A replica whose session was down in that window never receives the change, and no error is reported anywhere. That is why every one of these maps is backed by a durable table: the broadcast says only that something changed, and the table says what it changed to. A replica re-reads its table when its listener reconnects, so a missed delta is a delay and never a value that reads as though it had never been set. + ### Open Responses across replicas A response created by `POST /v1/responses` is held by the replica that served the request. A round-robin load balancer sends the follow-up poll, the `previous_response_id` chain and the cancel to any replica, so that metadata is replicated to every frontend and is also written to a `response_metadata` table in PostgreSQL. @@ -137,20 +156,26 @@ What crosses replicas and what does not: | Cancellation | The request is, the `CancelFunc` is not | The cancel is forwarded to the owning replica, which holds the function that stops generation | | Stream resume buffer (`starting_after`) | No | It is the full token log; replicating it would put every generated token on the bus. A resume request that lands on the wrong replica is refused with an explicit error, never with a silently truncated event list | -**Retention.** Rows carry the expiry of the response they describe, and each replica sweeps expired rows every five minutes. The expiry comes from the Open Responses store TTL, which is **`0` (no expiration) by default**: +**Retention.** Each replica sweeps dead rows out of `response_metadata` every five minutes, and a row is dead when either of two things is true. -```yaml -environment: - LOCALAI_OPEN_RESPONSES_STORE_TTL: "1h" -``` +- It carries the expiry of the response it describes, and that expiry has passed. The expiry comes from the Open Responses store TTL, which is `0` (no expiration) by default: + + ```yaml + environment: + LOCALAI_OPEN_RESPONSES_STORE_TTL: "1h" + ``` + +- It carries no expiry, because the TTL is `0`, and it is more than **24 hours** old. + +The 24-hour floor is the table's own bound and it is independent of the TTL. A TTL of `0` is a reasonable answer for the in-memory map it governs, which dies with the process; a table has no such bound, so without a floor `response_metadata` would grow for the life of the deployment and every restarting replica would re-hydrate every response the cluster had ever created. -Leave it at `0` in distributed mode and nothing ever expires: `response_metadata` grows for the life of the deployment, and every replica that restarts re-hydrates every response the cluster has ever created. Set a TTL that matches how long clients are allowed to poll for a response. +The floor never overrides a TTL you set. A row that names an expiry is judged on that expiry alone, longer or shorter than 24 hours. What the floor bounds is only how long a response stays resolvable **on a replica that did not create it**: the owning replica keeps it in memory for exactly as long as the TTL says. Set a TTL that matches how long clients are allowed to poll for a response. -These rows carry the request body and the generated output, not just identifiers. They live in the same database as the rest of the cluster state, and the TTL above is the only thing that removes them. +These rows carry the request body and the generated output, not just identifiers. They live in the same database as the rest of the cluster state. ### Agent tasks are scoped to their tenant -Every frontend replica keeps agent task definitions in memory so that `GET /api/agent/tasks` answers from any replica. That in-memory copy is kept current by a broadcast on the cluster bus, and the broadcast carries the owning user in the subject: +Every frontend replica keeps agent task definitions in memory so that `GET /api/agent/tasks` answers from any replica. That in-memory copy is kept current by a broadcast on the PostgreSQL carrier described above, and the broadcast carries the owning user in the subject: | Map | Subject it publishes on | Subjects it applies | |-----|------------------------|---------------------| diff --git a/tests/e2e/distributed/syncstate_distributed_test.go b/tests/e2e/distributed/syncstate_distributed_test.go index acd2797e6195..b0af894e7742 100644 --- a/tests/e2e/distributed/syncstate_distributed_test.go +++ b/tests/e2e/distributed/syncstate_distributed_test.go @@ -4,7 +4,7 @@ import ( "context" "github.com/mudler/LocalAI/core/services/distributed" - "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/pgbus" "github.com/mudler/LocalAI/core/services/syncstate" . "github.com/onsi/ginkgo/v2" @@ -42,48 +42,59 @@ func (a ftSyncStore) Delete(_ context.Context, k string) error { return a.s.Dele // This suite is the real-infrastructure counterpart to the fake-bus unit tests: // two SyncedMap instances stand in for two LocalAI frontend replicas, each with -// its OWN NATS connection to a shared NATS server and a SHARED PostgreSQL store - -// the exact distributed-mode invariant (single shared DB, per-replica process -// state). It proves the delta path works over the wire and that a late-joining -// replica recovers via store hydrate (the at-most-once gap a fake bus cannot -// exercise). -var _ = Describe("SyncedMap two-replica sync over real NATS", Label("Distributed"), func() { +// its OWN pinned LISTEN connection to the shared PostgreSQL and a SHARED store +// on the same database - the exact distributed-mode invariant (single shared +// DB, per-replica process state). It proves the delta path works over the wire +// and that a late-joining replica recovers via store hydrate (the at-most-once +// gap a fake bus cannot exercise). +// +// There is no NATS here any more. This family's deltas ride the deployment's +// PostgreSQL broadcast carrier, so a suite that still proved them over a broker +// would be proving something the product no longer does. +var _ = Describe("SyncedMap two-replica sync over the PostgreSQL carrier", Label("Distributed"), func() { var ( infra *TestInfra + db *gorm.DB ftStore *distributed.FineTuneStore ) BeforeEach(func() { infra = SetupInfra("localai_syncstate_dist_test") - db, err := gorm.Open(pgdriver.Open(infra.PGURL), &gorm.Config{ + var err error + db, err = gorm.Open(pgdriver.Open(infra.PGURL), &gorm.Config{ Logger: logger.Default.LogMode(logger.Silent), }) Expect(err).ToNot(HaveOccurred()) + Expect(pgbus.Migrate(infra.Ctx, db)).To(Succeed()) + ftStore, err = distributed.NewFineTuneStore(db) Expect(err).ToNot(HaveOccurred()) }) - // newReplica builds an independent "replica": its own NATS client to the - // shared server plus a SyncedMap over the shared store, started (hydrate + - // subscribe) and cleaned up automatically. + // newReplica builds an independent "replica": its own carrier, with its own + // pinned LISTEN connection, plus a SyncedMap over the shared store, started + // (hydrate + subscribe) and cleaned up automatically. + // + // No flush is needed where NATS needed one: pgbus.Subscribe returns only + // after the server has acknowledged the LISTEN, so a subscription that has + // been registered cannot miss a NOTIFY issued afterwards. newReplica := func() *syncstate.SyncedMap[string, *distributed.FineTuneJobRecord] { GinkgoHelper() - nc, err := messaging.New(infra.NatsURL) + bus, err := pgbus.New(infra.Ctx, pgbus.Config{DSN: infra.PGURL, DB: db}) Expect(err).ToNot(HaveOccurred()) sm := syncstate.New(syncstate.Config[string, *distributed.FineTuneJobRecord]{ Name: "finetune.jobs", Key: func(r *distributed.FineTuneJobRecord) string { return r.ID }, - Nats: nc, + Bus: bus, Store: ftSyncStore{s: ftStore}, }) Expect(sm.Start(infra.Ctx)).To(Succeed()) - FlushNATS(nc) // ensure the subscription is registered server-side before any publish DeferCleanup(func() { _ = sm.Close() - nc.Close() + bus.Close() }) return sm } @@ -102,7 +113,7 @@ var _ = Describe("SyncedMap two-replica sync over real NATS", Label("Distributed Expect(a.Set(infra.Ctx, rec("job-1", "queued"))).To(Succeed()) Eventually(func() bool { _, ok := b.Get("job-1"); return ok }, "10s", "50ms"). - Should(BeTrue(), "replica B must observe the job created on A via NATS") + Should(BeTrue(), "replica B must observe the job created on A over the carrier") got, ok := b.Get("job-1") Expect(ok).To(BeTrue()) From 0d2d4f07168938c1278ac150216a55b9fa19b831 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 4 Sep 2026 01:47:34 +0000 Subject: [PATCH 094/153] refactor(distributed): move job and agent fan-out onto the PostgreSQL carrier Five of the six families whose subscriber is an open HTTP response rather than a process-lifetime cache now travel on pgbus: jobs..progress, jobs..result, jobs..cancel, agent..events. and responses..cancel. Both ends of each move together, so there is no state where a publisher is on one carrier and its subscriber on the other. agent..cancel does NOT move, and the plan was wrong about why. Its only subscriber in the tree is the agent worker, which has no database and so cannot join the PostgreSQL carrier at all. Publishing that cancel on pgbus would have lost every cancel of a worker-run agent while returning nil, which reports a cancel that reached nobody as a cancel that was sent. EventBridge now names its cancel carrier separately, a frontend replica sets it to the carrier the worker reads, and it stays there until a cancel rides the worker's tunnel like every other verb addressed to a worker. The carrier drops at 256 rather than blocking, which is not safe on its own for a result: a lost result has no successor message. It is not the only path. The claiming replica persists the terminal line before it releases the claim, and an open progress stream re-reads the job row once after subscribing and then periodically, so a dropped terminal broadcast costs promptness and never the answer. Both per-request subscriptions close in a defer instead of on one return path, and pgbus grows Subscribers() so the leak they would otherwise cause can be asserted. It has no other symptom: only the first subscriber of a channel issues a LISTEN, so a leaked filter just adds one closure per notification for every stream the replica has ever served. Subscribe now issues its LISTEN before it registers, which makes that count a readiness signal rather than a figure to compare against itself. Two rules that were stated at several sites and pinned at none are now one each. The re-broadcaster is built beside the dispatcher and the bridge and handed to the dispatch loop, so no line is left that can point it at a carrier nobody subscribes to while every spec stays green. The set of statuses a job never leaves is one exported set that the SSE bridge and the store both read. The last hand-written subject filter in production code became messaging.SubjectAgentEventsWildcard. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/dispatch_loop_wiring.go | 18 +- core/application/dispatch_loop_wiring_test.go | 30 +- core/application/distributed.go | 20 +- core/application/fanout_wiring.go | 84 ++++++ core/application/fanout_wiring_test.go | 169 +++++++++++ .../endpoints/openresponses/sync_pg_test.go | 114 +++++++ core/http/routes/openresponses.go | 46 ++- core/http/routes/openresponses_wiring_test.go | 97 ++++++ core/services/agents/events.go | 140 ++++++--- core/services/agents/events_pg_test.go | 279 ++++++++++++++++++ core/services/agents/events_test.go | 32 +- core/services/jobs/dispatcher.go | 97 ++++-- core/services/jobs/dispatcher_pg_test.go | 271 +++++++++++++++++ core/services/jobs/dispatcher_test.go | 52 ++-- core/services/jobs/publish.go | 12 +- core/services/jobs/sse.go | 119 ++++++-- core/services/jobs/store.go | 25 +- core/services/jobs/store_test.go | 47 +++ core/services/messaging/subjects.go | 10 + core/services/messaging/subjects_wire_test.go | 33 +++ core/services/nodes/rebroadcast.go | 10 +- core/services/nodes/rebroadcast_test.go | 75 +++++ core/services/pgbus/bus.go | 63 ++-- core/services/pgbus/bus_test.go | 75 +++++ docs/content/features/distributed-mode.md | 21 +- .../e2e/distributed/agent_distributed_test.go | 67 +++-- tests/e2e/distributed/job_dispatch_test.go | 17 +- .../e2e/distributed/job_distribution_test.go | 42 +-- tests/e2e/distributed/sse_routes_test.go | 99 ++++--- tests/e2e/distributed/testhelpers_test.go | 26 ++ 30 files changed, 1907 insertions(+), 283 deletions(-) create mode 100644 core/application/fanout_wiring.go create mode 100644 core/application/fanout_wiring_test.go create mode 100644 core/http/endpoints/openresponses/sync_pg_test.go create mode 100644 core/http/routes/openresponses_wiring_test.go create mode 100644 core/services/agents/events_pg_test.go create mode 100644 core/services/jobs/dispatcher_pg_test.go diff --git a/core/application/dispatch_loop_wiring.go b/core/application/dispatch_loop_wiring.go index f797c616c798..b34fefc39479 100644 --- a/core/application/dispatch_loop_wiring.go +++ b/core/application/dispatch_loop_wiring.go @@ -8,7 +8,6 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/jobs" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "gorm.io/gorm" ) @@ -35,19 +34,30 @@ import ( // they reload. That is a whole feature lost to a nil field, with no error // anywhere, so it is refused here. // +// The re-broadcaster is taken already built, from newFanoutBridges, and is a +// *nodes.Rebroadcaster rather than the jobs.ProgressBroadcaster interface the +// loop stores it as. Both of those are deliberate. Taking it built leaves ONE +// expression in the tree that decides which carrier job and agent fan-out goes +// on, next to the dispatcher and the bridge that must read the same one, so +// there is no separate line here to point at a carrier nobody subscribes to: +// that mis-wiring publishes successfully, returns true, reddens no spec in any +// package, and shows up only as an SSE stream with no progress in it. Naming +// the concrete type is what makes the refusal below fire, too: widened to the +// interface, a nil re-broadcaster is a non-nil value holding a nil pointer. +// // The SELECTOR is built here rather than borrowed from newAgentControl, and // deliberately: nodes.AgentSelector holds no per-caller state, and sharing one // would couple the dispatch loop's lifetime to MCP's for nothing. func startJobDispatchLoop(ctx context.Context, cfg config.DistributedConfig, db *gorm.DB, store *jobs.JobStore, registry *nodes.NodeRegistry, conns nodes.AgentConnectionReader, - control *nodes.ControlClient, bus messaging.Broadcaster) (*jobs.DispatchLoop, error) { + control *nodes.ControlClient, broadcast *nodes.Rebroadcaster) (*jobs.DispatchLoop, error) { if cfg.InstanceID == "" { return nil, fmt.Errorf("the job dispatch loop was built with no instance id: its claims could not be told from ones a dead replica left") } if registry == nil || conns == nil { return nil, fmt.Errorf("the job dispatch loop was built with no way to find a connected agent worker") } - if bus == nil { + if broadcast == nil { return nil, fmt.Errorf("the job dispatch loop was built with no broadcaster: every job would run with its progress and its result reaching no SSE stream in the deployment") } loop, err := jobs.NewDispatchLoop(jobs.DispatchConfig{ @@ -57,7 +67,7 @@ func startJobDispatchLoop(ctx context.Context, cfg config.DistributedConfig, db Control: control, // The allow list lives in nodes and is keyed on the worker's node type; // nothing here decides what a worker may broadcast on. - Broadcast: nodes.NewRebroadcaster(bus), + Broadcast: broadcast, Store: store, }) if err != nil { diff --git a/core/application/dispatch_loop_wiring_test.go b/core/application/dispatch_loop_wiring_test.go index a42bc647f78a..f152fcd8b130 100644 --- a/core/application/dispatch_loop_wiring_test.go +++ b/core/application/dispatch_loop_wiring_test.go @@ -26,6 +26,7 @@ var _ = Describe("building the job dispatch loop", func() { var registry *nodes.NodeRegistry var conns *recordingConnections var ctx context.Context + var broadcast *nodes.Rebroadcaster BeforeEach(func() { if runtime.GOOS == "darwin" { @@ -36,6 +37,14 @@ var _ = Describe("building the job dispatch loop", func() { registry, err = nodes.NewNodeRegistry(testutil.SetupTestDB()) Expect(err).ToNot(HaveOccurred()) conns = newRecordingConnections() + + // A double is enough HERE, and only here. Which carrier this + // re-broadcaster publishes on is not this function's decision any more: + // it is handed one already built by newFanoutBridges, and that is where + // the carrier is pinned, by receipt on a second connection. What is + // left for these to say is that the loop refuses to be built without + // one and starts when it is. + broadcast = nodes.NewRebroadcaster(testutil.NewFakeBus()) }) // The silent one. A loop with no broadcaster dispatches work perfectly @@ -48,10 +57,25 @@ var _ = Describe("building the job dispatch loop", func() { Expect(err.Error()).To(ContainSubstring("broadcaster")) }) + // The nil that the interface would have hidden. The loop stores its + // re-broadcaster as the jobs.ProgressBroadcaster interface, and widened to + // that here a nil *nodes.Rebroadcaster is a NON-nil value holding a nil + // pointer, so the refusal above would never fire for the way one is + // actually absent: newFanoutBridges returns a typed nil alongside its + // error. This drives that exact value, which is why the parameter is the + // concrete type. + It("refuses a typed-nil broadcaster, which an interface parameter would have accepted", func() { + var absent *nodes.Rebroadcaster + _, err := startJobDispatchLoop(ctx, config.DistributedConfig{InstanceID: "replica-7"}, + testutil.SetupTestDB(), nil, registry, conns, nodes.NewControlClient(nil, "token"), absent) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("broadcaster")) + }) + It("refuses to build with no instance id", func() { _, err := startJobDispatchLoop(ctx, config.DistributedConfig{}, testutil.SetupTestDB(), nil, registry, conns, nodes.NewControlClient(nil, "token"), - testutil.NewFakeBus()) + broadcast) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("instance id")) }) @@ -59,7 +83,7 @@ var _ = Describe("building the job dispatch loop", func() { It("refuses to build with nothing to read connections through", func() { _, err := startJobDispatchLoop(ctx, config.DistributedConfig{InstanceID: "replica-7"}, testutil.SetupTestDB(), nil, registry, nil, nodes.NewControlClient(nil, "token"), - testutil.NewFakeBus()) + broadcast) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("connected agent worker")) }) @@ -81,7 +105,7 @@ var _ = Describe("building the job dispatch loop", func() { Expect(err).ToNot(HaveOccurred()) loop, err := startJobDispatchLoop(ctx, config.DistributedConfig{InstanceID: "replica-7"}, - db, nil, registry, conns, nodes.NewControlClient(nil, "token"), testutil.NewFakeBus()) + db, nil, registry, conns, nodes.NewControlClient(nil, "token"), broadcast) Expect(err).ToNot(HaveOccurred()) DeferCleanup(loop.Stop) diff --git a/core/application/distributed.go b/core/application/distributed.go index e803e1768b58..2225a9f65042 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -418,9 +418,6 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } xlog.Info("Distributed job store initialized") - // Initialize job dispatcher - dispatcher := jobs.NewDispatcher(jobStore, natsClient, authDB, cfg.Distributed.InstanceID) - // Initialize agent store agentStore, err := agents.NewAgentStore(authDB) if err != nil { @@ -428,15 +425,12 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } xlog.Info("Distributed agent store initialized") - // Initialize agent event bridge - agentBridge := agents.NewEventBridge(natsClient, agentStore, cfg.Distributed.InstanceID) - - // Start observable persister — captures observable_update events from workers - // (which have no DB access) and persists them to PostgreSQL. - if err := agentBridge.StartObservablePersister(); err != nil { - xlog.Warn("Failed to start observable persister", "error", err) - } else { - xlog.Info("Observable persister started") + // The job dispatcher and the agent event bridge, both on the broadcast + // carrier. See newFanoutBridges for why the two constructors are reached + // through one function that names *pgbus.Bus. + dispatcher, agentBridge, rebroadcast, err := newFanoutBridges(bus, natsClient, jobStore, agentStore, authDB, cfg.Distributed.InstanceID) + if err != nil { + return nil, err } // Initialize Phase 4 stores (MCP, Gallery, FineTune, Skills) @@ -476,7 +470,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // The consumer side of the claim queue, built and started in one act: see // startJobDispatchLoop for why those are not two lines. - jobDispatch, err := startJobDispatchLoop(cfg.Context, cfg.Distributed, authDB, jobStore, registry, clusterRegistry, controlClient, natsClient) + jobDispatch, err := startJobDispatchLoop(cfg.Context, cfg.Distributed, authDB, jobStore, registry, clusterRegistry, controlClient, rebroadcast) if err != nil { return nil, fmt.Errorf("wiring the job dispatch loop: %w", err) } diff --git a/core/application/fanout_wiring.go b/core/application/fanout_wiring.go new file mode 100644 index 000000000000..b14d94300e44 --- /dev/null +++ b/core/application/fanout_wiring.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: MIT + +package application + +import ( + "fmt" + + "github.com/mudler/LocalAI/core/services/agents" + "github.com/mudler/LocalAI/core/services/jobs" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/xlog" + "gorm.io/gorm" +) + +// newFanoutBridges builds the three surfaces whose cross-replica traffic is job +// and agent fan-out, on the deployment's ONE broadcast carrier. +// +// One function rather than two constructor calls in the start-up path, and the +// parameter type is the reason. jobs.NewDispatcher and agents.NewEventBridge +// both take a messaging.Broadcaster, which they must: neither may know which +// carrier a deployment runs, and their specs publish through a double. But that +// also means *messaging.Client satisfies them, so wiring either of them to the +// NATS client instead of the carrier COMPILES, passes every unit spec in both +// packages, and presents only as an SSE stream that stays empty while the work +// it is watching runs to completion on the other side of a carrier nobody is +// subscribed to. Naming *pgbus.Bus here is what makes that a build failure +// rather than a silent one, and it is why this exists as a function instead of +// as two lines and a comment asking the reader to be careful. +// +// The observable persister is STARTED here for the reason startJobDispatchLoop +// starts its loop: a bridge that is built and never subscribed captures nothing +// a worker publishes, and the only symptom is observables that quietly stop +// being written on a deployment that has workers. +// +// The re-broadcaster is built HERE and handed to the dispatch loop rather than +// built there from a carrier of its own, and that is the whole reason this +// function returns three things. It is the surface a reviewer skips: it is what +// turns an agent worker's progress line into a broadcast, so pointing it at a +// carrier nobody subscribes to leaves every unit spec in every package passing. +// The re-broadcaster publishes, the publish succeeds, Handle returns true, and +// the only symptom in the deployment is an SSE stream with no progress in it. +// Written as one expression shared with the dispatcher and the bridge, that +// mis-wiring stops being a line a spec has to guess at: there is no second +// carrier in scope to point it at. +// cancelCarrier is NOT the same carrier and must not be folded into bus. Every +// family this function wires has both of its ends on a frontend replica and so +// moves to the broadcast carrier whole, with one exception: the only subscriber +// to agent..cancel is the agent WORKER, which has no database and cannot +// join the PostgreSQL carrier at all. A cancel published on bus would therefore +// reach no worker, and every cancel of a worker-run agent would be lost while +// reporting success. It stays on the carrier the worker reads until a cancel +// rides the worker's tunnel instead of a broadcast. +func newFanoutBridges(bus *pgbus.Bus, cancelCarrier messaging.Broadcaster, + jobStore *jobs.JobStore, agentStore *agents.AgentStore, + db *gorm.DB, instanceID string) (*jobs.Dispatcher, *agents.EventBridge, *nodes.Rebroadcaster, error) { + // A nil check on the CONCRETE pointer, before it is widened. Once it is a + // messaging.Broadcaster a nil *pgbus.Bus is a non-nil interface holding a + // nil pointer, so every guard downstream reads it as a carrier that is + // present and every publish through it panics on a request instead. + if bus == nil { + return nil, nil, nil, fmt.Errorf("the job and agent fan-out bridges were built with no broadcast carrier: every job's progress and every agent's events would reach no SSE stream in the deployment") + } + + if cancelCarrier == nil { + return nil, nil, nil, fmt.Errorf("the agent event bridge was built with no carrier for agent cancels: every cancel of a worker-run agent would be published where no worker listens and reported as sent") + } + + dispatcher := jobs.NewDispatcher(jobStore, bus, db, instanceID) + bridge := agents.NewEventBridge(bus, agentStore, instanceID).WithCancelCarrier(cancelCarrier) + + // Warned rather than refused, and deliberately: the persister needs a store + // and a deployment without one still serves live SSE correctly. What it + // loses is the durable copy of a worker's observables, which is degraded + // rather than broken. + if err := bridge.StartObservablePersister(); err != nil { + xlog.Warn("Failed to start observable persister", "error", err) + } else { + xlog.Info("Observable persister started") + } + + return dispatcher, bridge, nodes.NewRebroadcaster(bus), nil +} diff --git a/core/application/fanout_wiring_test.go b/core/application/fanout_wiring_test.go new file mode 100644 index 000000000000..7472a0340202 --- /dev/null +++ b/core/application/fanout_wiring_test.go @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: MIT + +package application + +import ( + "context" + "encoding/json" + "runtime" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/agents" + "github.com/mudler/LocalAI/core/services/jobs" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" + "gorm.io/gorm" +) + +// The three fan-out surfaces, asserted from the OTHER replica's carrier. +// +// Everything here publishes on a second Bus and asserts on the effect the +// surface built on the first one had. One bus talking to itself would pass with +// the surfaces wired to any carrier at all, which is exactly the wiring defect +// these exist to catch: a dispatcher on one carrier and a publisher on another +// leaves every unit spec in both packages green and only an SSE stream empty. +var _ = Describe("wiring the job and agent fan-out bridges", func() { + var ( + ctx context.Context + db *gorm.DB + busA, busB *pgbus.Bus + jobStore *jobs.JobStore + agentStore *agents.AgentStore + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + ctx = context.Background() + + var dsn string + db, dsn = testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(ctx, db)).To(Succeed()) + + newBus := func() *pgbus.Bus { + b, err := pgbus.New(ctx, pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(b.Close) + return b + } + busA, busB = newBus(), newBus() + + var err error + jobStore, err = jobs.NewJobStore(db) + Expect(err).ToNot(HaveOccurred()) + agentStore, err = agents.NewAgentStore(db) + Expect(err).ToNot(HaveOccurred()) + }) + + // The cancel carrier is a SEPARATE argument, and its absence is refused + // separately. Folding it into bus would put every agent cancel on a carrier + // no worker can read, and CancelExecution would go on returning nil. + It("refuses to build with no carrier for agent cancels", func() { + _, _, _, err := newFanoutBridges(busA, nil, jobStore, agentStore, db, "replica-1") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("agent cancels")) + }) + + It("refuses to build with no carrier", func() { + _, _, _, err := newFanoutBridges(nil, testutil.NewFakeBus(), jobStore, agentStore, db, "replica-1") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("broadcast carrier")) + }) + + // S1. The dispatcher persists a terminal result broadcast by a peer, which + // it can only do if its wildcard subscription is on the carrier the peer + // published to. + It("subscribes the job dispatcher to results a peer replica broadcasts", func() { + dispatcher, _, _, err := newFanoutBridges(busA, testutil.NewFakeBus(), jobStore, agentStore, db, "replica-1") + Expect(err).ToNot(HaveOccurred()) + Expect(dispatcher.Start(ctx)).To(Succeed()) + DeferCleanup(dispatcher.Stop) + + job := &jobs.JobRecord{TaskID: "t1", UserID: "u1", Status: "running", TriggeredBy: "manual"} + Expect(jobStore.CreateJob(job)).To(Succeed()) + + Expect(busB.Publish(messaging.SubjectJobResult(job.ID), jobs.JobResultEvent{ + JobID: job.ID, Status: "completed", Result: "the answer", + })).To(Succeed()) + + Eventually(func() string { + stored, err := jobStore.GetJob(job.ID) + if err != nil { + return "" + } + return stored.Status + }, "20s").Should(Equal("completed")) + }) + + // S2. The observable persister writes what a peer broadcast, which it can + // only do if it was started AND is on the same carrier AND its filter has + // the right number of tokens. + It("subscribes the agent observable persister to events a peer replica broadcasts", func() { + _, bridge, _, err := newFanoutBridges(busA, testutil.NewFakeBus(), jobStore, agentStore, db, "replica-1") + Expect(err).ToNot(HaveOccurred()) + Expect(bridge).ToNot(BeNil()) + + Expect(busB.Publish(messaging.SubjectAgentEvents("a1", "u1"), agents.AgentEvent{ + AgentName: "a1", + UserID: "u1", + EventType: "observable_update", + EventSubType: "tool_result", + SourceInstance: "replica-2", + MessageID: "obs-1", + Metadata: `{"tool":"grep"}`, + })).To(Succeed()) + + Eventually(func() int { + records, err := agentStore.GetObservables(agents.AgentKey("u1", "a1"), 10) + if err != nil { + return 0 + } + return len(records) + }, "20s").Should(Equal(1)) + }) + + // S3, and it is the one this whole arrangement is for. The re-broadcaster + // is what turns an agent worker's progress line into a broadcast, and it is + // the surface a reviewer skips, because pointing it at a carrier nobody + // reads leaves every unit spec in every package green: it publishes, the + // publish succeeds, and Handle returns true. + // + // So these assert the RECEIPT on a peer's carrier and never Handle's return + // value, which is true for a publish that went nowhere. + DescribeTable("re-broadcasts a worker's line onto the carrier a peer replica reads", + func(subject string, payload string) { + _, _, rebroadcast, err := newFanoutBridges(busA, testutil.NewFakeBus(), jobStore, agentStore, db, "replica-1") + Expect(err).ToNot(HaveOccurred()) + + delivered := make(chan []byte, 4) + _, err = busB.Subscribe(subject, func(data []byte) { delivered <- data }) + Expect(err).ToNot(HaveOccurred()) + + rebroadcast.Handle(nodes.NodeTypeAgent, strings.ReplaceAll(subject, "*", "j1"), json.RawMessage(payload)) + + Eventually(delivered, "20s").Should(Receive(MatchJSON(payload))) + }, + Entry("a job's progress", messaging.SubjectJobProgressWildcard, `{"job_id":"j1","status":"running"}`), + Entry("a job's result", messaging.SubjectJobResultWildcard, `{"job_id":"j1","status":"completed"}`), + ) + + It("re-broadcasts an agent's events onto the carrier a peer replica reads", func() { + _, _, rebroadcast, err := newFanoutBridges(busA, testutil.NewFakeBus(), jobStore, agentStore, db, "replica-1") + Expect(err).ToNot(HaveOccurred()) + + delivered := make(chan []byte, 4) + _, err = busB.Subscribe(messaging.SubjectAgentEventsWildcard, func(data []byte) { delivered <- data }) + Expect(err).ToNot(HaveOccurred()) + + rebroadcast.Handle(nodes.NodeTypeAgent, messaging.SubjectAgentEvents("a1", "u1"), + json.RawMessage(`{"event_type":"json_message"}`)) + + Eventually(delivered, "20s").Should(Receive(MatchJSON(`{"event_type":"json_message"}`))) + }) +}) diff --git a/core/http/endpoints/openresponses/sync_pg_test.go b/core/http/endpoints/openresponses/sync_pg_test.go new file mode 100644 index 000000000000..ab4c73bc7376 --- /dev/null +++ b/core/http/endpoints/openresponses/sync_pg_test.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: MIT + +package openresponses + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/distributed" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// The same two-replica topology, on the carrier a deployment actually runs. +// +// The specs beside this one share ONE in-memory double, so a replica hears its +// peer through a function call. Here each replica holds its own LISTEN +// connection, which is the only arrangement in which a cancel can be published +// on a channel nobody listened to, arrive after the handler that would have +// applied it was closed, or be dropped for a subscriber that fell behind. A +// double cannot fail any of those ways. +var _ = Describe("ResponseStore cross-replica on the broadcast carrier", func() { + var ( + ctx context.Context + db *gorm.DB + store *distributed.ResponseMetadataStore + replicaA *ResponseStore + replicaB *ResponseStore + ) + + BeforeEach(func() { + ctx = context.Background() + + var dsn string + db, dsn = testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(ctx, db)).To(Succeed()) + + newBus := func() *pgbus.Bus { + b, err := pgbus.New(ctx, pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(b.Close) + return b + } + + var err error + store, err = distributed.NewResponseMetadataStore(db) + Expect(err).ToNot(HaveOccurred()) + + replicaA = NewResponseStore(0) + replicaB = NewResponseStore(0) + Expect(replicaA.EnableDistributed(ctx, newBus(), "replica-a", store)).To(Succeed()) + Expect(replicaB.EnableDistributed(ctx, newBus(), "replica-b", store)).To(Succeed()) + }) + + AfterEach(func() { + Expect(replicaA.Close()).To(Succeed()) + Expect(replicaB.Close()).To(Succeed()) + }) + + It("reaches the CancelFunc held by the owning replica over two LISTEN connections", func() { + const id = "resp_cancel_pg" + cancelled := make(chan struct{}) + replicaA.StoreBackground(id, &schema.OpenResponsesRequest{Model: "test-model"}, + &schema.ORResponseResource{ + ID: id, Object: "response", CreatedAt: time.Now().Unix(), + Status: schema.ORStatusInProgress, Model: "test-model", + }, func() { close(cancelled) }, false) + + // Waited for, not assumed. On the real carrier the metadata delta is + // asynchronous, so a cancel issued before it lands answers "not found" + // even though the durable row exists: the SyncedMap reads its own + // memory and goes to the table only on re-hydrate. The in-memory double + // the sibling specs share delivers synchronously and hides that + // entirely, which is why this spec exists. + Eventually(func() error { + _, err := replicaB.Get(id) + return err + }, "20s").Should(Succeed()) + + // The cancel lands on the replica that does NOT hold the CancelFunc. + resp, err := replicaB.Cancel(id) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Status).To(Equal(schema.ORStatusCancelled)) + + Eventually(cancelled, "20s").Should(BeClosed()) + }) + + It("does not report a cancel that reached nobody as a cancel that was refused", func() { + // The owner is gone, so nothing applies the broadcast. This carrier is + // at-most-once with no replay, and there is no reply to wait for, so the + // caller must still get a prompt terminal answer rather than an error + // that reads as the generation having declined to stop. + const id = "resp_dead_owner_pg" + replicaA.StoreBackground(id, &schema.OpenResponsesRequest{Model: "test-model"}, + &schema.ORResponseResource{ + ID: id, Object: "response", CreatedAt: time.Now().Unix(), + Status: schema.ORStatusInProgress, Model: "test-model", + }, func() {}, false) + Eventually(func() error { + _, err := replicaB.Get(id) + return err + }, "20s").Should(Succeed()) + Expect(replicaA.Close()).To(Succeed()) + + resp, err := replicaB.Cancel(id) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Status).To(Equal(schema.ORStatusCancelled)) + }) +}) diff --git a/core/http/routes/openresponses.go b/core/http/routes/openresponses.go index 4cf243864d82..235639a08b4d 100644 --- a/core/http/routes/openresponses.go +++ b/core/http/routes/openresponses.go @@ -1,6 +1,8 @@ package routes import ( + "context" + "github.com/labstack/echo/v4" "github.com/mudler/LocalAI/core/application" "github.com/mudler/LocalAI/core/config" @@ -19,21 +21,8 @@ func RegisterOpenResponsesRoutes(app *echo.Echo, // How the MCP endpoints reach an agent worker; nil outside distributed mode. agentControl := mcpAgentControl(application) if d := application.Distributed(); d != nil { - // Replicate response metadata across frontend replicas and subscribe to - // delegated cancels. Without this a GET, a previous_response_id lookup or - // a cancel that the load balancer sends to a replica other than the - // creator 404s, and the cancel never reaches the CancelFunc (#10993). - // Standalone deployments skip this entirely and stay process-local. - // - // The durable store is what a replica re-hydrates from after its - // subscription missed a delta; without it the same response_id answers - // 404 here and 200 on the peer that created it, forever. - var responseStore *distributed.ResponseMetadataStore - if d.DistStores != nil { - responseStore = d.DistStores.Responses - } - if err := openresponses.GetGlobalStore().EnableDistributed( - application.ApplicationConfig().Context, d.Broadcast(), application.InstanceID(), responseStore); err != nil { + if err := enableDistributedResponses(application.ApplicationConfig().Context, d, + openresponses.GetGlobalStore(), application.InstanceID()); err != nil { xlog.Error("Failed to enable cross-replica Open Responses store", "error", err) } } @@ -91,3 +80,30 @@ func setOpenResponsesRequestContext(re *middleware.RequestExtractor) echo.Middle } } } + +// enableDistributedResponses replicates response metadata across frontend +// replicas and subscribes to delegated cancels. Without it a GET, a +// previous_response_id lookup or a cancel that the load balancer sends to a +// replica other than the creator 404s, and the cancel never reaches the +// CancelFunc (#10993). Standalone deployments never reach here and stay +// process-local. +// +// The durable store is what a replica re-hydrates from after its subscription +// missed a delta; without it the same response_id answers 404 here and 200 on +// the peer that created it, forever. +// +// A named function rather than a block inside route registration, and that is +// the point of it. EnableDistributed takes a messaging.Broadcaster, as it must: +// its own specs publish through a double. So handing it the NATS client instead +// of the deployment's carrier COMPILES and reddens nothing anywhere, and the +// only symptom is a cancel that answers 404 on every replica but one. Registering +// routes needs a whole Application and therefore has no spec; this needs a +// DistributedServices and a store, and therefore has one. +func enableDistributedResponses(ctx context.Context, d *application.DistributedServices, + store *openresponses.ResponseStore, replicaID string) error { + var responseStore *distributed.ResponseMetadataStore + if d.DistStores != nil { + responseStore = d.DistStores.Responses + } + return store.EnableDistributed(ctx, d.Broadcast(), replicaID, responseStore) +} diff --git a/core/http/routes/openresponses_wiring_test.go b/core/http/routes/openresponses_wiring_test.go new file mode 100644 index 000000000000..1492511398cf --- /dev/null +++ b/core/http/routes/openresponses_wiring_test.go @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: MIT + +package routes + +import ( + "context" + "runtime" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/http/endpoints/openresponses" + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/distributed" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// Which carrier the Open Responses store is enabled on. +// +// EnableDistributed takes a messaging.Broadcaster, which it must: its own specs +// publish through a double, and it cannot be made to name a concrete carrier +// without dragging that dependency through the whole endpoint package. The +// consequence is that handing it the NATS client instead of the deployment's +// carrier compiles and reddens nothing, and the only symptom is a cancel that +// answers 404 on every replica but the creator. So it is pinned here, by +// watching what actually arrives on the carrier. +var _ = Describe("wiring the Open Responses store to a carrier", func() { + var ( + ctx context.Context + busA, busB *pgbus.Bus + store *distributed.ResponseMetadataStore + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + ctx = context.Background() + + db, dsn := testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(ctx, db)).To(Succeed()) + newBus := func() *pgbus.Bus { + b, err := pgbus.New(ctx, pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(b.Close) + return b + } + busA, busB = newBus(), newBus() + + var err error + store, err = distributed.NewResponseMetadataStore(db) + Expect(err).ToNot(HaveOccurred()) + }) + + It("enables it on the deployment's broadcast carrier and not on anything else it holds", func() { + // A DistributedServices holding BOTH, exactly as a running deployment + // does. That is what makes this an assertion about which one was + // chosen rather than about there being one at all. + d := &application.DistributedServices{ + Bus: busA, + DistStores: &distributed.Stores{Responses: store}, + } + + responses := openresponses.NewResponseStore(0) + Expect(enableDistributedResponses(ctx, d, responses, "replica-a")).To(Succeed()) + DeferCleanup(func() { _ = responses.Close() }) + + // A peer's carrier sees the metadata this replica mirrors, which it can + // only do if the store was enabled on the carrier and not on the other + // thing DistributedServices is holding. + mirrored := make(chan []byte, 8) + _, err := busB.Subscribe(messaging.SubjectSyncStateDelta("responses.metadata"), func(data []byte) { + mirrored <- data + }) + Expect(err).ToNot(HaveOccurred()) + + const id = "resp_wiring" + responses.StoreBackground(id, &schema.OpenResponsesRequest{Model: "test-model"}, + &schema.ORResponseResource{ + ID: id, Object: "response", CreatedAt: time.Now().Unix(), + Status: schema.ORStatusInProgress, Model: "test-model", + }, func() {}, false) + + Eventually(mirrored, "20s").Should(Receive(ContainSubstring(id))) + }) + + It("refuses to enable without the durable store a reconnecting replica re-hydrates from", func() { + d := &application.DistributedServices{Bus: busA} + + responses := openresponses.NewResponseStore(0) + Expect(enableDistributedResponses(ctx, d, responses, "replica-a")).ToNot(Succeed()) + }) +}) diff --git a/core/services/agents/events.go b/core/services/agents/events.go index 4682d2a0ef22..c6938d40f322 100644 --- a/core/services/agents/events.go +++ b/core/services/agents/events.go @@ -16,7 +16,7 @@ import ( "github.com/mudler/xlog" ) -// AgentEvent is the NATS message payload for agent SSE events. +// AgentEvent is the broadcast payload for agent SSE events. type AgentEvent struct { AgentName string `json:"agent_name"` UserID string `json:"user_id"` @@ -30,17 +30,41 @@ type AgentEvent struct { Timestamp int64 `json:"timestamp"` // Unix milliseconds (set by PublishEvent) } -// AgentCancelEvent is the NATS message payload for cancelling agent execution. +// AgentCancelEvent is the broadcast payload for cancelling agent execution. type AgentCancelEvent struct { AgentName string `json:"agent_name"` UserID string `json:"user_id"` MessageID string `json:"message_id,omitempty"` } -// EventBridge bridges agent events between NATS and SSE connections. -// It enables cross-instance SSE: user connects to Frontend 1, agent runs on Frontend 2. +// EventBridge bridges agent events between the broadcast carrier and SSE +// connections. It enables cross-instance SSE: a user connects to Frontend 1 +// while the agent runs on Frontend 2. type EventBridge struct { - nats messaging.MessagingClient + // bus is the fan-out carrier this bridge subscribes on. A Broadcaster and + // not a MessagingClient because fan-out is all a bridge needs: an agent's + // events go to whoever is watching, and there is nothing here to request or + // to queue. + bus messaging.Broadcaster + + // cancelBus is where agent..cancel is published and heard, and it is + // a SEPARATE field from bus because in this deployment the two ends of that + // family cannot be on the same carrier yet. + // + // Every other family here has both ends on a frontend replica, so both move + // to the broadcast carrier together. This one does not: its only subscriber + // is the agent WORKER (core/cli/agent_worker.go), the cancel has to reach + // the worker actually running the execution, and a worker has no database + // and so cannot join the PostgreSQL carrier at all. Publishing a cancel + // where no worker is listening would report a cancel that reached nobody as + // a cancel the execution declined, which is the one thing this whole + // programme may not do. + // + // So it is named, and it is separate, and it stays on the carrier the + // worker reads until a cancel rides the worker's tunnel instead. Folding it + // back into bus is not a simplification: it is silent loss of every cancel + // for every worker-run agent. + cancelBus messaging.Broadcaster // pub is where the events this bridge produces GO, which is not always the // bus. On an agent worker running a dispatched claim it is the response // body of the control RPC the claiming replica is reading, so the events @@ -58,21 +82,49 @@ type EventBridge struct { // stream-backed one. A copied sync.Map would swallow every cancel. cancelRegistry *messaging.CancelRegistry - // Background NATS subscriptions owned by this bridge + // The process-lifetime subscription this bridge owns. The per-request one + // SubscribeEvents opens belongs to the HTTP handler that opened it. obsPersisterSub messaging.Subscription } -// NewEventBridge creates a new EventBridge. -func NewEventBridge(nc messaging.MessagingClient, store *AgentStore, instanceID string) *EventBridge { +// NewEventBridge creates a new EventBridge on the deployment's fan-out carrier. +// +// Cancels go on that same carrier unless WithCancelCarrier says otherwise, +// which is right for the agent worker, where there is only one carrier to be +// on, and wrong for a frontend replica, which reads fan-out from PostgreSQL and +// has to reach workers that cannot. +func NewEventBridge(bus messaging.Broadcaster, store *AgentStore, instanceID string) *EventBridge { return &EventBridge{ - nats: nc, - pub: nc, + bus: bus, + cancelBus: bus, + pub: bus, store: store, instanceID: instanceID, cancelRegistry: &messaging.CancelRegistry{}, } } +// WithCancelCarrier puts agent..cancel on carrier instead of on the +// fan-out bus, and returns the receiver so it can be written as one expression +// with the constructor. +// +// A frontend replica needs it and an agent worker does not. The cancel has to +// reach the worker running the execution; a worker has no database and cannot +// join the PostgreSQL carrier; so a frontend that published its cancels there +// would publish them where no worker listens, and a cancel that reached nobody +// is not a cancel that was refused. +// +// A nil carrier leaves the bridge on the fan-out bus rather than on nothing, +// because a bridge that publishes cancels nowhere is the failure this exists to +// prevent. +func (b *EventBridge) WithCancelCarrier(carrier messaging.Broadcaster) *EventBridge { + if b == nil || carrier == nil { + return b + } + b.cancelBus = carrier + return b +} + // WithPublisher returns a view of this bridge whose events go to pub. // // Everything else is SHARED with the receiver, the cancel registry above all: a @@ -91,7 +143,7 @@ func (b *EventBridge) WithPublisher(pub messaging.Publisher) *EventBridge { return &view } -// PublishEvent publishes an agent event to NATS for SSE bridging. +// PublishEvent broadcasts an agent event for SSE bridging. // // Timestamp is emitted in Unix milliseconds to match the local dispatcher's // json_message events (see dispatcher.go) and the React UI, which feeds the @@ -106,8 +158,8 @@ func (b *EventBridge) PublishEvent(agentName, userID string, evt AgentEvent) err // PersistObservable publishes an observable_update SSE event for real-time UI // updates and, if a database store is available, writes the record to the DB. -// When the store is nil (e.g. on agent workers), the NATS event is still -// published so the frontend can persist it via StartObservablePersister. +// When the store is nil (e.g. on agent workers), the event is still published so +// the frontend can persist it via StartObservablePersister. func (b *EventBridge) PersistObservable(agentName, userID, eventType string, obs any) { payload := dbutil.MarshalJSON(obs) recordID := uuid.New().String() @@ -123,7 +175,7 @@ func (b *EventBridge) PersistObservable(agentName, userID, eventType string, obs }) } - // Always publish NATS event — enables real-time SSE and remote persistence. + // Always broadcast, which is what enables real-time SSE and remote persistence. b.PublishEvent(agentName, userID, AgentEvent{ AgentName: agentName, UserID: userID, @@ -135,7 +187,7 @@ func (b *EventBridge) PersistObservable(agentName, userID, eventType string, obs }) } -// PublishMessage publishes a chat message event via NATS for SSE bridging. +// PublishMessage broadcasts a chat message event for SSE bridging. // Uses "json_message" event type to match the React UI's expected SSE format. // Conversation history is managed client-side (browser localStorage), not server-side. func (b *EventBridge) PublishMessage(agentName, userID, sender, content, messageID string) error { @@ -172,10 +224,10 @@ func (b *EventBridge) PublishStatus(agentName, userID, status string) error { // SubscribeEvents subscribes to agent events for a specific agent+user. func (b *EventBridge) SubscribeEvents(agentName, userID string, handler func(AgentEvent)) (messaging.Subscription, error) { subject := messaging.SubjectAgentEvents(agentName, userID) - return messaging.SubscribeJSON(b.nats, subject, handler) + return messaging.SubscribeJSON(b.bus, subject, handler) } -// PublishStreamEvent publishes a stream event (reasoning, content, tool_call, done) via NATS. +// PublishStreamEvent broadcasts a stream event (reasoning, content, tool_call, done). // These are forwarded as "stream_event" SSE events matching the React UI's expected format. func (b *EventBridge) PublishStreamEvent(agentName, userID string, data map[string]any) error { return b.PublishEvent(agentName, userID, AgentEvent{ @@ -193,8 +245,13 @@ func (b *EventBridge) CancelExecution(agentName, userID, messageID string) error xlog.Info("Cancelled agent execution locally", "agent", agentName, "user", userID, "messageID", messageID) } - // Also publish via NATS for other instances - return b.nats.Publish(messaging.SubjectAgentCancel(agentName), AgentCancelEvent{ + // Broadcast so the replica that actually holds the execution can act on it. + // + // The error says whether the request was PUBLISHED and nothing more. This + // carrier is at-most-once with no replay, so a cancel that reached nobody + // and a cancel an execution declined are different facts that cannot be + // told apart from here, and neither may be reported as the other. + return b.cancelBus.Publish(messaging.SubjectAgentCancel(agentName), AgentCancelEvent{ AgentName: agentName, UserID: userID, MessageID: messageID, @@ -211,28 +268,32 @@ func (b *EventBridge) DeregisterCancel(key string) { b.cancelRegistry.Deregister(key) } -// StartCancelListener subscribes to NATS cancel events (broadcast to all instances). +// StartCancelListener subscribes to the cancel broadcasts every replica sees. func (b *EventBridge) StartCancelListener() (messaging.Subscription, error) { - return messaging.SubscribeJSON(b.nats, messaging.SubjectAgentCancelWildcard, func(evt AgentCancelEvent) { + return messaging.SubscribeJSON(b.cancelBus, messaging.SubjectAgentCancelWildcard, func(evt AgentCancelEvent) { if evt.MessageID != "" { if b.cancelRegistry.Cancel(evt.MessageID) { - xlog.Info("Cancelled agent via NATS", "agent", evt.AgentName, "user", evt.UserID, "messageID", evt.MessageID) + xlog.Info("Cancelled an agent execution on this replica after a broadcast cancel", "agent", evt.AgentName, "user", evt.UserID, "messageID", evt.MessageID) } } }) } -// StartObservablePersister subscribes to all agent events via NATS and persists -// observable_update events to the database. This runs on the frontend to capture -// observables published by workers (which have no database access). -// The subscription is stored on the EventBridge and cleaned up when the NATS -// connection closes. +// StartObservablePersister subscribes to every agent's events and persists the +// observable_update ones to the database. This runs on the frontend, to capture +// observables published by workers, which have no database access. +// +// The subscription lives for the life of this bridge and is one per replica, not +// one per request. func (b *EventBridge) StartObservablePersister() error { if b.store == nil { return fmt.Errorf("no store available for observable persistence") } - // Subscribe to all agent events using wildcard: agent.*.events.* - sub, err := messaging.SubscribeJSON(b.nats, "agent.*.events.*", func(evt AgentEvent) { + // The filter is the constant next to the builder it has to match. It used + // to be a literal here, four tokens spelled by hand three files from + // SubjectAgentEvents, and a filter one token short of its subject matches + // nothing at all with no error anywhere. + sub, err := messaging.SubscribeJSON(b.bus, messaging.SubjectAgentEventsWildcard, func(evt AgentEvent) { if evt.EventType != "observable_update" { return } @@ -267,7 +328,7 @@ func (b *EventBridge) StartObservablePersister() error { return nil } -// HandleSSE bridges NATS agent events to SSE for a specific agent and user. +// HandleSSE bridges an agent's event broadcasts to SSE for one agent and user. func (b *EventBridge) HandleSSE(c echo.Context, agentName, userID string) error { if agentName == "" { return c.JSON(http.StatusBadRequest, map[string]string{"error": "agent name required"}) @@ -275,8 +336,8 @@ func (b *EventBridge) HandleSSE(c echo.Context, agentName, userID string) error return b.handleSSEInternal(c, agentName, userID) } -// SSEHandler returns an Echo handler that bridges NATS agent events to SSE. -// This is the distributed version of the SSE endpoint. +// SSEHandler returns an Echo handler that bridges an agent's event broadcasts +// to SSE. This is the distributed version of the SSE endpoint. func (b *EventBridge) SSEHandler() echo.HandlerFunc { return func(c echo.Context) error { agentName := c.Param("name") @@ -348,9 +409,20 @@ func (b *EventBridge) handleSSEInternal(c echo.Context, agentName, userID string writeSSE("json_error", `{"error":"failed to subscribe to agent events"}`) return nil } + // Deferred, not called on the way out. This subscription is opened and + // closed PER HTTP REQUEST, and on the PostgreSQL carrier only the first + // subscriber of a channel issues a LISTEN while every later one registers + // an in-process filter, so a return that skipped this would leave a replica + // running one extra closure per notification for every stream it has ever + // served, and nothing in the tree would fail. + defer func() { + closed.Store(true) + if uerr := sub.Unsubscribe(); uerr != nil { + xlog.Warn("Failed to close an agent event subscription", "agent", agentName, "user", userID, "error", uerr) + } + }() + // Wait for client disconnect <-c.Request().Context().Done() - closed.Store(true) - sub.Unsubscribe() return nil } diff --git a/core/services/agents/events_pg_test.go b/core/services/agents/events_pg_test.go new file mode 100644 index 000000000000..db8176126590 --- /dev/null +++ b/core/services/agents/events_pg_test.go @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: MIT + +package agents + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "sync" + + "github.com/labstack/echo/v4" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// The event bridge against the carrier it runs on, across TWO connections. +// +// The wildcard rows below are the matcher's own rows re-asserted through the +// real carrier, and that is not duplication: a matcher that is right and a +// filter that is wrong look identical from inside the messaging package, and +// the symptom of the wrong filter is a persister that writes nothing at all +// with no error anywhere. +var _ = Describe("the agent event bridge on the broadcast carrier", func() { + var ( + ctx context.Context + db *gorm.DB + busA, busB *pgbus.Bus + store *AgentStore + bridge *EventBridge + ) + + BeforeEach(func() { + ctx = context.Background() + var dsn string + db, dsn = testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(ctx, db)).To(Succeed()) + + newBus := func() *pgbus.Bus { + b, err := pgbus.New(ctx, pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(b.Close) + return b + } + busA, busB = newBus(), newBus() + + var err error + store, err = NewAgentStore(db) + Expect(err).ToNot(HaveOccurred()) + bridge = NewEventBridge(busA, store, "replica-a") + }) + + // observable is the AgentEvent shape the persister acts on, minus the + // subject it travels on, so a spec varies only the subject. + observable := func(agent, user, id string) AgentEvent { + return AgentEvent{ + AgentName: agent, + UserID: user, + EventType: "observable_update", + EventSubType: "tool_result", + SourceInstance: "replica-b", + MessageID: id, + Metadata: `{"tool":"grep"}`, + } + } + + countFor := func(user, agent string) func() int { + return func() int { + records, err := store.GetObservables(AgentKey(user, agent), 10) + if err != nil { + return -1 + } + return len(records) + } + } + + Describe("StartObservablePersister", func() { + BeforeEach(func() { + Expect(bridge.StartObservablePersister()).To(Succeed()) + }) + + It("persists an observable a peer replica published on the built subject", func() { + Expect(busB.Publish(messaging.SubjectAgentEvents("a1", "u1"), observable("a1", "u1", "obs-1"))).To(Succeed()) + + Eventually(countFor("u1", "a1"), "20s").Should(Equal(1)) + }) + + It("ignores a three-token subject that a shorter filter would have swallowed", func() { + // agent.a1.events has one token fewer than SubjectAgentEvents + // builds. A filter of that shape matches this and NOT the real + // subject, which is the exact inversion the token count prevents. + Expect(busB.Publish("agent.a1.events", observable("a1", "u1", "obs-short"))).To(Succeed()) + // The sentinel, on the subject that certainly matches, orders the + // negative without a clock. + Expect(busB.Publish(messaging.SubjectAgentEvents("a1", "u1"), observable("a1", "u1", "obs-real"))).To(Succeed()) + + Eventually(countFor("u1", "a1"), "20s").Should(Equal(1)) + records, err := store.GetObservables(AgentKey("u1", "a1"), 10) + Expect(err).ToNot(HaveOccurred()) + Expect(records[0].ID).To(Equal("obs-real")) + }) + + It("ignores a five-token subject with the same leading tokens", func() { + Expect(busB.Publish("agent.a1.b.events.u1", observable("a1", "u1", "obs-long"))).To(Succeed()) + Expect(busB.Publish(messaging.SubjectAgentEvents("a1", "u1"), observable("a1", "u1", "obs-real"))).To(Succeed()) + + Eventually(countFor("u1", "a1"), "20s").Should(Equal(1)) + records, err := store.GetObservables(AgentKey("u1", "a1"), 10) + Expect(err).ToNot(HaveOccurred()) + Expect(records[0].ID).To(Equal("obs-real")) + }) + + It("persists for every agent and every user, not only the first it saw", func() { + Expect(busB.Publish(messaging.SubjectAgentEvents("a1", "u1"), observable("a1", "u1", "obs-a1"))).To(Succeed()) + Expect(busB.Publish(messaging.SubjectAgentEvents("a2", "u2"), observable("a2", "u2", "obs-a2"))).To(Succeed()) + + Eventually(countFor("u1", "a1"), "20s").Should(Equal(1)) + Eventually(countFor("u2", "a2"), "20s").Should(Equal(1)) + }) + }) + + Describe("SubscribeEvents", func() { + It("receives the agent and user it asked for and not another's events", func() { + mine := make(chan AgentEvent, 8) + sub, err := bridge.SubscribeEvents("a1", "u1", func(evt AgentEvent) { mine <- evt }) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = sub.Unsubscribe() }) + + peer := NewEventBridge(busB, store, "replica-b") + Expect(peer.PublishMessage("a1", "u2", "agent", "for another user", "m-other")).To(Succeed()) + Expect(peer.PublishMessage("a1", "u1", "agent", "for me", "m-mine")).To(Succeed()) + + var first AgentEvent + Eventually(mine, "20s").Should(Receive(&first)) + Expect(first.MessageID).To(Equal("m-mine")) + Consistently(mine, "500ms", "50ms").ShouldNot(Receive()) + }) + + It("closes cleanly and leaves the register where it found it", func() { + before := busA.Subscribers() + sub, err := bridge.SubscribeEvents("a1", "u1", func(AgentEvent) {}) + Expect(err).ToNot(HaveOccurred()) + Expect(busA.Subscribers()).To(Equal(before + 1)) + + Expect(sub.Unsubscribe()).To(Succeed()) + Expect(busA.Subscribers()).To(Equal(before)) + }) + }) + + Describe("the SSE handler's per-request subscription", func() { + // The second of the two subscriptions this deployment opens and closes + // PER HTTP REQUEST. A leaked filter has no symptom: the replica just + // runs one more closure per notification for every stream it has ever + // served, for as long as the process lives. + It("is closed however the handler returns", func() { + before := busA.Subscribers() + + out := &syncBody{} + req := httptest.NewRequest(http.MethodGet, "/api/agents/a1/sse/distributed?user_id=u1", nil) + reqCtx, cancelReq := context.WithCancel(ctx) + c := echo.New().NewContext(req.WithContext(reqCtx), out) + + done := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(done) + Expect(bridge.HandleSSE(c, "a1", "u1")).To(Succeed()) + }() + + Eventually(busA.Subscribers, "20s").Should(Equal(before + 1)) + + // Delivery first, so this is a spec about a stream that WORKED and + // then closed, rather than one that never started. + peer := NewEventBridge(busB, store, "replica-b") + Expect(peer.PublishMessage("a1", "u1", "agent", "hello", "m-1")).To(Succeed()) + Eventually(out.String, "20s").Should(ContainSubstring("hello")) + + cancelReq() + Eventually(done, "20s").Should(BeClosed()) + Eventually(busA.Subscribers, "20s").Should(Equal(before), + "a stream that has returned must leave no handler behind") + }) + }) + + Describe("cancel broadcasts", func() { + It("reaches a peer replica's cancel listener", func() { + peer := NewEventBridge(busB, store, "replica-b") + sub, err := peer.StartCancelListener() + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = sub.Unsubscribe() }) + + cancelled := make(chan struct{}) + peer.RegisterCancel("msg-1", func() { close(cancelled) }) + + Expect(bridge.CancelExecution("a1", "u1", "msg-1")).To(Succeed()) + + Eventually(cancelled, "20s").Should(BeClosed()) + }) + + // The one family whose two ends are NOT on the same carrier, and the + // spec that says so out loud. + // + // Its only subscriber is the agent WORKER, which has no database and + // therefore cannot join the PostgreSQL carrier. A frontend that + // published its cancels onto the fan-out bus would publish them where + // no worker is listening, every cancel of a worker-run agent would be + // lost, and CancelExecution would return nil throughout: a cancel that + // reached nobody reported as a cancel that was sent, and one step later + // as a cancel the execution declined. + It("publishes a cancel where the worker listens and not onto the fan-out carrier", func() { + // The worker's carrier. Not a second pgbus: the whole point is that + // a worker cannot have one. + workerCarrier := testutil.NewFakeBus() + + frontend := NewEventBridge(busA, store, "replica-a").WithCancelCarrier(workerCarrier) + + worker := NewEventBridge(workerCarrier, nil, "agent-worker-1") + workerSub, err := worker.StartCancelListener() + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = workerSub.Unsubscribe() }) + + cancelled := make(chan struct{}) + worker.RegisterCancel("msg-worker", func() { close(cancelled) }) + + // A listener on the FAN-OUT carrier, which must be shown nothing. + // Without this the spec would pass with the cancel published on + // both, which is the shape that hides the loss rather than fixing + // it. + onTheBus := make(chan []byte, 4) + _, err = busB.Subscribe(messaging.SubjectAgentCancelWildcard, func(data []byte) { onTheBus <- data }) + Expect(err).ToNot(HaveOccurred()) + + Expect(frontend.CancelExecution("a1", "u1", "msg-worker")).To(Succeed()) + + Eventually(cancelled, "20s").Should(BeClosed()) + Consistently(onTheBus, "500ms", "50ms").ShouldNot(Receive(), + "a cancel on the fan-out carrier reaches no worker and would be lost") + }) + }) +}) + +// syncBody is an http.ResponseWriter a spec may read WHILE the handler is still +// writing. httptest.ResponseRecorder's buffer is not safe for that, and reading +// it mid-stream is the only way to know the handler reached its wait rather than +// assuming it did. +type syncBody struct { + mu sync.Mutex + buf bytes.Buffer + header http.Header +} + +func (w *syncBody) Header() http.Header { + if w.header == nil { + w.header = http.Header{} + } + return w.header +} + +func (w *syncBody) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.buf.Write(p) +} + +func (w *syncBody) WriteHeader(int) {} + +func (w *syncBody) Flush() {} + +func (w *syncBody) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.buf.String() +} diff --git a/core/services/agents/events_test.go b/core/services/agents/events_test.go index f103f7982e57..95802bf8ba97 100644 --- a/core/services/agents/events_test.go +++ b/core/services/agents/events_test.go @@ -9,41 +9,25 @@ import ( . "github.com/onsi/gomega" ) -// fakeMessagingClient implements messaging.MessagingClient and captures the -// last published payload so tests can assert on it. -type fakeMessagingClient struct { +// recordingBus is a messaging.Broadcaster that captures the last published +// payload. A Broadcaster and not a MessagingClient because fan-out is the whole +// of what an EventBridge may do with a carrier. +type recordingBus struct { lastSubject string lastData any } -func (f *fakeMessagingClient) Publish(subject string, data any) error { +func (f *recordingBus) Publish(subject string, data any) error { f.lastSubject = subject f.lastData = data return nil } -func (f *fakeMessagingClient) Subscribe(string, func([]byte)) (messaging.Subscription, error) { +func (f *recordingBus) Subscribe(string, func([]byte)) (messaging.Subscription, error) { return &fakeSub{}, nil } -func (f *fakeMessagingClient) QueueSubscribe(string, string, func([]byte)) (messaging.Subscription, error) { - return &fakeSub{}, nil -} - -func (f *fakeMessagingClient) QueueSubscribeReply(string, string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return &fakeSub{}, nil -} - -func (f *fakeMessagingClient) SubscribeReply(string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return &fakeSub{}, nil -} - -func (f *fakeMessagingClient) Request(string, []byte, time.Duration) ([]byte, error) { - return nil, nil -} - -func (f *fakeMessagingClient) IsConnected() bool { return true } -func (f *fakeMessagingClient) Close() {} +var _ messaging.Broadcaster = (*recordingBus)(nil) type fakeSub struct{} @@ -57,7 +41,7 @@ var _ = Describe("EventBridge", func() { // React UI both expect Unix milliseconds. Nanoseconds also overflow JS's // safe-integer range. The timestamp must be in milliseconds. It("emits the timestamp in Unix milliseconds", func() { - fake := &fakeMessagingClient{} + fake := &recordingBus{} bridge := NewEventBridge(fake, nil, "instance-1") before := time.Now().UnixMilli() diff --git a/core/services/jobs/dispatcher.go b/core/services/jobs/dispatcher.go index 4d1f9222a297..8d9ff0dde016 100644 --- a/core/services/jobs/dispatcher.go +++ b/core/services/jobs/dispatcher.go @@ -67,7 +67,7 @@ type CancelEvent struct { // no concurrency limiter: there is nothing here to limit. type Dispatcher struct { store *JobStore - nats messaging.MessagingClient + bus messaging.Broadcaster db *gorm.DB instanceID string configLoader ModelConfigLoader // optional: to enrich job events with model config @@ -75,21 +75,44 @@ type Dispatcher struct { // Cancel registry (notetaker pattern) cancelRegistry messaging.CancelRegistry - // NATS subscriptions + // The broadcast subscriptions this dispatcher owns for the life of the + // process. The per-request one SubscribeProgress opens is not here: it + // belongs to the HTTP handler that opened it and is closed with it. cancelSub messaging.Subscription resultSub messaging.Subscription progressSub messaging.Subscription + // terminalRecheck is how often an open SSE stream re-reads the job row. + // Zero means DefaultTerminalRecheck. Set once, before Start, and read from + // HTTP handler goroutines afterwards. + terminalRecheck time.Duration + // Lifecycle ctx context.Context cancel context.CancelFunc } +// SetTerminalRecheck sets how often an open job-progress SSE stream re-reads +// the job row while it waits. +// +// It exists so a spec can drive the recovery from a dropped terminal broadcast +// without waiting out DefaultTerminalRecheck. Call it before Start. +func (d *Dispatcher) SetTerminalRecheck(interval time.Duration) { + d.terminalRecheck = interval +} + // NewDispatcher creates a new distributed job Dispatcher. -func NewDispatcher(store *JobStore, nc messaging.MessagingClient, db *gorm.DB, instanceID string) *Dispatcher { +// +// The carrier is a messaging.Broadcaster because fan-out is all this type does +// with it and all it may have: everything else a job needs from another replica +// travels on the claim row or on the control RPC's response body, and a +// dispatcher that could reach a request/reply or a queue group through this +// field would be able to reintroduce the publish-to-nobody the claim queue +// replaced. +func NewDispatcher(store *JobStore, bus messaging.Broadcaster, db *gorm.DB, instanceID string) *Dispatcher { return &Dispatcher{ store: store, - nats: nc, + bus: bus, db: db, instanceID: instanceID, } @@ -133,9 +156,15 @@ func (d *Dispatcher) Start(ctx context.Context) error { var err error // Subscribe to cancel events (broadcast to all — each instance checks its registry) - d.cancelSub, err = messaging.SubscribeJSON(d.nats, messaging.SubjectJobCancelWildcard, func(evt CancelEvent) { + // A cancel that does not arrive is not a cancel that was refused. This + // carrier is at-most-once with no replay, so a broadcast that is dropped or + // that lands while this replica is reconnecting reaches nobody, and the + // registry simply never hears about it. Nothing here may report that as the + // execution having declined to stop: the only thing this subscription can + // say is that a cancel DID arrive. + d.cancelSub, err = messaging.SubscribeJSON(d.bus, messaging.SubjectJobCancelWildcard, func(evt CancelEvent) { if d.cancelRegistry.Cancel(evt.JobID) { - xlog.Info("Cancelled job via NATS", "jobID", evt.JobID) + xlog.Info("Cancelled a job on this replica after a broadcast cancel", "jobID", evt.JobID) } }) if err != nil { @@ -144,15 +173,30 @@ func (d *Dispatcher) Start(ctx context.Context) error { // Subscribe to job result events from workers (persist to DB) if d.store != nil { - d.resultSub, err = messaging.SubscribeJSON(d.nats, messaging.SubjectJobResultWildcard, func(evt JobResultEvent) { - d.store.UpdateJobStatus(evt.JobID, evt.Status, evt.Result, evt.Error) + // The fan-out COPY of a terminal result, and never the only one. + // + // This carrier drops a broadcast rather than blocking when one + // subscriber falls behind, and a result has no successor message, so a + // subscriber that missed one would never hear about that job again. + // That is survivable here only because it is not the path the answer + // travels: the replica that claimed the work persists the terminal line + // through DispatchLoop.settleClaim BEFORE it releases the claim, so the + // job row already carries the answer when this broadcast is published. + // A dropped result therefore costs a live SSE stream its promptness, + // which jobs/sse.go recovers from by reading the row, and never costs + // the job its answer. Nothing may be moved onto this subscription that + // is not also written to a table first. + d.resultSub, err = messaging.SubscribeJSON(d.bus, messaging.SubjectJobResultWildcard, func(evt JobResultEvent) { + if err := d.store.UpdateJobStatus(evt.JobID, evt.Status, evt.Result, evt.Error); err != nil { + xlog.Error("Failed to persist a broadcast job result", "job_id", evt.JobID, "error", err) + } }) if err != nil { return fmt.Errorf("subscribing to result events: %w", err) } // Subscribe to trace events from workers (persist to DB) - d.progressSub, err = messaging.SubscribeJSON(d.nats, messaging.SubjectJobProgressWildcard, func(evt ProgressEvent) { + d.progressSub, err = messaging.SubscribeJSON(d.bus, messaging.SubjectJobProgressWildcard, func(evt ProgressEvent) { if evt.TraceType != "" && evt.TraceContent != "" { if err := d.store.AppendJobTrace(evt.JobID, evt.TraceType, evt.TraceContent); err != nil { xlog.Error("Failed to append job trace", "job_id", evt.JobID, "trace_type", evt.TraceType, "error", err) @@ -172,8 +216,8 @@ func (d *Dispatcher) Start(ctx context.Context) error { return nil } -// unsubscribeAll nil-checks, unsubscribes, and nils out each NATS subscription. -// Safe to call multiple times. +// unsubscribeAll nil-checks, unsubscribes, and nils out each broadcast +// subscription this dispatcher owns. Safe to call multiple times. func (d *Dispatcher) unsubscribeAll() { if d.cancelSub != nil { d.cancelSub.Unsubscribe() @@ -247,31 +291,44 @@ func (d *Dispatcher) Enqueue(jobID, taskID, userID string) error { return nil } -// Cancel publishes a cancel event to NATS (broadcast to all instances). +// Cancel broadcasts a cancel request to every replica. +// +// Its error says whether the request was PUBLISHED and nothing else. There is +// no reply, and there is deliberately no attempt to synthesise one: a cancel +// that reached no subscriber and a cancel an execution declined are different +// facts, and this carrier cannot tell them apart, so neither may be reported as +// the other. func (d *Dispatcher) Cancel(jobID string) error { - return d.nats.Publish(messaging.SubjectJobCancel(jobID), CancelEvent{ + return d.bus.Publish(messaging.SubjectJobCancel(jobID), CancelEvent{ JobID: jobID, }) } -// PublishProgress publishes a progress event for SSE bridging. +// PublishProgress broadcasts a progress event for SSE bridging. func (d *Dispatcher) PublishProgress(jobID, status, message string) error { - return d.nats.Publish(messaging.SubjectJobProgress(jobID), ProgressEvent{ + return d.bus.Publish(messaging.SubjectJobProgress(jobID), ProgressEvent{ JobID: jobID, Status: status, Message: message, }) } -// SubscribeProgress subscribes to progress events for a specific job (for SSE bridging). +// SubscribeProgress subscribes to progress events for ONE job, for SSE +// bridging. +// +// The subject is the exact one SubjectJobProgress builds and never the +// wildcard. On the wildcard this would be a stream showing every job in the +// deployment to every client watching any of them, which is a data-boundary +// rather than a display bug, and the subscription is opened and closed per HTTP +// request so the caller MUST close it. func (d *Dispatcher) SubscribeProgress(jobID string, handler func(ProgressEvent)) (messaging.Subscription, error) { - return messaging.SubscribeJSON(d.nats, messaging.SubjectJobProgress(jobID), handler) + return messaging.SubscribeJSON(d.bus, messaging.SubjectJobProgress(jobID), handler) } -// PublishTrace publishes a trace event for a running job via NATS. -// The frontend subscribes and persists traces to DB. +// PublishTrace broadcasts a trace event for a running job. The frontend +// subscribes and persists traces to the database. func (d *Dispatcher) PublishTrace(jobID, traceType, traceContent string) error { - return d.nats.Publish(messaging.SubjectJobProgress(jobID), ProgressEvent{ + return d.bus.Publish(messaging.SubjectJobProgress(jobID), ProgressEvent{ JobID: jobID, TraceType: traceType, TraceContent: traceContent, diff --git a/core/services/jobs/dispatcher_pg_test.go b/core/services/jobs/dispatcher_pg_test.go new file mode 100644 index 000000000000..77e65d0e8f96 --- /dev/null +++ b/core/services/jobs/dispatcher_pg_test.go @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: MIT + +package jobs + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "sync" + "time" + + "github.com/labstack/echo/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// The dispatcher against the carrier it runs on in production, across TWO +// connections. +// +// One carrier talking to itself is not the cross-replica case, and a double is +// less than that: it cannot drop, cannot reconnect and cannot fail the way a +// LISTEN connection fails. Everything below publishes on bus B and asserts on a +// dispatcher built on bus A. +var _ = Describe("the job dispatcher on the broadcast carrier", func() { + var ( + ctx context.Context + db *gorm.DB + busA, busB *pgbus.Bus + store *JobStore + disp *Dispatcher + ) + + BeforeEach(func() { + ctx = context.Background() + var dsn string + db, dsn = testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(ctx, db)).To(Succeed()) + + newBus := func() *pgbus.Bus { + b, err := pgbus.New(ctx, pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(b.Close) + return b + } + busA, busB = newBus(), newBus() + + var err error + store, err = NewJobStore(db) + Expect(err).ToNot(HaveOccurred()) + disp = NewDispatcher(store, busA, db, "replica-a") + }) + + Describe("SubscribeProgress", func() { + // The exactness of the per-request subscription, which is a data + // boundary and not a display detail: on the wildcard, every SSE client + // watching any job would be shown every OTHER job's progress, including + // jobs belonging to other users. + It("receives the job it asked for and not another job's progress", func() { + mine := make(chan ProgressEvent, 8) + sub, err := disp.SubscribeProgress("job-mine", func(evt ProgressEvent) { mine <- evt }) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = sub.Unsubscribe() }) + + // The other job first, then mine. If the filter is a wildcard in + // disguise, the first thing received is the other job's event, and + // no clock is needed to say so. + Expect(busB.Publish(messaging.SubjectJobProgress("job-theirs"), ProgressEvent{ + JobID: "job-theirs", Status: "running", Message: "not for me", + })).To(Succeed()) + Expect(busB.Publish(messaging.SubjectJobProgress("job-mine"), ProgressEvent{ + JobID: "job-mine", Status: "running", Message: "for me", + })).To(Succeed()) + + var first ProgressEvent + Eventually(mine, "20s").Should(Receive(&first)) + Expect(first.JobID).To(Equal("job-mine")) + Expect(first.Message).To(Equal("for me")) + Consistently(mine, "500ms", "50ms").ShouldNot(Receive()) + }) + + It("delivers what a PEER replica published, not only its own publishes", func() { + out := make(chan ProgressEvent, 8) + sub, err := disp.SubscribeProgress("job-peer", func(evt ProgressEvent) { out <- evt }) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = sub.Unsubscribe() }) + + peer := NewDispatcher(store, busB, db, "replica-b") + Expect(peer.PublishProgress("job-peer", "running", "from the other replica")).To(Succeed()) + + Eventually(out, "20s").Should(Receive(HaveField("Message", "from the other replica"))) + }) + }) + + Describe("the wildcard subscriptions Start opens", func() { + It("persists a trace a peer replica published", func() { + Expect(disp.Start(ctx)).To(Succeed()) + DeferCleanup(disp.Stop) + + job := &JobRecord{TaskID: "t", UserID: "u", Status: "running", TriggeredBy: "manual"} + Expect(store.CreateJob(job)).To(Succeed()) + + peer := NewDispatcher(store, busB, db, "replica-b") + Expect(peer.PublishTrace(job.ID, "reasoning", "thinking about it")).To(Succeed()) + + Eventually(func() int { + stored, err := store.GetJob(job.ID) + if err != nil || stored.TracesJSON == "" { + return 0 + } + return len(stored.TracesJSON) + }, "20s").Should(BeNumerically(">", 0)) + }) + + It("persists a terminal result a peer replica published", func() { + Expect(disp.Start(ctx)).To(Succeed()) + DeferCleanup(disp.Stop) + + job := &JobRecord{TaskID: "t", UserID: "u", Status: "running", TriggeredBy: "manual"} + Expect(store.CreateJob(job)).To(Succeed()) + + PublishJobResult(busB, job.ID, "completed", "the answer", "") + + Eventually(func() string { + stored, err := store.GetJob(job.ID) + if err != nil { + return "" + } + return stored.Status + }, "20s").Should(Equal("completed")) + }) + + It("leaves the register where it found it once the dispatcher stops", func() { + // Three process-lifetime subscriptions, and no more than three: a + // dispatcher that leaked one per Start would accumulate silently + // across the restarts a reconfiguration does. + before := busA.Subscribers() + Expect(disp.Start(ctx)).To(Succeed()) + Expect(busA.Subscribers()).To(Equal(before + 3)) + + disp.Stop() + Expect(busA.Subscribers()).To(Equal(before)) + }) + }) + + Describe("the SSE stream when the terminal broadcast never arrives", func() { + // The 256-deep per-subscriber queue DROPS rather than blocking, and a + // terminal event has no successor: a stream waiting only on the carrier + // would hang for ever and the user would read a job that finished as a + // job that produced nothing. Nothing terminal is published here at all, + // which is the strongest form of that loss. + It("ends the stream from the job row rather than waiting for an event that was dropped", func() { + disp.SetTerminalRecheck(50 * time.Millisecond) + + job := &JobRecord{TaskID: "t", UserID: "u", Status: "running", TriggeredBy: "manual"} + Expect(store.CreateJob(job)).To(Succeed()) + + before := busA.Subscribers() + body, handlerDone, _ := startStream(ctx, disp, job.ID) + + // This carrier has no replay, so a progress event published before + // the handler's channel is listened is simply gone. Subscribers() + // counts only live registrations, which is what makes it usable as + // the gate here rather than merely as a leak counter. + Eventually(busA.Subscribers, "20s").Should(Equal(before + 1)) + + // The stream is only proved to be PAST its post-subscribe read of + // the row once it has forwarded a live event. Without this the row + // could go terminal while the handler was still in that one-shot + // read, and the spec would pass with the periodic re-read deleted: + // it did, when this was written the other way round. + Expect(NewDispatcher(store, busB, db, "replica-b"). + PublishProgress(job.ID, "running", "step 1")).To(Succeed()) + Eventually(body, "20s").Should(ContainSubstring("step 1")) + + // The terminal state reaches the TABLE and no broadcast is ever + // published for it. This is exactly what a dropped result looks + // like to a subscriber: nothing. + Expect(store.UpdateJobStatus(job.ID, "completed", "the answer", "")).To(Succeed()) + + Eventually(handlerDone, "20s").Should(BeClosed(), + "the stream must end on the row when its terminal broadcast never arrives") + Expect(body()).To(ContainSubstring("event: done")) + Expect(body()).To(ContainSubstring("completed")) + }) + + It("closes its per-request subscription however it returns", func() { + // Two subscriptions are opened and closed per HTTP REQUEST in this + // deployment, and a leaked filter has no symptom at all: the + // replica just runs one more closure per notification for every + // stream it has ever served. + disp.SetTerminalRecheck(50 * time.Millisecond) + + job := &JobRecord{TaskID: "t", UserID: "u", Status: "running", TriggeredBy: "manual"} + Expect(store.CreateJob(job)).To(Succeed()) + + before := busA.Subscribers() + _, handlerDone, cancelReq := startStream(ctx, disp, job.ID) + + Eventually(busA.Subscribers, "20s").Should(Equal(before + 1)) + + cancelReq() + Eventually(handlerDone, "20s").Should(BeClosed()) + Eventually(busA.Subscribers, "20s").Should(Equal(before), + "a stream that has returned must leave no handler behind") + }) + }) +}) + +// syncBody is an http.ResponseWriter a spec may read WHILE the handler is still +// writing. httptest.ResponseRecorder's buffer is not safe for that, and reading +// it mid-stream is the only way to know the handler has reached its wait rather +// than assuming it has. +type syncBody struct { + mu sync.Mutex + buf bytes.Buffer + header http.Header +} + +func (w *syncBody) Header() http.Header { + if w.header == nil { + w.header = http.Header{} + } + return w.header +} + +func (w *syncBody) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.buf.Write(p) +} + +func (w *syncBody) WriteHeader(int) {} + +func (w *syncBody) Flush() {} + +func (w *syncBody) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.buf.String() +} + +// startStream runs the job-progress SSE handler for jobID on its own goroutine +// and returns a reader for what it has written so far, a channel closed when it +// returns, and the cancel that stands in for the client hanging up. +func startStream(ctx context.Context, d *Dispatcher, jobID string) (func() string, chan struct{}, context.CancelFunc) { + GinkgoHelper() + out := &syncBody{} + req := httptest.NewRequest(http.MethodGet, "/api/agent/jobs/"+jobID+"/progress", nil) + reqCtx, cancelReq := context.WithCancel(ctx) + c := echo.New().NewContext(req.WithContext(reqCtx), out) + c.SetParamNames("id") + c.SetParamValues(jobID) + + done := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(done) + Expect(d.SSEHandler()(c)).To(Succeed()) + }() + DeferCleanup(func() { + cancelReq() + Eventually(done, "20s").Should(BeClosed()) + }) + return out.String, done, cancelReq +} diff --git a/core/services/jobs/dispatcher_test.go b/core/services/jobs/dispatcher_test.go index 9628598c25ba..a68639a09ad9 100644 --- a/core/services/jobs/dispatcher_test.go +++ b/core/services/jobs/dispatcher_test.go @@ -2,6 +2,7 @@ package jobs import ( "encoding/json" + "sync" "time" . "github.com/onsi/ginkgo/v2" @@ -19,38 +20,35 @@ type publishCall struct { data any } -// fakeMessagingClient implements messaging.MessagingClient and records published messages. -type fakeMessagingClient struct { +// recordingBus is a messaging.Broadcaster that records what was published. +// +// A Broadcaster and NOT a MessagingClient, which is the point of the type +// rather than tidiness: the dispatcher may only fan out, and a double that +// still offered Request or QueueSubscribe would let a spec exercise a surface +// the production type can no longer reach. +type recordingBus struct { + mu sync.Mutex calls []publishCall } -func (f *fakeMessagingClient) Publish(subject string, data any) error { +func (f *recordingBus) Publish(subject string, data any) error { + f.mu.Lock() + defer f.mu.Unlock() f.calls = append(f.calls, publishCall{subject: subject, data: data}) return nil } -func (f *fakeMessagingClient) Subscribe(string, func([]byte)) (messaging.Subscription, error) { +func (f *recordingBus) Subscribe(string, func([]byte)) (messaging.Subscription, error) { return &fakeSub{}, nil } -func (f *fakeMessagingClient) QueueSubscribe(string, string, func([]byte)) (messaging.Subscription, error) { - return &fakeSub{}, nil -} - -func (f *fakeMessagingClient) QueueSubscribeReply(string, string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return &fakeSub{}, nil -} - -func (f *fakeMessagingClient) SubscribeReply(string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return &fakeSub{}, nil -} - -func (f *fakeMessagingClient) Request(string, []byte, time.Duration) ([]byte, error) { - return nil, nil +func (f *recordingBus) published() []publishCall { + f.mu.Lock() + defer f.mu.Unlock() + return append([]publishCall(nil), f.calls...) } -func (f *fakeMessagingClient) IsConnected() bool { return true } -func (f *fakeMessagingClient) Close() {} +var _ messaging.Broadcaster = (*recordingBus)(nil) // fakeSub implements messaging.Subscription. type fakeSub struct{} @@ -221,7 +219,7 @@ var _ = Describe("Dispatcher", func() { Describe("Enqueue claim kind", func() { var ( store *JobStore - fake *fakeMessagingClient + fake *recordingBus disp *Dispatcher db *gorm.DB ) @@ -231,7 +229,7 @@ var _ = Describe("Dispatcher", func() { var err error store, err = NewJobStore(db) Expect(err).ToNot(HaveOccurred()) - fake = &fakeMessagingClient{} + fake = &recordingBus{} disp = NewDispatcher(store, fake, db, "test-instance") }) @@ -265,7 +263,7 @@ var _ = Describe("Dispatcher", func() { Expect(disp.Enqueue(job.ID, task.ID, "user-1")).To(Succeed()) Expect(onlyClaim(db).Kind).To(Equal(string(ClaimKindMCPCI))) - Expect(fake.calls).To(BeEmpty(), "enqueueing must not publish: a queue subject nobody joined swallows the job") + Expect(fake.published()).To(BeEmpty(), "enqueueing must not publish: a queue subject nobody joined swallows the job") }) It("writes a plain task claim for a model without MCP servers", func() { @@ -294,7 +292,7 @@ var _ = Describe("Dispatcher", func() { Expect(disp.Enqueue(job.ID, task.ID, "user-1")).To(Succeed()) Expect(onlyClaim(db).Kind).To(Equal(string(ClaimKindTask))) - Expect(fake.calls).To(BeEmpty(), "enqueueing must not publish: a queue subject nobody joined swallows the job") + Expect(fake.published()).To(BeEmpty(), "enqueueing must not publish: a queue subject nobody joined swallows the job") }) It("writes a plain task claim when the model config is not found", func() { @@ -321,7 +319,7 @@ var _ = Describe("Dispatcher", func() { Expect(disp.Enqueue(job.ID, task.ID, "user-1")).To(Succeed()) Expect(onlyClaim(db).Kind).To(Equal(string(ClaimKindTask))) - Expect(fake.calls).To(BeEmpty(), "enqueueing must not publish: a queue subject nobody joined swallows the job") + Expect(fake.published()).To(BeEmpty(), "enqueueing must not publish: a queue subject nobody joined swallows the job") }) }) @@ -331,7 +329,7 @@ var _ = Describe("Dispatcher", func() { Describe("Enqueue event enrichment", func() { var ( store *JobStore - fake *fakeMessagingClient + fake *recordingBus disp *Dispatcher db *gorm.DB ) @@ -341,7 +339,7 @@ var _ = Describe("Dispatcher", func() { var err error store, err = NewJobStore(db) Expect(err).ToNot(HaveOccurred()) - fake = &fakeMessagingClient{} + fake = &recordingBus{} disp = NewDispatcher(store, fake, db, "test-instance") }) diff --git a/core/services/jobs/publish.go b/core/services/jobs/publish.go index d861433437f1..905246681bcf 100644 --- a/core/services/jobs/publish.go +++ b/core/services/jobs/publish.go @@ -5,7 +5,15 @@ import ( "github.com/mudler/xlog" ) -// PublishJobResult publishes a terminal job result event and a progress event via NATS. +// PublishJobResult broadcasts a terminal job result and the matching progress +// event. +// +// Both are FAN-OUT copies. The answer itself reaches the frontend on the reply +// line of the control RPC the claiming replica is reading, and is persisted +// before that claim is released, so neither of these publishes failing can lose +// a job its result. Errors are logged rather than returned for exactly that +// reason: a caller that treated a publish failure as the work having failed +// would report a job that ran as a job that did not. func PublishJobResult(pub messaging.Publisher, jobID, status, result, errMsg string) { if err := pub.Publish(messaging.SubjectJobResult(jobID), JobResultEvent{ JobID: jobID, @@ -24,7 +32,7 @@ func PublishJobResult(pub messaging.Publisher, jobID, status, result, errMsg str } } -// PublishJobProgress publishes a status-only update (no result) via NATS. +// PublishJobProgress broadcasts a status-only update, with no result attached. func PublishJobProgress(pub messaging.Publisher, jobID, status, message string) { if err := pub.Publish(messaging.SubjectJobProgress(jobID), ProgressEvent{ JobID: jobID, diff --git a/core/services/jobs/sse.go b/core/services/jobs/sse.go index f66d5ea331e0..d540b2893fbd 100644 --- a/core/services/jobs/sse.go +++ b/core/services/jobs/sse.go @@ -6,12 +6,26 @@ import ( "net/http" "sync" "sync/atomic" + "time" "github.com/labstack/echo/v4" + "github.com/mudler/xlog" ) -// SSEBridge provides an HTTP handler that bridges NATS progress events to SSE. -// This follows the notetaker pattern: subscribe to NATS, forward to SSE client. +// DefaultTerminalRecheck is how often an open job-progress SSE stream re-reads +// the job row while it waits for the broadcast that would close it. +// +// It exists because the broadcast carrier is at-most-once and drops rather than +// blocks: a terminal progress event can be lost, and a lost terminal event has +// no successor, so a stream waiting only on the bus would hang until the client +// gave up and the user would read a finished job as one that produced nothing. +// The row is where the answer is written before it is ever broadcast, so the +// stream ends on the TABLE and merely ends FASTER on the broadcast. +const DefaultTerminalRecheck = 15 * time.Second + +// SSEBridge provides an HTTP handler that bridges job progress broadcasts to +// SSE: subscribe to the carrier, forward to the SSE client, and close the +// stream when the job is done. func (d *Dispatcher) SSEHandler() echo.HandlerFunc { return func(c echo.Context) error { jobID := c.Param("id") @@ -49,18 +63,11 @@ func (d *Dispatcher) SSEHandler() echo.HandlerFunc { } // Send current job state first - job, err := d.store.GetJob(jobID) - if err == nil { - sendEvent("status", ProgressEvent{ - JobID: jobID, - Status: job.Status, - }) - // If already terminal, send done and close — no need to subscribe - if job.Status == "completed" || job.Status == "failed" || job.Status == "cancelled" { - sendEvent("done", ProgressEvent{ - JobID: jobID, - Status: job.Status, - }) + if status, terminal, ok := d.jobStatus(jobID); ok { + sendEvent("status", ProgressEvent{JobID: jobID, Status: status}) + // Already finished, so there is nothing to subscribe to. + if terminal { + sendEvent("done", ProgressEvent{JobID: jobID, Status: status}) return nil } } @@ -69,15 +76,18 @@ func (d *Dispatcher) SSEHandler() echo.HandlerFunc { // handler can return promptly instead of waiting for client disconnect. done := make(chan struct{}) closeOnce := sync.Once{} + finish := func(evt ProgressEvent) { + sendEvent("done", evt) + closeOnce.Do(func() { close(done) }) + } // Subscribe to progress events for this job sub, err := d.SubscribeProgress(jobID, func(evt ProgressEvent) { sendEvent("progress", evt) // Close the stream on terminal states - if evt.Status == "completed" || evt.Status == "failed" || evt.Status == "cancelled" { - sendEvent("done", evt) - closeOnce.Do(func() { close(done) }) + if IsTerminalJobStatus(evt.Status) { + finish(evt) } }) if err != nil { @@ -85,13 +95,74 @@ func (d *Dispatcher) SSEHandler() echo.HandlerFunc { sendEvent("error", map[string]string{"error": "failed to subscribe"}) return nil } - // Wait for client disconnect or terminal state - select { - case <-c.Request().Context().Done(): - case <-done: + // Deferred, not called on the way out. Every return below this line + // leaks a handler otherwise, and the leak has no symptom: on the + // PostgreSQL carrier only the first subscriber of a channel issues a + // LISTEN and the rest are in-process filters, so a replica that has + // served ten thousand of these streams simply runs ten thousand + // closures per notification and reports nothing. + defer func() { + closed.Store(true) + if uerr := sub.Unsubscribe(); uerr != nil { + xlog.Warn("Failed to close a job progress subscription", "job_id", jobID, "error", uerr) + } + }() + + // The row, once, immediately after subscribing. A job that finished + // between the read above and this subscription published its terminal + // event into the window where nobody was listening, and this carrier + // has no replay, so without this read the stream waits for an event + // that has already been and gone. + if status, terminal, ok := d.jobStatus(jobID); ok && terminal { + finish(ProgressEvent{JobID: jobID, Status: status}) + return nil + } + + // And the row again, periodically, for the rest of the stream's life. + // The carrier drops a broadcast rather than blocking when a subscriber + // falls behind, and a dropped TERMINAL event is a stream that never + // ends. Reading the row is what keeps a lost broadcast from reading as + // a job that produced no result. + ticker := time.NewTicker(d.terminalRecheckInterval()) + defer ticker.Stop() + + for { + select { + case <-c.Request().Context().Done(): + return nil + case <-done: + return nil + case <-ticker.C: + if status, terminal, ok := d.jobStatus(jobID); ok && terminal { + finish(ProgressEvent{JobID: jobID, Status: status}) + return nil + } + } } - closed.Store(true) - sub.Unsubscribe() - return nil } } + +// jobStatus reads a job's status from the TABLE and reports whether it is +// terminal. The third return separates "the row says pending" from "there is no +// row to read", which the caller must not collapse: a job whose row cannot be +// read has not been shown to have produced nothing. +func (d *Dispatcher) jobStatus(jobID string) (status string, terminal, ok bool) { + if d.store == nil { + return "", false, false + } + job, err := d.store.GetJob(jobID) + if err != nil { + return "", false, false + } + return job.Status, IsTerminalJobStatus(job.Status), true +} + +// terminalRecheckInterval is how often an open stream re-reads the row. +// Configurable so a spec can drive the recovery without waiting out the +// production interval, and never zero, which would spin. +func (d *Dispatcher) terminalRecheckInterval() time.Duration { + if d.terminalRecheck > 0 { + return d.terminalRecheck + } + return DefaultTerminalRecheck +} diff --git a/core/services/jobs/store.go b/core/services/jobs/store.go index 655847990c6b..0f3cb5b73fcf 100644 --- a/core/services/jobs/store.go +++ b/core/services/jobs/store.go @@ -229,6 +229,27 @@ func (s *JobStore) DeleteJob(id string) error { return s.db.Where("id = ?", id).Delete(&JobRecord{}).Error } +// TerminalJobStatuses are the statuses a job never leaves. +// +// ONE spelling for the whole package, because the rule was stated at four call +// sites and pinned at none of them: twice in the SSE bridge, which decides when +// to close a stream, and twice here, which decides when to stamp completed_at +// and which rows are still writable. A set that drifts between those two is a +// stream that closes on a status the store still considers open, or a row that +// accepts a second terminal write. Adding a status now means adding it here, +// and every reader follows. +var TerminalJobStatuses = []string{"completed", "failed", "cancelled"} + +// IsTerminalJobStatus reports whether a job in this status has finished. +func IsTerminalJobStatus(status string) bool { + for _, terminal := range TerminalJobStatuses { + if status == terminal { + return true + } + } + return false +} + // UpdateJobStatus updates just the status (and optionally result/error) of a job. func (s *JobStore) UpdateJobStatus(id, status, result, errMsg string) error { updates := map[string]any{ @@ -245,11 +266,11 @@ func (s *JobStore) UpdateJobStatus(id, status, result, errMsg string) error { if status == "running" { updates["started_at"] = &now } - if status == "completed" || status == "failed" || status == "cancelled" { + if IsTerminalJobStatus(status) { updates["completed_at"] = &now } return s.db.Model(&JobRecord{}). - Where("id = ? AND status NOT IN ?", id, []string{"completed", "failed", "cancelled"}). + Where("id = ? AND status NOT IN ?", id, TerminalJobStatuses). Updates(updates).Error } diff --git a/core/services/jobs/store_test.go b/core/services/jobs/store_test.go index 19f0971c2c17..6fff27790940 100644 --- a/core/services/jobs/store_test.go +++ b/core/services/jobs/store_test.go @@ -402,3 +402,50 @@ var _ = Describe("JobStore", func() { }) }) }) + +// The terminal-status set, asserted as ONE fact rather than as two agreeing +// lists. +// +// It was written out at four call sites and pinned at none: twice in the SSE +// bridge, which decides when to close a stream, and twice in the store, which +// decides when to stamp completed_at and which rows are still writable. Drift +// between those two is a stream that closes on a status the store still +// considers open, or a row that accepts a second terminal write. These drive +// both readers from the same exported set, so a status added to one of them +// cannot be missing from the other. +var _ = Describe("the statuses a job never leaves", func() { + var store *JobStore + + BeforeEach(func() { + var err error + store, err = NewJobStore(testutil.SetupTestDB()) + Expect(err).ToNot(HaveOccurred()) + }) + + It("names at least one status", func() { + // An empty set would make every assertion below vacuous. + Expect(TerminalJobStatuses).ToNot(BeEmpty()) + }) + + It("refuses a further write to a row in any of them", func() { + for _, status := range TerminalJobStatuses { + Expect(IsTerminalJobStatus(status)).To(BeTrue(), status) + + job := &JobRecord{TaskID: "t", UserID: "u", Status: "running", TriggeredBy: "manual"} + Expect(store.CreateJob(job)).To(Succeed()) + Expect(store.UpdateJobStatus(job.ID, status, "first", "")).To(Succeed()) + + Expect(store.UpdateJobStatus(job.ID, "running", "second", "")).To(Succeed()) + stored, err := store.GetJob(job.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(stored.Status).To(Equal(status), "a settled job must not be reopened") + Expect(stored.CompletedAt).ToNot(BeNil(), status) + } + }) + + It("does not claim a status a job can still leave", func() { + for _, status := range []string{"pending", "running", "", "queued"} { + Expect(IsTerminalJobStatus(status)).To(BeFalse(), status) + } + }) +}) diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index 7dfc07ba2ad4..224bae7bb98f 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -43,6 +43,16 @@ func SubjectAgentEvents(agentName, userID string) string { return subjectAgentEventsPrefix + sanitizeSubjectToken(agentName) + ".events." + sanitizeSubjectToken(userID) } +// SubjectAgentEventsWildcard matches every agent's SSE events for every user. +// +// It is a constant here rather than the string literal it used to be inside +// agents.StartObservablePersister, because that literal was the only +// hand-written subject filter left in the tree, and a filter that is not next +// to the builder it must match is a filter that outlives it. SubjectAgentEvents +// makes four tokens; this makes four, and SubjectMatches matches on token count +// first, so a three-token filter here would deliver nothing at all. +const SubjectAgentEventsWildcard = "agent.*.events.*" + // SubjectJobProgress returns the NATS subject for job progress updates. func SubjectJobProgress(jobID string) string { return subjectJobProgressPrefix + sanitizeSubjectToken(jobID) + ".progress" diff --git a/core/services/messaging/subjects_wire_test.go b/core/services/messaging/subjects_wire_test.go index 33a69aa17995..f6afa85ef5a6 100644 --- a/core/services/messaging/subjects_wire_test.go +++ b/core/services/messaging/subjects_wire_test.go @@ -2,6 +2,7 @@ package messaging import ( "encoding/json" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -114,3 +115,35 @@ var _ = Describe("per-tenant SyncedMap subjects", func() { Expect(ValidFilter(SubjectSyncStateTenantDelta("agent.tasks", ""))).To(MatchError(ErrUnsupportedFilter)) }) }) + +// The agent-events wildcard used to be a hand-written literal at the one place +// that subscribes to it, three files away from the builder it has to match. +// These pin the pairing itself, not the string: the filter is only useful if +// every subject SubjectAgentEvents can produce matches it, and if the near +// misses that a shorter filter would swallow do not. +var _ = Describe("the agent-events wildcard", func() { + It("matches what SubjectAgentEvents builds, for any agent and any user", func() { + for _, agent := range []string{"a", "my-agent", "agent.with.dots"} { + for _, user := range []string{"u", "", "user id"} { + subject := SubjectAgentEvents(agent, user) + Expect(SubjectMatches(SubjectAgentEventsWildcard, subject)).To(BeTrue(), + "filter %q must match %q", SubjectAgentEventsWildcard, subject) + } + } + }) + + It("has the same token count as the subjects it matches", func() { + // SubjectMatches compares token counts before anything else, so a + // filter one token short matches NOTHING and the persister goes silent + // with no error anywhere. Stated as a count so a builder that grows a + // token reddens here rather than in a suite that only checks delivery. + Expect(strings.Count(SubjectAgentEventsWildcard, ".")). + To(Equal(strings.Count(SubjectAgentEvents("a", "u"), "."))) + }) + + It("does not match a subject with the events token in another position", func() { + Expect(SubjectMatches(SubjectAgentEventsWildcard, "agent.a.events")).To(BeFalse()) + Expect(SubjectMatches(SubjectAgentEventsWildcard, "agent.a.b.events.u")).To(BeFalse()) + Expect(SubjectMatches(SubjectAgentEventsWildcard, "agent.a.cancel")).To(BeFalse()) + }) +}) diff --git a/core/services/nodes/rebroadcast.go b/core/services/nodes/rebroadcast.go index 10e7809c80e1..d3b9eff65ddc 100644 --- a/core/services/nodes/rebroadcast.go +++ b/core/services/nodes/rebroadcast.go @@ -18,9 +18,13 @@ import ( // subjects NOT added. var workerBroadcastAllow = map[string][]string{ NodeTypeAgent: { - "jobs.*.progress", - "jobs.*.result", - "agent.*.events.*", + // The constants and not the strings. A filter written out here is a + // filter that stops matching the day its builder grows a token, and + // SubjectMatches compares token counts first, so the drift presents as + // a worker being refused everything rather than as anything readable. + messaging.SubjectJobProgressWildcard, + messaging.SubjectJobResultWildcard, + messaging.SubjectAgentEventsWildcard, }, // A backend worker asks for no broadcasts. Spelled as an empty list rather // than omitted, so the reader sees the decision. diff --git a/core/services/nodes/rebroadcast_test.go b/core/services/nodes/rebroadcast_test.go index 6f01c8ac1c1b..6e5308a5b1db 100644 --- a/core/services/nodes/rebroadcast_test.go +++ b/core/services/nodes/rebroadcast_test.go @@ -1,6 +1,7 @@ package nodes import ( + "context" "encoding/json" "errors" @@ -8,6 +9,7 @@ import ( . "github.com/onsi/gomega" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/pgbus" "github.com/mudler/LocalAI/core/services/testutil" ) @@ -151,3 +153,76 @@ var _ = Describe("worker re-broadcast authorization", func() { }) }) }) + +// The re-broadcast path across TWO carriers, which is the shape production has +// and the shape a single in-memory double cannot have. +// +// This is the wiring failure that leaves every unit spec in every package green: +// point the Rebroadcaster at one carrier while the dispatcher subscribes on +// another and the rebroadcaster publishes, the publish succeeds, Handle returns +// true, and the only symptom anywhere is an SSE stream with no progress in it. +// So these assert the RECEIPT and never the return value. +var _ = Describe("re-broadcasting onto the carrier the subscriber reads", func() { + var ( + busA, busB *pgbus.Bus + rb *Rebroadcaster + ) + + BeforeEach(func() { + db, dsn := testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(context.Background(), db)).To(Succeed()) + newBus := func() *pgbus.Bus { + b, err := pgbus.New(context.Background(), pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(b.Close) + return b + } + busA, busB = newBus(), newBus() + rb = NewRebroadcaster(busA) + }) + + It("reaches a job-progress subscriber on the other replica's carrier", func() { + delivered := make(chan []byte, 4) + _, err := busB.Subscribe(messaging.SubjectJobProgressWildcard, func(payload []byte) { delivered <- payload }) + Expect(err).ToNot(HaveOccurred()) + + rb.Handle(NodeTypeAgent, messaging.SubjectJobProgress("j1"), json.RawMessage(`{"job_id":"j1","status":"running"}`)) + + Eventually(delivered, "20s").Should(Receive(MatchJSON(`{"job_id":"j1","status":"running"}`))) + }) + + It("reaches a job-result subscriber on the other replica's carrier", func() { + delivered := make(chan []byte, 4) + _, err := busB.Subscribe(messaging.SubjectJobResultWildcard, func(payload []byte) { delivered <- payload }) + Expect(err).ToNot(HaveOccurred()) + + rb.Handle(NodeTypeAgent, messaging.SubjectJobResult("j1"), json.RawMessage(`{"job_id":"j1","status":"completed"}`)) + + Eventually(delivered, "20s").Should(Receive(MatchJSON(`{"job_id":"j1","status":"completed"}`))) + }) + + It("reaches an agent-events subscriber on the other replica's carrier", func() { + delivered := make(chan []byte, 4) + _, err := busB.Subscribe(messaging.SubjectAgentEventsWildcard, func(payload []byte) { delivered <- payload }) + Expect(err).ToNot(HaveOccurred()) + + rb.Handle(NodeTypeAgent, messaging.SubjectAgentEvents("a1", "u1"), json.RawMessage(`{"event_type":"json_message"}`)) + + Eventually(delivered, "20s").Should(Receive(MatchJSON(`{"event_type":"json_message"}`))) + }) + + It("delivers nothing at all for a subject the worker is refused", func() { + // The negative half without a clock: a refused subject followed by an + // allowed one, and the allowed one arriving first is the proof. + delivered := make(chan []byte, 4) + _, err := busB.Subscribe(messaging.SubjectJobProgressWildcard, func(payload []byte) { delivered <- payload }) + Expect(err).ToNot(HaveOccurred()) + + rb.Handle(NodeTypeBackend, messaging.SubjectJobProgress("j1"), json.RawMessage(`{"job_id":"refused"}`)) + rb.Handle(NodeTypeAgent, messaging.SubjectJobProgress("j2"), json.RawMessage(`{"job_id":"allowed"}`)) + + var first []byte + Eventually(delivered, "20s").Should(Receive(&first)) + Expect(first).To(MatchJSON(`{"job_id":"allowed"}`)) + }) +}) diff --git a/core/services/pgbus/bus.go b/core/services/pgbus/bus.go index ab843449917e..a2c89faf00b6 100644 --- a/core/services/pgbus/bus.go +++ b/core/services/pgbus/bus.go @@ -406,38 +406,59 @@ func (b *Bus) Subscribe(subject string, handler func([]byte)) (messaging.Subscri done: make(chan struct{}), } first := len(b.subs[channel]) == 0 - if first { - b.subs[channel] = map[uint64]*subscription{} - } - b.subs[channel][sub.id] = sub b.mu.Unlock() - go sub.run(handler) - + // The LISTEN before the registration, not after it. + // + // Both orders satisfy "Subscribe has listened by the time it returns", so + // the CALLER cannot tell them apart. An observer can: with the registration + // first, Subscribers() counts a handler whose channel is not listened yet, + // so anything waiting on that count as a readiness signal proceeds into a + // window where a publish is silently lost, which is at-most-once delivery + // doing exactly what it says while looking like a broken subscription. + // Registering last makes the counter mean what its name says, and it + // removes the failed-LISTEN rollback: a registration never made needs no + // undoing. if first { b.barrier("issue", "LISTEN") if err := b.command("LISTEN " + pgx.Identifier{channel}.Sanitize()); err != nil { - // Not Unsubscribe: the ordering lock is already held here, and the - // connection was never listening on this channel, so there is - // nothing to UNLISTEN. - b.forget(sub) + close(sub.stop) return nil, err } } + + b.mu.Lock() + if b.subs[channel] == nil { + b.subs[channel] = map[uint64]*subscription{} + } + b.subs[channel][sub.id] = sub + b.mu.Unlock() + + go sub.run(handler) return sub, nil } -// forget removes a registration without touching the channel's LISTEN state. -func (b *Bus) forget(sub *subscription) { - sub.once.Do(func() { - b.mu.Lock() - delete(b.subs[sub.channel], sub.id) - if len(b.subs[sub.channel]) == 0 { - delete(b.subs, sub.channel) - } - b.mu.Unlock() - close(sub.stop) - }) +// Subscribers reports how many handlers are registered right now, across every +// channel. A counted handler is a LIVE one: its channel was listened before it +// was counted, so this number is usable as a readiness signal and not only as a +// figure to compare against itself. +// +// It exists to be asserted, because the leak it makes visible has no other +// symptom. Two subscriptions in this deployment are opened and closed PER HTTP +// REQUEST (a job's progress stream and an agent's event stream), and on this +// carrier only the FIRST subscriber of a channel issues a LISTEN: every later +// one just registers an in-process filter. An Unsubscribe that failed to remove +// its filter would therefore leave a replica that has served ten thousand SSE +// requests running ten thousand closures per notification, and nothing in the +// tree would fail. It would just get slower. +func (b *Bus) Subscribers() int { + b.mu.Lock() + defer b.mu.Unlock() + n := 0 + for _, channel := range b.subs { + n += len(channel) + } + return n } // barrier is a test seam and does nothing in production. diff --git a/core/services/pgbus/bus_test.go b/core/services/pgbus/bus_test.go index 899536c3b064..5a83769bb6d9 100644 --- a/core/services/pgbus/bus_test.go +++ b/core/services/pgbus/bus_test.go @@ -191,6 +191,81 @@ var _ = Describe("the PostgreSQL broadcast carrier", func() { }) }) + Describe("subscriptions that come and go", func() { + // The accumulation assertion, and it is deliberately NOT a delivery + // one. Two subscriptions in this deployment are opened and closed per + // HTTP REQUEST, so a filter that outlives its Unsubscribe is a replica + // that gets steadily slower and never fails. Delivery specs cannot see + // it: the leaked handlers deliver correctly, there are just more of + // them every request. + It("registers nothing that outlives its Unsubscribe", func() { + delivered := make(chan []byte, 8) + survivor, err := sub.Subscribe("jobs.churn.progress", func(data []byte) { delivered <- data }) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = survivor.Unsubscribe() }) + + start := sub.Subscribers() + Expect(start).To(Equal(1)) + + for i := 0; i < 1000; i++ { + s, err := sub.Subscribe("jobs.churn.progress", func([]byte) {}) + Expect(err).ToNot(HaveOccurred()) + Expect(s.Unsubscribe()).To(Succeed()) + } + + Expect(sub.Subscribers()).To(Equal(start), + "a subscription that is opened and closed must leave the register where it found it") + + // And the survivor is still exactly one subscriber, not zero and + // not a thousand. A counter rather than a boolean, because + // "something arrived" is true for every wrong answer here. + Expect(pub.Publish("jobs.churn.progress", map[string]string{"m": "after"})).To(Succeed()) + Eventually(delivered).Should(Receive(MatchJSON(`{"m":"after"}`))) + Consistently(delivered, "500ms", "50ms").ShouldNot(Receive(), + "a leaked handler would deliver the same notification again") + }) + + It("keeps the channel listened while a subscriber remains", func() { + // The churn above cycles LISTEN and UNLISTEN a thousand times on a + // channel that still has a live subscriber. If the refcount and the + // UNLISTEN it implies were not decided as one step, the channel + // ends up unlistened with a live subscription on it, and it does + // not self-heal: the next Subscribe sees first == false and never + // re-LISTENs. + out, _ := received(sub, "jobs.abc.progress") + for i := 0; i < 50; i++ { + s, err := sub.Subscribe("jobs.other.progress", func([]byte) {}) + Expect(err).ToNot(HaveOccurred()) + Expect(s.Unsubscribe()).To(Succeed()) + } + + Expect(pub.Publish("jobs.abc.progress", map[string]string{"m": "still here"})).To(Succeed()) + Eventually(out).Should(Receive(MatchJSON(`{"m":"still here"}`))) + }) + }) + + Describe("what Subscribe has finished doing when it returns", func() { + // The per-request subscriptions are why this matters. An SSE client + // connects and then triggers the work; if Subscribe returns before the + // LISTEN is on the wire, the first events go nowhere and the stream + // looks hung with no error on either side. + // + // There is deliberately no Eventually around the PUBLISH. Retrying the + // publish is exactly the thing that would hide an asynchronous LISTEN. + It("has already issued the LISTEN, every time", func() { + for i := 0; i < 50; i++ { + out := make(chan []byte, 1) + s, err := sub.Subscribe("jobs.race.progress", func(data []byte) { out <- data }) + Expect(err).ToNot(HaveOccurred()) + + Expect(pub.Publish("jobs.race.progress", map[string]int{"i": i})).To(Succeed()) + + Eventually(out, "10s").Should(Receive(), "iteration %d subscribed and then missed its own first message", i) + Expect(s.Unsubscribe()).To(Succeed()) + } + }) + }) + Describe("a handler that does not return", func() { It("does not stop delivery to the other subscribers", func() { // A carrier whose handlers run on one goroutine is a carrier one diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 8039857f11b4..d51cc24a5d8f 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -136,12 +136,31 @@ Several features keep state in a frontend's process memory and surface it over t | Agent tasks | `state.agent-tasks.delta` and `state.agent-tasks..delta` | | Open Responses metadata | `state.responses-metadata.delta` | +### Job and agent streams across replicas + +The same carrier moves the traffic whose subscriber is an open HTTP response rather than a cache: a job's progress stream, its result, its cancel, an agent's SSE events, an agent cancel and an Open Responses cancel. + +| Family | Subject | Read by | +|--------|---------|---------| +| Job progress | `jobs..progress` | `GET /api/agent/jobs/{id}/progress` on any replica, and the trace persister on every replica | +| Job result | `jobs..result` | The result persister on every replica | +| Job cancel | `jobs..cancel` | Every replica, so the one holding the run can stop it | +| Agent events | `agent..events.` | `GET /api/agents/{name}/sse/distributed` on any replica, and the observable persister | +| Agent cancel | `agent..cancel` | Every agent worker, and so still on NATS: see below | +| Open Responses cancel | `responses..cancel` | The replica holding the generation | + +One family on that list is not on this carrier. An `agent..cancel` has to reach the agent WORKER running the execution, and an agent worker has no database, so it cannot listen on PostgreSQL at all; that cancel is published on NATS, where the worker is listening, and it stays there until a cancel rides the worker's tunnel like every other verb the frontend addresses to a worker. Everything else in the table is on the PostgreSQL carrier. + +This is what lets a user watch a job or an agent on one frontend while the work runs against another. **No broadcast on this list is the only path to anything durable.** A job's terminal state is written to its row by the replica that claimed the work, before that claim is released, so a dropped result costs an open stream its promptness and never costs the job its answer: a stream that is still open re-reads the row and closes on it. A cancel is a request and not a verdict: if it reaches nobody it has not been refused, and nothing in the API reports it as such. + Two carrier details are visible to an operator. **The 8000-byte notification cap.** PostgreSQL refuses a `pg_notify` payload of 8000 bytes or more, and that limit is measured against the whole encoded notification, not just the value being replicated. A broadcast that does not fit is written to the `bus_messages` table and the notification carries the row id instead; the receiving replica reads the row and delivers the original bytes. This is an ordinary path and not an error: a fine-tune job carrying a long training message spills every time. Rows are retired ten minutes after they are written, by every replica, so `bus_messages` is a spill buffer and never a log of past events. **A broadcast is at most once, and is never replayed.** A `NOTIFY` reaches the sessions that are listening when it is issued and nobody else. A replica whose session was down in that window never receives the change, and no error is reported anywhere. That is why every one of these maps is backed by a durable table: the broadcast says only that something changed, and the table says what it changed to. A replica re-reads its table when its listener reconnects, so a missed delta is a delay and never a value that reads as though it had never been set. +**A slow subscriber loses broadcasts rather than stalling the carrier.** Each subscription buffers 256 messages; past that, its broadcasts are dropped and logged at error level on the replica that took them. One blocked SSE writer must not be able to stop delivery for the whole deployment, which is what the alternative would mean. The same rule follows from it: what must survive a gap lives in a table. + ### Open Responses across replicas A response created by `POST /v1/responses` is held by the replica that served the request. A round-robin load balancer sends the follow-up poll, the `previous_response_id` chain and the cancel to any replica, so that metadata is replicated to every frontend and is also written to a `response_metadata` table in PostgreSQL. @@ -212,7 +231,7 @@ Both **backend** and **agent** nodes are issued one. Earlier releases minted a c An agent worker's tunnel carries only the `http` tag: it runs no backend processes, so it does not offer the `grpc` tag at all. Its control server binds `127.0.0.1` on a port chosen by the kernel and advertises it nowhere, so an agent worker still opens no inbound port. -**An agent worker still requires `--nats-url`, and nothing the frontend sends it travels on the bus.** Every verb the frontend addresses to a specific agent worker is now a control RPC on the tunnel that worker holds: MCP tool execution, MCP discovery, the backend stop that flushes cached MCP sessions, agent execution, and MCP CI runs. What the bus still carries for an agent worker is the other direction and the broadcasts: cancellation, and the progress and result lines a worker asks the frontend to re-publish on its behalf. +**An agent worker still requires `--nats-url`, and one thing only still reaches it on the bus.** Every verb the frontend addresses to a specific agent worker is now a control RPC on the tunnel that worker holds: MCP tool execution, MCP discovery, the backend stop that flushes cached MCP sessions, agent execution, and MCP CI runs. The progress and result lines the worker asks the frontend to re-publish on its behalf travel back on that same response body, and the frontend re-publishes them on the PostgreSQL carrier, not on NATS. One subject in the other direction is still NATS and is the reason `--nats-url` is still required: `agent..cancel`, which the worker subscribes to so a cancel can reach the execution it is running. There is no `nodes..*` subject left, and an agent worker's minted JWT no longer grants `mcp.tools.execute`, `mcp.discovery`, `nodes..backend.stop`, `agent.execute` or `jobs.mcp-ci.new`. The `--agent-subject` and `--agent-queue` flags (`LOCALAI_AGENT_SUBJECT`, `LOCALAI_AGENT_QUEUE`) are gone: there is no subject for an agent worker to subscribe to and no queue group to be one of. diff --git a/tests/e2e/distributed/agent_distributed_test.go b/tests/e2e/distributed/agent_distributed_test.go index 9fd1d53c062c..dfb90ea8d927 100644 --- a/tests/e2e/distributed/agent_distributed_test.go +++ b/tests/e2e/distributed/agent_distributed_test.go @@ -106,36 +106,49 @@ var _ = Describe("Phase 3: Agent Conversations & SSE", Label("Distributed"), fun // Conversation history is managed client-side (browser localStorage). // No server-side conversation storage tests needed. - Context("Agent SSE Events via NATS", func() { - It("should bridge agent SSE events via NATS", func() { - bridge := agents.NewEventBridge(infra.NC, store, "instance-1") - - var received []agents.AgentEvent - sub, err := bridge.SubscribeEvents("my-agent", "user1", func(evt agents.AgentEvent) { - received = append(received, evt) + Context("Agent SSE events on the broadcast carrier", func() { + It("bridges an agent's SSE events from a peer replica's carrier", func() { + // Two carriers, because the whole point of this bridge is that the + // user's SSE connection and the replica running the agent are not + // the same process. + watcher := agents.NewEventBridge(infra.Bus(), store, "instance-1") + runner := agents.NewEventBridge(infra.Bus(), store, "instance-2") + + received := make(chan agents.AgentEvent, 16) + sub, err := watcher.SubscribeEvents("my-agent", "user1", func(evt agents.AgentEvent) { + received <- evt }) Expect(err).ToNot(HaveOccurred()) defer sub.Unsubscribe() - FlushNATS(infra.NC) - - // Publish events (simulating agent execution on another instance) - bridge.PublishMessage("my-agent", "user1", "user", "What's the weather?", "msg-1") - bridge.PublishStatus("my-agent", "user1", "processing") - bridge.PublishMessage("my-agent", "user1", "agent", "The weather is sunny.", "msg-2") - bridge.PublishStatus("my-agent", "user1", "completed") - - Eventually(func() int { return len(received) }, "5s").Should(Equal(4)) - Expect(received[0].EventType).To(Equal("json_message")) - Expect(received[0].Sender).To(Equal("user")) - Expect(received[1].EventType).To(Equal("json_message_status")) - Expect(received[2].Sender).To(Equal("agent")) + // Published on the OTHER replica, as an agent execution there would. + Expect(runner.PublishMessage("my-agent", "user1", "user", "What's the weather?", "msg-1")).To(Succeed()) + Expect(runner.PublishStatus("my-agent", "user1", "processing")).To(Succeed()) + Expect(runner.PublishMessage("my-agent", "user1", "agent", "The weather is sunny.", "msg-2")).To(Succeed()) + Expect(runner.PublishStatus("my-agent", "user1", "completed")).To(Succeed()) + + var evts []agents.AgentEvent + for i := 0; i < 4; i++ { + var evt agents.AgentEvent + Eventually(received, "20s").Should(Receive(&evt)) + evts = append(evts, evt) + } + Expect(evts[0].EventType).To(Equal("json_message")) + Expect(evts[0].Sender).To(Equal("user")) + Expect(evts[1].EventType).To(Equal("json_message_status")) + Expect(evts[2].Sender).To(Equal("agent")) }) // Conversation persistence removed — chat history is browser-only. - It("should cancel running agent via NATS", func() { - bridge := agents.NewEventBridge(infra.NC, store, "instance-1") + // Two FRONTEND replicas, which share a carrier. A cancel bound for an + // agent WORKER does not travel this way: a worker has no database, so + // it cannot listen on the PostgreSQL carrier and the frontend publishes + // its cancels where the worker is listening instead. That pairing is + // pinned in core/services/agents; this pins the replica-to-replica half. + It("cancels a running agent from another replica", func() { + bridge := agents.NewEventBridge(infra.Bus(), store, "instance-1") + canceller := agents.NewEventBridge(infra.Bus(), store, "instance-2") // Start cancel listener cancelSub, err := bridge.StartCancelListener() @@ -151,12 +164,12 @@ var _ = Describe("Phase 3: Agent Conversations & SSE", Label("Distributed"), fun }) bridge.RegisterCancel("test-msg-id", wrappedCancel) - FlushNATS(infra.NC) - - // Cancel via NATS - Expect(bridge.CancelExecution("my-agent", "user1", "test-msg-id")).To(Succeed()) + // Issued on the replica that does NOT hold the execution, which is + // the case the broadcast exists for. Its error says only that the + // request was published, so what is asserted is the effect. + Expect(canceller.CancelExecution("my-agent", "user1", "test-msg-id")).To(Succeed()) - Eventually(func() bool { return cancelled.Load() }, "5s").Should(BeTrue()) + Eventually(func() bool { return cancelled.Load() }, "20s").Should(BeTrue()) }) // Agent execution is now dispatched via AgentPoolService.dispatchChat(), diff --git a/tests/e2e/distributed/job_dispatch_test.go b/tests/e2e/distributed/job_dispatch_test.go index b35d93c23c28..c1c74d8ee0a3 100644 --- a/tests/e2e/distributed/job_dispatch_test.go +++ b/tests/e2e/distributed/job_dispatch_test.go @@ -53,7 +53,7 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { // not be told from ones a dead replica left. Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8080", "v1")).To(Succeed()) - dispatcher := jobs.NewDispatcher(store, infra.NC, db, owner) + dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, owner) task := &jobs.TaskRecord{UserID: "u1", Name: "dispatch-task", Model: "m1", Prompt: "p1"} store.CreateTask(task) @@ -100,7 +100,7 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { const owner = "plain-instance" Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8081", "v1")).To(Succeed()) - dispatcher := jobs.NewDispatcher(store, infra.NC, db, owner) + dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, owner) task := &jobs.TaskRecord{UserID: "u1", Name: "plain-task", Model: "m1", Prompt: "p1"} Expect(store.CreateTask(task)).To(Succeed()) job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "pending", TriggeredBy: "api"} @@ -158,12 +158,18 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { }) }) - Context("NATS job cancellation", func() { + Context("job cancellation", func() { // Cancellation stays a BROADCAST and is not part of the claim queue: the // replica holding a run is not the one an API cancel lands on, so the // signal has to reach every replica and every worker. + // + // Asserted across TWO carriers, because one carrier hearing itself + // proves nothing about the replica that actually holds the execution. + // A cancel that does not arrive is not a cancel that was refused, so + // what is pinned here is arrival and never the publisher's error. It("broadcasts a cancel for a job on the job's own cancel subject", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "cancel-instance") + publisher, listener := infra.Bus(), infra.Bus() + dispatcher := jobs.NewDispatcher(store, publisher, db, "cancel-instance") task := &jobs.TaskRecord{UserID: "u1", Name: "cancel-task", Model: "m1", Prompt: "p1"} store.CreateTask(task) @@ -171,7 +177,7 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { store.CreateJob(job) seen := make(chan string, 1) - sub, err := infra.NC.Subscribe(messaging.SubjectJobCancelWildcard, func(data []byte) { + sub, err := listener.Subscribe(messaging.SubjectJobCancelWildcard, func(data []byte) { var evt jobs.CancelEvent if json.Unmarshal(data, &evt) == nil { select { @@ -182,7 +188,6 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { }) Expect(err).ToNot(HaveOccurred()) defer func() { _ = sub.Unsubscribe() }() - FlushNATS(infra.NC) Expect(dispatcher.Cancel(job.ID)).To(Succeed()) Eventually(seen, "10s").Should(Receive(Equal(job.ID))) diff --git a/tests/e2e/distributed/job_distribution_test.go b/tests/e2e/distributed/job_distribution_test.go index 8c56e8730125..61853c130499 100644 --- a/tests/e2e/distributed/job_distribution_test.go +++ b/tests/e2e/distributed/job_distribution_test.go @@ -176,7 +176,7 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { const owner = "test-instance" Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8090", "v1")).To(Succeed()) - dispatcher := jobs.NewDispatcher(store, infra.NC, db, owner) + dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, owner) task := &jobs.TaskRecord{UserID: "u1", Name: "dispatch-test", Model: "m1", Prompt: "p1"} store.CreateTask(task) @@ -204,7 +204,7 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { const owner = "lossy-instance" Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8091", "v1")).To(Succeed()) - dispatcher := jobs.NewDispatcher(store, infra.NC, db, owner) + dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, owner) task := &jobs.TaskRecord{UserID: "u1", Name: "lossy-test", Model: "m1", Prompt: "p1"} store.CreateTask(task) job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "pending", TriggeredBy: "api"} @@ -231,9 +231,13 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { }) It("broadcasts a cancel on the job's own cancel subject", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance") + // Two carriers, because the replica that holds the execution is + // never the one an API cancel lands on. A cancel that does not + // arrive is not a cancel that was refused, so this pins ARRIVAL. + publisher, listener := infra.Bus(), infra.Bus() + dispatcher := jobs.NewDispatcher(store, publisher, db, "test-instance") seen := make(chan string, 1) - sub, err := infra.NC.Subscribe(messaging.SubjectJobCancelWildcard, func(data []byte) { + sub, err := listener.Subscribe(messaging.SubjectJobCancelWildcard, func(data []byte) { var evt jobs.CancelEvent if json.Unmarshal(data, &evt) == nil { select { @@ -244,14 +248,13 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { }) Expect(err).ToNot(HaveOccurred()) defer func() { _ = sub.Unsubscribe() }() - FlushNATS(infra.NC) Expect(dispatcher.Cancel("job-to-cancel")).To(Succeed()) Eventually(seen, "10s").Should(Receive(Equal("job-to-cancel"))) }) It("reports job progress on the job's own progress subject", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance") + dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, "test-instance") var progressEvents []jobs.ProgressEvent var mu sync.Mutex @@ -262,7 +265,6 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { }) Expect(err).ToNot(HaveOccurred()) defer func() { _ = sub.Unsubscribe() }() - FlushNATS(infra.NC) Expect(dispatcher.PublishProgress("progress-job", "running", "step 1")).To(Succeed()) Expect(dispatcher.PublishProgress("progress-job", "running", "step 2")).To(Succeed()) @@ -318,9 +320,9 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { }) }) - Context("Progress Streaming (NATS → SSE bridge)", func() { - It("should bridge NATS progress events", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance") + Context("Progress streaming to the SSE bridge", func() { + It("bridges progress events to a per-job subscription", func() { + dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, "test-instance") dCtx, dCancel := context.WithCancel(infra.Ctx) defer dCancel() @@ -335,8 +337,6 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { Expect(err).ToNot(HaveOccurred()) defer func() { _ = sub.Unsubscribe() }() - FlushNATS(infra.NC) - // Publish progress events dispatcher.PublishProgress("job-123", "running", "processing") dispatcher.PublishProgress("job-123", "running", "almost done") @@ -348,7 +348,7 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { }) It("should filter SSE events by job ID", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "test-instance") + dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, "test-instance") dCtx, dCancel := context.WithCancel(infra.Ctx) defer dCancel() @@ -362,8 +362,6 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { }) defer subA.Unsubscribe() - FlushNATS(infra.NC) - // Publish to both job-A and job-B dispatcher.PublishProgress("job-A", "running", "A progress") dispatcher.PublishProgress("job-B", "running", "B progress") @@ -379,7 +377,7 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { Context("Enriched claim payload (DB-free worker)", func() { It("stores the full Job and Task on the claim row, so the worker needs no database", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "enrichment-test") + dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, "enrichment-test") task := &jobs.TaskRecord{UserID: "u1", Name: "enrich-task", Model: "m1", Prompt: "hello {{.name}}"} store.CreateTask(task) @@ -406,7 +404,8 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { // every replica persists them, because the SSE stream a user is watching // may be open on a replica that claimed nothing. It("persists a result a worker's re-broadcast carried, whichever replica reads it", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "result-test") + dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, "result-test") + peer := infra.Bus() dCtx, dCancel := context.WithCancel(infra.Ctx) defer dCancel() Expect(dispatcher.Start(dCtx)).To(Succeed()) @@ -416,9 +415,11 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { store.CreateTask(task) job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "running", TriggeredBy: "api"} store.CreateJob(job) - FlushNATS(infra.NC) - jobs.PublishJobResult(infra.NC, job.ID, "completed", "job finished successfully", "") + // Published by a PEER replica's carrier: this is the fan-out copy of + // a terminal line the claiming replica already persisted, and the + // replica asserted on here claimed nothing. + jobs.PublishJobResult(peer, job.ID, "completed", "job finished successfully", "") Eventually(func() string { j, _ := store.GetJob(job.ID) @@ -430,7 +431,7 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { }) It("appends a trace a worker's re-broadcast carried", func() { - dispatcher := jobs.NewDispatcher(store, infra.NC, db, "trace-test") + dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, "trace-test") dCtx, dCancel := context.WithCancel(infra.Ctx) defer dCancel() Expect(dispatcher.Start(dCtx)).To(Succeed()) @@ -440,7 +441,6 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { store.CreateTask(task) job := &jobs.JobRecord{TaskID: task.ID, UserID: "u1", Status: "running", TriggeredBy: "api"} store.CreateJob(job) - FlushNATS(infra.NC) Expect(dispatcher.PublishTrace(job.ID, "reasoning", "thinking about the problem")).To(Succeed()) Expect(dispatcher.PublishTrace(job.ID, "tool_call", "calling search tool")).To(Succeed()) diff --git a/tests/e2e/distributed/sse_routes_test.go b/tests/e2e/distributed/sse_routes_test.go index a936eae880a2..5f93b9673716 100644 --- a/tests/e2e/distributed/sse_routes_test.go +++ b/tests/e2e/distributed/sse_routes_test.go @@ -32,63 +32,90 @@ var _ = Describe("SSE Routes", Label("Distributed"), func() { }) Context("Job progress SSE endpoint", func() { - It("should register job progress SSE endpoint when dispatcher active", func() { + // Frontend 0 publishes, frontend 1 is where the SSE reader is attached. + // One carrier hearing itself would pass with the dispatcher wired to + // any carrier at all, which is the wiring defect that leaves every unit + // spec green and only the stream empty. + It("delivers a job's progress to a reader attached to another frontend", func() { jobStore, err := jobs.NewJobStore(db) Expect(err).ToNot(HaveOccurred()) - dispatcher := jobs.NewDispatcher(jobStore, infra.NC, db, "sse-instance") + frontend0 := jobs.NewDispatcher(jobStore, infra.Bus(), db, "frontend-0") + frontend1 := jobs.NewDispatcher(jobStore, infra.Bus(), db, "frontend-1") dCtx, dCancel := context.WithCancel(infra.Ctx) defer dCancel() - Expect(dispatcher.Start(dCtx)).To(Succeed()) - defer dispatcher.Stop() - - // Subscribe to progress for a job — verifies the dispatcher can bridge - // NATS progress events that an SSE endpoint would consume - var events []jobs.ProgressEvent - sub, err := dispatcher.SubscribeProgress("job-sse-test", func(evt jobs.ProgressEvent) { - events = append(events, evt) + Expect(frontend1.Start(dCtx)).To(Succeed()) + defer frontend1.Stop() + + mine := make(chan jobs.ProgressEvent, 16) + sub, err := frontend1.SubscribeProgress("job-sse-test", func(evt jobs.ProgressEvent) { + mine <- evt }) Expect(err).ToNot(HaveOccurred()) defer sub.Unsubscribe() - FlushNATS(infra.NC) - - dispatcher.PublishProgress("job-sse-test", "running", "step 1") - dispatcher.PublishProgress("job-sse-test", "running", "step 2") - dispatcher.PublishProgress("job-sse-test", "completed", "done") - - Eventually(func() int { return len(events) }, "5s").Should(Equal(3)) - Expect(events[0].Status).To(Equal("running")) - Expect(events[2].Status).To(Equal("completed")) + // A reader on a DIFFERENT job, which must be shown nothing. The + // per-request subscription is a data boundary: on the wildcard, + // every client watching any job sees every other job. + theirs := make(chan jobs.ProgressEvent, 16) + otherSub, err := frontend1.SubscribeProgress("job-someone-else", func(evt jobs.ProgressEvent) { + theirs <- evt + }) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = otherSub.Unsubscribe() }() + + Expect(frontend0.PublishProgress("job-sse-test", "running", "step 1")).To(Succeed()) + Expect(frontend0.PublishProgress("job-sse-test", "running", "step 2")).To(Succeed()) + Expect(frontend0.PublishProgress("job-sse-test", "completed", "done")).To(Succeed()) + + var seen []jobs.ProgressEvent + for i := 0; i < 3; i++ { + var evt jobs.ProgressEvent + Eventually(mine, "20s").Should(Receive(&evt)) + seen = append(seen, evt) + } + Expect(seen[0].Status).To(Equal("running")) + Expect(seen[2].Status).To(Equal("completed")) + Expect(theirs).ToNot(Receive(), "a reader on another job id must be shown nothing") }) }) Context("Agent SSE endpoint", func() { - It("should register agent SSE endpoint when event bridge active", func() { + It("delivers an agent's events to a reader attached to another frontend", func() { agentStore, err := agents.NewAgentStore(db) Expect(err).ToNot(HaveOccurred()) - bridge := agents.NewEventBridge(infra.NC, agentStore, "sse-instance") + frontend0 := agents.NewEventBridge(infra.Bus(), agentStore, "frontend-0") + frontend1 := agents.NewEventBridge(infra.Bus(), agentStore, "frontend-1") - // Subscribe to agent events — verifies the bridge can deliver - // NATS events that an SSE endpoint would consume - var received []agents.AgentEvent - sub, err := bridge.SubscribeEvents("test-agent", "user1", func(evt agents.AgentEvent) { - received = append(received, evt) + received := make(chan agents.AgentEvent, 16) + sub, err := frontend1.SubscribeEvents("test-agent", "user1", func(evt agents.AgentEvent) { + received <- evt }) Expect(err).ToNot(HaveOccurred()) defer sub.Unsubscribe() - FlushNATS(infra.NC) - - bridge.PublishMessage("test-agent", "user1", "user", "Hello", "msg-1") - bridge.PublishStatus("test-agent", "user1", "processing") - bridge.PublishMessage("test-agent", "user1", "agent", "Hi!", "msg-2") - - Eventually(func() int { return len(received) }, "5s").Should(Equal(3)) - Expect(received[0].EventType).To(Equal("json_message")) - Expect(received[1].EventType).To(Equal("json_message_status")) + other := make(chan agents.AgentEvent, 16) + otherSub, err := frontend1.SubscribeEvents("test-agent", "user2", func(evt agents.AgentEvent) { + other <- evt + }) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = otherSub.Unsubscribe() }() + + Expect(frontend0.PublishMessage("test-agent", "user1", "user", "Hello", "msg-1")).To(Succeed()) + Expect(frontend0.PublishStatus("test-agent", "user1", "processing")).To(Succeed()) + Expect(frontend0.PublishMessage("test-agent", "user1", "agent", "Hi!", "msg-2")).To(Succeed()) + + var seen []agents.AgentEvent + for i := 0; i < 3; i++ { + var evt agents.AgentEvent + Eventually(received, "20s").Should(Receive(&evt)) + seen = append(seen, evt) + } + Expect(seen[0].EventType).To(Equal("json_message")) + Expect(seen[1].EventType).To(Equal("json_message_status")) + Expect(other).ToNot(Receive(), "another user's stream must be shown nothing") }) }) @@ -97,7 +124,7 @@ var _ = Describe("SSE Routes", Label("Distributed"), func() { appCfg := config.NewApplicationConfig() Expect(appCfg.Distributed.Enabled).To(BeFalse()) - // Without distributed mode, NATS-backed SSE routes are not registered. + // Without distributed mode, carrier-backed SSE routes are not registered. // Agent SSE events use the in-process LocalAGI SSE manager instead. // Job progress is tracked in-memory. Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) diff --git a/tests/e2e/distributed/testhelpers_test.go b/tests/e2e/distributed/testhelpers_test.go index 17ee72ce2e89..0049a77b8455 100644 --- a/tests/e2e/distributed/testhelpers_test.go +++ b/tests/e2e/distributed/testhelpers_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/pgbus" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -236,3 +237,28 @@ func FlushNATS(nc *messaging.Client) { GinkgoHelper() Expect(nc.Conn().Flush()).To(Succeed()) } + +// Bus opens a broadcast carrier on THIS spec's database. +// +// It is the carrier the job, agent and response families travel on, and it is +// what these specs must build their dispatchers and bridges with. Publishing on +// one carrier while the subscriber reads another is a defect with no error +// anywhere: the publish succeeds and the SSE stream is simply empty, so a spec +// that used the NATS client here would keep passing after production had gone +// silent. +// +// Every call returns a SEPARATE carrier on the same database, so a spec can +// build two and assert across them, which is the shape a deployment has. +func (i *TestInfra) Bus() *pgbus.Bus { + GinkgoHelper() + Expect(i.PGURL).ToNot(BeEmpty(), "Bus needs a database; use SetupInfra rather than SetupNATSOnly") + + db, err := gorm.Open(postgres.Open(i.PGURL), &gorm.Config{Logger: gormlogger.Default.LogMode(gormlogger.Silent)}) + Expect(err).ToNot(HaveOccurred()) + Expect(pgbus.Migrate(i.Ctx, db)).To(Succeed()) + + bus, err := pgbus.New(i.Ctx, pgbus.Config{DSN: i.PGURL, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(bus.Close) + return bus +} From 000545b0860e618b340f5288c5f63529fdf5d296 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 4 Sep 2026 02:55:15 +0000 Subject: [PATCH 095/153] feat(distributed): move the last nine fan-out families onto PostgreSQL Gallery progress and cancel, the operation cache's start and end, the model and backend cache invalidations, staging progress, and the prefix cache's observations and invalidations now travel on the LISTEN/NOTIFY carrier. No subject is published or subscribed on messaging.Client anywhere in the tree, which is what makes retiring that package a deletion rather than a migration: $ grep -rn 'natsClient\.Publish\|nats\.Publish\|\.Nats\.Publish\|QueueSubscribe\|SubscribeReply\|\.Request(' \ --include='*.go' core/ pkg/ | grep -v _test \ | grep -v 'c\.Request()\|ctx\.Request()\|Request()\.Context' \ | grep -v 'core/services/testutil/fakebus.go' core/services/messaging/client.go:168,170,172,227,234,236,250,252,254,268,269,287 core/services/messaging/interfaces.go:21,22,23 Every remaining hit is inside core/services/messaging itself. The production reads of the NATS client are now three, all of them the documented agent-worker exception: Close on shutdown, the agent pool's publisher, and the agent-cancel carrier passed to newFanoutBridges. Prefix-cache observations publish like every other family rather than through a method that refuses a message too large for a notification. The plan proposed such a refusal on the reasoning that a long prompt makes a chain of thousands of entries; ExtractChain caps a chain at Config.MaxDepth blocks, MaxDepth is a constant with no operator knob, and the chain reaching Sync.Observe has one source, the router's own extraction hook. A worst-case observation is a few kilobytes against an 8000-byte cap, so the hot-path spill the refusal was designed to avoid cannot occur, and shipping it would have added the programme's only deliberate message drop to guard a condition that cannot arise. pgbus gains FitsInline instead, a predicate that shares one size decision with Publish and decides nothing, and core/application refuses at startup to wire a prefix cache whose configured depth would put every observation over the cap. The carrier choice is no longer stated at four sites. StagingTracker.SetPublisher and SubscribeBroadcasts become one SetBroadcaster, so a tracker that publishes where its peers are not listening cannot be spelled; prefixcache.Sync gains SubscribeBroadcasts, which reads the carrier it publishes on; and the gallery service and the operation cache are wired by methods on DistributedServices that name no carrier at all, so the NATS client beside it cannot be handed over. OpCache.SetMessagingClient and GalleryService.SetNATSClient are renamed to SetBroadcaster so a missed call site fails to compile. Two pre-existing defects that the two-real-carrier specs surfaced are fixed. A progress tick published before a cancel and delivered after it cleared Cancelled and left the operation reading as still running on that replica; mergeStatus now drops a stale tick rather than merging it. GetStatus and GetAllStatus handed out the stored OpStatus pointer while the broadcast subscribers mutated it in place, so an /api/operations response could be marshalled mid-write; both now copy. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/cache_fanout_wiring.go | 165 ++++++++++ core/application/cache_fanout_wiring_test.go | 289 ++++++++++++++++++ core/application/distributed.go | 44 ++- core/application/startup.go | 10 +- core/http/app.go | 14 +- .../endpoints/localai/config_meta_test.go | 2 +- .../http/endpoints/localai/edit_model_test.go | 6 +- .../artifact_progress_coalescer_test.go | 4 +- .../services/galleryop/distributed_pg_test.go | 237 ++++++++++++++ .../galleryop/distributed_sync_test.go | 14 +- core/services/galleryop/history.go | 2 +- .../galleryop/model_revision_delete_test.go | 2 +- core/services/galleryop/operation.go | 46 +-- core/services/galleryop/service.go | 119 ++++++-- core/services/nodes/prefixcache/extractor.go | 4 +- core/services/nodes/prefixcache/sync.go | 93 +++++- core/services/nodes/prefixcache/sync_test.go | 236 +++++++++++++- core/services/nodes/staging_progress.go | 64 ++-- .../nodes/staging_progress_broadcast_test.go | 53 +++- .../nodes/staging_progress_pg_test.go | 86 ++++++ core/services/pgbus/bus.go | 56 +++- core/services/pgbus/spill_test.go | 41 +++ docs/content/features/distributed-mode.md | 51 +++- .../distributed/gallery_distributed_test.go | 62 ++-- 24 files changed, 1517 insertions(+), 183 deletions(-) create mode 100644 core/application/cache_fanout_wiring.go create mode 100644 core/application/cache_fanout_wiring_test.go create mode 100644 core/services/galleryop/distributed_pg_test.go create mode 100644 core/services/nodes/staging_progress_pg_test.go diff --git a/core/application/cache_fanout_wiring.go b/core/application/cache_fanout_wiring.go new file mode 100644 index 000000000000..29330a49bb9f --- /dev/null +++ b/core/application/cache_fanout_wiring.go @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: MIT + +package application + +import ( + "context" + "fmt" + "math" + "strings" + + "github.com/mudler/LocalAI/core/services/galleryop" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/nodes/prefixcache" + "github.com/mudler/LocalAI/core/services/pgbus" +) + +// The four process-lifetime caches, each wired onto the broadcast carrier by +// one function here, and none of them by a call site naming a carrier. +// +// Two are methods on DistributedServices and take NO carrier at all, because +// their call sites hold the whole struct and could otherwise reach for ds.Nats. +// The other two run inside initDistributed before that struct exists, so they +// take the CONCRETE *pgbus.Bus rather than messaging.Broadcaster. +// +// Concrete on purpose. *messaging.Client satisfies messaging.Broadcaster just +// as well as the carrier does, so an interface parameter at these sites lets a +// caller hand over the NATS client that is also in scope: it compiles, it +// starts, it publishes, it is delivered, onto a carrier the deployment is being +// taken off, and nothing fails until NATS goes away. With the concrete type +// that mistake is a build error rather than a deployment that looks healthy. +// +// The adopters themselves still take the interface, so their own specs drive +// them with an in-memory double. The narrowing is only here, where the wrong +// carrier is in scope. + +// wireGalleryBroadcasts puts the gallery service's progress, cancel and +// cache-invalidation traffic on the carrier, and opens the wildcard +// subscriptions that mirror a peer's. +// +// Set and subscribe in one call because they are one decision: a service that +// published where nobody listened would show every operation it started and +// none of its peers', which is what /api/operations looks like on a replica +// that has been load-balanced away from. +// +// The caller must have hydrated from the store and bound OnModelsChanged +// first; both are stated on the methods themselves. +func (ds *DistributedServices) wireGallery(gs *galleryop.GalleryService) error { + if ds == nil || ds.Bus == nil { + return fmt.Errorf("wiring gallery broadcasts: no broadcast carrier, so gallery progress and cancels would reach no peer replica") + } + if gs == nil { + return nil + } + gs.SetBroadcaster(ds.Bus) + return gs.SubscribeBroadcasts() +} + +// WireOpCache puts the admin operation cache on the carrier and starts it, +// which hydrates from PostgreSQL and subscribes. +// +// Exported, unlike its siblings, because the OpCache is built in the HTTP layer +// rather than in initDistributed. It takes neither a carrier nor a store: both +// come off this struct, so the HTTP layer cannot pass the NATS client that +// hangs off it beside them. +// +// A hydrate failure is the OpCache's own business and is logged there; a +// subscribe failure is returned, because a cache that hydrated and did not +// subscribe reports the operations it found at boot and never learns of another. +func (ds *DistributedServices) WireOpCache(ctx context.Context, cache *galleryop.OpCache) error { + if cache == nil { + return nil + } + if ds == nil || ds.Bus == nil { + return fmt.Errorf("wiring the operation cache: no broadcast carrier, so /api/operations would answer with whatever this one replica admitted") + } + cache.SetBroadcaster(ds.Bus) + if ds.DistStores != nil && ds.DistStores.Gallery != nil { + cache.SetGalleryStore(ds.DistStores.Gallery) + } + return cache.Start(ctx) +} + +// wireStagingBroadcasts puts file-staging progress on the carrier in both +// directions. The tracker's own SetBroadcaster is what makes those one call; +// see the comment there. +func wireStagingBroadcasts(bus *pgbus.Bus, tracker *nodes.StagingTracker) (messaging.Subscription, error) { + if bus == nil { + return nil, fmt.Errorf("wiring staging broadcasts: no broadcast carrier, so a staging transfer would show a progress bar only on the replica performing it") + } + if tracker == nil { + return nil, nil + } + return tracker.SetBroadcaster(bus) +} + +// wirePrefixCacheBroadcasts builds the cross-frontend prefix-cache layer on the +// carrier and subscribes it to peers, after refusing a configuration whose +// observations could not travel in a notification. +func wirePrefixCacheBroadcasts(bus *pgbus.Bus, cfg prefixcache.Config, idx prefixcache.Provider) (*prefixcache.Sync, error) { + // The configuration first, and the carrier second. A depth this carrier + // cannot hold is wrong whether or not a carrier was supplied, and naming + // the more specific fault is what makes the startup message actionable. + if err := requirePrefixCacheFitsInline(cfg); err != nil { + return nil, err + } + if bus == nil { + return nil, fmt.Errorf("wiring the prefix cache: no broadcast carrier, so each frontend would route on nothing but its own history") + } + sync := prefixcache.NewSync(idx, bus) + if _, err := sync.SubscribeBroadcasts(); err != nil { + return nil, err + } + return sync, nil +} + +// prefixCacheIdentifierAllowance is how many bytes of model id plus node id a +// prefix-cache observation is budgeted for when its worst case is checked +// against the notification cap. +// +// Both are operator-chosen strings with no enforced length, so no bound here is +// a proof. It does not need to be one: an observation that does not fit is +// SPILLED like any other broadcast, at the cost of a row and a SELECT, and is +// never lost. What the check exists to catch is the other failure, the one that +// has no symptom: a change to Config.MaxDepth that quietly puts every +// observation over the cap and turns the inference path into a table write per +// request. A generous allowance catches that and does not fire on a long model +// name. +const prefixCacheIdentifierAllowance = 512 + +// requirePrefixCacheFitsInline refuses a prefix-cache configuration whose +// observations would spill. +// +// prefixcache.ExtractChain caps a chain at Config.MaxDepth blocks, so an +// observation's size has a worst case that is known before the deployment +// serves a request: MaxDepth hashes at their widest decimal encoding, plus the +// identifiers. That bound is the reason Sync.Observe can publish like every +// other family instead of being given a way to refuse. +// +// It is a startup error and not a warning because the alternative reading is +// the one this programme exists to remove: a deployment that came up, spills a +// row and reads it back on every replica for every request whose prefix +// changed, and looks exactly like one that is merely slow. +func requirePrefixCacheFitsInline(cfg prefixcache.Config) error { + // The widest a uint64 encodes to in JSON, so the check does not depend on + // which hashes a workload happens to produce. + chain := make([]uint64, cfg.MaxDepth) + for i := range chain { + chain[i] = math.MaxUint64 + } + worst := messaging.PrefixCacheObserveEvent{ + Model: strings.Repeat("m", prefixCacheIdentifierAllowance/2), + Chain: chain, + NodeID: strings.Repeat("n", prefixCacheIdentifierAllowance/2), + Replica: math.MaxInt32, + } + fits, err := pgbus.FitsInline(messaging.SubjectPrefixCacheObserve, worst) + if err != nil { + return fmt.Errorf("sizing a prefix-cache observation: %w", err) + } + if !fits { + return fmt.Errorf("the prefix-cache depth is too large to broadcast: an observation for %d blocks does not fit in one notification, so every request whose prefix changed would write a row and every replica would read it back on the inference path", cfg.MaxDepth) + } + return nil +} diff --git a/core/application/cache_fanout_wiring_test.go b/core/application/cache_fanout_wiring_test.go new file mode 100644 index 000000000000..0cc078230f8d --- /dev/null +++ b/core/application/cache_fanout_wiring_test.go @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: MIT + +package application + +import ( + "context" + "runtime" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/galleryop" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/nodes/prefixcache" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// The four process-lifetime caches, each asserted from the OTHER replica's +// carrier. +// +// Every case here wires the cache on busA and drives it from busB. A cache +// talking to itself would pass with the wiring pointed at any carrier at all, +// which is the defect these exist to catch: the NATS client is in scope at +// three of the four call sites and satisfies the same interface, so a site left +// holding it publishes successfully and is delivered, to nobody the deployment +// will still be listening on. +var _ = Describe("wiring the process-lifetime caches onto the broadcast carrier", func() { + var ( + ctx context.Context + db *gorm.DB + busA, busB *pgbus.Bus + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + ctx = context.Background() + + var dsn string + db, dsn = testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(ctx, db)).To(Succeed()) + + newBus := func() *pgbus.Bus { + b, err := pgbus.New(ctx, pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(b.Close) + return b + } + busA, busB = newBus(), newBus() + }) + + // Each of the four refuses rather than coming up on nothing. A cache wired + // to no carrier has no symptom of its own: it answers from whatever this + // one replica happened to do, forever, and looks exactly like a fleet with + // nothing going on elsewhere. + DescribeTable("refuses to wire a cache with no carrier", + func(wire func() error, want string) { + err := wire() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(want)) + }, + Entry("gallery", func() error { + return (&DistributedServices{}).wireGallery(galleryop.NewGalleryService(&config.ApplicationConfig{}, nil)) + }, "gallery progress and cancels"), + Entry("operation cache", func() error { + svc := galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) + return (&DistributedServices{}).WireOpCache(context.Background(), galleryop.NewOpCache(svc)) + }, "/api/operations"), + Entry("staging", func() error { + _, err := wireStagingBroadcasts(nil, nodes.NewStagingTracker()) + return err + }, "progress bar only on the replica performing it"), + Entry("prefix cache", func() error { + _, err := wirePrefixCacheBroadcasts(nil, prefixcache.DefaultConfig(), prefixcache.NewIndex(prefixcache.DefaultConfig())) + return err + }, "its own history"), + ) + + // S2. The gallery service applies a peer's progress, which it can only do + // if the wildcard subscription wireGallery opened is on the carrier the + // peer published to. + It("subscribes the gallery service to progress a peer replica broadcasts", func() { + svc := galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) + Expect((&DistributedServices{Bus: busA}).wireGallery(svc)).To(Succeed()) + DeferCleanup(svc.CloseBroadcasts) + + Expect(busB.Publish(messaging.SubjectGalleryProgress("op-1"), galleryop.GalleryProgressEvent{ + JobID: "op-1", + Status: &galleryop.OpStatus{Progress: 42, Message: "halfway"}, + })).To(Succeed()) + + Eventually(func() *galleryop.OpStatus { return svc.GetStatus("op-1") }, 20*time.Second).ShouldNot(BeNil()) + Expect(svc.GetStatus("op-1").Progress).To(Equal(42.0)) + }) + + // S2, the other direction. A service that only subscribed would pass the + // row above and publish its own progress where no peer reads it. + It("publishes the gallery service's progress onto the same carrier", func() { + svc := galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) + Expect((&DistributedServices{Bus: busA}).wireGallery(svc)).To(Succeed()) + DeferCleanup(svc.CloseBroadcasts) + + out := make(chan []byte, 4) + _, err := busB.Subscribe(messaging.SubjectGalleryProgressWildcard, func(b []byte) { out <- b }) + Expect(err).ToNot(HaveOccurred()) + + svc.UpdateStatus("op-2", &galleryop.OpStatus{Progress: 7}) + + Eventually(out, 20*time.Second).Should(Receive()) + }) + + // S1. The OpCache is wired from the HTTP layer, and WireOpCache is what + // keeps that call site from naming a carrier of its own. + It("subscribes the operation cache to a peer replica's admissions", func() { + svc := galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) + cache := galleryop.NewOpCache(svc) + Expect((&DistributedServices{Bus: busA}).WireOpCache(ctx, cache)).To(Succeed()) + DeferCleanup(cache.Close) + + Expect(busB.Publish(messaging.SubjectGalleryOpStart, galleryop.OpCacheEvent{ + JobID: "job-9", CacheKey: "official@vllm", IsBackend: true, + })).To(Succeed()) + + Eventually(func() bool { return cache.Exists("official@vllm") }, 20*time.Second).Should(BeTrue()) + Expect(cache.IsBackendOp("official@vllm")).To(BeTrue()) + }) + + It("publishes the operation cache's admissions onto the same carrier", func() { + svc := galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) + cache := galleryop.NewOpCache(svc) + Expect((&DistributedServices{Bus: busA}).WireOpCache(ctx, cache)).To(Succeed()) + DeferCleanup(cache.Close) + + out := make(chan []byte, 4) + _, err := busB.Subscribe(messaging.SubjectGalleryOpStart, func(b []byte) { out <- b }) + Expect(err).ToNot(HaveOccurred()) + + cache.Set("llama-3-8b", "job-10") + + Eventually(out, 20*time.Second).Should(Receive()) + }) + + // S3, in both directions and as two separate specs. It used to be two + // calls, a publisher and a subscriber, and a tracker with one of them on + // each carrier shows a staging progress bar on the originating replica and + // nowhere else. SetBroadcaster is one method now, so that deployment cannot + // be spelled, but each half still has to be held on its own: a mutation + // that drops the subscribe leaves the publishing spec green and the reverse + // leaves the mirroring spec green. + It("mirrors a peer replica's staging progress into the tracker", func() { + tracker := nodes.NewStagingTracker() + sub, err := wireStagingBroadcasts(busA, tracker) + Expect(err).ToNot(HaveOccurred()) + Expect(sub).ToNot(BeNil()) + + Expect(busB.Publish(messaging.SubjectStagingProgress("model-x"), nodes.StagingProgressEvent{ + ModelID: "model-x", + Status: &nodes.StagingStatus{ModelID: "model-x", NodeName: "worker-7"}, + })).To(Succeed()) + + Eventually(func() map[string]nodes.StagingStatus { return tracker.GetAll() }, 20*time.Second). + Should(HaveKey("model-x")) + }) + + It("publishes the tracker's own staging progress onto the same carrier", func() { + tracker := nodes.NewStagingTracker() + _, err := wireStagingBroadcasts(busA, tracker) + Expect(err).ToNot(HaveOccurred()) + + out := make(chan []byte, 4) + _, err = busB.Subscribe(messaging.SubjectStagingProgressWildcard, func(b []byte) { out <- b }) + Expect(err).ToNot(HaveOccurred()) + + tracker.Start("model-y", "worker-8", 1) + + Eventually(out, 20*time.Second).Should(Receive()) + }) + + // S4, in both directions. The prefix cache is the family on the inference + // path, and a Sync wired to a carrier its peers do not read leaves every + // frontend routing on nothing but its own history while every publish + // succeeds. + It("applies a peer replica's observation into the prefix index", func() { + idx := prefixcache.NewIndex(prefixcache.DefaultConfig()) + sync, err := wirePrefixCacheBroadcasts(busA, prefixcache.DefaultConfig(), idx) + Expect(err).ToNot(HaveOccurred()) + + chain := []uint64{101, 202, 303} + Expect(busB.Publish(messaging.SubjectPrefixCacheObserve, messaging.PrefixCacheObserveEvent{ + Model: "m", Chain: chain, NodeID: "A", Replica: 1, + })).To(Succeed()) + + Eventually(func() bool { + return sync.Decide("m", chain, []prefixcache.ReplicaKey{{NodeID: "A", Replica: 1}}, time.Now()).HasHot + }, 20*time.Second).Should(BeTrue()) + }) + + It("applies a peer replica's invalidation, so a removed replica stops being routed to", func() { + // The invalidation half separately: a missed one leaves this frontend + // routing to a replica that is gone until the TTL, which is the reading + // of a missed message this programme forbids. + idx := prefixcache.NewIndex(prefixcache.DefaultConfig()) + sync, err := wirePrefixCacheBroadcasts(busA, prefixcache.DefaultConfig(), idx) + Expect(err).ToNot(HaveOccurred()) + + chain := []uint64{404, 505} + key := prefixcache.ReplicaKey{NodeID: "A", Replica: 0} + sync.ApplyObserve(messaging.PrefixCacheObserveEvent{Model: "m", Chain: chain, NodeID: "A"}, time.Now()) + Expect(sync.Decide("m", chain, []prefixcache.ReplicaKey{key}, time.Now()).HasHot).To(BeTrue()) + + Expect(busB.Publish(messaging.SubjectPrefixCacheInvalidate, messaging.PrefixCacheInvalidateEvent{ + Model: "m", NodeID: "A", Replica: 0, + })).To(Succeed()) + + Eventually(func() bool { + return sync.Decide("m", chain, []prefixcache.ReplicaKey{key}, time.Now()).HasHot + }, 20*time.Second).Should(BeFalse()) + }) + + It("publishes this replica's observations onto the same carrier", func() { + idx := prefixcache.NewIndex(prefixcache.DefaultConfig()) + sync, err := wirePrefixCacheBroadcasts(busA, prefixcache.DefaultConfig(), idx) + Expect(err).ToNot(HaveOccurred()) + + out := make(chan []byte, 4) + _, err = busB.Subscribe(messaging.SubjectPrefixCacheObserve, func(b []byte) { out <- b }) + Expect(err).ToNot(HaveOccurred()) + + sync.Observe("m", []uint64{909}, prefixcache.ReplicaKey{NodeID: "B", Replica: 0}, time.Now()) + + Eventually(out, 20*time.Second).Should(Receive()) + }) +}) + +// The size bound that lets prefix-cache observations publish like every other +// family instead of being given a way to refuse. +// +// The plan for this phase proposed a PublishNoSpill that would REFUSE an +// observation too large for a notification, on the reasoning that a long prompt +// makes a chain of thousands of entries. ExtractChain does not produce one, and +// the refusal would have been the only deliberate message drop in the +// programme, guarding a condition that cannot arise, with a counter nothing +// alerts on as its only symptom. This is what took its place: the same +// knowledge, asked at startup, where being wrong is a deployment that refuses +// to come up and says why rather than one that runs with no cross-replica +// affinity and looks healthy. +// +// It needs no bus. FitsInline is a pure function over the same encoder and the +// same constant Publish measures against. +var _ = Describe("the prefix-cache observation bound", func() { + It("accepts the depth the extractor actually produces", func() { + Expect(requirePrefixCacheFitsInline(prefixcache.DefaultConfig())).To(Succeed()) + }) + + It("refuses a depth whose observations would spill on every request", func() { + // An absolute depth, not one derived from the carrier's cap, so this + // row states a fact about this family rather than restating the + // constant it is measured against. + cfg := prefixcache.DefaultConfig() + cfg.MaxDepth = 100000 + + err := requirePrefixCacheFitsInline(cfg) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("write a row")) + }) + + It("is checked before the prefix cache is wired at all", func() { + // The check is worth nothing if the wiring runs anyway. Asserted + // through the same function initDistributed calls, and on the MESSAGE + // rather than on failure alone: this call has two things wrong with it, + // and a spec that accepted any error would pass on the carrier + // complaint with the bound check deleted. + cfg := prefixcache.DefaultConfig() + cfg.MaxDepth = 100000 + + sync, err := wirePrefixCacheBroadcasts(nil, cfg, prefixcache.NewIndex(cfg)) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("write a row")) + Expect(sync).To(BeNil()) + }) +}) diff --git a/core/application/distributed.go b/core/application/distributed.go index 2225a9f65042..92f11fee535a 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -507,8 +507,8 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // with --distributed-prefix-cache=false, which leaves prefixProvider and // pressure nil so the SmartRouter and reconciler behave exactly as the // round-robin floor (true no-op). When enabled we build the local index, - // wrap it in a NATS-backed Sync (publishes our observations, applies peers' - // via the subscriptions below), install the extraction hook used by + // wrap it in a Sync on the broadcast carrier (which both publishes our + // observations and applies peers'), install the extraction hook used by // core/backend/llm.go, and run a background eviction ticker on the app ctx. var prefixProvider prefixcache.Provider var pressure *prefixcache.Pressure @@ -522,8 +522,14 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade return nil, fmt.Errorf("invalid prefix-cache configuration: %w", err) } idx := prefixcache.NewIndex(prefixCfg) - prefixSync := prefixcache.NewSync(idx, natsClient) - pressure = prefixcache.NewSyncedPressure(prefixCfg.PressureWindow, natsClient) + // S4. One call puts this replica's observations and its peers' on the + // same carrier, and it takes the concrete carrier so the NATS client + // still in scope here cannot be handed to it by accident. + prefixSync, err := wirePrefixCacheBroadcasts(bus, prefixCfg, idx) + if err != nil { + return nil, err + } + pressure = prefixcache.NewSyncedPressure(prefixCfg.PressureWindow, bus) prefixProvider = prefixSync // Invalidate the prefix-cache index whenever a replica row is removed. @@ -546,20 +552,10 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade return prefixcache.ExtractChain(model, prompt, prefixCfg) } - // Apply peers' observations/invalidations to the same Sync. ApplyObserve - // and ApplyInvalidate update only the local index and do not re-publish, - // so there is no broadcast loop. - if _, err := messaging.SubscribeJSON(natsClient, messaging.SubjectPrefixCacheObserve, func(ev messaging.PrefixCacheObserveEvent) { - prefixSync.ApplyObserve(ev, time.Now()) - }); err != nil { - return nil, fmt.Errorf("subscribing to %s: %w", messaging.SubjectPrefixCacheObserve, err) - } - if _, err := messaging.SubscribeJSON(natsClient, messaging.SubjectPrefixCacheInvalidate, func(ev messaging.PrefixCacheInvalidateEvent) { - prefixSync.ApplyInvalidate(ev) - }); err != nil { - return nil, fmt.Errorf("subscribing to %s: %w", messaging.SubjectPrefixCacheInvalidate, err) - } - if _, err := messaging.SubscribeJSON(natsClient, messaging.SubjectPrefixCachePressure, func(ev messaging.PrefixCachePressureEvent) { + // The Sync owns observation and invalidation subscriptions. Pressure and + // reported residency are newer prefix-cache families and use the same + // PostgreSQL carrier so no cache signal depends on NATS. + if _, err := messaging.SubscribeJSON(bus, messaging.SubjectPrefixCachePressure, func(ev messaging.PrefixCachePressureEvent) { pressure.ApplyPressure(ev, time.Now()) }); err != nil { return nil, fmt.Errorf("subscribing to %s: %w", messaging.SubjectPrefixCachePressure, err) @@ -569,10 +565,9 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // their real KV state without coupling to router internals. Routing stays // on the guessed provider until a backend producer is available. reportedIndex := prefixcache.NewReportedIndex() - if _, err := messaging.SubscribeJSON(natsClient, messaging.SubjectPrefixCacheResidency, reportedIndex.Apply); err != nil { + if _, err := messaging.SubscribeJSON(bus, messaging.SubjectPrefixCacheResidency, reportedIndex.Apply); err != nil { return nil, fmt.Errorf("subscribing to %s: %w", messaging.SubjectPrefixCacheResidency, err) } - // Background eviction: sweep idle entries on the app context. Stopped // when the app context is cancelled (mirrors the reconciler loop which // also runs on options.Context). TTL/2 keeps stale entries from @@ -652,7 +647,8 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // Wire staging-progress broadcasting so file-staging shows up on every // replica, not just the one performing the transfer. Without this, a // /api/operations poll that round-robins onto a peer sees no staging row and - // the progress flickers. The origin publishes; peers mirror via the wildcard. + // the progress flickers. The origin publishes; peers mirror via the + // wildcard, on the same carrier. // A silently disabled safety check is how the original incident stayed // invisible for sixteen minutes. Say so once, loudly, at startup. if cfg.Distributed.DiskHeadroomDisabled { @@ -660,8 +656,10 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade "knob", config.FlagDiskHeadroomCheck, "env", "LOCALAI_DISTRIBUTED_DISK_HEADROOM_CHECK") } - router.StagingTracker().SetPublisher(natsClient) - if _, err := router.StagingTracker().SubscribeBroadcasts(natsClient); err != nil { + // S3, and it is ONE call rather than a publisher and a subscriber: see + // StagingTracker.SetBroadcaster for why a tracker that could name two + // carriers is a progress bar that only the originating replica shows. + if _, err := wireStagingBroadcasts(bus, router.StagingTracker()); err != nil { xlog.Warn("Failed to subscribe to staging progress broadcasts", "error", err) } diff --git a/core/application/startup.go b/core/application/startup.go index b985d7e55f0d..e2d99517147c 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -334,9 +334,10 @@ func New(opts ...config.AppOption) (*Application, error) { // Wire skill store into AgentPoolService (wired at pool start time via closure) // The actual wiring happens in StartAgentPool since the pool doesn't exist yet. - // Wire NATS and gallery store into GalleryService for cross-instance progress/cancel + // Wire the broadcast carrier and gallery store into GalleryService for + // cross-instance progress/cancel. The carrier is wired below, next to + // the subscriptions it feeds, so the two cannot name different carriers. if application.galleryService != nil { - application.galleryService.SetNATSClient(distSvc.Nats) if distSvc.DistStores != nil && distSvc.DistStores.Gallery != nil { // Clean up stale in-progress operations from previous crashed instances if _, err := distSvc.DistStores.Gallery.CleanStale(30 * time.Minute); err != nil { @@ -396,7 +397,10 @@ func New(opts ...config.AppOption) (*Application, error) { xlog.Warn("Failed to apply peer model config change", "error", err) } } - if err := application.galleryService.SubscribeBroadcasts(); err != nil { + // S2. One call sets the carrier and opens the wildcard + // subscriptions, and it names no carrier, so the NATS client on + // distSvc cannot be passed here by accident. + if err := distSvc.wireGallery(application.galleryService); err != nil { xlog.Warn("Gallery service subscribe failed", "error", err) } // Wire distributed model/backend managers so delete propagates to workers diff --git a/core/http/app.go b/core/http/app.go index 54ba3b205cfc..f61b7f459e15 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -476,17 +476,17 @@ func API(application *application.Application) (*echo.Echo, error) { var opcache *galleryop.OpCache if !application.ApplicationConfig().DisableWebUI { opcache = galleryop.NewOpCache(application.GalleryService()) - // In distributed mode, wire the NATS client + gallery store so this - // replica's OpCache stays in sync with peers — without this the + // In distributed mode, wire the broadcast carrier + gallery store so + // this replica's OpCache stays in sync with peers. Without this the // /api/operations endpoint returns whatever this single replica // happened to admit, and a load-balanced UI poll alternates between // "operation visible" and "operation gone" between replicas. + // + // S1. The carrier choice lives in core/application with the other three + // caches, and this call names no carrier at all, so the NATS client + // hanging off the same struct cannot be handed over here by accident. if d := application.Distributed(); d != nil { - opcache.SetMessagingClient(d.Nats) - if d.DistStores != nil && d.DistStores.Gallery != nil { - opcache.SetGalleryStore(d.DistStores.Gallery) - } - if err := opcache.Start(application.ApplicationConfig().Context); err != nil { + if err := d.WireOpCache(application.ApplicationConfig().Context, opcache); err != nil { xlog.Warn("OpCache distributed subscribe failed; running standalone", "error", err) } } diff --git a/core/http/endpoints/localai/config_meta_test.go b/core/http/endpoints/localai/config_meta_test.go index db7e0fd5fbdb..c90ca9997a3b 100644 --- a/core/http/endpoints/localai/config_meta_test.go +++ b/core/http/endpoints/localai/config_meta_test.go @@ -188,7 +188,7 @@ backend: llama-cpp lifecycle := &endpointLifecycleRecorder{} galleryService := galleryop.NewGalleryService(appConfig, nil) client := &endpointRecordingClient{} - galleryService.SetNATSClient(client) + galleryService.SetBroadcaster(client) endpointApp := echo.New() endpointApp.PATCH("/api/models/config-json/:name", PatchConfigEndpoint(configLoader, galleryService, appConfig, lifecycle)) diff --git a/core/http/endpoints/localai/edit_model_test.go b/core/http/endpoints/localai/edit_model_test.go index 223943e4619a..31f73e749712 100644 --- a/core/http/endpoints/localai/edit_model_test.go +++ b/core/http/endpoints/localai/edit_model_test.go @@ -126,7 +126,7 @@ var _ = Describe("Edit Model test", func() { Expect(loader.LoadModelConfigsFromPath(tempDir)).To(Succeed()) galleryService := galleryop.NewGalleryService(applicationConfig, nil) client := &endpointRecordingClient{} - galleryService.SetNATSClient(client) + galleryService.SetBroadcaster(client) app := echo.New() app.POST("/models/edit/:name", EditModelEndpoint(loader, galleryService, applicationConfig)) @@ -152,7 +152,7 @@ var _ = Describe("Edit Model test", func() { Expect(loader.LoadModelConfigsFromPath(tempDir)).To(Succeed()) galleryService := galleryop.NewGalleryService(applicationConfig, nil) client := &endpointRecordingClient{} - galleryService.SetNATSClient(client) + galleryService.SetBroadcaster(client) app := echo.New() app.POST("/models/edit/:name", EditModelEndpoint(loader, galleryService, applicationConfig)) @@ -276,7 +276,7 @@ var _ = Describe("Edit Model test", func() { Expect(peerLoader.LoadModelConfigsFromPath(tempDir)).To(Succeed()) galleryService := galleryop.NewGalleryService(applicationConfig, nil) client := &endpointRecordingClient{} - galleryService.SetNATSClient(client) + galleryService.SetBroadcaster(client) lifecycle := &endpointLifecycleRecorder{pendingCleanup: 2} app := echo.New() app.POST("/models/edit/:name", EditModelEndpoint(loader, galleryService, applicationConfig, lifecycle)) diff --git a/core/services/galleryop/artifact_progress_coalescer_test.go b/core/services/galleryop/artifact_progress_coalescer_test.go index 9c02b3e9a4c6..f26ed79d3eba 100644 --- a/core/services/galleryop/artifact_progress_coalescer_test.go +++ b/core/services/galleryop/artifact_progress_coalescer_test.go @@ -180,7 +180,7 @@ var _ = Describe("artifact progress coalescer", func() { progressClient := &recordingProgressClient{} service := NewGalleryService(&config.ApplicationConfig{}, nil) service.modelManager = &modelOperationProgressManager{err: installErr} - service.natsClient = progressClient + service.broadcaster = progressClient op := &ManagementOp[gallery.GalleryModel, gallery.ModelConfig]{ ID: "model-operation", GalleryElementName: "model", @@ -205,7 +205,7 @@ var _ = Describe("artifact progress coalescer", func() { progressClient := &recordingProgressClient{} service := NewGalleryService(&config.ApplicationConfig{}, nil) service.modelManager = &legacyModelOperationProgressManager{err: installErr} - service.natsClient = progressClient + service.broadcaster = progressClient op := &ManagementOp[gallery.GalleryModel, gallery.ModelConfig]{ ID: "legacy-model-operation", GalleryElementName: "model", diff --git a/core/services/galleryop/distributed_pg_test.go b/core/services/galleryop/distributed_pg_test.go new file mode 100644 index 000000000000..8c8affc4296e --- /dev/null +++ b/core/services/galleryop/distributed_pg_test.go @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: MIT + +package galleryop_test + +import ( + "context" + "fmt" + "runtime" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/galleryop" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// The gallery families on the real carrier, between two services on two LISTEN +// connections. +// +// The in-memory double in distributed_sync_test.go delivers synchronously and +// cannot spill, so it proves the merge logic and nothing about the carrier. +// Gallery progress is the family that crosses the 8000-byte notification cap in +// ordinary operation, at a few tens of workers, so the path a real fleet takes +// every tick is the spill path, and it is only reachable here. +var _ = Describe("the gallery families on the broadcast carrier", func() { + var ( + db *gorm.DB + dsn string + svcA, svcB *galleryop.GalleryService + // peerBus stands in for a replica that has no GalleryService in this + // process. It is how the backends family is driven: its publisher is + // reached only from deep inside backendHandler, which needs a whole + // gallery operation, while the shared publishCacheInvalidate it calls + // is already pinned end to end by the models row below. + peerBus *pgbus.Bus + ) + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + ctx := context.Background() + db, dsn = testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(ctx, db)).To(Succeed()) + + newService := func() *galleryop.GalleryService { + bus, err := pgbus.New(ctx, pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(bus.Close) + svc := galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) + svc.SetBroadcaster(bus) + Expect(svc.SubscribeBroadcasts()).To(Succeed()) + DeferCleanup(svc.CloseBroadcasts) + return svc + } + svcA, svcB = newService(), newService() + + var err error + peerBus, err = pgbus.New(ctx, pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(peerBus.Close) + }) + + spilledRows := func(subject string) int64 { + GinkgoHelper() + var n int64 + Expect(db.Model(&pgbus.BusMessage{}).Where("subject = ?", subject).Count(&n).Error).To(Succeed()) + return n + } + + It("carries a 60-node progress event to a peer through the spill table, byte for byte", func() { + // Sixty nodes is an ordinary fleet and roughly twice what the cap + // holds, so this is the path a real deployment's progress ticks take + // and not an edge case. Asserted as one row AND as identical content: + // a spill that lost an entry would still deliver something. + nodes := make([]galleryop.NodeProgress, 0, 60) + for i := range 60 { + nodes = append(nodes, galleryop.NodeProgress{ + NodeID: fmt.Sprintf("node-%02d", i), + NodeName: fmt.Sprintf("worker-%02d.fleet.internal", i), + Status: galleryop.NodeStatusDownloading, + FileName: "backend-image.tar", + Current: "512 MiB", + Total: "1.5 GiB", + Percentage: float64(i), + Phase: "downloading", + }) + } + status := &galleryop.OpStatus{ + Progress: 33.0, + Message: "installing on the fleet", + GalleryElementName: "official@vllm", + Nodes: nodes, + } + + svcA.UpdateStatus("op-fleet", status) + + Eventually(func() *galleryop.OpStatus { return svcB.GetStatus("op-fleet") }, 20*time.Second). + ShouldNot(BeNil()) + got := svcB.GetStatus("op-fleet") + Expect(got.Nodes).To(HaveLen(60)) + Expect(got.Nodes).To(Equal(nodes), "a peer must see the same per-node breakdown, not a truncated one") + Expect(got.GalleryElementName).To(Equal("official@vllm")) + + Expect(spilledRows(messaging.SubjectGalleryProgress("op-fleet"))).To(Equal(int64(1)), + "a progress event this size does not fit in a notification and must travel as a row") + }) + + It("carries a cancel to the peer holding the operation", func() { + // A cancel is a request and not a verdict, but it has to arrive: the + // replica that admitted the job is rarely the one the admin's click + // lands on. + svcB.UpdateStatus("op-cancel", &galleryop.OpStatus{Progress: 10, Cancellable: true}) + + Expect(svcA.CancelOperation("op-cancel")).To(Succeed()) + + Eventually(func() bool { + st := svcB.GetStatus("op-cancel") + return st != nil && st.Cancelled + }, 20*time.Second).Should(BeTrue()) + }) + + It("does not let a progress tick delivered after a cancel undo it", func() { + // The carrier puts no order on two subjects, and the owning replica's + // last tick is published BEFORE the admin's cancel and can arrive after + // it, on that replica's own echo as readily as on a peer. Driven here + // as an explicit late tick rather than by racing the two, so it states + // the rule rather than reproducing a window. + svcB.UpdateStatus("op-late", &galleryop.OpStatus{Progress: 10, Cancellable: true}) + Expect(svcA.CancelOperation("op-late")).To(Succeed()) + Eventually(func() bool { + st := svcB.GetStatus("op-late") + return st != nil && st.Cancelled + }, 20*time.Second).Should(BeTrue()) + + Expect(peerBus.Publish(messaging.SubjectGalleryProgress("op-late"), galleryop.GalleryProgressEvent{ + JobID: "op-late", + Status: &galleryop.OpStatus{Progress: 60, Message: "downloading"}, + })).To(Succeed()) + + Consistently(func() bool { + st := svcB.GetStatus("op-late") + return st != nil && st.Cancelled + }, 3*time.Second).Should(BeTrue(), + "a cancelled operation must not read as running again because a tick arrived late") + }) + + // A missed invalidation must never read as a cache that is valid, which is + // why this family is on Publish and spills rather than on anything that can + // refuse. These pin that the peer's refresh hook actually fires. + DescribeTable("carries a cache invalidation to the peer's refresh hook", + func(broadcast func(*galleryop.GalleryService), bind func(*galleryop.GalleryService, chan<- string)) { + fired := make(chan string, 4) + bind(svcB, fired) + + broadcast(svcA) + + var got string + Eventually(fired, 20*time.Second).Should(Receive(&got)) + Expect(got).ToNot(BeEmpty()) + }, + Entry("models", + func(s *galleryop.GalleryService) { s.BroadcastModelsChanged("llama-3-8b", "install") }, + func(s *galleryop.GalleryService, fired chan<- string) { + s.OnModelsChanged = func(evt messaging.CacheInvalidateEvent) { fired <- evt.Element } + }, + ), + Entry("backends", + func(*galleryop.GalleryService) { + Expect(peerBus.Publish(messaging.SubjectCacheInvalidateBackends, + messaging.CacheInvalidateEvent{Element: "official@vllm", Op: "upgrade"})).To(Succeed()) + }, + func(s *galleryop.GalleryService, fired chan<- string) { + s.OnBackendOpCompleted = func() { fired <- "backends" } + }, + ), + ) +}) + +// The OpCache's two subjects, likewise on the real carrier. The echo-loop guard +// stays on the in-memory double, which is the only carrier that can count +// publishes. +var _ = Describe("the OpCache on the broadcast carrier", func() { + It("propagates an admission and a dismissal to a peer replica", func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + ctx := context.Background() + db, dsn := testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(ctx, db)).To(Succeed()) + + newCache := func() *galleryop.OpCache { + bus, err := pgbus.New(ctx, pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(bus.Close) + cache := galleryop.NewOpCache(galleryop.NewGalleryService(&config.ApplicationConfig{}, nil)) + cache.SetBroadcaster(bus) + Expect(cache.Start(ctx)).To(Succeed()) + DeferCleanup(cache.Close) + return cache + } + cacheA, cacheB := newCache(), newCache() + + cacheA.SetBackend("official@vllm", "job-1") + + Eventually(func() bool { return cacheB.Exists("official@vllm") }, 20*time.Second).Should(BeTrue()) + Expect(cacheB.IsBackendOp("official@vllm")).To(BeTrue(), + "the peer must learn this is a backend install, not a model install") + + cacheA.DeleteUUID("job-1") + + Eventually(func() bool { return cacheB.Exists("official@vllm") }, 20*time.Second).Should(BeFalse(), + "a dismissed operation must clear from peer replicas too") + }) + + It("does not re-broadcast an event it applied", func() { + // The echo-loop guard, re-run on the new carrier's shape. applyStart + // writes the local maps and must not publish: two replicas that + // answered each other's broadcasts would never stop. + bus := testutil.NewFakeBus() + cache := galleryop.NewOpCache(galleryop.NewGalleryService(&config.ApplicationConfig{}, nil)) + cache.SetBroadcaster(bus) + Expect(cache.Start(context.Background())).To(Succeed()) + DeferCleanup(cache.Close) + + cache.Set("llama-3-8b", "job-2") + + Expect(cache.Exists("llama-3-8b")).To(BeTrue()) + Expect(bus.PublishCount(messaging.SubjectGalleryOpStart)).To(Equal(1), + "the replica's own broadcast came back to it and must not have produced a second") + }) +}) diff --git a/core/services/galleryop/distributed_sync_test.go b/core/services/galleryop/distributed_sync_test.go index ed88ab25be37..3572c81c4809 100644 --- a/core/services/galleryop/distributed_sync_test.go +++ b/core/services/galleryop/distributed_sync_test.go @@ -152,8 +152,8 @@ var _ = Describe("OpCache distributed sync", func() { svcB = galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) opA = galleryop.NewOpCache(svcA) opB = galleryop.NewOpCache(svcB) - opA.SetMessagingClient(bus) - opB.SetMessagingClient(bus) + opA.SetBroadcaster(bus) + opB.SetBroadcaster(bus) Expect(opA.Start(context.Background())).To(Succeed()) Expect(opB.Start(context.Background())).To(Succeed()) }) @@ -241,8 +241,8 @@ var _ = Describe("GalleryService broadcast sync", func() { bus = newFakeBus() svcA = galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) svcB = galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) - svcA.SetNATSClient(bus) - svcB.SetNATSClient(bus) + svcA.SetBroadcaster(bus) + svcB.SetBroadcaster(bus) Expect(svcA.SubscribeBroadcasts()).To(Succeed()) Expect(svcB.SubscribeBroadcasts()).To(Succeed()) }) @@ -329,8 +329,8 @@ var _ = Describe("GalleryService cache invalidation broadcasts", func() { bus = newFakeBus() svcA = galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) svcB = galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) - svcA.SetNATSClient(bus) - svcB.SetNATSClient(bus) + svcA.SetBroadcaster(bus) + svcB.SetBroadcaster(bus) }) AfterEach(func() { @@ -417,7 +417,7 @@ var _ = Describe("GalleryService cache invalidation broadcasts", func() { It("BroadcastModelsChanged is a no-op when NATS is not wired (standalone)", func() { standalone := galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) - // No SetNATSClient: must not panic and must simply do nothing. + // No SetBroadcaster: must not panic and must simply do nothing. Expect(func() { standalone.BroadcastModelsChanged("x", "delete") }).ToNot(Panic()) }) }) diff --git a/core/services/galleryop/history.go b/core/services/galleryop/history.go index 5617c74aa84d..76f94660780b 100644 --- a/core/services/galleryop/history.go +++ b/core/services/galleryop/history.go @@ -118,7 +118,7 @@ func newOpHistory(limit int) *opHistory { // add appends rec unless its job ID was already recorded. Returns false when // the record was a duplicate. The originating replica both evicts locally and -// receives its own NATS end broadcast, so without this every distributed +// receives its own end broadcast, so without this every distributed // operation would be recorded twice. func (h *opHistory) add(rec OpRecord) bool { h.mu.Lock() diff --git a/core/services/galleryop/model_revision_delete_test.go b/core/services/galleryop/model_revision_delete_test.go index 7da910102ed6..3daf9342f945 100644 --- a/core/services/galleryop/model_revision_delete_test.go +++ b/core/services/galleryop/model_revision_delete_test.go @@ -204,7 +204,7 @@ var _ = Describe("model deletion revision lifecycle", func() { service := NewGalleryService(appConfig, nil) bus := &countingMessagingClient{} - service.SetNATSClient(bus) + service.SetBroadcaster(bus) service.SetModelManager(manager) service.SetModelRevisionLifecycle(lifecycle) op := &ManagementOp[gallery.GalleryModel, gallery.ModelConfig]{ diff --git a/core/services/galleryop/operation.go b/core/services/galleryop/operation.go index 4322b626dae1..c177e4cd5da3 100644 --- a/core/services/galleryop/operation.go +++ b/core/services/galleryop/operation.go @@ -91,7 +91,7 @@ type OpStatus struct { } // opStatusWire is the JSON shape used when an OpStatus crosses a process -// boundary (NATS broadcast). The Error field on OpStatus is an `error` +// boundary (a cross-replica broadcast). The Error field on OpStatus is an `error` // interface, which json.Marshal flattens to `{}` because the concrete error // type usually has no exported fields — so a failed install replicated to a // peer frontend would arrive with a nil error and the UI would never surface @@ -165,7 +165,7 @@ func (o *OpStatus) UnmarshalJSON(data []byte) error { return nil } -// OpCacheEvent is the NATS payload broadcast by frontend replicas when an +// OpCacheEvent is the payload broadcast by frontend replicas when an // admin operation is admitted (SubjectGalleryOpStart) or dismissed // (SubjectGalleryOpEnd). Peers merge these into their local OpCache so a // load-balanced /api/operations poll never returns an empty list while a @@ -176,15 +176,15 @@ type OpCacheEvent struct { IsBackend bool `json:"is_backend"` } -// GalleryProgressEvent is the NATS payload for an OpStatus broadcast. It +// GalleryProgressEvent is the payload for an OpStatus broadcast. It // wraps OpStatus with the opID/JobID so subscribers reading the wildcard -// subject don't need to parse it back out of the NATS subject string. +// subject don't need to parse it back out of the subject string. type GalleryProgressEvent struct { JobID string `json:"job_id"` Status *OpStatus `json:"status"` } -// GalleryCancelEvent is the NATS payload for a gallery cancellation. The +// GalleryCancelEvent is the payload for a gallery cancellation. The // local cancellation func may live on a different frontend replica than the // one that received the UI cancel button click; the broadcast subscriber // runs the cancel func on whichever replica registered it. @@ -236,10 +236,14 @@ type OpCache struct { started *xsync.SyncedMap[string, time.Time] // Distributed sync (nil when standalone). - mu sync.RWMutex - nats messaging.MessagingClient - store *distributed.GalleryStore - subs []messaging.Subscription + // + // ONE carrier field, read by both the start/end publishes and by the + // subscriptions Start opens, so this cache cannot publish where its peers + // are not listening. + mu sync.RWMutex + broadcaster messaging.Broadcaster + store *distributed.GalleryStore + subs []messaging.Subscription } func NewOpCache(galleryService *GalleryService) *OpCache { @@ -252,13 +256,17 @@ func NewOpCache(galleryService *GalleryService) *OpCache { } } -// SetMessagingClient enables cross-replica OpCache sync. Once set, Set/ +// SetBroadcaster enables cross-replica OpCache sync on the deployment's +// fan-out carrier, which is PostgreSQL LISTEN/NOTIFY. Once set, Set/ // SetBackend/DeleteUUID publish OpCacheEvent messages that peer OpCaches // merge into their local maps. Call Start after this to subscribe. -func (m *OpCache) SetMessagingClient(nc messaging.MessagingClient) { +// +// messaging.Broadcaster and not the wider client: this cache needs Publish and +// Subscribe and nothing else. +func (m *OpCache) SetBroadcaster(b messaging.Broadcaster) { m.mu.Lock() defer m.mu.Unlock() - m.nats = nc + m.broadcaster = b } // SetGalleryStore enables PostgreSQL-backed OpCache persistence. @@ -273,17 +281,17 @@ func (m *OpCache) SetGalleryStore(s *distributed.GalleryStore) { } // Start hydrates the in-memory maps from PostgreSQL (if a store was wired) -// and subscribes to the broadcast subjects (if NATS was wired). It returns +// and subscribes to the broadcast subjects (if a carrier was wired). It returns // the first subscribe error; hydration errors are logged but non-fatal so // the frontend still comes up. // -// Safe to call exactly once after SetMessagingClient / SetGalleryStore. The +// Safe to call exactly once after SetBroadcaster / SetGalleryStore. The // ctx parameter is reserved for future cancellation — current subscriptions // live for the lifetime of the OpCache and are released by Close. func (m *OpCache) Start(_ context.Context) error { m.mu.RLock() store := m.store - nc := m.nats + nc := m.broadcaster m.mu.RUnlock() if store != nil { @@ -318,7 +326,7 @@ func (m *OpCache) Start(_ context.Context) error { return nil } -// Close drops all NATS subscriptions. Safe to call multiple times. +// Close drops all broadcast subscriptions. Safe to call multiple times. func (m *OpCache) Close() { m.mu.Lock() subs := m.subs @@ -421,7 +429,7 @@ func (m *OpCache) dropReplacedStamp(key, jobID string) { func (m *OpCache) persistAndBroadcastStart(key, value string, isBackend bool) { m.mu.RLock() store := m.store - nc := m.nats + nc := m.broadcaster m.mu.RUnlock() if store != nil { @@ -467,7 +475,7 @@ const ( // // Safe to call for an unknown job ID (no key, no record) and safe to call // twice for the same job (the ring dedupes), which is what makes it usable -// from both the local delete path and the NATS end event. +// from both the local delete path and the broadcast end event. func (m *OpCache) recordTerminal(jobID string, src terminalSource) { if jobID == "" { return @@ -628,7 +636,7 @@ func (m *OpCache) DeleteUUID(uuid string) { return } m.mu.RLock() - nc := m.nats + nc := m.broadcaster m.mu.RUnlock() if nc != nil { if err := nc.Publish(messaging.SubjectGalleryOpEnd, OpCacheEvent{JobID: uuid}); err != nil { diff --git a/core/services/galleryop/service.go b/core/services/galleryop/service.go index 6d6da5d1e404..6d742d6767f5 100644 --- a/core/services/galleryop/service.go +++ b/core/services/galleryop/service.go @@ -31,10 +31,14 @@ type GalleryService struct { cancellations map[string]cancellationActions // Distributed mode (nil when not in distributed mode). - // natsClient is the wider MessagingClient (Publisher + subscribe methods) - // when wired by the distributed startup path; broadcastSubs holds the - // progress + cancel subscriptions opened by SubscribeBroadcasts. - natsClient messaging.MessagingClient + // + // broadcaster is the deployment's fan-out carrier, and it is ONE field on + // purpose: the progress and cancel publishes and the wildcard + // subscriptions SubscribeBroadcasts opens all read it, so this service + // cannot end up publishing on one carrier and listening on another. A + // replica in that state shows every gallery operation it started and none + // of its peers', with no error anywhere. + broadcaster messaging.Broadcaster galleryStore *distributed.GalleryStore broadcastSubs []messaging.Subscription @@ -117,14 +121,18 @@ func (g *GalleryService) ModelArtifactMaterializer() config.ArtifactMaterializer return g.appConfig.ModelArtifactMaterializer } -// SetNATSClient sets the NATS client for distributed progress publishing. -// Accepting the wider MessagingClient (vs. plain Publisher) lets -// SubscribeBroadcasts wire the wildcard subscriptions that keep peer -// replicas' statuses + cancellations in sync. -func (g *GalleryService) SetNATSClient(nc messaging.MessagingClient) { +// SetBroadcaster wires the deployment's fan-out carrier, which is PostgreSQL +// LISTEN/NOTIFY. Both halves of this service's cross-replica sync ride it: the +// progress and cancel publishes, and the wildcard subscriptions +// SubscribeBroadcasts opens. +// +// It takes messaging.Broadcaster, which is Publish plus Subscribe and nothing +// else. Neither half needs request/reply or a queue group, and a wider +// parameter here would let this service acquire one without anyone deciding to. +func (g *GalleryService) SetBroadcaster(b messaging.Broadcaster) { g.Lock() defer g.Unlock() - g.natsClient = nc + g.broadcaster = b } // SetGalleryStore sets the PostgreSQL gallery store for distributed persistence. @@ -178,10 +186,10 @@ func (g *GalleryService) UpdateStatus(s string, op *OpStatus) { } g.statuses[s] = op store := g.galleryStore - nc := g.natsClient + nc := g.broadcaster g.Unlock() - // I/O happens after Unlock. The NATS broadcast loops back into our own + // I/O happens after Unlock. The broadcast loops back into our own // wildcard subscriber (mergeStatus), which would deadlock on this mutex // if we still held it. Holding the lock across a PostgreSQL round-trip // would also stall every concurrent reader on each progress tick. @@ -208,9 +216,14 @@ func (g *GalleryService) UpdateStatus(s string, op *OpStatus) { } } - // Publish progress to NATS in distributed mode. The payload wraps the - // OpStatus with the opID so peer replicas reading the wildcard subject - // don't need to parse it back out of the NATS subject string. + // Broadcast progress in distributed mode. The payload wraps the OpStatus + // with the opID so peer replicas reading the wildcard subject don't need + // to parse it back out of the subject string. + // + // A progress event carries one entry per node, so on a fleet of a few tens + // of workers it outgrows the notification cap and travels as a spilled row + // instead. That is the carrier's ordinary path and not an error: the peer + // receives the same bytes either way. if nc != nil { if err := nc.Publish(messaging.SubjectGalleryProgress(s), GalleryProgressEvent{ JobID: s, @@ -222,11 +235,17 @@ func (g *GalleryService) UpdateStatus(s string, op *OpStatus) { } // publishCacheInvalidate broadcasts a cache invalidation event so peer -// replicas refresh whatever in-memory state mirrors disk. No-op when -// natsClient is not wired (standalone mode). +// replicas refresh whatever in-memory state mirrors disk. No-op when no +// carrier is wired (standalone mode). +// +// An invalidation is not a hint and is never traded away for a cheaper +// publish: a peer that misses one keeps serving from a cache it believes is +// valid, which is the one reading of a missed message this programme forbids. +// It goes out through Publish, which spills a message too large for a +// notification rather than refusing it. func (g *GalleryService) publishCacheInvalidate(subject string, evt messaging.CacheInvalidateEvent) { g.Lock() - nc := g.natsClient + nc := g.broadcaster g.Unlock() if nc == nil { return @@ -242,7 +261,7 @@ func (g *GalleryService) publishCacheInvalidate(subject string, evt messaging.Ca // /models/toggle-state endpoints, which write the YAML and reload only the // local in-memory loader). Peers receive it via OnModelsChanged and refresh // their own ModelConfigLoader so a request load-balanced to any replica sees -// the same config. No-op in standalone mode (no NATS client). +// the same config. No-op in standalone mode (no carrier). // // op is "install" for a create/edit (the element must be (re)loaded from // disk) or "delete" for a removal (the element must be pruned from memory, @@ -262,7 +281,7 @@ func (g *GalleryService) BroadcastModelsChangedRevision(element, op, configRevis } // mergeStatus is the broadcast-side merge: it updates the in-memory map from -// a peer's GalleryProgressEvent without re-publishing to NATS or re-writing +// a peer's GalleryProgressEvent without re-publishing to the carrier or re-writing // to PostgreSQL. UpdateStatus is the local-write entry point and does both; // mergeStatus is what the wildcard subscriber calls. Splitting them avoids // an echo loop (replica publishes → its own subscriber receives → mergeStatus @@ -273,8 +292,22 @@ func (g *GalleryService) mergeStatus(opID string, op *OpStatus) { } g.Lock() defer g.Unlock() + prev := g.statuses[opID] + // A cancellation is terminal and a progress tick is not, and the carrier + // puts no order on the two. The owning replica's last tick is published + // before the admin's cancel and can be DELIVERED after it, on the owner's + // own echo as readily as on a peer; merging it wholesale would clear + // Cancelled and leave the operation reading as still running on that + // replica while every other one shows it stopped. A missed or late message + // must never read as an operation that was not cancelled, so a stale tick + // is dropped rather than merged. A terminal status that carries the + // cancellation still merges, which is how the final "cancelled" message + // arrives. + if prev != nil && prev.Cancelled && !op.Cancelled { + return + } if len(op.Nodes) == 0 { - if prev := g.statuses[opID]; prev != nil && len(prev.Nodes) > 0 { + if prev != nil && len(prev.Nodes) > 0 { op.Nodes = prev.Nodes } } @@ -322,18 +355,48 @@ func (g *GalleryService) UpdateNodeProgress(opID, nodeID string, np NodeProgress } } +// GetStatus returns a COPY of the operation's status, not the stored pointer. +// +// The copy is what makes the lock mean anything. Every caller of this and of +// GetAllStatus only reads what it gets back, but the broadcast subscribers +// mutate the stored OpStatus IN PLACE - applyCancel sets Cancelled on the +// struct a peer's event names, mergeStatus rewrites the fields a peer sent - so +// handing out the pointer let an /api/operations response be marshalled while a +// peer's cancel was being written into it. Returning the pointer under a mutex +// serialises the map lookup and nothing else. +// +// Nodes is shared with the stored status and not deep-copied: UpdateNodeProgress +// replaces that slice rather than writing through it, so a reader holding the +// old header sees a consistent older breakdown rather than a torn one. func (g *GalleryService) GetStatus(s string) *OpStatus { g.Lock() defer g.Unlock() - return g.statuses[s] + status, ok := g.statuses[s] + if !ok || status == nil { + return nil + } + copied := *status + return &copied } +// GetAllStatus returns a snapshot of every operation's status. Same rule as +// GetStatus, and for the same reason: the map and the statuses in it are both +// copied, because this one is handed straight to a JSON encoder while peers' +// broadcasts are still arriving. func (g *GalleryService) GetAllStatus() map[string]*OpStatus { g.Lock() defer g.Unlock() - return g.statuses + snapshot := make(map[string]*OpStatus, len(g.statuses)) + for id, status := range g.statuses { + if status == nil { + continue + } + copied := *status + snapshot[id] = &copied + } + return snapshot } // ReapStaleOperations marks abandoned in-progress operations (pending/ @@ -364,7 +427,7 @@ func (g *GalleryService) ReapStaleOperations(age time.Duration) (int64, error) { } // The database row is only half the picture. GET /models/jobs/ and // /api/operations read the in-memory statuses map, which is populated - // locally and via the NATS progress broadcast and never expires. An op + // locally and via the progress broadcast and never expires. An op // orphaned by a replica that died mid-download therefore kept serving its // last frozen tick (phase=downloading, processed=false, error=none) on // every replica indefinitely, long after the reaper had already given up @@ -430,7 +493,7 @@ func (g *GalleryService) stopOperation(id string, pause bool) error { delete(g.cancellations, id) } - nc := g.natsClient + nc := g.broadcaster store := g.galleryStore if !localExists && nc == nil { @@ -484,7 +547,7 @@ func (g *GalleryService) stopOperation(id string, pause bool) error { // applyCancel is the broadcast-side counterpart to CancelOperation. The // wildcard subscriber calls it when a peer publishes a cancel event: -// run the local cancel func if we have one (no echo via NATS), and reflect +// run the local cancel func if we have one (no echo via the carrier), and reflect // the cancellation in the local statuses map. Idempotent: a replica that // already cancelled this op locally treats the inbound event as a no-op. func (g *GalleryService) applyCancel(id string, pause bool) { @@ -699,7 +762,7 @@ func (g *GalleryService) Start(c context.Context, cl *config.ModelConfigLoader, // the pre-existing operations before live updates start flowing. func (g *GalleryService) SubscribeBroadcasts() error { g.Lock() - nc := g.natsClient + nc := g.broadcaster g.Unlock() if nc == nil { return nil @@ -752,7 +815,7 @@ func (g *GalleryService) SubscribeBroadcasts() error { g.Unlock() if cb != nil { // Run off-goroutine so a slow UpgradeChecker doesn't stall the - // NATS receive loop. Matches the local fire-after-install path. + // carrier's delivery loop. Matches the local fire-after-install path. go cb() } }) diff --git a/core/services/nodes/prefixcache/extractor.go b/core/services/nodes/prefixcache/extractor.go index 65531bafb7b3..e17ed2720371 100644 --- a/core/services/nodes/prefixcache/extractor.go +++ b/core/services/nodes/prefixcache/extractor.go @@ -24,7 +24,7 @@ import ( // limitation, since the cap bounds the chain length for very long prompts. // // xxhash is used (not hash/maphash) because the hash MUST be identical across -// frontend processes: peers exchange these hashes over NATS, and maphash uses a +// frontend processes: peers exchange these hashes on the broadcast carrier, and maphash uses a // per-process random seed that would make peers disagree. func ExtractChain(model, prompt string, cfg Config) []uint64 { if prompt == "" { @@ -38,7 +38,7 @@ func ExtractChain(model, prompt string, cfg Config) []uint64 { // state, so Reset()+Write produces the byte-identical value to a fresh // New()+Write. xxhash seed 0 is stateless, so output is unchanged while we // avoid allocating a Digest per block. The output determinism across - // processes (peers exchange these hashes over NATS) is preserved. + // processes (peers exchange these hashes on the broadcast carrier) is preserved. h := xxhash.New() chain := make([]uint64, 0, depth) prev := salt diff --git a/core/services/nodes/prefixcache/sync.go b/core/services/nodes/prefixcache/sync.go index 421fea7e2ad4..78d28f708e59 100644 --- a/core/services/nodes/prefixcache/sync.go +++ b/core/services/nodes/prefixcache/sync.go @@ -1,34 +1,99 @@ package prefixcache import ( + "fmt" "time" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/xlog" ) -// publisher is the minimal slice of messaging.Client that Sync needs. -type publisher interface { - Publish(subject string, v any) error -} - // Sync wraps an Index, broadcasting new/extended observations to peers and // applying peers' broadcasts. It is the cross-frontend coherence layer. +// +// It holds ONE carrier, and both directions use it: what this replica observes +// goes out on it, and what peers observe comes back in on it through +// SubscribeBroadcasts. Splitting those across two carriers would leave each +// frontend routing on nothing but its own history while every publish and every +// subscribe succeeded, so there is no way to spell that here. type Sync struct { idx Provider - pub publisher + // bus is messaging.Broadcaster, which is Publish plus Subscribe and nothing + // more. This package is reached from the routing hot path and a spec drives + // it with a two-method double; a wider parameter would hand the hot path + // request/reply it must never make. + bus messaging.Broadcaster } -func NewSync(idx Provider, pub publisher) *Sync { return &Sync{idx: idx, pub: pub} } +// NewSync wraps idx and puts its cross-frontend traffic on bus. A nil bus keeps +// the Sync local: it records and answers, and broadcasts nothing. +func NewSync(idx Provider, bus messaging.Broadcaster) *Sync { return &Sync{idx: idx, bus: bus} } + +// SubscribeBroadcasts applies peers' observations and invalidations into the +// wrapped index. ApplyObserve and ApplyInvalidate update only the local index +// and never re-publish, so there is no broadcast loop. +// +// It takes no carrier argument on purpose. The carrier is the one this Sync +// publishes on, read from the same field, which is what makes "this replica +// hears what it would have said" a property of the type rather than of whoever +// wired it. +// +// Returns every subscription it opened; on a partial failure it releases what +// it already opened and returns nothing, so a caller cannot be left holding a +// half-subscribed Sync it believes is whole. +func (s *Sync) SubscribeBroadcasts() ([]messaging.Subscription, error) { + if s.bus == nil { + return nil, nil + } + var subs []messaging.Subscription + release := func() { + for _, sub := range subs { + if err := sub.Unsubscribe(); err != nil { + xlog.Warn("prefixcache: releasing a partial subscription", "error", err) + } + } + } + + observeSub, err := messaging.SubscribeJSON(s.bus, messaging.SubjectPrefixCacheObserve, func(ev messaging.PrefixCacheObserveEvent) { + s.ApplyObserve(ev, time.Now()) + }) + if err != nil { + return nil, fmt.Errorf("prefixcache: subscribing to %s: %w", messaging.SubjectPrefixCacheObserve, err) + } + subs = append(subs, observeSub) + + invalidateSub, err := messaging.SubscribeJSON(s.bus, messaging.SubjectPrefixCacheInvalidate, func(ev messaging.PrefixCacheInvalidateEvent) { + s.ApplyInvalidate(ev) + }) + if err != nil { + release() + return nil, fmt.Errorf("prefixcache: subscribing to %s: %w", messaging.SubjectPrefixCacheInvalidate, err) + } + subs = append(subs, invalidateSub) + return subs, nil +} // Observe records locally and, if new/extended, broadcasts to peers. It returns // whether the local index treated the assignment as new or extended, so Sync // satisfies prefixcache.Provider. +// +// The LOCAL record happens first and is never conditional on the broadcast. A +// hint this replica could not share is still a fact this replica learned, and +// collapsing the two would cost the observing replica its own affinity for the +// very prompts it is already serving. +// +// The broadcast is an ordinary Publish, like every other family on the carrier. +// An observation is bounded by construction: ExtractChain caps a chain at +// Config.MaxDepth blocks, so the event a frontend publishes has a known worst +// case, and core/application refuses to start a deployment whose configured +// depth would push that worst case past what a notification can carry. Nothing +// here drops a message to stay under the cap: a drop would cost peers their +// affinity silently, and the startup refusal says so instead. func (s *Sync) Observe(model string, chain []uint64, key ReplicaKey, now time.Time) bool { changed := s.idx.Observe(model, chain, key, now) - if changed && s.pub != nil { + if changed && s.bus != nil { ev := messaging.PrefixCacheObserveEvent{Model: model, Chain: chain, NodeID: key.NodeID, Replica: key.Replica} - if err := s.pub.Publish(messaging.SubjectPrefixCacheObserve, ev); err != nil { + if err := s.bus.Publish(messaging.SubjectPrefixCacheObserve, ev); err != nil { xlog.Debug("prefixcache: observe publish failed", "error", err) } } @@ -37,7 +102,7 @@ func (s *Sync) Observe(model string, chain []uint64, key ReplicaKey, now time.Ti // Invalidate drops the local entry for one replica and broadcasts to peers. The // local drop is a no-op for models that were never cached (Index.Invalidate does -// not intern a tree). The broadcast is UNCONDITIONAL (when a publisher is +// not intern a tree). The broadcast is UNCONDITIONAL (when a carrier is // configured): the registry chokepoint fires for every replica removal, and a // peer frontend may hold a stale entry for the model even when THIS frontend // never cached it, so gating the broadcast on local-tree existence would drop @@ -45,9 +110,9 @@ func (s *Sync) Observe(model string, chain []uint64, key ReplicaKey, now time.Ti // until their TTL. func (s *Sync) Invalidate(model string, key ReplicaKey) { s.idx.Invalidate(model, key) - if s.pub != nil { + if s.bus != nil { ev := messaging.PrefixCacheInvalidateEvent{Model: model, NodeID: key.NodeID, Replica: key.Replica} - if err := s.pub.Publish(messaging.SubjectPrefixCacheInvalidate, ev); err != nil { + if err := s.bus.Publish(messaging.SubjectPrefixCacheInvalidate, ev); err != nil { xlog.Debug("prefixcache: invalidate publish failed", "error", err) } } @@ -58,9 +123,9 @@ func (s *Sync) Invalidate(model string, key ReplicaKey) { // coherence. A negative Replica on the wire means "all replicas of the node". func (s *Sync) InvalidateNode(model, node string) { s.idx.InvalidateNode(model, node) - if s.pub != nil { + if s.bus != nil { ev := messaging.PrefixCacheInvalidateEvent{Model: model, NodeID: node, Replica: -1} - if err := s.pub.Publish(messaging.SubjectPrefixCacheInvalidate, ev); err != nil { + if err := s.bus.Publish(messaging.SubjectPrefixCacheInvalidate, ev); err != nil { xlog.Debug("prefixcache: invalidate-node publish failed", "error", err) } } diff --git a/core/services/nodes/prefixcache/sync_test.go b/core/services/nodes/prefixcache/sync_test.go index 001454618c9e..87f09a56315c 100644 --- a/core/services/nodes/prefixcache/sync_test.go +++ b/core/services/nodes/prefixcache/sync_test.go @@ -1,6 +1,10 @@ package prefixcache_test import ( + "encoding/json" + "errors" + "math" + "sync" "time" . "github.com/onsi/ginkgo/v2" @@ -10,15 +14,71 @@ import ( "github.com/mudler/LocalAI/core/services/nodes/prefixcache" ) -type fakePub struct{ published []any } +// fakePub is the two-method carrier Sync is given: Publish and Subscribe and +// nothing else, which is the whole of messaging.Broadcaster. It delivers +// synchronously to matching subscribers so a spec can drive one Sync and read +// the other without polling. +type fakePub struct { + mu sync.Mutex + published []any + subjects []string + subs []fakePubSub + publishErr error +} + +type fakePubSub struct { + subject string + handler func([]byte) +} func (f *fakePub) Publish(subject string, v any) error { + f.mu.Lock() f.published = append(f.published, v) + f.subjects = append(f.subjects, subject) + err := f.publishErr + subs := append([]fakePubSub(nil), f.subs...) + f.mu.Unlock() + if err != nil { + return err + } + payload, merr := json.Marshal(v) + if merr != nil { + return merr + } + for _, sub := range subs { + if messaging.SubjectMatches(sub.subject, subject) { + sub.handler(payload) + } + } return nil } +func (f *fakePub) Subscribe(subject string, handler func([]byte)) (messaging.Subscription, error) { + if err := messaging.ValidFilter(subject); err != nil { + return nil, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.subs = append(f.subs, fakePubSub{subject: subject, handler: handler}) + return fakePubSubscription{}, nil +} + +func (f *fakePub) subscribed() []string { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]string, 0, len(f.subs)) + for _, sub := range f.subs { + out = append(out, sub.subject) + } + return out +} + +type fakePubSubscription struct{} + +func (fakePubSubscription) Unsubscribe() error { return nil } + // Sync must satisfy the Provider seam so SmartRouter can hold a single -// prefixcache.Provider that broadcasts via NATS. +// prefixcache.Provider that broadcasts to its peers. var _ prefixcache.Provider = (*prefixcache.Sync)(nil) var _ = Describe("Sync", func() { @@ -116,3 +176,175 @@ var _ = Describe("Sync", func() { Expect(idx.Decide("m", []uint64{3, 4}, cands, t0).HasHot).To(BeFalse()) }) }) + +// The observation family's own rules, and the reason it needs none of its own +// machinery. +// +// The plan for this phase proposed publishing observations through a method +// that REFUSES a message too large for a notification instead of spilling it, +// on the reasoning that a long prompt makes a chain of thousands of entries. +// ExtractChain does not produce one: it caps a chain at Config.MaxDepth blocks, +// and MaxDepth is a constant with no operator knob, so an observation's size +// has a known worst case six times under the cap. A refusal would therefore +// have been a deliberate, silent message drop guarding a condition that cannot +// arise, and the first change to MaxDepth would have turned it into a +// deployment that lost cross-replica affinity while reporting nothing. +// +// So Observe publishes like every other family, and the bound is the thing that +// is pinned: here, that a full-depth chain is what the extractor can produce, +// and in core/application, that a full-depth observation still fits in one +// notification. +var _ = Describe("Sync observation size", func() { + It("caps the chain a full-length prompt produces at MaxDepth", func() { + cfg := prefixcache.DefaultConfig() + // Ten times more prompt than the extractor will hash, so the cap is + // what decides the length and not the prompt. + prompt := make([]byte, cfg.WindowBytes*cfg.MaxDepth*10) + for i := range prompt { + prompt[i] = byte('a' + i%26) + } + + chain := prefixcache.ExtractChain("m", string(prompt), cfg) + + Expect(chain).To(HaveLen(cfg.MaxDepth), + "an observation's worst case is Config.MaxDepth entries; if it is not, nothing bounds what Observe publishes") + }) + + It("publishes a full-depth observation rather than trimming or dropping it", func() { + idx := prefixcache.NewIndex(prefixcache.DefaultConfig()) + pub := &fakePub{} + s := prefixcache.NewSync(idx, pub) + chain := make([]uint64, prefixcache.DefaultConfig().MaxDepth) + for i := range chain { + chain[i] = math.MaxUint64 - uint64(i) + } + + Expect(s.Observe("m", chain, rk("A", 0), t0)).To(BeTrue()) + + Expect(pub.published).To(HaveLen(1)) + ev := pub.published[0].(messaging.PrefixCacheObserveEvent) + Expect(ev.Chain).To(Equal(chain), + "a peer must be able to reconstruct the same prefix, so the chain travels whole") + }) + + // The load-bearing half. A hint this replica could not share is still a + // fact this replica learned, and an Observe that abandoned the local record + // when the carrier said no would cost the observing frontend its own + // affinity for the prompts it is already serving. + It("records locally even when the broadcast fails", func() { + idx := prefixcache.NewIndex(prefixcache.DefaultConfig()) + pub := &fakePub{publishErr: errors.New("carrier refused it")} + s := prefixcache.NewSync(idx, pub) + chain := []uint64{7, 8, 9} + + Expect(s.Observe("m", chain, rk("A", 0), t0)).To(BeTrue()) + + d := s.Decide("m", chain, []prefixcache.ReplicaKey{rk("A", 0)}, t0) + Expect(d.HasHot).To(BeTrue(), "the observing replica must keep its own affinity") + Expect(d.Hot).To(Equal(rk("A", 0))) + }) + + // Invalidations are not hints. A peer that misses one routes to a replica + // that is gone until its TTL, which is why Sync.Invalidate's own comment + // calls the broadcast unconditional. Asserted as the SUBJECT each call + // published on, so a family moved onto the wrong subject fails here rather + // than in a deployment. + DescribeTable("publishes every family on its own subject", + func(act func(*prefixcache.Sync), wantSubject string) { + idx := prefixcache.NewIndex(prefixcache.DefaultConfig()) + pub := &fakePub{} + s := prefixcache.NewSync(idx, pub) + + act(s) + + Expect(pub.subjects).To(ConsistOf(wantSubject)) + }, + Entry("observe", func(s *prefixcache.Sync) { s.Observe("m", []uint64{1}, rk("A", 0), t0) }, + messaging.SubjectPrefixCacheObserve), + Entry("invalidate", func(s *prefixcache.Sync) { s.Invalidate("m", rk("A", 0)) }, + messaging.SubjectPrefixCacheInvalidate), + Entry("invalidate-node", func(s *prefixcache.Sync) { s.InvalidateNode("m", "A") }, + messaging.SubjectPrefixCacheInvalidate), + ) +}) + +// Both directions on ONE carrier. SubscribeBroadcasts takes no carrier +// argument, so a Sync that publishes where its peers are not listening cannot +// be built; these pin that the subscriptions it opens are on the carrier it was +// given and that they carry both families. +var _ = Describe("Sync.SubscribeBroadcasts", func() { + It("subscribes both families on the carrier it publishes to", func() { + pub := &fakePub{} + s := prefixcache.NewSync(prefixcache.NewIndex(prefixcache.DefaultConfig()), pub) + + subs, err := s.SubscribeBroadcasts() + Expect(err).ToNot(HaveOccurred()) + Expect(subs).To(HaveLen(2)) + Expect(pub.subscribed()).To(ConsistOf( + messaging.SubjectPrefixCacheObserve, + messaging.SubjectPrefixCacheInvalidate, + )) + }) + + It("registers nothing at all when there is no carrier", func() { + s := prefixcache.NewSync(prefixcache.NewIndex(prefixcache.DefaultConfig()), nil) + + subs, err := s.SubscribeBroadcasts() + Expect(err).ToNot(HaveOccurred()) + Expect(subs).To(BeEmpty()) + }) + + It("carries a peer's observation into this replica's index", func() { + // Two Syncs, one carrier: exactly the production topology, with the + // publish and the subscribe on the same bus because neither end can + // name a different one. + bus := &fakePub{} + idxA := prefixcache.NewIndex(prefixcache.DefaultConfig()) + idxB := prefixcache.NewIndex(prefixcache.DefaultConfig()) + a := prefixcache.NewSync(idxA, bus) + b := prefixcache.NewSync(idxB, bus) + _, err := b.SubscribeBroadcasts() + Expect(err).ToNot(HaveOccurred()) + + chain := []uint64{11, 22, 33} + a.Observe("m", chain, rk("A", 3), t0) + + d := b.Decide("m", chain, []prefixcache.ReplicaKey{rk("A", 3)}, t0) + Expect(d.HasHot).To(BeTrue(), "the peer replica must learn where the prefix is warm") + Expect(d.Hot).To(Equal(rk("A", 3))) + }) + + It("carries a peer's invalidation, so a removed replica stops being routed to", func() { + bus := &fakePub{} + idxA := prefixcache.NewIndex(prefixcache.DefaultConfig()) + idxB := prefixcache.NewIndex(prefixcache.DefaultConfig()) + a := prefixcache.NewSync(idxA, bus) + b := prefixcache.NewSync(idxB, bus) + _, err := b.SubscribeBroadcasts() + Expect(err).ToNot(HaveOccurred()) + + chain := []uint64{44, 55} + a.Observe("m", chain, rk("A", 0), t0) + Expect(b.Decide("m", chain, []prefixcache.ReplicaKey{rk("A", 0)}, t0).HasHot).To(BeTrue()) + + a.Invalidate("m", rk("A", 0)) + + Expect(b.Decide("m", chain, []prefixcache.ReplicaKey{rk("A", 0)}, t0).HasHot).To(BeFalse(), + "a missed invalidation would leave this replica routing to a replica that is gone") + }) + + It("does not re-broadcast what it applied", func() { + // The echo-loop guard. ApplyObserve and ApplyInvalidate write only the + // local index; a peer that answered a broadcast with a broadcast would + // keep the carrier busy forever. + bus := &fakePub{} + b := prefixcache.NewSync(prefixcache.NewIndex(prefixcache.DefaultConfig()), bus) + _, err := b.SubscribeBroadcasts() + Expect(err).ToNot(HaveOccurred()) + + b.ApplyObserve(messaging.PrefixCacheObserveEvent{Model: "m", Chain: []uint64{1, 2}, NodeID: "A"}, t0) + b.ApplyInvalidate(messaging.PrefixCacheInvalidateEvent{Model: "m", NodeID: "A", Replica: 0}) + + Expect(bus.published).To(BeEmpty()) + }) +}) diff --git a/core/services/nodes/staging_progress.go b/core/services/nodes/staging_progress.go index 0a6ddc50ed60..9b11924cddc1 100644 --- a/core/services/nodes/staging_progress.go +++ b/core/services/nodes/staging_progress.go @@ -30,9 +30,14 @@ const ( // FileComplete, Complete) always publish so peers never miss them. stagingBroadcastInterval = time.Second // stagingRemoteTTL drops a mirrored (remote) op whose last update is older - // than this. NATS pub/sub is fire-and-forget, so a missed Done event would - // otherwise leave a phantom staging row on a peer forever; a live op + // than this. The broadcast carrier is at-most-once, so a missed Done event + // would otherwise leave a phantom staging row on a peer forever; a live op // refreshes its mirror at least every stagingBroadcastInterval. + // + // This is the direction the TTL is allowed to be wrong in, and it is the + // invariant this family has to keep: a missed event ages a mirror out, so + // a staging op that never happened is never invented, and one that is + // still running is re-asserted on the next tick. stagingRemoteTTL = 60 * time.Second ) @@ -51,13 +56,17 @@ type stagingEntry struct { // Used by SmartRouter to publish progress and by /api/operations to surface it. // // In distributed mode each frontend replica runs its own tracker. The replica -// performing a transfer owns the op locally and broadcasts progress over NATS -// (SetPublisher); peers mirror it via ApplyRemote (SubscribeBroadcasts) so a +// performing a transfer owns the op locally and broadcasts progress on the +// deployment's fan-out carrier; peers mirror it via ApplyRemote, so a // /api/operations poll that round-robins onto any replica surfaces the op. +// SetBroadcaster wires both halves at once, and it is one method for a reason +// written there. type StagingTracker struct { - mu sync.RWMutex - active map[string]*stagingEntry - publisher messaging.Publisher + mu sync.RWMutex + active map[string]*stagingEntry + // broadcaster is where this tracker publishes AND where it mirrors from. + // One field, set by one method, so the two cannot name different carriers. + broadcaster messaging.Broadcaster } // StagingProgressEvent is the wire payload a frontend replica broadcasts on @@ -76,19 +85,24 @@ func NewStagingTracker() *StagingTracker { } } -// SetPublisher wires the NATS publisher used to broadcast staging progress to -// peer replicas. No-op publisher (nil) keeps the tracker standalone. -func (t *StagingTracker) SetPublisher(p messaging.Publisher) { +// SetBroadcaster puts this tracker's publishing AND its mirroring of peers onto +// ONE carrier, and returns the mirror subscription for cleanup. A nil +// broadcaster keeps the tracker standalone and registers nothing. +// +// One method rather than a setter and a subscriber, because they were never two +// decisions. A tracker that publishes on one carrier and listens on another +// shows a staging progress bar on the replica performing the transfer and on no +// other, which is the exact symptom SubjectStagingProgress exists to prevent, +// and neither half reports an error while it happens. With one argument that +// deployment cannot be spelled. +func (t *StagingTracker) SetBroadcaster(b messaging.Broadcaster) (messaging.Subscription, error) { t.mu.Lock() - defer t.mu.Unlock() - t.publisher = p -} - -// SubscribeBroadcasts subscribes to peer replicas' staging-progress broadcasts -// and mirrors them into this tracker, so /api/operations on any replica surfaces -// staging ops it did not originate. Returns the subscription for cleanup. -func (t *StagingTracker) SubscribeBroadcasts(nc messaging.MessagingClient) (messaging.Subscription, error) { - return messaging.SubscribeJSON(nc, messaging.SubjectStagingProgressWildcard, func(evt StagingProgressEvent) { + t.broadcaster = b + t.mu.Unlock() + if b == nil { + return nil, nil + } + return messaging.SubscribeJSON(b, messaging.SubjectStagingProgressWildcard, func(evt StagingProgressEvent) { if evt.ModelID == "" { return } @@ -96,9 +110,9 @@ func (t *StagingTracker) SubscribeBroadcasts(nc messaging.MessagingClient) (mess }) } -// publishStaging emits an event to the per-model staging subject. The publisher +// publishStaging emits an event to the per-model staging subject. The carrier // is captured by the caller under the lock and passed in, so publishing happens -// outside the lock (a slow NATS link must not stall the staging copy loop). +// outside the lock (a slow carrier must not stall the staging copy loop). func publishStaging(p messaging.Publisher, evt StagingProgressEvent) { if p == nil { return @@ -121,7 +135,7 @@ func (t *StagingTracker) Start(modelID, nodeName string, totalFiles int) { // lastPub stays zero so the first UpdateFile tick always broadcasts. } t.active[modelID] = e - pub := t.publisher + pub := t.broadcaster snap := e.status t.mu.Unlock() @@ -167,7 +181,7 @@ func (t *StagingTracker) UpdateFile(modelID, fileName string, fileIndex int, byt var snap StagingStatus if time.Since(e.lastPub) >= stagingBroadcastInterval { e.lastPub = time.Now() - pub = t.publisher + pub = t.broadcaster snap = e.status } t.mu.Unlock() @@ -194,7 +208,7 @@ func (t *StagingTracker) FileComplete(modelID string, fileIndex, totalFiles int) s.Speed = "" e.updatedAt = time.Now() e.lastPub = time.Now() - pub := t.publisher + pub := t.broadcaster snap := e.status t.mu.Unlock() @@ -207,7 +221,7 @@ func (t *StagingTracker) Complete(modelID string) { t.mu.Lock() _, ok := t.active[modelID] delete(t.active, modelID) - pub := t.publisher + pub := t.broadcaster t.mu.Unlock() if ok { diff --git a/core/services/nodes/staging_progress_broadcast_test.go b/core/services/nodes/staging_progress_broadcast_test.go index 0f0f0db1ed39..77bfa70e52b5 100644 --- a/core/services/nodes/staging_progress_broadcast_test.go +++ b/core/services/nodes/staging_progress_broadcast_test.go @@ -7,6 +7,7 @@ import ( . "github.com/onsi/gomega" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/testutil" ) // decodeStagingEvents extracts every StagingProgressEvent the fake messaging @@ -33,13 +34,14 @@ var _ = Describe("StagingTracker cross-replica broadcast", func() { It("broadcasts staging progress so a peer replica surfaces an op it did not originate", func() { mc := &fakeMessagingClient{} origin := NewStagingTracker() - origin.SetPublisher(mc) + _, err := origin.SetBroadcaster(mc) + Expect(err).ToNot(HaveOccurred()) origin.Start("model-x", "worker-1", 1) origin.UpdateFile("model-x", "weights.gguf", 1, 5<<30, 10<<30, "100 MiB/s") events := decodeStagingEvents(mc) - Expect(events).ToNot(BeEmpty(), "writes must be broadcast over NATS") + Expect(events).ToNot(BeEmpty(), "writes must be broadcast to peer replicas") Expect(mc.published[0].Subject).To(Equal(messaging.SubjectStagingProgress("model-x"))) // A peer replica that never ran the op merges the broadcast. @@ -58,7 +60,8 @@ var _ = Describe("StagingTracker cross-replica broadcast", func() { It("removes the op from the peer when the origin completes it", func() { mc := &fakeMessagingClient{} origin := NewStagingTracker() - origin.SetPublisher(mc) + _, err := origin.SetBroadcaster(mc) + Expect(err).ToNot(HaveOccurred()) origin.Start("model-x", "worker-1", 1) origin.Complete("model-x") @@ -90,7 +93,7 @@ var _ = Describe("StagingTracker cross-replica broadcast", func() { }) }) - Context("when no publisher is wired (standalone mode)", func() { + Context("when no carrier is wired (standalone mode)", func() { It("does not broadcast", func() { mc := &fakeMessagingClient{} t := NewStagingTracker() @@ -98,6 +101,48 @@ var _ = Describe("StagingTracker cross-replica broadcast", func() { t.UpdateFile("model-x", "weights.gguf", 1, 1<<30, 10<<30, "") Expect(mc.published).To(BeEmpty()) }) + + It("registers no mirror subscription for a nil carrier", func() { + // The negative half of SetBroadcaster's one-carrier promise: a + // standalone tracker must not subscribe either. "Nothing was + // delivered" cannot tell that apart from a live subscription + // nobody published to. + sub, err := NewStagingTracker().SetBroadcaster(nil) + Expect(err).ToNot(HaveOccurred()) + Expect(sub).To(BeNil()) + }) + }) + + // SetBroadcaster is ONE method because publishing and mirroring are one + // decision. These two pin each half separately: a mutation that drops the + // subscribe leaves the publish spec green, and a mutation that drops the + // field assignment leaves the subscribe green, so a single spec over both + // would let either half go missing. + Context("both halves ride the carrier it is given", func() { + It("mirrors a peer's broadcast that arrives on the same carrier", func() { + bus := testutil.NewFakeBus() + peer := NewStagingTracker() + _, err := peer.SetBroadcaster(bus) + Expect(err).ToNot(HaveOccurred()) + + Expect(bus.Publish(messaging.SubjectStagingProgress("model-y"), StagingProgressEvent{ + ModelID: "model-y", + Status: &StagingStatus{ModelID: "model-y", NodeName: "worker-2"}, + })).To(Succeed()) + + Expect(peer.GetAll()).To(HaveKey("model-y")) + }) + + It("publishes onto the same carrier it mirrors from", func() { + bus := testutil.NewFakeBus() + origin := NewStagingTracker() + _, err := origin.SetBroadcaster(bus) + Expect(err).ToNot(HaveOccurred()) + + origin.Start("model-z", "worker-3", 1) + + Expect(bus.PublishCount(messaging.SubjectStagingProgress("model-z"))).To(Equal(1)) + }) }) }) diff --git a/core/services/nodes/staging_progress_pg_test.go b/core/services/nodes/staging_progress_pg_test.go new file mode 100644 index 000000000000..a1718c29023b --- /dev/null +++ b/core/services/nodes/staging_progress_pg_test.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: MIT + +package nodes + +import ( + "context" + "runtime" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// Staging progress on the real carrier, between two trackers on two LISTEN +// connections. +// +// One tracker talking to itself proves nothing here: the whole reason this +// family exists is that a /api/operations poll round-robins onto a replica that +// did not perform the transfer. An in-memory double cannot fail the way a +// carrier fails, and it cannot tell a tracker that publishes and listens on two +// different carriers apart from one that does not, which is the defect +// SetBroadcaster is shaped to exclude. +var _ = Describe("staging progress across two replicas on the broadcast carrier", func() { + var trackerA, trackerB *StagingTracker + + BeforeEach(func() { + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } + ctx := context.Background() + db, dsn := testutil.SetupTestDBWithDSN() + Expect(pgbus.Migrate(ctx, db)).To(Succeed()) + + newTracker := func() *StagingTracker { + bus, err := pgbus.New(ctx, pgbus.Config{DSN: dsn, DB: db}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(bus.Close) + t := NewStagingTracker() + sub, err := t.SetBroadcaster(bus) + Expect(err).ToNot(HaveOccurred()) + Expect(sub).ToNot(BeNil()) + return t + } + trackerA, trackerB = newTracker(), newTracker() + }) + + It("mirrors a peer's start, progress and completion, in that order", func() { + trackerA.Start("model-a", "worker-1", 2) + + Eventually(func() map[string]StagingStatus { return trackerB.GetAll() }, 20*time.Second). + Should(HaveKey("model-a")) + Expect(trackerB.GetAll()["model-a"].NodeName).To(Equal("worker-1")) + + trackerA.UpdateFile("model-a", "weights.gguf", 1, 5<<30, 10<<30, "100 MiB/s") + + Eventually(func() string { + if s := trackerB.Get("model-a"); s != nil { + return s.FileName + } + return "" + }, 20*time.Second).Should(Equal("weights.gguf")) + + // The Done event is what stops a peer showing a transfer that + // finished. A tracker that only ever received the start would pass + // every assertion above and leave the row on screen until the TTL. + trackerA.Complete("model-a") + + Eventually(func() map[string]StagingStatus { return trackerB.GetAll() }, 20*time.Second). + ShouldNot(HaveKey("model-a")) + }) + + It("leaves the mirroring replica's own operations alone", func() { + trackerB.Start("model-b", "worker-local", 1) + trackerA.Start("model-a", "worker-1", 1) + + Eventually(func() map[string]StagingStatus { return trackerB.GetAll() }, 20*time.Second). + Should(HaveKey("model-a")) + + own := trackerB.Get("model-b") + Expect(own).ToNot(BeNil(), "a replica must keep the transfer it is itself performing") + Expect(own.NodeName).To(Equal("worker-local")) + }) +}) diff --git a/core/services/pgbus/bus.go b/core/services/pgbus/bus.go index a2c89faf00b6..1e0ff6ddabbf 100644 --- a/core/services/pgbus/bus.go +++ b/core/services/pgbus/bus.go @@ -331,6 +331,53 @@ func (b *Bus) Close() { }) } +// inlineNotification encodes what Publish puts on the wire when a message fits +// in the notification itself, and returns it together with the caller's encoded +// payload so the spill path does not marshal a second time. +// +// It is the ONE place the wire form of an inline broadcast is built. Publish +// measures what it returns and FitsInline asks about the same bytes, so a +// caller that has proved its worst case fits cannot be proved wrong later by a +// change to the envelope's keys. +func inlineNotification(subject string, data any) ([]byte, json.RawMessage, error) { + payload, err := json.Marshal(data) + if err != nil { + return nil, nil, fmt.Errorf("pgbus: encoding a broadcast on %q: %w", subject, err) + } + encoded, err := json.Marshal(notification{Subject: subject, Data: payload}) + if err != nil { + return nil, nil, fmt.Errorf("pgbus: encoding the notification for %q: %w", subject, err) + } + return encoded, payload, nil +} + +// FitsInline reports whether Publish would carry this message in the +// notification itself rather than writing it to a row and notifying the id. +// +// It is a PREDICATE and never an action: nothing here refuses, drops or +// truncates a message, and no caller may use it to decide not to publish. The +// spill path is correct for every family on this carrier, and a message that +// does not fit costs a row and a SELECT, never its contents. +// +// It exists for the one question that is worth asking BEFORE a message is ever +// published: can a family whose payload has a known upper bound put that bound +// on the wrong side of the cap? A family whose worst case fits is a family that +// never spills, and a configuration change that moves the bound past the cap +// can then be refused at startup instead of turning every request on the +// inference path into a table write nobody notices. See +// requirePrefixCacheFitsInline in core/application. +// +// It shares its size decision with Publish rather than restating one: the same +// encoder, the same comparison against the same constant. A second comparison +// would be a second constant in disguise. +func FitsInline(subject string, data any) (bool, error) { + encoded, _, err := inlineNotification(subject, data) + if err != nil { + return false, err + } + return len(encoded) < maxNotifyPayloadBytes, nil +} + // Publish fans a message out to every subscriber of the subject on every // replica, including this one. func (b *Bus) Publish(subject string, data any) error { @@ -338,14 +385,9 @@ func (b *Bus) Publish(subject string, data any) error { if err != nil { return err } - payload, err := json.Marshal(data) + encoded, payload, err := inlineNotification(subject, data) if err != nil { - return fmt.Errorf("pgbus: encoding a broadcast on %q: %w", subject, err) - } - - encoded, err := json.Marshal(notification{Subject: subject, Data: payload}) - if err != nil { - return fmt.Errorf("pgbus: encoding the notification for %q: %w", subject, err) + return err } // The one size decision in this package. Several subjects on this carrier // exceed the cap in normal operation (a job result carries a whole LLM diff --git a/core/services/pgbus/spill_test.go b/core/services/pgbus/spill_test.go index 801b5f6e5f3d..9bdc851e8aad 100644 --- a/core/services/pgbus/spill_test.go +++ b/core/services/pgbus/spill_test.go @@ -127,6 +127,47 @@ var _ = Describe("broadcasts too large for a notification", func() { Expect(spilledRows()).To(Equal(int64(1))) }) + // FitsInline and Publish, asserted TOGETHER on the same payload. + // + // FitsInline exists so a family with a known worst case can prove at + // startup that it never spills, and that proof is worth nothing if the + // predicate and the publisher can disagree: a family told "you fit" that + // then spills on every request is exactly the invisible cost the check was + // added to prevent. Both rows below state an ABSOLUTE size and assert both + // halves, so a second size decision anywhere splits them. + DescribeTable("answers the same size question Publish answers", + func(total int, wantFits bool, wantRows int64) { + data := payloadEncodingTo(subject, total) + + fits, err := pgbus.FitsInline(subject, data) + Expect(err).ToNot(HaveOccurred()) + Expect(fits).To(Equal(wantFits)) + + _ = deliver(data) + + Expect(spilledRows()).To(Equal(wantRows)) + }, + Entry("one byte under the cap", notifyCap-1, true, int64(0)), + Entry("exactly at the cap", notifyCap, false, int64(1)), + ) + + It("refuses to call a megabyte inline", func() { + // Absolute, not derived from the cap, for the same reason the spill row + // above is: this must stay red-for-the-right-reason under a mutated + // constant. + fits, err := pgbus.FitsInline(subject, map[string]string{"k": strings.Repeat("a", 1<<20)}) + Expect(err).ToNot(HaveOccurred()) + Expect(fits).To(BeFalse()) + }) + + It("reports the encoding failure rather than guessing at a size", func() { + // A payload that cannot be marshalled has no size, and answering + // "false" would let a caller read an encoder bug as a payload that is + // merely large. + _, err := pgbus.FitsInline(subject, make(chan int)) + Expect(err).To(HaveOccurred()) + }) + It("stores the caller's payload in the spill row, not the envelope", func() { data := payloadEncodingTo(subject, notifyCap) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index d51cc24a5d8f..18ce8798dc4a 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -41,7 +41,7 @@ Each model gets its own gRPC backend process, so a single worker can serve multi - Each frontend replica holds **one extra PostgreSQL session** beyond its connection pool, pinned for the life of the process, and creates a `bus_messages` table. Both belong to the broadcast carrier that is replacing NATS for cross-replica fan-out; it already carries the four `state.*.delta` families (see [Cross-replica in-memory state](#cross-replica-in-memory-state)). Size `max_connections` for one additional session per frontend replica. - That session reports an `application_name` of `localai_pgbus_`, so `SELECT count(*) FROM pg_stat_activity WHERE application_name LIKE 'localai_pgbus_%'` counts the replicas currently listening. If the carrier loses its session it redials and re-registers on its own; a broadcast published while it was down is not replayed, which is why nothing that must survive a gap is carried by a broadcast alone. - `bus_messages` holds only broadcasts too large for a PostgreSQL notification, and every replica retires rows older than ten minutes. The table is a spill buffer, not a log: it is not a place to read past events from. -- **NATS** server - used for agent-worker coordination and the frontend's own cross-replica events. **Serve-backend workers do not connect to it at all**: every verb they take, and file staging with it, is an HTTP route on the worker's tunnel. Set no `LOCALAI_NATS_URL` on a `local-ai worker`. The frontend and any `local-ai agent-worker` still need one. +- **NATS** server - used for agent-worker coordination. The frontend's own cross-replica events do not use it: they travel on the PostgreSQL the deployment already runs. **Serve-backend workers do not connect to it at all**: every verb they take, and file staging with it, is an HTTP route on the worker's tunnel. Set no `LOCALAI_NATS_URL` on a `local-ai worker`. The frontend and any `local-ai agent-worker` still need one. - All services must be on the same network (or reachable via configured URLs) ## Quick Start with Docker Compose @@ -136,6 +136,47 @@ Several features keep state in a frontend's process memory and surface it over t | Agent tasks | `state.agent-tasks.delta` and `state.agent-tasks..delta` | | Open Responses metadata | `state.responses-metadata.delta` | +### Cross-replica caches + +A frontend also keeps caches that live for the life of the process rather than +for the life of a request: which gallery operations are in flight and how far +along they are, which admin operations have been admitted, which model files are +being staged onto a worker, and which replica already holds the KV/prefix cache +for a prompt. Each of those is kept current on every replica by a broadcast, and +**every one of them is on PostgreSQL**. Nothing in this table uses NATS. + +| Family | Subject | What a peer does with it | +|--------|---------|--------------------------| +| Gallery progress | `gallery..progress` | Merges the status so `/api/operations` answers the same on any replica | +| Gallery cancel | `gallery..cancel` | Stops the install, on whichever replica is running it | +| Operation cache admit | `gallery.opcache.start` | Learns that an operation was admitted, and whether it is a backend install | +| Operation cache dismiss | `gallery.opcache.end` | Drops the operation from its own map | +| Model cache invalidation | `cache.invalidate.models` | Reloads the model config from disk, or prunes a deleted one | +| Backend cache invalidation | `cache.invalidate.backends` | Refreshes its upgrade-available cache | +| Staging progress | `staging..progress` | Mirrors a transfer it did not perform, so the progress bar does not flicker | +| Prefix-cache observation | `prefixcache.observe` | Learns which replica already holds the prefix for a prompt | +| Prefix-cache invalidation | `prefixcache.invalidate` | Stops routing to a replica that is gone | + +**An invalidation that does not arrive must never read as a cache that is +valid.** The two `cache.invalidate.*` families and `prefixcache.invalidate` are +therefore published like any other broadcast: one too large for a notification +is written to `bus_messages` and read back by the peer, never dropped to save +the write. A missed staging event ages the peer's mirrored row out after a +minute rather than inventing a transfer, for the same reason. + +**Gallery progress spills, routinely.** A progress event carries one entry per +worker, so on a fleet of a few tens of nodes it is past the 8000-byte cap on +every tick and travels as a row. That is the ordinary path, not an error. + +**A prefix-cache observation never spills.** It carries one hash per prefix +block, and the extractor caps a chain at 64 blocks, so the largest observation a +frontend can publish is a few kilobytes and fits in the notification itself. +That bound is checked at startup: a build that raised the cap past what a +notification carries would put a table write and a read-back on the inference +path for every request whose prefix changed, so the frontend refuses to start +and says so rather than running slowly and quietly. Nothing here drops an +observation to stay under the cap. + ### Job and agent streams across replicas The same carrier moves the traffic whose subscriber is an open HTTP response rather than a cache: a job's progress stream, its result, its cancel, an agent's SSE events, an agent cancel and an Open Responses cancel. @@ -1638,7 +1679,7 @@ Notes: |---|---|---| | **Discovery** | Automatic via libp2p token | Self-registration to frontend URL | | **State storage** | In-memory / ledger | PostgreSQL | -| **Coordination** | Gossip protocol | The worker's own tunnel for serve-backend work; NATS for agent workers and cross-replica frontend events | +| **Coordination** | Gossip protocol | The worker's own tunnel for serve-backend work; PostgreSQL `LISTEN`/`NOTIFY` for cross-replica frontend events; NATS for agent workers | | **Node management** | Automatic | REST API + WebUI | | **Health monitoring** | Peer heartbeats | Centralized HealthMonitor | | **Backend management** | Manual per node | Dynamic via the worker's `backend.install` control route | @@ -1746,12 +1787,12 @@ queue depth, precise KV utilization, latency, and fairness signals. ## Roadmap: Routing and Caching Enhancements -The scheduling algorithm supports **prefix-cache-aware** routing: bias each request toward the replica that already holds the relevant KV/prefix cache (multi-turn conversations and shared system prompts), so backends reuse cache instead of recomputing it. A router-side radix tree maps prompt-prefix hashes to nodes, with longest-prefix match, a load guard that preserves round-robin behavior under imbalance, and NATS sync across frontends. It is purely a routing-layer hint (no backend changes) and never routes worse than round-robin. +The scheduling algorithm supports **prefix-cache-aware** routing: bias each request toward the replica that already holds the relevant KV/prefix cache (multi-turn conversations and shared system prompts), so backends reuse cache instead of recomputing it. A router-side radix tree maps prompt-prefix hashes to nodes, with longest-prefix match, a load guard that preserves round-robin behavior under imbalance, and cross-frontend sync on the PostgreSQL broadcast carrier. It is purely a routing-layer hint (no backend changes) and never routes worse than round-robin. -When the load guard must route away from a warm replica, the frontend emits a forced-disturb event. These events and the reset sent after a successful pressure-triggered scale-up are broadcast over NATS, so the rolling autoscale threshold is cluster-wide rather than per frontend. The Prometheus counter `localai_prefix_cache_forced_disturb_total{model="..."}` records events at their originating frontend; sum it across frontend replicas to inspect cluster pressure without counting the NATS copies. +When the load guard must route away from a warm replica, the frontend emits a forced-disturb event. These events and the reset sent after a successful pressure-triggered scale-up are broadcast over PostgreSQL, so the rolling autoscale threshold is cluster-wide rather than per frontend. The Prometheus counter `localai_prefix_cache_forced_disturb_total{model="..."}` records events at their originating frontend; sum it across frontend replicas to inspect cluster pressure without counting the broadcast copies. Backends can report exact KV-cache residency on the `prefixcache.residency` -NATS subject. The JSON event contract is: +subject of the PostgreSQL broadcast carrier. The JSON event contract is: ```json { diff --git a/tests/e2e/distributed/gallery_distributed_test.go b/tests/e2e/distributed/gallery_distributed_test.go index 100063ed52c2..b6bf9a6e2edc 100644 --- a/tests/e2e/distributed/gallery_distributed_test.go +++ b/tests/e2e/distributed/gallery_distributed_test.go @@ -68,8 +68,20 @@ var _ = Describe("Gallery Distributed", Label("Distributed"), func() { }) }) - Context("NATS progress updates", func() { - It("should publish progress updates via NATS", func() { + // The gallery families ride the broadcast carrier, not NATS. + // + // These used to publish and subscribe on infra.NC, which asserted that NATS + // delivers to itself and nothing about this deployment: they would have + // stayed green through the whole migration while the gallery service had + // already moved. Two carriers on the deployment's own database is the shape + // a fleet has, and it is the shape that fails when one end moves and the + // other does not. + // + // No flush, unlike the NATS version: pgbus.Subscribe has already issued its + // LISTEN by the time it returns, and Subscribers() counts only live + // handlers, so there is no window to wait out. + Context("gallery progress on the broadcast carrier", func() { + It("delivers a peer replica's progress updates", func() { op := &distributed.GalleryOperationRecord{ GalleryElementName: "whisper-large", OpType: "model_install", @@ -77,35 +89,28 @@ var _ = Describe("Gallery Distributed", Label("Distributed"), func() { } Expect(galleryStore.Create(op)).To(Succeed()) - // Subscribe to gallery progress + publisher, subscriber := infra.Bus(), infra.Bus() + var received atomic.Int32 - sub, err := infra.NC.Subscribe(messaging.SubjectGalleryProgress(op.ID), func(data []byte) { + sub, err := subscriber.Subscribe(messaging.SubjectGalleryProgress(op.ID), func([]byte) { received.Add(1) }) Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - FlushNATS(infra.NC) + defer func() { Expect(sub.Unsubscribe()).To(Succeed()) }() - // Publish progress events - Expect(infra.NC.Publish(messaging.SubjectGalleryProgress(op.ID), map[string]any{ - "op_id": op.ID, - "progress": 0.25, - "message": "25%", + Expect(publisher.Publish(messaging.SubjectGalleryProgress(op.ID), map[string]any{ + "op_id": op.ID, "progress": 0.25, "message": "25%", })).To(Succeed()) - - Expect(infra.NC.Publish(messaging.SubjectGalleryProgress(op.ID), map[string]any{ - "op_id": op.ID, - "progress": 0.50, - "message": "50%", + Expect(publisher.Publish(messaging.SubjectGalleryProgress(op.ID), map[string]any{ + "op_id": op.ID, "progress": 0.50, "message": "50%", })).To(Succeed()) - Eventually(func() int32 { return received.Load() }, "5s").Should(Equal(int32(2))) + Eventually(func() int32 { return received.Load() }, "20s").Should(Equal(int32(2))) }) }) - Context("NATS cancel across instances", func() { - It("should cancel operation across instances via NATS", func() { + Context("gallery cancel on the broadcast carrier", func() { + It("delivers a cancel to the replica holding the operation", func() { op := &distributed.GalleryOperationRecord{ GalleryElementName: "cancel-model", OpType: "model_install", @@ -114,24 +119,23 @@ var _ = Describe("Gallery Distributed", Label("Distributed"), func() { } Expect(galleryStore.Create(op)).To(Succeed()) - // Simulate another instance listening for cancel + publisher, subscriber := infra.Bus(), infra.Bus() + var cancelReceived atomic.Bool - sub, err := infra.NC.Subscribe(messaging.SubjectGalleryCancel(op.ID), func(data []byte) { + sub, err := subscriber.Subscribe(messaging.SubjectGalleryCancel(op.ID), func([]byte) { cancelReceived.Store(true) }) Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - FlushNATS(infra.NC) + defer func() { Expect(sub.Unsubscribe()).To(Succeed()) }() - // Send cancel from this instance - Expect(infra.NC.Publish(messaging.SubjectGalleryCancel(op.ID), map[string]string{ + Expect(publisher.Publish(messaging.SubjectGalleryCancel(op.ID), map[string]string{ "op_id": op.ID, })).To(Succeed()) - Eventually(func() bool { return cancelReceived.Load() }, "5s").Should(BeTrue()) + Eventually(func() bool { return cancelReceived.Load() }, "20s").Should(BeTrue()) - // Mark cancelled in the store + // The row is what survives a replica that was not listening. The + // broadcast is the hint to go and look at it. Expect(galleryStore.Cancel(op.ID)).To(Succeed()) updated, _ := galleryStore.Get(op.ID) Expect(updated.Status).To(Equal("cancelled")) From eaf4eb6d5bb83f572fe5424378ec13d08f61099d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 4 Sep 2026 03:51:06 +0000 Subject: [PATCH 096/153] refactor(distributed): delete MessagingClient and shrink the NATS client to fan-out Nothing in the tree publishes, subscribes, queue-subscribes or requests through the MessagingClient interface any more, so it is deleted rather than shrunk to Broadcaster: two exported names for one method set in one package is an invitation for the next author to pick whichever the surrounding file already imported. $ grep -rn 'messaging\.MessagingClient' --include='*.go' . core/services/syncstate/syncstate.go:54: // It is messaging.Broadcaster rather than messaging.MessagingClient because (one hit, a comment; no live referent. The naive grep in the plan also matches prose and the local test type names fakeMessagingClient and countingMessagingClient, so it can never be empty.) *messaging.Client is shrunk to exactly Broadcaster plus its own lifecycle. QueueSubscribe, QueueSubscribeReply, SubscribeReply, Request, Conn and the package helpers QueueSubscribeJSON and RequestJSON go with it; none had a production caller. Deleting the methods rather than only the call sites is what makes putting a family back on this carrier a build error instead of a line that compiles, publishes successfully, and is delivered onto a carrier the deployment is being taken off. Conn is in that list because while it existed every other name was one c.Conn().X() away; the flush-and-verdict that its real consumers needed is now ConfirmRoundTrip, which keeps the NATS JWT permission specs armed. The client, its options and its TLS plumbing are NOT deleted, and both processes stay on the bus. agent..cancel is the one fan-out family that could not move: its only subscriber is the agent worker, which has no database and cannot join the PostgreSQL carrier at all, so a cancel published there would reach no worker and be reported as sent. The frontend passes the client to newFanoutBridges as its cancelCarrier and the worker subscribes on it, so --nats-url stays required on agent-worker. Both go with the tunnel cancel verb. The struct field is renamed Nats -> CancelCarrier to say what it is for, and agentpool loses the messaging.Publisher it held only to be non-nil: it never published on it, and it was gating whether a frontend runs agents distributed or in an in-process pool. Retiring the bus would have flipped every replica back to the in-process pool silently. The gate now reads the agent store, which is the dependency the mode actually requires. Also deletes four subject builders with no production publisher (SubjectFineTuneProgress, SubjectFineTuneCancel, SubjectCacheInvalidateSkills, SubjectCacheInvalidateCollection), the queue and request/reply halves of the shared test double, and the e2e specs that were their only callers. Every surviving subject is now pinned to its exact literal, because a subject is a cross-version wire format and a rename that looks internal stops half a fleet hearing the other half. Docs: distributed-mode.md and cli-reference.md no longer claim NATS carries the agent-worker job subjects, the frontend's cross-replica events, or an agent worker's real work. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/application.go | 1 - core/application/cache_fanout_wiring.go | 3 +- core/application/distributed.go | 91 +++++++---- core/application/distributed_test.go | 21 +-- core/application/fanout_wiring_test.go | 42 +++++ core/cli/agent_worker.go | 27 ++-- core/cli/agent_worker_test.go | 51 ++++++ .../http/endpoints/localai/edit_model_test.go | 15 -- core/services/agentpool/agent_pool.go | 52 ++++--- .../agentpool/dispatch_chat_internal_test.go | 53 +++++++ core/services/agentpool/task_sync_test.go | 2 +- .../artifact_progress_coalescer_test.go | 20 --- .../galleryop/distributed_sync_test.go | 22 +-- .../galleryop/model_revision_delete_test.go | 15 -- core/services/messaging/client.go | 145 ++++++------------ core/services/messaging/interfaces.go | 67 ++++---- core/services/messaging/interfaces_test.go | 106 +++++++++++++ core/services/messaging/subjects.go | 24 +-- core/services/messaging/subjects_wire_test.go | 85 ++++++++++ .../nodes/control_worker_fake_test.go | 11 ++ core/services/nodes/unloader_test.go | 45 ++---- core/services/testutil/fakebus.go | 52 +++---- docs/content/features/distributed-mode.md | 14 +- docs/content/reference/cli-reference.md | 2 +- .../distributed/agent_native_executor_test.go | 2 +- .../distributed/finetune_distributed_test.go | 63 ++------ tests/e2e/distributed/foundation_test.go | 83 ++++------ tests/e2e/distributed/nats_jwt_test.go | 20 +-- .../e2e/distributed/nats_queue_reply_test.go | 77 ---------- .../distributed/skills_distributed_test.go | 74 ++------- tests/e2e/distributed/testhelpers_test.go | 7 +- 31 files changed, 677 insertions(+), 615 deletions(-) create mode 100644 core/services/messaging/interfaces_test.go delete mode 100644 tests/e2e/distributed/nats_queue_reply_test.go diff --git a/core/application/application.go b/core/application/application.go index 6a4f99f32a7a..4d14adbde5d5 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -636,7 +636,6 @@ func (a *Application) StartAgentPool() { if d.DistStores != nil && d.DistStores.Skills != nil { opts.SkillStore = d.DistStores.Skills } - opts.NATSClient = d.Nats opts.EventBridge = d.AgentBridge opts.AgentStore = d.AgentStore } diff --git a/core/application/cache_fanout_wiring.go b/core/application/cache_fanout_wiring.go index 29330a49bb9f..a351a8efcadd 100644 --- a/core/application/cache_fanout_wiring.go +++ b/core/application/cache_fanout_wiring.go @@ -19,7 +19,8 @@ import ( // one function here, and none of them by a call site naming a carrier. // // Two are methods on DistributedServices and take NO carrier at all, because -// their call sites hold the whole struct and could otherwise reach for ds.Nats. +// their call sites hold the whole struct and could otherwise reach for the +// cancel carrier. // The other two run inside initDistributed before that struct exists, so they // take the CONCRETE *pgbus.Bus rather than messaging.Broadcaster. // diff --git a/core/application/distributed.go b/core/application/distributed.go index 92f11fee535a..ee543713fd57 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -32,28 +32,49 @@ import ( // DistributedServices holds all services initialized for distributed mode. type DistributedServices struct { - Nats *messaging.Client - Store storage.ObjectStore - Registry *nodes.NodeRegistry - Router *nodes.SmartRouter - Health *nodes.HealthMonitor - Reconciler *nodes.ReplicaReconciler - JobStore *jobs.JobStore - Dispatcher *jobs.Dispatcher - AgentStore *agents.AgentStore - AgentBridge *agents.EventBridge - DistStores *distributed.Stores - FileMgr *storage.FileManager - FileStager nodes.FileStager - ModelAdapter *nodes.ModelRouterAdapter - Unloader *nodes.RemoteUnloaderAdapter - ModelCleanup *nodes.ModelCleanupService + // CancelCarrier is the NATS connection, and it is here for exactly one + // family: agent..cancel. + // + // Every other family this deployment fans out on moved to Bus below. That + // one could not, and the reason is structural rather than incidental: its + // only subscriber is the agent WORKER, which has no database and so cannot + // join a carrier that rides PostgreSQL. A cancel published on Bus would be + // published where no worker listens, succeed, and be reported as sent. + // + // It is named CancelCarrier and not Nats so that reaching for it to carry + // anything else has to be a decision. The field used to be Nats, and + // *messaging.Client satisfies messaging.Broadcaster, so any adopter that + // took it instead of Broadcast() compiled, started, published, and was + // delivered onto the carrier this deployment is being taken off, with no + // error anywhere. The client's queue and request/reply halves are deleted + // now, so the smaller mistakes are build failures; this name is what is + // left to make the remaining one visible. + // + // It goes when a cancel rides the worker's tunnel as a control verb. + CancelCarrier *messaging.Client + Store storage.ObjectStore + Registry *nodes.NodeRegistry + Router *nodes.SmartRouter + Health *nodes.HealthMonitor + Reconciler *nodes.ReplicaReconciler + JobStore *jobs.JobStore + Dispatcher *jobs.Dispatcher + AgentStore *agents.AgentStore + AgentBridge *agents.EventBridge + DistStores *distributed.Stores + FileMgr *storage.FileManager + FileStager nodes.FileStager + ModelAdapter *nodes.ModelRouterAdapter + Unloader *nodes.RemoteUnloaderAdapter + ModelCleanup *nodes.ModelCleanupService // Bus is the deployment's fan-out carrier, riding the auth database's - // PostgreSQL rather than a message broker. Nothing publishes on it and - // nothing subscribes yet; it is built at boot because its DSN has exactly - // one legitimate source and that has to be settled once, here, rather than - // invented by whichever call site is migrated onto it first. + // PostgreSQL rather than a message broker. Every cross-replica family the + // frontend broadcasts is on it now, with the single exception named above. + // Adopters reach it through Broadcast() rather than through this field, so + // that "which carrier does this family travel on" is decided once instead + // of at every adopter; its DSN likewise has exactly one legitimate source, + // settled in newBroadcastBus rather than invented per call site. Bus *pgbus.Bus // Cluster is the replica-membership registry: which frontend replicas are @@ -134,10 +155,10 @@ func (ds *DistributedServices) Shutdown() { if closer, ok := ds.Store.(io.Closer); ok { closer.Close() } - // AgentBridge has no Close method — its NATS subscriptions are cleaned up - // when the NATS client is closed below. - if ds.Nats != nil { - ds.Nats.Close() + // AgentBridge has no Close method: the cancel subscription it holds is + // cleaned up when the carrier below is closed. + if ds.CancelCarrier != nil { + ds.CancelCarrier.Close() } // The broadcast carrier holds a PostgreSQL session pinned for the life // of the process, plus the goroutine parked on it. A replica that @@ -181,7 +202,12 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } xlog.Info("Distributed instance", "id", cfg.Distributed.InstanceID) - // Connect to NATS + // Connect to the cancel carrier. + // + // This is the ONE bus connection a frontend replica still opens, and it + // carries one family: agent..cancel, whose subscriber is an agent + // worker that has no database and cannot read the broadcast carrier opened + // below. Everything else a replica fans out travels on that one. natsAuth := cfg.Distributed.NatsAuthConfig() if natsAuth.RequireAuth && (natsAuth.ServiceUserJWT == "" || natsAuth.ServiceUserSeed == "") { return nil, fmt.Errorf("LOCALAI_NATS_REQUIRE_AUTH requires LOCALAI_NATS_SERVICE_JWT and LOCALAI_NATS_SERVICE_SEED") @@ -191,9 +217,9 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade if err != nil { return nil, fmt.Errorf("connecting to NATS: %w", err) } - xlog.Info("Connected to NATS", "url", sanitize.URL(cfg.Distributed.NatsURL)) + xlog.Info("Connected to the agent cancel carrier", "url", sanitize.URL(cfg.Distributed.NatsURL)) - // Ensure NATS is closed if any subsequent initialization step fails. + // Ensure the carrier is closed if any subsequent initialization step fails. success := false defer func() { if !success { @@ -694,7 +720,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade success = true ds := &DistributedServices{ - Nats: natsClient, + CancelCarrier: natsClient, Store: store, Registry: registry, Router: router, @@ -762,10 +788,11 @@ func requireBroadcastCarrier(ds *DistributedServices) error { // service, the agent-task setter (twice, on two startup paths), the per-user // services manager and the Open Responses store. Every one of them takes a // messaging.Broadcaster, and *messaging.Client satisfies that interface too, so -// a site left holding ds.Nats compiles, starts, publishes and is delivered - -// onto a carrier the deployment is being taken off. Nothing would fail until -// NATS went away. Collapsing the choice to one function makes it a fact a spec -// can pin, which five scattered field reads were not. +// a site left holding ds.CancelCarrier compiles, starts, publishes and is +// delivered - onto a carrier only agent workers read, and only for cancels. +// Nothing would fail until NATS went away. Collapsing the choice to one +// function makes it a fact a spec can pin, which five scattered field reads +// were not. // // The return is the interface and not *pgbus.Bus on purpose: handing a nil // *pgbus.Bus to an adopter would produce a non-nil interface wrapping a nil diff --git a/core/application/distributed_test.go b/core/application/distributed_test.go index edcf7115a534..d3eb8167a344 100644 --- a/core/application/distributed_test.go +++ b/core/application/distributed_test.go @@ -118,12 +118,12 @@ var _ = Describe("shutting the distributed services down", func() { // service, the agent-task setter on two startup paths, the per-user services // manager and the Open Responses store. Every one of them takes a // messaging.Broadcaster, which *messaging.Client satisfies too, so a site left -// holding ds.Nats compiled, started, published and was delivered onto the -// carrier the deployment is being taken off, and nothing failed until NATS did. -// Collapsing the choice into one function is what makes it a fact these specs -// can hold. +// holding the struct's NATS field compiled, started, published and was +// delivered onto a carrier only agent workers read, and nothing failed until +// NATS did. Collapsing the choice into one function is what makes it a fact +// these specs can hold. var _ = Describe("handing the broadcast carrier to its adopters", func() { - It("returns the carrier the deployment opened and never the NATS client", func() { + It("returns the carrier the deployment opened and never the cancel carrier", func() { db, dsn := testutil.SetupTestDBWithDSN() cfg := &config.ApplicationConfig{} cfg.Auth.DatabaseURL = dsn @@ -131,10 +131,13 @@ var _ = Describe("handing the broadcast carrier to its adopters", func() { Expect(err).ToNot(HaveOccurred()) DeferCleanup(bus.Close) - // A NATS client is present on the struct, exactly as it is in a real - // deployment for as long as the request/reply and queue halves survive. - // Identity, not "is a Broadcaster": both fields satisfy that interface. - ds := &DistributedServices{Nats: &messaging.Client{}, Bus: bus} + // The cancel carrier is present on the struct, exactly as it is in a + // real deployment: agent..cancel is the one family that could not + // move, because its only subscriber is an agent worker and a worker has + // no database. Identity, not "is a Broadcaster": both fields satisfy + // that interface, which is the whole reason a field read could pick the + // wrong one and stay green. + ds := &DistributedServices{CancelCarrier: &messaging.Client{}, Bus: bus} Expect(ds.Broadcast()).To(BeIdenticalTo(messaging.Broadcaster(bus))) }) diff --git a/core/application/fanout_wiring_test.go b/core/application/fanout_wiring_test.go index 7472a0340202..6c3a77c3a2aa 100644 --- a/core/application/fanout_wiring_test.go +++ b/core/application/fanout_wiring_test.go @@ -101,6 +101,48 @@ var _ = Describe("wiring the job and agent fan-out bridges", func() { }, "20s").Should(Equal("completed")) }) + // S1b. WHICH carrier the cancel family rides, asserted by receipt on both + // carriers rather than by reading the argument list. + // + // The nil refusal above only says a carrier was passed. Passing bus for it + // - one token's difference at the one call site, and the natural edit for + // anyone finishing the migration - satisfies that refusal, compiles, and + // publishes successfully onto the PostgreSQL carrier, where the agent + // worker that has to act on the cancel is not and cannot be: it has no + // database. Every unit suite in agents and in application stays green and + // every cancel of a worker-run agent is lost while CancelExecution returns + // nil. + // + // So this asserts the cancel ARRIVES on the worker's carrier and, in the + // same spec, that it does NOT arrive on a peer replica's broadcast carrier. + // The negative half is the load-bearing one: the positive half alone passes + // for a bridge wired to both. + It("publishes agent cancels on the worker's carrier and not on the broadcast carrier", func() { + workerCarrier := testutil.NewFakeBus() + _, bridge, _, err := newFanoutBridges(busA, workerCarrier, jobStore, agentStore, db, "replica-1") + Expect(err).ToNot(HaveOccurred()) + + onBroadcast := make(chan []byte, 4) + _, err = busB.Subscribe(messaging.SubjectAgentCancelWildcard, func(data []byte) { onBroadcast <- data }) + Expect(err).ToNot(HaveOccurred()) + + onWorker := make(chan []byte, 4) + _, err = workerCarrier.Subscribe(messaging.SubjectAgentCancelWildcard, func(data []byte) { onWorker <- data }) + Expect(err).ToNot(HaveOccurred()) + + Expect(bridge.CancelExecution("a1", "u1", "msg-1")).To(Succeed()) + + // Expect and not Eventually: the worker's carrier here is the shared + // in-memory double, which delivers synchronously inside Publish, so by + // the time CancelExecution has returned the handler has already run. An + // Eventually would turn a mis-wiring into a timeout, which reports as + // slowness rather than as the wiring fact it is. + Expect(onWorker).To(Receive(), + "the agent cancel did not reach the worker's carrier; a worker has no database and cannot read the broadcast carrier, so this cancel reached nobody and was reported as sent") + Consistently(onBroadcast, "2s").ShouldNot(Receive(), + "the agent cancel was published on the broadcast carrier, where no agent worker is or can be subscribed") + }) + // S2. The observable persister writes what a peer broadcast, which it can // only do if it was started AND is on the same carrier AND its filter has // the right number of tokens. diff --git a/core/cli/agent_worker.go b/core/cli/agent_worker.go index 09f8c20bb2f1..161e67b89571 100644 --- a/core/cli/agent_worker.go +++ b/core/cli/agent_worker.go @@ -38,9 +38,15 @@ import ( // // It also holds one tunnel to the frontend, so the frontend can reach its // control verbs by RPC without the worker opening an inbound port. No verb the -// frontend addresses to THIS worker travels on the bus any more. The tunnel is -// still an ADDITION rather than a replacement: --nats-url is required, because -// every job and every fan-out event does. +// frontend addresses to THIS worker travels on the bus any more. +// +// --nats-url is still required, and for one thing only: agent..cancel. +// That family runs the other way, from a frontend replica to whichever worker +// holds the execution, and it could not move to the broadcast carrier because +// that carrier rides PostgreSQL and this process has no database. A worker that +// came up without a bus would register, serve, run agents and ignore every +// cancel, with nothing in the deployment reporting it. The flag goes when a +// cancel rides the tunnel as a control verb. // // Usage: // @@ -141,9 +147,9 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { go regClient.HeartbeatLoop(shutdownCtx, nodeID, heartbeatInterval, func() map[string]any { return map[string]any{} }) - // Resolve NATS credentials with precedence: explicit env override, then - // frontend-minted (auto-refreshed before expiry), then service fallback. - // Each static source must supply JWT and seed together. + // Resolve the cancel carrier's credentials with precedence: explicit env + // override, then frontend-minted (auto-refreshed before expiry), then + // service fallback. Each static source must supply JWT and seed together. natsTLS := messaging.TLSFiles{CA: cmd.NatsTLSCA, Cert: cmd.NatsTLSCert, Key: cmd.NatsTLSKey} var natsOpts []messaging.Option switch { @@ -228,9 +234,12 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { } }() - // The cancel listener is the ONE thing still on the bus here: a cancel is a - // broadcast to every replica and every worker, because the replica holding - // the run is not the one the cancel request lands on. + // The cancel listener is the ONE thing still on the bus here, and the only + // reason this process dialled one. A cancel is a broadcast to every replica + // and every worker, because the replica holding the run is not the one the + // cancel request lands on, and this worker cannot join the carrier the rest + // of the deployment fans out on: that carrier is the auth database, and an + // agent worker has no database access at all. cancelSub, err := eventBridge.StartCancelListener() if err != nil { xlog.Warn("Failed to start cancel listener", "error", err) diff --git a/core/cli/agent_worker_test.go b/core/cli/agent_worker_test.go index f941caf44ca0..7d2617d6ca07 100644 --- a/core/cli/agent_worker_test.go +++ b/core/cli/agent_worker_test.go @@ -6,9 +6,11 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "os" "strings" "time" + "github.com/alecthomas/kong" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -144,3 +146,52 @@ var _ = Describe("The agent worker's control-plane wiring", func() { Entry("mcp ci run", workerctl.PathMCPCIRun), ) }) + +// The last reason an agent worker dials a message bus, pinned so that removing +// it is a decision rather than an accident. +// +// Every verb a frontend addresses to THIS worker now arrives on the tunnel it +// dials, and the two queue groups it used to join are gone. One family is left, +// in the other direction: agent..cancel. Its publisher is a frontend +// replica and its ONLY subscriber is this process, and a worker has no database +// and so cannot join the PostgreSQL carrier every other family moved to. A +// worker that came up without a bus would register, serve, run agents, and +// ignore every cancel, returning nothing to say so - the cancel would be +// published, would succeed, and would reach nobody. +// +// So --nats-url stays required here, and it is required for this and for +// nothing else. When a cancel rides the tunnel as a control verb, this spec is +// what has to be deleted for the flag to become optional, and deleting it is +// then the visible half of that change. +var _ = Describe("The agent worker's remaining bus requirement", func() { + parse := func(args ...string) error { + // kong resolves env: tags from the process environment, and a + // LOCALAI_NATS_URL that is SET BUT EMPTY satisfies a required flag. + // Left in place, this spec would pass on a developer's shell and on + // nothing else. + if prior, had := os.LookupEnv("LOCALAI_NATS_URL"); had { + Expect(os.Unsetenv("LOCALAI_NATS_URL")).To(Succeed()) + DeferCleanup(func() { _ = os.Setenv("LOCALAI_NATS_URL", prior) }) + } + var cli struct { + AgentWorker AgentWorkerCMD `cmd:""` + } + parser, err := kong.New(&cli) + Expect(err).ToNot(HaveOccurred()) + _, err = parser.Parse(append([]string{"agent-worker"}, args...)) + return err + } + + It("refuses to start without a bus to hear cancels on", func() { + // Refused at parse time and not at first use. A worker that started + // and only failed to subscribe would already have registered itself as + // available to run agents nobody can cancel. + Expect(parse("--register-to", "http://frontend:8080")). + To(MatchError(ContainSubstring("--nats-url")), + "the agent worker started with no bus: every cancel of an agent it runs would be published to nobody and reported as sent") + }) + + It("parses once the bus is named", func() { + Expect(parse("--register-to", "http://frontend:8080", "--nats-url", "nats://bus:4222")).To(Succeed()) + }) +}) diff --git a/core/http/endpoints/localai/edit_model_test.go b/core/http/endpoints/localai/edit_model_test.go index 31f73e749712..d5d5461b4499 100644 --- a/core/http/endpoints/localai/edit_model_test.go +++ b/core/http/endpoints/localai/edit_model_test.go @@ -10,7 +10,6 @@ import ( "net/http/httptest" "os" "path/filepath" - "time" "github.com/labstack/echo/v4" "github.com/mudler/LocalAI/core/config" @@ -46,20 +45,6 @@ func (c *endpointRecordingClient) Publish(subject string, data any) error { func (*endpointRecordingClient) Subscribe(string, func([]byte)) (messaging.Subscription, error) { return &endpointSubscription{}, nil } -func (*endpointRecordingClient) QueueSubscribe(string, string, func([]byte)) (messaging.Subscription, error) { - return &endpointSubscription{}, nil -} -func (*endpointRecordingClient) QueueSubscribeReply(string, string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return &endpointSubscription{}, nil -} -func (*endpointRecordingClient) SubscribeReply(string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return &endpointSubscription{}, nil -} -func (*endpointRecordingClient) Request(string, []byte, time.Duration) ([]byte, error) { - return nil, nil -} -func (*endpointRecordingClient) IsConnected() bool { return true } -func (*endpointRecordingClient) Close() {} // testRenderer is a simple renderer for tests that returns JSON type testRenderer struct{} diff --git a/core/services/agentpool/agent_pool.go b/core/services/agentpool/agent_pool.go index 39803b0a2c15..0c515cef601e 100644 --- a/core/services/agentpool/agent_pool.go +++ b/core/services/agentpool/agent_pool.go @@ -19,7 +19,6 @@ import ( "github.com/mudler/LocalAI/core/services/agents" "github.com/mudler/LocalAI/core/services/distributed" "github.com/mudler/LocalAI/core/services/jobs" - "github.com/mudler/LocalAI/core/services/messaging" skillsManager "github.com/mudler/LocalAI/core/services/skills" "github.com/mudler/LocalAGI/core/agent" @@ -68,9 +67,17 @@ type localAGICore struct { actionsConfig map[string]string } -// distributedBridge connects to the NATS-based distributed agent system. +// distributedBridge holds what this service needs to run agents across a +// distributed deployment. +// +// There is no carrier here. There used to be: a messaging.Publisher named +// natsClient, which nothing ever published on. Its only job was to be non-nil, +// standing in for "this deployment is distributed", and it was the last hold a +// frontend's agent pool had on a message bus. That made the mode a deployment +// runs its agents in depend on whether a bus connection happened to be handed +// in, so retiring the bus would have flipped every frontend replica back to the +// in-process pool silently, on a code path with no error and no log line. type distributedBridge struct { - natsClient messaging.Publisher // NATS client for distributed agent execution agentStore *agents.AgentStore // PostgreSQL agent config store eventBridge AgentEventBridge // Event bridge for SSE + persistence skillStore *distributed.SkillStore // PostgreSQL skill metadata (distributed mode) @@ -124,7 +131,6 @@ type AgentConfigStore interface { type AgentPoolOptions struct { AuthDB *gorm.DB SkillStore *distributed.SkillStore - NATSClient messaging.Publisher EventBridge AgentEventBridge AgentStore *agents.AgentStore } @@ -141,9 +147,6 @@ func NewAgentPoolService(appConfig *config.ApplicationConfig, opts ...AgentPoolO if o.SkillStore != nil { svc.distributed.skillStore = o.SkillStore } - if o.NATSClient != nil { - svc.distributed.natsClient = o.NATSClient - } if o.EventBridge != nil { svc.distributed.eventBridge = o.EventBridge } @@ -176,7 +179,7 @@ func (s *AgentPoolService) Start(ctx context.Context) error { // Distributed mode: the frontend enqueues claims and agent workers execute // them. No LocalAGI pool, no collections, no skills service, all stateless. - if s.distributed.natsClient != nil { + if s.runsDistributed() { return s.startDistributed(ctx, apiURL, apiKey) } @@ -246,7 +249,7 @@ func (s *AgentPoolService) startDistributed(ctx context.Context, apiURL, apiKey // It needs DB access to list configs, update LastRunAt and write the claim // rows, because the worker has no database. The advisory lock ensures only one // frontend instance runs the scheduler. - if s.users.authDB != nil && s.distributed.natsClient != nil && s.distributed.agentStore != nil { + if s.users.authDB != nil && s.distributed.agentStore != nil { var schedulerOpts []agents.AgentSchedulerOpt if s.distributed.skillStore != nil { schedulerOpts = append(schedulerOpts, agents.WithSchedulerSkillProvider(s.buildSkillProvider())) @@ -391,12 +394,6 @@ func (s *AgentPoolService) Pool() *state.AgentPool { return s.localAGI.pool } -// SetNATSClient sets the NATS client for distributed agent execution. -// Deprecated: prefer passing NATSClient via AgentPoolOptions at construction time. -func (s *AgentPoolService) SetNATSClient(nc messaging.Publisher) { - s.distributed.natsClient = nc -} - // SetEventBridge sets the event bridge for distributed SSE + persistence. // Deprecated: prefer passing EventBridge via AgentPoolOptions at construction time. func (s *AgentPoolService) SetEventBridge(eb AgentEventBridge) { @@ -409,9 +406,28 @@ func (s *AgentPoolService) SetAgentStore(store *agents.AgentStore) { s.distributed.agentStore = store } -// Agent execution in distributed mode is handled by the dedicated agent-worker process -// using the NATSDispatcher from core/services/agents/dispatcher.go. -// The frontend only enqueues chat claims via dispatchChat(). +// runsDistributed reports whether this service runs agents across the +// deployment rather than in an in-process pool. +// +// The agent STORE is the condition, and it is the condition because it is what +// the mode actually requires: distributed mode reads and writes every agent +// config through it, its config backend is built from it, and the background +// scheduler cannot write a claim row without it. A deployment that has one runs +// agents distributed; a deployment that does not cannot, whatever else it was +// handed. +// +// It replaced a nil-check on a message-bus connection that nothing published +// on. That check gave the same answer for the wrong reason, and would have kept +// giving it right up until the bus was retired, at which point every frontend +// would have quietly started running agents in-process against a database full +// of distributed state. +// +// Agent execution in that mode is handled by the dedicated agent-worker +// process, which runs what a frontend replica claims and hands it over that +// worker's tunnel. The frontend only enqueues chat claims via dispatchChat(). +func (s *AgentPoolService) runsDistributed() bool { + return s.distributed.agentStore != nil +} // --- Agent CRUD --- diff --git a/core/services/agentpool/dispatch_chat_internal_test.go b/core/services/agentpool/dispatch_chat_internal_test.go index 0130d1c4c7c3..f144a0a8c502 100644 --- a/core/services/agentpool/dispatch_chat_internal_test.go +++ b/core/services/agentpool/dispatch_chat_internal_test.go @@ -6,12 +6,26 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gorm.io/gorm" + "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/agents" + "github.com/mudler/LocalAI/core/services/distributed" "github.com/mudler/LocalAI/core/services/jobs" "github.com/mudler/LocalAI/core/services/testutil" ) +// noopEventBridge stands in for the real bridge in the mode-selection specs +// below, which need every optional dependency present EXCEPT the one under +// test. It publishes nowhere on purpose: nothing here reads what it sent. +type noopEventBridge struct{} + +func (noopEventBridge) PublishMessage(_, _, _, _, _ string) error { return nil } +func (noopEventBridge) PublishStatus(_, _, _ string) error { return nil } +func (noopEventBridge) PublishStreamEvent(_, _ string, _ map[string]any) error { return nil } +func (noopEventBridge) RegisterCancel(_ string, _ context.CancelFunc) {} +func (noopEventBridge) DeregisterCancel(_ string) {} + // The third producer of a claim row, pinned separately from the other two. // // A rule stated at three call sites and pinned at two is how the third comes to @@ -74,3 +88,42 @@ var _ = Describe("Dispatching an agent chat in distributed mode", func() { Expect(n).To(BeZero()) }) }) + +// Which mode this service runs agents in, pinned against the dependency the +// mode actually requires rather than against whichever one happened to be +// non-nil. +// +// The gate used to be a nil-check on a message-bus connection that nothing ever +// published on. It gave the right answer for the wrong reason, and it would +// have gone on giving it until the bus was retired: at that moment every +// frontend replica would have started running agents in an in-process pool, +// against a database full of distributed agent state, with no error and no log +// line to say the mode had changed. The bus field is gone, so that particular +// mistake is no longer expressible; this is what keeps the replacement from +// drifting onto another incidental dependency. +// +// Each It below supplies EVERY other optional dependency and withholds only the +// store, which is what makes it discriminating: a gate moved to the auth DB, to +// the skill store or to the event bridge passes a spec that only checks the +// store's presence, and fails these. +var _ = Describe("choosing how a deployment runs its agents", func() { + It("runs them distributed when there is an agent store", func() { + svc, err := NewAgentPoolService(&config.ApplicationConfig{}, AgentPoolOptions{ + AgentStore: &agents.AgentStore{}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(svc.runsDistributed()).To(BeTrue(), + "a deployment with an agent store must run agents distributed; running them in-process would serve every replica its own view of state the store owns") + }) + + It("runs them in process when there is no agent store, however much else is wired", func() { + svc, err := NewAgentPoolService(&config.ApplicationConfig{}, AgentPoolOptions{ + AuthDB: &gorm.DB{}, + SkillStore: &distributed.SkillStore{}, + EventBridge: noopEventBridge{}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(svc.runsDistributed()).To(BeFalse(), + "the mode was decided by something other than the agent store; distributed mode reads and writes every agent config through that store and cannot run without it") + }) +}) diff --git a/core/services/agentpool/task_sync_test.go b/core/services/agentpool/task_sync_test.go index 15bb4c92fb8b..ba13e62e7428 100644 --- a/core/services/agentpool/task_sync_test.go +++ b/core/services/agentpool/task_sync_test.go @@ -155,7 +155,7 @@ var _ = Describe("AgentJobService task cross-replica sync", func() { // newTaskSyncServiceForTenant builds a per-user AgentJobService the way // UserServicesManager.GetJobs does: a user id, then the shared bus. Each // service gets its own task file so nothing converges except over the bus. -func newTaskSyncServiceForTenant(bus messaging.MessagingClient, userID string) *AgentJobService { +func newTaskSyncServiceForTenant(bus messaging.Broadcaster, userID string) *AgentJobService { tmpDir := GinkgoT().TempDir() sysState := &system.SystemState{} sysState.Model.ModelsPath = tmpDir diff --git a/core/services/galleryop/artifact_progress_coalescer_test.go b/core/services/galleryop/artifact_progress_coalescer_test.go index f26ed79d3eba..aa10b4366d5b 100644 --- a/core/services/galleryop/artifact_progress_coalescer_test.go +++ b/core/services/galleryop/artifact_progress_coalescer_test.go @@ -93,26 +93,6 @@ func (c *recordingProgressClient) Subscribe(string, func([]byte)) (messaging.Sub return nil, nil } -func (c *recordingProgressClient) QueueSubscribe(string, string, func([]byte)) (messaging.Subscription, error) { - return nil, nil -} - -func (c *recordingProgressClient) QueueSubscribeReply(string, string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return nil, nil -} - -func (c *recordingProgressClient) SubscribeReply(string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return nil, nil -} - -func (c *recordingProgressClient) Request(string, []byte, time.Duration) ([]byte, error) { - return nil, nil -} - -func (c *recordingProgressClient) IsConnected() bool { return true } - -func (c *recordingProgressClient) Close() {} - func (c *recordingProgressClient) Updates() []*OpStatus { c.mu.Lock() defer c.mu.Unlock() diff --git a/core/services/galleryop/distributed_sync_test.go b/core/services/galleryop/distributed_sync_test.go index 3572c81c4809..3828bccff6d8 100644 --- a/core/services/galleryop/distributed_sync_test.go +++ b/core/services/galleryop/distributed_sync_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "sync" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -17,7 +16,7 @@ import ( "github.com/mudler/LocalAI/core/services/testutil" ) -// fakeBus is an in-memory MessagingClient that delivers each published +// fakeBus is an in-memory messaging.Broadcaster that delivers each published // message synchronously to every registered subscriber whose subject filter // matches, using messaging.SubjectMatches so this double and the carrier agree // on what a wildcard filter means. @@ -78,25 +77,6 @@ func (b *fakeBus) Subscribe(subject string, handler func([]byte)) (messaging.Sub return &fakeBusSubscription{bus: b, subRef: sub}, nil } -func (b *fakeBus) QueueSubscribe(subject, _ string, handler func([]byte)) (messaging.Subscription, error) { - return b.Subscribe(subject, handler) -} - -func (b *fakeBus) QueueSubscribeReply(string, string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return &fakeBusSubscription{bus: b}, nil -} - -func (b *fakeBus) SubscribeReply(string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return &fakeBusSubscription{bus: b}, nil -} - -func (b *fakeBus) Request(string, []byte, time.Duration) ([]byte, error) { - return nil, nil -} - -func (b *fakeBus) IsConnected() bool { return true } -func (b *fakeBus) Close() {} - var _ = Describe("OpStatus JSON wire format", func() { It("round-trips a non-nil Error through Marshal/Unmarshal as a string", func() { original := &galleryop.OpStatus{ diff --git a/core/services/galleryop/model_revision_delete_test.go b/core/services/galleryop/model_revision_delete_test.go index 3daf9342f945..be86bc275961 100644 --- a/core/services/galleryop/model_revision_delete_test.go +++ b/core/services/galleryop/model_revision_delete_test.go @@ -5,7 +5,6 @@ import ( "errors" "os" "path/filepath" - "time" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" @@ -50,20 +49,6 @@ func (c *countingMessagingClient) Publish(subject string, _ any) error { func (c *countingMessagingClient) Subscribe(string, func([]byte)) (messaging.Subscription, error) { return nil, nil } -func (c *countingMessagingClient) QueueSubscribe(string, string, func([]byte)) (messaging.Subscription, error) { - return nil, nil -} -func (c *countingMessagingClient) QueueSubscribeReply(string, string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return nil, nil -} -func (c *countingMessagingClient) SubscribeReply(string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return nil, nil -} -func (c *countingMessagingClient) Request(string, []byte, time.Duration) ([]byte, error) { - return nil, nil -} -func (c *countingMessagingClient) IsConnected() bool { return true } -func (c *countingMessagingClient) Close() {} func (m *realDeletingManager) DeleteModel(name string) error { if err := gallery.DeleteModelFromSystem(m.state, name); err != nil { diff --git a/core/services/messaging/client.go b/core/services/messaging/client.go index 7450d8b17651..64c9b9968834 100644 --- a/core/services/messaging/client.go +++ b/core/services/messaging/client.go @@ -18,7 +18,22 @@ import ( // subscription was rejected (e.g. by JWT permissions) before returning to the caller. const subscribeConfirmTimeout = 5 * time.Second -// Client wraps a NATS connection and provides helpers for pub/sub and queue subscriptions. +// Client is a NATS connection, and it is the carrier for exactly one family. +// +// agent..cancel is the one fan-out family that did not move to the +// PostgreSQL carrier. Its only subscriber is the agent WORKER, which has no +// database and cannot join that carrier at all, so a cancel published there +// would reach no worker while reporting that it was sent. Both ends of that +// family still dial this client, which is why it, its connect options and its +// TLS plumbing are all still here. +// +// Everything else it used to carry is gone, and so are the methods that carried +// it: queue subscriptions became a claim on the job store, and request/reply +// became a streaming control RPC on the tunnel each worker dials. Deleting the +// METHODS rather than only the call sites is what makes putting a family back +// on this carrier a build error, instead of a line that compiles, publishes +// successfully, and is delivered onto a carrier the deployment is being taken +// off. type Client struct { conn *nats.Conn mu sync.RWMutex @@ -122,8 +137,8 @@ func New(url string, opts ...Option) (*Client, error) { // OnReconnect registers a callback invoked after the NATS connection is // re-established. It is consumed via an optional interface type-assertion -// (interface{ OnReconnect(func()) }) rather than being added to MessagingClient, -// so the messaging abstraction stays minimal and standalone/test clients are not +// (interface{ OnReconnect(func()) }) rather than being added to Broadcaster, so +// the messaging abstraction stays minimal and standalone/test clients are not // forced to implement reconnect semantics. A nil callback is ignored. func (c *Client) OnReconnect(cb func()) { if cb == nil { @@ -165,16 +180,6 @@ func (c *Client) Subscribe(subject string, handler func([]byte)) (Subscription, }) } -// QueueSubscribe creates a queue subscription. Within the same queue group, -// only one subscriber receives each message (load-balanced). -func (c *Client) QueueSubscribe(subject, queue string, handler func([]byte)) (Subscription, error) { - return c.confirmSubscription(subject, func(conn *nats.Conn) (*nats.Subscription, error) { - return conn.QueueSubscribe(subject, queue, func(msg *nats.Msg) { - handler(msg.Data) - }) - }) -} - // confirmSubscription creates a subscription via mk and forces a server // round-trip so that a permissions violation — which NATS otherwise reports // only asynchronously — is returned to the caller synchronously. The server @@ -219,91 +224,41 @@ func (c *Client) confirmSubscription(subject string, mk func(*nats.Conn) (*nats. return sub, nil } -// Request sends a request and waits for a reply (request-reply pattern). -// Returns the raw reply data. -func (c *Client) Request(subject string, data []byte, timeout time.Duration) ([]byte, error) { +// ConfirmRoundTrip forces a round trip to the server and returns whatever the +// server pushed back asynchronously, so that a refusal becomes an error a +// caller holds rather than a line in a log. +// +// It replaces a Conn() accessor that handed out the raw *nats.Conn. That +// accessor was the hole in this type's method set: every half deleted above is +// still one call away on a *nats.Conn, so a family could be put back on this +// carrier through it without a single build error, which is the whole thing the +// deletions are for. +// +// What it does is the publish-side twin of confirmSubscription. NATS reports a +// permission violation asynchronously and does NOT close the connection, so a +// denied publish is indistinguishable from an accepted one until something +// round-trips and reads the connection's last error. A flush that fails is +// returned as-is: the caller could not reach the server at all, which is a +// different fact from the server refusing it, and neither is evidence about any +// node. +// +// No production path calls it. The carrier's production users publish cancels +// and act on the delivery, not on the verdict; what needs the verdict is +// pkg/natsauth's permission grants, which are asserted against a real enforcing +// server and would otherwise be asserted against nothing, since an allow list +// that is EMPTY means unrestricted in NATS and a spec that only checks +// IsConnected cannot tell a granted publish from a denied one. +func (c *Client) ConfirmRoundTrip(timeout time.Duration) error { c.mu.RLock() - defer c.mu.RUnlock() - msg, err := c.conn.Request(subject, data, timeout) - if err != nil { - return nil, fmt.Errorf("request to %s: %w", subject, err) - } - return msg.Data, nil -} - -// SubscribeReply creates a subscription that supports replying to requests. -// The handler receives the raw request data and the reply subject. -func (c *Client) SubscribeReply(subject string, handler func(data []byte, reply func([]byte))) (Subscription, error) { - return c.confirmSubscription(subject, func(conn *nats.Conn) (*nats.Subscription, error) { - return conn.Subscribe(subject, func(msg *nats.Msg) { - handler(msg.Data, func(replyData []byte) { - if msg.Reply != "" { - if err := msg.Respond(replyData); err != nil { - xlog.Warn("Failed to send NATS reply", "subject", subject, "error", err) - } - } - }) - }) - }) -} - -// QueueSubscribeReply creates a queue subscription that supports replying to requests. -// Load-balanced across subscribers in the same queue group, with request-reply support. -func (c *Client) QueueSubscribeReply(subject, queue string, handler func(data []byte, reply func([]byte))) (Subscription, error) { - return c.confirmSubscription(subject, func(conn *nats.Conn) (*nats.Subscription, error) { - return conn.QueueSubscribe(subject, queue, func(msg *nats.Msg) { - handler(msg.Data, func(replyData []byte) { - if msg.Reply != "" { - if err := msg.Respond(replyData); err != nil { - xlog.Warn("Failed to send NATS reply", "subject", subject, "error", err) - } - } - }) - }) - }) -} - -// QueueSubscribeJSON creates a queue subscription that automatically unmarshals JSON messages. -// Invalid JSON messages are logged and skipped. -func QueueSubscribeJSON[T any](c MessagingClient, subject, queue string, handler func(T)) (Subscription, error) { - return c.QueueSubscribe(subject, queue, func(data []byte) { - var evt T - if err := json.Unmarshal(data, &evt); err != nil { - xlog.Warn("Failed to unmarshal NATS message", "subject", subject, "error", err) - return - } - handler(evt) - }) -} - -// RequestJSON sends a JSON request-reply via NATS, marshaling the request and -// unmarshaling the reply. This eliminates the repeated marshal/request/unmarshal -// boilerplate across all NATS request-reply call sites. -func RequestJSON[Req, Reply any](c MessagingClient, subject string, req Req, timeout time.Duration) (*Reply, error) { - data, err := json.Marshal(req) - if err != nil { - return nil, fmt.Errorf("marshaling request: %w", err) - } - replyData, err := c.Request(subject, data, timeout) - if err != nil { - return nil, fmt.Errorf("NATS request to %s: %w", subject, err) + conn := c.conn + c.mu.RUnlock() + if conn == nil { + return fmt.Errorf("confirming a round trip: nil NATS connection") } - var reply Reply - if err := json.Unmarshal(replyData, &reply); err != nil { - return nil, fmt.Errorf("unmarshaling reply from %s: %w", subject, err) + if err := conn.FlushTimeout(timeout); err != nil { + return fmt.Errorf("round trip to the NATS server: %w", err) } - return &reply, nil -} - -// Conn returns the underlying NATS connection for advanced usage. -// -// Deprecated: Prefer using the MessagingClient interface methods (Publish, Subscribe, etc.) -// instead of accessing the raw NATS connection. This method couples callers to the -// concrete Client type and bypasses the abstraction layer. -func (c *Client) Conn() *nats.Conn { - c.mu.RLock() - defer c.mu.RUnlock() - return c.conn + return conn.LastError() } // IsConnected returns true if the client is currently connected to a NATS server. diff --git a/core/services/messaging/interfaces.go b/core/services/messaging/interfaces.go index a3fdbdbbb49c..980927c02156 100644 --- a/core/services/messaging/interfaces.go +++ b/core/services/messaging/interfaces.go @@ -1,62 +1,47 @@ package messaging -import "time" - -// Publisher publishes JSON-encoded messages to NATS subjects. +// Publisher publishes JSON-encoded messages to broadcast subjects. type Publisher interface { Publish(subject string, data any) error } -// Subscription represents a NATS subscription that can be unsubscribed. +// Subscription represents a subscription that can be unsubscribed. type Subscription interface { Unsubscribe() error } -// MessagingClient is the full interface for NATS messaging operations. -// Consumers should depend on this interface rather than the concrete Client -// for testability. -type MessagingClient interface { - Publisher - Subscribe(subject string, handler func([]byte)) (Subscription, error) - QueueSubscribe(subject, queue string, handler func([]byte)) (Subscription, error) - QueueSubscribeReply(subject, queue string, handler func(data []byte, reply func([]byte))) (Subscription, error) - SubscribeReply(subject string, handler func(data []byte, reply func([]byte))) (Subscription, error) - Request(subject string, data []byte, timeout time.Duration) ([]byte, error) - IsConnected() bool - Close() -} - -// Broadcaster is the fan-out half of the messaging surface: a publish reaches -// every subscriber on every replica. It exists so a call site can be moved onto -// the PostgreSQL carrier without waiting for the request/reply and queue-group -// halves to be retired, because both *messaging.Client and *pgbus.Bus satisfy -// it. +// Broadcaster is the whole messaging surface: a publish reaches every +// subscriber on every replica. +// +// There is no second interface. Request/reply became a control RPC on the +// worker's tunnel, queue groups became a claim queue on the job store, and what +// is left is fan-out, which is one method more than Publisher. The wider +// MessagingClient that used to sit here is deleted rather than shrunk: two +// exported names for one method set in one package is an invitation for the +// next author to pick whichever the surrounding file already imported. // // IsConnected and Close are deliberately NOT here, and their absence is stated -// rather than left to be inferred, because a reader who knows MessagingClient -// assumes the smaller interface simply forgot them. +// rather than left to be inferred. // -// IsConnected has no production consumer: its only non-test occurrences are its -// implementations and the MessagingClient line itself. It is asserted by specs -// and read by logging, and a carrier's consumers must not branch on it. "The -// carrier is down" is not one of the four conditions a node's state can be in, -// and no code may turn it into evidence that a worker is absent. +// IsConnected has no production consumer. It is asserted by specs and read by +// logging, and a carrier's consumers must not branch on it. "The carrier is +// down" is not one of the four conditions a node's state can be in, and no code +// may turn it into evidence that a worker is absent. // // Close DOES have real callers, and they hold a CONCRETE type rather than this -// interface, which is why the interface can omit it. Before any of them is -// narrowed from MessagingClient to Broadcaster, re-run -// -// grep -rn 'Close()' --include='*.go' . | grep -v _test -// -// because that grep is the only thing standing between "the interface does not -// need it" and a lifecycle that silently stops running. +// interface, which is why the interface can omit it. type Broadcaster interface { Publisher Subscribe(subject string, handler func([]byte)) (Subscription, error) } -// The concrete NATS client is one of the two carriers this interface exists to -// make interchangeable. Asserted here rather than left to the first adopter, so -// a change to Client's signatures fails to compile in the package that owns the -// interface instead of in whichever call site is migrated next. +// One of the two carriers a deployment dials, asserted in the package that owns +// the interface rather than at whichever call site is migrated next. +// +// *pgbus.Bus is the other and cannot be named here, because it imports this +// package; it is asserted from interfaces_test.go, which also pins that +// *Client's method set is this interface plus its own lifecycle and nothing +// else. That pin is the guard, not this line: a conformance assertion stays +// true however many methods grow back, so it cannot say that the retired halves +// are gone. var _ Broadcaster = (*Client)(nil) diff --git a/core/services/messaging/interfaces_test.go b/core/services/messaging/interfaces_test.go new file mode 100644 index 000000000000..5a11fdbdbe76 --- /dev/null +++ b/core/services/messaging/interfaces_test.go @@ -0,0 +1,106 @@ +package messaging_test + +import ( + "reflect" + "sort" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/core/services/testutil" +) + +// The three things that must answer one contract, asserted where the contract +// lives rather than at whichever adopter is migrated next. +// +// *pgbus.Bus is the carrier a deployment runs on, *messaging.Client is the one +// carrier left that a deployment still dials (see below), and *testutil.FakeBus +// is what every consumer's spec runs against. A double that drifts out of the +// interface makes each adopter's suite fail in turn, in packages that do not +// own the interface and cannot say why. +// +// This file is package messaging_test because messaging cannot import testutil +// or pgbus: both import messaging. +var ( + _ messaging.Broadcaster = (*testutil.FakeBus)(nil) + _ messaging.Broadcaster = (*pgbus.Bus)(nil) + _ messaging.Broadcaster = (*messaging.Client)(nil) +) + +// exportedMethods names the exported method set of t, sorted, so a spec can +// assert what a type CAN be asked to do rather than what it happens to be +// asked today. +func exportedMethods(t reflect.Type) []string { + names := make([]string, 0, t.NumMethod()) + for i := range t.NumMethod() { + names = append(names, t.Method(i).Name) + } + sort.Strings(names) + return names +} + +var _ = Describe("the messaging surface", func() { + // Broadcaster is the whole interface. There is no second one, and the + // count is asserted rather than the membership alone, because a method + // added to this interface is a family being invited back onto a carrier + // this deployment is being taken off. + It("is fan-out and nothing else", func() { + Expect(exportedMethods(reflect.TypeOf((*messaging.Broadcaster)(nil)).Elem())). + To(Equal([]string{"Publish", "Subscribe"}), + "Broadcaster grew a method: request/reply became a control RPC on the worker's tunnel and queue groups became a claim queue, and neither may come back through this interface") + }) + + It("has no interface that can queue-subscribe or request", func() { + // MessagingClient is gone, and this is what says so from outside the + // package: a name reintroduced for the retired halves would have to be + // exported to be usable, and every consumer of this package holds a + // Broadcaster. + Expect(exportedMethods(reflect.TypeOf((*messaging.Publisher)(nil)).Elem())). + To(Equal([]string{"Publish"})) + Expect(exportedMethods(reflect.TypeOf((*messaging.Subscription)(nil)).Elem())). + To(Equal([]string{"Unsubscribe"})) + }) +}) + +// The NATS client survives this commit, and it survives for exactly one family. +// +// agent..cancel is the one fan-out family that did not move to the +// PostgreSQL carrier: its only subscriber is the agent WORKER, which has no +// database and cannot join that carrier at all, so a cancel published there +// would reach no worker and be reported as sent. Both ends of that family still +// dial NATS, which is why messaging.New, the connect options and the TLS files +// are all still here. +// +// What the client may no longer do is everything else. Its queue and +// request/reply halves are deleted, so a family being put BACK on NATS is now a +// build error at the call site rather than a line that compiles, publishes +// successfully, and is delivered onto a carrier nobody subscribes to. +// +// The method set is pinned by NAME and not by a conformance assertion, because +// a conformance assertion cannot express absence: *messaging.Client satisfying +// Broadcaster stays true no matter how many methods are added back. +var _ = Describe("the NATS client's method set", func() { + It("is a Broadcaster plus its own lifecycle, and nothing more", func() { + Expect(exportedMethods(reflect.TypeOf((*messaging.Client)(nil)))). + To(Equal([]string{"Close", "ConfirmRoundTrip", "IsConnected", "OnReconnect", "Publish", "Subscribe"}), + "the NATS client grew a method back: it is the cancel family's carrier and may carry nothing else") + }) + + It("cannot be handed a queue group or a request", func() { + // Named individually so the failure message says WHICH half came back. + // As a set assertion alone, re-adding Request reads as an off-by-one. + t := reflect.TypeOf((*messaging.Client)(nil)) + // Conn is in this list and is the reason the list is checkable at all: + // while it existed, every name above it was one c.Conn().X() away, so + // the deletions would have been a naming convention rather than a + // constraint. ConfirmRoundTrip is what replaced it, and it hands back + // an error rather than the connection. + for _, retired := range []string{"QueueSubscribe", "QueueSubscribeReply", "SubscribeReply", "Request", "Conn"} { + _, found := t.MethodByName(retired) + Expect(found).To(BeFalse(), + "%s is back on *messaging.Client; that half of the carrier was retired and its call sites moved to the worker's tunnel or to the claim queue", retired) + } + }) +}) diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index 224bae7bb98f..d7b35278dc47 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -31,7 +31,6 @@ func sanitizeSubjectToken(s string) string { const ( subjectAgentEventsPrefix = "agent." subjectJobProgressPrefix = "jobs." - subjectFineTunePrefix = "finetune." subjectGalleryPrefix = "gallery." ) @@ -63,11 +62,6 @@ func SubjectJobResult(jobID string) string { return subjectJobProgressPrefix + sanitizeSubjectToken(jobID) + ".result" } -// SubjectFineTuneProgress returns the NATS subject for fine-tune progress. -func SubjectFineTuneProgress(jobID string) string { - return subjectFineTunePrefix + sanitizeSubjectToken(jobID) + ".progress" -} - // SubjectGalleryProgress returns the NATS subject for gallery download progress. func SubjectGalleryProgress(opID string) string { return subjectGalleryPrefix + sanitizeSubjectToken(opID) + ".progress" @@ -103,7 +97,6 @@ const ( const ( subjectJobCancelPrefix = "jobs." subjectAgentCancelPrefix = "agent." - subjectFineTuneCancelPrefix = "finetune." subjectGalleryCancelPrefix = "gallery." subjectResponseCancelPrefix = "responses." ) @@ -129,11 +122,6 @@ func SubjectAgentCancel(agentID string) string { return subjectAgentCancelPrefix + sanitizeSubjectToken(agentID) + ".cancel" } -// SubjectFineTuneCancel returns the NATS subject to stop fine-tuning. -func SubjectFineTuneCancel(jobID string) string { - return subjectFineTuneCancelPrefix + sanitizeSubjectToken(jobID) + ".cancel" -} - // SubjectGalleryCancel returns the NATS subject to cancel a gallery download. func SubjectGalleryCancel(opID string) string { return subjectGalleryCancelPrefix + sanitizeSubjectToken(opID) + ".cancel" @@ -373,8 +361,13 @@ type RunningModelInfo struct { // reached through its tunnel, so no subject is minted for them. // Cache Invalidation (Pub/Sub — broadcast to all instances) +// Skills and collection cache invalidation are no longer minted here. +// cache.invalidate.skills and cache.invalidate.collections. had NO +// production publisher and no production subscriber: their only callers were +// e2e specs that published on a subject and subscribed to the same subject +// through one client, which passes for any literal at all. They are deleted +// rather than migrated, because there was no traffic to migrate. const ( - SubjectCacheInvalidateSkills = "cache.invalidate.skills" // SubjectCacheInvalidateModels is broadcast by the replica that completed // a model install/delete. Peers subscribe and re-run // ModelConfigLoader.LoadModelConfigsFromPath so a chat completion routed @@ -396,11 +389,6 @@ type CacheInvalidateEvent struct { ConfigRevision string `json:"config_revision,omitempty"` } -// SubjectCacheInvalidateCollection returns the NATS subject for collection cache invalidation. -func SubjectCacheInvalidateCollection(name string) string { - return "cache.invalidate.collections." + sanitizeSubjectToken(name) -} - // SyncedMap State Sync (Pub/Sub — broadcast to all frontends) // // The reusable syncstate.SyncedMap component publishes a {op,key,value} delta on diff --git a/core/services/messaging/subjects_wire_test.go b/core/services/messaging/subjects_wire_test.go index f6afa85ef5a6..aba9847d1f0a 100644 --- a/core/services/messaging/subjects_wire_test.go +++ b/core/services/messaging/subjects_wire_test.go @@ -147,3 +147,88 @@ var _ = Describe("the agent-events wildcard", func() { Expect(SubjectMatches(SubjectAgentEventsWildcard, "agent.a.cancel")).To(BeFalse()) }) }) + +// Every subject this package still mints, pinned to the exact literal it must +// produce. +// +// The table exists because of what is being deleted around it. Four builders +// went in this commit: fine-tune progress and cancel, whose families moved off +// this carrier, and skills and collection cache invalidation, which had no +// production publisher or subscriber to move. The only thing that had ever +// exercised any of the four was an e2e spec that published on a subject and +// subscribed to the same subject, which passes for any literal at all. A +// builder taken out by mistake alongside them would have had no unit failure +// anywhere: its call sites publish, the publish succeeds, and the symptom is a +// subscriber that never fires. +// +// So the pin is the LITERAL, not a round trip through the builder, and it +// covers every survivor rather than the ones a reviewer thought were at risk. +// A subject is also a wire format: two releases in one deployment agree on +// these strings and on nothing else, so a rename that looks internal is a +// rolling upgrade in which half the fleet stops hearing the other half. +var _ = Describe("the subjects this package mints", func() { + DescribeTable("builds the exact subject its subscribers filter on", + func(got, want string) { Expect(got).To(Equal(want)) }, + + Entry("agent events", SubjectAgentEvents("a1", "u1"), "agent.a1.events.u1"), + Entry("agent events, anonymous user", SubjectAgentEvents("a1", ""), "agent.a1.events.anonymous"), + Entry("agent events wildcard", SubjectAgentEventsWildcard, "agent.*.events.*"), + Entry("agent cancel", SubjectAgentCancel("a1"), "agent.a1.cancel"), + Entry("agent cancel wildcard", SubjectAgentCancelWildcard, "agent.*.cancel"), + + Entry("job progress", SubjectJobProgress("j1"), "jobs.j1.progress"), + Entry("job progress wildcard", SubjectJobProgressWildcard, "jobs.*.progress"), + Entry("job result", SubjectJobResult("j1"), "jobs.j1.result"), + Entry("job result wildcard", SubjectJobResultWildcard, "jobs.*.result"), + Entry("job cancel", SubjectJobCancel("j1"), "jobs.j1.cancel"), + Entry("job cancel wildcard", SubjectJobCancelWildcard, "jobs.*.cancel"), + + Entry("gallery progress", SubjectGalleryProgress("op1"), "gallery.op1.progress"), + Entry("gallery progress wildcard", SubjectGalleryProgressWildcard, "gallery.*.progress"), + Entry("gallery cancel", SubjectGalleryCancel("op1"), "gallery.op1.cancel"), + Entry("gallery cancel wildcard", SubjectGalleryCancelWildcard, "gallery.*.cancel"), + Entry("gallery opcache start", SubjectGalleryOpStart, "gallery.opcache.start"), + Entry("gallery opcache end", SubjectGalleryOpEnd, "gallery.opcache.end"), + + Entry("staging progress", SubjectStagingProgress("m1"), "staging.m1.progress"), + Entry("staging progress wildcard", SubjectStagingProgressWildcard, "staging.*.progress"), + + Entry("response cancel", SubjectResponseCancel("r1"), "responses.r1.cancel"), + Entry("response cancel wildcard", SubjectResponseCancelWildcard, "responses.*.cancel"), + + Entry("model cache invalidation", SubjectCacheInvalidateModels, "cache.invalidate.models"), + Entry("backend cache invalidation", SubjectCacheInvalidateBackends, "cache.invalidate.backends"), + + Entry("syncstate delta", SubjectSyncStateDelta("agent.tasks"), "state.agent-tasks.delta"), + Entry("syncstate tenant delta", SubjectSyncStateTenantDelta("agent.tasks", "u1"), "state.agent-tasks.u1.delta"), + Entry("syncstate tenant wildcard", SubjectSyncStateTenantWildcard("agent.tasks"), "state.agent-tasks.*.delta"), + + Entry("prefix-cache observe", SubjectPrefixCacheObserve, "prefixcache.observe"), + Entry("prefix-cache invalidate", SubjectPrefixCacheInvalidate, "prefixcache.invalidate"), + ) + + It("mints nothing for the families that left this carrier", func() { + // Stated as an absence of PREFIXES rather than of identifiers, because + // an identifier that is gone cannot be named here at all: the file + // would not compile. What can be asserted is that no survivor mints + // into the retired namespaces, which is what a half-finished deletion + // would leave behind. + // + // finetune.* went when fine-tune progress and cancel moved to the + // broadcast carrier's own subjects; cache.invalidate.skills and + // cache.invalidate.collections went with the skills and collection + // caches. + for _, subject := range []string{ + SubjectAgentEvents("a1", "u1"), SubjectAgentCancel("a1"), + SubjectJobProgress("j1"), SubjectJobResult("j1"), SubjectJobCancel("j1"), + SubjectGalleryProgress("op1"), SubjectGalleryCancel("op1"), + SubjectStagingProgress("m1"), SubjectResponseCancel("r1"), + SubjectCacheInvalidateModels, SubjectCacheInvalidateBackends, + SubjectSyncStateDelta("agent.tasks"), + } { + Expect(subject).ToNot(HavePrefix("finetune.")) + Expect(subject).ToNot(HavePrefix("cache.invalidate.skills")) + Expect(subject).ToNot(HavePrefix("cache.invalidate.collections")) + } + }) +}) diff --git a/core/services/nodes/control_worker_fake_test.go b/core/services/nodes/control_worker_fake_test.go index f5459e35130e..84fcfa0a557c 100644 --- a/core/services/nodes/control_worker_fake_test.go +++ b/core/services/nodes/control_worker_fake_test.go @@ -19,6 +19,17 @@ import ( "github.com/mudler/LocalAI/core/services/workerctl" ) +// requestCall records one control request this fake was asked to serve. +// +// It used to live beside a messaging double that answered NATS requests. That +// double's Request method is gone with the carrier's, so the type now belongs +// to the only thing that still records requests: the worker control fake, whose +// requests are HTTP routes on a tunnel. +type requestCall struct { + Subject string + Data []byte +} + // controlKey names one control verb on one node. // // It replaces the NATS subject the scripted double used to key on, and it is diff --git a/core/services/nodes/unloader_test.go b/core/services/nodes/unloader_test.go index e5f6fb7bba57..d3efd1d5f77c 100644 --- a/core/services/nodes/unloader_test.go +++ b/core/services/nodes/unloader_test.go @@ -72,20 +72,20 @@ func (f *fakeModelLocator) RemoveAllNodeModelReplicas(_ context.Context, nodeID, return nil } -// fakeMessagingClient implements messaging.MessagingClient, recording Publish -// and Request calls so we can assert on subjects and payloads. +// fakeMessagingClient is a messaging.Broadcaster that records Publish calls so +// we can assert on subjects and payloads. // // NO control verb reaches it any more, and the RemoteUnloaderAdapter cannot be // handed one: it holds no publisher. It survives here for the staging-progress // broadcasts in staging_progress_broadcast_test.go, which are cross-replica -// events rather than anything addressed to a worker. +// events rather than anything addressed to a worker. Its Request half went with +// the carrier's: a request/reply is a control RPC on the worker's tunnel now, +// and a double that still answered one would let a spec exercise a shape +// production can no longer take. type fakeMessagingClient struct { - mu sync.Mutex - published []publishCall - publishErr error // error to return from Publish - requestReply []byte - requestErr error - requestCalls []requestCall + mu sync.Mutex + published []publishCall + publishErr error // error to return from Publish } type publishCall struct { @@ -93,12 +93,6 @@ type publishCall struct { Data []byte } -type requestCall struct { - Subject string - Data []byte - Timeout time.Duration -} - func (f *fakeMessagingClient) Publish(subject string, data any) error { f.mu.Lock() defer f.mu.Unlock() @@ -118,27 +112,6 @@ func (f *fakeMessagingClient) Subscribe(_ string, _ func([]byte)) (messaging.Sub return &fakeSubscription{}, nil } -func (f *fakeMessagingClient) QueueSubscribe(_ string, _ string, _ func([]byte)) (messaging.Subscription, error) { - return &fakeSubscription{}, nil -} - -func (f *fakeMessagingClient) QueueSubscribeReply(_ string, _ string, _ func(data []byte, reply func([]byte))) (messaging.Subscription, error) { - return &fakeSubscription{}, nil -} - -func (f *fakeMessagingClient) SubscribeReply(_ string, _ func(data []byte, reply func([]byte))) (messaging.Subscription, error) { - return &fakeSubscription{}, nil -} - -func (f *fakeMessagingClient) Request(subject string, data []byte, timeout time.Duration) ([]byte, error) { - f.mu.Lock() - defer f.mu.Unlock() - f.requestCalls = append(f.requestCalls, requestCall{Subject: subject, Data: data, Timeout: timeout}) - return f.requestReply, f.requestErr -} - -func (f *fakeMessagingClient) IsConnected() bool { return true } -func (f *fakeMessagingClient) Close() {} type fakeSubscription struct{} diff --git a/core/services/testutil/fakebus.go b/core/services/testutil/fakebus.go index 1607a5e9c578..ed2a9eb4e591 100644 --- a/core/services/testutil/fakebus.go +++ b/core/services/testutil/fakebus.go @@ -3,12 +3,11 @@ package testutil import ( "encoding/json" "sync" - "time" "github.com/mudler/LocalAI/core/services/messaging" ) -// FakeBus is an in-memory messaging.MessagingClient that delivers each published +// FakeBus is an in-memory messaging.Broadcaster that delivers each published // message synchronously to every registered subscriber whose subject filter // matches. Matching is messaging.SubjectMatches, the same function the real // carrier uses, so a filter that fires here fires in production too. @@ -33,7 +32,7 @@ type FakeBus struct { // reconnectCbs back the optional OnReconnect/TriggerReconnect pair, letting a // spec exercise the component's reconnect re-hydrate path without a real - // NATS server. + // carrier. reconnectCbs []func() } @@ -43,14 +42,14 @@ type fakeBusSub struct { handler func([]byte) } -// The two interfaces this double stands in for. Broadcaster is asserted here -// and not left to the first adopter: every cross-replica map now takes a -// messaging.Broadcaster, and a fake that drifted out of that interface would -// break each adopter's suite in turn rather than the package that owns it. -var ( - _ messaging.MessagingClient = (*FakeBus)(nil) - _ messaging.Broadcaster = (*FakeBus)(nil) -) +// The one interface this double stands in for, asserted here and not left to +// the first adopter: every cross-replica map takes a messaging.Broadcaster, and +// a fake that drifted out of that interface would break each adopter's suite in +// turn rather than the package that owns it. Asserted again from +// core/services/messaging/interfaces_test.go, beside the same assertion for the +// two real carriers, so the double and the carriers are held to one contract in +// one place. +var _ messaging.Broadcaster = (*FakeBus)(nil) // NewFakeBus returns a ready-to-use in-memory bus. func NewFakeBus() *FakeBus { @@ -128,29 +127,22 @@ func (b *FakeBus) Subscribe(subject string, handler func([]byte)) (messaging.Sub return &fakeBusSubscription{bus: b, subRef: sub}, nil } -func (b *FakeBus) QueueSubscribe(subject, _ string, handler func([]byte)) (messaging.Subscription, error) { - return b.Subscribe(subject, handler) -} - -func (b *FakeBus) QueueSubscribeReply(string, string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return &fakeBusSubscription{bus: b}, nil -} - -func (b *FakeBus) SubscribeReply(string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return &fakeBusSubscription{bus: b}, nil -} - -func (b *FakeBus) Request(string, []byte, time.Duration) ([]byte, error) { - return nil, nil -} +// A queue group and a request/reply are NOT stubbed here, and their absence is +// the point. This double used to answer both, which meant a consumer that had +// not been migrated off them kept passing against a fake that load-balanced +// nothing (QueueSubscribe delivered to every subscriber) and replied nothing +// (Request returned a nil answer and a nil error, which reads as a peer that +// answered emptily rather than one that was never asked). Both halves are gone +// from the real carrier now, so a call site that needs either fails to compile +// against the carrier AND against the double. func (b *FakeBus) IsConnected() bool { return true } func (b *FakeBus) Close() {} -// OnReconnect mirrors *messaging.Client.OnReconnect so a spec can drive the +// OnReconnect mirrors the carriers' OnReconnect so a spec can drive the // component's reconnect re-hydrate path. The component detects this method via an // optional interface assertion; implementing it here keeps the fake a faithful -// stand-in for the concrete client. +// stand-in for the concrete carriers. func (b *FakeBus) OnReconnect(cb func()) { if cb == nil { return @@ -160,8 +152,8 @@ func (b *FakeBus) OnReconnect(cb func()) { b.mu.Unlock() } -// TriggerReconnect runs every registered reconnect callback, simulating a NATS -// reconnect event. +// TriggerReconnect runs every registered reconnect callback, simulating a +// carrier reconnect event. func (b *FakeBus) TriggerReconnect() { b.mu.Lock() cbs := append([]func(){}, b.reconnectCbs...) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 18ce8798dc4a..cd63f4f72816 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -5,7 +5,7 @@ weight = 71 url = "/features/distributed-mode/" +++ -Distributed mode enables horizontal scaling of LocalAI across multiple machines using **PostgreSQL** for state and node registry, and **NATS** for real-time coordination. Unlike the [P2P/federation approach]({{% relref "features/distributed_inferencing" %}}), distributed mode is designed for production deployments and Kubernetes environments where you need centralized management, health monitoring, and deterministic routing. +Distributed mode enables horizontal scaling of LocalAI across multiple machines using **PostgreSQL** for state, node registry and cross-replica fan-out. A **NATS** server is still needed for one thing: delivering an agent cancel to the agent worker running the execution. Unlike the [P2P/federation approach]({{% relref "features/distributed_inferencing" %}}), distributed mode is designed for production deployments and Kubernetes environments where you need centralized management, health monitoring, and deterministic routing. {{% notice note %}} Distributed mode requires authentication enabled with a **PostgreSQL** database - SQLite is not supported. This is because the node registry, job store, and other distributed state are stored in PostgreSQL tables. @@ -15,7 +15,7 @@ Distributed mode requires authentication enabled with a **PostgreSQL** database ![Distributed mode architecture: a load balancer fronts stateless SmartRouter frontends backed by a shared NATS/PostgreSQL/S3 plane, with generic workers running per-model gRPC backends](/images/diagrams/distributed-mode-arch.png) -**Frontends** are stateless LocalAI instances that receive API requests and route them to worker nodes via the **SmartRouter**. All frontends share state through PostgreSQL and coordinate via NATS. +**Frontends** are stateless LocalAI instances that receive API requests and route them to worker nodes via the **SmartRouter**. All frontends share state through PostgreSQL, which also carries every cross-replica event they broadcast. **Workers** are generic processes that self-register with a frontend. They don't have a fixed backend type - the SmartRouter dynamically installs the required backend by calling the worker's `backend.install` control route through its tunnel when a model request arrives. @@ -41,7 +41,7 @@ Each model gets its own gRPC backend process, so a single worker can serve multi - Each frontend replica holds **one extra PostgreSQL session** beyond its connection pool, pinned for the life of the process, and creates a `bus_messages` table. Both belong to the broadcast carrier that is replacing NATS for cross-replica fan-out; it already carries the four `state.*.delta` families (see [Cross-replica in-memory state](#cross-replica-in-memory-state)). Size `max_connections` for one additional session per frontend replica. - That session reports an `application_name` of `localai_pgbus_`, so `SELECT count(*) FROM pg_stat_activity WHERE application_name LIKE 'localai_pgbus_%'` counts the replicas currently listening. If the carrier loses its session it redials and re-registers on its own; a broadcast published while it was down is not replayed, which is why nothing that must survive a gap is carried by a broadcast alone. - `bus_messages` holds only broadcasts too large for a PostgreSQL notification, and every replica retires rows older than ten minutes. The table is a spill buffer, not a log: it is not a place to read past events from. -- **NATS** server - used for agent-worker coordination. The frontend's own cross-replica events do not use it: they travel on the PostgreSQL the deployment already runs. **Serve-backend workers do not connect to it at all**: every verb they take, and file staging with it, is an HTTP route on the worker's tunnel. Set no `LOCALAI_NATS_URL` on a `local-ai worker`. The frontend and any `local-ai agent-worker` still need one. +- **NATS** server - used for ONE subject family, `agent..cancel`, which is the only thing a frontend still sends over a bus and the only thing an agent worker still subscribes to. Everything else a frontend broadcasts travels on the PostgreSQL the deployment already runs, and every verb a frontend addresses to a worker is an HTTP route on that worker's tunnel. **Serve-backend workers do not connect to it at all**: every verb they take, and file staging with it, is an HTTP route on the worker's tunnel. Set no `LOCALAI_NATS_URL` on a `local-ai worker`. The frontend and any `local-ai agent-worker` still need one. - All services must be on the same network (or reachable via configured URLs) ## Quick Start with Docker Compose @@ -478,7 +478,7 @@ When a worker's tunnel goes, the frontend does **not** forget the worker. It rec That distinction exists because a worker loses its tunnel for entirely ordinary reasons. A frontend replica restarting during a rolling upgrade drops every tunnel it held, and each of those workers immediately re-dials the load balancer and lands on another replica. Treating that as "the worker is gone" would evict models mid-upgrade for a fleet that never actually went anywhere. -This applies to **backend** nodes only. An agent worker holds a tunnel and therefore has a `node_connections` row that ages exactly like a backend worker's, and nothing acts on it: an agent worker's real work travels on NATS, so a lost tunnel says nothing about whether it can run a job. The scheduler never sees agent nodes at all (every placement query selects `node_type = 'backend'`), and the health monitor skips them explicitly. +This applies to **backend** nodes only. An agent worker holds a tunnel and therefore has a `node_connections` row that ages exactly like a backend worker's, and nothing acts on it: the scheduler never sees agent nodes at all (every placement query selects `node_type = 'backend'`), and the health monitor skips them explicitly. Three things read this. The **scheduler** reads it before it places a cold load: a worker whose departure has outlived the grace is skipped and marked unhealthy, so every other frontend replica stops choosing it too. **LRU eviction** reads it before it hands back the node it freed capacity on, because a node full enough to be an eviction target is exactly the node the placement selectors never offer, so the scheduler's own check never sees it. The **health monitor** reads it on every cycle, which is what covers the case the heartbeat cannot see. A worker's heartbeat says its supervisor is alive; it says nothing about whether anything here can reach that worker's backends, because those are reached over the tunnel. A worker that heartbeats with a permanently dead tunnel (a proxy that stopped upgrading WebSockets, a rotated registration credential, a reconnect loop longer than the grace) is therefore marked unhealthy too, rather than staying listed healthy while every request for a model loaded on it fails "no route to that worker". @@ -585,14 +585,14 @@ A frontend replica that dies mid-load does not wedge the model: the job row carr ### NATS JWT authentication (recommended for production) -**This section is about agent workers and the frontend.** A serve-backend worker opens no NATS connection, so none of it applies to one; its own credential is the tunnel token it gets at registration, and its control plane is authenticated by `LOCALAI_REGISTRATION_TOKEN`. An agent worker now has both: a NATS credential for everything still on the bus, and a tunnel token plus the same `LOCALAI_REGISTRATION_TOKEN` bearer check in front of its control server. +**This section is about agent workers and the frontend.** A serve-backend worker opens no NATS connection, so none of it applies to one; its own credential is the tunnel token it gets at registration, and its control plane is authenticated by `LOCALAI_REGISTRATION_TOKEN`. An agent worker now has both: a NATS credential, which covers the one subject family still on the bus, and a tunnel token plus the same `LOCALAI_REGISTRATION_TOKEN` bearer check in front of its control server. -By default, NATS connections are anonymous: any client that can reach port `4222` may publish the subjects still carried on it. Those are the agent-worker job subjects and the frontend's own cross-replica events. `nodes..backend.install` and its nine siblings are **not** among them - they are HTTP routes on the worker's tunnel, see [The worker control plane](#the-worker-control-plane). Enable JWT auth to scope agent workers to their own subjects and give the frontend a dedicated service credential. +By default, NATS connections are anonymous: any client that can reach port `4222` may publish the one subject family still carried on it, `agent..cancel`. Anyone who can reach an unauthenticated bus can therefore cancel any running agent. Nothing else is on it: the agent-worker job subjects became rows in a claim table, the frontend's cross-replica events travel on PostgreSQL, and `nodes..backend.install` and its nine siblings are HTTP routes on the worker's tunnel, see [The worker control plane](#the-worker-control-plane). Enable JWT auth to scope agent workers to their own subjects and give the frontend a dedicated service credential. | Flag | Env Var | Description | |------|---------|-------------| | `--nats-account-seed` | `LOCALAI_NATS_ACCOUNT_SEED` | Account signing seed (`SU...`). The frontend mints a per-node user JWT at registration (`nats_jwt` in the register response). | -| `--nats-service-jwt` | `LOCALAI_NATS_SERVICE_JWT` | User JWT for the frontend (and optional fallback for agent workers) to publish install/upgrade and related subjects. | +| `--nats-service-jwt` | `LOCALAI_NATS_SERVICE_JWT` | User JWT for the frontend (and optional fallback for agent workers). The frontend publishes one subject family with it: `agent..cancel`. | | `--nats-service-seed` | `LOCALAI_NATS_SERVICE_SEED` | User signing seed (`SU...`) paired with the service JWT. | | `--nats-worker-jwt-ttl` | `LOCALAI_NATS_WORKER_JWT_TTL` | Lifetime of minted worker JWTs (default `24h`). | | `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | Fail startup if JWT credentials are missing when distributed mode is enabled. | diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md index 6d9aca0e1507..dcbdacd1fac9 100644 --- a/docs/content/reference/cli-reference.md +++ b/docs/content/reference/cli-reference.md @@ -224,7 +224,7 @@ LocalAI supports several subcommands beyond `run`: - `local-ai mcp-server` - Run the LocalAI admin tool surface as a stdio MCP server (controls a remote LocalAI instance over HTTP) - `local-ai worker` - Start a worker for distributed mode (generic, backend-agnostic; needs only an outbound route to the frontend, no message bus) - `local-ai p2p-worker` - Run workers to distribute workload via p2p (llama.cpp-only) -- `local-ai agent-worker` - Start an agent worker for distributed mode (serves agent execution and MCP CI runs as control verbs on the tunnel it holds; still needs `--nats-url` for cancellation and event broadcasts) +- `local-ai agent-worker` - Start an agent worker for distributed mode (serves agent execution and MCP CI runs as control verbs on the tunnel it holds; still needs `--nats-url`, for cancellation and nothing else: an agent cancel has to reach the worker running the execution, and a worker has no database and so cannot read the PostgreSQL carrier every other broadcast travels on) - `local-ai util` - Utility commands - `local-ai explorer` - Run P2P explorer - `local-ai federated` - Run LocalAI in federated mode diff --git a/tests/e2e/distributed/agent_native_executor_test.go b/tests/e2e/distributed/agent_native_executor_test.go index 8ac0ac109aeb..1d147bcef0d1 100644 --- a/tests/e2e/distributed/agent_native_executor_test.go +++ b/tests/e2e/distributed/agent_native_executor_test.go @@ -314,7 +314,7 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f }) }) - Context("NATSDispatcher", func() { + Context("WorkerExecutor", func() { It("should dispatch chat via NATS and receive response", func() { bridge := agents.NewEventBridge(infra.NC, nil, "test-instance") diff --git a/tests/e2e/distributed/finetune_distributed_test.go b/tests/e2e/distributed/finetune_distributed_test.go index e9392e1cc5c0..b1a1409e39a3 100644 --- a/tests/e2e/distributed/finetune_distributed_test.go +++ b/tests/e2e/distributed/finetune_distributed_test.go @@ -1,11 +1,8 @@ package distributed_test import ( - "sync/atomic" - "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/distributed" - "github.com/mudler/LocalAI/core/services/messaging" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -83,50 +80,22 @@ var _ = Describe("Fine-Tune Distributed", Label("Distributed"), func() { }) }) - Context("NATS progress publishing", func() { - It("should publish fine-tune progress via NATS", func() { - job := &distributed.FineTuneJobRecord{ - UserID: "u1", Model: "m1", Backend: "b1", - TrainingType: "lora", TrainingMethod: "sft", Status: "queued", - } - Expect(ftStore.Create(job)).To(Succeed()) - - // Subscribe to fine-tune progress - var received atomic.Int32 - sub, err := infra.NC.Subscribe(messaging.SubjectFineTuneProgress(job.ID), func(data []byte) { - received.Add(1) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - FlushNATS(infra.NC) - - // Publish progress events simulating training steps - Expect(infra.NC.Publish(messaging.SubjectFineTuneProgress(job.ID), map[string]any{ - "job_id": job.ID, - "status": "training", - "message": "Epoch 1/3, loss=2.5", - })).To(Succeed()) - - Expect(infra.NC.Publish(messaging.SubjectFineTuneProgress(job.ID), map[string]any{ - "job_id": job.ID, - "status": "training", - "message": "Epoch 2/3, loss=1.8", - })).To(Succeed()) - - Expect(infra.NC.Publish(messaging.SubjectFineTuneProgress(job.ID), map[string]any{ - "job_id": job.ID, - "status": "completed", - "message": "Training finished", - })).To(Succeed()) - - Eventually(func() int32 { return received.Load() }, "5s").Should(Equal(int32(3))) - - // Verify cancel subject is correctly formed - cancelSubj := messaging.SubjectFineTuneCancel(job.ID) - Expect(cancelSubj).To(ContainSubstring(".cancel")) - }) - }) + // The "NATS progress publishing" Context that stood here is deleted with + // the two builders it was the only caller of. + // + // What it did was subscribe and publish on messaging.SubjectFineTuneProgress + // through one client and count three deliveries, then assert that + // SubjectFineTuneCancel contained ".cancel". No production code had + // published on either subject since fine-tune progress moved to the + // broadcast carrier, so the round trip pinned the carrier and not this + // feature, and the string assertion passed for any literal ending in + // ".cancel". + // + // The carrier round trip is core/services/pgbus/bus_test.go, on two + // separate bus instances. That the surviving builders mint the exact + // subjects their subscribers filter on is now the literal table in + // core/services/messaging/subjects_wire_test.go, which is a stronger pin + // than a round trip through the builder could ever be. Context("Without --distributed", func() { It("should use in-memory state without --distributed", func() { diff --git a/tests/e2e/distributed/foundation_test.go b/tests/e2e/distributed/foundation_test.go index 244b5e6e0d6d..62c2a4b8348b 100644 --- a/tests/e2e/distributed/foundation_test.go +++ b/tests/e2e/distributed/foundation_test.go @@ -4,10 +4,10 @@ import ( "bytes" "context" "io" - "sync/atomic" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/advisorylock" + "github.com/mudler/LocalAI/core/services/agents" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/storage" @@ -83,16 +83,41 @@ var _ = Describe("Phase 0: Foundation", Label("Distributed"), func() { }) }) - Context("NATS client", func() { - It("should connect, publish, and subscribe", func() { + // The one carrier a deployment still dials besides PostgreSQL, and the one + // family left on it. + // + // agent..cancel did not move to the broadcast carrier: its only + // subscriber is the agent WORKER, which has no database and cannot join + // that carrier at all. So this round trip is no longer "the messaging layer + // works" - it is the cancel path for every agent a worker runs, and if it + // stops working the symptom is a cancel that is published, succeeds, and + // reaches nobody. + // + // Two Its that used to sit here went with the halves of the client they + // exercised. "should support queue subscriptions for load balancing" pinned + // that work reaches exactly one of N consumers; a queue group no longer + // selects anything and that property is now core/services/jobs + // claim_test.go's competing-claimants specs, which race eight claimants for + // eight rows and then for one. "should reconnect after disconnect" pinned + // that the carrier survives a drop, and said in its own body that it + // asserted nothing of the kind; the property is now + // core/services/pgbus/listener_test.go, which actually kills the session + // with pg_terminate_backend and waits for delivery to resume. + Context("the cancel carrier", func() { + It("connects, publishes and subscribes, which is the agent cancel path", func() { client, err := messaging.New(infra.NatsURL) Expect(err).ToNot(HaveOccurred()) defer client.Close() Expect(client.IsConnected()).To(BeTrue()) + // The real subject and the real filter, not a placeholder pair. + // A worker subscribes to the wildcard and a frontend publishes to + // one agent's subject, so a round trip on "test.subject" would + // stay green through a filter that no longer matches what the + // builder mints - which is the failure this family actually has. received := make(chan []byte, 1) - sub, err := client.Subscribe("test.subject", func(data []byte) { + sub, err := client.Subscribe(messaging.SubjectAgentCancelWildcard, func(data []byte) { received <- data }) Expect(err).ToNot(HaveOccurred()) @@ -101,58 +126,14 @@ var _ = Describe("Phase 0: Foundation", Label("Distributed"), func() { // Small delay to ensure subscription is active FlushNATS(client) - err = client.Publish("test.subject", map[string]string{"msg": "hello"}) - Expect(err).ToNot(HaveOccurred()) - - Eventually(received, "5s").Should(Receive()) - }) - - It("should support queue subscriptions for load balancing", func() { - client, err := messaging.New(infra.NatsURL) - Expect(err).ToNot(HaveOccurred()) - defer client.Close() - - var worker1Count, worker2Count atomic.Int32 - - sub1, err := client.QueueSubscribe("test.queue", "workers", func(data []byte) { - worker1Count.Add(1) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub1.Unsubscribe() - - sub2, err := client.QueueSubscribe("test.queue", "workers", func(data []byte) { - worker2Count.Add(1) + err = client.Publish(messaging.SubjectAgentCancel("a1"), agents.AgentCancelEvent{ + AgentName: "a1", UserID: "u1", MessageID: "msg-1", }) Expect(err).ToNot(HaveOccurred()) - defer sub2.Unsubscribe() - - FlushNATS(client) - // Publish multiple messages - for i := range 10 { - err = client.Publish("test.queue", map[string]int{"n": i}) - Expect(err).ToNot(HaveOccurred()) - } - - // Wait for all messages to be processed - Eventually(func() int32 { - return worker1Count.Load() + worker2Count.Load() - }, "5s").Should(Equal(int32(10))) - - // Both workers should have received some messages (load-balanced) - // Note: with only 10 messages, distribution may not be perfectly even - Expect(worker1Count.Load() + worker2Count.Load()).To(Equal(int32(10))) + Eventually(received, "5s").Should(Receive()) }) - It("should reconnect after disconnect", func() { - client, err := messaging.New(infra.NatsURL) - Expect(err).ToNot(HaveOccurred()) - defer client.Close() - - Expect(client.IsConnected()).To(BeTrue()) - // The reconnect behavior is tested implicitly by the RetryOnFailedConnect option - // A full reconnect test would require stopping/restarting the NATS container - }) }) Context("ObjectStore filesystem adapter", func() { diff --git a/tests/e2e/distributed/nats_jwt_test.go b/tests/e2e/distributed/nats_jwt_test.go index 1683773af4a6..46444c197ce2 100644 --- a/tests/e2e/distributed/nats_jwt_test.go +++ b/tests/e2e/distributed/nats_jwt_test.go @@ -23,9 +23,12 @@ var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { // route on its tunnel, and it no longer opens a bus connection at all; // the JWT is minted and unused. See pkg/natsauth.WorkerPermissions. Expect(infra.NC.Publish("_INBOX.probe", map[string]string{"path": "/tmp/model"})).To(Succeed()) - Expect(infra.NC.Conn().FlushTimeout(2 * time.Second)).To(Succeed()) - Expect(infra.NC.Conn().LastError()).ToNot(HaveOccurred()) - Expect(infra.NC.Conn().IsConnected()).To(BeTrue()) + // ConfirmRoundTrip is the flush AND the server's verdict in one call. + // Read separately they were two assertions that could drift apart; the + // verdict is the one that matters, because a permission violation does + // not close the connection. + Expect(infra.NC.ConfirmRoundTrip(2 * time.Second)).To(Succeed()) + Expect(infra.NC.IsConnected()).To(BeTrue()) }) It("denies a backend worker the file-staging subjects it no longer serves", func() { @@ -38,8 +41,7 @@ var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { subject := nodeSubjectPrefix(infra.NodeID) + ".files.stage" Expect(infra.NC.Publish(subject, map[string]string{"path": "/tmp/model"})).To(Succeed()) Eventually(func() error { - _ = infra.NC.Conn().FlushTimeout(500 * time.Millisecond) - return infra.NC.Conn().LastError() + return infra.NC.ConfirmRoundTrip(500 * time.Millisecond) }, "3s", "50ms").Should(HaveOccurred()) }) @@ -53,8 +55,7 @@ var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { if err == nil { defer func() { _ = sub.Unsubscribe() }() Eventually(func() error { - _ = infra.NC.Conn().FlushTimeout(500 * time.Millisecond) - return infra.NC.Conn().LastError() + return infra.NC.ConfirmRoundTrip(500 * time.Millisecond) }, "3s", "50ms").Should(HaveOccurred()) } }) @@ -66,15 +67,14 @@ var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { err = anon.Publish("nodes.any.files.x", map[string]string{"x": "1"}) Expect(err).ToNot(HaveOccurred()) - Expect(anon.Conn().FlushTimeout(2 * time.Second)).To(HaveOccurred()) + Expect(anon.ConfirmRoundTrip(2 * time.Second)).To(HaveOccurred()) }) It("denies backend publish to another node's subjects", func() { other := nodeSubjectPrefix("other-node-id") + ".files.stage" Expect(infra.NC.Publish(other, map[string]string{"stage": "nope"})).To(Succeed()) Eventually(func() error { - _ = infra.NC.Conn().FlushTimeout(500 * time.Millisecond) - return infra.NC.Conn().LastError() + return infra.NC.ConfirmRoundTrip(500 * time.Millisecond) }, "3s", "50ms").Should(HaveOccurred()) }) diff --git a/tests/e2e/distributed/nats_queue_reply_test.go b/tests/e2e/distributed/nats_queue_reply_test.go deleted file mode 100644 index 34c0b5c2e9b7..000000000000 --- a/tests/e2e/distributed/nats_queue_reply_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package distributed_test - -import ( - "sync/atomic" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -// What is left here is the messaging layer's own request-reply behaviour. -// -// The MCP specs that used to live beside it went with their carrier: MCP tool -// execution and discovery were the only subjects that combined a queue group -// WITH a reply, and they are now a selection plus an ordinary control RPC over -// the agent worker's tunnel. Their round trip is exercised against a real -// tunnel, a real connection row and a real relay in -// core/services/nodes/agent_control_test.go. -var _ = Describe("NATS queue request-reply", Label("Distributed"), func() { - var ( - infra *TestInfra - ) - - BeforeEach(func() { - infra = SetupNATSOnly() - }) - - Context("QueueSubscribeReply", func() { - It("should support queue subscribe with request-reply round-trip", func() { - // Subscribe with queue group - sub, err := infra.NC.QueueSubscribeReply("test.echo", "echo-workers", func(data []byte, reply func([]byte)) { - // Echo back the request data with a prefix - reply(append([]byte("echo:"), data...)) - }) - Expect(err).ToNot(HaveOccurred()) - defer func() { _ = sub.Unsubscribe() }() - - FlushNATS(infra.NC) - - // Send request and wait for reply - replyData, err := infra.NC.Request("test.echo", []byte("hello"), 5*time.Second) - Expect(err).ToNot(HaveOccurred()) - Expect(string(replyData)).To(Equal("echo:hello")) - }) - - It("should load-balance requests across queue subscribers", func() { - var worker1Count, worker2Count atomic.Int32 - - sub1, _ := infra.NC.QueueSubscribeReply("test.lb", "lb-workers", func(data []byte, reply func([]byte)) { - worker1Count.Add(1) - reply([]byte("w1")) - }) - defer func() { _ = sub1.Unsubscribe() }() - - sub2, _ := infra.NC.QueueSubscribeReply("test.lb", "lb-workers", func(data []byte, reply func([]byte)) { - worker2Count.Add(1) - reply([]byte("w2")) - }) - defer func() { _ = sub2.Unsubscribe() }() - - FlushNATS(infra.NC) - - // Send multiple requests - for range 10 { - _, err := infra.NC.Request("test.lb", []byte("req"), 5*time.Second) - Expect(err).ToNot(HaveOccurred()) - } - - // Both workers should have handled some requests - total := worker1Count.Load() + worker2Count.Load() - Expect(total).To(Equal(int32(10))) - // NATS typically distributes evenly, but we just check both got work - Expect(worker1Count.Load()).To(BeNumerically(">", 0)) - Expect(worker2Count.Load()).To(BeNumerically(">", 0)) - }) - }) -}) diff --git a/tests/e2e/distributed/skills_distributed_test.go b/tests/e2e/distributed/skills_distributed_test.go index 19d95cae4788..14929e6400f2 100644 --- a/tests/e2e/distributed/skills_distributed_test.go +++ b/tests/e2e/distributed/skills_distributed_test.go @@ -1,11 +1,8 @@ package distributed_test import ( - "sync/atomic" - "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/distributed" - "github.com/mudler/LocalAI/core/services/messaging" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -89,61 +86,22 @@ var _ = Describe("Skills Distributed", Label("Distributed"), func() { }) }) - Context("NATS cache invalidation", func() { - It("should publish cache invalidation via NATS on skill change", func() { - // Subscribe to skills cache invalidation - var received atomic.Int32 - sub, err := infra.NC.Subscribe(messaging.SubjectCacheInvalidateSkills, func(data []byte) { - received.Add(1) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - FlushNATS(infra.NC) - - // Save a skill and publish cache invalidation - rec := &distributed.SkillMetadataRecord{ - UserID: "u1", Name: "new-skill", SourceType: "inline", - } - Expect(skillStore.Save(rec)).To(Succeed()) - - // Publish invalidation (in production this is done by the service layer) - Expect(infra.NC.Publish(messaging.SubjectCacheInvalidateSkills, map[string]string{ - "user_id": "u1", - "name": "new-skill", - "action": "save", - })).To(Succeed()) - - Eventually(func() int32 { return received.Load() }, "5s").Should(Equal(int32(1))) - - // Delete and publish another invalidation - Expect(skillStore.Delete("u1", "new-skill")).To(Succeed()) - Expect(infra.NC.Publish(messaging.SubjectCacheInvalidateSkills, map[string]string{ - "user_id": "u1", - "name": "new-skill", - "action": "delete", - })).To(Succeed()) - - Eventually(func() int32 { return received.Load() }, "5s").Should(Equal(int32(2))) - }) - - It("should broadcast collection cache invalidation", func() { - var received atomic.Int32 - sub, err := infra.NC.Subscribe(messaging.SubjectCacheInvalidateCollection("my-collection"), func(data []byte) { - received.Add(1) - }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - FlushNATS(infra.NC) - - Expect(infra.NC.Publish(messaging.SubjectCacheInvalidateCollection("my-collection"), map[string]string{ - "reason": "skill_updated", - })).To(Succeed()) - - Eventually(func() int32 { return received.Load() }, "5s").Should(Equal(int32(1))) - }) - }) + // The "NATS cache invalidation" Context that stood here is deleted with the + // two builders it was the only caller of. + // + // Its two Its published on messaging.SubjectCacheInvalidateSkills and + // messaging.SubjectCacheInvalidateCollection through one client and counted + // their own deliveries back. Their own comment admitted the publish was + // simulated ("in production this is done by the service layer"), and no + // service layer published on either subject: the two subjects had no + // production publisher and no production subscriber, on any carrier. Only + // the model and backend caches have cross-replica invalidation (see + // galleryop.Service), and they are untouched here. + // + // So what these Its actually pinned was the carrier round trip, which + // core/services/pgbus/bus_test.go pins on two instances, and the subject + // literals, which core/services/messaging/subjects_wire_test.go now pins + // directly for every subject that survives. Context("Without --distributed", func() { It("should use filesystem without --distributed", func() { diff --git a/tests/e2e/distributed/testhelpers_test.go b/tests/e2e/distributed/testhelpers_test.go index 0049a77b8455..84b2f011281a 100644 --- a/tests/e2e/distributed/testhelpers_test.go +++ b/tests/e2e/distributed/testhelpers_test.go @@ -233,9 +233,14 @@ func SetupNATSOnly() *TestInfra { } // FlushNATS ensures all subscriptions are registered server-side before publishing. +// +// It asserts the server's verdict too, not only that the round trip completed: +// on a permission-enforcing server a denied SUB leaves the connection open and +// the flush succeeding, so a helper that checked the flush alone would let a +// spec proceed to publish into a subscription the server had already refused. func FlushNATS(nc *messaging.Client) { GinkgoHelper() - Expect(nc.Conn().Flush()).To(Succeed()) + Expect(nc.ConfirmRoundTrip(5 * time.Second)).To(Succeed()) } // Bus opens a broadcast carrier on THIS spec's database. From dc34e0ccc2bf4b6501784d3ff749a5eb36d2eac6 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 4 Sep 2026 05:18:02 +0000 Subject: [PATCH 097/153] feat(distributed): carry an agent cancel on the worker's own tunnel agent..cancel was the last family on a message bus, and the only reason an agent worker dialled one. Its subscriber is the worker running the execution, and a worker has no database, so the family could not move to the PostgreSQL fan-out carrier: a cancel published there would reach no worker while reporting that it had been sent. It is a control verb now. An agent worker mounts workerctl.PathAgentCancel on the loopback control plane behind its tunnel and applies the cancel to the same registry the executor registers a run on. The frontend issues it through nodes.AgentControlClient.CancelAgentRun. That call is a FAN-OUT and not a pick, because nothing records which worker holds a given execution: the claim row names the claiming replica, and it is deleted when the run ends. Every agent worker a live replica can reach is asked over its own tunnel, relayed by the peer mesh when a peer holds it, and each worker answers only for itself. The answers stay apart, which is why this family was held back. A cancel a worker made is nil. A cancel some worker could not be asked is ErrAgentCancelUndelivered, which is neither a refusal nor a missing run. A cancel every reachable worker declined to own is ErrAgentRunNotOnAnyWorker. A deployment with no agent worker is ErrNoAgentWorker. Neither new sentinel wraps ErrWorkerUnroutable and neither is a worker answer, so nothing is reaped, demoted or evicted because of a cancel. A worker in the ABSENT CONNECTION condition, one whose tunnel was lost inside the reconnect grace, counts as undelivered. It is not retried in the call and not queued: a retry would spend a budget the caller did not choose, and a queue would need durable state whose only consumer is a run whose control stream went with the tunnel. A worker whose departure has outlived the grace is the one routing fact a caller may act on and is excluded, or a single retired agent node would make every cancel undelivered for ever. The fan-out reads a different node set from the pick. A draining worker takes no new work but is still finishing what it holds, so it is offered the cancel; a pending one is refused by the tunnel route on every dial and is not. With that, nothing in LocalAI connects to NATS. The agent worker's dial, its credential ladder and its refresh loop are gone, and so is the frontend's cancel carrier. LOCALAI_NATS_URL is accepted and ignored everywhere, and distributed mode no longer requires it. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/agent_control_wiring.go | 7 +- core/application/agent_control_wiring_test.go | 36 ++++ core/application/dispatch_loop_wiring.go | 9 +- core/application/distributed.go | 121 +++++------- core/application/distributed_test.go | 15 +- core/application/fanout_wiring.go | 33 ++-- core/application/fanout_wiring_test.go | 83 ++++---- core/cli/agent_worker.go | 115 ++++------- core/cli/agent_worker_test.go | 97 ++++++--- core/cli/run.go | 4 +- core/config/distributed_config.go | 12 +- core/services/agents/dispatcher.go | 32 ++- core/services/agents/events.go | 186 +++++++++++------- core/services/agents/events_pg_test.go | 135 ++++++++----- core/services/agents/events_test.go | 2 +- core/services/agentworker/handlers.go | 16 +- core/services/agentworker/server_test.go | 13 +- core/services/messaging/client.go | 43 ++-- core/services/messaging/subjects.go | 33 ++++ core/services/nodes/agent_control.go | 106 ++++++++++ core/services/nodes/agent_control_test.go | 174 +++++++++++++++- core/services/nodes/agent_selector.go | 133 ++++++++++++- core/services/nodes/agent_selector_test.go | 158 +++++++++++++-- core/services/nodes/interfaces.go | 11 +- core/services/workerctl/paths.go | 7 +- docs/content/features/distributed-mode.md | 103 +++++----- docs/content/reference/cli-reference.md | 2 +- .../e2e/distributed/agent_distributed_test.go | 148 +++++++++++--- .../distributed/agent_native_executor_test.go | 76 +++---- tests/e2e/distributed/foundation_test.go | 50 +---- tests/e2e/distributed/sse_routes_test.go | 4 +- 31 files changed, 1369 insertions(+), 595 deletions(-) diff --git a/core/application/agent_control_wiring.go b/core/application/agent_control_wiring.go index f991ee87505f..af5e2113efd1 100644 --- a/core/application/agent_control_wiring.go +++ b/core/application/agent_control_wiring.go @@ -26,6 +26,11 @@ import ( // nothing to select from and a client with no transport reaches nobody, and // both would present as MCP being quietly unavailable in a deployment that // looks healthy. +// +// This client is also the deployment's agent CANCELLER, which is why the agent +// event bridge takes it: a cancel is a control RPC on the tunnels the workers +// hold, and the reconnect grace it is built with is what decides whether a +// worker that is not connected makes a cancel undelivered or is simply gone. func newAgentControl(cfg config.DistributedConfig, registry *nodes.NodeRegistry, conns nodes.AgentConnectionReader, control *nodes.ControlClient) (*nodes.AgentControlClient, error) { if cfg.InstanceID == "" { @@ -37,5 +42,5 @@ func newAgentControl(cfg config.DistributedConfig, registry *nodes.NodeRegistry, if control == nil { return nil, fmt.Errorf("the agent control client was built with no control transport to reach an agent worker over") } - return nodes.NewAgentControlClient(nodes.NewAgentSelector(registry, conns, cfg.InstanceID), control), nil + return nodes.NewAgentControlClient(nodes.NewAgentSelector(registry, conns, cfg.InstanceID, cfg.WorkerReconnectGrace), control), nil } diff --git a/core/application/agent_control_wiring_test.go b/core/application/agent_control_wiring_test.go index 3fb34022986c..f6a98354a720 100644 --- a/core/application/agent_control_wiring_test.go +++ b/core/application/agent_control_wiring_test.go @@ -5,12 +5,15 @@ package application import ( "context" "runtime" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/cluster" mcpremote "github.com/mudler/LocalAI/core/services/mcp" + "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/testutil" ) @@ -24,6 +27,12 @@ import ( type recordingConnections struct { owners []string seen chan string + // graces records the reconnect window each presence read was made with. It + // is the other argument with no other symptom: with a zero one every worker + // that lost its tunnel a moment ago reads as GONE, and a cancel addressed to + // it is reported as a run no worker is running rather than as one this + // deployment could not deliver. + graces []time.Duration } func (r *recordingConnections) ConnectedAmong(_ context.Context, _ []string, owner string) ([]string, []string, error) { @@ -37,6 +46,11 @@ func (r *recordingConnections) ConnectedAmong(_ context.Context, _ []string, own return nil, nil, nil } +func (r *recordingConnections) Presence(_ context.Context, _ string, grace time.Duration) (cluster.Presence, error) { + r.graces = append(r.graces, grace) + return cluster.PresenceReconnecting, nil +} + // newRecordingConnections returns a reader whose channel is ready BEFORE any // loop can be started against it. Creating it lazily from the spec goroutine // would race the loop's own goroutine reading it. @@ -89,6 +103,28 @@ var _ = Describe("building the frontend's agent control client", func() { Expect(conns.owners).To(ConsistOf("replica-7")) }) + It("makes the reconnect grace the one a cancel measures a lost tunnel against", func() { + // The other silent argument. With a zero grace every registered agent + // worker that is not connected right now reads as GONE, so a cancel + // addressed to one that is merely re-homing is reported as "no worker + // is running that execution" rather than as one that was not delivered. + // The only symptom is a wrong answer to a caller. + Expect(registry.Register(ctx, &nodes.BackendNode{ + Name: "agent-grace", NodeType: nodes.NodeTypeAgent, Address: "a:50051", + }, true)).To(Succeed()) + + conns := newRecordingConnections() + client, err := newAgentControl( + config.DistributedConfig{InstanceID: "replica-7", WorkerReconnectGrace: 7 * time.Minute}, + registry, conns, nodes.NewControlClient(nil, "token")) + Expect(err).ToNot(HaveOccurred()) + + // Driven through a real cancel, so what is asserted is the value that + // reached the presence read rather than one stored on a field. + _ = client.CancelAgentRun(ctx, messaging.AgentCancelRequest{AgentName: "a", MessageID: "m"}) + Expect(conns.graces).To(ConsistOf(7 * time.Minute)) + }) + It("refuses to build with no instance id", func() { _, err := newAgentControl(config.DistributedConfig{}, registry, newRecordingConnections(), nodes.NewControlClient(nil, "token")) diff --git a/core/application/dispatch_loop_wiring.go b/core/application/dispatch_loop_wiring.go index b34fefc39479..1dbf05e12d22 100644 --- a/core/application/dispatch_loop_wiring.go +++ b/core/application/dispatch_loop_wiring.go @@ -48,6 +48,13 @@ import ( // The SELECTOR is built here rather than borrowed from newAgentControl, and // deliberately: nodes.AgentSelector holds no per-caller state, and sharing one // would couple the dispatch loop's lifetime to MCP's for nothing. +// +// The reconnect grace it is built with is INERT on this path and is passed +// correctly anyway. The selector reads it only in Reachable, which is what a +// fan-out verb (an agent cancel) asks; this loop asks PickConnected, which +// never needs it, because a worker that is not connected cannot be picked +// whatever the reason. Passing a value this loop cannot observe is still +// cheaper than a second constructor. func startJobDispatchLoop(ctx context.Context, cfg config.DistributedConfig, db *gorm.DB, store *jobs.JobStore, registry *nodes.NodeRegistry, conns nodes.AgentConnectionReader, control *nodes.ControlClient, broadcast *nodes.Rebroadcaster) (*jobs.DispatchLoop, error) { @@ -63,7 +70,7 @@ func startJobDispatchLoop(ctx context.Context, cfg config.DistributedConfig, db loop, err := jobs.NewDispatchLoop(jobs.DispatchConfig{ DB: db, Owner: cfg.InstanceID, - Selector: nodes.NewAgentSelector(registry, conns, cfg.InstanceID), + Selector: nodes.NewAgentSelector(registry, conns, cfg.InstanceID, cfg.WorkerReconnectGrace), Control: control, // The allow list lives in nodes and is keyed on the worker's node type; // nothing here decides what a worker may broadcast on. diff --git a/core/application/distributed.go b/core/application/distributed.go index ee543713fd57..12271e3295d5 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -32,26 +32,6 @@ import ( // DistributedServices holds all services initialized for distributed mode. type DistributedServices struct { - // CancelCarrier is the NATS connection, and it is here for exactly one - // family: agent..cancel. - // - // Every other family this deployment fans out on moved to Bus below. That - // one could not, and the reason is structural rather than incidental: its - // only subscriber is the agent WORKER, which has no database and so cannot - // join a carrier that rides PostgreSQL. A cancel published on Bus would be - // published where no worker listens, succeed, and be reported as sent. - // - // It is named CancelCarrier and not Nats so that reaching for it to carry - // anything else has to be a decision. The field used to be Nats, and - // *messaging.Client satisfies messaging.Broadcaster, so any adopter that - // took it instead of Broadcast() compiled, started, published, and was - // delivered onto the carrier this deployment is being taken off, with no - // error anywhere. The client's queue and request/reply halves are deleted - // now, so the smaller mistakes are build failures; this name is what is - // left to make the remaining one visible. - // - // It goes when a cancel rides the worker's tunnel as a control verb. - CancelCarrier *messaging.Client Store storage.ObjectStore Registry *nodes.NodeRegistry Router *nodes.SmartRouter @@ -155,11 +135,9 @@ func (ds *DistributedServices) Shutdown() { if closer, ok := ds.Store.(io.Closer); ok { closer.Close() } - // AgentBridge has no Close method: the cancel subscription it holds is - // cleaned up when the carrier below is closed. - if ds.CancelCarrier != nil { - ds.CancelCarrier.Close() - } + // AgentBridge has no Close method and needs none: it holds no + // process-lifetime subscription of its own beyond the observable + // persister, whose carrier is closed below. // The broadcast carrier holds a PostgreSQL session pinned for the life // of the process, plus the goroutine parked on it. A replica that // leaves one behind on every restart runs the server out of @@ -202,30 +180,16 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } xlog.Info("Distributed instance", "id", cfg.Distributed.InstanceID) - // Connect to the cancel carrier. + // No message bus is dialled here, and there is none left to dial. The last + // family that needed one was agent..cancel, whose subscriber is an + // agent worker that has no database and so could not join the broadcast + // carrier below; it is now a control RPC on the tunnel that worker holds. + // A distributed deployment needs PostgreSQL and the frontends' own HTTP + // listener, and nothing else. // - // This is the ONE bus connection a frontend replica still opens, and it - // carries one family: agent..cancel, whose subscriber is an agent - // worker that has no database and cannot read the broadcast carrier opened - // below. Everything else a replica fans out travels on that one. - natsAuth := cfg.Distributed.NatsAuthConfig() - if natsAuth.RequireAuth && (natsAuth.ServiceUserJWT == "" || natsAuth.ServiceUserSeed == "") { - return nil, fmt.Errorf("LOCALAI_NATS_REQUIRE_AUTH requires LOCALAI_NATS_SERVICE_JWT and LOCALAI_NATS_SERVICE_SEED") - } - natsOpts := cfg.Distributed.NatsMessagingOptions("", "") - natsClient, err := messaging.New(cfg.Distributed.NatsURL, natsOpts...) - if err != nil { - return nil, fmt.Errorf("connecting to NATS: %w", err) - } - xlog.Info("Connected to the agent cancel carrier", "url", sanitize.URL(cfg.Distributed.NatsURL)) - - // Ensure the carrier is closed if any subsequent initialization step fails. + // success guards the carriers opened below, which must not be left pinned + // when a later initialization step fails. success := false - defer func() { - if !success { - natsClient.Close() - } - }() // Initialize object storage var store storage.ObjectStore @@ -451,28 +415,6 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } xlog.Info("Distributed agent store initialized") - // The job dispatcher and the agent event bridge, both on the broadcast - // carrier. See newFanoutBridges for why the two constructors are reached - // through one function that names *pgbus.Bus. - dispatcher, agentBridge, rebroadcast, err := newFanoutBridges(bus, natsClient, jobStore, agentStore, authDB, cfg.Distributed.InstanceID) - if err != nil { - return nil, err - } - - // Initialize Phase 4 stores (MCP, Gallery, FineTune, Skills) - distStores, err := distributed.InitStores(authDB) - if err != nil { - return nil, fmt.Errorf("initializing distributed stores: %w", err) - } - - // Initialize file manager with local cache - cacheDir := cfg.DataPath + "/cache" - fileMgr, err := storage.NewFileManager(store, cacheDir) - if err != nil { - return nil, fmt.Errorf("initializing file manager: %w", err) - } - xlog.Info("File manager initialized", "cacheDir", cacheDir) - // The frontend's control plane client. It reaches every worker over that // worker's own tunnel, on the same `http` stream tag the file stager below // uses, so a control RPC to a worker another replica holds is relayed the @@ -494,6 +436,33 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade return nil, fmt.Errorf("wiring the agent control client: %w", err) } + // The job dispatcher and the agent event bridge, both on the broadcast + // carrier. See newFanoutBridges for why the two constructors are reached + // through one function that names *pgbus.Bus. + // + // The bridge takes agentControl and not a carrier: a cancel is the one + // family whose far end is an agent worker, and it now rides that worker's + // tunnel as a control RPC. It is built ABOVE for that reason, rather than + // with the rest of the control plane below. + dispatcher, agentBridge, rebroadcast, err := newFanoutBridges(bus, agentControl, jobStore, agentStore, authDB, cfg.Distributed.InstanceID) + if err != nil { + return nil, err + } + + // Initialize Phase 4 stores (MCP, Gallery, FineTune, Skills) + distStores, err := distributed.InitStores(authDB) + if err != nil { + return nil, fmt.Errorf("initializing distributed stores: %w", err) + } + + // Initialize file manager with local cache + cacheDir := cfg.DataPath + "/cache" + fileMgr, err := storage.NewFileManager(store, cacheDir) + if err != nil { + return nil, fmt.Errorf("initializing file manager: %w", err) + } + xlog.Info("File manager initialized", "cacheDir", cacheDir) + // The consumer side of the claim queue, built and started in one act: see // startJobDispatchLoop for why those are not two lines. jobDispatch, err := startJobDispatchLoop(cfg.Context, cfg.Distributed, authDB, jobStore, registry, clusterRegistry, controlClient, rebroadcast) @@ -720,7 +689,6 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade success = true ds := &DistributedServices{ - CancelCarrier: natsClient, Store: store, Registry: registry, Router: router, @@ -784,15 +752,14 @@ func requireBroadcastCarrier(ds *DistributedServices) error { // carrier, and it exists so that "this family travels on the broadcast carrier // and not on NATS" is decided once instead of at every adopter. // -// It was five sites before this: the fine-tune service, the quantization +// It was five field reads before this: the fine-tune service, the quantization // service, the agent-task setter (twice, on two startup paths), the per-user // services manager and the Open Responses store. Every one of them takes a -// messaging.Broadcaster, and *messaging.Client satisfies that interface too, so -// a site left holding ds.CancelCarrier compiles, starts, publishes and is -// delivered - onto a carrier only agent workers read, and only for cancels. -// Nothing would fail until NATS went away. Collapsing the choice to one -// function makes it a fact a spec can pin, which five scattered field reads -// were not. +// messaging.Broadcaster, and the struct used to carry a second field that +// satisfied it, so a site that reached for the wrong one compiled, started, +// published and was delivered onto a carrier almost nothing read. That second +// field is gone with the last family that needed a bus. Collapsing the choice +// to one function is what keeps it a fact a spec can pin. // // The return is the interface and not *pgbus.Bus on purpose: handing a nil // *pgbus.Bus to an adopter would produce a non-nil interface wrapping a nil diff --git a/core/application/distributed_test.go b/core/application/distributed_test.go index d3eb8167a344..95449c4b0e07 100644 --- a/core/application/distributed_test.go +++ b/core/application/distributed_test.go @@ -123,7 +123,7 @@ var _ = Describe("shutting the distributed services down", func() { // NATS did. Collapsing the choice into one function is what makes it a fact // these specs can hold. var _ = Describe("handing the broadcast carrier to its adopters", func() { - It("returns the carrier the deployment opened and never the cancel carrier", func() { + It("returns the carrier the deployment opened", func() { db, dsn := testutil.SetupTestDBWithDSN() cfg := &config.ApplicationConfig{} cfg.Auth.DatabaseURL = dsn @@ -131,13 +131,12 @@ var _ = Describe("handing the broadcast carrier to its adopters", func() { Expect(err).ToNot(HaveOccurred()) DeferCleanup(bus.Close) - // The cancel carrier is present on the struct, exactly as it is in a - // real deployment: agent..cancel is the one family that could not - // move, because its only subscriber is an agent worker and a worker has - // no database. Identity, not "is a Broadcaster": both fields satisfy - // that interface, which is the whole reason a field read could pick the - // wrong one and stay green. - ds := &DistributedServices{CancelCarrier: &messaging.Client{}, Bus: bus} + // Identity and not "is a Broadcaster". There is no second carrier on + // this struct any more: the family that needed one, agent..cancel, + // rides the agent worker's own tunnel now. The identity assertion stays + // because what it pins is that adopters get THIS bus rather than + // anything else that satisfies the interface. + ds := &DistributedServices{Bus: bus} Expect(ds.Broadcast()).To(BeIdenticalTo(messaging.Broadcaster(bus))) }) diff --git a/core/application/fanout_wiring.go b/core/application/fanout_wiring.go index b14d94300e44..15ef31ca907f 100644 --- a/core/application/fanout_wiring.go +++ b/core/application/fanout_wiring.go @@ -7,7 +7,6 @@ import ( "github.com/mudler/LocalAI/core/services/agents" "github.com/mudler/LocalAI/core/services/jobs" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/pgbus" "github.com/mudler/xlog" @@ -44,15 +43,16 @@ import ( // Written as one expression shared with the dispatcher and the bridge, that // mis-wiring stops being a line a spec has to guess at: there is no second // carrier in scope to point it at. -// cancelCarrier is NOT the same carrier and must not be folded into bus. Every -// family this function wires has both of its ends on a frontend replica and so -// moves to the broadcast carrier whole, with one exception: the only subscriber -// to agent..cancel is the agent WORKER, which has no database and cannot -// join the PostgreSQL carrier at all. A cancel published on bus would therefore -// reach no worker, and every cancel of a worker-run agent would be lost while -// reporting success. It stays on the carrier the worker reads until a cancel -// rides the worker's tunnel instead of a broadcast. -func newFanoutBridges(bus *pgbus.Bus, cancelCarrier messaging.Broadcaster, +// canceller is NOT a carrier and must never become one. Every family this +// function wires has both of its ends on a frontend replica and so travels on +// the broadcast carrier, with one exception: the process that holds a +// worker-run agent's cancel function is the agent WORKER, which has no database +// and cannot join the PostgreSQL carrier at all. A cancel published on bus +// would reach no worker, and every cancel of a worker-run agent would be lost +// while reporting success. So it does not travel on bus, or on any bus: it is a +// control RPC on the tunnel the worker already holds, issued by the agent +// control client this takes. +func newFanoutBridges(bus *pgbus.Bus, canceller agents.AgentWorkerCanceller, jobStore *jobs.JobStore, agentStore *agents.AgentStore, db *gorm.DB, instanceID string) (*jobs.Dispatcher, *agents.EventBridge, *nodes.Rebroadcaster, error) { // A nil check on the CONCRETE pointer, before it is widened. Once it is a @@ -63,12 +63,12 @@ func newFanoutBridges(bus *pgbus.Bus, cancelCarrier messaging.Broadcaster, return nil, nil, nil, fmt.Errorf("the job and agent fan-out bridges were built with no broadcast carrier: every job's progress and every agent's events would reach no SSE stream in the deployment") } - if cancelCarrier == nil { - return nil, nil, nil, fmt.Errorf("the agent event bridge was built with no carrier for agent cancels: every cancel of a worker-run agent would be published where no worker listens and reported as sent") + if canceller == nil { + return nil, nil, nil, fmt.Errorf("the agent event bridge was built with no way to cancel a worker-run agent: every cancel of one would be sent nowhere") } dispatcher := jobs.NewDispatcher(jobStore, bus, db, instanceID) - bridge := agents.NewEventBridge(bus, agentStore, instanceID).WithCancelCarrier(cancelCarrier) + bridge := agents.NewEventBridge(bus, agentStore, instanceID, canceller) // Warned rather than refused, and deliberately: the persister needs a store // and a deployment without one still serves live SSE correctly. What it @@ -82,3 +82,10 @@ func newFanoutBridges(bus *pgbus.Bus, cancelCarrier messaging.Broadcaster, return dispatcher, bridge, nodes.NewRebroadcaster(bus), nil } + +// The real canceller, asserted where both packages are already imported. +// +// agents may not import nodes, so the port there is satisfied structurally and +// a signature drift between the two would otherwise surface as a nil field in +// this file's argument list, which compiles. Here it is a build failure. +var _ agents.AgentWorkerCanceller = (*nodes.AgentControlClient)(nil) diff --git a/core/application/fanout_wiring_test.go b/core/application/fanout_wiring_test.go index 6c3a77c3a2aa..8f13fb7ac96e 100644 --- a/core/application/fanout_wiring_test.go +++ b/core/application/fanout_wiring_test.go @@ -61,17 +61,18 @@ var _ = Describe("wiring the job and agent fan-out bridges", func() { Expect(err).ToNot(HaveOccurred()) }) - // The cancel carrier is a SEPARATE argument, and its absence is refused - // separately. Folding it into bus would put every agent cancel on a carrier - // no worker can read, and CancelExecution would go on returning nil. - It("refuses to build with no carrier for agent cancels", func() { + // The canceller is a SEPARATE argument, and its absence is refused + // separately. A bridge built without one has nowhere to send the cancel of + // an agent running on a worker, and the failure would present as + // CancelExecution reporting success on a cancel that reached nobody. + It("refuses to build with no way to cancel a worker-run agent", func() { _, _, _, err := newFanoutBridges(busA, nil, jobStore, agentStore, db, "replica-1") Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("agent cancels")) + Expect(err.Error()).To(ContainSubstring("cancel a worker-run agent")) }) It("refuses to build with no carrier", func() { - _, _, _, err := newFanoutBridges(nil, testutil.NewFakeBus(), jobStore, agentStore, db, "replica-1") + _, _, _, err := newFanoutBridges(nil, &stubCanceller{}, jobStore, agentStore, db, "replica-1") Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("broadcast carrier")) }) @@ -80,7 +81,7 @@ var _ = Describe("wiring the job and agent fan-out bridges", func() { // it can only do if its wildcard subscription is on the carrier the peer // published to. It("subscribes the job dispatcher to results a peer replica broadcasts", func() { - dispatcher, _, _, err := newFanoutBridges(busA, testutil.NewFakeBus(), jobStore, agentStore, db, "replica-1") + dispatcher, _, _, err := newFanoutBridges(busA, &stubCanceller{}, jobStore, agentStore, db, "replica-1") Expect(err).ToNot(HaveOccurred()) Expect(dispatcher.Start(ctx)).To(Succeed()) DeferCleanup(dispatcher.Stop) @@ -101,44 +102,35 @@ var _ = Describe("wiring the job and agent fan-out bridges", func() { }, "20s").Should(Equal("completed")) }) - // S1b. WHICH carrier the cancel family rides, asserted by receipt on both - // carriers rather than by reading the argument list. + // S1b. A cancel does NOT travel on a carrier, asserted by where it goes and + // by where it does not. // - // The nil refusal above only says a carrier was passed. Passing bus for it - // - one token's difference at the one call site, and the natural edit for - // anyone finishing the migration - satisfies that refusal, compiles, and - // publishes successfully onto the PostgreSQL carrier, where the agent - // worker that has to act on the cancel is not and cannot be: it has no - // database. Every unit suite in agents and in application stays green and - // every cancel of a worker-run agent is lost while CancelExecution returns - // nil. + // The nil refusal above only says a canceller was passed. What it cannot + // say is that the bridge uses it instead of publishing onto the broadcast + // carrier, which is the edit anyone finishing this migration would reach + // for: it compiles, it publishes successfully onto PostgreSQL, and the + // agent worker that has to act on the cancel is not and cannot be there. + // Every unit suite stays green and every cancel of a worker-run agent is + // lost while CancelExecution returns nil. // - // So this asserts the cancel ARRIVES on the worker's carrier and, in the - // same spec, that it does NOT arrive on a peer replica's broadcast carrier. - // The negative half is the load-bearing one: the positive half alone passes - // for a bridge wired to both. - It("publishes agent cancels on the worker's carrier and not on the broadcast carrier", func() { - workerCarrier := testutil.NewFakeBus() - _, bridge, _, err := newFanoutBridges(busA, workerCarrier, jobStore, agentStore, db, "replica-1") + // So this asserts the cancel reaches the CANCELLER and, in the same spec, + // that nothing is published on a peer replica's broadcast carrier. The + // negative half is the load-bearing one: the positive half alone passes for + // a bridge that does both. + It("sends an agent cancel to the agent workers and publishes nothing on the broadcast carrier", func() { + canceller := &stubCanceller{} + _, bridge, _, err := newFanoutBridges(busA, canceller, jobStore, agentStore, db, "replica-1") Expect(err).ToNot(HaveOccurred()) onBroadcast := make(chan []byte, 4) _, err = busB.Subscribe(messaging.SubjectAgentCancelWildcard, func(data []byte) { onBroadcast <- data }) Expect(err).ToNot(HaveOccurred()) - onWorker := make(chan []byte, 4) - _, err = workerCarrier.Subscribe(messaging.SubjectAgentCancelWildcard, func(data []byte) { onWorker <- data }) - Expect(err).ToNot(HaveOccurred()) - - Expect(bridge.CancelExecution("a1", "u1", "msg-1")).To(Succeed()) + Expect(bridge.CancelExecution(ctx, "a1", "u1", "msg-1")).To(Succeed()) - // Expect and not Eventually: the worker's carrier here is the shared - // in-memory double, which delivers synchronously inside Publish, so by - // the time CancelExecution has returned the handler has already run. An - // Eventually would turn a mis-wiring into a timeout, which reports as - // slowness rather than as the wiring fact it is. - Expect(onWorker).To(Receive(), - "the agent cancel did not reach the worker's carrier; a worker has no database and cannot read the broadcast carrier, so this cancel reached nobody and was reported as sent") + Expect(canceller.requests).To(ConsistOf(messaging.AgentCancelRequest{ + AgentName: "a1", UserID: "u1", MessageID: "msg-1", + }), "the cancel did not reach the agent workers, so it reached nobody and was reported as sent") Consistently(onBroadcast, "2s").ShouldNot(Receive(), "the agent cancel was published on the broadcast carrier, where no agent worker is or can be subscribed") }) @@ -147,7 +139,7 @@ var _ = Describe("wiring the job and agent fan-out bridges", func() { // only do if it was started AND is on the same carrier AND its filter has // the right number of tokens. It("subscribes the agent observable persister to events a peer replica broadcasts", func() { - _, bridge, _, err := newFanoutBridges(busA, testutil.NewFakeBus(), jobStore, agentStore, db, "replica-1") + _, bridge, _, err := newFanoutBridges(busA, &stubCanceller{}, jobStore, agentStore, db, "replica-1") Expect(err).ToNot(HaveOccurred()) Expect(bridge).ToNot(BeNil()) @@ -180,7 +172,7 @@ var _ = Describe("wiring the job and agent fan-out bridges", func() { // value, which is true for a publish that went nowhere. DescribeTable("re-broadcasts a worker's line onto the carrier a peer replica reads", func(subject string, payload string) { - _, _, rebroadcast, err := newFanoutBridges(busA, testutil.NewFakeBus(), jobStore, agentStore, db, "replica-1") + _, _, rebroadcast, err := newFanoutBridges(busA, &stubCanceller{}, jobStore, agentStore, db, "replica-1") Expect(err).ToNot(HaveOccurred()) delivered := make(chan []byte, 4) @@ -196,7 +188,7 @@ var _ = Describe("wiring the job and agent fan-out bridges", func() { ) It("re-broadcasts an agent's events onto the carrier a peer replica reads", func() { - _, _, rebroadcast, err := newFanoutBridges(busA, testutil.NewFakeBus(), jobStore, agentStore, db, "replica-1") + _, _, rebroadcast, err := newFanoutBridges(busA, &stubCanceller{}, jobStore, agentStore, db, "replica-1") Expect(err).ToNot(HaveOccurred()) delivered := make(chan []byte, 4) @@ -209,3 +201,16 @@ var _ = Describe("wiring the job and agent fan-out bridges", func() { Eventually(delivered, "20s").Should(Receive(MatchJSON(`{"event_type":"json_message"}`))) }) }) + +// stubCanceller stands in for the frontend's agent control client, which +// reaches workers over their tunnels and is driven over a real one in +// core/services/nodes. What these specs need from it is only that the bridge +// asks it at all. +type stubCanceller struct { + requests []messaging.AgentCancelRequest +} + +func (s *stubCanceller) CancelAgentRun(_ context.Context, req messaging.AgentCancelRequest) error { + s.requests = append(s.requests, req) + return nil +} diff --git a/core/cli/agent_worker.go b/core/cli/agent_worker.go index 161e67b89571..6df16fc21854 100644 --- a/core/cli/agent_worker.go +++ b/core/cli/agent_worker.go @@ -20,7 +20,6 @@ import ( "github.com/mudler/LocalAI/core/services/jobs" mcpRemote "github.com/mudler/LocalAI/core/services/mcp" "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/pkg/sanitize" "github.com/mudler/cogito" "github.com/mudler/cogito/clients" "github.com/mudler/xlog" @@ -40,20 +39,23 @@ import ( // control verbs by RPC without the worker opening an inbound port. No verb the // frontend addresses to THIS worker travels on the bus any more. // -// --nats-url is still required, and for one thing only: agent..cancel. -// That family runs the other way, from a frontend replica to whichever worker -// holds the execution, and it could not move to the broadcast carrier because -// that carrier rides PostgreSQL and this process has no database. A worker that -// came up without a bus would register, serve, run agents and ignore every -// cancel, with nothing in the deployment reporting it. The flag goes when a -// cancel rides the tunnel as a control verb. +// It dials NO message bus. The last family that needed one was +// agent..cancel, which ran the other way, from a frontend replica to +// whichever worker held the execution; it is now a control verb on this +// worker's own tunnel (workerctl.PathAgentCancel), so a cancel reaches the +// worker running the agent without either side touching a broker. // // Usage: // -// localai agent-worker --nats-url nats://... --register-to http://localai:8080 +// localai agent-worker --register-to http://localai:8080 type AgentWorkerCMD struct { - // NATS (required) - NatsURL string `env:"LOCALAI_NATS_URL" required:"" help:"NATS server URL" group:"distributed"` + // NatsURL is accepted and ignored, exactly as the backend worker's is (see + // core/services/worker/config.go). An agent worker connects to no message + // bus: every verb a frontend addresses to it arrives on the tunnel it + // dials, and a cancel now arrives the same way. It stays here, without + // required, so an existing command line or unit file that still carries + // --nats-url starts rather than failing to parse. + NatsURL string `env:"LOCALAI_NATS_URL" help:"Ignored. An agent worker connects to no message bus; the frontend reaches it over its outbound tunnel. Accepted so an existing worker command line still starts." group:"distributed" hidden:""` // Registration (required) RegisterTo string `env:"LOCALAI_REGISTER_TO" required:"" help:"Frontend URL for registration" group:"registration"` @@ -88,7 +90,7 @@ func (cmd *AgentWorkerCMD) natsAuthRequired() bool { } func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { - xlog.Info("Starting agent worker", "nats", sanitize.URL(cmd.NatsURL), "register_to", cmd.RegisterTo) + xlog.Info("Starting agent worker", "register_to", cmd.RegisterTo) // Resolve API URL apiURL := cmp.Or(cmd.APIURL, strings.TrimRight(cmd.RegisterTo, "/")) @@ -117,9 +119,14 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { shutdownCtx, shutdownCancel := context.WithCancel(context.Background()) defer shutdownCancel() - // Acquire credentials via (re)registration. When the bus requires auth and no - // static fallback is configured, wait through admin approval until the - // frontend mints credentials rather than starting unauthenticated. + // Register, and obtain this node's identity and its tunnel credential. + // + // The manager is still the NATS credential manager and still gated on the + // NATS auth flags, and that is deliberate rather than left over: what the + // gate decides is whether registration WAITS THROUGH ADMIN APPROVAL instead + // of returning a pending response, and that behaviour is unchanged by this + // worker no longer dialling a bus. What it no longer does is dial one: the + // only value read off it below is TunnelToken. credMgr := workerregistry.NewNATSCredentialManager( func(ctx context.Context) (*workerregistry.RegisterResponse, error) { return regClient.RegisterFull(ctx, registrationBody) @@ -147,42 +154,6 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { go regClient.HeartbeatLoop(shutdownCtx, nodeID, heartbeatInterval, func() map[string]any { return map[string]any{} }) - // Resolve the cancel carrier's credentials with precedence: explicit env - // override, then frontend-minted (auto-refreshed before expiry), then - // service fallback. Each static source must supply JWT and seed together. - natsTLS := messaging.TLSFiles{CA: cmd.NatsTLSCA, Cert: cmd.NatsTLSCert, Key: cmd.NatsTLSKey} - var natsOpts []messaging.Option - switch { - case cmd.NatsJWT != "" || cmd.NatsUserSeed != "": - if (cmd.NatsJWT == "") != (cmd.NatsUserSeed == "") { - return fmt.Errorf("LOCALAI_NATS_JWT and LOCALAI_NATS_USER_SEED must be set together") - } - natsOpts = append(natsOpts, messaging.WithUserJWT(cmd.NatsJWT, cmd.NatsUserSeed)) - case credMgr.HasCredentials(): - natsOpts = append(natsOpts, messaging.WithUserJWTProvider(credMgr.Provider())) - go func() { - if err := credMgr.RefreshLoop(shutdownCtx); err != nil { - xlog.Error("NATS credential refresh permanently failed; shutting down agent worker", "error", err) - shutdownCancel() - } - }() - case cmd.NatsServiceJWT != "" || cmd.NatsServiceSeed != "": - if (cmd.NatsServiceJWT == "") != (cmd.NatsServiceSeed == "") { - return fmt.Errorf("LOCALAI_NATS_SERVICE_JWT and LOCALAI_NATS_SERVICE_SEED must be set together") - } - natsOpts = append(natsOpts, messaging.WithUserJWT(cmd.NatsServiceJWT, cmd.NatsServiceSeed)) - case cmd.natsAuthRequired(): - return fmt.Errorf("NATS JWT+seed required: enable frontend minting or set LOCALAI_NATS_* env vars") - } - if natsTLS.Enabled() { - natsOpts = append(natsOpts, messaging.WithTLS(natsTLS)) - } - natsClient, err := messaging.New(cmd.NatsURL, natsOpts...) - if err != nil { - return fmt.Errorf("connecting to NATS: %w", err) - } - defer natsClient.Close() - // The executor and the event bridge the control plane serves, built BEFORE // the tunnel because a verb mounted with a nil handler answers a 404, which // a frontend reads as a worker too old to serve it. @@ -190,7 +161,7 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { // No ConfigProvider and no SkillStore: config and skills arrive in the // request body, exactly as they arrived in the job payload before, because // an agent worker still has no database. - eventBridge := agents.NewEventBridge(natsClient, nil, "agent-worker-"+nodeID) + eventBridge := agents.NewWorkerEventBridge("agent-worker-" + nodeID) executor := agents.NewWorkerExecutor(eventBridge, nil, apiURL, cmd.APIToken) mcpCIJobTimeout, err := time.ParseDuration(cmd.MCPCIJobTimeout) @@ -212,9 +183,9 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { // control plane rides the tunnel it holds. // // The credential is read through credMgr rather than captured from res, - // because every re-registration the manager performs ROTATES it and a - // captured value would lock this worker out of its own tunnel at the first - // JWT refresh. + // because every registration the manager performs ROTATES it: the frontend + // stores only the hash of the newest one, so a captured value would lock + // this worker out of its own tunnel after any re-registration. // // It is started AFTER registration, which is what supplies both the node // identity the dial names and the credential it presents. @@ -234,39 +205,20 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { } }() - // The cancel listener is the ONE thing still on the bus here, and the only - // reason this process dialled one. A cancel is a broadcast to every replica - // and every worker, because the replica holding the run is not the one the - // cancel request lands on, and this worker cannot join the carrier the rest - // of the deployment fans out on: that carrier is the auth database, and an - // agent worker has no database access at all. - cancelSub, err := eventBridge.StartCancelListener() - if err != nil { - xlog.Warn("Failed to start cancel listener", "error", err) - } else { - defer func() { _ = cancelSub.Unsubscribe() }() - } - xlog.Info("Agent worker ready, serving agent execution and MCP CI runs on its tunnel", "node", nodeID) - // Wait for an OS signal or an internal fatal condition (e.g. NATS - // credentials became unrenewable), so the worker restarts and re-acquires - // rather than lingering unable to serve. + // Wait for an OS signal. There is no internal fatal condition left to wait + // on: the one that existed was a NATS credential this worker could no + // longer renew, and it renews none. sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - var runErr error - select { - case <-sigCh: - case <-shutdownCtx.Done(): - runErr = fmt.Errorf("agent worker shutting down: NATS credentials unavailable") - xlog.Error("Internal shutdown requested", "error", runErr) - } + <-sigCh xlog.Info("Shutting down agent worker") shutdownCancel() // stop heartbeat loop immediately mcpTools.CloseAllMCPSessions() regClient.GracefulDeregister(nodeID) - return runErr + return nil } // The MCP verbs, written ONCE and served on two carriers. @@ -612,6 +564,11 @@ func agentWorkerControlHandlers(executor *agents.WorkerExecutor, apiURL, apiToke return agentworker.Config{ MCPTool: serveMCPToolRequest, MCPDiscovery: serveMCPDiscoveryRequest, + // The verb that removed this process's last reason to dial a bus. It + // reaches the SAME cancel registry the executor registers a run on, + // because it is the same bridge: a cancel arriving on the tunnel has to + // find an execution that is publishing onto a control stream. + AgentCancel: executor.Cancel, // Drops the MCP sessions cached for a backend that went away, on the // path a backend worker serves by killing the process instead. BackendStop: dropMCPSessionsForBackend, diff --git a/core/cli/agent_worker_test.go b/core/cli/agent_worker_test.go index 7d2617d6ca07..1e537999ad50 100644 --- a/core/cli/agent_worker_test.go +++ b/core/cli/agent_worker_test.go @@ -2,6 +2,7 @@ package cli import ( "bufio" + "bytes" "context" "encoding/json" "net/http" @@ -17,7 +18,6 @@ import ( "github.com/mudler/LocalAI/core/services/agents" mcpRemote "github.com/mudler/LocalAI/core/services/mcp" "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/core/services/testutil" "github.com/mudler/LocalAI/core/services/workerctl" ) @@ -83,14 +83,14 @@ var _ = Describe("The agent worker's backend stop", func() { // verb rather than as a wiring mistake. Nothing else in this repo would notice. var _ = Describe("The agent worker's control-plane wiring", func() { var base string + var bridge *agents.EventBridge BeforeEach(func() { mux := http.NewServeMux() // Built exactly as Run builds it, from an executor and the MCP CI // timeout, so a field this function forgets to set is a 404 here. - executor := agents.NewWorkerExecutor( - agents.NewEventBridge(testutil.NewFakeBus(), nil, "agent-worker-spec"), - nil, "http://127.0.0.1:1", "token") + bridge = agents.NewWorkerEventBridge("agent-worker-spec") + executor := agents.NewWorkerExecutor(bridge, nil, "http://127.0.0.1:1", "token") agentWorkerControlHandlers(executor, "http://127.0.0.1:1", "token", time.Second).Register(mux) srv := httptest.NewServer(mux) DeferCleanup(srv.Close) @@ -113,8 +113,50 @@ var _ = Describe("The agent worker's control-plane wiring", func() { // in the tree can tell the two apart and only this spec can. Entry("agent execute", workerctl.PathAgentExecute), Entry("mcp ci run", workerctl.PathMCPCIRun), + // The verb that removed this process's last reason to dial a bus. + // Unwired it answers a 404, which the frontend reads as a worker too + // old to serve it, and every cancel of an agent this worker is running + // is then reported as one that could not be delivered - for ever. + Entry("agent cancel", workerctl.PathAgentCancel), ) + // The cancel verb end to end through the mux, because the thing that must + // be true is that the path reaches the SAME cancel registry the executor + // registers a run on. Two bridges would compile, mount, answer 200 and + // cancel nothing. + It("cancels a run registered on the executor's own bridge, and says so", func() { + cancelled := make(chan struct{}) + bridge.RegisterCancel("msg-1", func() { close(cancelled) }) + + post := func(messageID string) messaging.AgentCancelReply { + GinkgoHelper() + body, err := json.Marshal(messaging.AgentCancelRequest{AgentName: "a1", UserID: "u1", MessageID: messageID}) + Expect(err).ToNot(HaveOccurred()) + resp, err := http.Post(base+workerctl.PathAgentCancel, "application/json", bytes.NewReader(body)) //nolint:gosec,noctx // httptest server, no redirects to follow + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply messaging.AgentCancelReply + Expect(json.NewDecoder(resp.Body).Decode(&reply)).To(Succeed()) + return reply + } + + Expect(post("msg-nobody-is-running").Cancelled).To(BeFalse(), + "a worker answers only for itself, and false is that answer rather than a failure") + Expect(post("msg-1").Cancelled).To(BeTrue()) + Eventually(cancelled, "20s").Should(BeClosed()) + }) + + It("fails to serve a cancel it cannot read, rather than answering that it found nothing", func() { + // The two are different facts. A 200 with cancelled false would be read + // as this worker's own answer about the run; a body it could not decode + // is not an answer about anything. + resp, err := http.Post(base+workerctl.PathAgentCancel, "application/json", strings.NewReader(`{"message_id":`)) //nolint:gosec,noctx // httptest server, no redirects to follow + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = resp.Body.Close() }) + Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError)) + }) + DescribeTable("answers a dispatched verb as a stream, so progress and the terminal line share one body", func(path string) { resp, err := http.Post(base+path, "application/json", strings.NewReader(`{}`)) //nolint:gosec,noctx // httptest server, no redirects to follow @@ -147,28 +189,23 @@ var _ = Describe("The agent worker's control-plane wiring", func() { ) }) -// The last reason an agent worker dials a message bus, pinned so that removing -// it is a decision rather than an accident. +// The agent worker dials NO message bus, pinned so that a flag reappearing as +// required is a decision rather than an accident. // -// Every verb a frontend addresses to THIS worker now arrives on the tunnel it -// dials, and the two queue groups it used to join are gone. One family is left, -// in the other direction: agent..cancel. Its publisher is a frontend -// replica and its ONLY subscriber is this process, and a worker has no database -// and so cannot join the PostgreSQL carrier every other family moved to. A -// worker that came up without a bus would register, serve, run agents, and -// ignore every cancel, returning nothing to say so - the cancel would be -// published, would succeed, and would reach nobody. +// Every verb a frontend addresses to this worker arrives on the tunnel it +// dials, and that now includes the cancel: agent..cancel was the last +// family in the other direction, from a frontend replica to whichever worker +// held the execution, and it could not move to the broadcast carrier because +// that carrier rides PostgreSQL and this process has no database. It is a +// control verb on the worker's own tunnel instead. // -// So --nats-url stays required here, and it is required for this and for -// nothing else. When a cancel rides the tunnel as a control verb, this spec is -// what has to be deleted for the flag to become optional, and deleting it is -// then the visible half of that change. -var _ = Describe("The agent worker's remaining bus requirement", func() { +// --nats-url is still ACCEPTED, and ignored, so an existing command line or +// unit file starts unchanged. +var _ = Describe("The agent worker's bus requirement", func() { parse := func(args ...string) error { - // kong resolves env: tags from the process environment, and a - // LOCALAI_NATS_URL that is SET BUT EMPTY satisfies a required flag. - // Left in place, this spec would pass on a developer's shell and on - // nothing else. + // kong resolves env: tags from the process environment, so a + // LOCALAI_NATS_URL inherited from a developer's shell would make the + // first spec below pass for the wrong reason. if prior, had := os.LookupEnv("LOCALAI_NATS_URL"); had { Expect(os.Unsetenv("LOCALAI_NATS_URL")).To(Succeed()) DeferCleanup(func() { _ = os.Setenv("LOCALAI_NATS_URL", prior) }) @@ -182,16 +219,14 @@ var _ = Describe("The agent worker's remaining bus requirement", func() { return err } - It("refuses to start without a bus to hear cancels on", func() { - // Refused at parse time and not at first use. A worker that started - // and only failed to subscribe would already have registered itself as - // available to run agents nobody can cancel. - Expect(parse("--register-to", "http://frontend:8080")). - To(MatchError(ContainSubstring("--nats-url")), - "the agent worker started with no bus: every cancel of an agent it runs would be published to nobody and reported as sent") + It("starts with no bus named at all", func() { + Expect(parse("--register-to", "http://frontend:8080")).To(Succeed(), + "an agent worker connects to no message bus and must not demand the URL of one") }) - It("parses once the bus is named", func() { + It("still accepts a command line that names one", func() { + // Ignored, not rejected. An operator upgrading a fleet must not have to + // edit every unit file in the same change. Expect(parse("--register-to", "http://frontend:8080", "--nats-url", "nats://bus:4222")).To(Succeed()) }) }) diff --git a/core/cli/run.go b/core/cli/run.go index 410e2f465106..4ed84c3d43b7 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -162,9 +162,9 @@ type RunCMD struct { DefaultAPIKeyExpiry string `env:"LOCALAI_DEFAULT_API_KEY_EXPIRY" help:"Default expiry for API keys (e.g. 90d, 1y; empty = no expiry)" group:"auth"` // Distributed / Horizontal Scaling - Distributed bool `env:"LOCALAI_DISTRIBUTED" default:"false" help:"Enable distributed mode (requires PostgreSQL + NATS)" group:"distributed"` + Distributed bool `env:"LOCALAI_DISTRIBUTED" default:"false" help:"Enable distributed mode (requires PostgreSQL; no message bus)" group:"distributed"` InstanceID string `env:"LOCALAI_INSTANCE_ID" help:"Unique instance ID for distributed mode (auto-generated UUID if empty)" group:"distributed"` - NatsURL string `env:"LOCALAI_NATS_URL" help:"NATS server URL (e.g., nats://localhost:4222)" group:"distributed"` + NatsURL string `env:"LOCALAI_NATS_URL" help:"Ignored. No component of a distributed deployment connects to a message bus; state and fan-out ride PostgreSQL and workers are reached over their own tunnels. Accepted so an existing command line still starts." group:"distributed"` DistributedAdvertiseAddr string `env:"LOCALAI_DISTRIBUTED_ADVERTISE_ADDR" help:"host:port other frontend replicas dial to reach this one (peer link). Empty = derived from the local address that routes to PostgreSQL, which only works when the database is on another host." group:"distributed"` StorageURL string `env:"LOCALAI_STORAGE_URL" help:"S3-compatible storage endpoint URL (e.g., http://minio:9000)" group:"distributed"` StorageBucket string `env:"LOCALAI_STORAGE_BUCKET" default:"localai" help:"S3 bucket name for object storage" group:"distributed"` diff --git a/core/config/distributed_config.go b/core/config/distributed_config.go index 976bb0caa3d2..fdf90d477874 100644 --- a/core/config/distributed_config.go +++ b/core/config/distributed_config.go @@ -22,7 +22,9 @@ type DistributedConfig struct { // answer is only usable when the database is remote, so a deployment with // a local or sidecar database has to set this. AdvertiseAddr string // LOCALAI_DISTRIBUTED_ADVERTISE_ADDR - NatsURL string // --nats-url / LOCALAI_NATS_URL + // NatsURL is accepted and ignored. No component of a distributed + // deployment dials a message bus any more. + NatsURL string // --nats-url / LOCALAI_NATS_URL StorageURL string // --storage-url / LOCALAI_STORAGE_URL (S3 endpoint) RegistrationToken string // --registration-token / LOCALAI_REGISTRATION_TOKEN (required token for node registration) // RegistrationRequireAuth fails startup when distributed mode is enabled but @@ -157,9 +159,11 @@ func (c DistributedConfig) Validate() error { if !c.Enabled { return nil } - if c.NatsURL == "" { - return fmt.Errorf("distributed mode requires --nats-url / LOCALAI_NATS_URL") - } + // No message-bus URL is required, and none is dialled. The last family + // that needed one was agent..cancel, which now rides the agent + // worker's own tunnel as a control verb; a distributed deployment needs + // PostgreSQL and the frontends' own HTTP listener. The flag is still + // accepted so an existing command line starts unchanged. // S3 credentials must be paired if (c.StorageAccessKey != "" && c.StorageSecretKey == "") || (c.StorageAccessKey == "" && c.StorageSecretKey != "") { diff --git a/core/services/agents/dispatcher.go b/core/services/agents/dispatcher.go index 6f46a0346842..82daac4be18e 100644 --- a/core/services/agents/dispatcher.go +++ b/core/services/agents/dispatcher.go @@ -278,6 +278,32 @@ func (d *WorkerExecutor) Execute(ctx context.Context, raw json.RawMessage, pub m return json.Marshal(map[string]string{"status": status, "error": errMsg}) } +// Cancel answers workerctl.PathAgentCancel on an agent worker. +// +// The reply is this worker's OWN ANSWER and travels as bytes on a 200: +// cancelled true when this process was running the named execution and its +// context has now been cancelled, false when it was not. False says nothing +// about any other worker and nothing about whether the run exists, and the +// frontend that fans the cancel out is the only thing that may assemble those +// answers into a verdict. +// +// A body this worker cannot read is the one thing here that is NOT an answer: +// it has learned nothing about any execution, so it is returned as an error, +// becomes a non-2xx over the tunnel, and is counted by the caller as a cancel +// it could not deliver rather than as one that found nothing. +func (d *WorkerExecutor) Cancel(_ context.Context, raw json.RawMessage) (json.RawMessage, error) { + var req messaging.AgentCancelRequest + if err := json.Unmarshal(raw, &req); err != nil { + return nil, fmt.Errorf("reading an agent cancel request: %w", err) + } + cancelled := d.eventBridge.CancelLocalExecution(req.MessageID) + if cancelled { + xlog.Info("Cancelled an agent execution on this worker after a control cancel", + "agent", req.AgentName, "user", req.UserID, "messageID", req.MessageID) + } + return json.Marshal(messaging.AgentCancelReply{Cancelled: cancelled}) +} + func (d *WorkerExecutor) handleJob(ctx context.Context, evt AgentChatEvent, bridge *EventBridge) (status, errMsg string) { xlog.Info("Processing agent chat job", "agent", evt.AgentName, "user", evt.UserID) @@ -302,9 +328,9 @@ func (d *WorkerExecutor) handleJob(ctx context.Context, evt AgentChatEvent, brid ctx, cancel := context.WithCancel(ctx) defer cancel() - // Register cancellation on the SHARED registry, which the bus-backed cancel - // listener also reads: a cancel arrives on the bus and has to reach an - // execution that is publishing onto a stream. + // Register cancellation on the SHARED registry, which the cancel control + // verb also reads: a cancel arrives on this worker's tunnel and has to + // reach an execution that is publishing onto a control stream. bridge.RegisterCancel(evt.MessageID, cancel) defer bridge.DeregisterCancel(evt.MessageID) diff --git a/core/services/agents/events.go b/core/services/agents/events.go index c6938d40f322..9b78f7f5587a 100644 --- a/core/services/agents/events.go +++ b/core/services/agents/events.go @@ -30,11 +30,24 @@ type AgentEvent struct { Timestamp int64 `json:"timestamp"` // Unix milliseconds (set by PublishEvent) } -// AgentCancelEvent is the broadcast payload for cancelling agent execution. -type AgentCancelEvent struct { - AgentName string `json:"agent_name"` - UserID string `json:"user_id"` - MessageID string `json:"message_id,omitempty"` +// AgentWorkerCanceller carries a cancel to the agent workers of this +// deployment, over the tunnels they hold. +// +// A narrow port rather than the control client itself, because this package +// must not import core/services/nodes, and because the only thing a cancel +// needs from the frontend's control plane is this one verb. +// +// Its error vocabulary is the caller's whole answer and each value means +// something different: nil is a worker's own answer that it cancelled the run, +// nodes.ErrAgentCancelUndelivered is a cancel that may have reached nobody, and +// nodes.ErrAgentRunNotOnAnyWorker is every reachable worker answering that it +// is not running that execution. Nothing here may collapse them. +// +// The real implementation is *nodes.AgentControlClient, asserted where both +// packages are already imported (core/application), so a signature drift is a +// build failure rather than a nil field. +type AgentWorkerCanceller interface { + CancelAgentRun(ctx context.Context, req messaging.AgentCancelRequest) error } // EventBridge bridges agent events between the broadcast carrier and SSE @@ -47,24 +60,20 @@ type EventBridge struct { // to queue. bus messaging.Broadcaster - // cancelBus is where agent..cancel is published and heard, and it is - // a SEPARATE field from bus because in this deployment the two ends of that - // family cannot be on the same carrier yet. + // workers is how a cancel reaches an agent WORKER, and it is a separate + // field from bus because the two travel on different carriers on purpose. // - // Every other family here has both ends on a frontend replica, so both move - // to the broadcast carrier together. This one does not: its only subscriber - // is the agent WORKER (core/cli/agent_worker.go), the cancel has to reach - // the worker actually running the execution, and a worker has no database - // and so cannot join the PostgreSQL carrier at all. Publishing a cancel - // where no worker is listening would report a cancel that reached nobody as - // a cancel the execution declined, which is the one thing this whole - // programme may not do. + // Every other family here has both of its ends on a frontend replica, so + // both live on the broadcast carrier. This one does not: the process that + // holds a worker-run execution's cancel function is the agent worker, and a + // worker has no database and so cannot join the PostgreSQL carrier at all. + // It holds an outward-dialled tunnel instead, and a cancel is an ordinary + // control RPC on it, addressed to the workers a live replica can reach. // - // So it is named, and it is separate, and it stays on the carrier the - // worker reads until a cancel rides the worker's tunnel instead. Folding it - // back into bus is not a simplification: it is silent loss of every cancel - // for every worker-run agent. - cancelBus messaging.Broadcaster + // It is nil on an agent worker, which has nobody to forward a cancel to: + // there, a cancel ARRIVES as that control verb and is applied to the + // registry below. + workers AgentWorkerCanceller // pub is where the events this bridge produces GO, which is not always the // bus. On an agent worker running a dispatched claim it is the response // body of the control RPC the claiming replica is reading, so the events @@ -88,52 +97,69 @@ type EventBridge struct { } // NewEventBridge creates a new EventBridge on the deployment's fan-out carrier. +// A cancel it cannot apply itself is forwarded to the agent workers through +// workers. +// +// The canceller is a CONSTRUCTOR PARAMETER and not a builder call, and that is +// the whole reason it is spelled here. As a WithWorkerCanceller line it is one +// statement whose loss compiles, passes every suite in this package, and turns +// every cancel of a worker-run agent into a cancel that reached nobody and +// reported nothing: the local registry has no entry, and there is no longer +// anywhere for the cancel to go. // -// Cancels go on that same carrier unless WithCancelCarrier says otherwise, -// which is right for the agent worker, where there is only one carrier to be -// on, and wrong for a frontend replica, which reads fan-out from PostgreSQL and -// has to reach workers that cannot. -func NewEventBridge(bus messaging.Broadcaster, store *AgentStore, instanceID string) *EventBridge { +// A nil workers is legitimate on an agent worker, which forwards nothing. See +// NewWorkerEventBridge, which is how a worker builds one. +func NewEventBridge(bus messaging.Broadcaster, store *AgentStore, instanceID string, workers AgentWorkerCanceller) *EventBridge { return &EventBridge{ bus: bus, - cancelBus: bus, pub: bus, + workers: workers, store: store, instanceID: instanceID, cancelRegistry: &messaging.CancelRegistry{}, } } -// WithCancelCarrier puts agent..cancel on carrier instead of on the -// fan-out bus, and returns the receiver so it can be written as one expression -// with the constructor. +// NewWorkerEventBridge returns the bridge an AGENT WORKER runs on. // -// A frontend replica needs it and an agent worker does not. The cancel has to -// reach the worker running the execution; a worker has no database and cannot -// join the PostgreSQL carrier; so a frontend that published its cancels there -// would publish them where no worker listens, and a cancel that reached nobody -// is not a cancel that was refused. +// It joins no carrier, because there is none for it to join: every event it +// produces is written onto the response body of the control RPC that asked for +// the work (see WithPublisher), and every cancel it must act on arrives as a +// control verb rather than as a broadcast. What it keeps is the cancel +// registry, which is the one piece of state a worker's bridge exists for. // -// A nil carrier leaves the bridge on the fan-out bus rather than on nothing, -// because a bridge that publishes cancels nowhere is the failure this exists to -// prevent. -func (b *EventBridge) WithCancelCarrier(carrier messaging.Broadcaster) *EventBridge { - if b == nil || carrier == nil { - return b - } - b.cancelBus = carrier - return b +// The publisher it holds until a verb hands it a stream REFUSES rather than +// drops. A worker that publishes with no stream to write to has produced an +// event that reached nobody, and a no-op default would make that indetectable. +func NewWorkerEventBridge(instanceID string) *EventBridge { + return NewEventBridge(unroutedCarrier{}, nil, instanceID, nil) +} + +// unroutedCarrier is the carrier an agent worker's bridge holds: there is none. +// +// Both methods fail rather than silently succeeding, because both would +// otherwise be undetectable. A dropped publish is an agent event nobody sees; a +// subscription that never delivers is a listener that never fires. +type unroutedCarrier struct{} + +func (unroutedCarrier) Publish(subject string, _ any) error { + return fmt.Errorf("agents: an agent worker tried to publish %q with no control stream to write it to: a worker joins no carrier, so this event would have reached nobody", subject) +} + +func (unroutedCarrier) Subscribe(subject string, _ func([]byte)) (messaging.Subscription, error) { + return nil, fmt.Errorf("agents: an agent worker tried to subscribe to %q: a worker joins no carrier, so nothing would ever be delivered", subject) } // WithPublisher returns a view of this bridge whose events go to pub. // // Everything else is SHARED with the receiver, the cancel registry above all: a -// cancel arriving on the bus must reach an execution that is publishing onto a -// stream, and a bridge that copied the registry would register the cancel where -// nothing looks for it. +// cancel arriving as a control verb must reach an execution that is publishing +// onto a stream, and a bridge that copied the registry would register the +// cancel where nothing looks for it. // // A nil pub returns the receiver unchanged rather than a bridge that publishes -// nowhere, because a handler that was given no writer still has the bus. +// nowhere: on a frontend that leaves the events on the fan-out carrier, and on +// a worker it leaves them on the carrier that refuses loudly. func (b *EventBridge) WithPublisher(pub messaging.Publisher) *EventBridge { if b == nil || pub == nil { return b @@ -238,26 +264,59 @@ func (b *EventBridge) PublishStreamEvent(agentName, userID string, data map[stri }) } -// CancelExecution publishes a cancel event and also checks the local registry. -func (b *EventBridge) CancelExecution(agentName, userID, messageID string) error { - // Try local cancel first +// CancelExecution stops one agent execution wherever in the deployment it is +// running, and reports which of three different things happened. +// +// The three, because a caller that cannot tell them apart is the defect this +// family has been held back for: +// +// - nil means the execution was cancelled. Either it was running in THIS +// process, or an agent worker answered that it had cancelled it. +// - nodes.ErrAgentCancelUndelivered means the cancel may have reached nobody: +// an agent worker that might be running it could not be reached. It is not +// a refusal and it is not a missing task. +// - nodes.ErrAgentRunNotOnAnyWorker means every agent worker this deployment +// could reach answered that it is not running that execution. +// +// The local registry is tried FIRST and short-circuits. A message id names +// exactly one execution, so a local hit is this process's own answer about it, +// and there is nothing a worker could add. +func (b *EventBridge) CancelExecution(ctx context.Context, agentName, userID, messageID string) error { if b.cancelRegistry.Cancel(messageID) { xlog.Info("Cancelled agent execution locally", "agent", agentName, "user", userID, "messageID", messageID) + return nil } - // Broadcast so the replica that actually holds the execution can act on it. - // - // The error says whether the request was PUBLISHED and nothing more. This - // carrier is at-most-once with no replay, so a cancel that reached nobody - // and a cancel an execution declined are different facts that cannot be - // told apart from here, and neither may be reported as the other. - return b.cancelBus.Publish(messaging.SubjectAgentCancel(agentName), AgentCancelEvent{ + if b.workers == nil { + // Not a cancel that was refused and not a task that does not exist: + // this process has nowhere to send the cancel, which is a fact about + // its own wiring and says nothing about the run. + return fmt.Errorf("cancelling agent %q for user %q: this process holds no way to reach an agent worker, so the cancel of message %q was sent nowhere", + agentName, userID, messageID) + } + + return b.workers.CancelAgentRun(ctx, messaging.AgentCancelRequest{ AgentName: agentName, UserID: userID, MessageID: messageID, }) } +// CancelLocalExecution cancels an execution running in THIS process and reports +// whether it found one. +// +// It is what an agent worker's cancel control verb applies, and the bool is +// that worker's whole answer: true is "I cancelled it", false is "I am not +// running it". False is deliberately not an error, because it is not one: a +// deployment fans a cancel out to every worker it can reach and all but one of +// them are expected to say no. +func (b *EventBridge) CancelLocalExecution(messageID string) bool { + if b == nil || messageID == "" { + return false + } + return b.cancelRegistry.Cancel(messageID) +} + // RegisterCancel registers a cancel function for a running agent execution. func (b *EventBridge) RegisterCancel(key string, cancel context.CancelFunc) { b.cancelRegistry.Register(key, cancel) @@ -268,17 +327,6 @@ func (b *EventBridge) DeregisterCancel(key string) { b.cancelRegistry.Deregister(key) } -// StartCancelListener subscribes to the cancel broadcasts every replica sees. -func (b *EventBridge) StartCancelListener() (messaging.Subscription, error) { - return messaging.SubscribeJSON(b.cancelBus, messaging.SubjectAgentCancelWildcard, func(evt AgentCancelEvent) { - if evt.MessageID != "" { - if b.cancelRegistry.Cancel(evt.MessageID) { - xlog.Info("Cancelled an agent execution on this replica after a broadcast cancel", "agent", evt.AgentName, "user", evt.UserID, "messageID", evt.MessageID) - } - } - }) -} - // StartObservablePersister subscribes to every agent's events and persists the // observable_update ones to the database. This runs on the frontend, to capture // observables published by workers, which have no database access. diff --git a/core/services/agents/events_pg_test.go b/core/services/agents/events_pg_test.go index db8176126590..b565dfcd4b25 100644 --- a/core/services/agents/events_pg_test.go +++ b/core/services/agents/events_pg_test.go @@ -5,6 +5,7 @@ package agents import ( "bytes" "context" + "errors" "net/http" "net/http/httptest" "sync" @@ -34,6 +35,7 @@ var _ = Describe("the agent event bridge on the broadcast carrier", func() { busA, busB *pgbus.Bus store *AgentStore bridge *EventBridge + workers *recordingCanceller ) BeforeEach(func() { @@ -53,7 +55,8 @@ var _ = Describe("the agent event bridge on the broadcast carrier", func() { var err error store, err = NewAgentStore(db) Expect(err).ToNot(HaveOccurred()) - bridge = NewEventBridge(busA, store, "replica-a") + workers = &recordingCanceller{} + bridge = NewEventBridge(busA, store, "replica-a", workers) }) // observable is the AgentEvent shape the persister acts on, minus the @@ -132,7 +135,7 @@ var _ = Describe("the agent event bridge on the broadcast carrier", func() { Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = sub.Unsubscribe() }) - peer := NewEventBridge(busB, store, "replica-b") + peer := NewEventBridge(busB, store, "replica-b", workers) Expect(peer.PublishMessage("a1", "u2", "agent", "for another user", "m-other")).To(Succeed()) Expect(peer.PublishMessage("a1", "u1", "agent", "for me", "m-mine")).To(Succeed()) @@ -177,7 +180,7 @@ var _ = Describe("the agent event bridge on the broadcast carrier", func() { // Delivery first, so this is a spec about a stream that WORKED and // then closed, rather than one that never started. - peer := NewEventBridge(busB, store, "replica-b") + peer := NewEventBridge(busB, store, "replica-b", workers) Expect(peer.PublishMessage("a1", "u1", "agent", "hello", "m-1")).To(Succeed()) Eventually(out.String, "20s").Should(ContainSubstring("hello")) @@ -188,63 +191,103 @@ var _ = Describe("the agent event bridge on the broadcast carrier", func() { }) }) - Describe("cancel broadcasts", func() { - It("reaches a peer replica's cancel listener", func() { - peer := NewEventBridge(busB, store, "replica-b") - sub, err := peer.StartCancelListener() - Expect(err).ToNot(HaveOccurred()) - DeferCleanup(func() { _ = sub.Unsubscribe() }) - + // A cancel no longer travels on any carrier, so what these pin is the split + // between the three answers a caller may be given. The CARRIAGE of a cancel + // to a worker is pinned where it happens, over a real tunnel and a real + // yamux session, in core/services/nodes/agent_control_test.go. + Describe("cancelling an execution", func() { + It("cancels a run held by THIS process without disturbing any worker", func() { cancelled := make(chan struct{}) - peer.RegisterCancel("msg-1", func() { close(cancelled) }) + bridge.RegisterCancel("msg-local", func() { close(cancelled) }) - Expect(bridge.CancelExecution("a1", "u1", "msg-1")).To(Succeed()) + Expect(bridge.CancelExecution(ctx, "a1", "u1", "msg-local")).To(Succeed()) Eventually(cancelled, "20s").Should(BeClosed()) + Expect(workers.calls).To(BeZero(), + "a message id names exactly one execution, and this process was running it") }) - // The one family whose two ends are NOT on the same carrier, and the - // spec that says so out loud. - // - // Its only subscriber is the agent WORKER, which has no database and - // therefore cannot join the PostgreSQL carrier. A frontend that - // published its cancels onto the fan-out bus would publish them where - // no worker is listening, every cancel of a worker-run agent would be - // lost, and CancelExecution would return nil throughout: a cancel that - // reached nobody reported as a cancel that was sent, and one step later - // as a cancel the execution declined. - It("publishes a cancel where the worker listens and not onto the fan-out carrier", func() { - // The worker's carrier. Not a second pgbus: the whole point is that - // a worker cannot have one. - workerCarrier := testutil.NewFakeBus() - - frontend := NewEventBridge(busA, store, "replica-a").WithCancelCarrier(workerCarrier) - - worker := NewEventBridge(workerCarrier, nil, "agent-worker-1") - workerSub, err := worker.StartCancelListener() - Expect(err).ToNot(HaveOccurred()) - DeferCleanup(func() { _ = workerSub.Unsubscribe() }) + It("forwards a run it does not hold to the deployment's agent workers", func() { + Expect(bridge.CancelExecution(ctx, "a1", "u1", "msg-remote")).To(Succeed()) - cancelled := make(chan struct{}) - worker.RegisterCancel("msg-worker", func() { close(cancelled) }) - - // A listener on the FAN-OUT carrier, which must be shown nothing. - // Without this the spec would pass with the cancel published on - // both, which is the shape that hides the loss rather than fixing - // it. - onTheBus := make(chan []byte, 4) - _, err = busB.Subscribe(messaging.SubjectAgentCancelWildcard, func(data []byte) { onTheBus <- data }) - Expect(err).ToNot(HaveOccurred()) + Expect(workers.calls).To(Equal(1)) + Expect(workers.last).To(Equal(messaging.AgentCancelRequest{ + AgentName: "a1", UserID: "u1", MessageID: "msg-remote", + })) + }) + + // The three answers, kept apart. A caller told any one of these in + // place of another acts on something that did not happen. + It("returns the workers' answer unchanged, whichever of the three it is", func() { + workers.err = errUndeliveredForSpec + Expect(bridge.CancelExecution(ctx, "a1", "u1", "m")).To(MatchError(errUndeliveredForSpec)) + + workers.err = errNotOnAnyWorkerForSpec + Expect(bridge.CancelExecution(ctx, "a1", "u1", "m")).To(MatchError(errNotOnAnyWorkerForSpec)) + }) + + It("refuses rather than reporting success when it has nowhere to send a cancel", func() { + // A bridge built with no canceller. Reporting Succeed here would be + // a cancel that reached nobody presented as one that was made, + // which is the exact failure this family was held back for. + orphan := NewEventBridge(busA, store, "replica-a", nil) + err := orphan.CancelExecution(ctx, "a1", "u1", "m") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sent nowhere")) + }) + }) - Expect(frontend.CancelExecution("a1", "u1", "msg-worker")).To(Succeed()) + Describe("the bridge an agent worker runs", func() { + It("applies a cancel handed to it locally and reports whether it found the run", func() { + w := NewWorkerEventBridge("agent-worker-1") + cancelled := make(chan struct{}) + w.RegisterCancel("msg-worker", func() { close(cancelled) }) + Expect(w.CancelLocalExecution("msg-other")).To(BeFalse(), + "a worker may only ever answer for itself") + Expect(w.CancelLocalExecution("msg-worker")).To(BeTrue()) Eventually(cancelled, "20s").Should(BeClosed()) - Consistently(onTheBus, "500ms", "50ms").ShouldNot(Receive(), - "a cancel on the fan-out carrier reaches no worker and would be lost") + }) + + It("refuses to publish when no control stream has been handed to it", func() { + // A worker joins no carrier. A default that DROPPED the event would + // make an agent run whose events reached nobody look identical to + // one whose events were delivered. + w := NewWorkerEventBridge("agent-worker-1") + err := w.PublishMessage("a1", "u1", "agent", "hello", "m-1") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("reached nobody")) + }) + + It("refuses to subscribe, rather than returning a listener that never fires", func() { + w := NewWorkerEventBridge("agent-worker-1") + _, err := w.SubscribeEvents("a1", "u1", func(AgentEvent) {}) + Expect(err).To(HaveOccurred()) }) }) }) +// recordingCanceller stands in for the frontend's agent control client. The +// real one is driven over a real tunnel in core/services/nodes; what this pins +// is that the bridge asks it at all, with the right request, and hands its +// answer back unchanged. +type recordingCanceller struct { + calls int + last messaging.AgentCancelRequest + err error +} + +func (r *recordingCanceller) CancelAgentRun(_ context.Context, req messaging.AgentCancelRequest) error { + r.calls++ + r.last = req + return r.err +} + +var ( + errUndeliveredForSpec = errors.New("this cancel could not be delivered") + errNotOnAnyWorkerForSpec = errors.New("no agent worker is running that execution") +) + // syncBody is an http.ResponseWriter a spec may read WHILE the handler is still // writing. httptest.ResponseRecorder's buffer is not safe for that, and reading // it mid-stream is the only way to know the handler reached its wait rather than diff --git a/core/services/agents/events_test.go b/core/services/agents/events_test.go index 95802bf8ba97..bc8ac5b7ec8f 100644 --- a/core/services/agents/events_test.go +++ b/core/services/agents/events_test.go @@ -42,7 +42,7 @@ var _ = Describe("EventBridge", func() { // safe-integer range. The timestamp must be in milliseconds. It("emits the timestamp in Unix milliseconds", func() { fake := &recordingBus{} - bridge := NewEventBridge(fake, nil, "instance-1") + bridge := NewEventBridge(fake, nil, "instance-1", nil) before := time.Now().UnixMilli() err := bridge.PublishMessage("agent", "user", "agent", "hello", "msg-1") diff --git a/core/services/agentworker/handlers.go b/core/services/agentworker/handlers.go index b933362dfb6d..fa44f93145bc 100644 --- a/core/services/agentworker/handlers.go +++ b/core/services/agentworker/handlers.go @@ -57,10 +57,6 @@ type StreamHandler func(ctx context.Context, raw json.RawMessage, pub messaging. // as "this worker does not serve that verb" (nodes.ErrWorkerControlUnsupported) // rather than as absence. It is why a later task can add two handlers without // any other file changing shape. -// -// workerctl.PathAgentCancel is named in the path table with no field here. It -// gets the same 404 for the same reason, and the path is fixed now so the two -// sides cannot disagree about it later. type Config struct { // MCPTool answers workerctl.PathMCPToolExecute. MCPTool UnaryHandler @@ -74,6 +70,15 @@ type Config struct { // pick a carrier. BackendStop func(ctx context.Context, req messaging.BackendStopRequest) error + // AgentCancel answers workerctl.PathAgentCancel. It is the LAST family a + // worker needed a message bus for: the cancel used to be a broadcast this + // process had to dial a bus to hear, and is now an ordinary control RPC on + // the tunnel it already holds. + // + // Unary and not streaming: a cancel has one answer and no progress. That + // answer is messaging.AgentCancelReply and it speaks only for this worker. + AgentCancel UnaryHandler + // AgentExecute answers workerctl.PathAgentExecute and MCPCIRun answers // workerctl.PathMCPCIRun. Both are nil in this task and both are set later, // which is where the queue subjects they replace become claim rows. They are @@ -99,6 +104,9 @@ func (c Config) Register(mux *http.ServeMux) { if c.BackendStop != nil { mux.HandleFunc(workerctl.PathBackendStop, serveBackendStop(c.BackendStop)) } + if c.AgentCancel != nil { + mux.HandleFunc(workerctl.PathAgentCancel, serveUnary(workerctl.PathAgentCancel, c.AgentCancel)) + } if c.AgentExecute != nil { mux.HandleFunc(workerctl.PathAgentExecute, serveStream(workerctl.PathAgentExecute, c.AgentExecute)) } diff --git a/core/services/agentworker/server_test.go b/core/services/agentworker/server_test.go index e4f4f002178e..9e278710efb3 100644 --- a/core/services/agentworker/server_test.go +++ b/core/services/agentworker/server_test.go @@ -93,6 +93,7 @@ func fullConfig() agentworker.Config { return agentworker.Config{ MCPTool: echoHandler(`{"result":"tool-ran"}`), MCPDiscovery: echoHandler(`{"servers":[]}`), + AgentCancel: echoHandler(`{"cancelled":true}`), BackendStop: func(context.Context, messaging.BackendStopRequest) error { return nil }, } } @@ -108,6 +109,13 @@ var _ = Describe("The agent worker's control verbs", func() { disc := post(base, workerctl.PathMCPDiscovery, `{"model_name":"m"}`) Expect(disc.StatusCode).To(Equal(http.StatusOK)) Expect(bodyOf(disc)).To(Equal(`{"servers":[]}`)) + + // The cancel is unary and its answer is the worker's own: a 200 body + // saying whether THIS worker cancelled the run. Anything else the + // frontend reads as an answer it did not obtain. + cancel := post(base, workerctl.PathAgentCancel, `{"message_id":"m"}`) + Expect(cancel.StatusCode).To(Equal(http.StatusOK)) + Expect(bodyOf(cancel)).To(Equal(`{"cancelled":true}`)) }) It("answers backend.stop with the 204 a BACKEND worker answers on that same path", func() { @@ -161,7 +169,6 @@ var _ = Describe("The agent worker's control verbs", func() { }, Entry("agent execute, whose handler is nil until it is wired", workerctl.PathAgentExecute), Entry("mcp ci run, whose handler is nil until it is wired", workerctl.PathMCPCIRun), - Entry("agent cancel, which has no handler field at all yet", workerctl.PathAgentCancel), Entry("a backend worker's verb, which an agent worker never serves", workerctl.PathBackendInstall), Entry("a path no build has ever named", workerctl.Prefix+"nothing/here"), ) @@ -178,6 +185,7 @@ var _ = Describe("The agent worker's control verbs", func() { Entry("mcp tool execute", workerctl.PathMCPToolExecute, func(c *agentworker.Config) { c.MCPTool = nil }), Entry("mcp discovery", workerctl.PathMCPDiscovery, func(c *agentworker.Config) { c.MCPDiscovery = nil }), Entry("backend stop", workerctl.PathBackendStop, func(c *agentworker.Config) { c.BackendStop = nil }), + Entry("agent cancel", workerctl.PathAgentCancel, func(c *agentworker.Config) { c.AgentCancel = nil }), ) // A rule stated at every verb has to be pinned at every verb. The exit @@ -213,6 +221,9 @@ var _ = Describe("The agent worker's control verbs", func() { return errors.New("the session cache is wedged") } }), + Entry("agent cancel", workerctl.PathAgentCancel, func(c *agentworker.Config) { + c.AgentCancel = failingHandler(errors.New("the cancel registry is unreadable")) + }), Entry("agent execute, before it has published anything", workerctl.PathAgentExecute, func(c *agentworker.Config) { c.AgentExecute = func(context.Context, json.RawMessage, messaging.Publisher) (json.RawMessage, error) { return nil, errors.New("no agent to run") diff --git a/core/services/messaging/client.go b/core/services/messaging/client.go index 64c9b9968834..47a98862e64e 100644 --- a/core/services/messaging/client.go +++ b/core/services/messaging/client.go @@ -18,22 +18,25 @@ import ( // subscription was rejected (e.g. by JWT permissions) before returning to the caller. const subscribeConfirmTimeout = 5 * time.Second -// Client is a NATS connection, and it is the carrier for exactly one family. +// Client is a NATS connection, and NO PRODUCTION PATH CONSTRUCTS ONE. // -// agent..cancel is the one fan-out family that did not move to the -// PostgreSQL carrier. Its only subscriber is the agent WORKER, which has no -// database and cannot join that carrier at all, so a cancel published there -// would reach no worker while reporting that it was sent. Both ends of that -// family still dial this client, which is why it, its connect options and its -// TLS plumbing are all still here. +// The last family that needed a bus was agent..cancel, whose subscriber is +// the agent WORKER: it has no database, so it could never join the PostgreSQL +// carrier the rest of the deployment fans out on. It does not need a bus either +// now, because it holds an outward tunnel and a cancel is a control verb on it +// (workerctl.PathAgentCancel). Nothing in core/ or pkg/ calls messaging.New. // -// Everything else it used to carry is gone, and so are the methods that carried -// it: queue subscriptions became a claim on the job store, and request/reply -// became a streaming control RPC on the tunnel each worker dials. Deleting the -// METHODS rather than only the call sites is what makes putting a family back -// on this carrier a build error, instead of a line that compiles, publishes -// successfully, and is delivered onto a carrier the deployment is being taken -// off. +// What survives here is this type, its connect options and its TLS plumbing, +// still exercised by the NATS JWT permission specs. Deleting them is a +// demolition of its own, together with the JWT minting at registration and +// pkg/natsauth's permission tables. +// +// The methods that carried everything else are already gone: queue +// subscriptions became a claim on the job store, and request/reply became a +// streaming control RPC on the tunnel each worker dials. Deleting the METHODS +// rather than only the call sites is what makes putting a family back on this +// carrier a build error, instead of a line that compiles, publishes +// successfully, and is delivered onto a carrier nothing reads. type Client struct { conn *nats.Conn mu sync.RWMutex @@ -242,12 +245,12 @@ func (c *Client) confirmSubscription(subject string, mk func(*nats.Conn) (*nats. // different fact from the server refusing it, and neither is evidence about any // node. // -// No production path calls it. The carrier's production users publish cancels -// and act on the delivery, not on the verdict; what needs the verdict is -// pkg/natsauth's permission grants, which are asserted against a real enforcing -// server and would otherwise be asserted against nothing, since an allow list -// that is EMPTY means unrestricted in NATS and a spec that only checks -// IsConnected cannot tell a granted publish from a denied one. +// No production path calls it, and this carrier no longer has production users +// at all. What needs the verdict is pkg/natsauth's permission grants, which are +// asserted against a real enforcing server and would otherwise be asserted +// against nothing, since an allow list that is EMPTY means unrestricted in NATS +// and a spec that only checks IsConnected cannot tell a granted publish from a +// denied one. func (c *Client) ConfirmRoundTrip(timeout time.Duration) error { c.mu.RLock() conn := c.conn diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index d7b35278dc47..b44bdb168856 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -270,6 +270,39 @@ type BackendStopRequest struct { Force bool `json:"force,omitempty"` } +// AgentCancelRequest is the body of an agent cancel control request. +// +// It carries the same three fields the agent..cancel broadcast carried, +// because it says the same thing; what changed is the carrier. The cancel used +// to be published onto a bus every agent worker had to dial, and is now a +// control RPC on the tunnel the worker already holds, which is why an agent +// worker needs no bus credentials. +// +// MessageID is what identifies the execution, and it identifies exactly one: +// an agent run registers its cancel function under it on the process that is +// running it, and nowhere else. +type AgentCancelRequest struct { + AgentName string `json:"agent_name"` + UserID string `json:"user_id"` + MessageID string `json:"message_id,omitempty"` +} + +// AgentCancelReply is an agent worker's OWN ANSWER to a cancel. +// +// Cancelled false is that answer too, and it means one thing only: this worker +// is not running that execution. It is deliberately not spelled as an error, +// and no caller may read it as "the run does not exist" on its own, because a +// worker can only speak for itself. +// +// There is no Error field, and its absence is stated rather than left to be +// inferred. A cancel this worker could not read, or could not serve, is a +// non-2xx like every other verb's failure to serve, which the frontend reads +// as an answer it did not obtain; there is no third thing a worker can learn +// by looking in its own cancel registry. +type AgentCancelReply struct { + Cancelled bool `json:"cancelled"` +} + type ModelStopRequest struct { ModelName string `json:"model_name"` ProcessKey string `json:"process_key"` diff --git a/core/services/nodes/agent_control.go b/core/services/nodes/agent_control.go index 846a5ec96ee1..89c7f7a2057f 100644 --- a/core/services/nodes/agent_control.go +++ b/core/services/nodes/agent_control.go @@ -8,7 +8,9 @@ import ( "fmt" mcpremote "github.com/mudler/LocalAI/core/services/mcp" + "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/workerctl" + "github.com/mudler/xlog" ) // maxAgentPicks bounds how many agent workers one verb is offered to before it @@ -29,6 +31,28 @@ const maxAgentPicks = 3 // and says nothing about any worker. var ErrNoAgentControl = errors.New("nodes: this deployment has no agent control client") +// ErrAgentCancelUndelivered reports that a cancel may have reached nobody: at +// least one agent worker that could be running the execution was not asked, or +// did not answer. +// +// It is the answer this whole family was held back for. A cancel that could not +// be delivered is NOT a cancel that was refused and NOT a run that does not +// exist, and a caller told any of those three in place of another acts on +// something that did not happen. It says nothing about any particular worker +// either, which is why it is its own sentinel and carries neither +// ErrWorkerUnroutable nor anything cluster.IsWorkerAnswer accepts: no node may +// be reaped, demoted or evicted because a cancel went undelivered. +var ErrAgentCancelUndelivered = errors.New("nodes: an agent cancel could not be delivered to every agent worker that might be running it") + +// ErrAgentRunNotOnAnyWorker reports that every agent worker this deployment +// could reach answered that it is not running the named execution. +// +// It is assembled ONLY from workers' own answers, and only when every worker +// was reached; the moment one was not, ErrAgentCancelUndelivered is the answer +// instead. It still does not say the run does not exist: it says no agent +// worker is running it, which is the largest claim the evidence supports. +var ErrAgentRunNotOnAnyWorker = errors.New("nodes: no agent worker of this deployment is running that execution") + // AgentControlClient issues the frontend's control RPCs to whichever agent // worker the selector picks. // @@ -64,6 +88,88 @@ func (a *AgentControlClient) DiscoverMCPTools(ctx context.Context, req mcpremote func(r *mcpremote.MCPDiscoveryResponse) string { return r.Error }) } +// CancelAgentRun asks the agent workers of this deployment to stop one +// execution, and reports which of three different things happened. +// +// A FAN-OUT and not a pick, which is what makes it the one agent verb that does +// not go through agentVerb. A message id names exactly one execution, running +// on exactly one worker, and no row in this deployment records which: the claim +// that dispatched it names the claiming REPLICA, not the worker, and it is +// deleted when the run ends. So the cancel is offered to every worker a live +// replica can reach, exactly as the broadcast it replaces was, and each worker +// answers only for itself. +// +// The three answers, and why none may stand in for another: +// +// - nil. A worker answered that it cancelled the run. That is a worker's own +// answer and it is conclusive, even if another worker could not be reached: +// the execution has been cancelled, and there is only one of it. +// - ErrAgentCancelUndelivered. Some worker that might have been running it +// was not reached. Nothing was learned, and the caller may not report the +// run as missing or the cancel as declined. +// - ErrAgentRunNotOnAnyWorker. Every worker was reached and every one of them +// answered that it is not running that execution. +// +// A worker that is RECONNECTING is counted undelivered, and this is the +// decision rather than an omission: it is not retried here and it is not +// queued. Retrying would hold the caller for the length of the reconnect grace +// with no bound it chose, and queueing would need durable state whose only +// consumer is a run whose control stream died with the tunnel. Reporting it, +// once, as a cancel that may not have arrived is the only thing this frontend +// actually knows, and it leaves the retry where the budget lives: with the +// caller. +func (a *AgentControlClient) CancelAgentRun(ctx context.Context, req messaging.AgentCancelRequest) error { + if a == nil || a.sel == nil || a.cc == nil { + return fmt.Errorf("control rpc %s: %w", workerctl.PathAgentCancel, ErrNoAgentControl) + } + reach, err := a.sel.Reachable(ctx) + if err != nil { + return err + } + + if len(reach.Connected) == 0 && len(reach.Absent) == 0 { + // Nothing was asked of anyone and nothing was learned. This must not + // become ErrAgentRunNotOnAnyWorker, which is assembled from workers' + // own answers: a deployment with no agent worker has produced no + // answers at all, and reporting one would tell a caller that a run it + // can still see is not running anywhere. + return fmt.Errorf("cancelling message %q of agent %q: %w", req.MessageID, req.AgentName, ErrNoAgentWorker) + } + + cancelled := false + // Seeded with the workers nobody could ask at all: a tunnel lost inside the + // reconnect grace, or a presence this replica could not read. They are part + // of the fleet this cancel did not finish asking, and dropping them here is + // what would turn an unfinished fan-out into "no worker is running it". + undelivered := append([]string(nil), reach.Absent...) + for _, nodeID := range reach.Connected { + var reply messaging.AgentCancelReply + if err := a.cc.Call(ctx, nodeID, workerctl.PathAgentCancel, req, &reply); err != nil { + // An unreachable peer, a refused stream, a tunnel that died between + // the connection read and the dial, or a worker too old to serve + // the verb. None of them is an answer about this execution. + xlog.Warn("An agent worker could not be asked to cancel an execution", + "nodeID", nodeID, "agent", req.AgentName, "messageID", req.MessageID, "error", err) + undelivered = append(undelivered, nodeID) + continue + } + if reply.Cancelled { + cancelled = true + } + } + + switch { + case cancelled: + return nil + case len(undelivered) > 0: + return fmt.Errorf("cancelling message %q of agent %q: %d of %d agent workers could not be asked (%v): %w", + req.MessageID, req.AgentName, len(undelivered), len(reach.Connected)+len(reach.Absent), undelivered, ErrAgentCancelUndelivered) + default: + return fmt.Errorf("cancelling message %q of agent %q: all %d reachable agent workers answered that they are not running it: %w", + req.MessageID, req.AgentName, len(reach.Connected), ErrAgentRunNotOnAnyWorker) + } +} + // agentVerb is the ONE place the select-call-retry rule lives, and the one // place the line between a retryable failure and an answer is drawn. // diff --git a/core/services/nodes/agent_control_test.go b/core/services/nodes/agent_control_test.go index aaeaa81f11a6..773460786602 100644 --- a/core/services/nodes/agent_control_test.go +++ b/core/services/nodes/agent_control_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "runtime" "sync/atomic" + "time" "github.com/gorilla/websocket" "github.com/libp2p/go-yamux/v5" @@ -21,6 +22,7 @@ import ( "github.com/mudler/LocalAI/core/services/agentworker" "github.com/mudler/LocalAI/core/services/cluster" mcpremote "github.com/mudler/LocalAI/core/services/mcp" + "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/testutil" "github.com/mudler/LocalAI/core/services/worker" @@ -39,6 +41,12 @@ import ( const agentControlToken = "agent-control-token" +// agentReconnectGrace is the window these specs read a lost tunnel as +// reconnecting rather than gone. Long enough that a worker registered in this +// spec and never connected is never GONE, which is what makes the undelivered +// answer reachable here. +const agentReconnectGrace = time.Hour + // fakeFrontend is the far side of a worker's tunnel: the real WebSocket // upgrade and the real yamux server handshake. type fakeFrontend struct { @@ -241,7 +249,7 @@ var _ = Describe("AgentControlClient", func() { agentControl := func() *nodes.AgentControlClient { return nodes.NewAgentControlClient( - nodes.NewAgentSelector(registry, clusterReg, selfInstance), control) + nodes.NewAgentSelector(registry, clusterReg, selfInstance, agentReconnectGrace), control) } toolRequest := mcpremote.MCPToolRequest{ @@ -362,6 +370,170 @@ var _ = Describe("AgentControlClient", func() { Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) }) + // The cancel, over the same real transport as everything above it. + // + // A cancel is a FAN-OUT and not a pick: no row in this deployment records + // which worker holds a given execution, so the cancel is offered to every + // worker a live replica can reach and each answers only for itself. What + // these pin is that the three answers stay apart, because a caller told any + // one of them in place of another acts on something that did not happen. + Describe("cancelling one agent run", func() { + // cancelHandler answers the cancel verb the way a worker does: true + // when it holds the named run, false when it does not. + cancelHandler := func(mine string, seen *atomic.Int32) agentworker.UnaryHandler { + return func(_ context.Context, raw json.RawMessage) (json.RawMessage, error) { + seen.Add(1) + var req messaging.AgentCancelRequest + Expect(json.Unmarshal(raw, &req)).To(Succeed()) + return json.Marshal(messaging.AgentCancelReply{Cancelled: req.MessageID == mine}) + } + } + + cancelOf := func(messageID string) messaging.AgentCancelRequest { + return messaging.AgentCancelRequest{AgentName: "a1", UserID: "u1", MessageID: messageID} + } + + It("reaches the worker holding the run over the tunnel THIS replica holds", func() { + var asked atomic.Int32 + startAgent("agent-holder", mine, agentworker.Config{ + AgentCancel: cancelHandler("msg-1", &asked), + }) + + Expect(agentControl().CancelAgentRun(ctx, cancelOf("msg-1"))).To(Succeed()) + Expect(asked.Load()).To(Equal(int32(1))) + }) + + It("reaches a worker whose tunnel a PEER holds, by relaying through that peer", func() { + // The hop the broadcast used to hide. This replica holds nothing, + // so the only way the cancel arrives is the connection row naming + // the peer and the peer's relay splicing the stream onto the tunnel + // it holds. + var asked atomic.Int32 + startAgent("agent-remote", theirs, agentworker.Config{ + AgentCancel: cancelHandler("msg-1", &asked), + }) + Expect(mine.Held()).To(BeEmpty(), "this spec is only about the relayed path") + + Expect(agentControl().CancelAgentRun(ctx, cancelOf("msg-1"))).To(Succeed()) + Expect(asked.Load()).To(Equal(int32(1))) + }) + + It("asks EVERY reachable worker, because nothing records which one holds the run", func() { + // A pick would ask one and, four times in five, report a run that + // is running as one no worker is running. + var first, second atomic.Int32 + startAgent("agent-a", mine, agentworker.Config{AgentCancel: cancelHandler("nothing-here", &first)}) + startAgent("agent-b", theirs, agentworker.Config{AgentCancel: cancelHandler("msg-1", &second)}) + + Expect(agentControl().CancelAgentRun(ctx, cancelOf("msg-1"))).To(Succeed()) + Expect(first.Load()).To(Equal(int32(1))) + Expect(second.Load()).To(Equal(int32(1))) + }) + + It("reports a run no reachable worker holds as exactly that, and never as undelivered", func() { + var asked atomic.Int32 + startAgent("agent-a", mine, agentworker.Config{AgentCancel: cancelHandler("some-other-run", &asked)}) + + err := agentControl().CancelAgentRun(ctx, cancelOf("msg-1")) + Expect(err).To(MatchError(nodes.ErrAgentRunNotOnAnyWorker)) + Expect(err).ToNot(MatchError(nodes.ErrAgentCancelUndelivered)) + Expect(asked.Load()).To(Equal(int32(1))) + }) + + It("reports a worker it could not reach as UNDELIVERED and never as a run that does not exist", func() { + // The refusing worker's tunnel is real and its control server will + // not open, which is what a worker with a dead control plane looks + // like from here. Nothing was learned about the run, so the answer + // may not be "no worker is running it". + var refusals atomic.Int32 + startRefusingAgent("agent-dead", mine, &refusals) + + err := agentControl().CancelAgentRun(ctx, cancelOf("msg-1")) + Expect(err).To(MatchError(nodes.ErrAgentCancelUndelivered)) + Expect(err).ToNot(MatchError(nodes.ErrAgentRunNotOnAnyWorker)) + Expect(refusals.Load()).To(Equal(int32(1))) + }) + + It("counts a RECONNECTING worker as undelivered, which is the decision this task made", func() { + // A registered, approved agent worker with no live tunnel and a + // departure inside the grace. Nobody may act on that condition, so + // the cancel is reported as one that may not have arrived rather + // than retried here, queued, or folded into "no worker holds it". + var asked atomic.Int32 + startAgent("agent-alive", mine, agentworker.Config{AgentCancel: cancelHandler("some-other-run", &asked)}) + registerAgent("agent-reconnecting") + + err := agentControl().CancelAgentRun(ctx, cancelOf("msg-1")) + Expect(err).To(MatchError(nodes.ErrAgentCancelUndelivered)) + Expect(err).ToNot(MatchError(nodes.ErrAgentRunNotOnAnyWorker)) + Expect(asked.Load()).To(Equal(int32(1)), "the worker that WAS reachable must still have been asked") + }) + + It("reports a worker's cancellation even when another worker could not be reached", func() { + // There is one execution and one worker cancelled it. Reporting + // undelivered here would tell the caller nothing happened when + // something did. + var refusals, asked atomic.Int32 + startRefusingAgent("agent-dead", mine, &refusals) + startAgent("agent-holder", theirs, agentworker.Config{AgentCancel: cancelHandler("msg-1", &asked)}) + + Expect(agentControl().CancelAgentRun(ctx, cancelOf("msg-1"))).To(Succeed()) + Expect(refusals.Load()).To(Equal(int32(1))) + Expect(asked.Load()).To(Equal(int32(1))) + }) + + It("reports an empty fleet as neither a route verdict nor a worker answer", func() { + err := agentControl().CancelAgentRun(ctx, cancelOf("msg-1")) + Expect(err).To(MatchError(nodes.ErrNoAgentWorker)) + Expect(errors.Is(err, nodes.ErrWorkerUnroutable)).To(BeFalse()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + }) + + It("refuses on a nil client rather than panicking inside a request", func() { + var missing *nodes.AgentControlClient + Expect(missing.CancelAgentRun(ctx, cancelOf("msg-1"))).To(MatchError(nodes.ErrNoAgentControl)) + }) + + // The reap guard's two predicates, asserted on the errors a REAL + // fan-out over a real tunnel actually returns. + // + // A cancel is offered to many workers and its answer is assembled from + // all of them, so no answer it produces speaks about any one node. The + // guard that reaps, demotes and evicts decides on exactly these two + // predicates, and a cancel that satisfied either would let one worker + // whose control server happened to be down take a node out of the + // deployment. + // + // The edit that breaks this is small and reads like an improvement: + // wrapping the per-worker Call failure with %w instead of naming the + // nodes with %v. cluster.ErrStreamTargetUnavailable then travels out of + // the fan-out, every spec above still passes, and an undelivered cancel + // has become a routing verdict about a node. + DescribeTable("produces no answer a reap guard may act on, whichever of the three it is", + func(fleet func(), sentinel error) { + fleet() + err := agentControl().CancelAgentRun(ctx, cancelOf("msg-1")) + Expect(err).To(MatchError(sentinel)) + Expect(errors.Is(err, nodes.ErrWorkerUnroutable)).To(BeFalse(), + "a cancel answer was readable as a routing verdict about a worker, which the scheduler may act on") + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse(), + "a cancel answer was readable as a worker's own answer, which a reap guard may act on") + }, + Entry("a worker that could not be asked", func() { + var refusals atomic.Int32 + startRefusingAgent("agent-dead", mine, &refusals) + }, nodes.ErrAgentCancelUndelivered), + Entry("a worker that is reconnecting", func() { + registerAgent("agent-reconnecting") + }, nodes.ErrAgentCancelUndelivered), + Entry("every reachable worker answering that it does not hold the run", func() { + var asked atomic.Int32 + startAgent("agent-a", mine, agentworker.Config{AgentCancel: cancelHandler("some-other-run", &asked)}) + }, nodes.ErrAgentRunNotOnAnyWorker), + Entry("a deployment with no agent worker at all", func() {}, nodes.ErrNoAgentWorker), + ) + }) + It("refuses on a nil client rather than panicking inside a request", func() { // The interface a caller holds this through is satisfied by a typed nil // pointer, which is not an untyped nil and passes every `!= nil` check diff --git a/core/services/nodes/agent_selector.go b/core/services/nodes/agent_selector.go index 3ac924b9ff2c..ecfca85345bc 100644 --- a/core/services/nodes/agent_selector.go +++ b/core/services/nodes/agent_selector.go @@ -3,12 +3,16 @@ package nodes import ( + "cmp" "context" "errors" "fmt" "math/rand/v2" + "time" + "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/xlog" ) // ErrNoAgentWorker reports that no agent worker in this deployment currently @@ -34,6 +38,16 @@ var ErrNoAgentWorker = errors.New("nodes: no agent worker holds a tunnel to this // core/services/cluster. type AgentConnectionReader interface { ConnectedAmong(ctx context.Context, nodeIDs []string, owner string) (held []string, heldByOwner []string, err error) + + // Presence is what separates a worker that is GONE from one that is + // RECONNECTING, and a fan-out verb needs that split where a pick does not. + // + // A pick only ever asks who can be reached now. A cancel has to say what + // happened to the workers it could NOT reach, and those are two different + // answers: a departure older than the grace is a routing fact the caller + // may disregard, and a departure inside it is an absent connection nobody + // may act on, including by reporting that the run was not found. + Presence(ctx context.Context, nodeID string, grace time.Duration) (cluster.Presence, error) } // AgentSelector picks an agent worker to send a control RPC to. @@ -54,13 +68,50 @@ type AgentSelector struct { // call would simply take a relay hop with nothing saying so. It is refused // where the selector is built instead. selfInstanceID string + // grace is how long a lost tunnel is read as reconnecting rather than gone. + // It is the operator's trade (DistributedConfig.WorkerReconnectGrace) and + // is only consulted by Reachable; a pick never needs it, because a worker + // that is not connected cannot be picked whatever the reason. + grace time.Duration } // NewAgentSelector returns the selector for the agent workers registry knows // about, reading presence through conns and preferring the tunnels // selfInstanceID holds. -func NewAgentSelector(registry *NodeRegistry, conns AgentConnectionReader, selfInstanceID string) *AgentSelector { - return &AgentSelector{registry: registry, conns: conns, selfInstanceID: selfInstanceID} +// grace is the window a departure must outlive before Reachable calls a worker +// gone; a non-positive one takes config.DefaultWorkerReconnectGrace rather than +// zero, because a zero grace would call every worker that lost its tunnel a +// millisecond ago GONE and quietly drop the cancels addressed to it. +func NewAgentSelector(registry *NodeRegistry, conns AgentConnectionReader, selfInstanceID string, grace time.Duration) *AgentSelector { + return &AgentSelector{ + registry: registry, + conns: conns, + selfInstanceID: selfInstanceID, + grace: cmp.Or(grace, config.DefaultWorkerReconnectGrace), + } +} + +// AgentReach is what this deployment can say about its agent workers right now, +// and it is deliberately two lists rather than one plus a count. +// +// Connected are the workers a LIVE replica holds a tunnel to. A verb may be +// issued to each of them, relayed by the peer mesh when the holder is not this +// replica. +// +// Absent are the registered agent workers that no live replica holds AND whose +// departure has not outlived the grace, plus those whose presence could not be +// read at all. Nothing may be asked of them and, far more importantly, nothing +// may be CONCLUDED from their silence: a fan-out that ignored them would report +// "no worker is running that execution" about a fleet it had not finished +// asking. +// +// Workers whose departure IS older than the grace appear in neither list. That +// is the one routing fact this system lets a caller act on, and leaving them in +// Absent would make every deployment that has ever retired an agent worker +// report every cancel as undelivered for ever. +type AgentReach struct { + Connected []string + Absent []string } // PickConnected returns the id AND the node type of an agent node whose tunnel @@ -139,6 +190,84 @@ func (s *AgentSelector) pickConnectedExcluding(ctx context.Context, tried map[st return picked, nodeType, nil } +// Reachable splits this deployment's agent workers into the ones a verb can be +// issued to and the ones whose silence proves nothing. +// +// It exists for a cancel, which is a fan-out and not a pick: the frontend does +// not know which worker holds a given execution, so it asks every worker it can +// reach and assembles their answers. What it must never do is assemble an +// answer out of a fleet it only partly asked, which is why the workers it could +// not reach come back named rather than dropped. +func (s *AgentSelector) Reachable(ctx context.Context) (AgentReach, error) { + if s == nil || s.registry == nil || s.conns == nil { + return AgentReach{}, fmt.Errorf("this deployment has no agent selector: %w", ErrNoAgentWorker) + } + agents, err := s.registry.cancellableAgentNodes(ctx) + if err != nil { + return AgentReach{}, fmt.Errorf("listing the agent workers of this deployment: %w", err) + } + ids := make([]string, 0, len(agents)) + for _, a := range agents { + ids = append(ids, a.ID) + } + held, _, err := s.conns.ConnectedAmong(ctx, ids, s.selfInstanceID) + if err != nil { + return AgentReach{}, fmt.Errorf("reading which agent workers are connected: %w", err) + } + connected := make(map[string]bool, len(held)) + for _, id := range held { + connected[id] = true + } + reach := AgentReach{Connected: held} + for _, id := range ids { + if connected[id] { + continue + } + p, err := s.conns.Presence(ctx, id, s.grace) + if err != nil { + // A read that FAILED is not an answer about this worker, so it + // cannot license the caller to conclude anything. Counted absent, + // which is the value nobody may act on, rather than gone, which is + // the one value that would let a cancel be reported as a run that + // does not exist. + xlog.Warn("Could not read an agent worker's presence; counting it as one this cancel could not reach", + "nodeID", id, "error", err) + reach.Absent = append(reach.Absent, id) + continue + } + if p == cluster.PresenceGone { + continue + } + reach.Absent = append(reach.Absent, id) + } + return reach, nil +} + +// cancellableAgentNodes returns the agent nodes a fan-out verb may be OFFERED +// to, which is not the same set as the ones work may be PLACED on. +// +// It differs from selectableAgentNodes in exactly one status, and the +// difference is the whole reason it exists. A DRAINING worker may take no new +// work, which is the operator's decision, but it is still finishing the runs it +// already holds. A cancel is not new work: it is a request about a run that is +// executing right now, and excluding a draining worker would report the cancel +// of a live execution as a run that no worker is running. +// +// StatusPending stays excluded. An unapproved node is refused by the tunnel +// route on every dial, so it can hold no execution to cancel; counting it would +// make every cancel in a deployment with one unapproved node report undelivered +// for ever. +func (r *NodeRegistry) cancellableAgentNodes(ctx context.Context) ([]BackendNode, error) { + var agents []BackendNode + if err := r.db.WithContext(ctx). + Where("node_type = ? AND status <> ?", NodeTypeAgent, StatusPending). + Order("id"). + Find(&agents).Error; err != nil { + return nil, fmt.Errorf("listing agent nodes for a fan-out verb: %w", err) + } + return agents, nil +} + // selectableAgentNodes returns the agent nodes a control RPC may be sent to. // // Two statuses are excluded and no more. StatusPending is a node an admin has diff --git a/core/services/nodes/agent_selector_test.go b/core/services/nodes/agent_selector_test.go index 959762cfc14f..b1c77323ff84 100644 --- a/core/services/nodes/agent_selector_test.go +++ b/core/services/nodes/agent_selector_test.go @@ -6,11 +6,13 @@ import ( "context" "errors" "runtime" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "gorm.io/gorm" + "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/cluster" "github.com/mudler/LocalAI/core/services/testutil" ) @@ -32,6 +34,31 @@ type stubConnections struct { owners []string // asked records the candidate list of each call. asked [][]string + + // presence is what Presence answers per node, and presenceErr makes it + // fail. graces records the window each Presence call was made with: a + // selector that passed zero would call every worker that lost its tunnel a + // moment ago GONE, and silently drop the cancels addressed to it. + presence map[string]cluster.Presence + presenceErr error + graces []time.Duration +} + +// testGrace is the reconnect window these specs pass. A real value rather than +// zero, because zero is what NewAgentSelector substitutes a default for, and a +// spec that passed it could not tell the substitution from the value. +const testGrace = 42 * time.Second + +// presenceOf answers what the deployment can say about a node's tunnel. The +// zero value is cluster.PresenceUnknown, which is the value nobody may act on, +// so a spec that forgets to state a node's presence cannot accidentally get the +// one answer (PresenceGone) that licenses a caller to conclude something. +func (s *stubConnections) Presence(_ context.Context, nodeID string, grace time.Duration) (cluster.Presence, error) { + s.graces = append(s.graces, grace) + if s.presenceErr != nil { + return cluster.PresenceUnknown, s.presenceErr + } + return s.presence[nodeID], nil } func (s *stubConnections) ConnectedAmong(_ context.Context, nodeIDs []string, owner string) ([]string, []string, error) { @@ -106,7 +133,7 @@ var _ = Describe("AgentSelector", func() { conns.held = []string{mine, theirs} conns.heldByOwner = []string{mine} - sel := NewAgentSelector(registry, conns, "me") + sel := NewAgentSelector(registry, conns, "me", testGrace) for range picks { id, nodeType, err := sel.PickConnected(ctx) Expect(err).ToNot(HaveOccurred()) @@ -126,7 +153,7 @@ var _ = Describe("AgentSelector", func() { theirs := register("agent-theirs", NodeTypeAgent) conns.held = []string{theirs} - sel := NewAgentSelector(registry, conns, "me") + sel := NewAgentSelector(registry, conns, "me", testGrace) for range picks { id, nodeType, err := sel.PickConnected(ctx) Expect(err).ToNot(HaveOccurred()) @@ -144,7 +171,7 @@ var _ = Describe("AgentSelector", func() { conns.held = []string{a, b} conns.heldByOwner = []string{a, b} - sel := NewAgentSelector(registry, conns, "me") + sel := NewAgentSelector(registry, conns, "me", testGrace) seen := map[string]int{} // 40 draws of a fair two-way choice miss one side with probability // 2^-39, which is far below the flake floor of anything else here. @@ -177,17 +204,17 @@ var _ = Describe("AgentSelector", func() { It("refuses when registered agents exist but none is connected", func() { register("agent-a", NodeTypeAgent) - sel := NewAgentSelector(registry, conns, "me") + sel := NewAgentSelector(registry, conns, "me", testGrace) assertNoWorker(sel.PickConnected(ctx)) }) It("refuses when the deployment has no agent nodes at all", func() { - sel := NewAgentSelector(registry, conns, "me") + sel := NewAgentSelector(registry, conns, "me", testGrace) assertNoWorker(sel.PickConnected(ctx)) }) It("refuses when it was built with no way to read connections", func() { - assertNoWorker(NewAgentSelector(registry, nil, "me").PickConnected(ctx)) + assertNoWorker(NewAgentSelector(registry, nil, "me", testGrace).PickConnected(ctx)) }) }) @@ -201,7 +228,7 @@ var _ = Describe("AgentSelector", func() { conns.held = []string{backend, agent} conns.heldByOwner = []string{backend, agent} - sel := NewAgentSelector(registry, conns, "me") + sel := NewAgentSelector(registry, conns, "me", testGrace) for range picks { id, _, err := sel.PickConnected(ctx) Expect(err).ToNot(HaveOccurred()) @@ -219,7 +246,7 @@ var _ = Describe("AgentSelector", func() { conns.held = []string{pending.ID} conns.heldByOwner = []string{pending.ID} - sel := NewAgentSelector(registry, conns, "me") + sel := NewAgentSelector(registry, conns, "me", testGrace) _, _, err = sel.PickConnected(ctx) Expect(err).To(MatchError(ErrNoAgentWorker)) Expect(conns.asked[0]).To(BeEmpty()) @@ -231,7 +258,7 @@ var _ = Describe("AgentSelector", func() { conns.held = []string{id} conns.heldByOwner = []string{id} - sel := NewAgentSelector(registry, conns, "me") + sel := NewAgentSelector(registry, conns, "me", testGrace) _, _, err := sel.PickConnected(ctx) Expect(err).To(MatchError(ErrNoAgentWorker)) }) @@ -247,17 +274,126 @@ var _ = Describe("AgentSelector", func() { conns.held = []string{id} conns.heldByOwner = []string{id} - sel := NewAgentSelector(registry, conns, "me") + sel := NewAgentSelector(registry, conns, "me", testGrace) picked, _, err := sel.PickConnected(ctx) Expect(err).ToNot(HaveOccurred()) Expect(picked).To(Equal(id)) }) + // Reachable is what a FAN-OUT verb asks, and it differs from a pick in + // exactly one way that matters: it has to say what happened to the workers + // it could not reach. A cancel assembled from a fleet it only partly asked + // reports "no worker is running that execution" about a run that is. + Describe("Reachable", func() { + It("separates a worker that is GONE from one that is merely reconnecting", func() { + // The whole invariant in one spec. A departure older than the grace + // is the one routing fact a caller may act on, so it disappears; a + // departure inside it is an absent connection nobody may act on, so + // it must be named. + live := register("agent-live", NodeTypeAgent) + reconnecting := register("agent-reconnecting", NodeTypeAgent) + gone := register("agent-gone", NodeTypeAgent) + conns.held = []string{live} + conns.presence = map[string]cluster.Presence{ + reconnecting: cluster.PresenceReconnecting, + gone: cluster.PresenceGone, + } + + reach, err := NewAgentSelector(registry, conns, "me", testGrace).Reachable(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(reach.Connected).To(ConsistOf(live)) + Expect(reach.Absent).To(ConsistOf(reconnecting)) + }) + + It("counts a worker that has never dialled as absent, not as gone", func() { + // PresenceUnknown is a worker that has never dialled or whose + // departure aged out of retention, and the registry cannot say + // which. Reading it as gone would license a caller to conclude + // something about a worker it has learned nothing about. + never := register("agent-never", NodeTypeAgent) + + reach, err := NewAgentSelector(registry, conns, "me", testGrace).Reachable(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(reach.Connected).To(BeEmpty()) + Expect(reach.Absent).To(ConsistOf(never)) + }) + + It("counts a presence it could not read as absent, because a failed read is not an answer", func() { + id := register("agent-a", NodeTypeAgent) + conns.presenceErr = errors.New("the database would not answer") + + reach, err := NewAgentSelector(registry, conns, "me", testGrace).Reachable(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(reach.Absent).To(ConsistOf(id)) + }) + + It("measures a departure against the grace it was built with", func() { + // A zero window would call every worker that lost its tunnel a + // moment ago GONE, and the cancels addressed to it would be + // reported as a run nobody is running. + register("agent-a", NodeTypeAgent) + + _, err := NewAgentSelector(registry, conns, "me", testGrace).Reachable(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(conns.graces).To(ConsistOf(testGrace)) + }) + + It("substitutes the deployment default for a grace of zero", func() { + register("agent-a", NodeTypeAgent) + + _, err := NewAgentSelector(registry, conns, "me", 0).Reachable(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(conns.graces).To(ConsistOf(config.DefaultWorkerReconnectGrace)) + }) + + It("excludes a PENDING node, which can hold no execution to cancel", func() { + // An unapproved node is refused by the tunnel route on every dial. + // Counting it would make every cancel in a deployment with one + // unapproved agent node report undelivered for ever. + pending := &BackendNode{Name: "agent-pending", NodeType: NodeTypeAgent, Address: "p:50051"} + Expect(registry.Register(ctx, pending, false)).To(Succeed()) + + reach, err := NewAgentSelector(registry, conns, "me", testGrace).Reachable(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(reach.Connected).To(BeEmpty()) + Expect(reach.Absent).To(BeEmpty()) + }) + + It("still offers a DRAINING worker, which may take no new work but is finishing what it holds", func() { + // The one status where a fan-out and a pick must disagree. A + // draining worker is still running the executions it took, so + // leaving it out would report the cancel of a live run as a run + // that no worker is running. + id := register("agent-draining", NodeTypeAgent) + Expect(registry.MarkDraining(ctx, id)).To(Succeed()) + conns.held = []string{id} + + reach, err := NewAgentSelector(registry, conns, "me", testGrace).Reachable(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(reach.Connected).To(ConsistOf(id)) + + // And the pick still refuses it, which is what the two sets are for. + _, _, err = NewAgentSelector(registry, conns, "me", testGrace).PickConnected(ctx) + Expect(err).To(MatchError(ErrNoAgentWorker)) + }) + + It("reports a connection read that failed as neither an answer nor a route verdict", func() { + register("agent-a", NodeTypeAgent) + conns.err = errors.New("the database would not answer") + + _, err := NewAgentSelector(registry, conns, "me", testGrace).Reachable(ctx) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeFalse()) + Expect(cluster.IsWorkerAnswer(err)).To(BeFalse()) + Expect(errors.Is(err, ErrNoAgentWorker)).To(BeFalse()) + }) + }) + It("reports a connection read that failed as neither an answer nor a route verdict", func() { register("agent-a", NodeTypeAgent) conns.err = errors.New("the database would not answer") - sel := NewAgentSelector(registry, conns, "me") + sel := NewAgentSelector(registry, conns, "me", testGrace) id, nodeType, err := sel.PickConnected(ctx) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("the database would not answer")) diff --git a/core/services/nodes/interfaces.go b/core/services/nodes/interfaces.go index 51663514a3fe..5ac85e87c10f 100644 --- a/core/services/nodes/interfaces.go +++ b/core/services/nodes/interfaces.go @@ -111,12 +111,11 @@ type NodeHealthStore interface { // ModelLocator is used by RemoteUnloaderAdapter for model discovery. // -// Get is here for one reason: backend.stop is the only control verb whose -// carrier depends on what KIND of worker it is addressed to, because an agent -// node's stop is still published on the bus rather than issued over the tunnel -// it now holds. The callers that come through NodeCommandSender carry a node id -// and nothing else, so the type is read here rather than threaded through every -// one of them. +// Get is here for the node reads that need the whole row rather than a routing +// answer. It is no longer needed to pick a CARRIER for backend.stop: an agent +// worker and a backend worker serve that verb on the same control path over +// their own tunnels, and the frontend states the fact without asking what kind +// of worker it reached (see RemoteUnloaderAdapter.stopBackend). type ModelLocator interface { FindNodesWithModel(ctx context.Context, modelName string) ([]BackendNode, error) RemoveNodeModel(ctx context.Context, nodeID, modelName string, replicaIndex int) error diff --git a/core/services/workerctl/paths.go b/core/services/workerctl/paths.go index fbc49ae12400..8803e2478035 100644 --- a/core/services/workerctl/paths.go +++ b/core/services/workerctl/paths.go @@ -88,10 +88,9 @@ func BackendPaths() []string { // cached for that backend. The frontend issues the same RPC to either and does // not branch on the node's type to pick a carrier. // -// PathAgentCancel is named here with nothing mounting it yet. The frontend -// therefore gets the catch-all's 404, which it already reads as "this worker -// does not serve that verb" rather than as absence, and the path is fixed now -// so the two sides cannot disagree about it later. +// PathAgentCancel is served by an agent worker and by nothing else. It is what +// replaced the agent..cancel broadcast, and moving that family here is +// what removed the last reason an agent worker dialled a message bus. func AgentPaths() []string { return []string{ PathMCPToolExecute, diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index cd63f4f72816..7ce0671035ba 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -5,7 +5,7 @@ weight = 71 url = "/features/distributed-mode/" +++ -Distributed mode enables horizontal scaling of LocalAI across multiple machines using **PostgreSQL** for state, node registry and cross-replica fan-out. A **NATS** server is still needed for one thing: delivering an agent cancel to the agent worker running the execution. Unlike the [P2P/federation approach]({{% relref "features/distributed_inferencing" %}}), distributed mode is designed for production deployments and Kubernetes environments where you need centralized management, health monitoring, and deterministic routing. +Distributed mode enables horizontal scaling of LocalAI across multiple machines using **PostgreSQL** for state, node registry and cross-replica fan-out. No message bus is needed: a deployment runs PostgreSQL and the frontends' own HTTP listener, and every worker is reached over the tunnel it dials outward. Unlike the [P2P/federation approach]({{% relref "features/distributed_inferencing" %}}), distributed mode is designed for production deployments and Kubernetes environments where you need centralized management, health monitoring, and deterministic routing. {{% notice note %}} Distributed mode requires authentication enabled with a **PostgreSQL** database - SQLite is not supported. This is because the node registry, job store, and other distributed state are stored in PostgreSQL tables. @@ -41,7 +41,7 @@ Each model gets its own gRPC backend process, so a single worker can serve multi - Each frontend replica holds **one extra PostgreSQL session** beyond its connection pool, pinned for the life of the process, and creates a `bus_messages` table. Both belong to the broadcast carrier that is replacing NATS for cross-replica fan-out; it already carries the four `state.*.delta` families (see [Cross-replica in-memory state](#cross-replica-in-memory-state)). Size `max_connections` for one additional session per frontend replica. - That session reports an `application_name` of `localai_pgbus_`, so `SELECT count(*) FROM pg_stat_activity WHERE application_name LIKE 'localai_pgbus_%'` counts the replicas currently listening. If the carrier loses its session it redials and re-registers on its own; a broadcast published while it was down is not replayed, which is why nothing that must survive a gap is carried by a broadcast alone. - `bus_messages` holds only broadcasts too large for a PostgreSQL notification, and every replica retires rows older than ten minutes. The table is a spill buffer, not a log: it is not a place to read past events from. -- **NATS** server - used for ONE subject family, `agent..cancel`, which is the only thing a frontend still sends over a bus and the only thing an agent worker still subscribes to. Everything else a frontend broadcasts travels on the PostgreSQL the deployment already runs, and every verb a frontend addresses to a worker is an HTTP route on that worker's tunnel. **Serve-backend workers do not connect to it at all**: every verb they take, and file staging with it, is an HTTP route on the worker's tunnel. Set no `LOCALAI_NATS_URL` on a `local-ai worker`. The frontend and any `local-ai agent-worker` still need one. +- **No message bus.** Nothing in a distributed deployment connects to NATS any more. Everything a frontend broadcasts travels on the PostgreSQL the deployment already runs; every verb a frontend addresses to a worker, including an agent cancel, is an HTTP route on that worker's own tunnel. `LOCALAI_NATS_URL` is accepted and ignored everywhere - on the frontend, on `local-ai worker` and on `local-ai agent-worker` - so an existing command line still starts. - All services must be on the same network (or reachable via configured URLs) ## Quick Start with Docker Compose @@ -52,10 +52,10 @@ The easiest way to try distributed mode locally is with the provided Docker Comp docker compose -f docker-compose.distributed.yaml up ``` -This starts PostgreSQL, NATS, a LocalAI frontend, and one worker node. When you send an inference request, the SmartRouter automatically installs the needed backend on the worker and loads the model. See the file for details on adding GPU support, shared volumes, and additional workers. +This starts PostgreSQL, a LocalAI frontend, and one worker node. The compose file still stands a NATS container up; nothing connects to it and you may delete that service. When you send an inference request, the SmartRouter automatically installs the needed backend on the worker and loads the model. See the file for details on adding GPU support, shared volumes, and additional workers. {{% notice tip %}} -Use `docker-compose.distributed.yaml` for quick local testing. For production, deploy PostgreSQL and NATS as managed services and run frontends/workers on separate hosts. +Use `docker-compose.distributed.yaml` for quick local testing. For production, deploy PostgreSQL as a managed service and run frontends/workers on separate hosts. There is no message bus to deploy. {{% /notice %}} ## Frontend Configuration @@ -66,11 +66,11 @@ The frontend is a standard LocalAI instance with distributed mode enabled. These |------|---------|---------|-------------| | `--distributed` | `LOCALAI_DISTRIBUTED` | `false` | Enable distributed mode | | `--instance-id` | `LOCALAI_INSTANCE_ID` | auto UUID | Unique instance ID for this frontend | -| `--nats-url` | `LOCALAI_NATS_URL` | *(required)* | NATS server URL (e.g., `nats://localhost:4222`) | +| `--nats-url` | `LOCALAI_NATS_URL` | *(ignored)* | **Accepted and ignored.** A frontend opens no message-bus connection. Kept so an existing command line still starts. | | `--distributed-advertise-addr` | `LOCALAI_DISTRIBUTED_ADVERTISE_ADDR` | *(derived)* | `host:port` the **other frontend replicas** dial to reach this one. See [Replica peer links](#replica-peer-links). | | `--registration-token` | `LOCALAI_REGISTRATION_TOKEN` | *(empty)* | Token that workers must provide to register | | `--registration-require-auth` | `LOCALAI_REGISTRATION_REQUIRE_AUTH` | `false` | Fail startup when distributed mode is enabled but the registration token is empty (node endpoints and worker file-transfer would otherwise be unauthenticated) | -| `--distributed-require-auth` | `LOCALAI_DISTRIBUTED_REQUIRE_AUTH` | `false` | **Umbrella switch.** Implies both `--nats-require-auth` and `--registration-require-auth` - one knob to lock down the NATS bus *and* the registration/file-transfer layer. Set this in production instead of the two granular flags. | +| `--distributed-require-auth` | `LOCALAI_DISTRIBUTED_REQUIRE_AUTH` | `false` | **Umbrella switch.** Implies `--registration-require-auth`, which is what guards registration, the worker control planes and file transfer. It also implies the inert `--nats-require-auth`. Set this in production instead of the granular flags. | | `--auto-approve-nodes` | `LOCALAI_AUTO_APPROVE_NODES` | `false` | Auto-approve new worker nodes (skip admin approval) | | `--distributed-shared-models` | `LOCALAI_DISTRIBUTED_SHARED_MODELS` | `false` | Assert that every node mounts the **same** models directory at the **same** path (a shared volume). When `true`, the router skips file staging entirely and workers load models directly from the shared path instead of re-downloading them. See [Shared models directory](#shared-models-directory). | | `--distributed-disk-headroom-check` | `LOCALAI_DISTRIBUTED_DISK_HEADROOM_CHECK` | `true` | Reject worker nodes that lack free space to store the model, at scheduling time rather than partway through staging. When `false`, node selection ignores free disk; the check still runs and warns when it would have rejected every node. Also toggleable at runtime via the `distributed_disk_headroom_check` setting. See [Disk headroom](#disk-headroom). | @@ -187,10 +187,9 @@ The same carrier moves the traffic whose subscriber is an open HTTP response rat | Job result | `jobs..result` | The result persister on every replica | | Job cancel | `jobs..cancel` | Every replica, so the one holding the run can stop it | | Agent events | `agent..events.` | `GET /api/agents/{name}/sse/distributed` on any replica, and the observable persister | -| Agent cancel | `agent..cancel` | Every agent worker, and so still on NATS: see below | | Open Responses cancel | `responses..cancel` | The replica holding the generation | -One family on that list is not on this carrier. An `agent..cancel` has to reach the agent WORKER running the execution, and an agent worker has no database, so it cannot listen on PostgreSQL at all; that cancel is published on NATS, where the worker is listening, and it stays there until a cancel rides the worker's tunnel like every other verb the frontend addresses to a worker. Everything else in the table is on the PostgreSQL carrier. +An agent cancel is deliberately NOT on that list. It has to reach the agent WORKER running the execution, and an agent worker has no database, so it can never listen on PostgreSQL; it is a control verb on that worker's own tunnel instead. See [Cancelling an agent run](#cancelling-an-agent-run). This is what lets a user watch a job or an agent on one frontend while the work runs against another. **No broadcast on this list is the only path to anything durable.** A job's terminal state is written to its row by the replica that claimed the work, before that claim is released, so a dropped result costs an open stream its promptness and never costs the job its answer: a stream that is still open re-reads the row and closes on it. A cancel is a request and not a verdict: if it reaches nobody it has not been refused, and nothing in the API reports it as such. @@ -264,7 +263,7 @@ A worker that presents a credential belonging to no node, or names a node ID the **A node that has not registered since upgrading cannot tunnel.** Its row has no tunnel credential and the column cannot be back-filled, because the plaintext only ever existed in the response that minted it. Such a node is refused with `401` until it registers again, which a worker restart does. The frontend does *not* fall back to the registration token for these nodes. -Unlike the agent worker's API key and its NATS credential, a tunnel credential **is** issued to a node still awaiting approval. It is inert until then: the tunnel route re-reads the node's status on every dial and refuses a pending one. Withholding it would instead strand workers that register exactly once, since approval on its own prompts no re-registration. +Unlike the agent worker's API key, a tunnel credential **is** issued to a node still awaiting approval. It is inert until then: the tunnel route re-reads the node's status on every dial and refuses a pending one. Withholding it would instead strand workers that register exactly once, since approval on its own prompts no re-registration. A tunnel credential does not replace `LOCALAI_REGISTRATION_TOKEN`. Without one, node registration itself is unauthenticated, so anyone who can reach the frontend can register a worker and be issued a tunnel credential for it. How far that gets them depends on auto-approve: with auto-approve on the node is healthy at once and the credential works immediately; with it off the node is pending and the credential is inert until an admin approves, so approval is the real gate. LocalAI warns about the missing token at startup. @@ -272,7 +271,21 @@ Both **backend** and **agent** nodes are issued one. Earlier releases minted a c An agent worker's tunnel carries only the `http` tag: it runs no backend processes, so it does not offer the `grpc` tag at all. Its control server binds `127.0.0.1` on a port chosen by the kernel and advertises it nowhere, so an agent worker still opens no inbound port. -**An agent worker still requires `--nats-url`, and one thing only still reaches it on the bus.** Every verb the frontend addresses to a specific agent worker is now a control RPC on the tunnel that worker holds: MCP tool execution, MCP discovery, the backend stop that flushes cached MCP sessions, agent execution, and MCP CI runs. The progress and result lines the worker asks the frontend to re-publish on its behalf travel back on that same response body, and the frontend re-publishes them on the PostgreSQL carrier, not on NATS. One subject in the other direction is still NATS and is the reason `--nats-url` is still required: `agent..cancel`, which the worker subscribes to so a cancel can reach the execution it is running. +**An agent worker no longer needs `--nats-url`, and connects to no message bus at all.** Every verb the frontend addresses to a specific agent worker is a control RPC on the tunnel that worker holds: MCP tool execution, MCP discovery, the backend stop that flushes cached MCP sessions, agent execution, MCP CI runs, and now the cancel. The progress and result lines the worker asks the frontend to re-publish on its behalf travel back on that same response body, and the frontend re-publishes them on the PostgreSQL carrier. + +#### Cancelling an agent run + +A cancel names one execution by its message id, and nothing in the deployment records which worker holds it: the claim that dispatched the run names the claiming *replica*, and it is deleted when the run ends. So the frontend offers the cancel to **every agent worker a live replica can reach**, over each worker's own tunnel (`POST /v1/control/agent/cancel`), and each worker answers only for itself. + +A caller gets one of three answers, and they are deliberately different facts: + +| Outcome | What it means | +|---------|---------------| +| success | A worker answered that it cancelled the run, or the run was held by the replica the request landed on. | +| *could not be delivered* | At least one agent worker that might have been running it was not reached: its tunnel was lost inside the [reconnect grace](#a-lost-tunnel-is-a-departure-not-an-absence), its control plane refused the stream, or a peer holding it was unreachable. Nothing was learned. It is **not** a refusal and **not** a missing run. | +| *no agent worker is running that execution* | Every agent worker was reached and every one of them answered that it does not hold the run. | + +A worker that is **reconnecting** always produces the second answer. The cancel is not retried inside the request and not queued: retrying would hold the caller for the length of the reconnect grace, and the retry belongs with whoever owns the budget. Re-issue the cancel once the worker is connected again. There is no `nodes..*` subject left, and an agent worker's minted JWT no longer grants `mcp.tools.execute`, `mcp.discovery`, `nodes..backend.stop`, `agent.execute` or `jobs.mcp-ci.new`. The `--agent-subject` and `--agent-queue` flags (`LOCALAI_AGENT_SUBJECT`, `LOCALAI_AGENT_QUEUE`) are gone: there is no subject for an agent worker to subscribe to and no queue group to be one of. @@ -467,7 +480,7 @@ Registering against an upgraded frontend **clears** a node's `address` and `http A worker on this release opens **no inbound listener on a routable interface**. Its backend gRPC processes and its HTTP file-transfer server all bind loopback, and the frontend reaches both through the tunnel. Concretely: -- **No inbound firewall rule, published port, Service or Ingress is needed for a worker.** A serve-backend worker needs outbound access to the frontend URL (`LOCALAI_REGISTER_TO`), and nothing else - not even to NATS. An agent worker binds only loopback too, and needs outbound access to both `LOCALAI_REGISTER_TO` (registration, heartbeats and now its tunnel) and `LOCALAI_NATS_URL`. +- **No inbound firewall rule, published port, Service or Ingress is needed for a worker.** A serve-backend worker needs outbound access to the frontend URL (`LOCALAI_REGISTER_TO`), and nothing else - not even to NATS. An agent worker binds only loopback too, and needs outbound access to `LOCALAI_REGISTER_TO` and nothing else either: registration, heartbeats, its tunnel and every verb the frontend addresses to it all go there. - **`LOCALAI_ADVERTISE_ADDR` and `LOCALAI_ADVERTISE_HTTP_ADDR` are gone.** There is nothing to advertise. Both are ignored if still set; remove them. - **`LOCALAI_ADDR` and `LOCALAI_SERVE_ADDR` are read for their port only.** The port is the base of the backend port range, and `port-1` is the HTTP file-transfer port. The host half names an interface nothing binds. - The node's `address` and `http_address` fields in `GET /api/nodes` are empty, and are cleared for nodes that reported them before the upgrade. @@ -583,50 +596,25 @@ The chat UI renders this state inline and retries automatically once the model r A frontend replica that dies mid-load does not wedge the model: the job row carries a heartbeat and another replica reclaims a job whose heartbeat has stopped. The heartbeat is time-based, not byte-based, because a checkpoint load legitimately transfers zero bytes for many minutes. {{% /notice %}} -### NATS JWT authentication (recommended for production) - -**This section is about agent workers and the frontend.** A serve-backend worker opens no NATS connection, so none of it applies to one; its own credential is the tunnel token it gets at registration, and its control plane is authenticated by `LOCALAI_REGISTRATION_TOKEN`. An agent worker now has both: a NATS credential, which covers the one subject family still on the bus, and a tunnel token plus the same `LOCALAI_REGISTRATION_TOKEN` bearer check in front of its control server. - -By default, NATS connections are anonymous: any client that can reach port `4222` may publish the one subject family still carried on it, `agent..cancel`. Anyone who can reach an unauthenticated bus can therefore cancel any running agent. Nothing else is on it: the agent-worker job subjects became rows in a claim table, the frontend's cross-replica events travel on PostgreSQL, and `nodes..backend.install` and its nine siblings are HTTP routes on the worker's tunnel, see [The worker control plane](#the-worker-control-plane). Enable JWT auth to scope agent workers to their own subjects and give the frontend a dedicated service credential. - -| Flag | Env Var | Description | -|------|---------|-------------| -| `--nats-account-seed` | `LOCALAI_NATS_ACCOUNT_SEED` | Account signing seed (`SU...`). The frontend mints a per-node user JWT at registration (`nats_jwt` in the register response). | -| `--nats-service-jwt` | `LOCALAI_NATS_SERVICE_JWT` | User JWT for the frontend (and optional fallback for agent workers). The frontend publishes one subject family with it: `agent..cancel`. | -| `--nats-service-seed` | `LOCALAI_NATS_SERVICE_SEED` | User signing seed (`SU...`) paired with the service JWT. | -| `--nats-worker-jwt-ttl` | `LOCALAI_NATS_WORKER_JWT_TTL` | Lifetime of minted worker JWTs (default `24h`). | -| `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | Fail startup if JWT credentials are missing when distributed mode is enabled. | - -### NATS TLS / mTLS (optional) +### NATS credentials (inert) -Use `tls://` in `--nats-url` / `LOCALAI_NATS_URL` for encrypted transport. When the server uses a private CA or requires client certificates, set: - -| Flag | Env Var | Description | -|------|---------|-------------| -| `--nats-tls-ca` | `LOCALAI_NATS_TLS_CA` | PEM file to verify the NATS server (private CA) | -| `--nats-tls-cert` | `LOCALAI_NATS_TLS_CERT` | Client certificate for NATS mTLS | -| `--nats-tls-key` | `LOCALAI_NATS_TLS_KEY` | Client private key (required with `--nats-tls-cert`) | - -The same env vars apply to backend workers and `local-ai agent-worker`. If the server cert is already trusted by the OS, `tls://` alone is enough. - -**Worker register response** (when minting is enabled and the node is approved): - -```json -{ - "id": "…", - "nats_jwt": "eyJ…", - "nats_user_seed": "SU…" -} -``` +**No LocalAI component connects to NATS.** The frontend's cross-replica fan-out is on PostgreSQL, a serve-backend worker takes every verb on its own tunnel, and an agent worker now does too, including the cancel that was the last family on a bus. -Agent workers connect with that JWT and seed automatically (shown once; store securely). Override with `LOCALAI_NATS_JWT` / `LOCALAI_NATS_USER_SEED` if needed. Set `LOCALAI_NATS_REQUIRE_AUTH=true` on an agent worker when the bus requires credentials; `local-ai worker` has no such flag, because it opens no connection to require credentials for. A JWT is still minted for a serve-backend node at registration and is simply unused; it grants nothing but that connection's own reply inbox. +Every `LOCALAI_NATS_*` setting is therefore accepted and inert, so an existing command line, unit file or Helm values file starts unchanged: -When `LOCALAI_NATS_REQUIRE_AUTH=true` and no static credentials are provided, an agent worker that registers while still **pending admin approval** keeps re-registering (with backoff) until an admin approves it and the frontend mints its JWT - it does not start unauthenticated. This retry is **bounded**: if the node is never approved (or no credentials are minted) after a large number of attempts, the worker exits non-zero so the failure is visible (a crash-looping or failed worker) rather than hanging silently. Minted worker JWTs are also **refreshed automatically** before they expire (the worker re-registers at ~75% of the JWT lifetime), so long-running workers survive past `LOCALAI_NATS_WORKER_JWT_TTL`; the NATS connection picks up the new JWT on its next reconnect. If refresh fails persistently, the worker exits (to restart and re-acquire) rather than drifting toward an expired, unrenewable JWT. Statically configured (`LOCALAI_NATS_JWT`) and service (`LOCALAI_NATS_SERVICE_JWT`) credentials are used as-is and not refreshed. +| Flag | Env Var | Status | +|------|---------|--------| +| `--nats-url` | `LOCALAI_NATS_URL` | Accepted and ignored on the frontend, `local-ai worker` and `local-ai agent-worker`. | +| `--nats-account-seed` | `LOCALAI_NATS_ACCOUNT_SEED` | The frontend still mints a per-node user JWT at registration (`nats_jwt` in the register response). Nothing consumes it. | +| `--nats-service-jwt` / `--nats-service-seed` | `LOCALAI_NATS_SERVICE_JWT` / `LOCALAI_NATS_SERVICE_SEED` | Accepted, unused: the frontend opens no bus connection to present them on. | +| `--nats-worker-jwt-ttl` | `LOCALAI_NATS_WORKER_JWT_TTL` | Lifetime of the minted-but-unused worker JWTs. | +| `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | On an agent worker this still makes registration **wait through admin approval** rather than starting against a pending node. It no longer gates any bus connection. | +| `--nats-tls-ca` / `--nats-tls-cert` / `--nats-tls-key` | `LOCALAI_NATS_TLS_*` | Accepted, unused. | -Generate operator/account material with [`scripts/nats-auth-setup.sh`](https://github.com/mudler/LocalAI/blob/master/scripts/nats-auth-setup.sh) (requires [nsc](https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nsc)). Configure the NATS server with account resolver JWTs before enabling `LOCALAI_NATS_REQUIRE_AUTH`. +You may stop running a NATS server, and remove these settings at your convenience. {{% notice note %}} -`LOCALAI_AUTH` (HTTP users/sessions) and NATS JWTs are separate: end-user API keys do not connect to NATS. HTTP registration still uses `LOCALAI_REGISTRATION_TOKEN`. +`LOCALAI_AUTH` (HTTP users/sessions) is unrelated. HTTP registration still uses `LOCALAI_REGISTRATION_TOKEN`, and every worker control plane sits behind that same bearer check. {{% /notice %}} ### Optional: S3 Object Storage @@ -676,7 +664,7 @@ during installation as well as the committed snapshot. {{% /notice %}} {{% notice warning %}} -The worker HTTP file transfer server is authenticated by `LOCALAI_REGISTRATION_TOKEN`. If the token is **empty**, the server **fails open** - anyone who can reach the port gets read/write access to the worker's models/staging/data directories (a remote model-poisoning / exfiltration vector), **and to the `/v1/control/` routes that install, upgrade and delete backends and stop the node**. The worker logs a loud warning at startup in this case. Always set `LOCALAI_REGISTRATION_TOKEN` in distributed mode, and set `LOCALAI_DISTRIBUTED_REQUIRE_AUTH=true` (frontend **and** workers) to make a missing token a hard startup error rather than a silent fail-open. On the frontend and on agent workers it also makes missing NATS credentials fatal; on a serve-backend worker it means the registration token alone, since that worker uses no bus credential. +The worker HTTP file transfer server is authenticated by `LOCALAI_REGISTRATION_TOKEN`. If the token is **empty**, the server **fails open** - anyone who can reach the port gets read/write access to the worker's models/staging/data directories (a remote model-poisoning / exfiltration vector), **and to the `/v1/control/` routes that install, upgrade and delete backends and stop the node**. The worker logs a loud warning at startup in this case. Always set `LOCALAI_REGISTRATION_TOKEN` in distributed mode, and set `LOCALAI_DISTRIBUTED_REQUIRE_AUTH=true` (frontend **and** workers) to make a missing token a hard startup error rather than a silent fail-open. On an agent worker it additionally makes registration wait through admin approval instead of starting against a pending node. By default the server binds loopback, so "anyone who can reach the port" means a process on the worker host, and no firewall rule is required. Setting `LOCALAI_HTTP_ADDR` to a routable address opts back out of that and puts the fail-open case back on the network - if you do it, firewall the port. {{% /notice %}} @@ -1254,12 +1242,11 @@ Agent workers are dedicated processes for executing agent chats and MCP CI jobs. ```bash local-ai agent-worker \ --register-to http://frontend:8080 \ - --nats-url nats://nats:4222 \ --registration-token changeme ``` Agent workers: -- Execute agent chat messages dispatched via NATS +- Execute agent chat messages dispatched to it as streaming control verbs on its tunnel - Run MCP CI jobs (with access to MCP servers via docker) - Handle MCP tool discovery and execution requests, which the frontend sends over the worker's own tunnel - Get auto-provisioned API keys during registration for calling the inference API @@ -1476,7 +1463,7 @@ local-ai worker \ --registration-token changeme ``` -**Multiple frontend replicas:** Run multiple LocalAI frontends behind a load balancer. Since all state is in PostgreSQL and coordination is via NATS, frontends are fully stateless and interchangeable. +**Multiple frontend replicas:** Run multiple LocalAI frontends behind a load balancer. Since all state is in PostgreSQL and coordination is via PostgreSQL and the workers' own tunnels, frontends are fully stateless and interchangeable. ## Model Scheduling @@ -1679,12 +1666,12 @@ Notes: |---|---|---| | **Discovery** | Automatic via libp2p token | Self-registration to frontend URL | | **State storage** | In-memory / ledger | PostgreSQL | -| **Coordination** | Gossip protocol | The worker's own tunnel for serve-backend work; PostgreSQL `LISTEN`/`NOTIFY` for cross-replica frontend events; NATS for agent workers | +| **Coordination** | Gossip protocol | Each worker's own tunnel for every verb addressed to it, agent workers included; PostgreSQL `LISTEN`/`NOTIFY` for cross-replica frontend events | | **Node management** | Automatic | REST API + WebUI | | **Health monitoring** | Peer heartbeats | Centralized HealthMonitor | | **Backend management** | Manual per node | Dynamic via the worker's `backend.install` control route | | **Best for** | Ad-hoc clusters, community sharing | Production, Kubernetes, managed infrastructure | -| **Setup complexity** | Minimal (share a token) | Requires PostgreSQL on the frontend, plus NATS if you run agent workers. Serve-backend workers need neither: only an outbound route to the frontend URL. | +| **Setup complexity** | Minimal (share a token) | Requires PostgreSQL on the frontend, and nothing else. Workers of either kind need only an outbound route to the frontend URL. | ## Troubleshooting @@ -1694,9 +1681,7 @@ Notes: - Ensure auth is enabled on the frontend (`LOCALAI_AUTH=true`) **NATS connection errors:** -- These concern the **frontend** and **agent workers** only. A `local-ai worker` opens no NATS connection; if one is failing to join, look at its tunnel and its `--register-to` instead. -- Confirm NATS is running and reachable (`nats-server --signal ldm` or check port 4222) -- Check that `--nats-url` uses the correct hostname/IP from that component's network perspective +- Nothing in LocalAI connects to NATS any more, on any component. If a release you are running still logs one, it predates the tunnel migration; on this release, look at the failing component's tunnel and its `--register-to` instead. **PostgreSQL connection errors:** - Verify the connection URL format: `postgresql://user:password@host:5432/dbname?sslmode=disable` @@ -1739,7 +1724,7 @@ Notes: - It is **not** the same as the worker being gone, and nothing acts on it as if it were. A model on an unroutable worker is not reaped, its rows are left alone, and the node is not demoted: doing any of those on a lost route is how a rolling frontend restart turns into a fleet-wide eviction. - Check the worker process is running and that it has an open tunnel (`opened a tunnelled stream to a worker` in the frontend log, and the worker's own dial/reconnect lines). A worker behind a load balancer that keeps reconnecting is usually an idle-timeout or WebSocket-upgrade problem at the proxy; see the tunnel section above. - **`no route` is not `gone`, and nothing in the frontend reads it as such.** A worker is declared **gone** by one mechanism only: no live frontend replica holds its tunnel *and* its departure is older than `--worker-reconnect-grace`. That is a fact recorded in the shared database, so every replica answers it identically. "No route" is one replica failing to reach a worker right now, and it is not evidence about the worker at all. -- Older releases decided absence from `nats: no responders available for request`, which was one frontend's observation that nobody answered *it* within a request budget. Two replicas asking at the same moment could disagree and demote each other's workers. That signal is gone from the scheduler; if you still see the message, it concerns only the subjects that remain on the bus (agent-worker jobs and MCP CI), never a serve-backend worker. +- Older releases decided absence from `nats: no responders available for request`, which was one frontend's observation that nobody answered *it* within a request budget. Two replicas asking at the same moment could disagree and demote each other's workers. That signal is gone from the scheduler, and no component opens a bus connection to produce it. **A worker fills its own disk over time:** - A request that carries a file (an image, an audio clip, a video) stages that file below the worker's HTTP staging or S3 cache `ephemeral/` directory. The frontend releases each request-owned input when inference finishes, and the worker reserves capacity before accepting it. diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md index dcbdacd1fac9..5478b6de0292 100644 --- a/docs/content/reference/cli-reference.md +++ b/docs/content/reference/cli-reference.md @@ -224,7 +224,7 @@ LocalAI supports several subcommands beyond `run`: - `local-ai mcp-server` - Run the LocalAI admin tool surface as a stdio MCP server (controls a remote LocalAI instance over HTTP) - `local-ai worker` - Start a worker for distributed mode (generic, backend-agnostic; needs only an outbound route to the frontend, no message bus) - `local-ai p2p-worker` - Run workers to distribute workload via p2p (llama.cpp-only) -- `local-ai agent-worker` - Start an agent worker for distributed mode (serves agent execution and MCP CI runs as control verbs on the tunnel it holds; still needs `--nats-url`, for cancellation and nothing else: an agent cancel has to reach the worker running the execution, and a worker has no database and so cannot read the PostgreSQL carrier every other broadcast travels on) +- `local-ai agent-worker` - Start an agent worker for distributed mode (serves agent execution, MCP CI runs and cancellation as control verbs on the tunnel it holds; it connects to no message bus, so `--nats-url` is accepted and ignored) - `local-ai util` - Utility commands - `local-ai explorer` - Run P2P explorer - `local-ai federated` - Run LocalAI in federated mode diff --git a/tests/e2e/distributed/agent_distributed_test.go b/tests/e2e/distributed/agent_distributed_test.go index dfb90ea8d927..d044eb89afeb 100644 --- a/tests/e2e/distributed/agent_distributed_test.go +++ b/tests/e2e/distributed/agent_distributed_test.go @@ -2,9 +2,18 @@ package distributed_test import ( "context" - "sync/atomic" + "net" + "net/http" + "net/http/httptest" + "time" + + "github.com/gorilla/websocket" + "github.com/libp2p/go-yamux/v5" "github.com/mudler/LocalAI/core/services/agents" + "github.com/mudler/LocalAI/core/services/agentworker" + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/nodes" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -111,8 +120,8 @@ var _ = Describe("Phase 3: Agent Conversations & SSE", Label("Distributed"), fun // Two carriers, because the whole point of this bridge is that the // user's SSE connection and the replica running the agent are not // the same process. - watcher := agents.NewEventBridge(infra.Bus(), store, "instance-1") - runner := agents.NewEventBridge(infra.Bus(), store, "instance-2") + watcher := agents.NewEventBridge(infra.Bus(), store, "instance-1", nil) + runner := agents.NewEventBridge(infra.Bus(), store, "instance-2", nil) received := make(chan agents.AgentEvent, 16) sub, err := watcher.SubscribeEvents("my-agent", "user1", func(evt agents.AgentEvent) { @@ -141,35 +150,66 @@ var _ = Describe("Phase 3: Agent Conversations & SSE", Label("Distributed"), fun // Conversation persistence removed — chat history is browser-only. - // Two FRONTEND replicas, which share a carrier. A cancel bound for an - // agent WORKER does not travel this way: a worker has no database, so - // it cannot listen on the PostgreSQL carrier and the frontend publishes - // its cancels where the worker is listening instead. That pairing is - // pinned in core/services/agents; this pins the replica-to-replica half. - It("cancels a running agent from another replica", func() { - bridge := agents.NewEventBridge(infra.Bus(), store, "instance-1") - canceller := agents.NewEventBridge(infra.Bus(), store, "instance-2") - - // Start cancel listener - cancelSub, err := bridge.StartCancelListener() + // The whole cancel path, end to end and with nothing doubled: a real + // agent worker holding a real WebSocket + yamux tunnel, a real + // connection row deciding which replica owns it, the real selection + // over the real node rows, and the real control client on top. + // + // A cancel does not travel on a carrier any more. Its far end is the + // agent WORKER, which has no database and so cannot join the carrier + // the rest of the deployment fans out on, and it holds an outward + // tunnel instead. This is what that is. + It("cancels an agent run on a real worker over the tunnel it holds", func() { + const replica = "instance-1" + + registry, err := nodes.NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + clusterReg := cluster.NewRegistry(db) + Expect(clusterReg.Register(infra.Ctx, replica, "10.0.0.1:8080", "v1")).To(Succeed()) + tunnels := cluster.NewTunnelRegistry(clusterReg, replica) + + // The worker's own bridge, and the run registered on it. This is + // the state a dispatched agent execution leaves on a worker. + workerBridge := agents.NewWorkerEventBridge("agent-worker-e2e") + executor := agents.NewWorkerExecutor(workerBridge, nil, "http://127.0.0.1:1", "token") + cancelled := make(chan struct{}) + workerBridge.RegisterCancel("msg-e2e", func() { close(cancelled) }) + + node := &nodes.BackendNode{Name: "agent-e2e", NodeType: nodes.NodeTypeAgent, Address: "agent-e2e:50051"} + Expect(registry.Register(infra.Ctx, node, true)).To(Succeed()) + registered, err := registry.GetByName(infra.Ctx, "agent-e2e") Expect(err).ToNot(HaveOccurred()) - defer cancelSub.Unsubscribe() - - // Register a cancellable context - _, cancel := context.WithCancel(infra.Ctx) - var cancelled atomic.Bool - wrappedCancel := context.CancelFunc(func() { - cancelled.Store(true) - cancel() - }) - bridge.RegisterCancel("test-msg-id", wrappedCancel) - // Issued on the replica that does NOT hold the execution, which is - // the case the broadcast exists for. Its error says only that the - // request was published, so what is asserted is the effect. - Expect(canceller.CancelExecution("my-agent", "user1", "test-msg-id")).To(Succeed()) + frontend := newTunnelFrontend() + rt, err := agentworker.Start(infra.Ctx, agentworker.Options{ + FrontendURL: frontend.URL(), + NodeID: registered.ID, + TunnelToken: func() string { return "tunnel-secret" }, + ControlToken: "control-token", + Handlers: agentworker.Config{AgentCancel: executor.Cancel}, + }) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = rt.Close() }) + _, err = tunnels.Attach(infra.Ctx, registered.ID, frontend.Session()) + Expect(err).ToNot(HaveOccurred()) - Eventually(func() bool { return cancelled.Load() }, "20s").Should(BeTrue()) + control := nodes.NewControlClient(nodes.WorkerNetDialerFor(func(nodeID string) func(context.Context, string, string) (net.Conn, error) { + return cluster.NewWorkerDialer(tunnels, nil).DialerFor(nodeID, cluster.StreamTagHTTP) + }), "control-token") + agentControl := nodes.NewAgentControlClient( + nodes.NewAgentSelector(registry, clusterReg, replica, time.Hour), control) + + // Issued through the frontend's own bridge, which is what a cancel + // request landing on a replica reaches. + bridge := agents.NewEventBridge(infra.Bus(), store, replica, agentControl) + Expect(bridge.CancelExecution(infra.Ctx, "my-agent", "user1", "msg-e2e")).To(Succeed()) + Eventually(cancelled, "20s").Should(BeClosed()) + + // And the second of the three answers, from the same live fleet: + // a run no worker holds is NOT reported as cancelled. + err = bridge.CancelExecution(infra.Ctx, "my-agent", "user1", "msg-nobody-holds") + Expect(err).To(MatchError(nodes.ErrAgentRunNotOnAnyWorker)) + Expect(err).ToNot(MatchError(nodes.ErrAgentCancelUndelivered)) }) // Agent execution is now dispatched via AgentPoolService.dispatchChat(), @@ -206,3 +246,53 @@ var _ = Describe("Phase 3: Agent Conversations & SSE", Label("Distributed"), fun }) }) }) + +// tunnelFrontend is the far side of a worker's tunnel: the real WebSocket +// upgrade and the real yamux server handshake, with no LocalAI frontend behind +// it. It is what lets these specs put a REAL agent worker on a REAL tunnel +// without starting a whole server. +type tunnelFrontend struct { + srv *httptest.Server + sessions chan *yamux.Session +} + +func newTunnelFrontend() *tunnelFrontend { + GinkgoHelper() + f := &tunnelFrontend{sessions: make(chan *yamux.Session, 4)} + upgrader := websocket.Upgrader{} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != cluster.ConnectPath { + w.WriteHeader(http.StatusNotFound) + return + } + ws, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + sess, err := yamux.Server(cluster.WebsocketConn(ws), nil, nil) + if err != nil { + _ = ws.Close() + return + } + select { + case f.sessions <- sess: + default: + _ = sess.Close() + } + })) + DeferCleanup(f.srv.Close) + return f +} + +func (f *tunnelFrontend) URL() string { return f.srv.URL } + +// Session waits for the worker to dial in and hands back its tunnel session. +// Waited for on a channel rather than slept on: the dial is the worker's own +// and nothing in this process orders it against the next line of the spec. +func (f *tunnelFrontend) Session() *yamux.Session { + GinkgoHelper() + var sess *yamux.Session + Eventually(f.sessions, "20s").Should(Receive(&sess)) + DeferCleanup(func() { _ = sess.Close() }) + return sess +} diff --git a/tests/e2e/distributed/agent_native_executor_test.go b/tests/e2e/distributed/agent_native_executor_test.go index 1d147bcef0d1..1df3b32adecc 100644 --- a/tests/e2e/distributed/agent_native_executor_test.go +++ b/tests/e2e/distributed/agent_native_executor_test.go @@ -316,7 +316,7 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f Context("WorkerExecutor", func() { It("should dispatch chat via NATS and receive response", func() { - bridge := agents.NewEventBridge(infra.NC, nil, "test-instance") + bridge := agents.NewEventBridge(infra.NC, nil, "test-instance", nil) configs := &mockConfigProvider{configs: map[string]*agents.AgentConfig{ "test-agent": { @@ -378,28 +378,24 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f }) It("should handle cancellation via EventBridge", func() { - bridge := agents.NewEventBridge(infra.NC, nil, "cancel-test") + // The local half: a run held by THIS process is cancelled without + // anything leaving it. The cancel of a run held by an agent WORKER + // travels over that worker's tunnel and is driven end to end, on a + // real one, in agent_distributed_test.go. + bridge := agents.NewEventBridge(infra.NC, nil, "cancel-test", nil) var cancelled atomic.Bool bridge.RegisterCancel("test-msg-id", func() { cancelled.Store(true) }) - // Start cancel listener - cancelSub, err := bridge.StartCancelListener() - Expect(err).ToNot(HaveOccurred()) - defer cancelSub.Unsubscribe() - - FlushNATS(infra.NC) - - // Cancel the execution - Expect(bridge.CancelExecution("test-agent", "user1", "test-msg-id")).To(Succeed()) + Expect(bridge.CancelExecution(infra.Ctx, "test-agent", "user1", "test-msg-id")).To(Succeed()) Eventually(func() bool { return cancelled.Load() }, "5s").Should(BeTrue()) }) It("should execute agent chat from enriched payload without ConfigProvider", func() { - bridge := agents.NewEventBridge(infra.NC, nil, "enriched-test") + bridge := agents.NewEventBridge(infra.NC, nil, "enriched-test", nil) // Executor with NO ConfigProvider (simulating DB-free worker) executor := agents.NewWorkerExecutor(bridge, nil, "http://localhost:8080", "test-key") @@ -576,7 +572,7 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f Context("Full Distributed Chat Flow", func() { It("should dispatch chat via NATS, execute, and publish response via EventBridge", func() { - bridge := agents.NewEventBridge(infra.NC, nil, "flow-test") + bridge := agents.NewEventBridge(infra.NC, nil, "flow-test", nil) // Store agent config in PostgreSQL cfg := agents.AgentConfig{ @@ -676,7 +672,7 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f }) It("should dispatch background run via NATS with system role", func() { - bridge := agents.NewEventBridge(infra.NC, nil, "bg-test") + bridge := agents.NewEventBridge(infra.NC, nil, "bg-test", nil) cfg := agents.AgentConfig{ Name: "bg-agent", @@ -996,7 +992,7 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f llmURL, llmShutdown := startAgentMockLLMServer("I found the answer using my skills.") defer llmShutdown() - bridge := agents.NewEventBridge(infra.NC, nil, "full-e2e-test") + bridge := agents.NewEventBridge(infra.NC, nil, "full-e2e-test", nil) // Subscribe to agent events var receivedEvents []agents.AgentEvent @@ -1041,32 +1037,38 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f // it travels on the response body rather than only as a publish. Expect(string(reply)).To(ContainSubstring(`"status":"completed"`)) - // Wait for the full execution: processing + agent response + completed - Eventually(func() int { + // Waited for by CONTENT and not by count. A run with streaming on + // publishes an unbounded number of stream_event lines before its + // terminal status, so "at least three events have arrived" is + // reached long before the completed status is, and the assertions + // below then read a snapshot that cannot contain it yet. That is a + // race in the spec rather than in the executor, and it fails about + // one run in five. + seen := func() (agentMessage, completed bool) { eventMu.Lock() defer eventMu.Unlock() - return len(receivedEvents) - }, "15s").Should(BeNumerically(">=", 3)) - - eventMu.Lock() - defer eventMu.Unlock() - - var hasAgentMessage, hasCompleted bool - for _, evt := range receivedEvents { - if evt.EventType == "json_message" && evt.Sender == "agent" { - hasAgentMessage = true - Expect(evt.Content).To(ContainSubstring("found the answer")) - } - if evt.EventType == "json_message_status" && evt.Metadata != "" { - var meta map[string]string - json.Unmarshal([]byte(evt.Metadata), &meta) - if meta["status"] == "completed" { - hasCompleted = true + for _, evt := range receivedEvents { + if evt.EventType == "json_message" && evt.Sender == "agent" { + agentMessage = true + Expect(evt.Content).To(ContainSubstring("found the answer")) + } + if evt.EventType == "json_message_status" && evt.Metadata != "" { + var meta map[string]string + // A metadata blob this spec cannot read is not a + // completed status, and saying so beats failing the + // whole run on one malformed event. + if err := json.Unmarshal([]byte(evt.Metadata), &meta); err == nil && meta["status"] == "completed" { + completed = true + } } } + return agentMessage, completed } - Expect(hasAgentMessage).To(BeTrue(), "should receive agent response message via EventBridge") - Expect(hasCompleted).To(BeTrue(), "should receive completed status via EventBridge") + + Eventually(func() bool { _, completed := seen(); return completed }, "15s"). + Should(BeTrue(), "should receive completed status via EventBridge") + agentMessage, _ := seen() + Expect(agentMessage).To(BeTrue(), "should receive agent response message via EventBridge") }) It("should execute background agent run via NATS dispatcher with mock LLM", func() { @@ -1074,7 +1076,7 @@ var _ = Describe("Native Agent Executor", Label("Distributed", "AgentNative"), f llmURL, llmShutdown := startAgentMockLLMServer("All systems operational. No issues detected.") defer llmShutdown() - bridge := agents.NewEventBridge(infra.NC, nil, "bg-e2e-test") + bridge := agents.NewEventBridge(infra.NC, nil, "bg-e2e-test", nil) // Subscribe to agent events var receivedEvents []agents.AgentEvent diff --git a/tests/e2e/distributed/foundation_test.go b/tests/e2e/distributed/foundation_test.go index 62c2a4b8348b..c96bd71e54e0 100644 --- a/tests/e2e/distributed/foundation_test.go +++ b/tests/e2e/distributed/foundation_test.go @@ -7,8 +7,6 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/advisorylock" - "github.com/mudler/LocalAI/core/services/agents" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/storage" . "github.com/onsi/ginkgo/v2" @@ -83,16 +81,6 @@ var _ = Describe("Phase 0: Foundation", Label("Distributed"), func() { }) }) - // The one carrier a deployment still dials besides PostgreSQL, and the one - // family left on it. - // - // agent..cancel did not move to the broadcast carrier: its only - // subscriber is the agent WORKER, which has no database and cannot join - // that carrier at all. So this round trip is no longer "the messaging layer - // works" - it is the cancel path for every agent a worker runs, and if it - // stops working the symptom is a cancel that is published, succeeds, and - // reaches nobody. - // // Two Its that used to sit here went with the halves of the client they // exercised. "should support queue subscriptions for load balancing" pinned // that work reaches exactly one of N consumers; a queue group no longer @@ -103,38 +91,12 @@ var _ = Describe("Phase 0: Foundation", Label("Distributed"), func() { // asserted nothing of the kind; the property is now // core/services/pgbus/listener_test.go, which actually kills the session // with pg_terminate_backend and waits for delivery to resume. - Context("the cancel carrier", func() { - It("connects, publishes and subscribes, which is the agent cancel path", func() { - client, err := messaging.New(infra.NatsURL) - Expect(err).ToNot(HaveOccurred()) - defer client.Close() - - Expect(client.IsConnected()).To(BeTrue()) - - // The real subject and the real filter, not a placeholder pair. - // A worker subscribes to the wildcard and a frontend publishes to - // one agent's subject, so a round trip on "test.subject" would - // stay green through a filter that no longer matches what the - // builder mints - which is the failure this family actually has. - received := make(chan []byte, 1) - sub, err := client.Subscribe(messaging.SubjectAgentCancelWildcard, func(data []byte) { - received <- data - }) - Expect(err).ToNot(HaveOccurred()) - defer sub.Unsubscribe() - - // Small delay to ensure subscription is active - FlushNATS(client) - - err = client.Publish(messaging.SubjectAgentCancel("a1"), agents.AgentCancelEvent{ - AgentName: "a1", UserID: "u1", MessageID: "msg-1", - }) - Expect(err).ToNot(HaveOccurred()) - - Eventually(received, "5s").Should(Receive()) - }) - - }) + // + // The third went with the family it carried. A cancel used to be published + // on NATS because its subscriber was an agent worker that could not read + // the broadcast carrier; it is a control verb on that worker's own tunnel + // now, and the deployment dials no message bus at all. The path is driven + // end to end, over a real tunnel, in agent_distributed_test.go. Context("ObjectStore filesystem adapter", func() { var store *storage.FilesystemStore diff --git a/tests/e2e/distributed/sse_routes_test.go b/tests/e2e/distributed/sse_routes_test.go index 5f93b9673716..2e7b64a9a911 100644 --- a/tests/e2e/distributed/sse_routes_test.go +++ b/tests/e2e/distributed/sse_routes_test.go @@ -86,8 +86,8 @@ var _ = Describe("SSE Routes", Label("Distributed"), func() { agentStore, err := agents.NewAgentStore(db) Expect(err).ToNot(HaveOccurred()) - frontend0 := agents.NewEventBridge(infra.Bus(), agentStore, "frontend-0") - frontend1 := agents.NewEventBridge(infra.Bus(), agentStore, "frontend-1") + frontend0 := agents.NewEventBridge(infra.Bus(), agentStore, "frontend-0", nil) + frontend1 := agents.NewEventBridge(infra.Bus(), agentStore, "frontend-1", nil) received := make(chan agents.AgentEvent, 16) sub, err := frontend1.SubscribeEvents("test-agent", "user1", func(evt agents.AgentEvent) { From 60221f07325a076b85ce79770e76596a3eab06f3 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 4 Sep 2026 06:33:14 +0000 Subject: [PATCH 098/153] fix(distributed): read a departed agent tunnel as the routing fact it is Task 4 gave agent workers tunnels and deliberately left the NodeType skip in HealthMonitor.tunnelDeparted, with a spec asserting that an agent node whose presence reader answers PresenceGone is NOT marked unhealthy. That spec was scaffolding. It was true while an agent worker took its jobs and its verbs over the message bus: a departure row for one said nothing about whether it could work, and an early bug in the new tunnel client could otherwise have demoted a fleet of healthy agent workers. There is no bus. An agent worker is reachable through its tunnel and through nothing else, so a departed agent tunnel means exactly what a departed backend tunnel means: no live replica holds it, the departure has outlived the reconnect grace, and that is a routing fact the scheduler and a reaper may act on. The skip would now hide the only symptom an unreachable agent worker has. This is the deliberate removal Task 4's M6 predicted, and task-4-report.md is where that mutation already stands recorded red against the spec this commit deletes. The skip existed at ONE site. router_liveness.go has none: its candidates come from queries that already filter node_type = 'backend'. The two skips in managers_distributed.go stay, because an agent worker still runs no backend processes, so it has no backend to list and no backend op to apply. Two node types can depart now, which is why the second half exists. Before this, one type could depart and every per-node cache a departure left stale was dropped from wherever its owner happened to notice, so a reader could not tell which caches a demotion invalidated by reading the demotion path. Departure gets ONE notification point. DepartureNotifier is edge triggered, because the monitor runs on a ticker and a departed node stays departed; its subscribers are NAMED, because what has to be caught is a forgotten cache and a count can say only that one of four is missing; and NewHealthMonitor takes it as a required positional argument, so a caller that does not pass one fails to compile. Four caches subscribe: prefix-cache affinity in every model, probe freshness at every address, in-flight staging operations, and the per-node breakdown of every open gallery operation. The prefix-cache one is registered only when prefix-cache routing is enabled, so --distributed-prefix-cache=false stays a true no-op. The notification carries the node's name as well as its id, because the staging tracker keys on the name and the other two key on the id, and a subscriber should not have to read the registry from inside an eviction hook. A departure notification is an act on absence, so it fires only on the routing fact. A tunnel lost inside the grace, a worker that never dialled, a presence query that failed and a stale heartbeat all announce nothing, asserted per node type. The stale-heartbeat branch is excluded on purpose: it already marks the node offline, which deletes its rows and runs the registry's replica-removed hooks, so firing there too would double-evict and make the notification mean two different things at its subscribers. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/absence_wiring_test.go | 6 +- core/application/distributed.go | 99 ++++++- core/application/distributed_test.go | 88 +++++++ core/application/startup.go | 6 +- core/services/galleryop/service.go | 38 +++ core/services/nodes/departure.go | 153 +++++++++++ core/services/nodes/departure_test.go | 243 ++++++++++++++++++ core/services/nodes/health.go | 71 +++-- core/services/nodes/health_test.go | 178 +++++++++++-- core/services/nodes/prefixcache/index.go | 16 ++ core/services/nodes/prefixcache/provider.go | 7 +- core/services/nodes/prefixcache/sync.go | 14 + core/services/nodes/probe_cache.go | 21 ++ core/services/nodes/router.go | 14 + core/services/nodes/router_test.go | 7 + core/services/nodes/staging_progress.go | 23 ++ docs/content/features/distributed-mode.md | 6 +- tests/e2e/distributed/cluster/cluster.go | 30 +-- tests/e2e/distributed/cluster_control_test.go | 54 ++-- tests/e2e/distributed/cluster_tunnel_test.go | 23 +- 20 files changed, 1012 insertions(+), 85 deletions(-) create mode 100644 core/services/nodes/departure.go create mode 100644 core/services/nodes/departure_test.go diff --git a/core/application/absence_wiring_test.go b/core/application/absence_wiring_test.go index 5ca33122c2f6..f7754d053cc8 100644 --- a/core/application/absence_wiring_test.go +++ b/core/application/absence_wiring_test.go @@ -77,7 +77,7 @@ var _ = Describe("stamping the absence wiring onto the scheduler's options", fun var _ = Describe("the absence wiring a distributed deployment refuses to start without", func() { present := func() (*nodes.SmartRouter, *nodes.HealthMonitor) { router := nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{Presence: presenceStub{}}) - health := nodes.NewHealthMonitor(nil, nil, time.Second, time.Minute, "", false, presenceStub{}, time.Minute) + health := nodes.NewHealthMonitor(nil, nil, time.Second, time.Minute, "", false, presenceStub{}, time.Minute, nil) return router, health } @@ -98,7 +98,7 @@ var _ = Describe("the absence wiring a distributed deployment refuses to start w It("refuses a health monitor built without one, and says what it would do", func() { router, _ := present() - blind := nodes.NewHealthMonitor(nil, nil, time.Second, time.Minute, "", false, nil, 0) + blind := nodes.NewHealthMonitor(nil, nil, time.Second, time.Minute, "", false, nil, 0, nil) err := requireAbsenceWiring(router, blind) Expect(err).To(HaveOccurred()) @@ -112,7 +112,7 @@ var _ = Describe("the absence wiring a distributed deployment refuses to start w // the monitor's leaves it listed healthy while its models are unreachable. It("names the scheduler and the health monitor as separate requirements", func() { blindRouter := nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}) - blindHealth := nodes.NewHealthMonitor(nil, nil, time.Second, time.Minute, "", false, nil, 0) + blindHealth := nodes.NewHealthMonitor(nil, nil, time.Second, time.Minute, "", false, nil, 0, nil) router, health := present() Expect(requireAbsenceWiring(blindRouter, health)).ToNot(Succeed()) diff --git a/core/application/distributed.go b/core/application/distributed.go index 12271e3295d5..09aa565299f5 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -154,7 +154,13 @@ func (ds *DistributedServices) Shutdown() { // Returns nil if distributed mode is not enabled. // configLoader is used by the SmartRouter to compute concurrency-group // anti-affinity at placement time (#9659); it may be nil in tests. -func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoader *config.ModelConfigLoader) (*DistributedServices, error) { +// +// galleryProgress is the gallery service, narrowed to the one method a node +// departure needs. It is a PARAMETER and not a later setter because the +// registration of every per-node cache a departure evicts happens here, in one +// place, and a cache registered somewhere else is a cache a reader cannot find +// by reading this function. +func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoader *config.ModelConfigLoader, galleryProgress nodeProgressDropper) (*DistributedServices, error) { if !cfg.Distributed.Enabled { return nil, nil } @@ -391,6 +397,12 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // says the worker's supervisor is alive, presence says whether anything // here can still reach its backends, and a worker can be the first without // being the second indefinitely. + // + // The departure notifier is built HERE, before its only caller, and its + // subscribers are registered further down once the caches they drop exist. + // One object, one caller, so "what does a departure evict" is answered by + // reading registerDepartureEvictions and nothing else. + departures := nodes.NewDepartureNotifier() healthMon := nodes.NewHealthMonitor(registry, authDB, cfg.Distributed.HealthCheckIntervalOrDefault(), cfg.Distributed.StaleNodeThresholdOrDefault(), @@ -398,6 +410,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade !cfg.Distributed.DisablePerModelHealthCheck, clusterRegistry, cfg.Distributed.ReconnectGraceOrDefault(), + departures, backendClients, ) @@ -508,6 +521,11 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade var prefixProvider prefixcache.Provider var pressure *prefixcache.Pressure var prefixCfg prefixcache.Config + // The CONCRETE Sync, declared out here so the departure wiring below can be + // handed it. Concrete and not prefixProvider, because a nil interface value + // carrying a nil *Sync is not nil, and the disabled deployment would then + // register an eviction that dereferences it. + var prefixDrop *prefixcache.Sync if !cfg.Distributed.PrefixCacheDisabled { prefixCfg = prefixcache.DefaultConfig() if cfg.Distributed.PrefixCacheTTL > 0 { @@ -526,6 +544,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } pressure = prefixcache.NewSyncedPressure(prefixCfg.PressureWindow, bus) prefixProvider = prefixSync + prefixDrop = prefixSync // Invalidate the prefix-cache index whenever a replica row is removed. // AddReplicaRemovedHook fires from the single chokepoint all removal paths @@ -658,6 +677,12 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade xlog.Warn("Failed to subscribe to staging progress broadcasts", "error", err) } + // Every per-node cache a departure leaves stale, onto the one notification + // point, after the router that owns two of them exists. + if err := registerDepartureEvictions(departures, prefixDrop, router, galleryProgress); err != nil { + return nil, err + } + // Create ReplicaReconciler for auto-scaling model replicas. Adapter + // RegistrationToken feed the state-reconciliation passes: pending op // drain uses the adapter, and model health probes use the token to auth @@ -722,6 +747,78 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade return ds, nil } +// nodeProgressDropper is the gallery service narrowed to the one method a node +// departure needs. An interface so the wiring below cannot reach for anything +// else on the gallery service from inside an eviction hook. +type nodeProgressDropper interface { + DropNodeProgress(nodeID string) +} + +// The names each per-node cache is registered under. They are constants because +// they are asserted: a wiring spec that spelled them itself would keep passing +// after a subscriber was renamed and lost. +const ( + departurePrefixCache = "prefix-cache" + departureProbeCache = "probe-cache" + departureStagingTracker = "staging-tracker" + departureGalleryNodes = "gallery-node-progress" +) + +// registerDepartureEvictions registers every per-node cache that a node's +// departure leaves stale on the deployment's one departure notification point. +// +// One function, and every subscriber in it, because the reason the notifier +// exists is that a reader cannot otherwise enumerate what a demotion +// invalidates: before it, one node type could depart and each stale cache was +// dropped from wherever its owner happened to notice. Adding a per-node cache +// without adding a line here is the failure this shape exists to make visible, +// which is why the subscribers are NAMED and the names are asserted. +// +// It REFUSES rather than skipping when the router or the gallery service is +// missing. A deployment whose departed nodes keep their probe entries, staging +// rows and per-node operation progress does not fail, log or slow down: it +// answers with state for a node that left, indefinitely. +// +// prefix may be nil, and only prefix. That is --distributed-prefix-cache=false, +// where there is no index to drop from, and it stays a true no-op: nothing is +// registered rather than a hook registered onto nothing. It is the CONCRETE +// *prefixcache.Sync for that decision to be safe, since a nil provider inside +// an interface would compare non-nil here and dereference on the first +// departure. +func registerDepartureEvictions(departures *nodes.DepartureNotifier, prefix *prefixcache.Sync, router *nodes.SmartRouter, gallery nodeProgressDropper) error { + if departures == nil { + return fmt.Errorf("wiring departure evictions: no departure notifier, so a departed node would keep every per-node cache entry it has for the life of the process") + } + if router == nil { + return fmt.Errorf("wiring departure evictions: no router, so a departed node would keep its probe-freshness entries and its staging operations") + } + if gallery == nil { + return fmt.Errorf("wiring departure evictions: no gallery service, so a departed node would stay in every open operation's per-node breakdown") + } + // S1. Inside a nil check and not inside the prefix-cache-enabled block, so + // that "the disabled deployment registers nothing" is a fact a spec can + // hold rather than a property of where a line was written. + if prefix != nil { + departures.OnDeparture(departurePrefixCache, func(node nodes.DepartedNode) { + prefix.DropNode(node.ID) + }) + } + // S2 and S3 are two registrations and not one, because they are two rules: + // a probe entry is keyed by node ID and a staging op by node NAME, and a + // single hook doing both would hide which of them was lost. + departures.OnDeparture(departureProbeCache, func(node nodes.DepartedNode) { + router.InvalidateNodeProbes(node.ID) + }) + departures.OnDeparture(departureStagingTracker, func(node nodes.DepartedNode) { + router.StagingTracker().DropNode(node.Name) + }) + // S4. + departures.OnDeparture(departureGalleryNodes, func(node nodes.DepartedNode) { + gallery.DropNodeProgress(node.ID) + }) + return nil +} + // requireBroadcastCarrier refuses to hand back a distributed deployment whose // broadcast carrier is missing. // diff --git a/core/application/distributed_test.go b/core/application/distributed_test.go index 95449c4b0e07..731b86cf29bb 100644 --- a/core/application/distributed_test.go +++ b/core/application/distributed_test.go @@ -9,7 +9,9 @@ import ( . "github.com/onsi/gomega" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/pgbus" "github.com/mudler/LocalAI/core/services/syncstate" "github.com/mudler/LocalAI/core/services/testutil" @@ -173,3 +175,89 @@ var _ = Describe("handing the broadcast carrier to its adopters", func() { Expect(func() { Expect(m.Set(context.Background(), "v")).To(Succeed()) }).ToNot(Panic()) }) }) + +// The registration of every per-node cache a node's departure evicts. +// +// Four subscribers, registered in one function, on a notifier the health +// monitor is then handed. Every one of those is a line that compiles, starts +// and serves when it is missing: a deployment whose departed nodes keep their +// probe entries, staging rows, prefix affinity and per-node operation progress +// does not fail, log or slow down, it just answers with state for a node that +// left, for the life of the process. +// +// Asserted by NAME and not by count. A count says a cache was forgotten; only +// the names say which, and "which" is the entire content of the failure. +var _ = Describe("wiring the per-node caches a departure evicts", func() { + // bootDistributed brings a real distributed deployment up against a fresh + // database, which is what makes these assertions about production wiring + // rather than about a notifier a spec assembled itself. + bootDistributed := func(arm ...func(*config.ApplicationConfig)) *DistributedServices { + GinkgoHelper() + db, dsn := testutil.SetupTestDBWithDSN() + ctx, cancel := context.WithCancel(context.Background()) + DeferCleanup(cancel) + cfg := &config.ApplicationConfig{DataPath: GinkgoT().TempDir(), Context: ctx} + cfg.Auth.Enabled = true + cfg.Auth.DatabaseURL = dsn + cfg.Distributed.Enabled = true + for _, a := range arm { + a(cfg) + } + ds, err := initDistributed(cfg, db, nil, galleryop.NewGalleryService(cfg, nil)) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(ds.Shutdown) + return ds + } + + It("registers every one of them, on the notifier the health monitor fires", func() { + // Reached through the health monitor and not through a local variable, + // because registering the four on a DIFFERENT notifier than the one the + // monitor was built with evicts nothing while every count still reads + // four. + ds := bootDistributed() + + Expect(ds.Health.Departures().SubscriberNames()).To(ConsistOf( + departurePrefixCache, + departureProbeCache, + departureStagingTracker, + departureGalleryNodes, + )) + }) + + It("registers no prefix-cache eviction when prefix-cache routing is disabled", func() { + // --distributed-prefix-cache=false stays a TRUE no-op: there is no + // index to drop from, so nothing is registered rather than a hook + // registered onto nothing. The other three are unaffected, which is the + // half that makes this a statement about S1 and not about the feature + // flag switching the whole mechanism off. + ds := bootDistributed(func(cfg *config.ApplicationConfig) { + cfg.Distributed.PrefixCacheDisabled = true + }) + + Expect(ds.Health.Departures().SubscriberNames()).To(ConsistOf( + departureProbeCache, + departureStagingTracker, + departureGalleryNodes, + )) + }) + + It("refuses a deployment with no router, naming what its departed nodes would keep", func() { + err := registerDepartureEvictions(nodes.NewDepartureNotifier(), nil, nil, galleryop.NewGalleryService(&config.ApplicationConfig{}, nil)) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("probe-freshness")) + }) + + It("refuses a deployment with no gallery service", func() { + err := registerDepartureEvictions(nodes.NewDepartureNotifier(), nil, nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}), nil) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("per-node breakdown")) + }) + + It("refuses a deployment with no notifier at all", func() { + err := registerDepartureEvictions(nil, nil, nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}), galleryop.NewGalleryService(&config.ApplicationConfig{}, nil)) + + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/core/application/startup.go b/core/application/startup.go index e2d99517147c..b9d371336bec 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -271,7 +271,11 @@ func New(opts ...config.AppOption) (*Application, error) { // the model configs are loaded, so it is declared out here. var revisionStore modeladmin.RevisionStore - distSvc, err := initDistributed(options, application.authDB, application.ModelConfigLoader()) + // The gallery service is handed in rather than set afterwards: it owns one + // of the per-node caches a node departure evicts, and every one of those is + // registered inside initDistributed. It exists by now because start() built + // it above. + distSvc, err := initDistributed(options, application.authDB, application.ModelConfigLoader(), application.galleryService) if err != nil { return nil, fmt.Errorf("distributed mode initialization failed: %w", err) } diff --git a/core/services/galleryop/service.go b/core/services/galleryop/service.go index 6d742d6767f5..9614abf50055 100644 --- a/core/services/galleryop/service.go +++ b/core/services/galleryop/service.go @@ -355,6 +355,44 @@ func (g *GalleryService) UpdateNodeProgress(opID, nodeID string, np NodeProgress } } +// DropNodeProgress removes nodeID's per-node rows from every operation that is +// still open. +// +// A per-node row is a claim that a node is doing something. When the node +// departs, nothing will ever move that row off "downloading": the operation +// stays open in /api/operations with a bar that never advances, and the node it +// names is not in the fleet any more. +// +// Only OPEN operations. A processed operation's breakdown is the record of what +// each node did, and rewriting it because a node later left would be reporting +// history rather than state. The aggregate mirror fields are left alone for the +// same reason: they are the last tick that happened, not a claim about now. +func (g *GalleryService) DropNodeProgress(nodeID string) { + if g == nil || nodeID == "" { + return + } + g.Lock() + defer g.Unlock() + for _, status := range g.statuses { + if status == nil || status.Processed { + continue + } + kept := make([]NodeProgress, 0, len(status.Nodes)) + for _, np := range status.Nodes { + if np.NodeID != nodeID { + kept = append(kept, np) + } + } + if len(kept) == len(status.Nodes) { + continue + } + // Replaced rather than written through, which is the rule GetStatus + // documents: a reader holding the old slice header sees a consistent + // older breakdown instead of a torn one. + status.Nodes = kept + } +} + // GetStatus returns a COPY of the operation's status, not the stored pointer. // // The copy is what makes the lock mean anything. Every caller of this and of diff --git a/core/services/nodes/departure.go b/core/services/nodes/departure.go new file mode 100644 index 000000000000..f826a61c20d8 --- /dev/null +++ b/core/services/nodes/departure.go @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: MIT + +package nodes + +import ( + "sync" + + "github.com/mudler/xlog" +) + +// DepartedNode identifies the node a departure is about. +// +// It carries the NAME as well as the ID because the per-node caches this +// notification exists to drop are not keyed alike: the prefix index and the +// probe cache key on the node's ID, and the staging tracker keys on the node +// NAME a transfer was attributed to. A payload carrying only one of them would +// leave the other subscriber to look the node up in the database from inside an +// eviction hook, which is a query on a path whose whole point is that it is +// cheap and edge triggered. +type DepartedNode struct { + ID string + Name string + Type string +} + +// departureSubscriber is one registered eviction, with the name it is reported +// under. The name is not decoration: it is what a wiring spec asserts on and +// what names the offender when one of them panics, and a bare count could say +// only that some cache was forgotten, never which. +type departureSubscriber struct { + name string + fn func(DepartedNode) +} + +// DepartureNotifier is the ONE place a node's departure evicts per-node state. +// +// A departure is a routing fact: no live replica holds this node's tunnel and +// the departure has outlived the reconnect grace. It is NOT an absent +// connection (a tunnel lost inside the grace), NOT an unreachable peer and NOT +// the node's own answer, and this type is reachable from exactly one caller for +// that reason. A cache that wants to be dropped when a node goes registers +// here; it does not get to decide for itself what "gone" means, because +// deciding that in four places is how the four conditions collapse into two. +// +// EDGE TRIGGERED. The health monitor runs on a ticker and a departed node stays +// departed, so a level-triggered notifier would re-evict every cache every tick +// for as long as the node is away. Subscribers are invoked on the transition +// into departure and again only after the node has been seen present. +// +// Safe for concurrent use, and safe on a nil receiver: a monitor built with no +// notifier evicts nothing rather than panicking on the first departed node. +type DepartureNotifier struct { + mu sync.Mutex + subs []departureSubscriber + // departed is the edge. A node in it has already had its departure + // announced; Present is what takes it back out. It is bounded by the number + // of distinct nodes that have departed and not returned, which is the same + // order as the fleet. + departed map[string]struct{} +} + +// NewDepartureNotifier returns a notifier with no subscribers. +func NewDepartureNotifier() *DepartureNotifier { + return &DepartureNotifier{departed: map[string]struct{}{}} +} + +// OnDeparture registers a subscriber under name. Every registered subscriber is +// invoked; registering one never displaces another, which is the bug +// NodeRegistry.AddReplicaRemovedHook's own comment records from its single-slot +// ancestor. +// +// Register at wiring time, before the health monitor starts. +func (n *DepartureNotifier) OnDeparture(name string, fn func(DepartedNode)) { + if n == nil || fn == nil { + return + } + n.mu.Lock() + defer n.mu.Unlock() + n.subs = append(n.subs, departureSubscriber{name: name, fn: fn}) +} + +// SubscriberNames returns the names of the registered subscribers, in +// registration order. +// +// Names and not a count, because what the wiring spec has to catch is a +// FORGOTTEN cache and a count can only say that one of four is missing. This is +// the whole symptom: a deployment whose departed nodes keep their per-node +// state does not fail, log, or slow down, it just answers with entries for a +// node that left. +func (n *DepartureNotifier) SubscriberNames() []string { + if n == nil { + return nil + } + n.mu.Lock() + defer n.mu.Unlock() + names := make([]string, 0, len(n.subs)) + for _, s := range n.subs { + names = append(names, s.name) + } + return names +} + +// Departed records that node has departed and invokes every subscriber, once +// per departure. A second call for a node that has not been seen present since +// invokes nothing. +// +// Each subscriber is invoked with the lock released and its panic contained: an +// eviction is best effort, and a cache that panics must not strand the caches +// registered after it. +func (n *DepartureNotifier) Departed(node DepartedNode) { + if n == nil || node.ID == "" { + return + } + n.mu.Lock() + if _, already := n.departed[node.ID]; already { + n.mu.Unlock() + return + } + n.departed[node.ID] = struct{}{} + subs := make([]departureSubscriber, len(n.subs)) + copy(subs, n.subs) + n.mu.Unlock() + + for _, s := range subs { + invokeDepartureSubscriber(s, node) + } +} + +// invokeDepartureSubscriber runs one subscriber and contains its panic, naming +// the subscriber that failed. Extracted so the deferred recover is scoped to a +// single subscriber rather than to the whole loop, which is what makes the +// remaining subscribers run. +func invokeDepartureSubscriber(s departureSubscriber, node DepartedNode) { + defer func() { + if r := recover(); r != nil { + xlog.Error("A departure subscriber panicked; the node's state it owns was not dropped", + "subscriber", s.name, "node", node.Name, "nodeID", node.ID, "panic", r) + } + }() + s.fn(node) +} + +// Present records that nodeID has been seen present, arming the next departure. +// Called on every tick for a node that is not departed, so it is cheap and +// invokes nothing. +func (n *DepartureNotifier) Present(nodeID string) { + if n == nil || nodeID == "" { + return + } + n.mu.Lock() + defer n.mu.Unlock() + delete(n.departed, nodeID) +} diff --git a/core/services/nodes/departure_test.go b/core/services/nodes/departure_test.go new file mode 100644 index 000000000000..210483d737b2 --- /dev/null +++ b/core/services/nodes/departure_test.go @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: MIT + +package nodes + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/galleryop" + "github.com/mudler/LocalAI/core/services/nodes/prefixcache" +) + +var _ = Describe("DepartureNotifier", func() { + It("invokes every registered subscriber, because registering one must never displace another", func() { + // The single-slot ancestor of AddReplicaRemovedHook is the bug this + // asserts against: a second registration silently replaced the first, + // and the cache registered earlier was simply never dropped again. + n := NewDepartureNotifier() + var first, second []DepartedNode + n.OnDeparture("first", func(node DepartedNode) { first = append(first, node) }) + n.OnDeparture("second", func(node DepartedNode) { second = append(second, node) }) + + n.Departed(DepartedNode{ID: "n1", Name: "worker-a", Type: NodeTypeBackend}) + + Expect(first).To(HaveLen(1)) + Expect(second).To(HaveLen(1)) + }) + + It("fires once per departure however many times a departed node is reported departed", func() { + // The health monitor runs on a ticker and a departed node stays + // departed. Level triggered, every cache in the deployment would be + // re-evicted every tick for as long as the node is away. + n := NewDepartureNotifier() + var fired int + n.OnDeparture("counter", func(DepartedNode) { fired++ }) + + n.Departed(DepartedNode{ID: "n1", Name: "worker-a", Type: NodeTypeBackend}) + n.Departed(DepartedNode{ID: "n1", Name: "worker-a", Type: NodeTypeBackend}) + n.Departed(DepartedNode{ID: "n1", Name: "worker-a", Type: NodeTypeBackend}) + + Expect(fired).To(Equal(1)) + }) + + It("fires again once the node has been seen present, so the edge re-arms", func() { + // The negative control for the spec above: a notifier that fired once + // and never again would satisfy it, and a worker that leaves, comes + // back and leaves again is the ordinary case. + n := NewDepartureNotifier() + var fired int + n.OnDeparture("counter", func(DepartedNode) { fired++ }) + + n.Departed(DepartedNode{ID: "n1", Name: "worker-a", Type: NodeTypeBackend}) + n.Present("n1") + n.Departed(DepartedNode{ID: "n1", Name: "worker-a", Type: NodeTypeBackend}) + + Expect(fired).To(Equal(2)) + }) + + It("keeps one node's departure edge separate from another's", func() { + n := NewDepartureNotifier() + var seen []string + n.OnDeparture("collector", func(node DepartedNode) { seen = append(seen, node.ID) }) + + n.Departed(DepartedNode{ID: "n1", Name: "worker-a", Type: NodeTypeBackend}) + n.Departed(DepartedNode{ID: "n2", Name: "worker-b", Type: NodeTypeAgent}) + + Expect(seen).To(Equal([]string{"n1", "n2"})) + }) + + It("runs the remaining subscribers when one of them panics", func() { + // An eviction is best effort. The panicking one is registered FIRST so + // the assertion is about containment and not about ordering luck. + n := NewDepartureNotifier() + var ran []string + n.OnDeparture("panicker", func(DepartedNode) { panic("cache exploded") }) + n.OnDeparture("second", func(DepartedNode) { ran = append(ran, "second") }) + n.OnDeparture("third", func(DepartedNode) { ran = append(ran, "third") }) + + Expect(func() { + n.Departed(DepartedNode{ID: "n1", Name: "worker-a", Type: NodeTypeBackend}) + }).ToNot(Panic()) + Expect(ran).To(Equal([]string{"second", "third"})) + }) + + It("passes the departed node's identity through for a BACKEND node", func() { + n := NewDepartureNotifier() + var got DepartedNode + n.OnDeparture("sink", func(node DepartedNode) { got = node }) + + n.Departed(DepartedNode{ID: "n1", Name: "gpu-worker-1", Type: NodeTypeBackend}) + + Expect(got).To(Equal(DepartedNode{ID: "n1", Name: "gpu-worker-1", Type: NodeTypeBackend})) + }) + + It("passes the departed node's identity through for an AGENT node", func() { + // Two node types can depart now, and a subscriber that could not tell + // them apart could not log which fleet it just evicted for. + n := NewDepartureNotifier() + var got DepartedNode + n.OnDeparture("sink", func(node DepartedNode) { got = node }) + + n.Departed(DepartedNode{ID: "a1", Name: "agent-worker-0", Type: NodeTypeAgent}) + + Expect(got).To(Equal(DepartedNode{ID: "a1", Name: "agent-worker-0", Type: NodeTypeAgent})) + }) + + It("reports its subscribers by name, in registration order", func() { + n := NewDepartureNotifier() + n.OnDeparture("one", func(DepartedNode) {}) + n.OnDeparture("two", func(DepartedNode) {}) + + Expect(n.SubscriberNames()).To(Equal([]string{"one", "two"})) + }) + + It("ignores a departure with no node ID rather than arming an empty edge", func() { + n := NewDepartureNotifier() + var fired int + n.OnDeparture("counter", func(DepartedNode) { fired++ }) + + n.Departed(DepartedNode{Name: "nameless"}) + + Expect(fired).To(Equal(0)) + }) + + It("evicts nothing rather than panicking when there is no notifier at all", func() { + // A single-node deployment builds the health monitor with no notifier, + // and the monitor calls these on every tick. + var n *DepartureNotifier + + Expect(func() { + n.Departed(DepartedNode{ID: "n1"}) + n.Present("n1") + n.OnDeparture("x", func(DepartedNode) {}) + }).ToNot(Panic()) + Expect(n.SubscriberNames()).To(BeEmpty()) + }) +}) + +// One spec per row of the eviction table. Each drives the PRODUCTION sink the +// wiring in core/application registers, through a real departure, because what +// has to hold is that the departure reaches that cache and not that a closure +// was called. +var _ = Describe("what a node's departure drops", func() { + It("drops the departed node's prefix-cache entries in EVERY model", func() { + // Every model, and that is why this is not the registry's per-model + // InvalidateNode: a departure names no model, and the rows that would + // have enumerated them are exactly what the departure puts in doubt. + cfg := prefixcache.DefaultConfig() + idx := prefixcache.NewIndex(cfg) + sync := prefixcache.NewSync(idx, nil) + now := time.Now() + chain := []uint64{1, 2, 3} + gone := prefixcache.ReplicaKey{NodeID: "n1", Replica: 0} + stays := prefixcache.ReplicaKey{NodeID: "n2", Replica: 0} + idx.Observe("model-a", chain, gone, now) + idx.Observe("model-b", chain, gone, now) + idx.Observe("model-a", chain, stays, now) + + n := NewDepartureNotifier() + n.OnDeparture("prefix-cache", func(node DepartedNode) { sync.DropNode(node.ID) }) + n.Departed(DepartedNode{ID: "n1", Name: "worker-a", Type: NodeTypeBackend}) + + Expect(idx.Decide("model-a", chain, []prefixcache.ReplicaKey{gone}, now).HasHot).To(BeFalse()) + Expect(idx.Decide("model-b", chain, []prefixcache.ReplicaKey{gone}, now).HasHot).To(BeFalse()) + // The negative control: a drop that emptied the index would satisfy + // the two assertions above and destroy every other node's affinity. + Expect(idx.Decide("model-a", chain, []prefixcache.ReplicaKey{stays}, now).HasHot).To(BeTrue()) + }) + + It("drops every probe-freshness entry the departed node had, at every address", func() { + // Keys are "|". A departed worker that returns with + // recycled ports would otherwise serve one request per still-fresh + // address with no probe at all. + router := NewSmartRouter(nil, SmartRouterOptions{}) + router.probeCache.markFresh("n1|127.0.0.1:50100") + router.probeCache.markFresh("n1|127.0.0.1:50101") + router.probeCache.markFresh("n2|127.0.0.1:50100") + + n := NewDepartureNotifier() + n.OnDeparture("probe-cache", func(node DepartedNode) { router.InvalidateNodeProbes(node.ID) }) + n.Departed(DepartedNode{ID: "n1", Name: "worker-a", Type: NodeTypeBackend}) + + Expect(router.probeCache.IsFresh("n1|127.0.0.1:50100")).To(BeFalse()) + Expect(router.probeCache.IsFresh("n1|127.0.0.1:50101")).To(BeFalse()) + // A prefix scan and not a substring one, and not a wipe: another + // node's entry at the same address survives. + Expect(router.probeCache.IsFresh("n2|127.0.0.1:50100")).To(BeTrue()) + }) + + It("ends every staging operation attributed to the departed node", func() { + // A staging op has no terminal state of its own when its destination + // leaves: the copy loop that would call Complete is blocked on a worker + // nothing can reach, so /api/operations shows it copying forever. + router := NewSmartRouter(nil, SmartRouterOptions{}) + tracker := router.StagingTracker() + tracker.Start("model-gone", "worker-a", 2) + tracker.Start("model-stays", "worker-b", 2) + + n := NewDepartureNotifier() + n.OnDeparture("staging-tracker", func(node DepartedNode) { tracker.DropNode(node.Name) }) + n.Departed(DepartedNode{ID: "n1", Name: "worker-a", Type: NodeTypeBackend}) + + Expect(tracker.GetAll()).ToNot(HaveKey("model-gone")) + Expect(tracker.GetAll()).To(HaveKey("model-stays")) + }) + + It("removes the departed node from every open operation's per-node breakdown", func() { + gs := galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) + gs.UpdateNodeProgress("op-1", "n1", galleryop.NodeProgress{ + NodeID: "n1", NodeName: "worker-a", Status: galleryop.NodeStatusDownloading, Percentage: 12, + }) + gs.UpdateNodeProgress("op-1", "n2", galleryop.NodeProgress{ + NodeID: "n2", NodeName: "worker-b", Status: galleryop.NodeStatusDownloading, Percentage: 40, + }) + + n := NewDepartureNotifier() + n.OnDeparture("gallery-node-progress", func(node DepartedNode) { gs.DropNodeProgress(node.ID) }) + n.Departed(DepartedNode{ID: "n1", Name: "worker-a", Type: NodeTypeBackend}) + + status := gs.GetStatus("op-1") + Expect(status).ToNot(BeNil()) + Expect(status.Nodes).To(HaveLen(1)) + Expect(status.Nodes[0].NodeID).To(Equal("n2")) + }) + + It("leaves a finished operation's breakdown alone, because that is a record and not a claim", func() { + gs := galleryop.NewGalleryService(&config.ApplicationConfig{}, nil) + gs.UpdateNodeProgress("op-done", "n1", galleryop.NodeProgress{ + NodeID: "n1", NodeName: "worker-a", Status: galleryop.NodeStatusSuccess, Percentage: 100, + }) + done := gs.GetStatus("op-done") + done.Processed = true + gs.UpdateStatus("op-done", done) + + gs.DropNodeProgress("n1") + + status := gs.GetStatus("op-done") + Expect(status.Nodes).To(HaveLen(1)) + }) +}) diff --git a/core/services/nodes/health.go b/core/services/nodes/health.go index 8b9061e9428e..4ab3efe78d15 100644 --- a/core/services/nodes/health.go +++ b/core/services/nodes/health.go @@ -47,10 +47,13 @@ type HealthMonitor struct { // tunnelDeparted. nil disables the second mechanism entirely. presence NodePresenceReader reconnectGrace time.Duration - missesMu sync.Mutex - misses map[modelKey]int // consecutive failed-probe counts; reset on success or model removal - cancel context.CancelFunc - cancelMu sync.Mutex + // departures is the one place a node's departure evicts per-node state. + // See DepartureNotifier: this monitor is its only caller. + departures *DepartureNotifier + missesMu sync.Mutex + misses map[modelKey]int // consecutive failed-probe counts; reset on success or model removal + cancel context.CancelFunc + cancelMu sync.Mutex } // NewHealthMonitor creates a new HealthMonitor. @@ -68,7 +71,14 @@ type HealthMonitor struct { // is correct for a single-node install and wrong for a distributed one; a zero // grace with a non-nil reader takes the documented default, because a zero // window would make every departure a verdict the instant it was stamped. -func NewHealthMonitor(registry NodeHealthStore, db *gorm.DB, checkInterval, staleThreshold time.Duration, authToken string, perModelHealthCheck bool, presence NodePresenceReader, reconnectGrace time.Duration, clientFactory ...BackendClientFactory) *HealthMonitor { +// +// departures is positional and REQUIRED for the same reason, not a setter and +// not an option: a caller that does not pass one fails to COMPILE. A monitor +// with no notifier is a deployment where a departed node's per-node caches are +// never dropped, and there is no other symptom. Passing nil is still legal and +// still says something, namely that this deployment has no caches to drop; a +// distributed one always passes the notifier its subscribers were registered on. +func NewHealthMonitor(registry NodeHealthStore, db *gorm.DB, checkInterval, staleThreshold time.Duration, authToken string, perModelHealthCheck bool, presence NodePresenceReader, reconnectGrace time.Duration, departures *DepartureNotifier, clientFactory ...BackendClientFactory) *HealthMonitor { checkInterval = cmp.Or(checkInterval, 15*time.Second) // Heartbeat checkpointing lets last_heartbeat sit up to one checkpoint // interval behind by design, so a hardcoded 60s fallback here would mark @@ -91,6 +101,7 @@ func NewHealthMonitor(registry NodeHealthStore, db *gorm.DB, checkInterval, stal perModelHealthCheck: perModelHealthCheck, presence: presence, reconnectGrace: cmp.Or(reconnectGrace, config.DefaultWorkerReconnectGrace), + departures: departures, misses: make(map[modelKey]int), } } @@ -105,6 +116,20 @@ func NewHealthMonitor(registry NodeHealthStore, db *gorm.DB, checkInterval, stal // what a healthy fleet looks like. func (hm *HealthMonitor) ReadsAbsence() bool { return hm != nil && hm.presence != nil } +// Departures returns the notifier this monitor fires departures on. +// +// Exposed so the wiring can be asserted on the object production built rather +// than on a notifier a spec assembled itself: the subscribers are registered by +// core/application on a notifier it then hands to this constructor, and a +// deployment that registered them on a DIFFERENT notifier evicts nothing while +// a listing of that other notifier's registrations still names all four. +func (hm *HealthMonitor) Departures() *DepartureNotifier { + if hm == nil { + return nil + } + return hm.departures +} + // tunnelDeparted reports whether this deployment has decided that a node's // tunnel is gone: no live replica holds it and the departure has outlived the // reconnect grace. @@ -115,22 +140,20 @@ func (hm *HealthMonitor) ReadsAbsence() bool { return hm != nil && hm.presence ! // Acting on any of those would demote a fleet for a reason that has nothing to // do with any worker, which is the collapse this whole mechanism replaced. // -// Backend workers only, and since agent workers hold tunnels too that check is -// now the RULE rather than an optimisation. It used to save a query: an agent -// worker dialled nothing, so it had no departure to measure and would have -// answered PresenceUnknown regardless. It now has a real node_connections row -// that really does age past the grace, and demoting an agent node for it would -// be a verdict about a route the agent's actual work does not travel on: an -// agent worker still takes its jobs and its verbs over the bus. Deleting this -// check would silently make every agent worker whose tunnel dropped go -// unhealthy, and its next heartbeat would make it healthy again. +// EVERY worker type, and that is the change this rule most recently absorbed. +// It used to skip agent nodes, because an agent worker dialled no tunnel and +// took its jobs and its verbs over the message bus instead, so a departure row +// for one said nothing about whether it could work. There is no bus left: an +// agent worker is reachable through its tunnel and through nothing else, so a +// departed agent tunnel means exactly what a departed backend tunnel means and +// the skip would now hide the only symptom an unreachable agent worker has. +// +// The rule itself did not widen, only the set of nodes it applies to. The three +// answers nobody may act on stay unacted on for both types. func (hm *HealthMonitor) tunnelDeparted(ctx context.Context, node *BackendNode) bool { if hm.presence == nil || node == nil { return false } - if node.NodeType != "" && node.NodeType != NodeTypeBackend { - return false - } p, err := hm.presence.Presence(ctx, node.ID, hm.reconnectGrace) if err != nil { xlog.Warn("Health monitor could not read node presence; leaving the node's status alone", @@ -258,11 +281,18 @@ func (hm *HealthMonitor) doCheckAll(ctx context.Context) { if hm.tunnelDeparted(ctx, &node) { if node.Status != StatusUnhealthy && node.Status != StatusOffline { xlog.Warn("Node is heartbeating but its tunnel has been gone longer than the reconnect grace; marking unhealthy", - "node", node.Name, "nodeID", node.ID, "grace", hm.reconnectGrace) + "node", node.Name, "nodeID", node.ID, "type", node.NodeType, "grace", hm.reconnectGrace) if err := hm.registry.MarkUnhealthy(ctx, node.ID); err != nil { xlog.Error("Failed to mark a departed node unhealthy", "node", node.Name, "error", err) } } + // The ONE place a departure evicts per-node state, and it is fired + // from THIS branch only. The stale-heartbeat branch above already + // marks the node offline, which deletes its rows and runs the + // registry's replica-removed hooks; firing here as well would + // double-evict and, worse, would make the notification mean two + // different things at its subscribers. + hm.departures.Departed(DepartedNode{ID: node.ID, Name: node.Name, Type: node.NodeType}) // No re-promotion, and no per-model probes. The probes would dial a // worker there is no route to, once per model per tick, and decline // to count any of it; the re-promotion below is what used to undo @@ -270,6 +300,11 @@ func (hm *HealthMonitor) doCheckAll(ctx context.Context) { continue } + // Present, so the NEXT departure of this node is announced again. This + // runs on every tick for every live node and must stay a map delete: + // the notifier is edge triggered, and re-arming here is the edge. + hm.departures.Present(node.ID) + // Heartbeat is fresh and the tunnel is not gone: the node is alive if node.Status == StatusUnhealthy || node.Status == StatusOffline { xlog.Info("Node recovered", "node", node.Name) diff --git a/core/services/nodes/health_test.go b/core/services/nodes/health_test.go index 8ddfc8491b16..9a6d8172a8f9 100644 --- a/core/services/nodes/health_test.go +++ b/core/services/nodes/health_test.go @@ -35,7 +35,7 @@ var _ = Describe("HealthMonitor", func() { // Use a 30-second stale threshold for tests. // Pass nil db to avoid advisory lock path (no distributed mode in tests). - hm = NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, nil, 0) + hm = NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, nil, 0, nil) }) makeNode := func(name, address string, vram uint64) *BackendNode { @@ -431,6 +431,12 @@ var _ = Describe("HealthMonitor and a worker whose tunnel is gone", func() { registry *NodeRegistry clusterR *cluster.Registry hm *HealthMonitor + // departed is what the monitor ANNOUNCED, which is a different + // question from what it demoted: a departure notification is an act on + // absence, so the three answers nobody may act on must announce + // nothing even where they also demote nothing. + departed []DepartedNode + departures *DepartureNotifier ) const ( @@ -450,7 +456,10 @@ var _ = Describe("HealthMonitor and a worker whose tunnel is gone", func() { Expect(cluster.Migrate(ctx, db)).To(Succeed()) clusterR = cluster.NewRegistry(db) Expect(clusterR.Register(ctx, instance, "10.0.0.1:8080", "v1")).To(Succeed()) - hm = NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, clusterR, grace) + departed = nil + departures = NewDepartureNotifier() + departures.OnDeparture("spec", func(node DepartedNode) { departed = append(departed, node) }) + hm = NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, clusterR, grace, departures) }) // register creates a heartbeating backend worker. Its heartbeat stays fresh @@ -464,6 +473,16 @@ var _ = Describe("HealthMonitor and a worker whose tunnel is gone", func() { return node } + // registerAgent creates a heartbeating AGENT worker, the same way register + // creates a backend one. + registerAgent := func(name string) *BackendNode { + GinkgoHelper() + node := &BackendNode{Name: name, NodeType: NodeTypeAgent} + Expect(registry.Register(ctx, node, true)).To(Succeed()) + Expect(node.Status).To(Equal(StatusHealthy)) + return node + } + statusOf := func(id string) string { GinkgoHelper() n, err := registry.Get(ctx, id) @@ -550,6 +569,22 @@ var _ = Describe("HealthMonitor and a worker whose tunnel is gone", func() { hm.doCheckAll(ctx) Expect(statusOf(node.ID)).To(Equal(StatusHealthy)) + Expect(departed).To(BeEmpty()) + }) + + It("leaves an AGENT node whose tunnel went inside the grace alone, and announces nothing for it", func() { + // Asserted per node type because this task widened the set of nodes the + // rule applies to, and a reconnecting agent worker is a case the rule + // was never reachable by before. A departure notification is an act on + // absence: firing one here would evict the caches of a worker that is + // re-dialling right now. + node := registerAgent("agent-re-homing") + departTunnel(node.ID, grace/2) + + hm.doCheckAll(ctx) + + Expect(statusOf(node.ID)).To(Equal(StatusHealthy)) + Expect(departed).To(BeEmpty()) }) It("leaves a node that has never dialled a tunnel alone", func() { @@ -560,28 +595,125 @@ var _ = Describe("HealthMonitor and a worker whose tunnel is gone", func() { hm.doCheckAll(ctx) Expect(statusOf(node.ID)).To(Equal(StatusHealthy)) + Expect(departed).To(BeEmpty()) }) - It("leaves an AGENT node alone even when its tunnel would read as gone", func() { - // Agent workers hold a tunnel of their own now, and still take their - // jobs and their one remaining node verb over the bus. So a departure - // row for one is real and is still not a fact about whether the agent - // worker can work. This spec is what stops a bug in the agent tunnel - // client from demoting a fleet of perfectly healthy agent workers. - node := &BackendNode{Name: "agent-worker", NodeType: NodeTypeAgent} - Expect(registry.Register(ctx, node, true)).To(Succeed()) - departTunnel(node.ID, grace+5*time.Second) + It("leaves an AGENT node that has never dialled a tunnel alone", func() { + node := registerAgent("agent-never-dialled") hm.doCheckAll(ctx) Expect(statusOf(node.ID)).To(Equal(StatusHealthy)) + Expect(departed).To(BeEmpty()) + }) + + It("stops reporting an AGENT node healthy once its tunnel has been gone past the grace", func() { + // The inverse of the scaffold this replaces. That spec asserted an + // agent node was NOT demoted, and it was true while an agent worker + // took its jobs and its verbs over the message bus: a departure row + // said nothing about whether it could work. There is no bus. An agent + // worker is reachable through its tunnel and through nothing else, so + // its departed tunnel means what a backend worker's does. + node := registerAgent("agent-worker") + departTunnel(node.ID, grace+5*time.Second) + + hm.doCheckAll(ctx) + + Expect(statusOf(node.ID)).To(Equal(StatusUnhealthy)) + }) + + It("does not touch a BACKEND node in the same fleet whose tunnel is fine", func() { + // The negative control for the widening: demoting everything would + // satisfy the spec above. + agent := registerAgent("agent-worker-2") + backend := register("backend-worker-2") + departTunnel(agent.ID, grace+5*time.Second) + _, err := clusterR.Claim(ctx, backend.ID, instance) + Expect(err).ToNot(HaveOccurred()) + + hm.doCheckAll(ctx) + + Expect(statusOf(agent.ID)).To(Equal(StatusUnhealthy)) + Expect(statusOf(backend.ID)).To(Equal(StatusHealthy)) + }) + + It("announces a departure for a node whose tunnel is gone past the grace, naming it", func() { + // The eviction edge. Every per-node cache in the deployment is dropped + // from here, and from nowhere else. + node := register("announced") + departTunnel(node.ID, grace+5*time.Second) + + hm.doCheckAll(ctx) + + Expect(departed).To(ConsistOf(DepartedNode{ID: node.ID, Name: "announced", Type: NodeTypeBackend})) + }) + + It("announces an AGENT node's departure with its type, so a subscriber can tell the fleets apart", func() { + node := registerAgent("announced-agent") + + departTunnel(node.ID, grace+5*time.Second) + + hm.doCheckAll(ctx) + + Expect(departed).To(ConsistOf(DepartedNode{ID: node.ID, Name: "announced-agent", Type: NodeTypeAgent})) + }) + + It("announces a node's departure once, not once per tick, however long it stays gone", func() { + // Level triggered, every cache in the deployment would be re-evicted + // every health interval for as long as the worker is away. + node := register("still-departed") + departTunnel(node.ID, grace+5*time.Second) + + hm.doCheckAll(ctx) + hm.doCheckAll(ctx) + hm.doCheckAll(ctx) + + Expect(departed).To(HaveLen(1)) + }) + + It("announces a second departure after the node has been seen present again", func() { + node := register("left-twice") + departTunnel(node.ID, grace+5*time.Second) + hm.doCheckAll(ctx) + // Back, and SEEN back: the re-arm happens on a tick where the node is + // present, not on the claim itself. + epoch, err := clusterR.Claim(ctx, node.ID, instance) + Expect(err).ToNot(HaveOccurred()) + hm.doCheckAll(ctx) + Expect(clusterR.Release(ctx, node.ID, instance, epoch)).To(Succeed()) + res := db.WithContext(ctx).Exec( + `UPDATE node_connections SET disconnected_at = now() - make_interval(secs => ?) WHERE node_id = ?`, + (grace + 5*time.Second).Seconds(), node.ID) + Expect(res.Error).ToNot(HaveOccurred()) + Expect(res.RowsAffected).To(Equal(int64(1))) + + hm.doCheckAll(ctx) + + Expect(departed).To(HaveLen(2)) + }) + + It("announces NOTHING for a node whose stale heartbeat took it offline", func() { + // Firing here too would double-evict and, worse, would make the + // notification mean two things: an offline node's rows are DELETED and + // the registry's replica-removed hooks already run for it. + node := register("dead-supervisor") + departTunnel(node.ID, grace+5*time.Second) + res := db.WithContext(ctx).Exec( + `UPDATE backend_nodes SET last_heartbeat = now() - make_interval(secs => 600) WHERE id = ?`, node.ID) + Expect(res.Error).ToNot(HaveOccurred()) + Expect(res.RowsAffected).To(Equal(int64(1))) + + hm.doCheckAll(ctx) + + Expect(statusOf(node.ID)).To(Equal(StatusOffline)) + Expect(departed).To(BeEmpty()) }) It("leaves every node alone when it has no presence reader", func() { // A single-node deployment has nothing that can say a worker is gone. node := register("no-cluster-registry") departTunnel(node.ID, grace+5*time.Second) - plain := NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, nil, 0) + plain := NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, nil, 0, nil) plain.doCheckAll(ctx) @@ -595,7 +727,7 @@ var _ = Describe("HealthMonitor and a worker whose tunnel is gone", func() { // above passes an explicit grace, so nothing else reaches this. node := register("no-grace") stub := &stubPresence{answer: cluster.PresenceConnected} - defaulted := NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, stub, 0) + defaulted := NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, stub, 0, nil) defaulted.doCheckAll(ctx) @@ -603,15 +735,23 @@ var _ = Describe("HealthMonitor and a worker whose tunnel is gone", func() { Expect(stub.graces).To(ContainElement(config.DefaultWorkerReconnectGrace)) }) - It("leaves a node alone when the presence query fails", func() { - // A database hiccup must not demote the fleet. Driven with a stub, - // because a real registry cannot be made to fail on demand. - node := register("query-fails") + It("leaves a node alone when the presence query fails, and announces nothing", func() { + // A database hiccup must not demote the fleet, and must not evict its + // caches either. Driven with a stub, because a real registry cannot be + // made to fail on demand. Both node types are in the fleet, because the + // rule now reaches both. + backend := register("query-fails") + agent := registerAgent("agent-query-fails") + var announced []DepartedNode + notifier := NewDepartureNotifier() + notifier.OnDeparture("spec", func(node DepartedNode) { announced = append(announced, node) }) broken := NewHealthMonitor(registry, nil, 15*time.Second, 30*time.Second, "", false, - &stubPresence{err: errors.New("connection reset")}, grace) + &stubPresence{err: errors.New("connection reset")}, grace, notifier) broken.doCheckAll(ctx) - Expect(statusOf(node.ID)).To(Equal(StatusHealthy)) + Expect(statusOf(backend.ID)).To(Equal(StatusHealthy)) + Expect(statusOf(agent.ID)).To(Equal(StatusHealthy)) + Expect(announced).To(BeEmpty()) }) }) diff --git a/core/services/nodes/prefixcache/index.go b/core/services/nodes/prefixcache/index.go index fe629ff56844..644080ae926b 100644 --- a/core/services/nodes/prefixcache/index.go +++ b/core/services/nodes/prefixcache/index.go @@ -120,6 +120,22 @@ func (ix *Index) InvalidateNode(model, nodeID string) { } } +// DropNode drops entries for ALL replicas of nodeID in EVERY model. +// +// InvalidateNode's sibling, and the difference is the caller. InvalidateNode +// serves the registry chokepoint, which fires per model because a replica row +// IS per model. A departure names no model at all: the node is gone from every +// model it ever served, and asking the caller to enumerate those would make the +// eviction depend on a registry read that a departed node's rows may already +// have left. Like its sibling it never interns a tree. +func (ix *Index) DropNode(nodeID string) { + ix.mu.RLock() + defer ix.mu.RUnlock() + for _, t := range ix.trees { + t.RemoveFunc(func(k ReplicaKey) bool { return k.NodeID == nodeID }) + } +} + func (ix *Index) Evict(now time.Time) { ix.mu.RLock() defer ix.mu.RUnlock() diff --git a/core/services/nodes/prefixcache/provider.go b/core/services/nodes/prefixcache/provider.go index 976713aaca88..ea64c86cfbcd 100644 --- a/core/services/nodes/prefixcache/provider.go +++ b/core/services/nodes/prefixcache/provider.go @@ -17,8 +17,13 @@ type Provider interface { Observe(model string, chain []uint64, key ReplicaKey, now time.Time) bool // Invalidate drops all entries for ONE replica. Invalidate(model string, key ReplicaKey) - // InvalidateNode drops entries for ALL replicas of a node. + // InvalidateNode drops entries for ALL replicas of a node, in ONE model. InvalidateNode(model, nodeID string) + // DropNode drops entries for ALL replicas of a node, in EVERY model. It is + // what a node DEPARTURE evicts through: a departure names no model, and a + // caller made to enumerate them would have to read rows the departure may + // already have cost it. + DropNode(nodeID string) // Evict sweeps expired entries for all models. Evict(now time.Time) } diff --git a/core/services/nodes/prefixcache/sync.go b/core/services/nodes/prefixcache/sync.go index 78d28f708e59..19558ee3d898 100644 --- a/core/services/nodes/prefixcache/sync.go +++ b/core/services/nodes/prefixcache/sync.go @@ -131,6 +131,20 @@ func (s *Sync) InvalidateNode(model, node string) { } } +// DropNode drops the local entries for ALL replicas of nodeID in every model. +// +// It does NOT broadcast, and that is the one place it parts company with its +// per-model siblings. A departure is decided by the health monitor, which runs +// under an advisory lock so exactly one replica reaches this; the peers keep +// their entries for a node that the same decision has just marked unhealthy in +// the database, and selection filters on healthy status in SQL, so those +// entries cannot route work to the departed node and expire on their own TTL. +// Inventing a node-wide invalidation event to carry this would add a wire +// meaning whose only reader is a set of entries that are already unreachable. +func (s *Sync) DropNode(nodeID string) { + s.idx.DropNode(nodeID) +} + // ApplyObserve applies a peer observe event locally (no re-broadcast). func (s *Sync) ApplyObserve(ev messaging.PrefixCacheObserveEvent, now time.Time) { s.idx.Observe(ev.Model, ev.Chain, ReplicaKey{NodeID: ev.NodeID, Replica: ev.Replica}, now) diff --git a/core/services/nodes/probe_cache.go b/core/services/nodes/probe_cache.go index a5b3e2cb0573..511666b13f14 100644 --- a/core/services/nodes/probe_cache.go +++ b/core/services/nodes/probe_cache.go @@ -1,6 +1,7 @@ package nodes import ( + "strings" "sync" "time" @@ -68,6 +69,26 @@ func (c *probeCache) Invalidate(key string) { delete(c.seen, key) } +// InvalidateNode drops every cached probe for nodeID. +// +// Keys are "|", so this is a prefix scan and not a lookup: a +// departed worker that comes back with recycled ports would otherwise serve one +// request per still-fresh address with no probe at all, which is the one moment +// the cache is asked about a process that certainly is not the one it saw. +func (c *probeCache) InvalidateNode(nodeID string) { + if nodeID == "" { + return + } + prefix := nodeID + "|" + c.mu.Lock() + defer c.mu.Unlock() + for k := range c.seen { + if strings.HasPrefix(k, prefix) { + delete(c.seen, k) + } + } +} + // DoOrCachedResult returns true if key is fresh; otherwise it runs probe // (coalescing concurrent callers via singleflight) and caches a successful // result. Failed probes invalidate the cache, so a transient miss does not pin diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index ef3add61644d..735ab9730ff7 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -304,6 +304,20 @@ func (r *SmartRouter) Unloader() NodeCommandSender { return r.unloader } // StagingTracker returns the staging progress tracker for UI visibility. func (r *SmartRouter) StagingTracker() *StagingTracker { return r.stagingTracker } +// InvalidateNodeProbes drops every cached probe freshness entry for nodeID. +// +// The router owns the probe cache, so this is how a departure reaches it. It is +// a method rather than an exported cache because the cache's keys are the +// router's own composition of node ID and worker-local address, and a caller +// that had to build one of those keys would be a second place that decides the +// key format. +func (r *SmartRouter) InvalidateNodeProbes(nodeID string) { + if r == nil || r.probeCache == nil { + return + } + r.probeCache.InvalidateNode(nodeID) +} + // scheduleLoadResult holds the result of scheduling and loading a model on a node. type scheduleLoadResult struct { Node *BackendNode diff --git a/core/services/nodes/router_test.go b/core/services/nodes/router_test.go index 80bf628c0c07..e66428cc6684 100644 --- a/core/services/nodes/router_test.go +++ b/core/services/nodes/router_test.go @@ -1390,6 +1390,7 @@ type fakePrefixProvider struct { observed []observeRecord invalidated []invalidateRecord invalidatedNode []string + droppedNodes []string decision prefixcache.PrefixDecision } @@ -1411,6 +1412,12 @@ func (f *fakePrefixProvider) InvalidateNode(model, nodeID string) { f.invalidatedNode = append(f.invalidatedNode, model+":"+nodeID) } +// DropNode is the departure eviction: every model, one node. Recorded the same +// way its per-model sibling is, so a spec can tell the two apart. +func (f *fakePrefixProvider) DropNode(nodeID string) { + f.droppedNodes = append(f.droppedNodes, nodeID) +} + func (f *fakePrefixProvider) Evict(_ time.Time) {} var _ = Describe("SmartRouter prefix-cache routing", func() { diff --git a/core/services/nodes/staging_progress.go b/core/services/nodes/staging_progress.go index 9b11924cddc1..860038601f7b 100644 --- a/core/services/nodes/staging_progress.go +++ b/core/services/nodes/staging_progress.go @@ -230,6 +230,29 @@ func (t *StagingTracker) Complete(modelID string) { } } +// DropNode ends every staging operation attributed to nodeName. +// +// A staging op has no terminal state of its own when its destination node +// leaves: the copy loop that would call Complete is blocked on a worker that is +// no longer reachable, so /api/operations shows it copying files forever. +// +// It drops what THIS tracker holds, locally owned ops and mirrored ones alike, +// and broadcasts nothing. A Done event could not end a peer's locally owned op +// anyway (ApplyRemote treats a local op as authoritative, on purpose), and the +// peer performing such a transfer ends it itself when the transfer fails. +func (t *StagingTracker) DropNode(nodeName string) { + if nodeName == "" { + return + } + t.mu.Lock() + defer t.mu.Unlock() + for modelID, e := range t.active { + if e.status.NodeName == nodeName { + delete(t.active, modelID) + } + } +} + // ApplyRemote merges a peer replica's staging broadcast into this tracker. It // never re-broadcasts (no echo loop). A locally-owned op is authoritative: a // remote event for the same model is ignored, so the origin replica receiving diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 7ce0671035ba..bda4ebf1c2be 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -491,11 +491,13 @@ When a worker's tunnel goes, the frontend does **not** forget the worker. It rec That distinction exists because a worker loses its tunnel for entirely ordinary reasons. A frontend replica restarting during a rolling upgrade drops every tunnel it held, and each of those workers immediately re-dials the load balancer and lands on another replica. Treating that as "the worker is gone" would evict models mid-upgrade for a fleet that never actually went anywhere. -This applies to **backend** nodes only. An agent worker holds a tunnel and therefore has a `node_connections` row that ages exactly like a backend worker's, and nothing acts on it: the scheduler never sees agent nodes at all (every placement query selects `node_type = 'backend'`), and the health monitor skips them explicitly. +This applies to **both worker kinds, under the same grace**. An agent worker holds a tunnel of its own and is reached through it and through nothing else, so its `node_connections` row ages exactly like a backend worker's and means exactly the same thing: past the grace it is reported `unhealthy` in `GET /api/nodes`. Earlier releases exempted agent nodes by type, because an agent worker took its work over a message bus and a departed tunnel said nothing about it; there is no bus any more, so that exemption would hide the only symptom an unreachable agent worker has. What still differs is placement, not liveness: the scheduler never sees agent nodes (every placement query selects `node_type = 'backend'`), and backend listing and backend install/upgrade/delete still skip them, because an agent worker runs no backend processes. + +**A departure evicts the per-node state the frontend was holding.** On the transition to gone, and once per departure rather than once per health cycle, the frontend drops that node's prefix-cache affinity entries in every model, its cached backend-probe results, its in-flight file-staging operations, and its rows in the per-node breakdown of every operation still open in `GET /api/operations`. A worker that comes back is re-probed rather than trusted, and an operation that was waiting on a node that left stops reporting it as still copying. Nothing is evicted for a worker that is merely **reconnecting**: the eviction is an act on absence, so it fires only where the demotion does. Three things read this. The **scheduler** reads it before it places a cold load: a worker whose departure has outlived the grace is skipped and marked unhealthy, so every other frontend replica stops choosing it too. **LRU eviction** reads it before it hands back the node it freed capacity on, because a node full enough to be an eviction target is exactly the node the placement selectors never offer, so the scheduler's own check never sees it. The **health monitor** reads it on every cycle, which is what covers the case the heartbeat cannot see. A worker's heartbeat says its supervisor is alive; it says nothing about whether anything here can reach that worker's backends, because those are reached over the tunnel. A worker that heartbeats with a permanently dead tunnel (a proxy that stopped upgrading WebSockets, a rotated registration credential, a reconnect loop longer than the grace) is therefore marked unhealthy too, rather than staying listed healthy while every request for a model loaded on it fails "no route to that worker". -The log lines are `Scheduled node has no tunnel and its departure outlived the reconnect grace, marking unhealthy and re-scheduling`, `Eviction target has no tunnel and its departure outlived the reconnect grace, marking unhealthy and evicting again`, and `Node is heartbeating but its tunnel has been gone longer than the reconnect grace; marking unhealthy`. Each names the grace it used. +The log lines are `Scheduled node has no tunnel and its departure outlived the reconnect grace, marking unhealthy and re-scheduling`, `Eviction target has no tunnel and its departure outlived the reconnect grace, marking unhealthy and evicting again`, and `Node is heartbeating but its tunnel has been gone longer than the reconnect grace; marking unhealthy`. Each names the grace it used, and the last one names the node type as well, so an agent worker's demotion is not read as a backend worker's. **The demotion is a status change, not a deletion.** The worker's `node_models` rows survive it. Status is enough to unwedge the node, because request routing and LRU eviction both select only healthy nodes, so the model stops being served from there and the next request places it somewhere reachable. Deleting rows on a presence read would give any future defect in that read the widest possible blast radius, for nothing the demotion does not already deliver. diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 861b47f2456a..fe48d5672ceb 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -48,13 +48,12 @@ type Options struct { // alongside the backend workers. // // They exist so a spec can hold the two kinds of worker side by side in one - // cluster. An agent worker still speaks NATS, and now holds a tunnel of its - // own as well, which is exactly the shape the tunnel-departure rules must - // not act on: it gets a real node_connections row whose departure really - // does age past the grace, and demoting it for that would take out a fleet - // whose actual work travels on the bus. A spec that asserted only on - // backend workers could not tell "agent workers are unaffected" from - // "nothing here looks at them". + // cluster. An agent worker holds a tunnel of its own and is reached through + // it and through nothing else, so it gets a real node_connections row whose + // departure really does age past the grace and it is subject to every + // tunnel-departure rule a backend worker is. A spec that asserted only on + // backend workers could not tell "the rule reaches both fleets" from + // "nothing here looks at agent workers". // // They register through the same WorkerFrontendURL hook as backend workers, // so a spec that puts a balancer in front of the fleet gets one for its @@ -396,18 +395,19 @@ func (c *Cluster) startWorker(i int) (*Process, error) { // startAgentWorker starts agent worker i. // // It is `local-ai agent-worker`, not `local-ai worker`, and the difference is -// the whole point of having it here: an agent worker REQUIRES a NATS URL and -// runs no backend, so it is the control for every rule this phase added about a -// worker whose tunnel is gone. It dials a tunnel of its own, but binds only -// loopback for the control plane behind it, so there is still no port to -// reserve and no reachable readiness endpoint to wait on; a spec learns it is -// up by finding it in the roster. +// the whole point of having it here: an agent worker runs no backend processes, +// so it is the second worker KIND every rule about a departed tunnel now has to +// hold for. It dials a tunnel of its own, but binds only loopback for the +// control plane behind it, so there is still no port to reserve and no +// reachable readiness endpoint to wait on; a spec learns it is up by finding it +// in the roster. func (c *Cluster) startAgentWorker(i int) (*Process, error) { name := agentWorkerName(i) cmd := exec.Command(c.opts.Binary, "agent-worker") cmd.Env = append(cmd.Environ(), - // The bus, which is what makes this worker the control: a backend - // worker in this same cluster is given none. + // Still set, and still ignored. It is left here so this suite keeps + // covering the promise that an operator's existing --nats-url does not + // break an agent worker; nothing in the process reads it any more. "LOCALAI_NATS_URL="+c.opts.NatsURL, "LOCALAI_REGISTER_TO="+c.workerFrontendURL(i), "LOCALAI_NODE_NAME="+name, diff --git a/tests/e2e/distributed/cluster_control_test.go b/tests/e2e/distributed/cluster_control_test.go index 8f9e472a73cc..9daad1b5c2bb 100644 --- a/tests/e2e/distributed/cluster_control_test.go +++ b/tests/e2e/distributed/cluster_control_test.go @@ -382,8 +382,8 @@ func withReconnectGrace(d time.Duration) func(*cluster.Options) { return func(o *cluster.Options) { o.ReconnectGrace = d } } -// withAgentWorkers adds agent workers, which still speak NATS for everything -// this phase has not moved, to a cluster. +// withAgentWorkers adds agent workers, the second worker kind, to a cluster. +// They hold a tunnel and are reached through it and through nothing else. func withAgentWorkers(n int) func(*cluster.Options) { return func(o *cluster.Options) { o.AgentWorkers = n } } @@ -645,20 +645,21 @@ var _ = Describe("Control plane over the worker tunnel", Label("Distributed"), L "the survivor must serve the re-homed worker") }) - // Scenario 5, the wedge task 6 fixed, plus the agent-worker control. + // Scenario 5, the wedge task 6 fixed, now for BOTH worker types. // // A worker with a FRESH heartbeat and a tunnel that is gone past the grace // must stop being reported healthy. Before task 6 it stayed healthy // forever, with every request for a model on it failing "no route", because // every reaper keyed on the heartbeat and the heartbeat was fine. // - // The agent worker in the same cluster is the control for the other - // direction. It holds a tunnel too now, and the balancer below blocks its - // dial exactly as it blocks the backend worker's, so its departure ages - // past the same grace. Its real work still travels on the bus, so a rule - // that read "no tunnel" as "gone" would take the whole agent fleet down - // with it. - It("stops reporting a heartbeating worker healthy once its tunnel is gone past the grace, and leaves agent workers alone", func() { + // The agent worker in the same cluster used to be the control for the other + // direction: it was skipped by name, because its real work travelled on the + // message bus and a departed tunnel said nothing about it. There is no bus. + // The balancer below blocks its tunnel dial exactly as it blocks the backend + // worker's, its departure ages past the same grace, and it is now reachable + // through that tunnel and through nothing else, so it must be demoted for + // the same reason and on the same evidence. + It("stops reporting a heartbeating worker healthy once its tunnel is gone past the grace, for agent workers too", func() { var balancer *frontendBalancer c, dsn := startClusterOnFreshDB(2, 1, withBalancer(&balancer), withAgentWorkers(1), withReconnectGrace(10*time.Second)) @@ -669,6 +670,8 @@ var _ = Describe("Control plane over the worker tunnel", Label("Distributed"), L Should(And(ContainElement(c.WorkerName(0)), ContainElement(c.AgentWorkerName(0))), probe.describe) nodeID := probe.idOf(c.WorkerName(0)) Expect(nodeID).ToNot(BeEmpty()) + agentID := probe.idOf(c.AgentWorkerName(0)) + Expect(agentID).ToNot(BeEmpty()) owners := newTunnelOwners(openClusterDB(dsn)) var owner int @@ -678,6 +681,17 @@ var _ = Describe("Control plane over the worker tunnel", Label("Distributed"), L }, tunnelOwnershipTimeout, tunnelOwnershipPoll).Should(BeNumerically(">=", 0), owners.describe) survivor := 1 - owner + // The precondition the agent half of this spec rests on, asserted and + // not assumed. The balancer forwards to the FIRST target that accepts a + // connection, so both workers land on the same replica and killing it + // takes both tunnels; if that ever changes, the agent worker's tunnel + // survives on the replica this spec keeps, its departure never ages, + // and the assertion below would be waiting for a demotion that is + // correctly not coming. + Eventually(func() int { return owners.ownerIndexOf(c, 2, agentID) }, tunnelOwnershipTimeout, tunnelOwnershipPoll). + Should(Equal(owner), + "the agent worker's tunnel is not on the replica this spec kills, so its departure would never age and the agent assertion below would prove nothing") + // Take the tunnel away permanently: block the dial, then kill the // replica holding the live session. The worker keeps registering and // heartbeating through the balancer, which still proxies everything @@ -705,12 +719,20 @@ var _ = Describe("Control plane over the worker tunnel", Label("Distributed"), L "the worker's heartbeat is stale, so it was demoted for being gone rather than for having no route") // The agent worker, in the same cluster, under the same grace, on the - // same health monitor, is untouched. Its tunnel is blocked by the same - // balancer, so this is a node whose departure really has outlived the - // grace and which must still not be demoted for it. - Consistently(func() string { return atSurvivor.statusOf(c.AgentWorkerName(0)) }, "20s", "2s"). - Should(Equal("healthy"), - atSurvivor.explain("an agent worker was demoted by a rule about tunnels, and agent workers never hold one")) + // same health monitor. Its tunnel is blocked by the same balancer, so + // its departure really has outlived the grace, and the skip that used + // to exempt it by node type is gone: an agent worker is reached through + // its tunnel and through nothing else now, so a departed tunnel is the + // only symptom an unreachable one has. + Eventually(func() string { return atSurvivor.statusOf(c.AgentWorkerName(0)) }, departedTimeout, departedPoll). + Should(Equal("unhealthy"), + atSurvivor.explain("an agent worker whose tunnel is gone past the grace is still reported healthy, so nothing in the deployment can tell that it is unreachable")) + + // And it was the ROUTE, for the agent worker as much as for the backend + // one. Without this the assertion above is satisfied by an agent worker + // that simply died. + Expect(atSurvivor.heartbeatOf(c.AgentWorkerName(0))).To(BeTemporally(">", time.Now().Add(-1*time.Minute)), + "the agent worker's heartbeat is stale, so it was demoted for being gone rather than for having no route") // And the demotion reverses when the route comes back, so it is a // statement about the route rather than a one-way condemnation. diff --git a/tests/e2e/distributed/cluster_tunnel_test.go b/tests/e2e/distributed/cluster_tunnel_test.go index dd77707db556..294f1b72151d 100644 --- a/tests/e2e/distributed/cluster_tunnel_test.go +++ b/tests/e2e/distributed/cluster_tunnel_test.go @@ -386,13 +386,17 @@ var _ = Describe("Worker tunnel end to end", Label("Distributed"), Label("Cluste // Scenario 1b. The AGENT worker, which used to be gated out of tunnels at // the credential mint site and now dials one like any other node. // - // A wrong implementation is silent in both directions. If the mint gate is - // still closed, the agent worker registers, heartbeats and works exactly as - // before, with its tunnel dial refused 401 forever in a log nothing reads; - // no other spec in this suite would notice. If the absence rules were - // widened to follow, the agent worker would instead start being demoted for - // a tunnel its real work does not travel on. - It("holds a tunnel for an AGENT worker, and still does not judge it by one", func() { + // A wrong implementation is silent. If the mint gate is still closed, the + // agent worker registers, heartbeats and works exactly as before, with its + // tunnel dial refused 401 forever in a log nothing reads, and no other spec + // in this suite would notice. + // + // The healthy assertion at the end is the NEGATIVE CONTROL for the rule + // that now demotes an agent worker whose tunnel has departed past the + // grace: a rule that condemned agent workers outright, rather than for + // their route, would take out an agent worker whose tunnel is right here + // and held. The demotion itself is asserted in cluster_control_test.go. + It("holds a tunnel for an AGENT worker, and reports it healthy for as long as it does", func() { c, dsn := startClusterOnFreshDB(1, 0, withAgentWorkers(1)) client := controlSession(c) @@ -409,8 +413,9 @@ var _ = Describe("Worker tunnel end to end", Label("Distributed"), Label("Cluste Eventually(func() int { return owners.ownerIndexOf(c, 1, nodeID) }, tunnelOwnershipTimeout, tunnelOwnershipPoll). Should(Equal(0), owners.describe) - // And it is still an agent worker: holding a tunnel changed nothing - // about how this deployment decides whether it is present. + // A live tunnel is a live route, so nothing demotes it. This is what + // separates "demoted for having no route" from "demoted for being an + // agent worker", and only the two specs together can say which. Consistently(func() string { return probe.statusOf(c.AgentWorkerName(0)) }, "10s", "2s"). Should(Equal("healthy"), probe.describe) }) From b91ed7a5c28a8610e39e5146620f91d98888d1a0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 4 Sep 2026 07:46:08 +0000 Subject: [PATCH 099/153] chore(distributed): stop telling an operator to run a NATS cluster Every carrier had already moved and no process opened a bus connection, but the surface an operator reads still described a deployment with a broker in it: a compose service, a 220-line credential-generation script, two CI steps pulling a container nothing started, two flag tables offering --nats-url, an architecture diagram with a NATS box wired to the workers, a join-command generator in the Nodes page that emitted --nats-url for agent workers, and a test suite that stood a NATS server up for specs that no longer used it. That is the one way this programme could still fail invisibly. Every test passes, every binary works, and every production deployment goes on running and paying for infrastructure that carries nothing. Nothing in this repository starts a NATS server any more. The compose file is four services, the docs say to shut the broker down and what to keep, and the e2e suite runs on one PostgreSQL container. The three LOCALAI_NATS_*_TIMEOUT env vars are KEPT, and are now documented twice as being kept. They were never broker settings: each names a control-RPC budget the frontend applies to a worker, still read and still enforced. They carry the prefix only because they arrived with the bus, and renaming them would break every existing deployment for cosmetics. The agent worker's join command was the last surface still emitting the flag, two tasks after the agent worker stopped dialling. The Playwright spec that covered it asserted the opposite of what is now true, so it is inverted rather than deleted, and it reads the rendered command string rather than the component's variables: the variables are what the fix removes, so a spec reading them would have stopped compiling instead of failing, and a compile error is not evidence about what an operator is shown. nats_jwt_test.go and its helpers are deleted. They pinned a real server ENFORCING the minted permissions. The CONTENT of those allow lists is still pinned, untouched, by pkg/natsauth's own suites, including the spec that refuses to let the agent lists go empty, since an empty allow list in NATS means unrestricted. The enforcement half is retired rather than moved: enforcement is a property of a connection, and nothing opens one. The suite's own NATS container goes with them, which the brief left for the next task. Removing the pre-pull while BeforeSuite still ran the image would have defeated the step rather than cleaned it up, and this change removes the last reader of TestInfra.NC. agent_native_executor_test.go and mcp_ci_job_test.go are moved onto infra.Bus() instead of deleted: they were the last two specs building a bridge and a dispatcher on a client nobody uses, which is exactly the drift TestInfra.Bus's own comment warns about. cluster.Options.NatsURL is now fed a deliberately dead address rather than a live container's. Frontends and agent workers still receive LOCALAI_NATS_URL, because that is the coverage for the promise that an existing command line still starts; sourcing it from a running server would have let a regression that actually dialled it pass. The control in cluster_control_test.go keeps its assertion and loses its explanation, which claimed the deployment had a bus and no longer could. One latent spec race surfaced and is fixed: the background-run spec waited for a COUNT of events and then read a snapshot for the terminal status, which is the last event of a run and therefore always arrives after the count is met. Its immediate twin had already been fixed this way. Nothing in production changed. pkg/natsauth keeps its files. It is reachable from production only through the natsauth.Config parameter thread, and that thread is the next task's. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .github/workflows/tests-e2e-distributed.yml | 14 +- README.md | 2 +- core/http/react-ui/e2e/nodes-roster.spec.js | 33 ++- core/http/react-ui/src/pages/Nodes.jsx | 8 +- core/http/react-ui/src/utils/format.js | 3 +- docker-compose.distributed.yaml | 51 ++-- docs/content/features/distributed-mode.md | 48 ++-- .../features/distributed_inferencing.md | 4 +- docs/content/features/text-generation.md | 5 +- .../diagrams/distributed-mode-arch.html | 25 +- .../images/diagrams/distributed-mode-arch.png | Bin 328769 -> 287200 bytes pkg/natsauth/permissions_coverage_test.go | 39 +--- scripts/nats-auth-setup.sh | 220 ------------------ .../distributed/agent_native_executor_test.go | 114 ++++----- .../e2e/distributed/cluster_baseline_test.go | 14 +- tests/e2e/distributed/cluster_control_test.go | 7 +- tests/e2e/distributed/foundation_test.go | 13 +- .../distributed/gallery_distributed_test.go | 5 +- .../e2e/distributed/mcp_ci_job_helper_test.go | 52 +++-- tests/e2e/distributed/mcp_ci_job_test.go | 44 ++-- .../e2e/distributed/nats_jwt_helpers_test.go | 156 ------------- tests/e2e/distributed/nats_jwt_test.go | 176 -------------- tests/e2e/distributed/testhelpers_test.go | 133 ++++------- 23 files changed, 291 insertions(+), 875 deletions(-) delete mode 100755 scripts/nats-auth-setup.sh delete mode 100644 tests/e2e/distributed/nats_jwt_helpers_test.go delete mode 100644 tests/e2e/distributed/nats_jwt_test.go diff --git a/.github/workflows/tests-e2e-distributed.yml b/.github/workflows/tests-e2e-distributed.yml index 3f952ee42843..f1a1a1a60b9d 100644 --- a/.github/workflows/tests-e2e-distributed.yml +++ b/.github/workflows/tests-e2e-distributed.yml @@ -56,11 +56,12 @@ jobs: - name: Pre-pull test images # Pulling here rather than inside the suite keeps container-start timing # out of the spec timeouts and makes a registry outage read as a - # setup failure instead of a test failure. These two are the only images - # the suite needs once the testcontainers reaper is disabled below. + # setup failure instead of a test failure. This is the only image the + # suite needs once the testcontainers reaper is disabled below: the + # suite stands up no message broker, because nothing under test dials + # one. run: | docker pull postgres:16-alpine - docker pull nats:2-alpine - name: Distributed E2E # TESTCONTAINERS_RYUK_DISABLED keeps the pre-pull above meaningful. The # reaper exists to clean up leaked containers on a long-lived host, but @@ -91,9 +92,9 @@ jobs: # # Separate job from tests-e2e-distributed so the fast in-process suite is # not held behind a Go build of local-ai. Serial on purpose: each Ginkgo - # process would get its own PostgreSQL and NATS container and each spec - # spawns two or three local-ai children, so --procs on an unmeasured runner - # is a change to make with numbers, not by default. + # process would get its own PostgreSQL container and each spec spawns two or + # three local-ai children, so --procs on an unmeasured runner is a change to + # make with numbers, not by default. # # The two timeouts bound different things and are not alternatives. Ginkgo's # --timeout=20m bounds the SUITE only; this job timeout must additionally @@ -160,7 +161,6 @@ jobs: # setup failure rather than a test failure. run: | docker pull postgres:16-alpine - docker pull nats:2-alpine - name: Cluster E2E env: # No LOCALAI_E2E_BINARY and no separate build step: make test-e2e-cluster diff --git a/README.md b/README.md index e4c1da1e354f..96ec4c6da905 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,7 @@ For older news and full release notes, see [GitHub Releases](https://github.com/ - [Object Detection](https://localai.io/features/object-detection/) - [Reranker API](https://localai.io/features/reranker/) - [P2P Inferencing](https://localai.io/features/distribute/) -- [Distributed Mode](https://localai.io/features/distributed-mode/) — Horizontal scaling with PostgreSQL + NATS +- [Distributed Mode](https://localai.io/features/distributed-mode/): horizontal scaling on PostgreSQL, with no message broker to run - [Model Context Protocol (MCP)](https://localai.io/docs/features/mcp/) - [Built-in Agents](https://localai.io/features/agents/) — Autonomous AI agents with tool use, RAG, skills, SSE streaming, and [Agent Hub](https://agenthub.localai.io) - [Backend Gallery](https://localai.io/backends/) — Install/remove backends on the fly via OCI images diff --git a/core/http/react-ui/e2e/nodes-roster.spec.js b/core/http/react-ui/e2e/nodes-roster.spec.js index 2ef2a1dc1d50..b4bcc589be36 100644 --- a/core/http/react-ui/e2e/nodes-roster.spec.js +++ b/core/http/react-ui/e2e/nodes-roster.spec.js @@ -46,9 +46,17 @@ test.describe('Nodes fleet roster', () => { test.describe('Nodes join command', () => { // The panel emits BOTH the backend and the agent join command from one - // component, so the bus flag has to differ per tab rather than be deleted. - // Backend workers connect to no NATS server; agent workers still do. - test('omits the NATS flag for a backend worker and keeps it for an agent worker', async ({ page }) => { + // component. Neither worker kind dials a message bus any more: each holds one + // outward tunnel to --register-to and takes every verb on it. A join command + // carrying --nats-url would tell an operator to stand up, secure and pay for a + // broker that nothing in the deployment connects to, which is the one way this + // migration can still cost money after the code stopped using it. + // + // Asserted on the RENDERED command text rather than on the component's + // variables, because the variables are what the fix deletes: a spec reading + // them would stop compiling instead of failing, and a compile error is not + // evidence about what an operator is shown. + test('emits no bus flag for either worker kind', async ({ page }) => { await mockCluster(page, []) await page.goto('/app/nodes') @@ -57,14 +65,21 @@ test.describe('Nodes join command', () => { await expect(backendCli).toContainText('local-ai worker', { timeout: 15_000 }) await expect(backendCli).not.toContainText('--nats-url') const backendDocker = page.locator('.p2p-cmd pre').nth(1) + await expect(backendDocker).toContainText('LOCALAI_REGISTER_TO') await expect(backendDocker).not.toContainText('LOCALAI_NATS_URL') + // The agent tab is the one that regressed: it was the last surface still + // emitting the flag, and it kept emitting it for two tasks after the agent + // worker stopped dialling. await page.getByRole('radio', { name: /^Agent$/ }).click() const agentCli = page.locator('.p2p-cmd pre').first() await expect(agentCli).toContainText('local-ai agent-worker', { timeout: 15_000 }) - await expect(agentCli).toContainText('--nats-url') + await expect(agentCli).toContainText('--register-to', + ) + await expect(agentCli).not.toContainText('--nats-url') const agentDocker = page.locator('.p2p-cmd pre').nth(1) - await expect(agentDocker).toContainText('LOCALAI_NATS_URL') + await expect(agentDocker).toContainText('LOCALAI_REGISTER_TO') + await expect(agentDocker).not.toContainText('LOCALAI_NATS_URL') }) test('does not advertise flags the CLI does not have', async ({ page }) => { @@ -83,8 +98,12 @@ test.describe('Nodes join command', () => { // command carrying them fails at kong before LocalAI does anything. await expect(card).not.toContainText('--distributed-nats') await expect(card).not.toContainText('--distributed-db') - // And the worker step no longer tells an operator to point a backend - // worker at a bus it does not dial. + // Neither step tells an operator to point anything at a bus. The FRONTEND + // command is asserted first and by itself: it is the one that used to carry + // --nats-url as a required flag, so an operator following this card would + // have stood a broker up before starting LocalAI at all. + await expect(card.locator('.p2p-cmd pre').nth(0)).toContainText('--auth-database-url') + await expect(card.locator('.p2p-cmd pre').nth(0)).not.toContainText('--nats-url') await expect(card.locator('.p2p-cmd pre').nth(1)).not.toContainText('--nats-url') }) }) diff --git a/core/http/react-ui/src/pages/Nodes.jsx b/core/http/react-ui/src/pages/Nodes.jsx index ec2209cef1ea..0d79a9c753fe 100644 --- a/core/http/react-ui/src/pages/Nodes.jsx +++ b/core/http/react-ui/src/pages/Nodes.jsx @@ -27,8 +27,6 @@ function WorkerHintCard({ addToast, nodeType = 'backend', hasWorkers }) { const { selected, setSelected, option, dev, setDev } = useImageSelector('cpu') const isAgent = nodeType === 'agent' const workerCmd = isAgent ? 'agent-worker' : 'worker' - const natsFlag = isAgent ? ' --nats-url "nats://nats:4222" \\\n' : '' - const natsEnv = isAgent ? ' -e LOCALAI_NATS_URL="nats://nats:4222" \\\n' : '' const flags = dockerFlags(option) const flagsString = flags ? `${flags} \ ` : '' @@ -41,10 +39,10 @@ function WorkerHintCard({ addToast, nodeType = 'backend', hasWorkers }) {

CLI

+ --registration-token "$LOCALAI_REGISTRATION_TOKEN"`} addToast={addToast} />

Docker

@@ -58,7 +56,7 @@ function ScaleOutCard({ addToast }) {

Distributed mode is not enabled

Distributed mode spreads models across worker machines and routes inference across the fleet. Start LocalAI with it enabled, then register a worker.

- +

See the Distributed Mode documentation for production setup.

) diff --git a/core/http/react-ui/src/utils/format.js b/core/http/react-ui/src/utils/format.js index 67e5a374f941..9756d6663650 100644 --- a/core/http/react-ui/src/utils/format.js +++ b/core/http/react-ui/src/utils/format.js @@ -16,7 +16,8 @@ export function percentColor(pct) { // milliseconds, regardless of its encoding. The agent SSE bridge emits the // json_message timestamp in three different shapes depending on deploy mode: // an RFC3339 string (standalone agent pool), Unix milliseconds (local -// dispatcher), or Unix nanoseconds (older NATS path). A numeric value is +// dispatcher), or Unix nanoseconds (releases before the tunnel migration, +// which a rolling upgrade still has in flight). A numeric value is // classified by magnitude (s / ms / us / ns) so any of them yields a sane // epoch. Falls back to Date.now() for null/empty/unparseable input. export function normalizeTimestampMs(ts) { diff --git a/docker-compose.distributed.yaml b/docker-compose.distributed.yaml index 91e4d83fc990..ac50dac1f720 100644 --- a/docker-compose.distributed.yaml +++ b/docker-compose.distributed.yaml @@ -1,7 +1,11 @@ # Docker Compose for LocalAI Distributed Mode # -# Starts a full distributed stack: PostgreSQL, NATS, a LocalAI frontend, -# and one llama-cpp backend node. +# Starts a full distributed stack: PostgreSQL, a LocalAI frontend, one +# llama-cpp backend node and one agent worker. +# +# There is no message broker in this file and none is needed. PostgreSQL carries +# every cross-replica broadcast, and each worker dials one outbound tunnel to the +# frontend and takes every verb on it. # # Model files are transferred from the frontend to backend nodes via HTTP # — no shared volumes needed between frontend and backends. @@ -28,13 +32,6 @@ services: timeout: 3s retries: 10 - nats: - image: nats:2-alpine - ports: - - "4222:4222" # Client connections - - "8222:8222" # HTTP monitoring (optional, useful for debugging) - command: ["--js", "-m", "8222"] # Enable JetStream + monitoring - # --- LocalAI Frontend --- # Stateless API server that routes requests to backend nodes. # Add more replicas behind a load balancer for HA. @@ -52,7 +49,6 @@ services: environment: # Distributed mode LOCALAI_DISTRIBUTED: "true" - LOCALAI_NATS_URL: "nats://nats:4222" LOCALAI_AGENT_POOL_EMBEDDING_MODEL: "granite-embedding-107m-multilingual" LOCALAI_AGENT_POOL_VECTOR_ENGINE: "postgres" LOCALAI_AGENT_POOL_DATABASE_URL: "postgresql://localai:localai@postgres:5432/localai?sslmode=disable" @@ -68,7 +64,7 @@ services: # Force pure-Go DNS resolver. The default cgo resolver follows the # container's nsswitch.conf and ends up forwarding to host # systemd-resolved (127.0.0.53), which isn't reachable from inside - # the container — failing every postgres/nats hostname lookup at + # the container, failing every postgres hostname lookup at # boot. The pure-Go path reads /etc/resolv.conf directly and uses # Docker's embedded DNS at 127.0.0.11. GODEBUG: "netdns=go" @@ -83,13 +79,12 @@ services: depends_on: postgres: condition: service_healthy - nats: - condition: service_started # --- Worker Node --- # A generic worker that self-registers with the frontend. # The same LocalAI image is used — no separate image needed. - # The SmartRouter dynamically tells workers which backend to install via NATS. + # The SmartRouter tells a worker which backend to install over that worker's + # own tunnel. # # Model files are transferred from the frontend via HTTP file staging. # The worker has its own independent models volume. @@ -115,10 +110,9 @@ services: # tunnel session or when a backend it serves no longer answers locally, so # `unhealthy` means the frontend cannot reach this worker or its data path. # - # No LOCALAI_NATS_URL and no dependency on the nats service: a backend - # worker connects to no bus. Everything the frontend asks of it travels the - # tunnel this container dials out to localai:8080. The frontend and the - # agent worker below still need NATS. + # This worker connects to nothing but the frontend. Everything the frontend + # asks of it travels the tunnel this container dials out to localai:8080, so + # the only service it depends on is localai itself. environment: LOCALAI_SERVE_ADDR: "0.0.0.0:50051" DEBUG: "true" @@ -186,15 +180,15 @@ services: # LOCALAI_NODE_NAME really must differ: the registry upserts by name, so two # workers sharing one steal each other's row and each other's tunnel credential. # - # Workers are generic — no backend type needed. The SmartRouter - # will dynamically install the required backend via NATS when - # a model request arrives. + # Workers are generic: no backend type needed. The SmartRouter installs the + # required backend over the worker's tunnel when a model request arrives. # --- Agent Worker --- # Dedicated process for agent chat execution. - # Receives chat jobs from NATS, runs cogito LLM calls via the LocalAI API, - # and publishes results back via NATS for SSE delivery. - # No database access needed — config and skills are sent in the NATS payload. + # The frontend claims a queued run from PostgreSQL and drives it as a + # streaming control RPC over this container's own outbound tunnel; progress, + # agent events and the terminal result all come back on that same response. + # No database access needed: config and skills are sent in the request. agent-worker-1: # image: localai/localai:latest-cpu @@ -212,11 +206,10 @@ services: - | apt-get update -qq && apt-get install -y -qq docker.io >/dev/null 2>&1 exec /entrypoint.sh agent-worker - # The agent worker is NATS-only — no HTTP server to probe. The image's - # healthcheck detects that mode and reports healthy rather than probing a - # port that will never bind, so no override is needed here. + # The agent worker binds its control server on loopback only and publishes + # no port. The image's healthcheck detects that mode and reports healthy + # rather than probing a port that will never bind, so no override is needed. environment: - LOCALAI_NATS_URL: "nats://nats:4222" LOCALAI_REGISTER_TO: "http://localai:8080" LOCALAI_NODE_NAME: "agent-worker-1" LOCALAI_REGISTRATION_TOKEN: "changeme" # Must match frontend token @@ -226,8 +219,6 @@ services: depends_on: localai: condition: service_started - nats: - condition: service_started volumes: postgres_data: diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index bda4ebf1c2be..ee4bdab330da 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -13,7 +13,7 @@ Distributed mode requires authentication enabled with a **PostgreSQL** database ## Architecture Overview -![Distributed mode architecture: a load balancer fronts stateless SmartRouter frontends backed by a shared NATS/PostgreSQL/S3 plane, with generic workers running per-model gRPC backends](/images/diagrams/distributed-mode-arch.png) +![Distributed mode architecture: a load balancer fronts stateless SmartRouter frontends backed by a shared PostgreSQL/S3 plane, with generic workers reached over the tunnel each one dials out and running per-model gRPC backends](/images/diagrams/distributed-mode-arch.png) **Frontends** are stateless LocalAI instances that receive API requests and route them to worker nodes via the **SmartRouter**. All frontends share state through PostgreSQL, which also carries every cross-replica event they broadcast. @@ -38,10 +38,10 @@ Each model gets its own gRPC backend process, so a single worker can serve multi ## Prerequisites - **PostgreSQL** (with pgvector extension recommended for RAG) - used for node registry, job store, auth, and shared state - - Each frontend replica holds **one extra PostgreSQL session** beyond its connection pool, pinned for the life of the process, and creates a `bus_messages` table. Both belong to the broadcast carrier that is replacing NATS for cross-replica fan-out; it already carries the four `state.*.delta` families (see [Cross-replica in-memory state](#cross-replica-in-memory-state)). Size `max_connections` for one additional session per frontend replica. + - Each frontend replica holds **one extra PostgreSQL session** beyond its connection pool, pinned for the life of the process, and creates a `bus_messages` table. Both belong to the broadcast carrier that carries every cross-replica fan-out, including the four `state.*.delta` families (see [Cross-replica in-memory state](#cross-replica-in-memory-state)). Size `max_connections` for one additional session per frontend replica. - That session reports an `application_name` of `localai_pgbus_`, so `SELECT count(*) FROM pg_stat_activity WHERE application_name LIKE 'localai_pgbus_%'` counts the replicas currently listening. If the carrier loses its session it redials and re-registers on its own; a broadcast published while it was down is not replayed, which is why nothing that must survive a gap is carried by a broadcast alone. - `bus_messages` holds only broadcasts too large for a PostgreSQL notification, and every replica retires rows older than ten minutes. The table is a spill buffer, not a log: it is not a place to read past events from. -- **No message bus.** Nothing in a distributed deployment connects to NATS any more. Everything a frontend broadcasts travels on the PostgreSQL the deployment already runs; every verb a frontend addresses to a worker, including an agent cancel, is an HTTP route on that worker's own tunnel. `LOCALAI_NATS_URL` is accepted and ignored everywhere - on the frontend, on `local-ai worker` and on `local-ai agent-worker` - so an existing command line still starts. +- **No message broker. Do not deploy one.** Everything a frontend broadcasts travels on the PostgreSQL the deployment already runs; every verb a frontend addresses to a worker, including an agent cancel, is an HTTP route on that worker's own tunnel. If you are upgrading from a release that ran one, see [Migrating off the message broker](#migrating-off-the-message-broker): your existing command lines keep working and the broker can be shut down. - All services must be on the same network (or reachable via configured URLs) ## Quick Start with Docker Compose @@ -52,7 +52,7 @@ The easiest way to try distributed mode locally is with the provided Docker Comp docker compose -f docker-compose.distributed.yaml up ``` -This starts PostgreSQL, a LocalAI frontend, and one worker node. The compose file still stands a NATS container up; nothing connects to it and you may delete that service. When you send an inference request, the SmartRouter automatically installs the needed backend on the worker and loads the model. See the file for details on adding GPU support, shared volumes, and additional workers. +This starts PostgreSQL, a LocalAI frontend, one worker node and one agent worker. Those four services are the whole deployment: there is no broker in the file and none to add. When you send an inference request, the SmartRouter automatically installs the needed backend on the worker and loads the model. See the file for details on adding GPU support, shared volumes, and additional workers. {{% notice tip %}} Use `docker-compose.distributed.yaml` for quick local testing. For production, deploy PostgreSQL as a managed service and run frontends/workers on separate hosts. There is no message bus to deploy. @@ -66,11 +66,10 @@ The frontend is a standard LocalAI instance with distributed mode enabled. These |------|---------|---------|-------------| | `--distributed` | `LOCALAI_DISTRIBUTED` | `false` | Enable distributed mode | | `--instance-id` | `LOCALAI_INSTANCE_ID` | auto UUID | Unique instance ID for this frontend | -| `--nats-url` | `LOCALAI_NATS_URL` | *(ignored)* | **Accepted and ignored.** A frontend opens no message-bus connection. Kept so an existing command line still starts. | | `--distributed-advertise-addr` | `LOCALAI_DISTRIBUTED_ADVERTISE_ADDR` | *(derived)* | `host:port` the **other frontend replicas** dial to reach this one. See [Replica peer links](#replica-peer-links). | | `--registration-token` | `LOCALAI_REGISTRATION_TOKEN` | *(empty)* | Token that workers must provide to register | | `--registration-require-auth` | `LOCALAI_REGISTRATION_REQUIRE_AUTH` | `false` | Fail startup when distributed mode is enabled but the registration token is empty (node endpoints and worker file-transfer would otherwise be unauthenticated) | -| `--distributed-require-auth` | `LOCALAI_DISTRIBUTED_REQUIRE_AUTH` | `false` | **Umbrella switch.** Implies `--registration-require-auth`, which is what guards registration, the worker control planes and file transfer. It also implies the inert `--nats-require-auth`. Set this in production instead of the granular flags. | +| `--distributed-require-auth` | `LOCALAI_DISTRIBUTED_REQUIRE_AUTH` | `false` | **Umbrella switch.** Implies `--registration-require-auth`, which is what guards registration, the worker control planes and file transfer. Set this in production instead of the granular flags. | | `--auto-approve-nodes` | `LOCALAI_AUTO_APPROVE_NODES` | `false` | Auto-approve new worker nodes (skip admin approval) | | `--distributed-shared-models` | `LOCALAI_DISTRIBUTED_SHARED_MODELS` | `false` | Assert that every node mounts the **same** models directory at the **same** path (a shared volume). When `true`, the router skips file staging entirely and workers load models directly from the shared path instead of re-downloading them. See [Shared models directory](#shared-models-directory). | | `--distributed-disk-headroom-check` | `LOCALAI_DISTRIBUTED_DISK_HEADROOM_CHECK` | `true` | Reject worker nodes that lack free space to store the model, at scheduling time rather than partway through staging. When `false`, node selection ignores free disk; the check still runs and warns when it would have rejected every node. Also toggleable at runtime via the `distributed_disk_headroom_check` setting. See [Disk headroom](#disk-headroom). | @@ -85,6 +84,8 @@ The frontend is a standard LocalAI instance with distributed mode enabled. These | `--worker-reconnect-grace` | `LOCALAI_WORKER_RECONNECT_GRACE` | `90s` | How long a worker whose tunnel was lost is treated as **reconnecting** rather than **gone**. Only after this window may the scheduler stop placing work on that worker, clean up its rows and release its models. Set it below the worker's own reconnect worst case and you will condemn workers that are re-homing exactly as designed. Measured on the database clock, so every replica agrees. See [A lost tunnel is a departure, not an absence](#a-lost-tunnel-is-a-departure-not-an-absence). | | `--expose-node-header` | `LOCALAI_EXPOSE_NODE_HEADER` | `false` | When enabled, inference responses carry an `X-LocalAI-Node` header with the ID of the worker node that served the request. Coverage spans the OpenAI-compatible endpoints (chat completions, completions, embeddings, audio transcriptions, audio speech / TTS, image generations, image inpainting), the Jina rerank endpoint (`/v1/rerank`), the VAD endpoints (`/v1/vad`, `/vad`), and the Anthropic Messages (`/v1/messages`) and Ollama (`/api/chat`, `/api/generate`, `/api/embed`) shims. Useful for debugging, observability and load-balancer attribution. Off by default: the node ID reveals internal cluster topology and should not be exposed on a public endpoint. Best-effort: under heavy concurrency for the same model across multiple replicas, the header may reflect a recent routing decision rather than this exact request's. Acceptable for observability and debugging. | +The three `LOCALAI_NATS_*_TIMEOUT` names above are **control-RPC budgets, not broker settings**, and are still read and enforced. They carry that prefix only because they were introduced alongside the message bus that distributed mode used to require; renaming them would break every existing deployment for cosmetics. Do not delete them when you [shut the broker down](#migrating-off-the-message-broker). + ### Replica peer links Frontend replicas record themselves in an `instances` table and open direct links to each other, so that a request arriving at one replica can be served by state another replica holds. Each replica publishes one address for this, and every other replica dials it: it is the address **peers** use, which is not necessarily the address the process binds. A replica behind a Kubernetes Service, a load balancer or a NAT binds one and is reached at another. @@ -127,7 +128,7 @@ The peer link is served at `/api/cluster/peer` and authenticates with `LOCALAI_R Several features keep state in a frontend's process memory and surface it over the API: fine-tune jobs, quantization jobs, agent tasks and Open Responses metadata. A round-robin load balancer sends a follow-up request to any replica, so each of those maps is kept current on every replica by a broadcast. -**Those four families travel on PostgreSQL, not on NATS.** Each mutation is a `NOTIFY` on the database the deployment already runs, and each replica holds one `LISTEN` session for it. There is nothing to configure: the carrier uses the same database URL as `--auth-database-url` / `LOCALAI_AUTH_DATABASE_URL`. +**Those four families travel on PostgreSQL.** Each mutation is a `NOTIFY` on the database the deployment already runs, and each replica holds one `LISTEN` session for it. There is nothing to configure: the carrier uses the same database URL as `--auth-database-url` / `LOCALAI_AUTH_DATABASE_URL`. | Map | Subject | |-----|---------| @@ -143,7 +144,7 @@ for the life of a request: which gallery operations are in flight and how far along they are, which admin operations have been admitted, which model files are being staged onto a worker, and which replica already holds the KV/prefix cache for a prompt. Each of those is kept current on every replica by a broadcast, and -**every one of them is on PostgreSQL**. Nothing in this table uses NATS. +**every one of them is on PostgreSQL**. | Family | Subject | What a peer does with it | |--------|---------|--------------------------| @@ -480,7 +481,7 @@ Registering against an upgraded frontend **clears** a node's `address` and `http A worker on this release opens **no inbound listener on a routable interface**. Its backend gRPC processes and its HTTP file-transfer server all bind loopback, and the frontend reaches both through the tunnel. Concretely: -- **No inbound firewall rule, published port, Service or Ingress is needed for a worker.** A serve-backend worker needs outbound access to the frontend URL (`LOCALAI_REGISTER_TO`), and nothing else - not even to NATS. An agent worker binds only loopback too, and needs outbound access to `LOCALAI_REGISTER_TO` and nothing else either: registration, heartbeats, its tunnel and every verb the frontend addresses to it all go there. +- **No inbound firewall rule, published port, Service or Ingress is needed for a worker.** A serve-backend worker needs outbound access to the frontend URL (`LOCALAI_REGISTER_TO`), and nothing else. An agent worker binds only loopback too, and needs outbound access to `LOCALAI_REGISTER_TO` and nothing else either: registration, heartbeats, its tunnel and every verb the frontend addresses to it all go there. - **`LOCALAI_ADVERTISE_ADDR` and `LOCALAI_ADVERTISE_HTTP_ADDR` are gone.** There is nothing to advertise. Both are ignored if still set; remove them. - **`LOCALAI_ADDR` and `LOCALAI_SERVE_ADDR` are read for their port only.** The port is the base of the backend port range, and `port-1` is the HTTP file-transfer port. The host half names an interface nothing binds. - The node's `address` and `http_address` fields in `GET /api/nodes` are empty, and are cleared for nodes that reported them before the upgrade. @@ -598,22 +599,26 @@ The chat UI renders this state inline and retries automatically once the model r A frontend replica that dies mid-load does not wedge the model: the job row carries a heartbeat and another replica reclaims a job whose heartbeat has stopped. The heartbeat is time-based, not byte-based, because a checkpoint load legitimately transfers zero bytes for many minutes. {{% /notice %}} -### NATS credentials (inert) +### Migrating off the message broker + +Earlier releases of distributed mode required a NATS cluster alongside PostgreSQL. **They no longer do. Shut the broker down.** Nothing in LocalAI opens a connection to one: the frontend's cross-replica fan-out is on PostgreSQL, queued work is a claim on a PostgreSQL table, a serve-backend worker takes every verb on its own tunnel, and an agent worker does too, including the cancel that was the last family on a bus. -**No LocalAI component connects to NATS.** The frontend's cross-replica fan-out is on PostgreSQL, a serve-backend worker takes every verb on its own tunnel, and an agent worker now does too, including the cancel that was the last family on a bus. +There is no migration step and no cutover window. Stop the broker, delete its service from your compose file, chart or unit files, and delete the credentials you generated for it. A deployment that keeps running one is paying for infrastructure that carries nothing. -Every `LOCALAI_NATS_*` setting is therefore accepted and inert, so an existing command line, unit file or Helm values file starts unchanged: +**Your existing command lines still start.** Every `LOCALAI_NATS_*` setting below is parsed and then ignored, so an unedited command line, unit file or Helm values file needs no change on the day you upgrade. Remove them at your convenience. | Flag | Env Var | Status | |------|---------|--------| -| `--nats-url` | `LOCALAI_NATS_URL` | Accepted and ignored on the frontend, `local-ai worker` and `local-ai agent-worker`. | +| `--nats-url` | `LOCALAI_NATS_URL` | Accepted and ignored on the frontend, `local-ai worker` and `local-ai agent-worker`. The value is never dialled, so it may point at a broker that is already gone. | | `--nats-account-seed` | `LOCALAI_NATS_ACCOUNT_SEED` | The frontend still mints a per-node user JWT at registration (`nats_jwt` in the register response). Nothing consumes it. | | `--nats-service-jwt` / `--nats-service-seed` | `LOCALAI_NATS_SERVICE_JWT` / `LOCALAI_NATS_SERVICE_SEED` | Accepted, unused: the frontend opens no bus connection to present them on. | | `--nats-worker-jwt-ttl` | `LOCALAI_NATS_WORKER_JWT_TTL` | Lifetime of the minted-but-unused worker JWTs. | -| `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | On an agent worker this still makes registration **wait through admin approval** rather than starting against a pending node. It no longer gates any bus connection. | +| `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | On an agent worker this still makes registration **wait through admin approval** rather than starting against a pending node. It gates no connection. | | `--nats-tls-ca` / `--nats-tls-cert` / `--nats-tls-key` | `LOCALAI_NATS_TLS_*` | Accepted, unused. | -You may stop running a NATS server, and remove these settings at your convenience. +{{% notice warning %}} +`LOCALAI_NATS_BACKEND_INSTALL_TIMEOUT`, `LOCALAI_NATS_BACKEND_UPGRADE_TIMEOUT` and `LOCALAI_NATS_MODEL_LOAD_TIMEOUT` are **not** in the table above and must **not** be removed. Despite their names they were never broker settings: each one is a control-RPC budget the frontend applies to a worker, and each is still read and still enforced. They are documented with the other frontend flags in [Frontend Configuration](#frontend-configuration). The names are kept because renaming them would break every existing deployment for cosmetics. +{{% /notice %}} {{% notice note %}} `LOCALAI_AUTH` (HTTP users/sessions) is unrelated. HTTP registration still uses `LOCALAI_REGISTRATION_TOKEN`, and every worker control plane sits behind that same bearer check. @@ -657,7 +662,7 @@ absolute snapshot path and skip transfer. Otherwise, the controller stages the complete snapshot tree to each worker before loading the backend. With an object store configured the controller uploads to the bucket and commands the worker to fetch over its tunnel; without one it pushes the files to the worker's HTTP file -transfer server directly. Neither path uses NATS. +transfer server directly. {{% notice warning %}} Every controller and worker must have enough disk space for its own snapshot @@ -715,7 +720,7 @@ local-ai worker \ --registration-token changeme ``` -There is no `--nats-url` here. A serve-backend worker connects to no message bus: it dials one outbound tunnel to `--register-to` and serves every request the frontend makes of it over that. The flag is still accepted and ignored, so an existing command line keeps working. +There is no broker flag here. A serve-backend worker dials one outbound tunnel to `--register-to` and serves every request the frontend makes of it over that. A `--nats-url` left over from an older command line is still accepted and ignored; see [Migrating off the message broker](#migrating-off-the-message-broker). | Flag | Env Var | Default | Description | |------|---------|---------|-------------| @@ -732,7 +737,6 @@ There is no `--nats-url` here. A serve-backend worker connects to no message bus | `--distributed-require-auth` | `LOCALAI_DISTRIBUTED_REQUIRE_AUTH` | `false` | Umbrella switch implying `--registration-require-auth` | | `--heartbeat-interval` | `LOCALAI_HEARTBEAT_INTERVAL` | `10s` | Interval between heartbeat pings | | `--worker-tunnel` | `LOCALAI_WORKER_TUNNEL` | `true` | Hold one outbound multiplexed tunnel to the frontend and serve its requests over it, so this worker needs no inbound port (see [Worker tunnels](#worker-tunnels)). Setting it to `false` is a **fatal startup error**, not a degraded mode: the frontend has no path that dials a worker's advertised address, so a worker without its tunnel is a worker nothing can reach. To run without tunnels, run the pre-tunnel release on both the worker and the frontend. | -| `--nats-url` | `LOCALAI_NATS_URL` | *(ignored)* | **Accepted and ignored.** A serve-backend worker opens no NATS connection. Kept so an existing worker command line still starts. | | `--backends-path` | `LOCALAI_BACKENDS_PATH` | `./backends` | Path to backend binaries | | `--models-path` | `LOCALAI_MODELS_PATH` | `./models` | Path to model files | | `--vram-budget` | `LOCALAI_VRAM_BUDGET` | *(empty)* | Cap the VRAM this node advertises for model placement, as a percentage (e.g. `80%`) or an absolute amount (e.g. `12GB`). Empty uses all detected VRAM. See [Per-node VRAM budget](#per-node-vram-budget). | @@ -1384,7 +1388,7 @@ engine_args: The ds4 backend (DeepSeek V4 Flash) supports **layer-parallel** distributed inference: a single model that is too large for one machine is split by transformer layer across several machines. Each machine must have the GGUF present locally, but loads **only its own slice** of the layers. This lets you run a model whose weights exceed any single host's memory. -This is **not** routed through the SmartRouter: it is a model-internal split, configured manually (Phase 1). It is unrelated to the NATS/PostgreSQL distributed mode described above. +This is **not** routed through the SmartRouter: it is a model-internal split, configured manually (Phase 1). It is unrelated to the PostgreSQL-backed distributed mode described above. ### Topology @@ -1682,8 +1686,8 @@ Notes: - Check that `--registration-token` matches on both frontend and worker - Ensure auth is enabled on the frontend (`LOCALAI_AUTH=true`) -**NATS connection errors:** -- Nothing in LocalAI connects to NATS any more, on any component. If a release you are running still logs one, it predates the tunnel migration; on this release, look at the failing component's tunnel and its `--register-to` instead. +**Message-broker connection errors:** +- Nothing in LocalAI connects to a broker any more, on any component. A release that logs such an error predates the tunnel migration; on this release, look at the failing component's tunnel and its `--register-to` instead. See [Migrating off the message broker](#migrating-off-the-message-broker). **PostgreSQL connection errors:** - Verify the connection URL format: `postgresql://user:password@host:5432/dbname?sslmode=disable` @@ -1747,7 +1751,7 @@ Notes: - The HTTP file transfer server runs on the base port - 1 (default: 50050) - All of those bind loopback, so a firewall cannot be the cause. What can is another service on the same host already holding a port in the range: move the worker's range with `LOCALAI_ADDR` (see [Worker Port Configuration](#worker-port-configuration)) or bound it with `LOCALAI_GRPC_MAX_PORT` - Verify the backend gallery configuration is correct -- The worker needs OUTBOUND network access to the gallery and to `LOCALAI_REGISTER_TO`. It needs no inbound access at all, and no access to NATS +- The worker needs OUTBOUND network access to the gallery and to `LOCALAI_REGISTER_TO`. It needs no inbound access at all ## Routing pipeline diff --git a/docs/content/features/distributed_inferencing.md b/docs/content/features/distributed_inferencing.md index f30b97a6035d..c14c01072d78 100644 --- a/docs/content/features/distributed_inferencing.md +++ b/docs/content/features/distributed_inferencing.md @@ -9,7 +9,7 @@ aliases = ["/features/distribution/"] ![Federated vs worker mode: federated routes a whole request to one node; worker shards one model across nodes](/images/diagrams/federated-vs-worker.png) {{% notice tip %}} -Looking for production-grade horizontal scaling with PostgreSQL and NATS? See [Distributed Mode]({{% relref "features/distributed-mode" %}}). +Looking for production-grade horizontal scaling backed by PostgreSQL alone? See [Distributed Mode]({{% relref "features/distributed-mode" %}}). {{% /notice %}} ## Choosing a distributed mode @@ -19,7 +19,7 @@ LocalAI can spread inference across multiple machines in three ways. Pick the on | Mode | Best for | Guide | |------|----------|-------| | **P2P / Federated inference** | Ad-hoc clusters, community sharing, quick experimentation. Nodes discover each other via a shared libp2p token, with no central server. | This page | -| **Distributed Mode (PostgreSQL + NATS)** | Production deployments, Kubernetes, and managed infrastructure. Stateless frontends behind a load balancer, workers self-register, and state lives in PostgreSQL. | [Distributed Mode]({{% relref "features/distributed-mode" %}}) | +| **Distributed Mode (PostgreSQL)** | Production deployments, Kubernetes, and managed infrastructure. Stateless frontends behind a load balancer, workers self-register over an outbound tunnel, and state lives in PostgreSQL. No message broker is needed. | [Distributed Mode]({{% relref "features/distributed-mode" %}}) | | **MLX distributed** | Apple Silicon clusters running MLX models over the MLX distributed runtime. | [MLX Distributed]({{% relref "features/mlx-distributed" %}}) | For the low-level protocol and endpoints used by P2P workers, see the [P2P API reference]({{% relref "reference/p2p-api" %}}). diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md index 490877e21b1d..ee3f980a2248 100644 --- a/docs/content/features/text-generation.md +++ b/docs/content/features/text-generation.md @@ -393,8 +393,9 @@ replicas, so retrieval, `previous_response_id` chaining and cancellation work regardless of which replica the load balancer picks: - `GET /v1/responses/{id}` returns the response from any replica. -- `POST /v1/responses/{id}/cancel` is delegated over NATS to the replica that is - actually generating, so generation really stops. If that replica is gone, the +- `POST /v1/responses/{id}/cancel` is delegated over the PostgreSQL broadcast + carrier to the replica that is actually generating, so generation really + stops. If that replica is gone, the response is reported as `cancelled` without blocking. - **Streaming resume (`?stream=true`) is served only by the replica that created the response.** The event buffer lives in that process's memory and is not diff --git a/docs/static/images/diagrams/distributed-mode-arch.html b/docs/static/images/diagrams/distributed-mode-arch.html index f1e49b7087da..39758350de2e 100644 --- a/docs/static/images/diagrams/distributed-mode-arch.html +++ b/docs/static/images/diagrams/distributed-mode-arch.html @@ -54,7 +54,7 @@

One control plane, many workers

-
Stateless frontends, a shared NATS/Postgres plane, and generic workers running per-model backends.
+
Stateless frontends, a shared PostgreSQL plane, and generic workers reached over the tunnel each one dials out.
localai.io/features/distributed-mode
@@ -99,15 +99,17 @@

One control plane, many workers

// ---------- STATE PLANE ---------- txt(560,30,"SHARED STATE PLANE",{w:700,sz:13,ls:".2em",fill:SOFT}); -const SPX=560, SPW=300, SPY=46, SPH=470; +const SPX=560, SPW=300, SPY=46, SPH=326; shadowRect(SPX,SPY,SPW,SPH,PAPER,INK,4); svg.appendChild(el("rect",{x:SPX,y:SPY,width:SPW,height:58,fill:RUST})); svg.appendChild(el("line",{x1:SPX,y1:SPY+58,x2:SPX+SPW,y2:SPY+58,stroke:INK,"stroke-width":4})); txt(SPX+22,SPY+38,"Control plane",{f:"Bricolage Grotesque",w:800,sz:26,fill:PAPER}); // chips +// No broker chip. Every cross-replica broadcast and every queued job is a row +// or a NOTIFY on the database below, and no worker connects to this plane at +// all: a worker is reached over the tunnel it dials out to a frontend. const chips=[ - {n:"PostgreSQL", s:"shared config & state"}, - {n:"NATS", s:"jobs · messaging bus"}, + {n:"PostgreSQL", s:"state · registry · broadcast · jobs"}, {n:"S3 (optional)", s:"model & artifact store"}, ]; const CHX=SPX+24, CHW=SPW-48, CHH=104; let cy=SPY+82; @@ -151,14 +153,15 @@

One control plane, many workers

arrow(FX+FW, y+FH/2, SPX, ty, INK); }); -// NATS messaging bus -> workers (dashed). Workers coordinate via NATS; -// PostgreSQL is the frontends' shared state, not something workers connect to. -const natsY = SPY+82+CHH+18 + CHH/2; // NATS chip center y -wY.forEach((y)=> arrow(SPX+SPW, natsY, WX, y+WH/2, RUSTD, "2 8")); -// label the NATS bus arrows -const labW=140, labH=26, labX=(SPX+SPW+WX)/2-labW/2, labY=natsY-46; +// frontends -> workers over the tunnel (dashed), routed in the clear band BELOW +// the state plane. The route is the point: control verbs go frontend to worker +// and never touch the plane, and the worker opened the connection outward, so +// nothing dials into a worker and no port is published on one. +const tunY = 430; +arrow(FX+FW, tunY, WX, tunY, RUSTD, "2 8"); +const labW=250, labH=26, labX=(FX+FW+WX)/2-labW/2, labY=tunY-40; svg.appendChild(el("rect",{x:labX,y:labY,width:labW,height:labH,fill:PAPER,stroke:RUSTD,"stroke-width":2})); -txt(labX+labW/2,labY+18,"backend.install",{f:"Bricolage Grotesque",w:700,sz:14,a:"middle",fill:RUSTD}); +txt(labX+labW/2,labY+18,"backend.install \u00b7 worker-dialled tunnel",{f:"Bricolage Grotesque",w:700,sz:14,a:"middle",fill:RUSTD}); // ---- annotated arrow: frontend -> worker : LoadModel (gRPC) ---- arrow(FX+FW, fY[2]+FH-24, WX, wY[2]+WH-30, COLD, "4 7"); diff --git a/docs/static/images/diagrams/distributed-mode-arch.png b/docs/static/images/diagrams/distributed-mode-arch.png index 52117732ed129a8e4bb804e706c9cfd9f585ebd9..e4b1d530d6ac8be68a875507ca12ead101c99aaf 100644 GIT binary patch literal 287200 zcmeFZbwJeH*Ds7HsD!9U2m*ov(o)i(ARwJX4Bg#5z<^TH(#=T6(9IA7NJ!_<-7pN@ zdFMRObAIQX-@Wg9?;C$U-+zG3ci4Ncz1C-aR&0V5q_ax&%zt~{ZLt0AC-k@Sd@ovXRD*(-S`ABxtV_=f zJ)6j9kaLr|d!45so|=h<@}~K4)QPQishiRN_E&*3r^@$m|1m-8XN>1vniO{&m(wM!mF z3cA3D{}1i{-_QbJ*%MZ8xcL*4_#dm~4aWHIfxk)Q{}*|5=W*a1LMEt@&7_U$Xv;wP6Wg+$qDpM2qLMHNOvo4S`>9*YZKRL^`?HqKqD^A zVlbnPzI!4+jiBR5(Qsvq3*)d)IHKd>nW@@Ne-R^;ePT?*pHYLbBqBIOHig?#v`UTX zvg_yI0F>J`=>=)pE25RY!iKY9;r9GNR!0Pm7S3p>lV$P~J%BiUSKeoWj9>n9_iejG zK4y(rW;d~GAg?1fql11c$7{lB>q?kwKjgJ38ofxiw~JuZ}8yPti=baZu&iY)$NUZ1?eJk&znYjO{LJOFD-<;T4?`k>LJv zbZJnW($@3nHX4}Ty)AKd9ouTH&p_1BfstRDh6zvVj^O^I22c8X|5C?57EFy_;nlWl z$c4rjgoBPtn;x6{wQ<@eo66~+fQUwgk63zj(96?M+7y4fWB-fM65xB=#+du{@jH-O z_)Y6>F_G&ibjGJ+gjV!{IUQBPWge71m+vLHWuNZB&7HU6oJ*$ObU%rplSJaOlAW8- zbQTOm_}MsgR^F5 zvK(UVLlF4{%I?nY+`n91N61OHFcl*x$9`(NA%WaqrA^^-F6AsNc+=#`MdSuGc$je6 z3=-u!Wuf%Udtj#jjmP+Xl}moEce=DE*(;s8?iW>jM)$AMClqfEMi;!iSNAi8uZBZ+ zQ|Cv&jg%Xn%*kx?mjpq$Hu^m~Phd;CUd=@=R=Roz5^6DujNAD>jb73~NS84|W8Es! z!9|R`~xXk60@)vaX=$sF2xWNIcLdWT?xrasHa&QxFwp!CXQ)OS)r+Vy_ zHb7uUF;@uy{CMs|(Ea`8ZN*y7xuH2^ZaIHt-4NZS%v87S9K~7brQl3Gri=INMT=0| zzw`>G)oxy%r(3TLn0|6iAq@^K25pTlG&V}Hz2Go5 zEfyJ?!}rjH9?rgxn70c|O_j@;n|rUJ?q0@l=hA!M4SaO^0n0PXsj$|@V>^7AtiF;5 zrPF-)^^D+mx{p1(skMiJ!>~S%pk{4z)1{xY?la1@auRj3fE4Lo7@tz&nQb9Q$gRV& zvAX*0EoYCsYY5)&H+cVA?@xiM*Pnefpz)1mrWWZd_~0dnjFshIG+c6P9e@CjK(dCB zH#OUlkXyc`fQA0hgmPRUG@eZxUNde0(%8%{H+ZbXsATlJ8`@J)Moz+01o9s3%jR=> zaZJO7($>+Qzmao6`V`eM@)`W7*5@!QV%o}}nOxL1P{4hJD~GpNH?FF)Q^piB&S*#_ za^roDdqnbkOL^m!6(X7TDikKG94=sc+_(DM|6R432}>{YTt$o=#dC~^Vr?~hgKd9Oc5DBD2;4m< zIuU?s_BE?V8h$*9x_V9HwF%MO%>_Sn^l4rBIqD2|gpzDRoielBdXBhZ;rS2e zu{5{7RL&1|y=(;xz{dSNw;G;TNy=LLtrdQpcBHgC{;Il-j#p6+WY4RzI@fj=K)mOG zW{bOlUP99|AvM&eHCmBus;PPEvbOZxWRIKcE`@QIbYt!@Q?nP$I*r~ox_N0muKRwq zXMSWj3LC9-ki*&hn*fj{cwSA&$Eyt1 zCGbSHeyM`W);>`?-Q_M##7@Xj)1bD3I6;vc=9kDlWIbx@D0B$MD&H2eyM@3^PipZ>&&Om^#?)uOv{Ngj`lQ4;Xgc~osgKj(-u0xIG`#z#kM93V7Mu6) zG4uBRNT{rl5urmoo*3LNUVtz<4xDhBIqw=73)H!K>g;)E;5~F4ce`NO9xauZK^A1k zxvpB6(fhlkD=X=|<#aSCo;ftyiYk-Q7)u@dUKNwcUtH?a8`%+XV`a1^UCg8{9INK8 zs1(rO06yJPN~2GzErR9~+Zx=x9aFREu5{+f@>wpbN+EgWy~Kw~2qx*rdHht&pV3*{ zF+n{yqtlvVXikBs7*!!!_^~*!FmiWeibS0p?qC!th~Q4b;!Df&QTsJ;4Ev=nN}p30 zM;vn4*De3`p1c2FzWF!;gE)0eBVH3Y=^;9q?@{3G%%LpaHJCDMvLoQ>X`yN7HxiDh zCo)N+|J*T{^lG*jb1B%tNMOO1ljvo@tH8!0fTm=LWwc}@?pdT1MasCv)4BM1H<<(d zf}fnLYVJdisa#H_GScYBR*4hp^NeFY2o}j0Fe%j@Z`q~a30j$zciWz+cSLI0pB&WA z4uO+AhKnB6<8wW{7uRVkI_kS#N(K#u3P>trMlW|+JP>U4V=MUpeioAzF;-oeKk2ym zrDYhX1gm}{Ohgn%O!s0pULLXo^QXV!pJG_uk|h1td4Al>$Ht`B?B&EyOUc3tizBpE zj7%XW+efxWVA*s4*3_aVjXp)^CO5LeX$2X;+IwGPhn|^f+aD)h{!$0Pl9oiVWdS6WdN7wjtAIIt$Do0{jlwz~fN@kL5%jh1~2+f2elCY+N25FMBm4*uqGh}__UE?YgT%|pB_p$yG z4tC)7;wxx1EVtmuKe6upO9R}y@8@NY9QJ=FmJci1Sica8csX-NY@=#V1^-OMZ#945 z)aK8@ox{ZA(i7p|F(_qInw2AyBES!tyNw_k&`gM{e9dUna$io&Heg1hXB-Y%h?#jJ z`CQe4fDN=8&3->2SgAv@^TCeDNY}6_V<8a@ur8g^aUNe~T)57vw10`t=p=A$#3c0T zwiG>u%jv9xMRydtSpE)zI(}zw9NyG)1VykpIA&kf{(>!aD1grlA(aIqCCQ$>%xR<& z$u+Fkt8TcS5Xt}Inze$=>7EYNmd}%aSG|SfzYIEUX_Lz_*^sEoN$c~hRi_wOx(HVY z4}03tgQLXfQ-S-KB#wxJ@RE0mVLe3B-S;n$oV7ck*R!3h(#(1s&-z>ksbCqNdy=en z3706wkI${Tef7MpYYF2r;19DhE^~buqZR;NePE$FA|?C%%{}H}^Rg4c+UkPo%|1J& z{BmaRh0EzT6Ipo_+*|f@Mc-G;IkQfE=Ga4YmC87bfYU|#>^}l$aR`(dEqr26^WJ#= zMChA4jv6SX2%<6s@Z8{uTJtN+>rF^kDU#t!e@gFOG(L5Oe(FrF1YIGMp0IsN$#`Gs zY^6J)A)|3URoR9u)>^P>p<-JvY~y;}5(gxYhxs^&H{@Z)JMZ0_uCf3|DNTS8@#fX=8Y2d_^B5=al7h)Huy7xelQM&28v`e*M(VR9U1Y7en2 zs>p20fR{dH{2G_)mv)|xGEkeF4DK#O`T~FKtmTa+^zrGz!843NA+*v}l^Au)V`o`6 zMF#L_qA)eu@QVV|paQjwqXU90jLW~P^ZB|g;JWilwmprvT_Y8mD~9bYFCCc#&G~i| z)H1kIsUbdn-A)T(k#~zImR`=jCVKXs+CA`bD=dX7+1vDZaXU%9+52t;3EnPBvZY~A_*fGIo=MzH=*=)kh(ThmZbd-%RIHJDLhFU_ZYqnTwZRX&_vWJ zajsl{4p;0Wq!(Bdy!n@tc>m&MxWol#3OQM}OpYS6#$5aqj#{nSq-DRI1?+9@x0zpB zts~BIWkAntoP^|6;zhYf`mLIUzO3rShcN!B$fpk<6vkc?`;~WZZTLLlPw|$sCe}o{0A1oGzJd}rCblzAuW<345K zX7Xt0xP6BGU(lidh$FeL?MBwL()n4B1rXP!y0el$TC4TbR~tl)Gtfp0j8t@lR?{TZ zu)q=N%g-^esc2r>M@Z)&q>Y*+u@j4l)HH|RWg+z4M96%w;5nT%+C}W1D&zC=Sn{3y zWIQESIj?L+>QrJ}H&!W{ZK3Cv6EMCFi6@H9?cjizT>L{KVVd`bJpnwk4%bUVzKjk% z7^N{vdZRft;SjZz;0Jz;pt01@Y#HR>c+LCqWK@n8$1AefDa&~8SeL{5qmjg7k{>s> zYWtejs=g=yQk2v7CvYi6rZV2tX1ETX7UhJxl?vIPZyE@m58$ReCmEo4>ryYMKYJZh zDk3$9Wa<+UoVT$X6uMg;krOyfnu?%B|kel{9if&Hdi zG})u4N{J@Rfo%il7&rzWk=LSCSq|8|43z|M+sULw;Iz59(Z ze+qgLED^J;Y$6qmZcBJNm#Lf%ENYz>uuo^M0T&q55iAtS}UskT2pT2yv zFoAt3fq^TJBl{%+PtSIfRKLb#-MSz#k#qscYErCqchov?5xrlYWeCExIv1x$axq1$ zM%#Gg@O5?vH-&kT<8N){@JaF?UA_d_kO0z>s&79gFw}&=UJWNS&H|DwR#YDTG>o5sUi-!Vv=6~&TZcW zB>QD=mQDjY(#CtVwaqBFBrOqj^`o}sOgtvVa``ugOQ#Xj2 zTk7}K;TX)2+8T&a7ywyIl0X%)fb}*!eRUN&vlXM06CIPLOeye}BgjsY#dlSq%X`bC z!Auu3rO!1!LEdgZGBT3MLwZu2wdWndHZy9$trMeVX=ASihBn=fXRO8p^ALrIPeI#u zQL}CD#V~LOLcUN4NvV`AefhAxI5HTzG0T_RW$$OsJX7Uup|-(Lh-T8DbiHM&s27lT z(r2!+?}>eyN-SLm=Z^6LF1FGJiOF;f-Pt_r!F$`^9G4#ax}=g}lc6A_7PS6dyuWDj zyYF0ICqb<*xFBoxIeA5LC4A=OAWBFNkR~|J?u+OYahPtOGU$L>3dmIkf085maFtq4 zWw)?Uu0`fESI^~i%;Lv^GJBuwiECUq#;(K|HNh^Uata%%7;S?oc*Uz1DeQTff!PoE zF*Q&iFNb!NpgdUKR~Qw%HhzexU6Q?Zm`hD?C*hC{!vWgb z>+5Tvl~whb6yEL#V>wF=QIXnJZD)<>FPGL{ zqd98$fTmch{_WY$s#Ng(=H)=NW7ZL6EEI-4DD27!#J&b)OGDU=uDWI<7X*3_-kVHU z823&O@bP7&#TLZ>Ujl!%k#XCq0XCFNZ!JWl_`fm?>w zHC7lk=5pv_7!`@Ab!c8hW`5x2Mj-KXYgE;&-a@&{Tvp9urm}ewwfuk*m91lBOZ+eP zIZ{~giP^2Igr!fg=^4W>E-oypTT$J`i#74z$xfE8)wEh!Y&^wFPL;Ki7sYbwxyZI2 z6SZJa`*P26BaGv1AD5y~@3L{^zl7}ACi1zji(6PAKcECT@T}+Rjn^O}^z)7)sir!iLUJ04FO%uwd|+5nyx2X+;EFKB`|+?!H9P>uAe&Q20lj2{3Y5 zzpd34WN?OKAJ_ zoR~Q?>KW&2T%5A8mF^x2v%j`DE^zQ;3|ct-{TXq31KL9+j!xV#c{*@Eu02A>q;anc zLwD%9!joT4KqjKl=YA?^f9`lD|7U@qrNz?R+}y%qOR%G>s|%15hc=|PU!pmoiuv{R znx%G;M%6^K994(~OMJWP^~&j9hidFvJPE&o3{+6fvepQZj34QTOIU&Ds)O*XLNo|M5L_o%!9@j zd^3wR2S`=!P-t92zac z+TVCvH$nnNIg8Y6my)nvUE0`)#_C-OX;^KCSfUc}L~PejMxBM`%d0^*kq-qBPY+U@ zhlAZtXJ-{PtOyyKHQclNB3(8ju}tXY)Rfor^deo|n7n3L=w*zSziz)@N++V+UKL!l zS&o1mS)QGk9V?`IsRA{&0_7XbFJN*$y0ub-#AKsUxwSxCfClXfv9w=z&II4HrbPw0lF&&TO$6eRpG)11%-0GQ#Zm8b{{ zx}vU8o3||Pq-96W@H0?0cQf~!ax^r_buSK2PO7S_v+4MY-n(}j1bS0*lgFKl0BS(F zeooZHOMd_Z^Tjnf~GOTU811{f; zkq56wJZ9&R%mx@7?Vfv|j55u4ZvsDdn7`{mI#BofGY~9{%#^-gT`TI@b{2ZQeQeQm zny+kOmi|FHMrpQOt|qMLq=S%9h?`Cbol*Jyo$P8ztkMit&1ZBxDPYII0s_Hx3@D9k zbmVO!aV_YmoHnr7#j=VM(ebo+R)Vm1wsIPa(MH!ym+M5MT}zmN*?2&=H#m#>{o0|5 zErV|PzIwx1#`vM2mB(3V{zwV`+U1s?xy-|7=-R`M`5Q9@HJ>Kf(P&;e7gNQ;MxKsP zmx^JpcWow?p{~_8`hsJs4y3fT)-|2e)BIaBvADSS2y@_(27ekI@0U3-M<)06;{svK z`LMUxmrVB8FX5iFmRG%YJJUdFGcfA$1jKRUTB#@brG=!bg zyd4|@ta}-p>k7@=nOQr)d$6xQ?L|{}?$S5cxD7SqJ;2vq(|zze7enPNQbroOJe@?< z{edXiY>)npE4I}54`4-AZ528Gjn$wM`5$p@O5*8GgZ0n$eC}2cEwvz=5B<}@U3+_i zeAc9hxR`DyHdcYw6;ttm(%rFYH9|gtZrTIh9D0F%2Or+0alS00xhf~T$$u3OS`>%n zFl4fbe?`kfqgX}j$HCuGCsjZu3ySTaN#UI8k@#V+`~*WSbgY%IoB>^5UpHm99veqU zB?y~c-OebqIdkzJo8ry1VqF{MQeExLSOm;C!gZV6`g6CFZ4Gh%sAg@XvL76;Jb#`R zdS{+B;ITQ9v~`uh+N*J4r9*&_z+9%IOLaKy!&MkRVPDlzfjP!xjJhED>JPk07#>q` zn; zk8CBZRZ<5lTv%EJkhhr8k_wJf&lHBcSuL~4F=PFf{t9Qmzki7e^)&xxVVNmwCDW{b z^5q%B3yze2ikqi$xq%VNTG(Qqcb|XZsAq~)*(>T6(Yebesy*E2Kb?@F1NOIY~z^J^}}+k@!ZgJ>nTs#uFK z4H0H}>N`S;0Nr_GK=8i3j=u!OGolZynowK(w+GiCCc9}}8JGGDSx}NHX<;~VoCFEF zwi^-oii&D>E1>O2V84jFda544hms`Ug7vToGFo3x<~v9kJi8IfJ-B}2(gfLY2mhaT z+o?BKBR^bUlmrPVBzd>>@!X!Iw!AjAD61l#wM71y1lsq`&XW$$@^@I<%Ggq{c9{q| zS2rET_kWm7!9`8Ok|#=)cfa7*6T}HDf^Mfbf4|t`(5VZR<^jW;UvAj@wd!%s(F@uzd}cg5MI?1F-VX59ti zSq}1PChO|>F%B}|)!Q05Yh=tM*#<|gc-@nPl?q@{+s!w`ud181GSPqZ1G*Ds4L7?> zmIY$OPaGZ{;SvzUu!?4&Z3Wbn(L}YJ;jiXt*Ix5nn_{gl8**%8Lnv3gRtFSd{GdU$ zdPjWPwB-Abo}zt5x=L>>W)^Bv-u;skd|doL3=;+$Pu41SV2-KJXdTi0CIeH`)~?x% z$w}K7vz$6>q`JraCuE|(E&`6Rv*unDK5eZQMSG|Y2N0-JxB(-cE|J)OKJwou3nKQP z*2mP?e-Ja3ag&qmJ$Uuco)_H?^9s99`5BnRf^Fty1o{@{pLp?}{-Z5Mw_|hY+Ybz4 z!C&T?m(%wDY?nlf$B3L4J;5VCu`8K2pI`U>ZLzOrT9g7Fr5iZr`5_CHs8P!`Dbk6yGED6d+} z$n%zOlDxZ~!W{7<7S)(yFaK@z)3~@UxsU05u@v6(s_I!qJ!>7=;`*On6Jo}XM?V89 zUS3|`zI}VJ2&N0di0`*VIWaKmZNqSJbhYeHvnv`o?YD<%35|Z{j z(*H9t`p3(AOOy8_ZvFO2{lsnu$A*SBl)n~Ye9Z{^=kZd~1w9CpK*zq6hFoN*+(cPN zOpULYVpACy@g3Ssftc#W8;Vo9>IVPYw%GrRo+(vz6$_pK9F2X_m|<(4oMBU>69eF5 zmU?rQPJ@!)1$E{>`sem3B`Uo!C9)8YTHd@8&<7enD0AM)6r0V!Q2PVjux|7QPq}6l zr{-_}_kFtm>jkZ`n%k78WIk5IZMg}fyV{5FzxzEqB#A2;$N2g!GtiyH?NurE(eR-9 z-{g(qj+-S$z#ATTD2g@$u?y&H7e$E7Ru%9obp)~h-(GC-UKFSs650%99OIAzZUrQL-L*mvzXu=X4MXX174*#Ye?_E^x_4=_q zI+=n9AGOCoLoSmCHKNOJIkzgzPn!tlUDl2tExq!dYA$MPjIJmwJd|!45|@blE+?D* zgM&RjQ~G|3Imfh|Qo$vdMigz$$Nw~v`K$&U4NA*T<^{9Y*LqYJ%)fH=lfIn{E>xM_ z=%u^whQvO-klCtKXVW}fI@GHnCT(ca;qf@F$0gSGfDx`$7sXu!&=O~$$r@v^bRK@j z#{_G-tSQ?NX$co5o$vDc-{1VxmV;UsNm~xTU@^-qsr&edYF=*tNIqc`I{dy18HB`4 zx=nT~pZ5CPUi`k&Dg>p;K+135Y#qwBrIPQ#&Kzmki+U*77QlBE%2(Zt#dL~!h^^;r z%~w&#T$S}l@4*Fk!wF2)K%L$N&@vUe z?x?d-e7QCUNEmD~^@{8$WOQr1UVKBvP+Dd(S93l94}u>pixXZI`I#_i-<_CxmsZAF z&qUiL@r%n#vB*qqca*|Y5H5zRsR<$=BbrY0KNoL1&V& z{tbN$IFeDh4<0^z>{h9|*TYzwlFNS^NRJv-f8MVAp7GCI04mmL7i1>e)?W)xpQrUR z>7A*|T0K0FnhOkJ4$x~lo|POp=CRIMs5=j@re{qh8NITOzt$5*RyL~Lrt4gcGjuZ! z(?dVmIV9$f6vpP}!tfnk-h9kKY&#cz1R&vsA8o$ZuoUJ1wmSQo9b3yCb``An0`Gvy zT9?|4@6CU#{r>stPnnvHp{8bLC*L}m>cufSFF*KCES(+HuPPONQS_L2H@CFJIsa8o zXxQ+{)8fZcX`XWhGP~C>8Pi^%rgIA;3}2r@<7)f}+SAaGw>g!dcN0{pLf6^3=}*me zlQ2lkc59urlFE%txQ%w(7ME%g_r_~z){U_s@-8zi9bjzW}%D5Jg55sk2$W-yj?S`P)0x=atIALUK zmg=d0%0JOkx6QJ=rplRte-#~|sx^rC5|MJQoxebIrkzSrTMdLH?CKpvoAw>h4+wb= zT(9KGzjAnoNt7sf7xi#$*5#xGkkVj*V87VDBF^?dF5}=F6lQlt97dNwGxu_S#OB!F zOr)S$+3p5r8>>k*q-wlN)+#ct;TjB#oXEQM$hOYJ#*@7E#UlAKIH|6pW%Zs^j;`|9PQ^*Z{QGATV&dmynYj4);lY#)e{*v7 zoL#qmK%27t2Z(kj+d4;t&YtNctO1b-cUWr}$tgxG?x<|&G%mI$l^+`NT(%*SxSyzS zJF!9p#3xonVUrF~j7o-xc6oWJ^!Mm)gX!3@gKi^qRbU|+egU;PMIF)P4W_(=;wa_N zO0qI+J-^2^U6D}+21&C48m4#07DTU%JH6;&Pg(I8uC>9!54ck2Mro=k#| zggnG^Z4H-I*u`s)old4Zy-zu)U^=$G;8_zh5wGR$QSW*LX+~Uv0Ecl2Fv*t65eMP{ zlD10uq_e!ke$T29_MrE$7WJ+ieYl`KV(9RQ4iVW8G_p^S=G;dF{ZdknFu<;iNF`{&SG^ zubt>mzUNbI!Fh5ji&dmX-Mt}v#)gO?5Qb;|9Q3dO%&4)(vPQ)WG+l0^yMAU=;=kx6 zt1Ksqgit`Tk=0P$pK70Y3mgaYteke}Blul!S8Kx5_50epgAKJNULMx&FgBSlC+beL zS`G=6B*^Dfk?YrfVx?!a@bDcr*SpvIj>(t%yygMUvzBknaNDWd+aSno1-Npv_7e1~ zXK&(5LV}+6RkDb#WA`_wKiIFY!OE!sRfY3JW_^AAK{6AY;6Z!LU;KWNxr@@CvIOsW zwBX%^xXK0_7%AT+| zB*k;1@~*)o2Xk-L^3g8IX2@txSpv1Ex9JJ3ks+vF3pzh7xAup{Oy@VePT-8InjKtK zya(<6gS(~;sEAK5et1we6jVIp%IbAeoVMlZ4+W^HHJuOg%-`*gB1`L+4`tY%Vg!C2 zsR)69rzS^>H_~nd{^I7xe#04<@e;@bXBqvzS*!F!Pmig1K(B~)qoJId97JZ5r`daB zp?a^;efM*ab98fI3bxnRsAv4@0J)0TV!h_63@gL79S;VIre~62QYra5_47j>*efNF z3Z{*Yu+6my7VIF4d$2DVs(&7W(L>qLRE&7n_krC%zn5Zw6!l7dcDhMQo3>cpvP=2V z1>loD+*0kspcHB8o(C9qwITw4Bfz#R8sC2ZA}Wh`6qe*MfR9VUa$zzhGkCLhc+nL( zdcxlTUa?YO9w>2Z%t$$ovNt&Y9evlEM~>94d)711P53W8_njf7kbY{xO|WkAg_txQe2_kwmqrt%zi{3;huY{!p! zOB2|yrf@zNdB=@L4gJ2@G14fdF_YX83S=c7z5HG|D5hgX4#k?Lmh;|b%;ty8s?SY( z+{}|3OTf<$pP_)Q>^ln=*#5I9^d5_kL!DvYv&Rag_?KSNWqF^oX%yR*X@K-X4%O$s zVi50CS5*n#9*ZP+9aQfZB~MS^HK3Yj?CMXap==}=jW3A)3jG$CjS*f&~oNf=g;B?28KO(mKd&R@%T;FU|b4S^VY2iRvLXA zs)wyIo6eQ0<*}^b$}YWge1om#baifQe#yv;Nw_LZ=@~!u4(LWx#!K(xnUvt6W6^9dwVE*X^&oeP)fN^m zxI&ZxVHWB6%3Cis!qKk0JA1_A{k*YrzHd(S!OVShtwan`iox%JHj{^p-Ux$}WIfw+ zI2=gEjOpvfr>G2611tVZ>o9DM9E*H&9DFdChRTc69gsc=MWs6?N#BF>azBSOZy6Fc zBo*s-Z8MAQ;R236pk2IFMylj)>DFtP^b5dgys~In72Z z6cB_CB<5D0f*{cdXSt*;t!<+lzs47khRKh(*#ONMZq`P_C4#VQ zwo2!DojPY3(%9%zIj=G$h5H;6Cw?^XH+8+z;Av%uv-Ds7@&^uiq+YlTI2HL4z0HEf zv3{n1e?0u{Y0!1q<%UnwX#^>k@a+y%vr8T#u<`j`i=j>#E6dziNqYS(z(Ga4zk{i3ow*7|lIQA=FHSOJG9t>lF{KEewp74ZQb zhm`iB=#W03yEOpAAY@c5;a5*Tq@<@jg;)UOwHj4loMbhT-5d5nGyIa#$c<-gXF5@b zZ*THhM6Cu4U%tS(sYY%`&5fumb3@R6BF+d%#T4pXiFkd(DuaY`#{tx!9Y1-hH{EdMpb} zm&J_UGU>OI&*oGeDu4}8^gX$%w*d4J0or4`Rm|u-%iurpBCgU)?H0nfocYflK3D>y z{oQs{S2|}>f98UrWJg#?I4c|Uov5XN4y9|5r;z<>6QyZuYusz!c3iK(v2tRLjbe z1%O~->HqXbvan4QYmriJJy61L*z`TuOS$GxUvHE3eTqN$W+8{h6qO8$M{Bvbr3QaG zuM&`+F8q#_e;5vGDxM#*uEzHFuM(!)nfXJz-~T3+J)NQ}9w)rVoFP+xx=UU&dkc!B zxWHSx&|GVp6IhtKu1<+a$~60(Fm$OnH9j`>-K^c{ zItn^bG?S2|qoSe?(6O=q-XBtPti`2Qh`uWQpLYa4yKg2AKh8_%xQZn-!@|aIu9O>; zmhSb@uC!e#)nZ~Sw6wG$pe)*Ut;wRNAR-}g8mn`#u!zZ%mwJqY^PKXn5bg|k2B4^@ zt?l*mXTC=1o^T9n6Y+CHqm>m&S_M+K9;q%#i6+1h5 zVAZU5H!yH=RVm}6p#ryqS6394k3tHX3w>LDe?d)Vn{Va2@8B(gbMMo-Bx$a<-4FaG zs-je$1Ix|hLkuf@VjpfjaDX{TQ4_`>*p&=_l)psZ?`STauIN_=*wj*w+=qrd!VL)y z5i|vU23p!1He#d#Ze|9i!;f1w}ZI9xDX|Jc#%{Mi(BO2$tP!f7}L@QdnyZhm$!T*`O_8V8|M^ zUJ#pJhmpq0%j(*Gz$y7k;Fmi0_hyUjLo-EwB%o+PqtZLiIVc?4Z%(SrkCB<{^=JB< z_@Ur#>h2Y>XnMTDp#KO20?E;6*=ia2pv?b~RMTd-db>A4bMt&~2yK*wo1piEN4e0o z_)AjGyIrjbGokv-8Bf-^`W?U)4T!{ZV9kktcR{%Ze6eoZX(Bn;d_LA*lLRzUAokXB z6A48amI0Xq$E`qHK7ln`6ibC|EL$e=V+W^@>{`}c`#VPZThm?E-#H`LPnzM);n+c+ zR&(Qp6Aba%1kP30ZBq5bvyWh0(h7q6Ot_s|*Kwel_ zNW=miwUigO7@t*95cD|xN%?-DXW(XQ1K-ML`bV=TL~*x$s!JDi;E|W=-e{kPn zo)=F3!KS2?hqv4vXxUScmui0yu)(X;3Oz#BdK2t9MSZFc+M7$LW@9o#4U&%+>99FP zWp##caazA1CqJ6}<%1FwU-xqw39h_#?+_Q7gW2AcLRfTIQQ?{Dz447Arq16K%`wh0 z2i6DRb!c9HdkrtT?Foq6;6F7>8l0#qD7bT5K{WO4_q&OJ>vv}KYOm>5qXy`3S$UCv zL|5`kmz%}6r}xw3k+WCfD1ng)yU9k|n|1G`s{j>4N%27u@7=frJzhiqBa@aF6s{l- z_);AGIQ21@W2JzEtUp)SOr{xiH^x3JGzW~S&O1)ogM#Pgt*cMKBDd7A_M?Fip%rI( zLyd>cI+~X^WJaG1AWoN7-?HSBh1Nr4Fsn*RyR~c0>@GUPtKm5*GO2y|GWYkD=m+I_ z&zJ$9>+IwE=xvY1Ndgt*P$t+$1_lS}3-QGln!gEoX*h0YIadaS#eS<@pSZ=PeCUOU z7+xN?;||1&z|nW0t}YV%pQ}K;suHw_n!s)%X9_9Jy6>VchrGd-^;dW4BzGpo#m{iC z!e8+nL}(cgFx%Lz%iP(1(DJuF*hm*zp4>XzV7s%JYYo@3bj#u4es|77Q6*(0@rvio zX`jc9mv}AQ5;%QJdAxP8*E)aJ(!(+%4Zmszb+HaZlkPz5ytlt=BR^=n_{nv?tkhZj z{ae$nnZ`IqEqDKj4Fs!BK&3f%*?g5Q{942HK^hKqTv;z|3ZvP>RS* zJEcav>gL;(njv@Xi_H#Y+qJ;nNN-IiJ;ceD%yK+Z6SHl$53$u&Ht%Uy%7d(HSAP+P zAO0`@{|aAilUb@~$JgkJ0hM?bZy(vrWopWeEnN83pw%GTl(I=&gTL+NH+sq@16yk< zAMJ@eHrT~!?P%IdhiGBt)U#P8w={=yb&bP>Ij{a+Ez};P*MLeX3p$m2)TD6~KkIhD zDS)bxb-7N1G`ZVG&ka?9kAA7o+AUx^9Dj4W^+d{w8!2824eB_}NV;~wXf_(Jhu=6! ze*d&KMWU;7+&wa&lBo&F`~W! z;k-ML+>7Hh8%gx2pSW3C7g^9#RAhe9(=!m)nPCR8x8AF~sd1`JZ5_iMKspE( z)Yo6cO)PkCz(f6?qa9N^7`)U&H7@9Qr(@I=LCktM*F%*P=Kzbzn%BQxZ=lMNF*a^T z)oOdT)|;K5|IWAG{B%|6q<6hRPVO^Dce#{sl9JP#9V~gX7Lfu{py10Wx?b#pfG zBIz$?F=o2%Ye#I&;Qe$F1xPv_b`C(~`nc0)e$c`7N2!>U#YMv&TRdxmp5tx!gx;2R zq=^eMFfw@kSFzbS9LpG)hdun zt&jP$BKgd5fCk82p0I%W`gOR3N_WT}f$gl|K}5=%T={3L$_KrH*W!?*-b=00VW5_J zR#@jaP6#5xYeDnQ$@>hy-_R z`~<4|MMVKo{HB$KwW{-731TGsyf)=Y>}F>q`mj~RezoGK_&d4?Yj)SpZ?3Q*?Ji8U zjQVSIM1v0|bZN~#^tKi*+ch)kU4<6wYD)A+P_ya5Ua;y2Z@cBuX}in+tn{`pX|^0= zpWFM|ucp;m$u_?8g@ofA3$)9``Y_%H*)H=@h*3i6vhX_3#RI+n1ZAHls|pQ99@{9^ zdqBC7xPDUSG5^1I+Duj?2jS@`=g zL%E?Dc%opuh~=tqK(6A7IJ%?I?b^9szuUzr-WkX)m(41gH!7X*v{)@iCH%W=!+G)r zVZ+-hyt&y(X(*XL%cJm~=9zJH4Hlh8881`QQioe#oQc+tDhR9%2kog)9~Jm${R0nGJW*9!se>vi0xwZNI^vK?zleF2c1}jcr3-C z;N6#tbqXVs7e)$==ZM2Ng1NJe$CBZMerS@o2DNLvLQF7e0WV9c-*wGYC#?nOK5sX?q88UiZ|i(S{tPB|?q&<>w?M?U`yB<(CO zW%qIvkZ^M`jzss2w*M?=G8&Y@U2sOU!hbC*z_l-KrUh)_OW&M`#wr~i?+KTKFA&+D zdoTmXZPg`ikb4J_*QxmJ@}R$HZgd3&feYd?O%08e06t+QnrnQiR;CbIYi- zITIq3zRgkTIDh*Q@`mRgoyydi6(i3z|7jr;u<}tl!klZ@<3NvFAZ-?ftaPfG_gH8b zuniVo`^?@LkSkj_%~(5TC}5;i-iHxSUHXwXFQ?;&-n*mP#24Uuoow$E1#%~$(3gzUYyWAA;6GP91&LH2gCImbEY_v-!mefA!9;4Gef{Dpi*&rW~+7uU4I*8lLU!utFz z{f_Tx>8Ai%SS4+8KUr@YUaa%&T`^s98U<8O{JWSHiV_(TvbML0gx&0A1o=}o>ynT9 zukD(QQBUs1Sz&T&Qjp$R0w-5u0_X}3N&rO}D*JWq1M;sPGEW2rD>%<0XPcT|9;`1H z@z-+h!mpPb!)a1|NiDqTHaN{N8*20Iy^BS}NmziXet>~$hsAx-B_MP(flN7g$4BaD zTW`EYpa$*h8J{{kJMaC2q|Zqi|M&yyl%`@(@cvulHDH{RLQl!5e>GwI*^fmhZ`ZV~ zJ}6deopGB>$Tazr@QOM{Z7Ik!GZo~uEg845dA@W~;!B1-r17p@*p2lQ5^Ac!qPvPg zA&XN}mrRE7D?UY_fK8lP$l$feidT%fghjt=;{4eW8W>V`!2JXM(g21T)ap0b2WpYT zwgl?1X;5H*$r33SCzY8CdVzSe@HZ9YzrXJWgIs;=C^=tvIpBcb5h2oa{)VRUDZXKH zvYyD24-$Lunkmy*;YVnG_Y+cE=wj_icbFoas2+W%2YYlPjwfzN?8JU?N^ktx1 z7dqZOZZBg&krA@mLQeyMY}6dFF9^QYsFd(I#H^n~x^5Ck2rqu_$?`18>#FDnA>X1Z z%mN9y4v!TB+O>D$9Uv1VAg@J+V9@o;k!)0Sru;C<_QNQfh z;d;3|6Yk;ahu7?Nd0UW@e!ve!0QL{t$qlZ#Sp?Bj_#fQo{~E?nbrn-w-uf0OMpFXh zo2RMC>8mffDsP^M^BY~EE?&t6Z(+M|K~4E+>47+jd3o%hjE3 zv0_$VL1LoE5_m>jg9y1o=D%GJ!7^QrP=zIq`)+WXgK2{oS*3a8T>mch@Au$D$mVLu ziV|ZSblQ=>_I4AqRdIAQ@r(V(+vdju2a|jt+++qNRJdj(r4Z>bswIGPU!R`M&Usa^ z&-gQWm&?HGl=R2WJUpuaxgrJrWNTDzHdGZf+t-64e=Z)8#jPJI??c|Feu;djye~Ci z!#%jb^*+P>I>K%F22A?~Ph)%s}w?PNu{QckDCIVDBBFh1B1x3-?Vr|z3 zAPBl?*3@It+ko)Sox7VU5Oh}~*qGMzLJ#X|v%Rt6e!6st-JDV3lhSBKd^lPaUd(%7 zAx32ucy|0G`6w}1-{>^F%ii7IliF-}A@>x(!tOqqZ}dmgU)~%_*GM4|t37&q`{$QZ zXB*)$CZ98d+YUMuG7ro98Qp^kovv~WGl_r_btR=F0>6UV63mLNmQPE8@njn^1+V`VZn+)oEQ+?4(J}wCXSi82H zGvL~Pd(gV^5294`l4uoL`;6AFY?G-G(cL3OafX_M%p6|s$p1!6-KBwvKKBnOxEhqY z(XDq;=?-yB{g(JE@5xZPZhJM?;rnmU;o$K7*V;1({tq?|-yVHG>H8RN`Q_Hk-zIOg zIvb%P&g9QIyR8KNfn3?GQkkLi|&(m_T-(3oQpQ)bhl{#AlB^bf1qqMC(9 zLE%IH)lF*)Pi_L8kJDSrIn^6bxJi!mih{Lm#lDW5Zm^*Q(%+tUNL4NHA<(Zd(~0Q}1fGo*!1?b(Wb*;JB410uh`FGsB=)!T z@Q~Tcp*_&ZH8L~W|9ryRxvit46Nter@pR>VCJO#LVMiWI8+bH-`gz~o>ii~hn%+30 z<7sbnH>#aiK&4goM=7WX0ghPrawzKvfyBCJ1687Wor!o|T-EH_-y0_)9z-@+^&qH4 zS-|Ia*`zup?^M&BY$=HruT8R^;vz*3$V92)l2;XVwJaM87qLaU%CK=pMiA=tGeHqL zk}RC(CmHddLLwwIE9wrK+B5?Oln;R_Y7sfS?&nI*M3~o1s9!IDXw^s)k+^90{OTH7 zHrguGoq)B(_VCbT0LgtkB%kdU9 z<;_{w&Iq(D#BgdxeIQOhcNobvk{A?1>Fkwj>e=!plWXG z8L!6k&4_K`GTwhJAgt&-bXx8u|F33d=wtgv@pAx&)XCJ1| z*B97wu%*@p^;q;u!5ClrpM#jK^@dI9J1dv;Gmt^k}fg*Xk=sye)8vP zZh89^1x1sf7;d?I-jSa&_3XOn9jqW8n|Bz z7+&19F^vYDr5wx5IIN9j6 znY)d**fev`G(&jcdeEkGQ0BU1nHNKpD2>_$qtxR2pwYs~b0`T7;XS~2r)!oS(yvai zvXSzY`jP*#*y(4kD>NE(S<1x_d_y3wQfzlXj+yb-^|M3ee3eY^k(4q#yDlU<$Cw@Z zN?-XcUf^9dUSi0k#c-Y1h2#jSZx?&8m8FJRAr?Uj{`3(t9A^F(<4wny*yYi1z-CO# zukZecBRF*>HL;MH`>VwGtq~%^q!(15v$-s)d^ktQsiy!N0pDUYVdSX9DFk;J=}yv}*XMTIXT{<{18r~r*@Q)|7;Hsiw}QP-}teSC1zBrP#}R>7uc z(%hd5R@mks>NaOWJi9E^i1pu_kg}8bI}){UCTaCd(8T3BmplD0WcHrH?o<@i6_ipp zZg!Xx-EEu`)DcaW_JA^^RL?#0{O)LFY*!(M+rXu@BJds%sk zz}O{3kk+YTud+3-u2b^|$)A^KP*#t_JvYi*CN((+0>ny(WLCChyU2|E^S6 z0>f{Vvw=QiQu)q4(keu{^wK-Aym-~`_yHNj~ z-!JWDb`G=dc^d7XANp6$IH!!Wh)+9f@rV8hxfDqI8q8Zf=|~XX-5p5vo$GKO>KAmD z)>s&}pA`TxHrHRhRyNc7h>zjS+uY0=Wv4HZU_|HnsY5@pr34X5~|@PNxMX z;MZo&i@iM8gk$s8C!`NX*2YIo^~^BmEu)Qp;@y_QK_V{8ysvjw+_;WD?8-(<85!%E zJ~s^`ccL51GF&Rtd0Tjf7V0T>^eOS}c7)Cv+^nT!_?!m{sJ*&4ywOOF`l!>sr`V8? z;@-8N^?@6WRa_M&5^;5P&^7oxIrEP@jGstMbGEO^h_jCycGXVV)(tn_*+yjDjsJ0{QQzu z#2-nX2*_&#p0BEY568Mld4+|A0kn29g)o}>88BnD`z$G} z3I~M#)3tajGIL&21;hLA?xPt)+WEjRYI`CweasuG5_YZP;i0=7@HR#`>N z3l;Asu>5+^65`WKWauybQtfkcWczf>_MR!+!*5Il=Sb6vygyZftiG8hh!{LSi;aO%%BDWIfiWSa+RHlVzhOR{ClL_D{^uC#{|l-$ zUfR}9f}PL;Vy=>`yB72U-u3R~jU0-|u~D7Ifzj;1z89=tLUyPpaITwigb}^DPF1E& z7nh~k@X0DyaGU9fE{IJgREpt-M?wLnrm5V{c@8KAf^oYVRi!Vfk=Wly(9(%q-#%Ho-ea)&t|q@?Io8l9@>`&4W0l;(T5cfj(FMm=Zw6=Qk9I2^Rm?JsEc3 zcY__pI>Wo{L<1geX#UO14V)-vV-yKNwEhFVsUpCVD>Yc7*4(r4bI(@yDjbhbeZ0;= z*^Z97l;!W8&yV6)_ww?xIzQnPDPL}Hh)sAP7b&F?{%F!3O7RC0Trki!0o%;@Lo*-O z5Odcb&tBKH&#ub(?UGe%Ob8iqpQA4#7$nONp9rN^UYWP5oyTSaywCjb{a?e~ll8>?k>jZ3{C|)?U&VKjRYULLPhm6mD-m+Po zFMb;8TJdu5Z#~%oCt>jB@@)+irnjo%`!70HPgX@fCy&|^V-WM7Y5`_x`&EXA-3VE_ zyDp!=2BdzKPwx+9mguNQ{bOIH!PtC(jL?kiT^lH*!m#wfEvCW+CS<~W^x)#( zW&YjW`sF_(f({-DgQfdj+#rVw<<3^d{$FaKAt%9a>DL-da$K{oYU^n0NP$pI1@gpZ z1__pj?P^Qv8eihnXtPCaRb_z*S;^=+mcGNuG5f%$Zm&NV*yx#<#QQUgc5Aw>LHP)8 z075vNPTlp2cvVXUK8)-iyQ>?aBwe#o>ks61VLL+-ZvuzM_)wPTW3bG?diN$kcftY^ z75IL=ZBo;*R!{P2`U3gUwQD!yY$-6;-_p*_qx#z8!sAfdd9HO6R0C_Us}Z>OA&!d2!_uqoG_bWMccs$0bbdrf z$0RpB8Bm38^%hP=nX_gS5Y8)3p&Oo_jJ0}fC=idcY__O z`pixvzF5P>HNIwf?3^2>)`@Im;?Bf~t=5{`+bgPI@}r^(t?JWTM~l2f3JQ1ne<`N| z-eeVd`Cwb0!ok73qQ+6-Q2sa2N4*;MLvL5tncqM0Iczx2ZL93rWh)5vB7dC&c>{A%H_(4L6^tcQbwX zUHnE+eCcvz)`c-5eR$}y^KZQ-ub_X=9JG4JhH&@SzG->sXYiPE>rUr~QP7XqJ^^S5 zv>D@LaqTPAK@kLgvtUhDy&~%A16!^{Jq;m7fV)8N`fq??0nj+j@ ztEh{Hlz91HC$}sIL+Ui`0pU>Kg>8wd_RM~dfJGc; zHz9h|+w_GKWE22iTux`CXuqAmpC*&`fA--V6Bk7i0Fb1(j;*-Vvsb-m4i30rr&UR^ z*L5doA=&10USbRHr#?Q#i?Djpx(7@=JXLDtw49s(cj6<`6w8t;ts!FR9B46RY^HNO z+y?-6-@;GBoKuG6SQOex--WSiylY2AjYpC%)GX{dlDW<$;07O?1b1nc%{xNp(|N1F zEiDW;@*b+-gXC0z1vKXq_^h5Ico@bekb_awcvR-9=LS~01ubl~?vfKQb%F0hLb{Cv z#3x(>rkhvl%^}0a>#7g515gT2N6qhZVI7-%R130FAcq;;R4RIU7VqC{2vB=wSKhXP zNQ?Og->HpCzEj(N!RXv<7zHz7p*SJApLH@9$|MQ#R$MqORvCpzdlfu2Tgp)l_NjKg zF`OWy_g=q0G`!yo)*+}=Xb{6hZfE=^m1Xia{k@@hbWXZ+SaS|XQW}Wp#ziRw9^`+I zmptl=pNxy=HFE2$+8Kp5V`s*0=_?Xc0&_U&Cgn*gtq0=w@}vR1tPU$Z02p>K1R95w zPM|Pt|1&p!Z&GDN6C2Y0^9dLB{*xXbdEe!^+Z9&x{jc?_%%P#=V6-$7uYRksRZv!L zZf=46SvZ!k)_9?tFEcR_WU^=zbUfB#ez^((fuFvha#;uDC@77S&&RTDIiFlrV7kJ~ z-ySk9qMvz05FNE)dQQh4*JC!E`I^lI0ohxlUwMIik>qOeHf~u3Rngn!yCfhb(DZPnd!3sW~>JK#?xVfDLeXpS*zl%vqaO*VvXUX=%s=p1dIv2_>FShWb(4GM?<4Qxz zA=H69pID?*CM1H*-DG&dM*b?+iQjm*n^gL*hek~3_RnSSf%M+&6_R}(V9$CDj4s*x z_D0RdGE=RAE)g9Hg<4usjRGsrW?P~fruTiNPCgs?3mQ?~$vn;Vnyze#q_iIZl!%Bx;l;rlbiTTY^!d{Eu?FUD0=)m!`fbX2qj_VK88px_oF z&U{OJ9g^HnV+=sJQt7k$Z|bGN$89i&Pn6Pkua~?>7q0f35w8iXZ-55geZ6q%31G-e zr>Q1Ay*=+oP0e)Bi|Pa}%o~R1y$kp(Lxg12-6sjwo6{e^scXeadd6u!7lCCeE=IC? zi7fx^Z=9k?7m&Uf0BJ}n{jfVb)UPq)LpeM=SPj(=ROzsyDSpQKREYO{Co*>0KD0fy z#s3h10ja;LFru^N>lAC=-jo1SxZq0>79H7+d;h+<7qdf8g8l<)oTDZSCwCmf& zTr{t!;y^PS7&c-{j+wm}T9|ICeafcKKh?;{w14645F)u*hrjU3tD~%qi zj&d%yUp&ps-1qUF*Gs_q$-^m^6z^lEWCkJK^9KH_HabcUEt4jh-5lk`;umqL3r+L~ zVGwusgU*$E^Ir+%d17J;aK`OQqb$6lZfC_L0Y>a^pxHqmhAE+Xu%*i{Od=rm7m`kj(w?cD+>Mv> z6)nhb4?f5W;~ttVkjD)uftu(by}36wkk3o@g6GRz%^JKxggipT-N_U0byF@A7!oI5 zJ?~tovtAn$vz|h3-~HF1Kd6Z5ea7nbk`aB%A+C@2zeyUCFBtPWT;-3>$NCi4pl zzSZK$z#4pabacf~UB#+PRu$D9X1+hmqRoB?xh{UviCKQ0|JK}hf>EBQ;Ke1v8{C<| zNdWh|NH#FN8rnw=WkE3EBmG5cRMn@wdmxIDnn}m%jxEpYQw=F`e@|u4z8ux`(DO(YFc;H>(LG8)JOg%8ZZ$?BrNy<&T2An0HDJMl;x~f8vKx+WUc5 zf2L>LOG>@|OmQfJJdgby$~m%~?g9)1B^CW#ozrCrsKsTWQ?g8lVmkV=K*kE$jX53b zDp+?sC`(HP`90S(ABhaadJd^!vbPq4wJ$zbN20FOVGp$7&yNB5%}tx#Q0_|qm>b-z z#if09#y0cQ%YvqonQy5zWw&#?n%g}{H<$W-8?F0)W{XLC!SC!o^xa6+pM6J^vLK0e z^f{_nOx)dfmN;;wIVJIxJgy`+laY(16}gT_XJAkvXPh$}``zML4Tx_4+u__5P*tqN z_Sf6~rT7#2cen(94MsIr83%oi&XLjk@(CH5Ty^cAl#o~THb8#k=2EVImx;VjCGO4K z1V_H2#uQ;1cY*B!|j*PBoA!ANGL9B4jaM3rN9AYQ<`#5^){~Kc=Gf+@dMH z7Z!q*_s}XHCG2I8vTtg6#w}$b{Zd+t<+=-Za~kRk5<`DonfG?5(b3T2_c})0x}t%8u2bZr1?KApehRAP+jm|0`pr-{;vP} ztrTu;-y7z5{e0C1#ny`Q63!D-z=*emH#!4^2b~aPe z+0Q95GG-JXeJU#(Xax3_Qq_zEagX}m zCzZZ72akxe#X$43*^6cY6Jo&3sBcn`+k1iE9YJQR=29p`)ZTn2!~38y)!fgNt8ieY z(YnN|d+!&((Fz`>O*M>ZUhHHGqD70*4Mll*t@-&(11_PKh2(2BZF}>qnnUxWI+& z!62eVs@abugK&cufu?^ROxvsV`V6_2;x>QKn1SOmP>K-}NZUgHU9|yzbi^e_V zhh;LCUV`2j+HZVyNQSvSdUfEpa~!VDBie83Y54}^((u+KpVi^aSj&zVfK~6^-pj6R6^|~j)ttE5B7SnfW&;f=TW+o@Y%`pS z>?e{RB(+kGgV>*ZZdWx5^zLAdE}n3{K-xK-l(5<~m{80fc_)5TB3G;LaL!Z?xNot0 zX%}r?Ze0+BZ;VqXQ~%;5I;W_qk=q6TnfoVc1bMqniVwM>RtJGKdAA$;RcpFfC{k^PR9$=DLZTVUKhC4Pbd$fhm=7 zM++x4q?E*KIs7n#EG;UR7`0IHHMNs=bK~Sj>!a}foF94d#J3F}JAkT2<}%%Xh-o#% zwK&bsCr+gD-o5+4XWsR{5U6h>_FncfWhDemuYc6t)(3-Ca#hOjTKJOyh*7+}KLBlt z5Y);h5!5^wX;s1yzqD$%8Y1?6-Y9fDTdvEGH)$O67@cYU!^z z_?fCxe^HW7s{nogI$rBBNU(<-uNOf%Ga zXx{GTXqSzFLj8EH%l={e$q9jMn~7VuG?&eN&b&Hv+%OT>ggsSRaznjKYT0ys_e~dFe}hqr{AnjU1m+in?u>P!d8^ zOL3m*lnuHD@wz@1cd(j|bFvwAt|~v|sI>h(j0~Rt(<+Cw7K%IuaB@*ufOZTdwxF?r zF=~c|E$H3XHa_GQ6=r>7>9X$Aer%23m|?!>C-!PM_J=n#zu>vsM5%YSqj{_-cCHZF zrgJ72oq*pR_qT3(`Y6QGa*5$rOdPNpv0^rxEm+LFoGuS2l|OSGG|$U0OMIB7?Q?N> zG@8T0Lz!G1K-jBIv0VTNj_2Jl2JN32B$WaxdrDNBomL#|e*XTS&+jE5Fa+YXjDaQg zjZt%Kw3P%3~B9*EQs2Lz=LY{i*gyp@sWm_UX3+pMG^mk?rXu}Z3%jqf~9 zv}BnI2L-jCUfUjynssdQ(N*e~GE}wRl9RZ`yPa88S6?4c)Ki1Ic)!hSWAm{n-#6b> z_3WL4g~apM+(J!4uG0+d+QSSFbZIq|RKT83WjznRmakNq@<6biow5$n1`EEGCiwNh zZFU{@3&1s3Vxa(K5^#dNB(Wn+?6QA3eg~@Ydetx=!S9gB%-oYav%$s7bFO-8IQGmj znYR>&ow|VfD?bit@;(QJoG$@HNrjj72y25ohkE6qX`t14=c5MOJ45FFBWZ8*O{Wgl zH!jWKKl|^6W$+9cmlFI#>BbRFnJAO_ znf!`e*08Y-7tGIa=LH(Suwc(9;alWjk5Rs2>A-J~W`R_LMnR!#QM6R)s)fz!Hps_R zFD^<9<@(FRMM6p$YuHS4-voD(2X7AT=A7P%9@Xf4CnJsUiLx>WoLJvBj(a#VChif5 z4D5a4kql1B$oPJjjf^Fw!qGr7^he||lVS|Gn#F4As0%J1bklY=wbRG8+ZUH;j$H6i zlF=GknE1~KiuhS~8;D^8lG1+ge_+`Tv=`^WjNfxz&K=^Syf)%R$g})pkuA0 zqM`x@@83gdG|+NBjG?p<_4++s1lJ>;o}k5(m}v ztmj$D!YuMyUJ8xL`kqQy>o*sFnKriQYA(*tDMx8p11BuLr3laO=6nz+(J;Hl(8@;x zvXy9=XAY{2{VhEO`1AArffqKwUyT`?Q|Dy-#3o*4KE&$KX8xmiUBL1eYrBD5qf9{s zb`fBQKe2nDt{c>jw3`(7MwBhvy-ZG`JtYCS!0^sbv|#|}EZ8P)ASmmvFT~Dj%sw?W zzB28qWr#OCgQVK1eoX!7k?3>y^Nzrb^46loTPgW@W4C7V5y|>}fo2W{2O;g9#%~X0 zbq>~bw(y4nrY_9yl=(6xva%CfM9u`w;i(k}>}n_~8JKdTj>E_AscjMVHTU3LSl4mii5avj1}PJ*<_CDe!F)SPRE5U55<1#Rk`+9`n2uw&#SgaQFb$9CC|B#?qD-`}iq zgCpy?&E_+5!_Y>Y52D_0%LUS-aQ79mJ*|$=(I}e5!^%2yP=kP3J4H~tEDV+a1eN&0*4>2!Q0g~AY{&pMR&^lT1H67syuUSyu3$~hP zd@b&7TUbeSJuxxG)T{Wx4d4fKcYnw>^MLntT|iomGG*mU%I#8pK!Bx$5699HtZW_Y zgxnn9Gy~+WW#r_W>&gssv+V{(+}z~|2bw5ri`dl3Yp&j}(ovTQN^cvs^|L*D!)^)H zx-g&c94GMCF}SCe&My7IhV!T-8U%pEJ5aN1+p~P@a<>nVzh(A1ppQIq>gGEWi3gCt zc*Ly~kY--9lhOj~CIod|o?WTOVI25@^Fg_YNbA-nx&MI||G-tnZ8^B17IJ8Q7K;3M&hUB z?a}{$JoPeY3txHl^tbL$5ou3v8+!JUcritt(oq9^G^rv$DabM<2KJqtpAQ*F?RfDH zhm@!)`&6S1EiCd1JglIb18FHTmV-~@v@Wd&iUcx8N*XJSYWpR4GrvapZeicr6}5PU zgoLQ61<$4&Sh&mU!2X@2Z#wEnv%Z@VgXV(ozS2^%WKl9uaRQmHM4nlm27Ni+`yy`3 zCnoy(%@g<>?Y673i3$B+tHVDPf(0^X?>ZapeH*Skss|q`-&bu@NNe2R-fQ8~4=e_})wx-Q3xjjcYl>!s~-6 zQT3BM1#Uz!qUtjJdSukzCAPeel^<+U9s!TBbD)zb{~LSW(K057 zpVEVFx8tOx98A2HMg(V7gR!Xo#-%K;%n zXY&cbGGl6@#s7u4i(Sa$Q{tgFK;LjK)D3@r83Q8UBiMlqfzfcRl(6ng=DRQAHP5aK3z{y>%0QV9IZQhpmsuUP#zq zkpXZAG;*P?uDx^Y2lfk{9NmLVx1kw7uPR1N0zp5&&8n3Rd%ba8ZyQ${^wFVBo>e>} z+cVgk0#&){CeXWi{u(q6;LP3$+0h>%C1v62c4LRWq8{ zRXW~XNge&~5R{?fQE^|tJ|q?r^lL0LQCG(THa~Oh;ic71bZiP;X>D($x8}Vl?I6@& zoit$`MMaXZo6ux}+V_AgfGP;J3E1d<{^lzv`adjS{K<~9hF$IlELL)-^tEcu*hnhP z_j>(V!Pf1-XX+$M>4r`InUzMFnQFg?#8LN}1-33Jzbn_0Sg?HQran2qX*&sKR@Es*C0RbKrlZw5cVMk3b zwN)>4fpN%GFMaeENO%b^pLvmZ;3lt94<`N$ot~QE7$ScsD<}6k`VQI8$Qk>mMQT4j z2~~4USvGMz;7wCy&7LW=OUirpi*C`Pt^%z!{I=?^2w}U`o9BM5oylH(?3>m&pYVTo z)EOJkEFAuvXStvVs%4QE@#+g<;qLVl5XSSGc$J5+Xe)6x*HCgSvRP9ZRHRNJb?i~m zX3j^eR)TTsHmP$B{Y!<#pFP*6ENPv|? z2Ld_Wi$!W9-f=O1u;J!*bZ|b~)s6bCYskHEtVi%!wH8UdM_zVi_O{{ILhjSZnx4d( z+#!No>pQVtC;PCk*&Z^rdE7~w*o zC=rdP{zDK{v8$Wy5C}OQ<69rj{_pG0V`GgNixKM0yPZ&MAeK75~utkN9Eg?&kgy+#TjQ5A?X85q<&$=Up=<0i~;)C;(^Ao9Tk**O-r%O<3GiE^rdOY+MZ570 zR(Qapq5C4uxIB6Z))(`G7em#QK3-8kPx!@>T|*lkg+7QY-$h-jy9ZUq!9d99FDXl)wX%PVQA#r^TW1E!7V zrGU>HX6Y1QYXnp1=ros;3tg&BQZBAHLYHk-YZKggF!SR9yF5XqrK6+q5Fmc0$mj#gNdIRj&9U#nE<3 z+KN`iqVCa2JZ`p}fW$VC8>{2~KII59LA-1^3Fgt3M$Lv6^ri-ure~Hf#XxIi&HD4} ztft0}b!?d}Sf77Fe3V-0v}*P%vCW;0#=%7OtYAcbq&Uz-xn5hpxoyZRUDu48^N~6P ztk7^c`^Z5JvF9aF5;q8(T@LO;PCg*)M553F1uHkV@x96W_WR;UvIUtu`x;{-)$phA ze4cSxgmG4X(Ce_vtukjEH*oXNnfMqhE~vxMt${*Vtv888e=MPsP{5MA9MFD;Z*N?- zeK8xBUEQa9Dls%?AkQV|YQFtk+;$$X5@=ts*|LW}yY0i^jPWi8!Seg^?T6!fySlUDr8T*K zoiK8frI;Bx>Bb!X;y?#8#2DM<_8b zki=#V`BvVn73!gliBx{it4Lol?N)>P#ABb$(*lnj-J%^ipSJOSuoBfWZC7QEbuiyQ z`(`A9!R#q19(~uxOu|5hjr|8TDdD42d$+vZ-S^3=3`}bFW#x?B?&?SUd=aDZ_vhsb zQ5q1FR{5ab7;Voaf6e74d&?^=j~8ng2p=nmu|Agdv#p@qi?>*vQAqLL#ZyE1fk0yGtp9 zBckX9$luvJM4!Wib)Qm%eynjD_<2uA!Y27dDAntp1!5@ZJd&TCc1YI8VslM;`H#r0#4U-f4K zWUSvT;zI0?mWVl>1kTs$i~FSjb&9xA`hU#B`kQO5?Y-rvOD1C;(j(oQXL(6{b!yGZi27>i%$PQ zY%_%>HW#``OGma@eftvz|G>?qabDb2Z;{X5l?ORxFFdRAWP<$sYbHahWyK=wmh_*e zPQ;BV&3pUgVc*|8p&;ph1|U0W;^H=+SPflt;gfqnJVT{^rZVL-(;sjPrLj4veY|x^ z3Gu8}IpLo-CDV7%*VjVl)BZ(rM)Jkf+Q0)d%sssoP82T~h@+#z8Y-YpGdGPPURJX3 zv^&?Uoj~N4riEr`f)vL!fqXxsh?!_{v$_XfPJBG?)vL=72di>nH8pWNx&MvIMriOh zCM4EfStfstS=BI&4tI;4OWJRdo{T+avRF04ebwJ?R61Dw%UAcbT-*omRG(>yJMx{_ zpW?S}qa1x^+@$d9pzpYepgxk=N_du{yKKARp0$1yE>t1d@S?Uw$}>dGlJ-(PuGy|wv`EG+I2=biB= zVCBdDK`p0}_jTUz!Y)5_-b#tnnA>oDK8hz^1?-p&YBD4ilON$<9CX5ToQEV5I$rp% z3%K(crgEXI(p?YI;v4Md)h@{%q~zEzb_vl@Swa-fv)H@7z8m0?w1H!qk5R=voJWZ%VnbSKK;~( zVrt}_cUo!P(O)|^ES@8p%-y9q)?^xs=hITbT)+(7Qxk3&LCl@@E5WVsPwQ$U#Et?z z9mVgIh5rf__;2V2e$PobtJ_?wJodRlqCcjO2dpk8)7lfRX5ykOHTk6afdy&Oz(8#j zaa*nvAfKj`Q!_LyE4ZcKcci*nnE;Q1NuO(@#~!U(Dp@PWsefTiq?Wc-?Eyly;2+9! zL6?j}j6xx*sH0%n=9^E?SGkL?$siEeMBzFCq2$H$IIWDCL+5^Vn_{Xko#dO1z|E$X z;UP+0cN!6&-6k8hSSYW|uH*`8$E)-2j%3n@-)b-uS zGG!|HO!m5_@Nnn}LYz$cjuIT@nzh?JgI!o!cXd0TkMV;w{%27& zc$rRb{Lt}J*z7BVO}(cTZI54xbJTaRYq-5V9YGw=o4on5 zE}1qG;d)MXcbAs$-WRD=RA6ioQAIOuzmCkOMd8*>;+FX<=|PxKn$4?Bx@jf;-sh=S zujV0mhU(tXhsF;B9s&5#Y|<2kJRbL;Vzjic-mU$^OR2}iBN_f*sZj|EY(tD}qeDZF zv#NPq8^FLo^ zassDB6HOF(uDP7=)KwHnm^kd;gyMIFBzJ8GUR~ZRa~>HJLxzmozdqqN3S21Zl3%cR z`0ALQQ5r_&-*ovkjA#Mh5U7lp2b2Scci6lVn-V4$YW~^5sEAtRn~WPYM@D4B$Cbzf z+c`ve)wOOxrNA(EeT20|rHrOmaweaxZ{)?yEf3QY$^ zxXAg$SFoej))aSYi78(1_ND`oJU{sz;Ki7Vv({-ln_1*=1>(Y_@FdFhFbalE`QHl~ ztx?N=#y`EBi#B<<@VlgUwfxVQ@go%uf-(tW&H1ychDD-UXZlA_3rNJR$f~-+O64~- z$(6B)7)`~JB6Z1yOp-Y7;W`4#50Di=R_02V{%IR9taJmAJ-gZa(;0d?VOO!$tMmfM zzxPfG{(O4xCEHmbvm?XB<8o4C5|Ste*78s?577dy`qiY(mCA)WCts)A9Q@`ek-CnW zsSX6cjt<1IOC5V-vq?mjl+^bt_VExU#o7O3?>(cM`nL8_EQo@F3W%tH2uPEnl+Yt0 zA|OpVNbfcD7D5y$0s<;cy7b;lfIuR>gCMFLk7Yy(tzC1vJ{ zr6xLapibnfW4C}r`n&75Hf0%q4-W1xmD7&m_HL)Cnhac!UCN#a3=F)6Jps;$vIk&a zVzPpYFSuCE=_E$5`+geJ;+4ggd-th@#h4v&YTFa~vo2Yf5-maE*haA`=ccoV6TT#L z8L}5jXN=qh*i-83$L5Z;3n{;%&5v@`F5C;RtxpGpYO3nm9IKB5w8EO!Yqu8`R!S22 zQGO+b-zp`I|D6jM`?B`)LkE445zEt#Ro!g)&E?h1h>+twtewp!OGHM0xA=pf-}J+u zxV?2z0olx((I!iW2g=6g=ReO8dpNq){ZcVg0b>9saf-z&yJiHE{9;dmhylvVEoJj2ePTFu7s%+zavUDiDa9H2d{9+Z)BUl#%d+H}&7s0X zz;$Pb@k;1(E=Q6k#BQ90WLAQIz7$3+)UCIO=`6 z(eV4+2H%QnM2Jyrfr@e0*mU*v4J7|*J>hf>(4Em*Xr(Q^I3X47*sT(ukIZvqD;w6< zWFXf$4Srd`U}Y@`VvNu^E!eumTFr8U9RG2+VJTls^QTLFqDkXZ+{(tn?miQ)AOojXgvNeq zU*FwbDWEBx<&u9xM-%|d$*dS|@WgPzzkVbuK&k3DE4a+>3BY`P1E3E?RJq`mS!^S$ zsj$5&pebhO{fjz4+!3hwy_@#3@5gzX(NWPKnT^)sko@;-A>YNaaO@VcC+7E@H?M=; z?MyRj7xaBTKG(6!wTMp=RSKpew4pBQNUO2HF4YkaBt)2%Uy*PVfAMiiJv6@>5q@p8 zw%U7akp}(qxojHZ8!fn{_Z;AvO6APQg$uL6MP5J3=A_`DQp->V7r7i%uWyOd@Mu|XS8f2+z6a_dZB3?YgWd|p<^f_J8n5@&m311-tN_mOgaMMr zAZ&NoYDFi%;1bKDfZG7?b@0lSR{=Z3e&~K&`P$L?ck91~VHK1yE_IoDvzN9MC)cup zlXbUS%FEv@{MvotE zywLpglj$GcQ-@K=^S~Q{{NFjxy=V-V(<%A~Y@>&Bz5-a%^eiJT#w7>DVFAPlwqTI< zOVoKBLvXz#AwmOVJ%sGKO1H}$&fIxhQyS$*$<`Y6LsBOh@oJ?VVp+^lq#r2ZuRKKY{H zbcKfID8QPWtkAxGaai1A35RWV@C296sSpv6x%*3Y@G_Cg=P`?XXKzHjZ->*hNWOrmOS;EGjJ1YRSHADJD!Hm z5RCW)d}~e)f`YC*y;@3FJDJ1OaN3v1=l%}1`!%Qhc%?&kw^1%-=43-SZA!i(l6G6o z(79AiKGO`6?eF&{_DVO4GHIWlUVSD>B#?X_P>Jp0l2!WghO~9x%CTAxi^AE4Q9HL; z?$}6kTvL?PlT>hCDwQG)2m*j26NHLEF?q+O5-q!^SKaXyWri4Q;Yk2Osp-DsYI}Pg zS|^`)m+uTG`&eB@*3Q*iJ*3b-ER7Npx%Kjm_$!WR;@*ldTdOoT2oi*)vF8^TR zcH-2d-8L#B^MZyd$Mn}Dpx3j3bG$$i25786090&BYR+OC&dV2jAp8jc>NXe)KGaZ; znjgr+^hLfSU8n?NHYbf-&nkQpwP{g z;xM=?g13710<9_=A}d_vKl%?I8}XDSJ^zN z?I%$eCmK`167B%&Y)jJ25G94&At@7MF3w35+$Y?7wzwFtpkUE*HHmF&cw`n6@~HtZ}}1;ikt5o#n!XzEsxU*)|+`y=9$#9>;3 zl%3t^J-upMo36AoU1qhX%ndF1_Tkw;?~jW^&E=O&HeSb0M?a>=+!(RBX6zklQoE25 z)e=wx2rgO-M}n^{0bBZ=^HGsNP7dC2yECU(^69hm#C1T?-DIF$kq)+Rq2y5H)Fq^C z04xGPn=$@ig_pPtJq=r2qBVp~r>1%x&QM!1jjr?nl)o7mCPO&UA0-@dnn?f}T{i5O zld4D}R0rO+k|{ul-7+P7zQPg`g;-I*y6vcdjd-KKyC>nV1O=`0qE@Gfe!C&6^9Y8u zK?MRHf{h3bS+}SLH0SkqYS5V)y52DWwwG&bMuhc&B&Rn(GJKKdpFSV>5k+SL#-@Mz zHFHI1JdAZ*)O!%kp6WHFZN!wHr)mVL!8Wpkjn0l@;58>hQJZ!!dFX^IA1Wa#kuT|4 zcQT$V`NSvOjRx;Fpx;2${cfalINPjZwb^Np76TB#la45@33ia<1ef_Ppjv6lk)#73 zFV*#{m)vVC?0DuhKp27?=Ozwd|GkTO%iA>?RzVw%l&XAub$Z-=Oka7S5TGNiY zMOF2Mq-oY$FT51y=(>BeLc6<BKTHNz_vU7mzeA zcug|9@S@zyGtw)j%^&@J(?=%))H^jo z){CEjVW+R?1zoutv&t#Weh>z2Y_8sQ(Rmi_iXgZ)n4AOnmo?B@I*cR8Vv^@qS3s+;&N6*`1M@w?wLB5)BR>VMTXUb#9tda!9@#Z zZbTBI$+-Wo#hZkv-wdrGrg!dEUNx4+omOvv>?cf}L>_bu3=Of~@KTIpPlbu{0+hX{ zQctC{a(>6gXhv&m#Ky+ZF{$ph>dMjaiCFCqbT)0^_CmvG!l@PW^X-7+>jzA#$G!u| zXjHv1A6Rfi*x~r(4$p0cOh(B807%2Sk}R-;fxsRXRitNm>~xBqH-QZg+7*dTr-6;@ zj>@w}Fnlu^6HGU^R9#+2XVrO{t*5x&{=?wvXKO{TCpCu?iH_&LV-pYP~w|mXzw}U`Xg2)~R`$ZiYS@?1KMse*)I>j=X*yoB;I`=UQ<4-XgEjf~c zX#gLe8Ky-o;jkWka-stUV>YL7NmKQqtJ8-Y}Bo(XoaWmpucidYt?;JxWnf{D^CrC^Xlpn-q9>#^UrcQ?*0SAQ6D( zP&=ZeM5+^gSOE!v#Y9>_31&^g*<7D%NSh5yI%axMO%l6%!~i{F@&KPx%{`*tsUJ3S zUCba`lmA)~T_X0{>*eY|*F@csJdnfML5x^*JqmcKQ1g=HGhWSPq(>o$eM!VQ1>&F^QlZY{U-E4}({mmWjOVOws$D5pV-v z$PEk(K8nLT4)2uuENbzJ?!Z!|fG!Fdo)Y!N)lxpxX+l?HG&On>kX}n&YzJJyMwha> z`@}q3sYk&hzekN%M$R2^E9Z_6M7j@!R+7c60>Vba&nMtrk}vsOsVIr=FJC$G7*goO zza$Auc~vTPv`6K)scv-jmDME0WQL4soO^JUTW3?G-zt$b-g+;cls(yLiop{@$GS>% zY+w<4q9;OZq1NipO)%6;(pLSVjVL&3orq(ExosISzMdtApal#M@*fV9laurM{@y^J zH*fe~22@FXH;}|*BM`x7d{EYcjm^R$i9cI^@IDZnJ}MXPXs<4&3*;1R5?BY8UG!M@31VuwNW-|K4Kg#-VtJPvk6D zP?%Bc&U6MXozRVS9MlUtgPqk~K^^hXH8^i4HD_yR(O?NzOn+yu?7(Ngepk{)EmzKv zX!+GkzH+xzC9r7H+F7(NS~pZQc*xmuj}&ivi7WSZs}wf#&41N@`#@zw`jTKFbpHd^ zQ1YPXH@Q`BSL(Zz>BP&n)8+1?W_BBlpUX_%z?TwM(c|8Ggwg`;1-+l43FT+2HYE~% z?FO78%(-Gm(VcRPzed!cFkge4cRs6;a3d;NH6#VN~fq79YsDS4u=Z2?-BPYGdRY z4p`FkA@ysZ$gR7+bI&SgE3xmKkJHQN;s(Yi;JpF}R&5HMhKb2nC|;nZfBOsP<?<>kZpIzx@Thy4hBU zgk^mY7%lXA3h3~GKRt#5pL257o%F8U5IRbRAQvMMGhg2YTeu$~{cud6E1diBDh3SZ zxzrIx4ERS>gevK_yTe+Qc8DVDyH(;I+m6;x__!rE03qIC$Zf|DEX+H;=WNmmy}1c9 zm3!k4ox!L5#0abEgFf%+=ow2t^8E#e_c=mScg(-+y(`-u7jWU*fxR$`VGtc!)K+$c zF~=7JVo|3fUY3b|UEBv;kL8%&gq~B34X21I{n~=;Y3|rSZLAi4-}hI`eFp=C@6j;g z)33|RPFoRPDTxOB9$>2{`sMtZ7Ydtn1%gu3axJyKh%N0^y{8pg>4i*q?-(^r+I?)1 zP0*j;%zDZgHgmo``$eP9$>hrqgUFKm5-tSVILTcCn&R8-={`iJwR$SRn{n?7X*}WI zI4YSuMib*yS3>X!DdR&!=jtCc^47x#fDnM+=OZ|u_XZ35l%pGav?X-9_d4m2G0wtO!V&mtjckuE=*PEz| zt49M=#4TL&%H@SNZIoAU&u7N~_;ykP{G>z=xPld>6>>UJp z(;wH&H+_G;Cj|NQK$3xe?4p*2mU7>|3cS!>xBfJx(J~o7!6M~x^nUrrI+Nd%)llw^ z@7ijNpJPtU-mq}x)2GSorZ>MGG4pZ@<4JJUNfM9)f0EsSt2mYRdL89m(t9cH7X9Qq z9jh13(D=}lRIcc7Q^L9|{Jd}0SbVd7bVZ2~$+xzFhIu<=zhY)Srb%jlQF5~*|cpN$hb^4*VV|Vo`+SPJ2DL$z0?fK%OIs7b&Nx|-CLI3@Jn86p} z9l!RhDC~6KlL(~U(Xur9a>8nxL+ib@&zJW&J4P>R>Gen)3`*xE%jxbL2v1lT0o})5n5RTSAwaR;n*ho?%djSr;ER z0^<^sBQQJ4hkQ0(zWHMYA{aJqAW6yk&5Fiec)rIgbUf~|uHYfR+xtKDl`aj4TKP=l1nUCYs)vyoA> z>X$VJ%cb~qAuSC3aoAEzA@ITZSfzNc&+SoLh5$0TH zeD=E2Qs-Yr3@xm|d`;<7Y3=0X>Ms=T?+?Bgudi(CdA9o@0vxzjAxQIbFn0nuX+R(q!)m`&MpDrOXE0nVL9@CKHXD5y1 zfAgH0m|I!FmlRVC>sSF;kNgSF3hdFC*mf4&H-MbQ5ZdT+R1)v4;8IUk&-vgRr;TIz z-`^3oAb!sKHESPPii!!p8h7d+Put6`?am)hpvWke`=P(Rtw{5#ZI||m*lh%v*;3MI zDt)FN5Kxrh7p$;MQNO;P&5nF$uiMit^LROb!?@MR^=mGDP7aE>VQM~|Md~aK-@eE? zn6R1RlnuK;c2!eJ|7@-x;=B=IY2-AsNZaj-)fcVJ%3?cM(PV@U^#~X>oZxqKUGb62 z1|IT_l{Qb~e(#<{eO>-qqJGHv#y}gk$6-*<|2lA9)b&*oyJk2vhS%T$tK$F$RdME? zSJE03r>^0(m;;`{G)*xVl=@hH`O$Wx?2^tnDJeH$PTht)laig=u5);Yn?9-onx6dt zdbpXXx2YI1zLKi%y4`q63qZ=BXWP=#{N%dF#PIy<-o)et5{U%9Yh2uY`n8qu0q=bZ zH|g)9mrL*FmY2^Pp|Mr!Mn_%7_PvSX=j>p@aoVqgS(4w`N+HF&qp3#;#@7uF_OPj5 zO?TbFw;I^-d6BT)C?*(OS=odCalVxnev%4}a%!C01G8~+Z^-Px`-PZ@vv=KOMpkyc zUHyEw(B{5KRlG9L^XmX3f)aZmFRHI5W?ur!Bkr{q`|W&{dvfh9d)+!}Z@52v$$4uW zey~2IXy%hswITfdXxnLYV!`rZ<;>ee;YHWYscGZly_TMsdDWm(77j zFQJttP#VHkJ9shbd)QSgB!I#<6UQpp%lujM3o92{n0!|3l%Y#^;S_zM8;kii zSxm&9(S*d`lVi>6W)ZmgmEwpy0wEVs1c>qzzFy)G2F~(88S-eZg*xICXCj5wV+#MV$kTx&}f?B$(l|-XX1P`SAvbr!nUDz)TWq3-0SE3#+3xP;nzY06&4LN|;W*UBj>q;C?dOSk*)^$@Ux-K_0S6?-{Kg|tyDrQgvT|F#4Yx3vZsd7>9&duN%SfyN&9j= zrpxjm6PhIhv4mc42!3Get3BEQ8{6TlgRFH$AmmG>Li{d>2amnJ<2RdZ2R<0CrnCE~ z*4@`{v>#I7QqXk;^(!+aN#Q2OmI~k%V_2sc+n*YFZD!_uFJS;s z7@n4?S`yG1NJ=<9Zu44xGfcek%orSiGq!{it0u1ZMZv>-qOzigIYT5R_xz(B<@DSe=-dM-jZ- z>G;beAoW)-1YaKF0TCX*c4}a=eaiu#S)$WK?T_73gxv%;=F&q%B;l`??%Ge{Ezw6x zzxyOOKO8j51npZ2FZJ4YC5}gfn#xss|D6k{;DlJJ&-7&1KNS)ZzJ6Ez?;?DmsOa_c zJI>$#hjRe(l6S3WaFeHKmBoX8yEha%EwM2%?n~l)sO?}Q(XHr(-xGkKF-x595d{;z zOA36EiM*_Z?5LC34zcUxej~#O8yTh^~i^ot^j}Abf`VONh(&`$bulluTqn zLkOd@HVAC%w?)=9O!}(lAh$Mg)ZGZaWv8t0+Y;?Wm|DF;>?fXtH2q?xLqy&=MPg1i z3sfYTJKU<%X(kNM2ML}F*~&#V$4Day+g5AHE_!~97DHSH^t<)wKi67YV2?_C)5~34 z_{F+opPcMbq3g}nHnRy`3)@9DyQj||(}QU@wu5SJGfS(`1#n)~;g8--Z^~jqa#;;2 zMbI&7HQ*?icoIP3%R@=&UXeD~0D%>Xn)9#isb~A1)i@kH$kvTr^lMx8isOj|?jVjy zbc7)j#!oxPz`)=$N$a*(iruq!N?&b%WCQVm z9)8?x2#sUnqI&E8Xk{?p|Q;y1Z?c2KTT&}pVWa+PW@In5RWl!@nq%J z9(18`t^KI2@*7~{4hBbnLmZR_dIwhJc7rPVz3{rgq()ytq3O#4SAzXZ08BgzNPVD) zK>9=N6Uvl87|*z@t!(J$Txa)TuYz`qk9&YtQ#F=JL8?#WXZNJ*BS;1kDGrdc_x+sH zSL+&%gNZI%Xv{7i-XImCmsePL2*J3I2dAVrfC8yFT^pTi02|XgiQ5JU8Q6uyQAp@^ zIRbySynG`u)^9bxK6Ie%O@^4qmfyM}Mu8(+*r7M!)~HNZKUsGyPZ5&XXiTH9+;-I3 zuUq-#ZdE8Px17;2PISqF>D=kJ08m^y2XEG#&lE?rOjO=iSW73>%t70>0fC~t{GonS z)1{^WL$R|g#d&v`VFCRVs8m``00=EIPqD=JB-OXKJUhJK@ocT%L$X@!|fPXz)1O>c57=9y1VLv8NCTsX#1(VkV)ts_jh6UzE-+7V9pt%%o@KKz*7Ck zI>jq_3ncDaRtI$5cigzk|BB_|=kO6Au3w6L@RN(1r+0J5MeJC6!8@D{C!jzn9n2)P za*&E2gU+~*6Y+ZyP~!e<1KgYAwU)f|Kt64^Heiujs13`2sO3W06W#hO}`U!W{H zmkfb+e?lJamwV~|yoWRU@i}@w^a$un`>xL1FDdwBdV2Z@>CrA=W#~7P9?1Fnwm>~D zG&D;aT{746I~Csv7m{jVCa`N2in$yoVl?WE3A+`*AtJzNim`>+RqyBex~;D*4AUIM zo`NF6wEKa>p%6rb+mj~zTQwjSWJDx%GaOwz6iw8BCpe!IMY*I%7y0bHCXG>;_;{K> zjx%OUe8WG}!a^3~@Z}L(aKHW`ikk{OKe&aMHuNOQAerEgUMsBOnsW-hTMwg@U&}l4 zsnhl)eYGYj;M_)uD=zno?+Y+NylnAd@DnpU3p~7K+ScridW3lO@>B&?SKL!gu1G^# z29FY$?+T1+8`phl>nLLj=kwOkZ!P^)U5m*8UnlUVo-$ZiYcX45QuV%d%-WWrW^1W! zAlF;orYqrvu|&s2?OKF!y=6;;grzaS(SQCO>Vxeo|4j0*HX7uv4rW;SI(2^WBzCR2h){nr{|1bCj>Gk@7YGVd-oINlF&Yu z!{uyYu}^eM-8$1*VtKhZIVsx^z5NcS-FQjTdSav8{rQ?HFt9K(BE?*WLzm==^ialx zyVJSblbcL_=35i(sh?+>n4Afk+;ORX>mgQ2eqEJnZjWR3&I=s)>QCa%nS6Yjv)NO; ze7;xdY24MTaU%Giy2kPvmfE&{bzNb+^X3hi1@}Gh8Dd2{c`v)J?(8Jh>vun4Yqxpj zx=>YK?NTIS+W9;gD|Ix~Mt6mcG2Q)FlPn-1Em~-0fm)9tB{dt3SHod2bcTfuOH^>e zf&-$&0IShMUtV4wA!^$iU*bJ%rwq<2v`=FEq*90xmOIQY^TG$>1vom^u%5yHM0ac-8s_?d9v@D4rqH4(ynhM>WAL7@-Ejo>%=`XU z^=Fz!_(35c>y9UIG32Ek4qUu*jhpLwvfsM=0Q2gNTOdKf&6(*TKp)<2v^Xee8PHqk zT(S_z&sK@o6NRl0IYSG)F2;*vOIMu+NLOAm?KoObO;7*2QwbmvKU_+FtL8H=t(Vl_1_2Xuoko33)GF)_$FHxj3b1d%Pm2zCOo#03P;2 zmv(6>MxE*fJr=W>0TdX@iygJi%=%-QC=s-@v`V~u$q%Lv6&2GVm9}R???p8FlbmNn zt&WJg2s3?GyP6Ewx9YBnhptpU+UEe{jGLR=|1-Zax6er$+4_bjqS|vjG-z~W=4sE+ zxA}BeBjUvB&BMd}k$S&ofka`*+9$W3WC8Iyvv@Tjp1WAfzVC$UZr;z7b535}BXk~{ zV}M9k*p|bVdX93ultWi^9<4yOQg>uVqSHL+P3t{8NU!X8NIb7VCl%_anB>Y>=Jr5H z+xrv8cVS>LB~vpTS5^fYP0bi;Gx?QOXJj##q}O}371oD_`sb-XURol(UcjvNTh4&t zs{M!h{ zOfuk`BPuK1gh8HFRaSI&mVvxXOw3X=%~DM4lFG2)=82})Sm7%F#%9XLg4bs1nR%RN z5UU+DgHrEwaYJ-BWdS`5WeBxe_$Z!_ zdb~(OCmRez4l`t}wF8YM+CajL@>}fkhoU|@I$F{%ejkw!I#UE~TC8HC$D{;8N$zC4 zeW!7kA>XR+>Rsw3i>v|Wu~8D$WwxpAE`^|LTSqaT5;`fHk>BZGIm@fLiMaWdmD#WP z&#ImUDh!HY-P%O0h z0YayQ@p@ZlE&a2F4xe&-Ib?pF^*~ik#PwAg#Y4cNFX*^>snP|= z0*R?7fGfRzH!3DZXue_}Pa;xr)%XI*^EwGMTk)OJbSqQ#LQ&!O*K!I9=0b87;>+B- zM^8;AuV2~rB%ZgUrBKDCrKP#0IaWDM zGlXPvC{x_Jbjpy81O3>Uo&=9^Oza&p^P7JFu&=}V`uf@#_38SPa+2yFzrU6{x3_qH z<4s86=H{vPyFc|zlIMjtdv1Wwm-r8Yftp|#xuVZUT}kGmbX8}yLkucFOCxtBwQ03u zb!-(5B|hQgj9}MqX3p~dRXsXfWd*!=8VX|zntvYVu2KwP7}|7tlhd%?Lh|^>bR?Mm zfrjf0u5G@y;xFpps{iuHNWd}`qDnRx@DvZFNdfv)YhY_&-~jwQO22{1ka0X}Y2EwX zQ0f#HFK>dN&D-aX?@i$?tXLyRwB*&EU%B=3BFXzouf>7(h4`%#?FW*sZHFX}lZuOj z*i$JfQwC3inLK9h|7F6i*6Iuo&x|mB4yit_Uj1V3Qjl9BUzDPuR8fC0t0j(GS57KmJ`#>syZ2^~4Xsa!C%yo4Non#+QUzwRu71 zGMT)GA7Nj`FVlEE`p@G?e5xvgKK8`8xd8u-u0aGAp?tNRb4xkS&~eb$n|9^}t4oVo z{XMeZ^h0cI=*LLsQi_XjrN+d?(S4ErBl!IAh~V2UnVihxx#!oqL}z8w6dYoMl_oc~ zSXp5@tyPKUeRuQEbAC$EE?Hns({BS?tX4^+B;abN`=r+@)zfWdXh4^IZH(|@3c)I#dby&`haz4 zWX^L}C6p97EIJfw&;`WHk^u4vZ+BD>7|D4FAO#BpGUwDJ&fPq`V)F@+vAo}z{tnBJ z`~@j{iqq8YfkgFj2m-a$FhB-GqHMCVr)fau0q2jyO-)TDF#_sy3OgSSE|J>4o1?g5h%diu=<&XxAj7sPzJrT}r(?T69>?&G5%QwW9rfLvM%;WpGv zUvm0!(2G{dNS`{@O^1X6qO^&$v^2ko5S8tP0?FI-d%A3FvKQ!I{vCK>QPEb)IpYK8 zYHo~oo@G*eO{~#DS==@XJHA0I*vg9YfBJOkWS9bTT0OO2{^zTZ+#Vem7{ET{<}RqM z5)!){(%s+9eVvvjfP6nAa{s`!!WH^WUo`5mg@rqrZo_y{ai8Qsn!w-yJ*+k5ac__!t}$xIv0Gl;MA4u zwRLxwBZj$oc)Su57ke}icMX9EZg!Y+5)rxV=I##Q4qgU7e0@Te+Avqx)P#z-b?zg( z>7ZFqip8Y78zqFh$Xad3XO(7VaVz9RVw`y}b>)kP(nNHfdwz z2SMt4i;9X$&J}HL7OsAwH7AwUc1xKNMu>@H4_OgnViS{-`pv!MFK&Kql;#p%@?E!@n+r zgkLX4t|XLlfDFlKfZQlU3rp;29JuzgGqjQS=H`pow)TuB8S1Dr{2A+WF4kLq_o#p% zSg(0*B*mlDgX|ixtG2E=(gyjQEB17Eu5Ix@MS|*tI1362a++Htp~r3>+aWJ~CZ=pR zR;?)zLW07i!9m9UmalQS&6Q!DDsfY6m63&o+=-#FE0n;7==51e4@UN z!1-QOR`xH*I&-Me;ljp`RqIILrS3fkWQ(=b0K4=sIu(s*Zb5C1WXeaC`~MKtYuVDnd={&iUTc(82n1(wN; z4e`VT@kIS{8B*^k_7x>wsy`F;5D&0~Ye}>Q>yXIAM7<1|FF#hPp=XNM1NSL_pM94D z7gwmbQc^VFzw>gsTUM__>+0%`H$&V!UOoDHJhP}(Q&dz{<<5bCFCbX<=DhObjh0LN@rJb8T&HU?6FlzkfrKS%_p|97>z`)Sxb(^Q-(?N7}>}B&i|IkzZjJ`P>@tyM0 z%ST^>f;v?3XE#b}YD#m)i%_dthLGS8z=8p8F$yTcY_6^GmkHbkLf2}~N<>vvFt92{ z%>7MSy82QC=Ox(m{&|ysW&~I~L3WVH%jUVu%Rnf?TKwb3k5YaO)h}^Ocs>vpyLb-p zfWpEQN@^r9XTVfVO`4d>$;mX+PlYiK~f5%afqFd!es9%m9i5(dU z3Th*Hm6id(lvqbiOf|f9ZJt~H*R7}iRDTF?k4jHLL}g`VkiJi)FrvD;y0W@5;QY4U z0P#q|dW~Up5TsIrb>85I^-CuK&gIfSVbtH@Iazmh_ZNJkq>cJ`8L&}^%Hx&_A5*>F zA|+rNi}0MqDe`>a2*NGL?Ow>%I7@A-AzN$fg@5ed-y1ySJtaLIU=CT-($}8^mWGkK ze#2?epLvXRI&@Z4SC>&SEZ`{zwGIoos<;3fkNaUobTkd=f4$N7hg_?gx@u2{u5Zmv z55Ny(mM*pZeOXTGM!}yXVUHQdI-4^Yaq`uc2$whCN5?q4hJ?=bk41ATz~ z!+O>-k`h$a#M2ONeYXoNBWuaYtYrO{Vfy>fQf)O8 z4#0`51Lkpqq^ycFVmX$_5TF_nl5QBQuE0!CgQF@jPL4-R2ed6flDE(OE6U?hlO_mAS+Q2m*8n}~uugj(@T#BKdVJs(Ru zJ8(y5(QO`MiwFPNRV0tUrz`>)JGZOZDoU#%z#J(l^}-k)|9S0-jz_{J!_+hDuqo=1 ziOD?E{=&IvR>U7eL9zeei-{yS6abwJ4t@i+P%iMQYilixu%Dnvh0Rl9E`9=IYeNab zRs+WWjCZEankm-`bi z{BsTF7FM!~Ee3a8dIt1h`}bd;fBT=8N%D>Lk16`+`oj7@U-CWcHQ*rfuZvawfA0BE z{y#gAghc-Ke>N}4>u>-2i2u(_)Z39hCC|$X!)5F#Amg4(o`h%rewQTAMq3na*j)I} zB6%ney`*!Rwr~9CF8i?h2a|S47H02JELipG9AIGFS-wUYk-6WBJDXjr~_P)t}NEPTdjPg)t}+(H+<;ZHj( zbzgYI^~67Z4b1K>;AF6Io=euRb||s4x3|wPfpKzhmVAHK;4*@f)l7SMfsZfKDl1wm z`qij>6UBv~xJ5di!BPiB2a(&OikE9+h5>emp|RZ-AdhxNCp0u8WKkl1xXn8_Gj#xn zpEba#Dv*?a9EDaSGi%u=UR(yWRHU52vJxC+f=;dSDo;=<@s@q=do!cfs zbDylOof)DoKaItlkKd3$twR;FXff_l79f7n(vW1`4482*`MSR<&O=UBB5UqS)ZBHVNev|Dacq*{Au< z29%nhP$QI@*HeJ;n(x`>rV=qb z1~&lAU2BL{if5ae%<$gVIyt|yqpx=ct+s0coLr}-%pGprCIgtb7alp(rl($jL0=RV(rru*ei{tkRAL`>j6cu&wVuVxEryx zN))$htR1a~oI%1LV!(oi$G}fxm|DePh zbB(Vg{PotQQP_FVjfNeR>Cw5Ai?goo=IAQY%dh?8{f)OSd%hejC0iyXBrk0t=;KL{ z#7HH!KU8za3)6aVh0lD-UZUaMwk0ls=BLQeNrwug1`bUEi!r{~s6fCETroA<)y$Ni zeGHds#fypSDP|n5on%H*@oQ+LZVSLM1SDyREz3lY{Y=`-nr$N>GX45fVF+jY#6uMZ{~L z12q?F%-FQ~n~s;F9~9jDMdU4jat3a6Ohy z6+=jrIS_L|_R}>mG+5jYlMSF!^LD90_!GpicCt8oZ7hpEVv5iJ!Bi5KGSXF- zrq*?7X)S7{Wu~i(6046!J>0Pd#D>Aag8pQtc8ZucsZ~br#cp2%ZRc*lLnfe(C&T-Z zhSiz&ErDN4%b?%s=ojP$y z-BFF`N-sq#`b*NoK^=tId2FvU@8!FONk)}tPzd3mY)1s%IZL`2q)@QZOOE!HV7fLs zJhC%6z8HPH`SL7hNu&2(dGL=QfC{e%HlC`6Zclo=Rq8RY>oW9Az2xCJZ2(OtS}ILU zK5OMiB5CB8`h*AY=5EYH*Nu^~iE&09&{j@iHfw}?lJD?H>2@~65DIr7VnmlJmbH!! zy=ekA(%E3O`P0rjRT!$k8R%U3;6utzMv_e5OO!$5q3M&Z)|Spmle-*LHk4jq7?W0} zmA>uN&SCjp$g0&>*P|Hr>wCyYRbc92uNfceLi6Bn#|L5lUY=U=fVF?R9?2Ka0M#z^ zd{!Ls>B{TYTWWiUx_rdUiX8Evvt88i%;8u#NV0S{R466`7M_;58un$d?pcG~eW2{f znz`}Vj7wSPO!>zQ%<}X^M7^u$nB}){lL(qZ)-xsLcbMR+yNxnSlLci z(+1pq3mr6bo)VS60N&zGtg3lzwRJWl8;p>s$~wv|Z`cs5hvBO)l}m>AOB&oa)UC!& z$l7hS!OH5B60%(9kzR&R?Y1k`j3iW!Cbq9344~68=gONh2Yg;w3-cAgb>v)pB66NDjejOscz!yHf~gqY(8IRi6Ys?qKDqD5 zY-Ms2iqce61f+@drXnIBpj7EiKzb*%5ETId0g>LNcOoF2 z5CQ2;dI=-~LZk!;Aqk|O3-15@yw5t%TJMK*)_K>-C)NsF>}${7vu9?{{05~L#5r5` zvh@aN*%qeESOVUo2}3h#?@aX>M_GWN$68vz>%444R8Gl=rMRRL9GEF2CTLaVvnzoNHo39O9*|p{LB>5 z_x?+`E0xfD0T*bb=8Pd}X)z8X8XYxlUs$j=PU{GMhkdBPvkO-}#2rv4r2##B)= zOLlFemYjGG`gUsQ^Vv5!#wj9l*uiCX{gHH5ZhNETCpXs2PKmY3z+X>FI2a`Q6m3Vd z&9)I@eS=sY74v$N()bOxhs>s-FkA4PymDsarc3C4S$&qyJoN!2nw_;=G+;vreO5+N zCZN==Ua@7wzx-ucDPzQ%^+JnRd)Eb3Q5wZ5E822RnR^;pAON?-lTH z>zZ$(PD7ogT^GpLpIim{T5^6ZORWP1q6y4Y@MS4iW4NfPfh@$jLaLb2OaV@b`g`1{ z{f=lhZ)3sDX}|roj4CS9m3)=dU#f)f7aBJ8-{FjoDwYAv#$m@}(b!6Ce2#@Qd=-4! z*MHQw!P*;A;ZG9h7p?dxYg)3JYA{f*G>M=)58olZnxShZX4JduyE)pa-kBkD?#&_E z=yA*rmR?)wG~QgaA-C^AM68C}^=JaCQrqEqPmE5sVv>uDa3`$Ypj8jQN*J5sg)DDBeQW zq(VOZjjR~N6cqK6-MMOhY(2n01vz#nqs7VkHO4k5ACXnZvamP8vXCedF!dQoUgY&1 ztO@fj`ceJPMzp|&P!onrseH+Ue)MKBCR)^Fqn>dK0oqe=#@6&dx8B;7WOAb&glBs$ z;)&35Q(v$2k-?LjTQ@L6qY!Ugkclc!2q-1i2xWOs9&uc`?a!lC_6l=^R9P6;V?HC- zcn?=s2;bm%3Z9}}a%(kDFu(Xqdv{w9v9eD7Dxc<=h=r{uAcjvioi3^mEwGtuCmwQ* zS>&~$99=Q;yQR=dX2#H^3x~2#bcbct5$W5|?3fVs8gD|}gNTOPd*t5xMm431@DlCj zc>1*}#X&Ky3wU)>>Z{8grc>^S1}$`=wy3)1X}tzFy_ZHjAM2PI7+$4%q`TcK`u%?8 z&7~3tH`=H>IcDkASy*N>6aDL613Qv`Guh%+0C|efp=X-zj)nC2*hCUp_@2uiC1TCtxZ!C6W zbdcDHAQJg)m07(L@&kEqE~tu3P!umtKRb%uRomi|Ql%+xfX@c9r#4o$ zjHt%JIuM(dP@79cU(Vv(2D|j6m)!?}v-Y;OezgT^5E7Eq%>CDDW2RAfGsGRXi#fy0 z(tsBxeU>Mu_7}x@$??IBrZVA+$XV0{1MSzTIhR-1lG#=_UEk%`*0uQ-+?F|97)~ng zs<(RDRub5U;)+kS?4{f(_6Stp`|H8>@^Cn2kz?Z};1(q9RyKGpQ$a3gpYSIoxsXOg zl|0)&zR+*<+yr#^IX|U{r3){dG2=ue1t)L0wQjXIALVmdENnvxsK;qnJS;@M_izqa z6HD*y=Cd9Y{_gL7eq%8_1^S3x|Lg^)Uf~QSs7v6d%&~R5TCXWQE%CMc55We{@gf*{ z!hor~++MZJi^p*@%>o|cJf0?bOH}$(@+Gj_omGVPco#0Q0yE4EKR||4<)LTYQTnLq zJ)^&LVc0#&8+fg-89mtSNC9X8vM=?S-4Ic7A83HI<+`JcAw76$5z4PF1FRB^zR%B( zUU5~>CH~T~BzJYEn%^+WCcV|kQhxNb)qCU9r24k0MaWbsZah%2!DV)qmrQv-fw1| z4WDgL<5l7CWCfTHt)5th6<>w;_aE5kRBxP4%zH#s04vl z|LnXvMUP8WE!*8F9^6~a?pCIb(Iaz4;@I8p@{aWc5qbI=)QtYpQYALV84}pJz&`N< zH<&p;|NQ)3-4@3={;{m5V_rqj=3iq>Q$Hps#_P5#Ax$m`FWibUg_q=I;KP=Z5%v?KF1p|$z;l>bOe*)(3R+Zu?r1rL5PfLsD zTKP7AT9)f)a^LgG_sH3EMs+J&kU5UXt4`PBdqd!-2Zs7$ic9)<84Bh}(0O+G z+E#G)%7xL8G00>6YRc#6>4L)Ou0$vzaWaX~-4*b%uiO9w2CPrVSkPbi-ziW_1d*p_ z)$)GTM$VmZ;I$gGK~ zsY73G^xWvi__mW^388CD=$>KAu10^Jpj$V+vB9TKLou0-1Ia5&)R&fePI%C~KQ79c zoh$x<(9313Lk)`60Ctg3!)}SD$U-9?LEQ5tTt#}|+&7j2EFRj~(W_rL*xG)(v5A;Q zCl8y038ji=(q`!*M&1ni=WGQdyeVVGd4S3!x9vQ<6L{+Wn4BS5!n4q7W60Qp5QMD# z>SYDihm16|1uk6jG}qPgy_Eqt(T|Bnj2b>H5+6gq;;!VVJ+{4GTu@^_9JM@^=|}l| zALb@E<*KpXANsW(u!f&q(iqq3zMTE_vCbEf)<2Q1%E4c^Q^XnZ#|2LSE5Qd?3H8k$ zn+!l+$HRH-#PBeZU6J%#5_i$LuhfI|xv7vifzSRe zBep8G1*QmSDcAL2zwI@TzSct4Fg`d28I4vs9(fUdxVgF0AzMLEnc5=SlslM;1}|Kg zLgH{J;T#b5vkY{oAb5Fg%ZX)ryTTpOvi~@0*-LskmVs^AWVWUjg-u7d5h7q&SEeU> za0QwDP2(!&$E_I?w-MXoVn-4|sIBwDiCexf$tPQTUHq`%TZ+Zs09=(bY)ss9zB+nynPPiHxBVKXqpCG}-EHUg69K z4X-EjnqJwd0Tgi%`|h->@My$|NujZ}TIM0+Mi*M*HT6D2*INd)MT|Nw8(=f|R=X=i zPvVy0^$t^>;4=C=+{@6Hf`6q81lmlRH71%d)p)HSUi355`>@GdfD(X;`P*OS?*k#W%CW9u%aHXccVs>F zSN4O#gZ?Lbf@+;Q&S-RWgx}hcQe~aGXaH>EARTtQijeb z&Ja6Wo6%FQW!vOu<*SB7YDbgH6s0V)PSHqAZezMKdJ{I!onBi}=JTfk4A|9EVK_Z6 ztb(L-i_d`_Z|k=P&(ihf5m{U|lUsw;FcBq;m z-ivW|Z*w=j`7~!|O5PoYsYm2@3d5Vh)GvRr8V+ZZcYQJQ*YlDDi~-m6ecNkU0eYTl zJrw4k7fi0-NdQ%pYoTMEZ)nU86A1EK(dRC?IdFm|lTGVvXnSDD!~!{^MY@4GyA(gG zjP!A*m(su42?<`5#cQoHSOsgUMBb<)&hO*o36;y4>G{cBjmL8V#eEfI>fS6gTdfG^ zP&W%owdVRAH*(jZ>T^ zn#hjE6da1+R~Jq`J7#o~^hYF{_|(l6{min>1I3!%5x{Tep^I-5kaugrt`?hyh8BJ$ zrS2bXw=7`0S-9yqbPHW{6uZ~pbZ|vTkU%wnY}YC8&*tMrw~P2+YP>y+u`we04;P-yy> znwg`;zv1a2j)hlTu=z4i&|gXGLY!f>hR{E8_v&0*fq3}-N+^P0ZuQ~5i*XQ+2wgx# zT1@P92|3(~e>xzXB@eMe4CSte8IaiwURJ-JYJ1X7gy~P#$R?>+@?Y`5Qi`A<)E<_p zqWz_SPoL|!P4?1jwbv~>>l)}35GRYtaiIov?*(2SdufSCt$7-_IG-?L7D)F6V{{2co+()`6e%&0 zfPo*fTgDlYY1T{v=QQqr;BXZTUj^#VwqS5f`Ns{OW-N|nfuk(d@Ws!B+)Wpx1=*`Z z`kPB{3tu(6uJMcnsKC=S_|V^=vsYU6VWagZs?H0++xg_Jq+}80?K~!%Z=-X~;tm@P z?g+%E4JwO^Y_5GGcQwoP*@cC-OtXO_zNj|-sl?1CcT!8aPDXadl95hdNR7kw+HESv z`;xZE$Q&H0y>4G>q;X39Hn!&{@bf&np? z-tREfuh|9Xv{F7CNBrc`ijEBr0d^UK+WL}orn`%j{Ptu{Fbk34~iI_OuA%sXg@7mm;m0#@_Re3><70z z?QCbdhGcP>lX28vG}sB3dH;brOW4X>=2+D8g0{8qeuYI@KD4yx&l-r!hQh}Bp3WHB zf{fu13i`Hu%y^*8n`aK*UhBw&K8LWH{2q;#-+~b|S>{uhqR|gEru=3<*A5%Q+)!7M zfe0;WsQ-2VlbTGlatUV~pcg?V-)(H}nEDaqE%dk{R$>ia_**I23eIqTAE~X9h34h& zf9Geec^AhBzE3r+-u);}iVw6N>PxYlqMImo;N|OKUlrvxjZ%IO%HhWbVz1nbKkt3r zw81U<99Hz78;A7-q>uV%#z_^Ymk@eBS^J`3N|gl%j-lZzX@+&2K&@VGC)abgO4;?~ z0|#6_Q^0_#P;rdgbw;guz|ELk2Zih`*%C~RvqqWDD|XF$en0(t&wwnKr1K;X*p!x6 zHT0`DCAVpRfJ=e8*h$(mvt_3Z?tBfuvm>HQ5*@(39wTc1|MpFHT+4SsawiGTgkw9+J_dSEXkbxFxtCew0or;7yLc%-0Lxf6Ib-WT_iwr0t0_hM(S~qZTQPy#!tk!V{qT8Z!l}uM zmrBkkUf;L{`sTy7T6tuFy@Nb>QkIvxDS!LVuoSJV5`BL+0YV=Y8a4s@AADK=R-B5q zz`lhB@774>lhV{XfLXP0(n!=Z5eK*L>WKyt0aIzMeAxV+Dn`ftsEbbto;aK>KY3hFN{vG80!;H6T4 zAcwJdR5=sO)eJ^)csjB1|Pc7p2E)PBKbV}A*i7{hc8an_j@W_$=_5=?lAqwU)Wvlj(gk` z;9%gH7h!LgI>y$^%6=Y4tvoT=@NAGz{ zA#?WLw^o&q(crdA4|mCZ)P8xaU%mPtVRv+Pth#SZO#C$|CNHImF5|cfc;4g=35N!?(Pov*h?s>sJN3U zHvNUKl35FDsX(O0Al|pqmvCD8a=I#O`C6cJ$HEE?Mz8nYYzjNf#ZYVNoNGCMEh~8R z_P4~~;D}+VPkJv6EnXU45UZLM5cix7Se2Z<)Jehh)nVh163| z=t@9pb!em>a`Xtl@?UfKf6vzX-+iIxt^ig1?}=jn|IHqj>6LN`Cgf?NE)|%9w&BL@ zxsU!$e3j|pg6)z~B(K9Mdv)h1Gx}T&kiH%p`xJ1>1@qE@v*pdl+y5iewL5wcndIVSwgkopU#Tv8@34Ilmond66*djI+S zKgwr8+~Bj-@G{JV}>`< zilt976bwfoAu9kF;~F5p$W7QbeV;Iw@)rXK)WUnNkG{7uavUg-`hP8rYj|yi#Z-YL zK+sQWFky3KUWzj?+K4w4N^32EKVoez{8%NGHIW$zsIyra%|pNR&_xZ)R><5!ajS-wLf%BWWC zEH`KY$T|Q%|JjLe{8>jbB+x@>Bc58PodTe2^3QV_=hfdy1No3)_)?`PWl_fzLTwqI znz}Zw6|{oA#k5MKHywoMYy%b4M^8G(d{he|Rzr#E!-=%`11N|evJf}pgH7B>r5xsW zjB>LZ+LGtgy}85UJHH>FY05d|F0Hr!6iRsXrd~dLOs8PF_e_~66--$x4>@3E9Z=eR zn=aid()&70QsSsK>Wxa$QD&@_3~PC;nHSzbHZ1I+S53_!%nt-2PSFp*pCWgQ4Of$s zjwHkoA$X*Jzkhl(UMx)e6_lW3iID&(Zz~4R&fObLw2eLY&-ks6PJuVqg_}ra^Gi$C z_NS(&q}+r3bb{2ZhbqQSM6gJj`juF1WuW%3M}lk2LMSJI2SE9TW0{M=I>5F*3OL6O z2nWbg34?HY&lHN{F7`gLq9IY9;_A*VRs?`LFc1px+Xs3=XtVtyfXDJkqKu>lB(A7B zM7J$s=pfS|YSlI7nwr|;3MKnFp*Yb@^91Q81^&xd#Z>%*FvDctc~7$WC?+^{rNCX2 zlT_)TCBN7hm2a3VV$6}Q1pfZx&ppm5MPvGoZ5?Qy+Xr;Cg2ZkgnJd@A0glO}1Kh8q zR1`@<<5ASnH}BON{d!35^sn9^m7xIft6}}Z0@*;%ANlPds}eha-G}u?d07LDNtM=| z){gbGQS0^7J-*vbd0UHW(v)8)x$vjxS1T*SSNB%=S5^rSuYO4&K>P!fG&kYKq2SI7l=k{_3P2tN3GLvWEn* zY!Rx<`7ZwW+<{Lvk-gO=u0i_A2Yef4peE9`?(RLH!8n+}EqKSg@tWLV(@qS~1c|!l zPQUgv3jk`Sdk?4k^+%4f1g6eDG8lKZpf~?Hzc2xeb}o00M-v6Cz`fdRFfAZVa#CgJ zyt(~eU0$cPyBse6@eH{iI=-0b~Loe(LzAe;>Q_PhtfsvYRM+ya8L?8B%ogKbx*#w&3uMISSS%62jBJ zVJ#tCXE(Y`X&l~^f_Id5KJPP3M&Pwt32pgX!O_87pR5T1t&7Q7rSxtuydM%L7@!HE48*DY*xOvyAz>};TBC9Pz3iKPwnW6vN5 z0O$_2Y%W<%oN+L$+~4HCs9}j00Cqi=8vx(^8w)Vlj)c$>T^t$jipk59;=6L%)}_f5 zZ-DKdcQ|T>v1docY$y8#1Vqv$g;ooIgW(dg&33oAmy`WRHN z_gV+0Y(#PWJU)?H*>HJ-dbx-Tv0_jp_gA7L^MTWXB^sgHX#%vptHc)*PQ}R$d!~tQCa$!d%+3~E!7>eSlv?{w`*&> z*;+R+?RU5c-O`s0N4V_62_(`VKlx{B5r=544>}t4-$GUflBZf}?(bft(uW6X$7&Z2 zPp}6$nTKy7^Za0}OS}iPy*5wXPk6{R1zvOJ8%{7a(+97X+v;p=Ap=mpItBzT(l=s{>h1i)e{jAnn*kakTee}0UF_knY4EN>ud(S-ozZ45?KGqQ4!F{}3< z{bs~-h(KT*F@CA&J$Jfi8X01AB;mE|N3O)*tomAUvTd*sqL|k4TTUba{&TEP0!U#95?&>htC;k31!Oi{lPr-T!vflI=XCT z=!^OGU&x_&j9ORAHwl@yex-Zvnd9Nr`E9!gpAW=Iqb=kbkgt>^0V|M|H z#A|}Jv1|~8zdi`F%n7i3SRlXkK<1SCh$`CtIEsWMAl%|MO%H5a0ToK}VuH5b_=6en zdO#9f@1nP0nsWcPwo|*y{CTX|6%pM)ZNi7KGk&zBXYq1>YMVnB%5O(DWiHK1KIGDE zEHudS^BR-|AwhH+VA6>qcYEgKw|ErVf>+`JdkOFxf#ZVZ9}De)3ciBfQNx%)NJzfr z6>>f`!GhB1s-n2j#0fF*T^A5XA%Cw7T+Q|u|Qj zEVu9-`^CSiQ#mLgu;rp*x9_r}IoR~8wXJ7_LWa0dhv)>?|Z-BdSQb#aTHs51vaH>-D_-QBqrL{jlYu_772*p_#v;`Z_ zj&->rK${_y$QhjvlO`|qeI!mTed-Xd^ugw(iel#>bUBWWXh@?kNuB>JN7|(`tlWm< zK-+_(68gy^9fbudv zt35quVPbEvC^fhs={VUpo1!%1Kjy!^3mXR+SAC zR|T1;ifN;Dl4#iV)q>2tj%%#V^2Bd$3vA#|NeSln=sTaAyw5n>vksMsWO~1J!+zon z(7ti3V$|KS%lllktd5)UTA3&vEcUJ9rLcckS=o_)@Ra4q_TVU~z2&#@xT%!MiaCiq zJ3c41iF;R%sXHy~@urAbh0rlId0{QeN4r19v+`QYHRY#pk{a*EOkL1^z_}^lq@5Br z4bNzH?bSZ%9i{j61fWo}cB5?hp)W1&xx1AF+7Id|6m7mc?&Y748r!|C6ML{Xe^kJJ z$6!|SP{tfC+l5vC9SqI-AGR-uf*7CWVGs)^Y_2^8fpclxLENrd?J~8+b-+BnUlO&` zWM`e6xUUQ^#P$_4>I?x~jDreA3?G-d&p9k|SLimQ=PCG3hnQZ=4Q_E;@r2!;8*e5X zw@NRdWnXRx*U3d}LByO=FPmsDmgqhPpyxv}xV_SzN}d(3C2^9la6_xY)2!U|x=`#F ziHpPgopCC-{j!!8At1?A$Ru8?;_5s;^DwRe6gQhX4aomnh-md?cy-1)tF>?CgkEty zY)|)mMBhMMDqx9`c^ce;u2Yr>$-<)dm8d=oR2J2qO~>d38$GwaQ?OQF&{m{uo6tq0 zwpS(HFoKAJmzav6uxUfA%t*V5`|@L>0=}mj+SZGU?(lDb5%nZfqNe3KWlR&m`2jnE zF<%TAw7hguN@eqjql(C<88D-oU6oRbmNn@k4NNFpysh!SinO!t)OdI2NuFXL**TyK zp>5CUJa+?5oF?8*cI(_bCXL^pG^L?I54m=WwbvkKFBK)^^m)AXm&0#?k5%NF$o30~ za)wmPs`!L*IJ%r&`dZ9)@uQKe{1feZ?h3$|ZnCq(;9J7e)1`g#&9byQ3)90rfACj{ zm^R;BEvS0xfLleIWG_rKE~Wtvo=F42h2l)Z1U2zhjxEN7Xhz?cX5pFyAPJL`jUl$3 z&HRV7k=WP|G{hvg59V}pa|^-e2_3VAyhIh1GyTcR%?kcnOhgSQqZ>|$S*-jHzqh}$ z{9LK6zd!vl5J_UK?7gkQycMYZ4<0&An0tkL#98Mb)$+5dhevr+3;iSW{>po!A6LE$ z$_8)!iVrd&)K^Xdz&xBWfRjns-EEc&clmS2;X_e20K|Le|LVkA@*Z_{Ybzct`+7sT zFp4E-ywwr+T+h}`7A$u)A)rAL-KJzuDs%0L)l#Mg0K#V8`d(e`tRtq8j~ui4mYW28 zvbYycN-gZ*j+9+DvvND&pT2qP2MxLb^xW~*U3+(6P@Hw~BhbI67C4DQdm1n^qr@-N z!sgs;`4!fl<0?XBrLnSYw!|$f7hjh6(Y_qr3k6yyD;i1!$|Hix6O8WiI_bUw-NX1E zHU@fDX%4o|>a!;F+xy$9kf=M8#vFY^h@%fcV;cKyIA_T zZMpMvk7AvJ3!0(d`rhE#3<3sU02EUxKz%)bedx5|e-z~+hOTM=fu7tl$Izbbz&)sn zFZ`u9!MV1kmN70)_8u5e=3Ekt#?2lfgFd$KrWCWUt0*`=c`&7_c7l20P_SRia*dX)5eWn$~(P^#0 z#@KmTasI<7O<1*w*ZTUO5-KjMeP-Pyn)`b?9lJ0>b zr0otk*r<;Z5LzrkL4bar8#T3A4v&cse{cF-FjYx-bzi-yNxUb8x7r7=!@4sk##>%l zScrw&bT0$oZnjo3Mn&-!U^8~TD9>X#Q=}CW!i;V`&>Daq%$!y}m_FEUJPMA!+3P(G zyw;nC0Pt!888W?>V0p#c$yy5!X!6eCAb=>1v#eJj2BUg`I1;NGG##6+rwO!S!WqYG zNCRiX6dfH6X|7$JpYHiHEiAOrI}%LkgV3g`09+sa&*uZ&IS9E>JjIOuXYQ%Drda)) z<=hqTV8D}(&!Ijps4jKw(VRUv81==EEfA*^cRW7b0N%fnN=b>M&86)24;Y5fBsi<6 zSOT4_huVLRA3U+$=#olf`BGSpZ!qG@)&R^7>*{Gyn}qXf{8_ z5K%^8=RN3xX8U8m#|PY!&svY>E5EkZ`$@N>EcQ+_8W--WO8;o~Y?k>8pCOd`c`T!VcG`m)JJ z*KPCC!JP8Kg^x|E4pPh#k|yr~hGP3;OZP%1_6fzEpC65KdDxpF7O@?Ool@56A5Lra3OLxc1CA~=ewYUiXi+o} zsur@k9Ga7KNsyy#xx=RfMy*Qs9ubkNu-_@@&o|GJ+TI2yW{lTg)c2Z-}iwT$pM0J0Ml+X_29Gr zefPOmS`hsz*IB@-Yi8ulZeC?=Pt7>LA?*CKXMz8|xBcEE2uV&6=;>)$moqiVCIe2P zx+B;+2v>wg{isU%Hx>|u9**^WfvMrvvvpIq7&9az;W zIqc+wWtSl^UkAG`F5Yx;MbdT=y|voqpk{t49u>R?R>BW$ABI?89SV^ZP!)#5y+&{S zki_q+^j2}wwXvV)!3*uK@D;Y=LesuQ*QPaT*{-gLEN?@xAA;OZUV%EBAr$nmhYW+v z<=?C)d^xAwxfB))l@&oDn?IkCw*W^TPKcbko;>3={bW85=xdQp=?1Oy z_ydJv06jc#wVCtXb(7!gL_mEiVe8xp^jQnuW!oWPO_{Caiy5?KWnOv@v~K%Z1!sGB z@su(tP4uwg3UGIm_{?0XMPcfFGJ6h+yEdl$O8{aH9bWp;B9PQ{J$nW_HZBUx2XfQS z`W}Y#8c0Ckk1mg@O zxI6-mOO@g2g@x9G1Ckougj_6d(%{C_{Wc2{U`4d#&GE(_NQ}rRW=kdOI^U>jpcz5| zPWe(UO9%z%oMqSV+0&irP0rc!r=`#@0XkUB2@wMF9>9%^04I@ZEeoeY9S};-fDlll zabEOzZahA>>@D}n6|RqLUrB(<^;igvd#3EEHMs^AH2>zIyY-3F6!5uPrWLPHE1cco zLJTtBE)N+?Ni_O;oVvZ+5x$^RZF1cSilX>X#h5<2STy9XWv+ARxFD-#r!F~&WJ##n zs$FUyW{#`k-akET*5+bDFP^r5FXp}#Aa;a7Qzh(nYSQZ!{n5OD=MT3G5`!FSC&siR zi|D4+fEGLwKqmrtUyaovW&^E(9Jx9xC%aEcle1%Evtx5F;N^>b#p}EW&Y!+i#r7s! z(naJFxs*Wt>C3o&4ezK&z=e#lal)M6mJ0%_>rW;uY-GDY9DhG~BbSlxuDrc;&6C)Q z>A&WgwYbym425ht(9LT+LPj%X!z9Xqh%+B^45r1;cYDpLe4%=ID@cuw3sZ~ zm>^CKH)BU1>SNw-u)VQnXs}{)bD6ec(-)^e+V9<`0O-Aoy;Bq*ga57u`ZAs@4Qgna zR>J;bRF*pE+#oQYHo4fZj_gM z#r|8z!uQo$#|hw`1+bH4yHtZrOy#d$m0w@aw{v_+{P8-_<#=lEuDwXA1fTy>*TYNe z_3)n)ZB8Ed%eI<^M9(5_xi5qd`BrDoRD*F$;$u=P^c)zO!KG%9&OrMqiJi9f>fBk{d~meYVx{y zTuNPSEMD66z`@6^&e1Vw?2e>h|bgZWZ{AJN76rU3CMPD$LwFn zhG06i`n_~HkeD~LS3aD1zJI7e7#RM&MSa6lDs*@3SpmzPs?{{+ zAA8@fmu2va#r7(tUAti3>nHbYst%O)}_qI*?#nlv1+85_nlUln*rN11EC}%M`FC!lBZddSus-Nj9(|Y zr0nUy{i@UbXU?&#)Gw3|Dmi$$?T0j(GL$JLKV#GJEIy%z3(1QLx)U|zWmlc)?-F(RJ@EN{t$Xhj zvmbxjt!|8uYI;TTaZZaX&)JHTt`>Q8pcR)Pv~e&uJM`j8V!6%POG;&P7rpa`tKX@- zzkMRD;7U&$brp@$leKoX)7{lJ0fKmUlm!ib(kL zIl(mBlA6*hcKLP!{)>#BT?dg#<*7m~f~!QbM({id*>CLSH zEh_K5_Z4=%@k=9F?CKM%fsLfNm^3kw8&WsC9vv4+ShyUusg>iVq+hmG!>20ib%N)w z*oc&zhzGy8FY^2fb2yB*0-xp~b&lnsIRl$}U#$!rA0|1qn)ZyOpHZ)VY4cP4JQok^ zi@~{RPfuTqCy^bm_Z~Gr(sxQHTOWVSu>y5UfRHH%zNO|Ly=I<#CgIA^u|xj5wd}e2?4ll)KNDrX~L5KnYdCMR&n#pTSvibLZ z^JE`78C(~Ld|DoHZW9j%D#(BXzwOPTuKs@QdB9@aw(|b&Go^)qhB6AS=l(b}Jvw|8 zmt)!VDtV>m;OA-8Zk@Qhs5F$9&i;-xuM)R)ZZn5De}C_J^$!NR@Oc0Q+>$4 zJ+ZXV9mn-<@uPpi@Mh~5i81Jf`fYV?Uv=%IcjrTaw5V9?!n=8{QRX+V4w%U#q;h5Dlmxq0QT5(Xh2|Lfy*hI9HBKFawQI3*KcludF z=r-Tsi!V_yx%1+( zYID4H8LZWHykXU->y$C+S5|wsC713>P5=ax%M;vMC_y)CF@8mLpdt}H$gW=ps*l4M zBmUgZ&EY#HmS1Hv-Mlc;T8B#`o^#bG=U4oayiM{U=a0C7_qrBXLK>kA&OY~t{z-M3OGxQ@R?v0%p zT#cH@+m@!BuMM<&jF!xuyl*a~t$xvX|9xKp#}vD1auKDT6oiWu7Ys(0gt-(b&MtEa2V&0@ci99{1^tAxX=y=$5Oj-2m7uCl?GR`#dlqdp3= z&~Yr{$Hbnnv-aY?DNC#?ZlphDDOtN$TGRTZrEItP_K@>pw{{?@gYMS`?$*6LIF&RK zd6m1_!|*|BbfrNwz{NfuIFiXKDc#>SA3i!gTu7rhAj3LH=af&oD24~cy&52zmoNh^F&r>wX$X3M6MFW8NSn(fC%HE^ z;P=RQ+&5&o0)a3f$|XL(`$aXQE%>W1@7m^2`8NG_&zHaXX`#F{$qY?~lQ#>$-Owou z5TBL9&PRYeeJe{fbUoLhQF=XIybyeSScu^(*!l);%v-tu&7CcUkBOiv$PeZgp2(nq z0zvg_amhuy&L>(V$8#P+;bPW*jU)pT_uc9rrT3svP=>l(dIpD+=@=Eie*NLG7qlut zW6waRz(9?%UP{V6p>wWEevrMTl5BmWJl2}si;?Mlo~)DTJ)5*)u+Xfm!!vlQk<_UJ=+R&H`K|BXo86ES z(N#72py90Ud|fE)rkaa??wbwK-%iSgvH>T8^a=`WZNA)Wcv*DDvF}l-ljfOcp=zdmTZy_oCJ`*I>&|f!OKKpK5i{4LT+QY(%i+yv> zuU~|;SGOZA?%ohOUu^dQkh}52cr3ac>Uz&|pudQF1N3h!09ZaOJUOoRoPw8VG9pqK zoiJtG+Cbc6j;-ZEyPPot7O48YsI(;e4;uOxa?Qbsm?W@E@g4PE@etDS3!}$ed4mjC z9dEBG3d-DgugY-pb{(acPH^vayYWM6eH+v!v5?!TV9KI2P+EC1skv0Qtiowxp(@2? z-_q$%xou|!w`r}w_Wl~PHK28VdfTncU!j znBoOxjkrU6!mkC6Efs2R@uIB_UOzEN^IP9Tjti{G+IT)p>I;!4#bx;r^JW|3n12~t zWUYmWAH+wayT5@m4C5hNMByOFLdvLLe7gMFnof$Ydhza_@VA@Gzb@pfyv%NNu3gs; zs`I!l^otOdAjd26{Oitwu;>%N%{*QGWQ~5!UDb$s*M$s6KT23sXvT;#m!j#+EH+P5 zSDU_%XunjfYJW2H+F7@~nFA&?-PNmPCBHxx8jH~qFuGEIPxd;FJywGJQ_z@%1Pj;z zKAEJ`mTN7=R9jGWcnuY|Hz%(DyI^wxwD+qi(I?@pVB?}~@o?&WNcYSgw_^4`K|Z&G zYe`bp%8_6HIU;8#14E^E;Wc`TGwC3TYktZJXpFI+H@AM1!&$J2oxgvW~BUJ zKF<5c-d-|`*0+Z|B_~$6SGLRGrGmY!pERFUcMoM?9PFPy^c&=+XL_!Nfp4S4^`c@O z^cMtnuCbLm+682{O(c7|3S+_rFBg=%k58nF>J+Mkp|I`pEZWi;3<;5OcBul8TW0HyHuEbu>J?dgKeSJOWl;h_pDyD>sr?=G*XW65oAs&}%W}b(ry| zDNaa2-n8BOJo`Cc&L8m32bkYtn6=iddv3XInW5!5h5Wd>x`;yoMjL$;MNux}QAQDa2mRhh0@xahVxpS2>Ib;4wYzy{`O8?1)T9Psz2uKBS6MInb7ew} zaWWBEKCU)pqBpXsH=Tg>`B+|UZKo|8H|>O<+x$=3&P8F=ZU*jG{WFz(+XQ>Z2K<`( z`~ovtzGi99!3HA|#ohgl?0NFliuBTsq;`bK?r6MrqXWAtiev4v z{MDaT>Y>|c&Uh;@@H}BEW(EJJwQBgw57TEvo@spz5z{V=8ZVA!gLXeo103GYadeAE zUuN;V4J}%e>vvx(j4={k+ZvvyT<|Iu1)&6@{8!@+39oDUG(6x%!0hhu?E9tq=GD{D%WWqvf*r!o(`5GsBF_O?E$y) znN0@>yM#dR5wgfaAIx#o0sS~jf<|WZDPdPn4g_%RGJ+_-zFgsO8=V7hz z{XPiT$;dV4dQvjxzb?)~3Dh9iUxL60?lUuyELzKHC%w zZpnuhTMx|pwGBvky#kq&+fU{pT4|TeP{*dgS>ax{GNE{%e!jeziKynntT^OOEGTfe zd3#=#KWrqde$J8^C-3POVCSs+*@!p zXD;q#bEUSt)A0XNXQ-#Q=HfUqi8`70PlIAwn#HM9M&W(v`YKCpun9v1k`2K8E2EsWtWXot|5$>}icwAX`Tc4K294Cf++MO{9 z7zFEfvQ{E0+)l&U)?1v8zqpqYX#*N8<}Lq(PTcEy*X4AKUDx<;k9Az67z;F{v zgqE$y1d>qZK=F}B8DO)iP2A#Rh0JIrZTFJ0)br+|nXPkr2qmlCaNzfvkI!ARm4(Ih zxQ^0#@N`NL<98McJsB9rR=3H*&q`d%h;xvbB$NfC=^GIVSUVFXGjJ5Md zBe99C-R#WXaBQTHMUYijJwZ10c9eCpEb&W7y13=Z?NzNqVdVwW*p2fjRHst$tcF!L zpOL?_cjYrk0(f1tSaQ;GNjDxpIw#cpEq!aif&}Olmj&CqJVyPOSz)KYfBH3_JgftQ z&|iE|B@uMnTfZs~tQWG|UIRBoAuO8f1sD2L1AqQK0>r56t5>%eTO0$gO;Yj>!=4vw z!RN3VOmqxaM0@j5whxL{DP}6-VkmjdoBP!?9;M65pDUv4H;1A%H?Wp#rXewQ`;N_m zSw_Ctvt~(_+O)#En8L!3KgHI&oqk$x(7)-B`!-`I6tvNVj`*a?Wg*Nt4}TL}GQ~KB zKErq*0$v{yVdT8?gQauPGOm)Ff7%-grnV#Ee`OD2ejaT{~5E8gye*eV8d8??SD&e+-*el;@HR?7ko9bI`cp!94; zvY0p{^*V%7w)HEyaaGlgnDvDdAU-USd?!n5hZ+5FO{m-x3wTZvF zXgLV-gUkrlU8chQX3pxOTQJng4T=RZ6CQh8qF>6_ZdWw57rZ85`>{rFoMm=BxWi*> zB`2t*f_jMbk&hp_b!>M)3if+K0#;%Rla?LK>^}V$e(Nl%)8@0cdJVmKE5drZ_QDU_ z{li}Cxz-x$Xs${`o!RgZz#1|;zX#Muvw|7;~puJ7jrhF>fEhg&rZ?k(akNt_P$C&`(-!`!*&E!kcsYxF3E3A@*c)3yMW-&=gAAZz4!)Hj2d_#>T#l% z0bQ0})&Mx}dq$wBoIA{dOlb%{faQF14O z*~929+PK$t`qQI+dX2q3un2#O<4P}w*|B}z+asdn)UrUCIZOkXMq-4LS=z9?vnkyt z;u&~hZR{bx)s@$+BFOeK4}FOZXL>_DeO;$^e(Z-&-OaTr*QlEFl z7LNN@rx4V5%}y&TUs($mSfBnp0nU`I+$an!R4coAI++n!2B4J6Pm!a42$=nIRhub- zWDp|tHir>~pV+kJ@%LJ}iD?;E*$yfT+XYm87RNZ#JBve)TiSb{h|(zWn1IAnccLkx zsh9+l^_&IU94vl@b7MS2*12y6D?}ny^^;5~0}PpDw9iS6LC_hy`{qr?+hP`?o@It2 z4T9N~Bz6E#rk3kd5wZ0ucP`@giMep;`e2L$EB_%`9qGmnEEaR0-K7?~X-#E7a!l&G zg1QYc56M9y9D{e$n^A`L4fwtK7-7cpI%M0Gy@Ub+#DwOJ1!ToZQ*ayq<}$jL3U@~` z@)ZG(x%KkKs%?kkR!NNH$@m+kdAB@F3V~bQV$YPW532M4U#TfhR^#_7_~8Bew=m)B z?U~dzhew6_;^Nu2gA32fBiYFqG)KQbR!k$WhWGwXr9{K~OJa=Qkoko~6nY(!Gm_3! z=8^#=76P4~=u~7sXGm&^*fQ`>){i2jYd$8RzVtCEcvIWs)>*VG z-JxCkYUWoi4V%T?CYMrY9k^&INdQo`U~VU(l?j z&bY-8bdlH~%a01ivn3u?jJHX{+gh1KKdUzUDAI=2cL?PuQ&@3lUaw1y@Ul579i_&& zcHYjj-7Aq6;djL6Vj9OuETR@~6(DF4!u#vC>$=zDhKcDy&p;bez8?p7`P9S1vi?h) zdsHw9%x)9D@eJN$-NzUm#Sr9%+$24$S*qV>r&PS*(1NCn zJ0FcY_Ij~s*V}yv%+=7C`gm5C5G-8SXCY!PkGlqZ`J%=`GgS4OIL-fceqR&4VjDH} zz{-tR82i>jb5!=$k+Bct!%v)T%iFZ-Bl`I_QC#sDdh!T?R*%JY|C8YIlYCiagH_}y z$+$a}0GluyTf2Xj?bZSx1=cTagP{+|Ux3^m-4T5r0t$KfiMN4TnN*gh9C$0WytAJa zFkWdN#1K-R-X+W(MLEEQVSUE$;E^w*H(pv?_D&XW+R>I#xnjAT@@X80-(ic^Wu7?7 zyz2@K`QtR~g5X~cO;^gT`CfHhXN=rZ?8yY)thkz^fhA9|q6rRx9IC`-KIPaCk3{!}X;bBW<+{|$E=CiUC<;v_;+4tf~M-^?C54y@S=j^PuSN(!}Yr?z5Bzkg4V4;M{)PvG<4;-p7gFW4j@o}J$$N(7HLYsGuL zOX)Px%zs@;kns@QEG*T zj(7)TK%Hpyn~=PRktzD+h z266@x~`gip}=5N?E^oLv;Qm_$Ry2SgU=UMUaPK-F|fmg3_XK<;a*Mrc2>7?XcJHtX&l?KU-vOde0R&_z0n_G`Q|St zE{zL*hmG^yM#~23zxMY50b(fKi!u~7FiUH|MvI@ zKB_wFTV7~HDz73&#aJd;{t z77~)MP?tHxWXAo%qr-7z23u=C&OBfmpmROOU2#fL1=|*L+^cAGST?t})VwL%(Gsrf z%C|}tqvpIG*B>kzrXm~=bc)P)(YxyT{pLaSHmQ$Afyp;%00uF`>pkjeAM5u+@Mfy* z#-t;ZfQJo^nFL9hJGbOt9Zx*eZruHsv;_5$pYJhJjsJ?LF<_jSFb>0HqppH~G91YE zA^!N((I#2ceSa!L!P%*L=EiwNJUWE!BJwE&u3>{)#3<}8-1H>cbeWML1=Bspz_p-vbw6Sp=RE2ny>A;9z&&t zgFycBYr*Tm^~_q8@8!QV(dBTeobPn0^3N=Ntk#q?1qM`Ij3H0&qrL zI`2}E6M&CPw)(}1M0agYO%Pmi9=|%@#z96D2MD}-`a5hS@P5qRi%xjEeSGX+V7TO% zr4%30vOfzaT0`_j{i?1ei;uKZn3n-*#o69|j~W_&@gV+^^X;0Lh%6iXBB3`+ z`P3Ml1!d|HrNDS6qC-sc^$31ArvwovC$4%V3qI$31z6ztJee(XfEE!>Gtc(wHQn|V zNxVgWisH~@4g9%lv0yCD__XnTFx%^3clJp$4jt?2kHJ@i_cX1m$54>iZAE!@8ge^V zH%Z-g2}q}|C)A8_(aZJMJM%R(#n1R@c0P&!z2yA6wp8MR%(jn;XIe{QnkciD@pvK0 zdt0w%?uLt2ns}6N49h9VaPfsye$PRHnEx!Ye_8h{y1q>i(^5;WTf- zfvv;asI23o;!%QXuYl?fYwu5K=@hjHUh8F4kR4oJ-TH~2s(j$iqhzWVxLhFl*s`jcqvjpq(eBa4H zq0>w?HT1FDO)py7zEU5h)f}A>n0oC#%+rzKL5X0D=YO3nXl-H3hOo1S9rIfb*hB_D zZQP_Lt8U^AM62^rXESr|furuTkL}GC()VTzt$8qwyn|EPLuB4U^};6Hl9JUlEG#Yb zg^~lP#1pa@8f;Fic&g7{-9)ETt~Rdo9Z}r%HO?{fkv#xo=o;F7ON6}M%^wmP74MqV zU*yzDR8!O=QsHbQx3LA+A6M#9Rr26*=0z$WF3GltmaEL^&Q^E+abcr&3E#&f3dqTg z?*~D;5062@!tX^-mLeEA^esf#uMR`QiEN*KA#hyR zZd*07UM}*k-t-nO-kPx`W>u#Mo3;D7NNQhP;*az+$^PjV8*Z8$?SQ!dW4FUewQHE| ze5BrUe|jRnAak-_&`AnCfv9p)%Jz8G-xB(Gl_V7g^dCo%nj1z>`gwbJbPq>XpOi&C zKXAKvCx_z%&abQ6Trz$-JJ0D^@EBq178_~g&8EkZqYt{M)6;O?_xj{a>Cn}wDG}<< z^OXv<@EACM>4d`G^s4<^$(Yi3Q%YQ13U$k8ofS!u5JLN#?cKV1^cLdsSbiU9DpHNE zpOc8dq#6CKMJM%uky=!PV}bhuhO56uqIF>mqUoqBeC+v@p-B^=g*cd)q&(9j*yl7 z`KweYGn?S`IXd>8!W*sop+m1Vl0+(0Ij+ky4*i*W07^a1FIMbnzg?#E@AT1Lhdiot zz+>~9quGq91XQ2A+_O6CMj_t#Hh439LA7m+krvI14!BEbq!@ zcl1Dyck0wXx!E>;rN4Xe^0T`er2ct5N^Uu?Cr<6VOyMq=@VDN$5yw~8)V0W6+4hVBN4-efVSKfqnPqRr@Nay)$5qe)=M`C ziXy%vU;R#qXD8-Z6&qBXQnNM5uG$4QL>%@S_sfXK-N~2*e=h+>H3~F*Ce#>_-1@}!4Gb|9u!!2`?L$^ z7{54Z2zM;!RmX|6zI!)SGSMT2LCBFDkVkBwKhuz5q&8H7pC7CJJk^~jD7UORWfq94 z5B0kRQsn*)bcv%mXd?o?*-C-2_x%nqr8V%Q8qpx$oZxLdJcPv3@A#dy7GPhxX%q)* zujSupu&q(}MAiU=kYP9zITG>mZ`=Yo=-O!I3`U#9psI9B6VeOz%T#%aZ zD@(d?wQLe$)4$^qRT^~~ly_qf7Bq;X!hPv(^lUT8QBB0oDVOZ6`u2FLx1+zYs5?s# zgvO|P%rMjGC)m$xVl4ETK{<0AqwQ>8_#nvym$#z8S@w<5B3^#_7@MS|upf9WG+J*p z_get6_q{8Vb2hZ=h>P--TV{=dZlwjk*DvNn^lEqXNFT^*5y;w?xB3w)%N})iTjun* z`$@UK*620jr0>dhDgD0Z81#o>hp1O+#mzw@*3tCctNhXD`-z*{)ky7{Yf3^bsb{$& z3pnO8b5QEculhHhLP94oPxdRVl^(jsUqvv)Lb_d>Ad>>9#oc}QmmT<_Vq4V*uZ~B* zS_&cVs3|vXJr(GU$MVh7_9S)vY!05-dKz-$_2L2*7_=lGx%sTd8dIH`8gB`VxO^Xp z_#DZI+&y6Gl;f@~#&GcJ+kHyO7Sjrn7^@?u9KT$M>l6|4>c}lrC65^OXjxn@Hur|X z>H$S^I>iZE!wG~Hazq*4p7w!GM;mW~PD$LNi%+ zkt{^t{MCcE(I1rhG5o}De<%QUzRqk!le>IiUs^IX z5gsj|(mkn4=BKzfz(MBsu$;6v#>E)Zb8o(yY~%~%z;c#w-v{?#b+==lp;Gp;(Qfq- zR%SxT^+l`XFnvhxudXaC^WSL0L$p6p)(?|%;XTFEohO1G)b-|{MOhoQjY+#~q@U3P z>L$Q2TGwtmoevE$svz0yqKK;s*qR$n@h=@mRGdK4o}{+iOaA>?$hV7QT=cJiDou1Mn$^@yymE_6+v0O70|c%;Pps( zHF-gu6gG_dj7j5LW#kXNSjTz4_PvaHjZz&B&$hdbpjw2Z^>OH)P0oduq^?vm8X@Mu<8?yet0&o>vdO022#_cjIq$&_wZ*nB1FBeWyB2>XDp754X8C1e^$rqRZvYr zKFxaDtNY$E?m`kxO$~n2z`JbG$A9dda1-y8=I3bB;gsG<3Xu281-cItEAvn`1TX& zG~w;3naq<8SA}ouBr)u-g#J0}Qqu46w5n1eW+ua%be{_qzAF|l%%&Ee8aFQvl&s7) zkZAdIM0x&0J}sPeSVmE17!~NCHdJa>x9t;IO0+IcOh)z+?*-%T#?ol?VV=3c36|h? zI6xU7>VP4*(M?`R`C2?P57tBAN;kvv!sOt!k@3^P^biQq?Z}gW>u&`5O}YlwXFLG# zyH18Yr~}6DxQp+}f5VGEr@%iiypn2hKBcPPT?Vfi3nTY)RnEO%X_)skt~;=+*9@r= zA5S{6KA)^>8&QyC^7bx{lOQqtqLG+&XEctzCx^~N-5%kxQNx{Xypqv*)Q<9$?100< z@%dlq+;*$?Q^{xR5dM_#2_a{omp2w3`c&;uR<9}tZ zFoYi8hQW4F;-Z9X0aM~%=68@_rPJ*mJ^JN!z@TQaI$n(GB|uAMOAChL*IV-)+H%;}Z>EkOeH z9{sW4KZNfIgQSMoQc37Y@*?oEJ3Kb@P9`@^zA2Il3M?!~o=9v#HXc;Jp-1#}n1(;z z9`MF@G>FfJ+8Nxlg}}Z1ML`<&SI~#R%H@nV&g6(628KHU3%2ih zA}Ta49UerUg>4BQzbWF%q>&fCGTpD9^IbcH4Pkx_=6G=@k>JRg$4yHMe9>zg`?Xn7 zn_t$+tkPPwbjm+ln2`zO4mo^wcgr?E7Xc=}u{+w1w1~5b!8^{Rn4+2+xIz!VpviqO zokbA3=51K#sBc%rBso*SP^fHXQww~rqJW<*phO(3%|U4F8A~;7+B3EMXh-q7+UrCR z6;quo6mYVjjZ892MCBp%rm=w(cvdk^CSOp<0RJKcsMSKXca(o!_JO&<@jJ?^NzOxd zi&W;P{WNLTY}GNy73lOML?NRTIHRX-a9LDBtGl06c)PL4c$@X($m;$?eO=QZwzjF{!xF?t-R=K&mn2iuFiR{ukPpH zkmLVpUzvD+(-hj9?xsR5rJcB!gxpA|FU?{;r7^5O*ok?ffVLlMzsIx@0HZmlX~y7P z<~GjLytpC`?s3MWaTHUDCcrt^v2u_){S@V*F^KkUunfcsBCmBXLDZ>C2f0MKWVy|1+TQip{W!K8-0+J zM^(GJB=-GZy;hg1e#pjxFX{ZF_egOMjU7UC-IU&!aN2DS2zB0Ot3&gbwO=Tg8+LT9 zD-cJJ_$_aumoF;L7soQk99ei&J{kPXkaBUg#77h=80X-c`D=_zdGITo;D4SDm2X4! z7s2H&{4SWac!oHS%v+Uh;mdauZVs;nD`+@1YHyKChG1k z(-E9<{DR5vcA6=9jr`}_(!1(M4t8Q@Xze!U5xQbNzLbmiuPg7qgXWd95l(cS*AETL zu{ThXP9VvuC7}5YN2AM!uJNMO;b_zpY(#}*H~Z=3p_~WI70E8K8A8^lJ&V&OLM?Cf z`!AY(FMU7SueZVMOUA~0+6qO^FG?fDypLEg#L6o8=4wrcu5_yQYpt=r9j-QPCi{~V zWJa+KpT%ULwEZWR;4|qXroj8Mvny~d_iRKkijJCeDu9l3|11GJH%9z+<_tV^@B92G zG9Qzzl;tR0R(eWWJ}C{LBHADVw79%SQOR}|Q|7BPdg`llezEY$h5X{;p6jaYO z-_{s$feF8B*N}Ltb<&JLcF6aB^tiO;ltdDLa_2#IA+kZ9^@S*?f0J zIAt&)*IKpPEMF2d+%iTXO=L?507_0fO(!NKfjh4mHs_Ssx^JL9C#z6AlBZ6>mQKBQ zSPl#8-A4o0jcjV-{!a*g7b-#{K?Tl-%`0#X8;PENG(DbEke5aFxZw>bpz>?8jcbI( zOu5y^n5SMYh*Wd&;O*fF140tC`kGpK@czrV?=|)RQ4ju0>8Sqh-<^r-eWfh6AumUz zofy32cSA@is{fvn_=Ys?1q0gOi^Pm7_tF1$|MT-}=>O|p&1*le z?A14uMVzpIPx&e>EsX+nM*CZj`Q5}Z{+LCm-uGCHm%L?14WWW;th{Nc3m7=j(a}ep z0|IcVKbg5GA>NJooBjP?Es#R&Pi2q)efj_OWd9xQ^9SAk?-f-4|DV$mV>CD{UxM@+ z78jP7J6zWg@`Ovv%m3Km|E^GAhvjdx3#|!N(dp_=8i-GrxV1xs*co*sT7?ZY=STk4 z&i);K@9W8*@{AAlKD}=^K0vzLF)*I+J*J!eR6pKROUt0b5A8!8;4||n&%Q^FS+%OZ+?`YLB3)G$~CiU--e}3QI`#Wd< zKn$rMT}wO`y%HU(HvAe0T9*93hWeyu-T_K(898w-<~*V`^ukV2l<7@|fNxS5#Cq`>e`06cOpthpQuu+Eo8(f$7Bs4+y8b6EPm~b!6Uh zue@7ae;N9Rhof%N*g6m!yWZ|UrnT;eM15mEzI0>^gJNmtoyYlyUdy>Qn69=J&K_oi zhl0yN(dd*XyalPyQB(hc;Ia`UB;rw&!P{QJ>UlDVR7^TJ@rnZ7Exr|dVw+_9nLvJP z=^~wca5=P8Pr`tPfz?q3ugB=TO;`tIv9u4&CGEGa$KD`(OI5DyP+On<{UQe$+Uas^ z4*bq$f!z;@id#ctA7`uiaQ2dH-W84SAIWoT0)m}S*3zKfFz9iAeiq0BO73b=;d2nQ zQtF-5Vr_vq%7K7@J}1*iKZ)9>Bu)GMt)QiVZ~3O`;)?!xycgoPl=T-YSM7t5FP~Nu zO&Hd_w!vv{POwB=T~cas`&!VqoqQ^YC*wY;B<+(1A}f4~YLZWmUI)&;kr!;; z1aDeHqWeG-2g)(m5}N;u0~)>iqavfFSAG9#4**^FA`!Ra$bejpxNmtz4P({z&8qsaqqz zAEo7LI#Yd3Qvq{oyXWW9yyk)#o~eyeokHa-;dR5V|E0LzL!V=-or}q`F zf>iU`nq&YrIYFV=(f~8#c5dmlqITmQ#tU|#dK(u{M?e5aqs-d7|Ipg+dv?9LKz&Us z=#7{C-Oo}!L;o1a)`Dy!ISAftbh@sbB_fYT>TW7-dSKt6^;sF91hk$>Zp=?WBR6+; zNIVY7lzye0-B}VJIeKBu#k-zv`q@*9>9y9>myUns!=L*R3;mHjXtl<@A*-I?%;|4^ zo#!XY%Cs4^Gm3^fmlv+3-;~pD=4_E0E^qVKOPQD$HMtWA8VkMl2_)a@Tupd*U{SQF z{+iK{gx=n2drYi}WMQRAx`64w88XmNsG2&&5|MtYpW%^Aj-RT;*BWL@v=Q{RY$&x< zS!hamrTip*kXMI`Dw@;zII*-@%cUS1!%#)xqDh8w2Wu03Za6Xmf@y9YH|JvzeM!5K zp-&sM;N0flgeBFQ=RYe_GJfKm6>*+v`_{;dOV% zLunCRSbbZ~h+3oRAeao0%5FXt%m#q*?ae1>-L1e)TSPGn21*M-Z9cl^>8qfhXNl_G z`TX9~AmwD+ods#(i&()UCC3{o4bNnK)uxz8yoUa`nX{G6WfXi|VZ-egRX4NMyL^P$ z+-BCnKbocz%RjE#egT3F22pH~cOG|YuQk-&$oZU6G*4{V**Fb}^^#Y?SL+;LpC_7aY?jht z;ClYP2igf}h?C_Y%Oul@iBVFDA!Sb5&bn&Y1~zbwc6S>t|JeGmwKd~SF7cl)5co3q zcX$*AfvY7rj1u5+1iC}jg1Im6U%>|iICK82=bQKYwZ_OP!<}GGdopAr3nVFK5`lGA zUhn9VVO3HA_F`jQ`;lHYlQLZH-J}t5SCwEe3-36yhQ-m1HDG5^lBS%3pjye#2LZy~ z(F+6cBo!hXl+@sTHMGXeuIH`ww9)1$XXZ+lZJKvUr;m?%b9?@xO0Pv;FDojJk0-Sp z2|DG2o>U!XppIjJXgqIzzeacub;BMsOjJ8hBdtS~K6U&Cz%8f?yQD$;mr4nA5TAV&bAJpToG9YCo8l zl}YKqI%B??v^!MmLCLNJd|Ra>kyADwLvFFJ_C4dYf)1g;0@snZ z5TB=KRxHuFUf226_7k*_r=L3BWd7G4pZk;66_RA&i{;mR=ez2pm0!OdQN6u=kPocI z&Z@rnL=r72eRqAD&C>nYV+)=L!am6VfxirJ(RcgYNI#l7em-OV#inM$kl8vN9C5o` zV{WRaE+>%+9o%+QYADnH5^!bWy>+yn?f~Y z+gSr4cpxzP=?MBdXjZ^3HxqeSz_g63+QRa7y@B?sr(YRipVk=gy=%x>wMP+%^ic&2 z0L^bx%Vd9=8iYghrXWdH!!GKTZm&F|WJ+qzs)Pet9*nxmo>* zu@M&4i0Q~B8`J8%9p+RkHR zW!VN zAiY_YBwzvXA${11!o9EP$3ijN^lW<<^sb%M45i-{q+>JZ)-Q2?(e#K@Q};+oOH0N^ zy}I{t@O{(2Sit|hCi}%7PqcWCCD8ADI?2Ab<@RBxU@UA+kZmwFk|yX`I=xvse8BPj z$a0C&3qjVwwbwIdJBO2o;Uyg%qKXO&i;E z%U0qB(A<4nMkczuZ>n}p^`4uQrKS z+L&t3vtU0cg}w+dfK^_B5K^+m5?tg0xrkwNa2+L&i-g_S5J&Z-sfqMom(AaIAtwDp zrlLyc(g+^PO4wziL1tm)0(0R`miI9mNq(I|&L!St4UR-zZ z1I>21)gn5+{NA(etCmsm0`+)zuhF#?(kX4W)bjjLChs3WR0laY%ohcFO?N)w06dF|h{b#w<8R?o=qrR{+L za0QOy$vo=8XmVjzR$f1es_1CU4*|+KDii`PsIw8e0wH@pIXD8C|9!6}hCkn`J6v-T z2x{F!HrP$W``hCkC@9(*HH87@Q(l}i3)Mo49{Kv&?`+D7tyS>y7Sv4@n`@3@9)`|p zIg(;D1mj=<_`@u#Xn7=e;-(_qHXm>beH3apaC|jxHt1|sxtC zXi4ufdO=QbH4J0ZT}LK1inOMr$r@F8ISs?ju~}<6kZrzM`POM&MyNsvNK!mJQ55|e zef?{x5Yu4a>V&<49k^1%| zA!Ymw8e{WjOlEDyDvDw zYW}CN@#IHhtq|VU0m?W_;;ydEy`L4SQX%L(jj31_w*x)7aXme#N*cxi0jdGU&Qcbd z?)4v60@Fw#U-AmMJZ3 zH@^u4EddpZzL9zx0;*GbXTN>?#Gb9dFx}Pjjn1|83KgbeV z;>buzJyNH0r&2uA6a6`Ug}0{$%1|b0ITSL@MJOMG=O4$AR#L3{1YZ9*{F=0--oAA~ z=?m6sxH3nVhSd&58!yziA6O*-6KDDw3(C0laQFPx)XJ9QZV!6})%=p8?O8r*0Nvp4 zeDoGRY=SM$^Jm0Ws)by)1*A_Qa(bsPJzBbllW3C#YmsviL=4TFjoJbmeQ_}Pe!oT?2 zHsRx3qrP{62Bc&g#|KbVloz~L%leX#Fv(i)(zu1 zq>EVhENFidj$fg!cfX9S^is|4m}&|@+=BHy$%F&;kLH2z9{-1B1}gpG{6T}q?N3CY z;M@KEv2`D!%dFo`EQbmU`F~rjwTf#b@AVzqxS3&7mUHJywC)VJ z9LW|sY6>3E+Yq20*+Cnv;(roNa`Yu2w+&K34ZBD>oX)2w9XTt#NFR8%v4YIBXl4VOcL|SaZ(2**c>Y-PU?=ukmx>NlvQ88m}aX@7l&t z>f0MG`GmN%ti()lpZt=N+gM>358iav+knGel4*M7Y|crPw0B?~uT=FmPgKpzFEi;M z(IT42Mr?FuwL#PWAY{Tuc|1{F?_3cA);2cOdZB#56hy3q&MU@Ce=F@ zG4iqAH{Ss*3uDK)x98NTw+xc)&|9$rdP;fl^tiAS%XBDlwgvEI8Y!mkR1t!i{haKMC zj`h&r!PfTnoX$?<-|w#`pYUI}uV~!@3J-Dbyyid2ypCU%4lG&n6q64JJUqYRs4<*4 zWI@pwoE8Ydz2Y1V7avWUx>BbJcv{JNQ`ne&J-vi;5AnTnXfHI{Vz%-PBW1l?QV&=8 zSv1OazSeccj5~XW)Vd#MS+@e7s$3uE*IwqneMhrdPd-?Bo7Zl}GC~`(j~eN5EaYDX zkIpkCX}6iCb=4~I<9j>$t}lAE^X;P3MTsRxIm*3gS<_QvzOOQSsRJaxDIG~d z8><&WVb@o^h1)!IxVmnS2qkygG=NEg9>q0T`2XVQHA;87@_TWE!1=Ik;joE4x%=Tr zVy%S*v~iyXmuJ(>kVc!m?UKjWY8jM@AW{DExRc)Hy@!W%sgO!&fjzoeQ~yv?y{SXU z*`z3Aj3M$O2z)5fQ;~5=WiHY6NrR*N9kSgDJ=I0_bXd4-{)4l-X(j=DC~?J%&eH0k zh5&wJW-K4b*=Ul^(J5;8LR-_3e+{#Z^XncKydu$fWgTHWq1;D#!h@Arytee`3u9%+ zZpvY|QRI_*X!rSMv`%KL#h3>w&UH=H1?FoNj3OXfSdkV~bm5m}_PK^Oc2+p-7Kg*) z3uLlMf3{~XUgb5>k`%9QDHye|c zHR@mhdosQTpABc4#bDh(*i344PZFQit2uvE>ws(v*`RHpj8?KRizSe9{M9=#)Hx9e zSXJp19jB$4tV0KV5DIE2N2JxZoxou20rs12q(yNqr0IHmk2yiymYap-Yj>7)Bv}aP z%^M3nJwcbQVR0-6^P9bL`lgAdw)tjMK;oM6UF-H~qmDIPB*0Q{gDL+koc+H$Mf8oq z2XCWO8<{FX8ooj$RxO$U+gnLGyVTHYv1l{D0<-2v=3<@@anVI~)z_z($w3T}yH{_q zu^lc?YpnG|Lqq&A&(9)29X2*@rXvS|Z+o1Vf=>biC28#?C1ExvvxKhmrGA7O4lrQq z(|>o+`gaHB`ua|%Kf`*cGeUyT8Br$(Rq&`&oNzn|Q2@3xXsuC29CioFYIk80@P=D$ zjvLEuvkpp+j0Cu&hM15c%;{p8xwvfXq=r{l^ZLWmK>0zeOjK@Tmi(6|^+cXiml_HR zkrR9@#n&cRCl?rm8VB2^U(!GQ=gf5S{wnN-zA<2MA`!Ku1e~RMi(qqOW1U+B`V(?F z(oo9`M7}=4SU%7QG4mev8DvN73yR7l@kBfEaYoyvN|Eu~EdrNss>%q7iuzPg;D>RVVn~T}}t@Ffgc5kwSSX6criM&2ZBzm?k8N^lOwq6R#e{g$9wogt8Jw11|5Q-Wq zsbNmnzw&=68dQ1ARH~q6;A}LY^-pxLh{vK80JK z2kflzT^z3gq}2IiWn)+4-l2%ds!ev`To_&}2Vy11I^SADy1TOzy7H=GQbU_36Y2YR z4|jL%R$OZiczGKhrAPs(JwzZ-qUgPudS`UgRQ;2t*y1awc2^}bY_N#*>q_xq-tjH+0A(Q4FK-} zu9)FyPyT$vHG0@kqF+5*c@Prl@%1X$H^CTu&(Q)#_ejf6=AFiJtZM4}r(%4he z(1EWHsb@VeQfn9o{=4xkZ*$9%db2^jlBF|BnTjgj)I32K&22Yc7N{D5NpXMOv07h1(Z3nAdo{5*=s zDiRuihV~k0P4z9krOnH6<#GZLRP{|weB3MQGQE|BWBcVooC%;D7=`6$S4tWLuH482;2Tu&0&W-6`dZV4TMkB^@rD)JTJ>wLu+TcspdR(h2K zO9)$>l8UyhMNrif<=k;Iv`7Dmc6br}Rp3gBzk5kYa9QQ0Oh$W*Z;4~)!~-!*V4$li z&d#e@G_k14lA?kZP*X$uWekv>Zvr5u2H>$PXe3g_<>s<;_P;km$M5b&IMv_q=%|0! zLoPazi=y7X$=g#W*IHYAR`$3T5Q0g-o*68&$ybDY9vvMW88NvohC&OAho{3ao_9n3lp*-HGiYX z*^58?$U~)ELYF@&xym%Qc2*>JwEX);@ZR4DX11mTJtZx(iJgO?bka*v;FoE!ipt9F z`xBlg+uT#m)oFmkxYP9ZEuPSSKJ9zg6-Y;C(sMV?cp*t`2p0_nID>C9C8mAfD^sCR z2xKiQly}IZ9Sh^eeT}!Os_J%wz)Ays&F47i@p&MD!^=3>7cYMMLlpcwGgOBgkd?*@ z4Yk#`oK*7#@N@z1l5I`EWM!HxDA4^P7`x$-+Z|Ip6p}vP_Vl!ulE4xu71;4uk2b#S zoxFdqNF}oB3IPp?&SPQE|5krlIzAa^UYPr%8<(=0pX87=K zXfVS!a2KX{HdCgEEQVH3p4jaZ*gqC4Tb>(lRD;-usOlmuEFchi*zPbO@9&NQji!uR z+&|V%MG2?iDMsOwFAg&=`s)?~OoEUp`9X)?z6W54T2<<}0qtsVhm6j~G1-h;J z^(I5`>x8Y3_I;?&H>wIK3{?2VSLZi>R2d1pZ&H`~TY9h4{MWrM$t1^=0}i|V1vLq)Hus^ z%$_E~?qiJq6syS*Mnc>#yE~tXbhw<;ElskzYPgHUef$3O4kXG8kdzRv9Sgz(YbNPs zWZn|iZZ?``fVmH*r8U#8{XXTiQ&5)P)BEl}zjx?pVD!RJVu5@wC_yQL*;T>#0sCVE z1OAxm_Pr0l{_dgNyJMp?E|<$7;|0LS^hF2VM()!RB_v2fd9I;#=jSr#LZLybR~>^LDIbhly10I=!$7G!>XG_@n(Ot!%+BMwm2lQ& ze-$m`LLxHht5q-bu43)+xpuzm8RmiFF1E7)S$_4@9PW_KG*N9=6WvawGD<{X1=+>w zInhZ{Z5XApm3QetH3D`qFtKGMHd!d{>N0f&J$Y;A2Bt0#*_m% z4}}wPerd^sTRg8H7}L9u^)-{jMeM)!MF~z~a9^9Q#zpt~TIO-KH0ue#B9}{t0zNv& zEBp;MiPqh>Jt=(u3Wzi=hYu|kuNE+X8+zC?1AAc~(FEA!r(@mZF;n$nw`|h<^9XM*D?#0oR414u)zoa}nqewfGu2zhzT3PIHL9*A=#1+@EcMo#(ADmU zLlT6Y*-i@gtj7*wqOL-8l8w6c&9kln3@v91%MCUx=^N(B!M;r{r;`=aRFnMN;puBH zd^kMt=MZ%DgEC2ClBb!d@_L{7M%v+IhFhwW&REMzleVpq zbdC~Z71{vbSwXf|s6D~gJQnEmMpq+{J+$KS)v+E`U@Tv)EU(rB=> z`=yK=Zag-ZSfMmqagSJ4VV8%FmV9Rm(W81Ia-&Nhft#sgRsJ40n}=ZnYOah+Wpcz` zU)UM>LVzEAJmDN>aC{`-1r^dW`8z^k@4P?et8x^ySo`y_Bq!aS)yTPD8??0 zEsUg3tUBj+ouNOKgLppKN0@GTHE7jqwgnpx&x;jrJ zBL5xyHbU$4YN635hdOh}^X9~HC7Wz(dW)Z8??Z0rT-D4gaxmT2ZiAR@-%s__ub&Rh z-{TKL6d~Om)FpFS!#8l6Z6p?7v6lH_jT%Mc z&qlkw2~a(9It!Z39*v_h9N8G?kwvLKMyKaJ*bBUwP|>|Mx-FbK*M``A{doNhAWy}e z9e983w99*W*)X%?sb1&!-;}D=CTCkiBofb4;Bfbk`_WlKB~jznb}uRLp?eH=1J2|g z6SE0F_7iUQA$p>Hn`cW>t=qFRDKq;ra`Cl0KZFbnC%TIkR@$R>eqE%x?OesCf_XAX z=RdBHfjM9sI%Eg}#+A8PewUkCQKPdex(pd^h-}Z)?}e%`t<$Bz#^U=gT7DUgT_xGM z$?lg*6Y=sCu$sx$Y+$#2dm(2?o3`6{_wsB{2Ww|2MSUcX1p}gEFBjeB4`H`5(@SAbB&E|Ms{%!t} zmHpBkK6U&&%eO@QVP*}_3bjI6*I?B*`n#!DGcnV#p*J4OJ;w;kJ`3DvMF+F1omy5y zwm>>@?o4Q!2!-qyv9x~pP}=hClp8dqzy4cPT%10V!@m2`t*ya3zrMb|twCQsO~7V0 zekHk$yt>GtjO_TUpVlcHes5+UEO^rhTw*uu9VUH?@qSEARL!q`)5`nYsLy%R$CDQ?A1aw8bFsZ)t_wpQb35E)wq5RRI;dSqO(6aXmU1K2 za1acglt`&Ku``_@oaNTO7}MKxGcU%gnw=s~8MJ)Ltv}ybemPD+J-$eEdi`!=z!G+? zJCPNxmGmudGDYKR+3S6=ZFKF$&?1U+@%AxM1&4KchDIBEWh~ZnbuY#W4%+e~wcR8R z>QzIuE-|34uJ}#us$@OpbM@2*OG^tYM&-0lm}5V3Xz{%O*!hyL+Ka1gJ;j(>it}vrUTv{p@tv#r#5N8N!SZ{2>DCW{uwS!dAYOzTR`D)e%-V!@5#hFzuhk8-tg0`2U&K~Lztg>t ze$BN-Upz9SEU~Z8vX^1;$NK1iB({7~v?&p5wy3T{kkffw1iMClTrn9BweosZ_xFep0`MQ+VHdO ztj?UX%H*4dFn{95t`diD23}+4FX!uOjK5^#r|Y(rY&yox8uD~D>%H#Jucl36wU#n- z!l8Frdp_{?ZTb8WM__c)43|iHGyTW#M7#9H`XBS3vp(00X4g(IarNdkC|!TwIe18P zd8{od$A=Ca4BFpR(NKyN69c7aIlVtt??ps`OIP!0VP3?P?DqQI(~$}0I8YUkmv=D) zOcn=^>^(E28zs;sfC75_?QuI+l`rUtAkJhu4rXNgQrVOI5vXK37}uUSv@koOIwUA( z;)QI=r;s1}A$S|+UpP}dUyi+F5O#igW@aO-jV_uVRc)Cxa^?4En!>yD#d=ESU=N(s zPLo1dzt*U162D;!3A7Pr`kcUDW(RYcpDBh^n%_YL`zl@6XqN0( zjrDS=k9B&HUKesKgh!S}rQP=RJhd}2a=eQ=s&66UGhG)QDUdZ4^W)LWS`L^&jkv>V z9z!I{8+Ltfl}-@^HP}{#s5xUzCa}$#jBoEWslu~5L{L56ef}g=_7YMn{Pe*BnH22g zJGUiPg7_!3Xz?u1!$q}A~cW4+7M z|BY?|D0=?mgX35(QDgOqiEG%;CCbRU8AGqw?#&^rJiqbOoVsKusk``DN(L0;4+qB*&k1>{rO0+9sYqkO&H(FV>YEH`^b6A)t*8VtycRS4IFH7%BnP=0CggILj zb-sQS{5oRME`wm68JzL?EIMmxJp@Fq`1?y)ez-vHbatgsj4P)$=a{xltq@U(o_7aI zymGm28K>`nXaVoDiIEl~m+9j@GMN9Yxhjl2pI0j1*+UYVGLasCEaka{IN5&T$fjAa zF_syhiiU=eR38e=KV!@~Q<(&pr+eHtW2WSVhP|T?;=J{D$o>MIn1I*Qaw!CD@uuve2iY_Tk5C2T5+sn8L@Wy=rtZ;ZH#LOblN$7 zrD|4X$XYiUz+_Gf^72c|mQO(F#fwntXJb-6-BBfyBvqh7j&`$YWdX{z^hM^&g7&BJ zgJ($%-aRq8g6yW3pJ?451QM_qjsD!R*XxTWArzB{-I3{6TEPqg;bOyNIyn zFB&bv_|(L|M;*g4#VfXGVBrs9mW5xxZ6)_lz|L8;scP_5`aHG&=pq=6k$IR*I2^=v zGx6!U_UYHT=M{=c@pkXm#`g}4FrigVHQgcZy8|wBV^EwBl<}jD@DIqJG27%1UDx3- zcdpJvyNB&UXVd^S7}|V+uV@q%0ZmqW5pu-Bj2$8Vwfd->dx3RGNg>cabF+U{ z9c)H$TqqJ{SIhcXIC6&2Xs3fH7D2Su{#~eTA9mD#n~a zGTf@Uq$%doM%QrCJ;i~8*@pNEl^M9P=ZVh-vNs2R=z~N2rWyb~Q>1N=&iiIu-nrVB zFgUaUyIcy|c0NvEaF^PMq44eA4V!L#W?E>VbLqJZ5l6CqnL6<5-+KQ_Hz^Z!6lstrJ*}Z4N&Dti=*j;GbMK>~ zME;;=J@T%BT7ChE6SvFhfrjHs zYI@7_)vmEwm(3K{`J5|LJ+|JL0D(&0`o4PvART09^9Dm5rPdVxmd5q7$I%c5z6AgN z(Nr=wp%g8N9i)>(Fn)UB>@rGC-#5Fv9Ktl+Adx_pg`Fuf?WSowPoKIYHfT>THCaD* zkhgl|c=5QW&h?!BhA&53{G2|NH_UaV;vJhXTxjKlyh@)b@AsOmH%-Loc@4{)Gzfli z7)!o0we@CRG@oStvirC=AgO}km({&8z^2;d3KCGQxhU6GSAVRxqKe}gCUbLzh=zfx zZOA<;mKtjt2#?J|hZxSVu!ELTILJ&*{gY<)a6d&8m&g6s4Ht+=lBwuw?>qEa#nB2s zZ&mRIpsA&ZO?iR+6Spb(z>rsXM8ojMy>T=d{f!;s6W3$0$!Clfx@Iq9KM1`S=d-5K znDu*xr)Q_7cH)AT2=s8ArQaGgZ*b~_0W&%cbj@l$e;k+}29v8l+j+HRF)s-ND62!O zc$6^OJr8uqX#=e=w4LT`{c} zq{cv*#!DtPB&HMJz8Ly+kJW7c(GyNu(U2{6`Bi<6F-v_hOBEMA)M>{XJBtlo9|~jG zar*^4hM==dW8W?XOWSFE=5uCHls!~*W;|AfpEJjNtE|xI#rD9Z{6Pf+lzMC}So6G1 zgd6+wQ!EB(5f&G^hQb3AW>U)MIuQQm{$97B+U0=}J$=Xl6U1QQHP=Sh&Gqig;OER| zN)kYvYtQFoCeZ-rlQbkWVxTcA4_Yu?WuTYWOL9@00fGC=#xgz zhpjs4k{Gf?yf+z@Rn5Xa*@Z2!mu->$TJ-kWMX!6_+WEw#QT6qkl0OLW zMfa(MR?qJz+}!^bVcjc0oyIAp?UpC zbrsqc;5yd{r$4;3j&J4EzQlJ|BawR*fQ|XO*&@`7p)n&C5}MKO&D( z+uRsJ?sLR!s6$n{Adcm6qLHBn7u)b}BQ{Nsj`_0W%SZ2e(mL4)+U>mo<<;sGv#&Mn#LMJ7AUSsFv|}2LV(!icapP= z+C5a!&=0p~f4lmtDN-1Il9RB59SH$lg$u9di0yjzGvLL$X{WD7&@>Kc(+0*p<-tq* zCp*i1M5fxIQf&M*0kXHB1*-UO3!K6&mQ+R)U&Vi{Zw-%Cb!i@QH z#KgC|`Oj)DzH4NShLe$0SP{Tx5)|;Hyu1y{{LeqBc`r40*h#qj+SHQk@vnpU#%ZE^ z=P$HEf!|xf=jca)=j&u?b~Bg<5#oxc&3+zxogJ8kju8yzV8;=BPnR9 zWGXIz0o+g9`Ezu;k81q87tqPd%1LDF5F-9&AILJQM6DG3%hyJ~JbyGzNv}~}Etghd zRA;hZ3?-+-#6L?a-(qySG#v<=*T}^D&_~!uWR&40up}lL>$R0QRyYE%pq-sn$wo;) z2QtIdhY8HePS(lZ5BLA%VmkST^zmSVp{}6Wrt<@Sm2Rn2Lv=FB=n+=as)>;HgOzez z_eQ1v?f36DFTDTlP)hJpXO&YAoLya{ zB)H3noukfXf{zS@v*qgwqcMFqc1yxNisf>bQQ{RL+5u(O7`rwf8ngT%v~*O(1MMyN zPdg{0nGPCQ3iXBCee98mY>*cZFxc91EURx}pMRH*)!%@f@4iibR#9snqg74V(xTX( zw_kIjsfNmA*+THOiRw|z_sGfF3+OwDvZ!r@#|nlK`f-h-AG1;=Eme$7AdwOUXeN&N z1`!D^gni$qp4g96_SjLGka+HrjQsng+dq87xP$6(=(04KfW!6KL$wTF?ohWCL_;EN zeLiUp8YQe31dy;wOcPI9ZMyK!jc96+$yK58ePu7I-K$#`$<+K%xNyqhaP;*U@*u=Z z+7-HNs1Ak}K?+OzuH5#5l~Rz;Y%g6y{6Y&qb%;)<<>lu?>|WxW^$~ZUjqd-i=#>F zMWdFq1elmsVcn9;1$D1L_%U+sGT}}3pv(7Eg%u1@PFECPt9X%O1bReKP>@aZF&kJ( zS+M>f4A7SUSvZP+or3qYyh9b%!!KXx-tU>jJ~WBPTZ@l^pO(=A$?Op0 z^%~lc%e3(Kgfb|UMIt^V^a_gUW^DBv$UFrr)qAr|W6yO4lrW72;fA$Nb#>z$5qRc= z8!o|Fnrm+6?k>IW;Wzc-rif&{;omkk zK$ib|K`p&x78MdGV=oo!n1m2^`C`R{LBd9MEpAtLKjRm&H;Ly`dP-l`g`?MMeC#8p z6hR$vFRQCq;xdHnW6Q|Sa7>IdAge0lEy!+9q*0_eOa&f&6rj$Uj$=k{^cEft*A3IkRHoR7 z+Q&nj8%y&q$u!f}=+NIxihQ^+??5twK zwZ&`H+%X?lw-8^rk*rjOi|zBRWnAXcMJPa0B`SB)(DbW4iYmWqC|KGjOG29_z-GCQ zTY>Eu0#?j2QD$W^bt!iu?_XRd<#ydw-DE!DB%(HMqMpa zMidl8^z=N~qGP2#fOhc`^WxL+Yhr=H%8-8FWV6k29t?<5Wm$shQbdekj9huM~(@UcTdSR7ki5U|?vf-KPBRr3+T^b?}haaG!)N(u_kQHZL*W;UdHtg5DRn!|FOTBmIPmC zZw9Ie8AU4efn(yc&bCxj+-{oZ821L4JH36aik`T}^B3Y?s!7ho?ZEF2Ax1Xh$Eh5ABW2k(X7w+wTGk(41CwT zs@gqE8KWb|UiDqh+=#Yg9uN*3w%<3Yvxj>jslXEq;6?#&r4?bMymm0_{<;VTD!i+7 znafzqBA)K}*RI{7+8cEf2sDi1@H+#@2m`WhZN3G545?DGF+UBBiS5FhYN(6%l&6M6 zwIZvpjfZD^Mz;1)UCZbJeQGse J?#58Db!55j^-j|5Y-)_l3QuWJ~5f~FeBjn0k8=XFkSC55o0ym${{Z)Tg0a_YBJgkQJR-q+jcUY!$o z8jNUc59ZNOclS#@C1=N4`7?>lbD?^<+N-Dfw%0^hhu}7ldbe!t) zJ>UxAYvay>Ry>?mk~G!msMFhP0Ha4pR2}2f6ADrdT+cyDMU({(rs_2w7W$bv3s_oQ z=uL3gXl_1Vw;RYC!a3vh^dJ{Zwp_?VJj=m+^nc9IZRf+IC)zsudZ$jh9yJX zRB~gcVjDrT?J^L_&#cPWk{}*;d|J}HuWOZoE6=&ASlHQE`o|_6zb$RA1yutkW0g)^P6?qhw6u-sh}8WLGvj)ZMKDPwzjZ5G`fVg=VBd<9Pp&B8^DkZVv0c!9m2Mk`KhK zPrfovJISCJpeBrT`^LNiQHsWhcaeZ%0Q~YNOTjuQu}iwu&YI=Yv?N%1GN;nq8{J@b zFu%4+Kx>w7SKn69pf-av(V$ZYdARKw*Ma9>k^SW1(uY6e)d(xABQx}1Ff?3|f}2wv zu@01CgeY_#Mla`~dhddFdx6V$t}NX>YL5ZsqmhsJUIaNTDB@;Jd;c-zx@%*jF0sjk zl=}pM-De5JqX&KHrfZjnPU>TuOq3E&I=NGu>`N4d|9P!@zI0Jv02JXGQdU-wHC*#V z06<3*mxMmTM`wB&#)hWEs$aMm2~Nc8dzhI~OpeOSOQQ!P*eS!z!TJF$^G}l`Atr%x zU}!g@LF(e72Fb=s0i?f`CB)1*;PakBlHVTd)S!f-1QNy}6-{Mwee?|elm4Lj?{O}AmdT&e4=waO5Xa{C2FKiBO5 zynv{WAa$3oS23_>UFctXChFD#OH^i)S9L8cjwXUZeBzMzyyV7T^nC0l3_;%j zaK75R4pNpgx=5{ymJ=BnN!&IX(96aqg=)m?ew46oQUIg^n<)JP^nX!{u6ltbdi5ij zZ|OxEeS9rG1M6IqoGo!MyAs&NZ9{`QP>eh&=?q^qqbNpEovtOD?%aGjWG|J5 zpzjuzo6d#Q#<)QGTj}wDCL}><*j4U0J4MXxuS7(A_6aoj(b@BAwaOemCoE=2?ZvfMiwmdy=baodJoFj+uX(i)0eNF!3^r)dAR67m&6T;F z!l|dZ5{SFl3|lYHg9{Cl`W5@kGE}6m6f$%T>}G}Oz=>+5^&ldT{mERxB;^r zXVNe@y9Oge%q}UFGT6Ww%ezqW9Z)&Ppe})g$4F z+D7hG@8j3lON-pL9-2S1!I_0ak@B)ybW!jP{YqAe?7EFFM-e(m%h>x-32VJt~+{CD`e&eO&1ip%+lN=1CyMk(3nvWnblkjOt;O zv9+UAV926fgq*?(dG>OgbHEw13Z1J%ObZr^<+Y~3x)7>k1MFHqWDP4k+OEvXWbsK` z?#GlyYrV3w_teR5hIDErb{0q zVr9TIIZs_rf1NTIc|Ccwv^ae2DhJk;ZqLUypYQOo z#zhvelNztcOWMPJm{X9eup8Br8AL0lJKKJQL%~ue_`H0B1h5*5`$_i%)hu~09s#p? zfe0Kwt*4#Rh)VePwCXC`v`rv4>2FsY^ov4kPTJ`TMw0&ARQ#kIkvD6WW;k>qXc@!T z*&&ep0@Fn%ba`dF%7!Y^Zw5Pmgeo6dsiTsbS(t5eP`odWqPEaQ59lx{rO3>ZaUB}4 zUHpB|@q(`pm35>`(zNJB{6KX3K9?i(`W&5;8x{NFi@bV0cQC8G+QYuifz^9#?7!ir z>M7S>j(cA`S!nw>JS=o_1ic6J1Ug$qdHmf%r-YLSu}Fl&#b?!dP-Ewf+oB1FsAh3nrl! zWG0xYjV6NqS&C-v1T6*+?^BbrS-epT5g%E>=ZY9AZJ`ukpym3NR}xdU+dt|KBaQ3g zR7)qz6@27RCv|RYn(>JaQSM^rmi19%QXK4J*jZ>AfY#F9{?%CzG{lL@7?d5W-&lTZM6s*TWSEUc1)`w$NhW{!Br`ZLHa2pu(-fpN9#_go z;Bk{zSk{|TpYs($PaS5=AM7})299@UlN(ZzBY`~mAl+SVeyI2uClP{Zo^|40LTlz3nD4T7NN_ z8cKM*iURaE0(V5?XTBwie1^X*+WaA)U2ifPUfGiGyFZ0FBgP*)3aelplMFn_mOke` zhTz9+H-1Amn2Z=QR(>t{db{oDDscmTAiP{)VmCY$I2M@*deeQ(pMabST9*oDjd+&r zfo-OG=a4aDAQj2Q8=Z+Eu@D23(mN=J2Rvq$6sknLlydR%(R9|ye z$3tjpW8xs_DFTcG`J}l`XDp;=O?}D;7KPFvNF=rN5ctK9k1H}J zk)@&tRlL;Q*hiwF#<7q9kw%Zhm`$)k2bd#={%p4MtLo6JVy)6|J zlGrXt3Ye!~#}rj;mRgqtdFes5DTj6ks=UW7?MFxbIt* zc(REFlz{>|pQz$d36QL-|E^|lYdCifrTcn@Zf7ZZ&+P0A=N|Vb*xhdmwH@_-Ds4(`Po&;ZjOb6>a)zkhH_(c&XY8A#K472n}nC z*(VeY(Gby(VSr1^&h)t=LlJ57V}irA1{l6l+*sH=Q6&N4#Y3>Z$(G=oto6(pHyZ7vzxdniln{XE1HSyV%cIY`7|>3pQFXebi57Lp=VdwI0PZVb%#Bpxig%U@2Asl z?K~yc)M7QB1Ow@l^Giw0C7SO`)#-HMSc0nL>K$+O&wPIc`KK_E*&1LDMcH*@ik|N_Rku`ow-m_RT+ia?dW5pzX^`pi5Hs4%8RD%?Of{ z&+Fn_F6e&7lG>K&uWlQ*S@fHirBH*f8_8f^&zFRdHvbgbfoB}-APzFR zKi0DPJo`+lfm69 z^iT{nivVD9+qUiOy^?fok-6*{IQn@jxw)WC{MB8l8nJ{eMR+4n|R zZvF8*s(zt2GcXtl+ZJE5IG2Dj>0NbX?G@%$k<_h%p2&6$u|vv}nr!CxIfc|HOg)Jf z7kfUib`zRc6)+ryN-lS@4=XpD@b(t`$O;{^r+3d5H6pmyH9b@b#eG2|#%VK!|CEuC zUf$ztn}YsYJ#~)?+0dU;kr`|3K2QC_Q@OUs%NKc)^aj{*#w#JI*mJYu?zHZ?a(fDH zig3ugf3N_o8wSpeD3zguCF=)wIFpO?QCS2PqEi9X`*VEYnnpw5(rqfak5n^1V<}rH zSStjq&yWyhFeYIKt0B-#CNQgC)o>~P$X>*{^xF_KT8d~|hpRM>>^T$Iz16pSw!wlD zFg_*_ZpTmWnux7p-}%ZOw7k6V3dRTbX{Z^J+%q~k-UI|nX0!U9fqaSm7N`%87;$QR zZ7`lv-hXrQx%-PK3RqB`2?`sHK#x2#Rv9{!*d|=51_fbPl;<`3UGQ-nrjQCwj@-IA#&K z(CVaTy6B-lpDvb(OM56gh@?*h$?5gVCVJu>%|{lu4d#>J!bpeXjx%`#JA!cuIa|kO z5i&|@#IcgSgt8ot2&ZJb{qSOyc`d?hZH>@}s_T~v7dKQa?46E1l7&bVt|H0V(zOvw z`K4cUPNQWQM9 zj$=|e!~VM^5EESxl(UyJcF49{N>Y|}d2@WPv%-kz*=7x6$otIGVSl{HMNPeybZL0V zj{ncldOh{S)0(*lZH{C;&8I8MVKO62`dvLN+-D&N_V2;Ed!2sPHxw*tUYZdK`GkEf zARuk%YxYH!xt^XnEGK{BuwDf*2q(X;GpoY{sqphh&eFZ*nyfiRb2N%Lk5T7f!+T^w zaBGUr{%eYg-UeJg!eYXW>At=Be(&uUs~_aD&$k;wiyIRedZhDe3kJm4S@Kp5L9VW- znNkYe+~lcY;e77_So)l+E_|GmqnyJ1oj8ZfkN%$S+An(ZZl)$6Jb(0d{nE|KSOKZ3 zg2*j>jtkki<=N6~EvjCc{xMX?Ln2(lMH5VGlcHXUUW(C?qCxaLN4@(rAXjMkLPOD} zRIPAFrzjq~jc9vkNfV~wrIr?5`>4hnqvskaCLRfNk$Y;Q2}u~V!O&I zO>y!9cTJwk)zj-XL%-vrL-ev{c)kcH)6+IjB(ZjI|BH`Ov83)z$64ai zWPX1p7xcI;5k5Bi!TsdDU)Q?4O$|+4YK39f$HWJh$hX!(nwO_Wi-F`#ua-779ZzJW zQB$pl4Ed@^Jhbpj1CALBY$Ir6D|IXJP1>U)`Qzc)uX4W$)6;1%%Jr*HOC)=`ZBVBs zu|TG?-)`|ZnGXc?zYs3LxeT{{CG+8+y<@m62di;tw6|qCrG08!n$^ zvuU(-Efjdqu9ZiiHitwlxi7kcLSc1eqFSO4iho)G)p(zEHkyy@D#PKR?D%|@1n+G$ zCE@TIRgKr3y(58o25N=@>~8v5{SyKA;da{Cpf?0L#cIWdOf0G262;3!BwswT;JTG2 z(w_g5CN(2>(&Sq5;6CRt1zfG+ceulXQLix*d)L`27>S?j7kPyLZ+m+;fQz> z+yz;i)0)~ARNVX2XoD(9vKxO%K0U@1t;*R7WF!;kTn3K|p*F}Y!*oMh<@bCdXe(r1 zT5a!lKr6e7LlclL^lPab$hB@?x2vr zW>?hD#-f9-220r5f~D1hJ^BSv9WltMVKk@!Mw~ZdfG@?w&XSU>^GG|@SN@G*t|n_a z3AM$tls5Byg5x~gP-#h*qp+@3o60d6L#4ijgP#ulyr@Huqo2Bt?kvGrX2)tSF_&CB z^+GSOJbIpd8jFrE5@jDW6cBg#@eyK-7F1Gmow7Jp`GPhYcVjH|ry4W+pp+%Ci8>{G zEcPWc1UDzS@H6I?N7r*%gZO;PC3er256q18zMhv!!e83O6a4WA-8(NSC3A$&6DK-dd;bC-PVrPuh|SsI3Q6qF%}RyioAVB)w+C0L6L~-UQ2I!^HnIWe7i&uO19~{7Bz#npi<3PR;z8i^P0ms0ljSb7? zzIPFA8#9USW2P9;^X*xXY*K@Iu}N|Z!7Ju$#~~HGD1u2TNM3jj2yvuLEsfDS5`g{VkzSGMfMFXC`%MgXABkd8rI9_)ed`|$k2}oF{ zmRv&CRCpEUHV;itC&P7-Y-1{b-@Tqtr9@H-O-p+4z_$8c zd)8qF-j!t&nUb_z$78Sofo|uXb#>Jvj>B6-bu;E4Do z=B7E%5sO>PImb7si)*uQ=?HRi*z#Om@=Ouz0oC63d|w(EeCa0Dec6=0A@N+;?}o=3 z((g9z!)mhrTH9Sff|fcep4$Ew`mz1{w^r_csNBgsModD<4N@5-NRC&LM2!>{o+3Zu zf5>Jj$3zyzN8BLmd^q-qJG_4d;Sg?afH+%K`ls*7t*Bl;k*#kt=n#F!&&lmsYWedE zhnNHDnr`|U)vz7AFMMHUZ*`s$Kd?O70Ml0R(`fF_^g-L278xcg`J%F;tY;TVYvk z7GwnR3O*lO* z*QAO`7Q79)^6T|FLsQSD_8%*pq5<n|^*<&D?1}dIiG@w#e zWoxBdh2EJCfo3O)yH7;-U}`17^o#_*F>3k4mnmMDp3@0zG1XD|qe+?)A{N>srL(mY~Q)!FkXQKX@05-AVDIH}rWpc8c^% z@gDq#^*`$kJFI?2pAr-Q;4c0n9BW_B5bYcc9!=A=aqE;Fl491*DHM(sqKqBt)9RM& zim)q9;<5y^GNq|u70CewwkJEct~|d~evw7jxV?|^XuPWs%9#Imb-+7K%Zynud^`46 zH~H4u|K!S(pi8Pz933wjBx9_=eS*Snb50PxJo;s0m53cao7QDxR1U94?WjRrAADCS z5nT`u*s9$BkFKwdi?Zn&#sGto?vU>8RFN*}?vn0Kk&=|9Te@rM29ajzZlpna>H3yM z@7w2j-f#Z7_O~F36NR9tMjmZF?r6A~#;X0KA-E8)sf%7Nza#ptvX;tZdxUfSN}T}1?p5fL zQ+-pcuhqPfsGPD+VY=qKpFi0~Q8S7~W4y^%Ru8k`8E(d6=ps!GTvVS!5r!52I%mLjuRLKT?Af_N zJTiF5@kqVOxa1;c_=IIfN{cqwjt%LxT;pK~5l)MX_-)LuimoUbiV*pi4RlY(<16gj zHBw7GtEOtOB*^6$GO+{?cdk3IBGRsDUtbx%O9l1`CwmwRR2p_%5ABh)Vrx=%4b)6{ z3|XFsP7;OeO*|?0a9t-7Q*+~ZeOUT=y1=w=fxUlfi*RAsTX9Y(aZAndHi4h0bT&<& zB0wqHPCN%UeGx_U?hOOCA%%})xjcXB=-Og?fjzJvVOxyL!4R*R(=%4c$%>Qe34tJX@NZ;brR}O@)_)I;o8{x=l)Rk2lG`!QOz&3d z-z#JxixrHH6W1W$XGi8pHq>#gZmO1FM zL>+1PB=h-ulxHQ3vP(V4gv+x^j^uD%af7dMD0;png8NWA-eXnQ8KL`6_9GgYP(EKx zl%C9scJv!pCk`gZ=9gxb9;7uS(LCZ!fRB;6*kTtL)?H;=4S@tEdfTNdVGc3vSe7M5+7*oHl&R6(R8 zHf1CMHn;t1YI2{5Y#0Y#9E;`F`GCl4#gK3!ZZ+eXy;scizTPXOl8<3piN8CXzO*?d zVNzUvirstg3lgfo^#e4<2w2ptN;F!?ct5#bLMM=tji@|_PC zlX|QK@bil%Pgm$)wb*7sW$8zR1ct=X_ph;)D`@pVwSO?@@~-_^(2A57ewjeXKgceaA-TM zpiUblC!@(a=x%hmzB1W)XX#>+`xFx0ypYv0N0k6BGRwncAmVn*4tV%2%>TuXbEmMX zz?vdp48$vJY?kCILWk47i^8rtE&EA?XY?eA|kSH@Y&<@Qp z1<`qxCCx9hWJ*XnMayZ?_R4AK^pR01m)Xy^t5D7{)3c0>eo_oZq7?IuqjUf0msx!} zewdD8tNW9p%5;!tG+#Rb=$&2?nnry%&Sd2=t-0g1Vy)yw2c~f zIw*7Y`2c}TYp7cpG&pPceD8)&2L@*X5(9J4aD69?e!J62)5*_?`5rujT-8`2r2L?UW*4N`Pas%3dyZ zx53n*kz{Tc`*k)KPFn2uDL_FEAqo@!_gJG3v*E5MI0udlZr#AjwDcm zTzs(>62reQ2ZtJRce8R=4EZ^j#Bp6iH)^KTh{T%`9mTJ@jhL#Q* zyW!1QxoE$tt^4+DVE=oQ)gE4*n=YB}HtY65jYR>~WpHMzz8HtT6TiYGSAV>kcoCJp zGX}8dcDr5z zQXXX#gY&Mzc%Xt|^-uKtn5<+-INBh`YhzIScN1)OpeVh|8TpoiImEp&_N!4c7r@rn z0c795Q-~RA{EyY2PkK)PWa3~!HRC6$q`pwA5iksdPbq(fg;;YyzB)oqzLj5WhG(_HiQQ5C=QIa?LKR+m?A0=zXfpMF*iL#aDI98?~_ z=7dDmM^CA`E3OI@X4@i&UJ8$qG`Vyd$rQ0G?v}p{X7a$~M<>#EP(5d+jg{T>_nSJv z8Jx#3D|v@;FKEMCXfX#`%eG`~@UEYn5hZL)hM6GUFVua}{R#~=a+$KezD_&c`G}6A zuV=J9<-Y38pbbfyR+R&ga*HGPD@I`~ZkIV#TSiRZ-T^uPPd=+P(ugoSwv;x6PcGa}$Q*sAgYb54E`p=vW-ZD{(i(Z0H(>`7 znuCy4Z#eUe9@n(Y{5l`m>#PIq-#lZjuL2n;HnjjU)n?cB5FU3QsDVdT24&3gKtn-&=Us%} z)kh(Ws72b8^!)DEtIK}oTPpDIz1wP}`a}wtn+643q$s;&a$t>&@IUs(DTC64%FbR#Z&IJlG?6zf_cSUi*jfivM!sz z5DYa@o^G@eKfjQ!yZ2p`eEU{E_LxLY7dyNlNRFnfVJ zegV;NRoxoOXmE;j8_=3KHGAy^)$)q`k`?Eea&}FBFD`1=5Kg5QMjn4QqM;C*bysD7 zH}RR|%Z!~YrPupwzpGfOruo70VV+2o2i(w79@&mSH8@`&q6^at*htft`$FX zp7H?p^vW)-b$i1*BE@?RvwJ9Suf?P7)G{m*%)4HMG#m8Hk)wIK(*hy#frr90^dL(c zh$+G!?L^VL7sErOGhJUi&T{t`$UVbeW3wO6JF@DiNF0|+9Q9RJg&)BiN&M_SS$+2G z^77f`kI+}DS`!dn^Jl*^JDL`Ky&Zjv6a{tknxtWN%(#T*2_;-t)IQNB9txKSm`cYS zO$Oci`X{AVN(~_IC>B-!*f{DY_XSE109L5nvf|+pB7Y`>Qaj zB(A5A6-;RSsuevY^zZ}2RqLz*OxUC9!4j9#k{O}j&4i7@k}1&p)RuaVkr9G$aB%yb z4V?^P5Fk(vWE-uRAyL9no3vva{MM+htd^O@9+*gbuy%D^_ZY-{w)Uc)3w=Cr!g|Id zyQJw;2_tHNj5Ub}&aUnt@0O>OyIij;pX{cE5(a)$ObQ4$J$FfgU= z!TIfsNv-j#T;nTLSKckeXvyxp_*WzDR9JU%wuwF0n)Nmt6J&LA21eWK#nh1z9na~= z{1lJO1HES&nG1){(^VNfxx)#Cm`K(cK=0j9K7-B(N)vJ-2{U$9RVwVIko;fV5){Vr zvjGM-90<`B`p5v+5OMG7J5<4qbUB89k(%l^fdYrpWn$*=k1kFILKl0phn(9#XF0ty zIb~;mVdod8zDp2ejSxE>JALT@BVBJdU}#-=T@Wz1)p7Xj8AgwexxlrgchqwK25&_Q z0=)U~2~Z-x5x84t6Z9h4HfxVAJz$L1$IM^J*8jGqc-)hGqB2x`DPG1^R!U`J^777pTYd)}v98bSprosl&f1VJ+1p4P zX@@oSw?+nWeIHUEt#rPRAY8*^WnuGUjTR*|w0Y8il$IYoa{W$xASN{G#Waoz`H?!j z=+qY?{O&X4Rnez#??Y|%Ah-6~>8Fp8LGN|*UcS{d_!xr2?R&p2t;G)}g`^l5EdIcW zJZ3Vkp-wo1dqq8r4>!6sxTqu3WXo#QG20*D;aZge)Xp!W%^shLN9l{~u+!z`w&NOj zEc+-G*YCiC=!Vh8(#MibFv_Op3t_Hl}Y&`oD{J-{_4X)V%;q zIz6o-jVFZC3I*UlulwA-ym!`td`{T-u`22J2TtUlYKTS>nqvfvg-E01r57oeSsD@d z`SX+i{07V>G##wZEeY*aqA6)7(Z3jX-;>WR0`wb<@3~!b!t{Pm>ECaBYk64akYvqr zQv)%od)wK+%aHd2i5AABxhKwlmDV>8oH#rr*HN8Hhe()3HQM4Rlok1_67fP-EYI4a8y%yipZR)6TFRO?rNccv!xOQRe>Z{rhWp*h zXk&H0*;AvGxpgt_znUPpK_>#!_YKdRsdX?A!AK$o*qr_evPD!BX)vIOg^gpiNKvZ^ zdGkp|y4?N3e`AE=-V-`n+Q6C3eZfQoBgV$-`zW2zviV+Ki`R8P=l2pK@c#ZrZ%-x% zoc{B7znlGjV84W-&ZAgy9N&X*-n|1n{>nh&|Jp1l5K|C{{7cNyaimwGTs!mbW13v1 z$Uhf{hpv3yKKR&)Z6!>Pb;5)S{)b-_B-z_jsLRV&AkaQP%wNwywhW(0H;(TvK%ikvKtghSBrf9AiTAmJ@X1lM0Gv z?2zhB#@bs5DH+m$PU%y$xXL5ip~VOUg1%UW5x)M!wM^OOxhi)eNK>J`)^Wl6ez1=O zE)Fd9b8wDhgE~;}-xjoig>Yiu&K~@@)+dTJ64Wj+%4#+7nhRO~bI>q#7B$^#A&*_!-3N!aiZh+h@iG4NeDk~G1T|Mk$KFKHjw$4# zv6RKf6QkVCKgB&s0)NvikZ8d7UMriJpXF(_*~Tr0BD<-wQ=W+B*byM3 zV5BpaC=1bWXuCZ#qb7V|*sPS)LC1n+`qJz{s=t=UoIrDC+t2^iLq`&f9hoe0qfKdmpo)l?*XvZQN``1M-eg zu)5vKufdIiO4z8W5>}#k)a_q__61e%8`$YIJ8j(M&rP0MD>3ECM!a;|CusUJE?pkz zxOB(H#rtRrz_?9~Po*fD?;{$)F!`&itI?A;Wjn`lWU(oBq(}~;unTT*F<~R*eBK{( zZL(8oeZEkj`(cp~LuzAz$L-Iewjw~s+uGfof#c2e3L2lUH7FOL92cvkqBuD|K86-v zT>NCRBVm8fVwyX9;R8KQeoaCTEdyU0+48f){GckjIeX~E7ecz!ok`Wh%c zdJ;-nHn&UDq-ty(wngV}L*k@x_9sw>KcFY`RI`_l8g3WBH+Z+adCvJW)tdpO8crfB z<2Z1T5aQO8MHjY%R(Jhr^V@`BfT<+ucKzIrAW`UG9t8Jr+ZW8YGq&uXShzFm%Ee*b ztqiOq>aA7iJ36$NFM2nrEnXX!%)v0KaI{~a$i?M$+_)XMh4;5=sB`-drZgKqPMg1& ze_{JY?P%n&*Hwm_EfLOMW>milVcPT%UsA`Wdz>J5Te`i@Hju%}Ql}|jasq4xQ<%exGiY<#F?ib~?{qD6ju z+g(3dnm5v#io4A64i@DGl0w0+Sfe0LklDJiB0mn6j8n#PCPkRnDzQH20DRg6Fp`Xo zsM@_D;S3uW?+VYZTy?K+5eW9RZ}IeOY$O%A8Uqcz>B9jz^wX#&ics-(DVo0yqPFSqTAVvm*J zOz}FJQgZG}mF9RqGcrAzmNtr!6<{U}nw`@9zWbK{%}?R)gT^`n}xR z3zSK8BCCvgwzCi%Kwh|K)~B1$AmHhs2!3>YDoIH}fFWeU3xi zj83)T7avRJ0j#BjGa_pdGZ~p?@fMo` zy1>8Cw&WNP&^z~|zkdBWdVAZXVE>pp7dtg4wdHz=tm9t!vE>$Rz)jXOOgh}cV(Re^ zPWJoZ3iOeYBU1{2p0WJs7OP16+6$s0m)#CXp-Z_{2%&il60-R(9UaYQ%SH;K8z)-P z+D8Suzj%Fib~djmvpGP2zDu8FM9!^bnGL{|3{UGt)FRY1cK}tfxo_x-_I3sDW1uY2 zI#6KL)opRYOKNb4Ds3+qd}o*Rumtfbvnv`=ccJohcwlQX5(z}4=TXtnWz$;c6b$g& z74dr-EibJQp{%bB6fsIma(h?gxT3MIC5KDCu;N$m*Gy~XrQrcvSXVRKw|u}Br0Jji zM_Sm9{NQ|RHD51`sB6nLVda5!e|j>ZQq%NiS)L^PGS~|@C7Pn`PFP(f82lJ6Dn`45 z_goK6a#;&rJG6wa%IGXIHY+G*GJ|q2Y%@64LOR)x{o4EjBVeRn5qjqE%afg0mM&T>$BNGE_cVZ>&>?TBYM9NXnU zOk9#^exn|)jyp-4A!K>AMfa^?Uhb1p*MqjBzV%{Rp{l$LOM9F496IdTdVoa@GyM8m z&i;V8S9x=pSiwsUmSmm%y-bEnRn#rFTQSLxMcZcYl=^)q1URDC)+;+T>HPGX8Nh|J zQN z_WOy|#43KqJG-|R8%dy1&S3dqbkHNZs)U`<8c|_MEVrYE(dJhm>vHZZ;eB7dI8lE! zNCfPMqKo~Ix!{eLSETxU)+F}QhR}VD2o+qC>F{ae`gL2yoa^w)6Z}eX*g&7$1tj8i zz2`afcMiB9cqybf6Jhm&`8>SsF$((P<^y9RjSH&D!^o7b#tL-57h3w}G?jx%5;W~iebG0)zq~E6Df@TWyMA86*W(-6wNZ!ik46I=RFM@BM{8$f*w+UzR{PeLUYm+{;Vilqi@YY6xxYS2Q!8c1 z^>p!0VQU@BxCc%Jio!dXQFA}1gtd?QnY8mdpH?Aw+aTEz#H*|{;@fyiBXhcvd+CfY z0NHF2D5(RZc=5pFOD_L_IwyR3soCM%fzTxSHVPvx$EM&ZCeZEtR$pcYXF`*X!}{K3 z9yZNCF#J~fz{wJ`CK+G9y&GP*XYO_VfDSmMHbK?|#8ULVJhl8g-_{c_Ci_H}Humud ze~~XaZJ;-&1IhSnfRzRHG6fbEMWs;_F&v2vL6?B->VkLM1*5#D+FO+*+nx2n;Tyeh z80EXWQeR_ChBgOIT>5p0s-MJ(ib?Kf1SgSWFc0`n#3-WA(V2!Es~#lNg<1Vb8z0y$A?5Jc__zHIRYruB&+=NGDjO>T6p8GoE%RIcBIrVf}e3m_+g7uA; z`HJ}@9Gw%5Z`D6cS34Ksi}3Cs#+vjT+m;1eQ!{55Q59MfSMPqIiV9b9^z^UH_q@=q z7t#i%Xc1tmjSSR%9}tBX^>$mB+8X4UEL2~292!JQJZ%^a6cx5+@osVOnvTxY|7Zcd zVnD#;BkT5i3dq8=vsX*el$mMX9Pi|^Y5#bG1gleQAOwiatY+2=Aw zpd=Dnn^85m>hAj&l8|q`u6j2U$!iKwRuV^)p8Ce5DIcP@gIBkzeYai~0z26k8jm?U z9XnrfqE|m(N6h1F-srR`;5ghNownf*#|3dBT)r+@uL6M%Q&5M5FG{o*MR&nhaurh$ zfANQS0eMEOJTr0btWB;r+X|=>+rCexV(Pf*)3LrEg`giGG|QdUaC|l|=45V{NRl*P z07xUUZ!=727JvMh_M;bY*&Mpw8B35CYt}AnQ4y}xw6>!?q0dq@9lgaHjtLi1?2Hq- zOAO9N$p5&$FSAI1vAZ+B+ce#FeUouq@Iq90(fS`y{Z5qtqo(HYi0(de7K{%Hh~ncy z(_JV^_FQd3^w&6&f=N%MMBD4`iO#FklrEE`vZDHacnxQ5#$B<#GW=wVsx34}Pg5m+ zX86xcJPd@YTwL5`h?_Q~ecie5CE(=tPU{388 zOJ^#q2kT|ukr?f(I{YGFzZp?Xp)gK#FWRQpxuZ{{8g&*>Pn5#3AH7~oY38AsIr+Z8 z=G~GPB*!Kyoc8b3+lo%MLo(6>5MZ0CQiZZxnqY4T7KRp=Eo)CX`B@c8*X}RYXUCnl z(0>tsS2BTBrm4t>LVMOk=dxY<^Jx2|d(dN)mDk9ylq-!15+%^&U+SN~ohykQA+EP6 z;(z-cm)~*WDB=5rnfOTmzJOq=-UO8&g8A&}6k?b2Fxv6=dd*J)<6$RqN|c0x?sOpx z{OwEONDcOTMIQv8DOk(xq<9*>tll{n%Z2YsLQ@3-qJd^{NlNtl?I$>__135F+?5K# zG1c7lSm8NTG$`}iy025t!t1pQa@4%s@#4%uA77<#?5kG&>YmNt!E=Z!HB)56RwDux zhShsU#s27A=h)|kk7IW3JfCs6a?ZUA58h*TZrOw@lj5dPM>ySO3d&bPY`2uqT@rN|a1 zK%AW*m8;&$Bta|gaonqq(Gs}hezW7bIi1_S1=Sle>F_x(XS)g%da~pzAbEGB%o$D0 zN8|@ZloJ$+pRcGpHPV1?4*N){f$JOB;-w|wcDp;TJNvbXgxj{-dG$3IKsN0}weI3s z_ReMp;5`9|(LXuYNAOk%_sn%L8|AXGu$rVlpq+E1MPQkQ`7#cG>th|?sQmb%z>HcTm&rkM=F9>0)2kMp}e6;LxMk} zBCZvgpj5yC6je9~|BC zGbKIBb0#DISx%*A?kld98VjH|PYK`o z)w;I>R^prx&8DI@V=Px@q%NfCR@%0FPcb{ww$L`=LRYAA7WpOqw0Xtzr#-OJH8Xg1 zY~t|Qu4kwF%vS-DJ{}j=YkE4M`bo)7`FJ5; zmIohekE6lTLbPhNv*R;ojHM)+pZiQ6E{I>bc>V@*9IS!RBvz1gut6!P%f|UyF2>gj zqifM4G913)nc277<$i}&3fHy8){&?>Lr#V^^YfMYq!kdr|AYYk_%!5pPKyv@yMWO0 z*^p+25iszu3)!O?_kP>|-r)c7NX6^dKvv_}GKj~i#GErPLHd3rI#==g$)lNQjrJJ^Zf0w8uA4fD--9oHN3`R~tx9F<` zuBkA9D@mEfL5Zle&@Y-vyu649C>^l%}dCemNVK-=mfm(o8i?}cw>-Ml`WATd$e z;%$~%)H4EQ<0+TI4hniSDlGWN-Z0kRf;u&wiJIZ_2YL$377GvtnOCZ)8T4=W2j1Xg z%nk z)_RcPmHXXVh%XP^xq*5F(u(t(GC8emuq;=KaxqB623CPIg(sGtRy??cFm4J!T<%nE z?T(c}C)Mmlmkw+@A7PuiPLQO4bt0qOmOKXj)GzIF%u1na%o&K^nl*CG($-(NRDHQI z;NF@xKK?F=1~BM#wj=$ecj#$TeT0}qJ3J{eYZdD}(E?tUn(ravNEc^-%1&Ph31iOEn# z7on?@wqQ&P3xw5n?CFvv959niP_@-Mm)YAzrc7C0#61Su83rs?ah^r?OkE$gr7T43 z<1Npbyyc_~9Ex`x6tLU)m{M%Ei^1#)LSyxnBBYMq!mQuk?~Cm^uXQGiRafP5xs-C% zo$jOUtc10#cPx^ry-E-zd6L4@Vz`XTaWNgkyyEe(^Vq;$nE&dU-&yoIK6pWxbuzwZ zuqc(Y*`PF`Mc|2@qC*JxRW=%0#LJ2r$c-LaTFdN2l@?rCR7J({d98A^`@#xy#Rqkz zK!5iAGDKv&qQ9O$E}eo`rP0fu(vI~YcDa|nAg10L8?Vv1Zi<^-0m;vrFJ>0S zTl)GK)hk!aPl@&unNFk5(kO34=Ras^$rW3a{4i{DvU$a&>cs46vXx#ON{d zK}DWo9J5xxOmi-oH_Rrdq_7C$cH-LUro%iY zXI^k3U?e!3cG2wauq6I)(c+SG%h{}o$L^=5+12iB&3^3)l8}89_~&*^gn(E8_5YvX z>e!zMo8|2<4M1W)ZmMuYa)&~a6)t^)flA4Xu`QmNq<`if<>iG>+0xdVed%GkBhPG+ za#1~C%usAl&cO$2KZhg+nYyGR6CSr*K~-q<-9)SC8lGlTJQEkQ`}lt{U|xbRZC_f#XzZHM~fX787p%JkGBQUK+@)BlZPFbL1?CDCtMBnJD_IUq^}sH(+h!KAIyjL5S1cDs#~ zOjYo6b_qirVA3rRrhU|;@!oo%rn|T4x@(xRe)WpP-wx>{?So^1p?d+&o|F2xaoyInMss%#$qQ= zNR;WeVNbSwRh6U8j>(XZ+$)MmcB{T-}RWdnCYhR;YXBjdwW?84>RJQhd%kEU>eeSP=CfaL3nNdwz31f-Awth z2bc-y?%-8z6UiA;m4~Swx;%Zvgp<^A?r(zmkDdk_8w%??lo%BWBfBP%mf_}1kuZ`! z!1?_YdIcqp-Xp9P`C)G#W&YN#{b|)Y^q`YY3k5?as!M#|`0v7`_@FPQv2k!<#zk+jznr^JR#yH`H64uZ#|NdgxS1JT z6*E1O>3(pA${^d3}=17 z&hH!yRlBjFT`@)9T=%_A&=d(^|4XDwaXLAv!DZiRc;BY!5marDtTtU|<`|`I@KNj) z&Lx9p~br;ZYIuLr4Ek5Mr43@NyEQ_FL-%E%~>aDH=L|&ppWeIaw=CaOsQb zJRAA{8&f_!=;G_HuStY6-<8@*eG~j6?)u#q_D?9F1>$1gLV^Rve_O_HRloTm)SR~u z3#041L%qf8eS4ov4F)C@2H%O+K>3d>ApE3$GL0Pa|BZ~iR*l|@^F4xGLgVTVZH$bX zK30AE$;%aq`;iA7;W1wTu;$pZ#HjdZ-wYQ*w zbBAiI7P4@>ZTjQ?2e9A=z|zw5hsxUW`QOu{Odcqn-Cicx3qWpdDb6^VTSg{q@@kNiZ>*@wM0NfB7rmv=+LU4 z`FMV8=ySK3z)JBC_1XNCJacd?)%udb}1P&kQ=zSL#N^4}zZ>47LU<2kG#2k8BeIg1l?7z=U902xaJcmk*d z{I9tNVBMp5KmHg;^`gOm$OdfgT($cH3=_*NM zW+RM>G)n4jIog>`O(}7?K7Vz7{}X``x*&{_gZ;{dNnqT41jKtR{kyayLD!@YP?9T; z*Z%G^^zJYe9i?+Lt#U^SL71SPRm49_r0WC4uO_uDP3nzWmE7 zW|H5|PE>jo)6O05|Jl`$5uiQHF{eO6Q&T=?a&w2YvZmsHP#FaZHuDk+O~}yvGOz!! z?s%93f(8`wrQ%{zIUFMYi@?oL%+MCXYYm19&HkP__Cw{u>r(6P-kwo&2ZzpLJ8$-D-RQr}V530!LTCj<20}6@ z2^z$j6f?5Rf6qWcCmPNkXR^OX@f_^De1CI?@zHx&BG?nelb-x7x3CEdC!c>QxsMmI z%v`PBpb~X<%1)EnZm+9VWp-*b&a-MOF8Pda?w}Fo)vc zm)%}>e=1t`bL!)8i&-1umg7VHEw0-&&ON-@*slc=b9Z~3Dh@P(IJ~imaS7)erO#yW z8@`e&QB6FEqYG$zWjcOy4VkR(YDt2Wvp{^^TAE@zUJn$kcev&&l%8TX)yq+ytHux;CvgOu>+^ zf~T@+UyOBr8IJsCLtQ?bN-Bx{ z6ZD$tY4?#q1^DdS*;DVcQ@-d*5{`03>&K#Z2Qc+OrIDf)#-@WWFSZH?XuqE9)FW$i ze(=1{Q3a#h{Tl|^=jM=4uVnt&G%#Q^y!}_s^-fBr%3;xau>NF%Lhle1)^e4H2B`|W zbCZVor)FPZga!B(6lEFS@NW`g{v2sOLkAR@`@5m3K9f3d%Ak{wvstf-tLgj_bU`y{ zb?srVe{b^z*`AOC9FnFZ6;+bV)%9}Hzy;~_?<4KTZ*6aXSLDoBRgor}mW_;|n^6gs z^oQDoemzc1#WP{HgUi5aVB z(z8E9JI+H(#coSvPk#hI0Q` zO6pkU)C2r*hvg;uBurUY>CTwa5t$`k7njh`_K1NRBNOA~KFEFl zRjgl!v-2u*nADq>rN5+$i0?-OBZPl{f`j|l?vdgJ1S#ME>k^t_suXd((S992?S66N z<54m|oK2ynm8sJnd4+XQLh*7J5G?-%z_qQ7^I^=;P-R=D%Mc|-WFeVz9hu~&ufi(ZL0j0&x2eMLgsG^yhUf2u)qXBKF*Qn2 zSG8)BToD4w&u5-tXe2NYG~Xa7+h+QUwMVb-zm` z#+=VaR&+5^l4ET@1K@w|l1=lR~bcbZ01Obu4T(yN0X0VbqFYnF%rBA zAINFV4UkfxEu^@x^BT9nqJ)w!Cci4UGIT_CL#oZ<6MUMRUHY53MT#l_OzTpnGW+Rp z3mn#6qm(5+7Rppu4PNk_0B8Qyl0fHPBxaYkjI@}jiFbajZ0XJO{krx-+r@I%5b@cP8JHcKJvoV>d`2F9ixV z8n2Biz*c_7rh(V5+2t#YI$53!O&UR!k&jJXycnq`O{iDN>CiWbNkGU8ztP*t6{R)R zUVhqQdxk&bivF{IvBqNFg>7*t#@2QwMu$Cues5Go@T4>2l@ND@z;>oQ`rS$_sgag6 zg(PnSEoc+nU&V{QG2@~%lgve15=JuFn zM5@;q6)QRjn^xKgLCl9e^4dKFBvyS zcYA^FdiwNI_N3~GphxY+pf3Yz89VHu30p)OyGrkOtmaDNX-#ju$Lmo!x0Plmx4si; zjYP9Y!^f}TtaOzoOmcFJgM-8YJa?@QVE6FPP};!r!dLeMpf*^#>b6mCD;6H`3$^Pd1d2RIeS5=v{y~n zG1uFSX)Tr1`5|&hvtIyCO^0W!6{$BCyP)=`En`*9WdfiAMQS= z&h*(z!}ca4?mhn08v@`5Wzum6=)D+1MtjwtsK)+EKV)|uoU9;6%~gHXKDJ?eg&+74n94G1On7cBfRfrMtW`A4J}w) zI@nZnh07;{_>dEy8(Zp>>Gqr%AgBYe9HnA;h3|j6S410~6<)wrC{loD8;NPSrQ8cV zMY6S>?3XDqbTW%EGFq41x7)sUl~#@vMQgA{I;h#N^tcEkAQEhW^RUO?-~{o zOYx6LxR~*hH@;?}HTe<5H@wVttC!tu<;Nz~;~Sym^Wm8@khD%!e(3D=ZHR_^!Si@w z?O3hph|JvyEc_d%lln#0fbl8Ko5tU0Ch|%$)tNXVM~jXohK7f;Te3pJft3GL;GH{V zCa%ZGA6Yq1E|s88hCLGHc;$!oH+*eeUZO8H3M6@TI0Ze&>kF`|o0};|e;n%y z>s>Frzl0~9QH;RZBD)CVpKZ?8L1)$NAU(U>G4Vr(_4X<|e9yVZD4#>HP)j-rM!Q+q zF|an6=9NR{h^^op{Gi1Uqezjfg@pB@NUyN@Xenn=+r{J#em`w0cJ596TXOF6t%WX~ zxtsHu#asL7v0%5)vQb6V9Gtl3Aj#sk~5fG&4W-UGc z?l;jALgktQyk=3$O@E5Pjx$|vuzUDa&jWAwmsS4?iIQjo5N!6zLp&E{Ez2lIfBWi4 zsByM}>*o(pK$JLs%Yp6ffTny@T4JK6SQ4$g(QVmDSCFImJBuJ@nr!*hG#B#GH;HY4 z4mSRAg_!fcdZJAxUKj^KSc zUx)2&b1sjGC2>@FXUtYR!5L}wfx55I5W{gsSZR#Y)Ci;YbM@$425-VnHfi}eRVmZT<;fjnjWxdCQlhtTAi`G z<=#VR**9k>cRwEoNE{xT%A3T-L^T9eQw*J<`(xGUZ4ls~yjt2oqo`B~l9UI? z%d<^hZI-@JPMxw{Pd~V7uV)2E$1jrn9WkB zN}__Qyl9cVw^L=A_f2yCTn)nA6+(Y=Sx(Cl?7H z4Fy+wbh22-wba|aNFu@6Yoi`f^dsD)!utLVxoHoca~&S{=xE7%>CQFO&uBJM1gWxe zHpcA@qDL{{okp*lk@Dz@ccwF6EV0Zs7sl>Bai(gpDh+W2vh1%`$8jbP>D01V?nf4p zGX^-=)=%Sku8I`#c-Sh<%8r{Xc%G~&)SK_sxXiAL&Rpj8PnGYb)2s2I9q0>O9}*qo zBhoE`Op@1`9j%VtKgwjU`FTlPgee3Lj%cO=k&=k9DQc42mdFYV`U^)O$Unu!edT0Z z#6BKG!-${XXhU(;Gil6l#PWJ)eCTT&poyJ)e-&(M&A+}jvxQ-#v|N3seNF%QQdUwy z*lO82u}iOoQzw3ib*lV%E{i^NUH{98z-e0!ay8{?VQ5)~TtSFb_vHUYV zYWKU)|qK(N3dE73~ z*M6Y)dX9v*4UZa2Yje@^^bb#+lCsLk}WZ0@F!K)m_(OFEFHLP}=l zrJA^~+C{w_wrF}y=4nw^=e$lUNp^~ClVpYDk6@pPC<_0^%CES5K%`I`EB-KrXnbCA z6^Y=rJ71YgiP=?F$MHDLi6};*Z`wbR$Kt3HItN4ZQyO z{d#rGWB0&=;%g3}*Y1fWxT>fzr}XNe$CYj81A+UJg@B`*@&SBWb3M)bnurP&uYw|1 zi*}4Ag$>|Xzr3)Ru8K)w(?}Q6$jB7k&HM%?>VoaCs9b`S_eYgPYtydGHgS0$>)a4!IUzIk<{3HQa`Ukfz>aGgF%PXDDK|s?IC# zYgSPEPRZng>W$AQl$UuoB-e@E)b!nk}4Blg?SAvN$Lx@@XL@5MdFA?s>}8nn`G0iCNkL)$RNLuvTDB6n@Z3RrjHCmHOxUy&-*Kf*n zZ{vslA6s7;5ZAJF%Lz#c65KTecY-?vf?IHRcXycxPH-JGxVyXS0E4>(cO86?H_5r* z`OdrV?O*#}_wKG;UA=16s`l-oY}R9KuB;6?O;&LDPCz=5)6G=PlP}Plo*pESs_jyd z8e3cO=Ac>`?QlLKGhQ2E@y+-pZVlk>X_Q?@o_U;3 zFi8(j!XhVaNnv`k&WBeY>`mt_=~+rk3mx9ix(n$OqgoX+m&k#ON$xaA1T|elyLKsOtFXKXIp>1 zYj-&OAwLtCy+tg|xqkcl9DZd_Ls?x77(+o<4r{r6?&yL_ILglBuHQz`_ku16I?StI|Kt);%Uh#IRi(j z?-sN|uWf4dRL5x99xt zT#t7s2|h&7?(cHXrlnpz(6E3}jxuAeyiJJIsZORyeK4`8)e_T|9>(fGC0u1K1D92t zEkb`#Nm$JKMmFPJv?47XEezNSjDM2?PEs=~^6q)V=YxE#qmb9+3veKvD1ymXg3;+r zv`^w<)OM7D@upiTD$45BwaiIJt$w=AV~dfK23KmWM@lMww&tDNEtr+EXu%h#ilegg zB$tNv+T0hUW~OEdACZLHJj8#dQ+(P!)RTmi|r0`^RH6UYo*L;dd?!%iik~;DY|JjwdnX#z`?N)b8jB> zj=^fxA+>$gaL=5+qh(G2e_R-k=031~Z-T5Nw00;%BlEnu{&Ay70Rd@h>a@us+u~4n z!HQl{VX>LvIJQvQf;aMk3Qf_Ur&@)!K!37*t3$Y-_g-E*sQxon$-FfBcJIEo-@JZB zJ8p8zp{otHG^pF;e(Os1IJP14NuXV2)O)S7VfK^cLS1lkO7RqX`%;2ymXrfp^^4Om zZ)YpRXc%&zaI|`5b})q<6z{u5Kh{ci+BpoX$lAO=*mpp0{+g zqE|W9Uq7jecLA?538f9{3PYjS!D1K{f+B6d3TdCNdTxaI)TBpLocE?iMc0=Y9_1VD zvsCL_C(fB#R4k!=#;eqiBwf{8l8h(~3XM`s-||!^OTz@b1|)6T5F5YIkVRCe2U?O z(8kr$$4W~YN4`ejt}`RzQVZWqjq8fTK%YvxHFVvxwJtqX1KFgF^aIqlv0a-yesHk zKOR!D%gQ_0YMc`W)4^pne#Qx_Mjq*d9?&kbu4SEGFko||+{Kgz`zELa9a(Z?+aPgRw&;J_ek{WcgO zB{N1jKJR4la%KlPh~?<4+u-T!H0suaWs&nG#|($2=ckve{9OxQIZP={iGNhgJdK4y zuBih8M=PhyR}aJyJnE>UZGGW0eRz0AKw~geq+43WrE*jtzzc673qCt(cVUA-I|T;GoHbils;p! z_1yM1LN;dHojM^4FCO4$%r^bl&-2$&Wh4XnUg-&Sbu(pc^=Bv>6T~}mtCI~H%)gVK ztZ9GJp(|Oc#l|cKS@nud$bXgkn_!_c8>Sa$Q97u;lI+lRvi8`EA40YeR}4d)Mkd#~ z2m3B$y2`qBPkgFEoT8gz){m2OrwoNMClhAUH~M~1AuLuYl1gjzi|W$F2q!})eD7#M zCtIV8Rc%vkEp zvcI7v6W9uZNH>I>={HtLX*nM!rO){nVt_apz^3PEZGo#sH9O@gio!@)*`-03FA}yd9l*^HCctaIsK-wDL))o-lYiB@ z%)tyd>XCNCzHNKV>fPv_`$fmtyC3{X4{q!E;4r3 z`%#N;bvc_Dq|$hPY4fcB{0EvjxfUIr%ps5%mGo0PZ;xt4sl5=2X`ok9nD)-j2*6yy zMw*@`GY`zu61yf8Imehqcjs($j$+4c78p3RQ6HL_Jj>^lyO#TNip7?B8XB?hzxGy% z4rHT~p>-R?+frYz-581MiLMm1WA(kXwLWxjd**;X>Lj#d76YQJo{Oh%O=cT&%E=#+ z|AN3Q@wy#7>fKjKh%9|V^)Z=@ogbsFDl7|xPw*$vcyh~r5+;)eI8@e@ivIpx@T-5f zb9gfGuyh^^to}@nx6S}EF0{*o{OH>)`kq=oG#PtivhC1q@r7pWL*sN}IMsC)1dsU# zuA>=n0F9Jik|~RYL7Fx`IFv=-hGc)-XEEDrO4K&~6cTxy0ucg z`T$E7A-Z8^|LgVh7mswM@H^liTW-4URa>_K&fczC;!25L%n{Tq+xQ=NMRpTa%=(M` z2CNzi=6-n03i>M!=;aPSAoI^U@O)x#%wT(-6AZn90=1b+bu?E6=Yu9|O_o?D_2zb7 zE4RV{J9BtGyu&Kt;VOA=;l+)9r&j#J>ok(GDJpfSe_R4d?612p`GuL49&Hq zifaM2^52Upt3_A%1mjHU`MAgyECjBeP||fkN?#S-RW}JGa^3~Tf9T=)#%eS`AQ&#L zQ`_%3AJRva(nLGN1Y#}$YqkV8UP{z|q{6LR6VF$ckzwbI>YXoiJa1bB? zf#Fs|Y-TGy(~^N4!n_|qgKynrl>GAt_uiZ40j)@crNs^sf=Xhgk)M`pot7#>C0D=bO}!pBb8K=hM|%agx)%9ZSF5ddc7Ry`MYwbg@~85#cy| zaZM^eNkj9n(-)VKKXFIOp`EI-Xgf(MPirQTcfM6BeZ~+W=sfv+w4qx~x|ma=>L>*| z!{ww|5K7Yevi+$0W!nUP%aX_UWJV2s(OOAw74a-N37%2lSpG#?Y(0{`ErN}8nymZ2 zE#c#fg|R3>3>tLK-SwuiGA-qFz zUyZkgIi82?Fgpa`15qY4tq>MwEAcg1SN|2&7Xw61NatOXa3c0{* zbP8#F$3n$q;4+iEy@RzqD?nAz8zX)R5$rXa%*Fu+U#>>jhHY%FJ-J3|pM_#QZBdXb z@TnxG@rY|s{=hj)`vZarZlB8&C6^RQbB_Mye0RwyUE88Vwi<4+@Mxz=;&CSTR~-=N zV!3}#J!tRR>3WkWfwM&D$6PotAAH47!Te>x5j-p~dhe~g z#3*NjTVMu_rk0mPdEWe8U)qECn=@W!0^vJ05$YyOX(}6JJ2`1snL$?8VNaoN}mE1 z>he^#XFXcPs~|Kl`}@{0mWF>}Sw6Xw2ch%{n$`S6Xzxlos`0$kfg1e)MUCn&VclhYmFSGv?%4K**#3CpR!@*E_ zCqQ`n2DF>^@);)$0De8p5S+HNEJuVku7;4Ize=aEhZFFuv;yPzOm!Ja{EW?QNN+4R zSzdX~A*ub<#nJR$T8DB6Pd+JJgSTJPg&hjF zVFwL1LhZy>3hbrWhQi@XoIz8cRzC`{_4ErKvvPrrzx}Z|*(Ild+ zNuM}vihWRj(bje~`B&og*QaaBG8cssA>q2*VVH>M^NW{L{J9VUDvJ#~op4Jt4ew$)i_qMDhZOFZni>s$rkD9dEl= z?I)Plq*wU2Z+=Btv^u+YvN<`WEt*7$oGjy^l-ve0#ZqHqh1-$~`wB`1sk-ych;~Pp zNI!m`y6Arhn90XSyea1h1&SZ-36jV`gSgAHKP>%3|46_0JH|xG7twDPO zvhF!FyC#$bhWprLp`Wt^hw|Gw*dyFF@uG zbC&o0A1uH>F)W;Yij7@`y&^BapDF>rvA;ks!zvB0T>s$~j0JAwazq|HN0_`iC>Q^X*w zr*b#mPdV%slv2a5u$+80IA34Zi}M=6aqJ+|fT8A^bf|f^4}H;TtQZrmwdKSXB+VSY zOM<^ag1$k`qRInoq*;+7GO`$0_Dno!Sj@8zTRn6TllU1HaJ1F;ZPrrBZ@Xw~cC&AF zbfJ#7-DI=>#xUNr5YZ}{GM9|XHlE40z@npQ9EboQxUtOG8Y_RLVCsgIfb<4$;|wXtyWh~v!S+` z8?NfKT05E)jnMUnEgyi^mbtoT2@Yt~Vj`e6pk`4pZ&*qw`2-Ths-hv1S+~+V)Y|5g z`b>^(0jLY_ctY%mB-Ef^sOy}Wp1-Uk{4OzIT>`oQl!*%DGDHl2 z@dS?e`awpLdYMeFAH7aQ^EItDN;#cUJ03Ehml(vFg(f^8cs!GlaD#IrvTqHWUf%ML z%Z*iI5QdE5gJkK7*(a5;mb_q=P8w}tqv?6DZzj_ej@d?shJt+ zyiW>K!&Oe~&QTNteGvss)Z$w4a(Ns~KR-2)(yK&qLhzVin2o6UaI6c$4HkQ0E?bi{ z7HU1k-z2P`2#j~DfBD9idLYEpTFs$mC%HS{6l_)029~z2NGtpA*4GMms3|K1dck1l$Sad(%QZ;a8j+#;9KI->1Xikxi zVU#<*eIu|!x5arzwd+ketzOd240q{4(+dT=p}Wn6B<%)lep|MV?L~MHOoisNvV=_B zj`HZlb4HC8u>ix5fns`rMw)$5&@Xt~mcpkelJPF*ixjf>v@9mmHAf?;m4e^=CvIHX znP{I6UOFG%OL^O4zRE7D&#=1BFaN3xKOb7`)$F;0UkLx!?c1W28ivC`aj*)a z@t?9sSH=LG122Ej#Rwo|m(rHXjU6{C_g%jA<%wml9`1o{;NN`rFJ4GtVKpunSA>!g zgq*z8DV-uVWHqO5;%*Yk)}qG7y2d+owXw_>ABIQDbQBcVJQeYBt}tvoSnh4UF{0|J zhHM^<=Cp}+)tY`-VhMBAe%ow?Gc$xYX1nHclr~m~&^E&K{=MO6Jyh(+yVXrV`+zg* z8?=Ztw!5!d7^@s51+ny2+`zYUgPcT(J8CKBdCK z0J2cFG_g&{o z-LoaMVb#43+`es0F@&N{WOy5s!c^_p*`=Co7a9QsJY6k0q(BZ~F12UkWhTnbwxvTF z23}~-t_9$;HS;a4w!%olXv3-%3x%ZvP=%hqBIkP}_IO>GrM~9M z;-J~7a<%Qin)t)iNTFv>YJtL%zkajm3`72;{6}0a_voxQ1NNl2T3QOS$MG0RUeL(R z3Xo0;XbeK8Fu@tko*SaR`2qgUW|KgJ8jJR2aSTS>V4GiwEVz37pde7T{EI5N)&f~= z$MYJuR5Y`aAM(9u$=K`1v)YsUeJbvl(0!`4wbEg%6v0&?pG);lfDp$jpS?jTP< zGBKi{4j#{2(;Q)DE+kZ7mG0wIo|Hrh92tRiNR?~SajGOmgkyc<$TQpZQ3w{Ty{?L# zM0x>te*`b~a!XVlPo*lYXL-+|6zLCDyMNdH2I8KbrtJJ3uM`N_zRKY!IM@KS==+=E z87;J&hiIyr?3G60oc~Ip`{qv!s7#Nz2|ynZ%rhQFzDt70CcVC`EsIk*TUNswv;T?R z^*)#S?$_)tn*9ta=dJ}EYJSdv^7I2WMaqZvdny~PR$A>Sv3r_0RN zQ+Fx@TIG8AottSaC87O-xh-`VvFN0)9aL=1AZ241Ze_DOVI|jpP`#M4qxTLhwVHYa znSHs52ha{P)U#pw=Emcrq}dQ|Xa&73u8*3#&M2FHOfz2J0A=z}*zUh<7xTdTEsOB| zk5HUph6xQ=iD-8;L&gwJI^2XMjn|b!jrZ^l4cBMSAicKsJTkde8KPM-(oqTRiGX5R z8f)GmhT?>wljUe|0bgSZzm_OBAwbmzqB|>Lr}?{#COL7G47;MBhN7H1?(!mwz0a5Q zv}AsjNP4&D>1*bZ!^bFDE)Ou6+Rv+$@N%1hMvei6Rw7N-ErpjAumvJ%@UsLPM7T$WmZLi^5T=`Lm9k*#V z%+v?(^dF+!C3#{?W&_0#CQ)YC>G5a|e_ypL;$;Jko@#FWO{C+_f)@+Q+aK=EMa5)` z=|JUl?3eFBK0}*R@X&om)eHcyY{xMEn(&Xc_O*n(BH*1jmSd~Q1C)-8OQ5RMO9k)2 z?0PaNyIY$#ZgahVeO$us=(dIo3OX7Hsa99lrHU&FfptBf3%%gC8U-BmBq(CAPk{T7 zd~AoZ1}=3-y7M2;HaTD|5yAeRV@JW9cc83_=3MKFy-_z^dhLp=HASIdflJ3QhNi7BU^jo1V8#DLo=Hxou`Mp_?1Mq-U1DSxOyYo5O4(at16d2flY^iT@tnnu0v2kNro8HO%?#0j_E z!SG#c#mQ?wFM98|lder452=bme~DBf8PuDkYxABI(ac@*tyw2U1C$>w<}4y!mhEZK-UePNt}qZcBJ~NJI{BIt*Ig!29$IJlW|&9mz4W}OB@)z5_ZHSsr>qY|8p(ni5 zOD?t2@CcPhJnin^`4W4ocRam;wLzmQm{V7^r&ra>#fcWUsP?u!!AzhTn=i47a`mT{ zKYr^wvmX7Su$uG8aXc=Ch09%sBfH;H07Gpl3JUb)hcvjY7x^OZzaq| zi_tdeR*DpPJgHq~bJD{MtB3E8BtfR z49uKlKE6Hon6OdDrY2Q9H0D4e*|8IoC(|RAE`DlT%pNg~VGFY{Ol1f&VcB9>X%%}E zNc#*%g4q*H0E-L8{r-s;D$C9l;Sn- zIh7Bagl6M}drpW=k>_Ufo+>!rf6@X+sCu!^2W1n6chVbB{Jh3L4xz-DUE6wG$v)_b0o z&5gw3B2mh?27z7Z2%6|N9c-R^{J(3zz6?bY3rgFi@%!+7^&i-19H+FU3|&$*lZpTr z71A-qO))v5zip>x9%RVVuirv=Ij%R8ke~5%VJS4K?9H!iK2)6rdQhb{7%P3>7Pu|A z-5%)gT46dsHaxrFq+1aUM>-l{hBv00L#8j9;Y%KK(%kfpKG!cm2sg<=(Sxq|X>lUn zH_$#c`+hk?j85ioP*PoP6H6tznrB-^YXP-l1AG52JyPDGun+e$x z{k4bnNE3^i;y)!`aGh^V_t_NoY+aFj99VqB@Lo1Pd=bg00`hjldE{_!Yw-n@-d#e( zQZ>fj)^l-@fOu_ZH{4P?_*%VY;0h{gwPgSkgNh^cYM-^6zWJ;@Z1s+D>+545nftEr zo)|&Mo*Z{+GMTRT2Yd)%9?|N4pA}Ye*5LhuQyN0dS0We<}HoU_Wk%6^}Rw>wvxX>-xSb)Yi}yn{+(7Gsq9U zw>dRT`lzC=Ulcz*u38(b8b2hJe9|W@zR-up{g7;I$Qrm>{>5x_aH2HG-=!<#M#FSe zlZWB`^k`6sN~8fTqUL$`aP&zeMduZepW)Z;k=!scl~Vy*$@}+w*QF+%%Wt4A)5hl6 z?3sQ<7-tuz9xDucVWr_qQIl`+_V7N8mYD3ith`v37PvY#gYjls(C<%vVqKu%38)cU|(U!QfC6$Tit2#?M?o0FIA+ zh!0O00AgdZ4z2ygeex$8)j4Q_1=*QHbcnTgZaJ)3J9#hI=NzSZHsOIhE6Z?EBYOdo zVXwY`AT#8qlzj31__L#-q2s18y1YhcITW1UcuTyG%f4%N$LJ54G&aU9XWfX=;6O98 zvR1Y=ws!s)p{h)wj)^<4ntr-*d`gCcfw?!^H|ij`pJSxw5lnPGy&kTJ0dW!|UGO-^ zctKK{jwc97lD?7;(1i8fWeqUtSoeJO1QDyq8w#9rwl1O55Ym(Mk&oEoXZ>xRA_l(& z4_*`HpWY5Rh%29bQqr(rx*k{!(|P%NEJoO}pX>*aHN(V2{^zq667Cq%t${W@m5Hj? z$tpx`P-c&RyUgP>XIiG9#2BmxwhIkB_F?AH7dS({*!N`su+o z4$We;J2??5jdSti*2KVW5rd{W%6CiX1|&>6Je#ghr*UcjB77TjxM5X8Q_)X6&DGpz z5?G)4av4Dyt+JIuG=Q?^%Y~?xvGG@q?xM|<+MZnTxD^GGY}`6}{`-Z^jj*}dkyJ|C zrg{c&hwo8M1V5%_to3YM;@F!#*Ywmw9P|TmHsm4|^$g8t<(&ycrKfAgYwjSP`T}=ni2bq6tPy%UdQr>ayg0T*YIPp{%QlZ5k#YXDiqEN=~gj7z4f82y#t% zVEyL3m~M3YA1i>v*G$w+>U=XwVN6QUuX#c~A&Q^c^^j{@G0;`fqw z!u=_7m2-yGseBKPR9e~{O;<29tZXe{Osl84dTF;nt|i?~H@b=bA;R&gd*A;1ZoUBS zz-XYXa#7D-D9efG`35Sk!LNSujicCEj+nWvfldo;X*QKbL(5z&CHaL4A8#WTqAwvyb$tBcO&iTJ&b^7b3=2!*w@s{V zGe6N!5yT0)vSptGZe3yWYI-|DZTH%9&pY5WjB#>x`e`GSgVz`HfBr zb*=@Tyi(iQ9$|x>QYO8!hZ!0DH0H2OQG@-JplHfgeBSd24IXe(9%@x`62->eG8bWI zxzBaC65+5$wj2SlJEDQr*+DR90k8gdhv4A+{w|i+_7i)oirXzbzF8Iw?u4sBY=s7< z3%9xg8reS(sWk58rKggT$$dwwuL;$@n&hwf;W5m25!2A2iM2lc01rDt_k)zFDcVNK zclML?Q*0IQaY0Po!Ccq{BMNf)*sp7_#~utpK2hSSPAHDuc>a~!#|h*?vC>>06rUk= zLwbzkPEVUGRv-f_jrlZ_LjI-kNv4gf2(mWzwWTy}=Zx!%B{IhAk5>%;p8B53AATDQ zyQ|UED0{E{Ch>LANJREC_dw9KM+Ao3%yM>42gUj2L;c-(lW2|>wQLl^Y}<3+ z)cR%LgyuIZ7C_@>#L&s{02(NbEplZkNH@x7_4X)=#G#{>HaC3yog_LRpbvpxXsL1D z;&VjV-_gW>zP1PRcRjZ2c7qtlCfaI?A#+@43_Ai^qRQY_Ldt-4UTrBQFDE;DORk7A z6YiW;f}A8b9yiNZ?lo9l@UP?TjWL+8GP*}MGC!0m5)Z4>R;xk=KcM|zy28ZCMnZdw`22Ir0* zt2aiD9v;3S(N#IQVp+}&ukq9O_WTfu%}-@my)#iduLFfz#9e@f>3dVed7A@*B*E!8 z2lhqm+whqutah31+Kr)0pYb&C&&Rb##k5jV@RO2Llr@=f=0Xx+#bF*EE*ST(K8Xjs zZd|}e!@XXN`M0gW!D^8@D@{#IwhepSD~&%8$U6g-%Pnau)QY4OKQNUE5c8s{;}-V^ zS$m-+@eZ#}j7{~6QYWcRa}1}xK{3a{QPU`=Qd&w{?uhO*?A>U->$N1U8FGZ7q*0zC z5Y3OT@njKBLxxfi2uvB9n7?AANcQSZ5;c-4)o`Ls(lmFx)jA=^A{`g_N95r5c4QqU*}}IhwqR@ z^KXv@8ySBA=9%$dOElfym!)BF&dkgw8>5zgYuSBsl0nJmOzuAyagf6NUL{AMKR}q9 zhVh4qDIv1PSFrM^iOhPv@CND*7a>cC*(uI_RG*R(2eK*jwd5l^kK;8k{5h0mt~6P0 z^(Upf9!m7wo2m=?3uDbd|5=HjK=K#KTi@?DhFsHpvr z4PpGt*Wo5KV>7cdSA<%OI|e(W|A8+48IXb$Mhy@nWOwc#X)wN}ex0|-`y~2`Lh%CD6qaaTpCI^Wp*iy8}v%ctq!F-Eaih$JCxR|EcaWd;W>l_ zS`-1<9ji6h`L~h_Y^*OBu--;r$VbdZa(A=STewxpB=H|_2+FVTM@K59<1z_HVY<}6 zMl-iIjXqj6R08?>JdaDpkk`RcTT6pBAhVaRzo+g4=i;Vd-k7QTztxoBv|(NkMNLlf zlxFv~kwTiovXIpLfwLUt=N)eGudI znDflDMxQ=@E9JOlw(QF1mHG2H8xhCkJjR^*(9U;f9V-Zh5u#a=roEbfM>_v39b5qG z8#ol{oNwLT)Dwvbu{i!7zcwaB6N_B+74YJb@H^5+=4CsCLeve<#`Z_LJesVAwdw+WIh(m|xruV7yK7U&aKNlJx?&bKG=a zfa*)VYQom9ui+ha1r;1AssuwxPj2sWLhP+0$7v?3j@SM|POSJMMl${xMvnZRS>Xzk zHCZkmlwM4o5CP71_khN-AhZGMt)VBcomDx#o)^5leO&Hk#F&{^e10_~DaH@JVX#%6 z_LW3_0dIH1f)Fp0picqr^6z@k`hVsG0~S~*7aEbirCBbTy&{m5Rh_;LOEYb5#U@8t zrTnYK4Zl0TUj@Nm@A`NhkSQeuRM9H8*W0>l{9*datUo^=iFZ^afEX2=rBcL1B-vge zn3dCt>v7zAn`pmT)*29QkT;}x{f+kPgc5SrDJK`#(^|RU^mTC38rbls_xu0dz29#l z48EbLtdzVw7rK@ysTXYG+w!jL97AzapJu)aq29Ap>b`X63cX0F&)nW9Mot;P8J$wP zW`3a6h}98^+pBguZ|3fd+=*b+;z|K%EWXg;a(8zX83?9&jQ=ro?bT*-EI9#LZtHO`Epg(!)bk!^Qio$qm^*H!b{&>QYTP_!?FwyKeyxQU+fu7+A(+2WCQ|Jl9Y!Y6I zp`#YAp*||9QRUZH__`AOmy7+P@b@sb__eRHQ?}$AP{wlAhi7V$O{zGaF~`34>Y=<;pa|R}cl=M0+&_ys z{+s9|rW7Y5Zy+3I^&t#hjG&a7>R7Hh;`-z-MW!Da2d1tsJ0+x@zkce+kUPj$P{YcK z@Wnt*%zG7%E|x>ZMgS`le@7zVT3TD0w)nIchVXwy9k6lsU>8P1sq6}~KEoDkbNQac z|G@%+sX5R)jVo;#tx^&WUYjjV+Aa?~xOllH~s<4a3=+2VcI-qx~W2{gbXZq5G=-PYx9e(H8{kW(UTV=_>XWQ$)?E@PIN1+Hd z6!+CRr?@8$Sn?CWL*{ud1TAHCtZ--2a;u%LQmKB%XDDl&@L@~J8(Hosp{$waJJu3s4zZSNJ))vKm8+iF2p2I&M`V*F1GL(u97jKH- zI4;-sSBiA)n8Q{7_?NFn3ZC^8vJRVa4l(huC3Up4vyD|hN9_%g4^OgKjlgRoU}Z+Y z8O+~a{EXzV+^@9^%Vbbe2ki|o2MAqVc`TV;c(?WQpEPc6<7xcY2?h7_JFIE+YvlY0 z7gdqC<=amOL6_eSQmC_wG4$w_BqH+Te{S5CrZEo-#sdrZn=>Oy)-V@auCuP(oY)2eSK{w95nH?G6QOwOaAA zWG;np+%aAH%INK0fbaL`13w{Dn7)ctk>yRs_vvdN+>vu!@Be3QV9hZ*4ZARc9>wry z`#wbkKHoi!6ludQJ9C)GAW5i_jXLj3h5Kleo0-rp*}RBF%Y;|#tnX~M(<F2nfh>(GfJN}rSIggf(wmYZQa;8OHVCzV(|4Pj#WswY`c0+JU9JSfr%y$ zt80yVSh1i5TUba@G5ZvAXq{Vf{LA}L&G&b=Q!GWtu!f*ufh+v~@`Z3czkOl1YMr`Ra3*Jh|l6Q{6%(N<#Lw4(+u`>PP%oepD2?HD5 z=NZ~XECb~%t>WfT5rCJ+qePLMf*oLU2Cq>U_5VB6;#MB_a~Lhm0RscwTr;1(+oOBn*Ny# zfnqY`@9pjz(@hY7es;ZL3OXlXm60tsK9s(=xcB#FdsBB*7M3f(a{B)~og64H;Kb5o zbT6;2V{`uMLQ_Ob-7<9?{FrS&$V4juOm?R)P_Qp-jnBSsXzRdH1YYXaCNXShBABVI zmdBkV@W<*jY_ZwF$=)z&5u=7n7w$8wIWGe_fxm+yCd3`35_+{H(<45oV@oTjZ{e32 zG8xeD+R_<8-OA+fMVp5KmDunipj_DhAZ=$TJ{ggv+>5Of{JWj#7a%NvdO9EGpTo4b z%m8O{L&$70c3SxU?H*WR{dB)PO>>^*8sNoD* zB+_6_4fW;upv9B6{leY{p}DwSyPDG3Pz~|I;7SI_7GGe6wR6RTT@TI+Kk1^+?PpQk z);3lLIs!MB^cqQ7mv)CG4i6<%etAyh*VQ}c(`)Ji&m)_^cVXmw1i2zV->hCh*!$=b zSW{(_0r2e;ll}K~3m?W<@2q@ZQ6@VpASJY4q|+C8mongxdo$Fhh6ls%;E&R8gvW-* za_Xb5ulz695eCPd*aaa`n*~>d9Z>&)b1aLwHi(R==O63FvV3x_FR0MwpeGc5x;zw) zO!snu!U$3SJU`jPV7I)%8jrOs3P4(Zy^fZ)-}R{*E-tR=-ry)hwjG#-QU9(t zoXq#KNlyQ_QteL$UawcR$K1;H%wD&lFsWCKBGrHU6k6Di1c&cbt}A>0Yt3N1y>|U^ zE=boeyvoSrM{Bdssl(0c^W41bp9U!SIK;$;dsGQBkFe7qN#N4dTKo0C{}#R9@W~Ix zA-a8VnO9CxQRw|7OkjR2gEs+%GlTR*QSiJl2C=v|b_U#W59WSW)6)}!&9L)}RUFM| zCim3fORu{?w8tx;q7byXs_Ng@Z4sj1xlD=ZqqNo;me$F_pLdh4a?#HvEW(mO`MVkw z_ntjEFi#VX zfw^kLM(jqZZ%pG?*ie_fCJdwxcBl2ySlK`1RH%zR@9#9$v~OZ|92ow!NciW!yN^qJ zS=zdO`YxH8Vy|RSU0r*>1BzN%xibx^so8ArILQph8wJ>`bjZpKqsY5&YfolVF&KEB zoChbZ00u}rqRAE1GOAB^g`XIs>o^^|6NS%BYX!gB7lDN?&*wC#n1mbUeE0i0JZt2E z6-v7AV=bo~P%=7us=RR3(hGC5mBin~9a1`O`g1Nu9j4v--`Bky3}`acf^{%>Y{&oWFKUq@7;cAtSd z?tC;K?Gy!>x<_2Kcm8|F!|m~n@Z&8&xMW7Yc50fN0TX;sq)A8lHc0(0KvJ`fCTA z%E@cq-4AX!687vJ=ZHN>(a;AxH;4_)wYAMI48_27K9!YD^5z+m9Kt}whgztS-4kC` zOu8xBO8+;EMe+M><0uxt)k>cRM6zg*k;9*TFZ5NX(@i~%w5op}*~mL!|IKu?DSLCm z6gkM`WN@ah@eGAxskdhcYKNvqbo|e__Kn6yey=n9@3ri2)wm}ivb82^AMj(lQZNq_ z@544O`gbe@+Jo&N!3+tdxpj=dgif&jZmOEvXhhr6lTTid`NxBlZ@I!kprLlaOzi$D zOpD#{r&Ckfl|x0F$2u%o4~6MP_l}$5J+-#(eI0nJ+S#YNVIh7!LzTrsYig#&>n0<^ z5Bhp~YGxrFBz&Gg0&eqauT%kTyEt_`{NjN|B(0^LaA4Mdkv3JM<28QV3@|(9TAYdy z$Ii~4zVJSfBi2}hd#b$C{byywaQ>JAtm0{SFzIJXlF;McN>Ln5tkLkCUN(DH$Y|cD#gR`@H{*6d0S4OeTV1zjDmJ5>v?0P#$iU3b zuLW>>5=pz)K+0gXHySw+M%n3oN zDS9%hkG4+cMN{y4X#hgczZ#)pye~O&xdDLVQJzr$nt?C}eSkZMiWVufLPH_6u;$fD zgWcu>9PU4VH@?X4yfX?#gHALvtlQJG_6-;&wjF0AU-K$axN+~-|DozD!`kecZd+(e z3lv&tacPUYyFD%L4n>2z6xWanQoOiBae})ALUDJO;2PW|keu{;-uFA_?4Mlx$i4UM znb|XI)>=O7@k>nt+?<$?BK8e0n#p>9D#DCs) zTGnbRB8!b<0n4%c8)r{jRD7<3?`wJ|7R}ub#tEqn&hOjAz4voHM%yYym-2xfVI^&V z*Wl-m&>5~7R~DevZnV72G}}BvDKXvE?X$3M^7hl&6H6{9nw*=s>hj;I;w^H;CA10{1*=gh%?_+E$H2WEP!pOZ z@xSZYo4=|GxTXztp(O@hzqWr*{}sv@K|)(LFQ zedNvk6o1qTvcLus+f5+6(ccS;nb>W>(cP7Ay&J3fYAkwt>0qo3?4}#rUEcdaby^lNH|(Uv0F zSKcKwuS!&Js{V5NW&W++JU6}R$rGw5ht_gS21EJL@*;2?KLJY=+o-KlSEHPSU#g4c z5P*=P-}glHigS-WlG6D%<)N)n#}IW*DIx zur)qO6=(Y{EAu3guXk+4J&F%2KTWC2HLaQR0zod%{XpEcb6uDSBi#sQ`(F=$`4_bP zHCutaal)>|a!%s}9<_a3c|3;fReQ=>rw=^nBIQ1F7h=sdT&}zf$&f&2&KQ{Y zeXDd+EVlR>~_d%9O5!#p5f4VL)g`2H})Zmj%l>G3&+O# z40T*8h*lu~im=D8p*L15_MpZOk8-$Z9HAi~vD3D^F4(F%wZ}jTQ za_@8?|+u6?)Ae+VI_$XVq>bMF+VKR~17!u5B zrXsT9@^1Pq#p%3{?zw!L`Rr}3SF>*p3FUoV2ES`fT*P5XIV6qkaV=n5+S&FN#o&5L zd{e<231g_Vj+;qNZO*Rx_9)^_!;3BF#455VS{4!A^LLzIn(yP#V>13{d$R!0`9_tb z1U~Zfxruh#7n|$e6+^azoB?g4wQ0fO4dPGU=GLt@Lt<_+29qibY|IME9Q0y;)}}oe z+UWYlToaFgDdL)iet4HZBz@gh;y94w{Pn+k!#L1x4%cWRtUF6gU^R&?CH`gKNi^wk z^eL9|HovDvpyWi5-Gk@+rQ1$;4+nYVuJC_-0WQg35=Bnh>7=)$cQfZv+r+gfue4fx z+2?tdgG(;5`|l!y(Yi$T0KaHl>d37<-Rna~WxyVoqv>L}hq6s`Lig~T$WZ9yvJzas6Wtv1&%T4EsN=tIFn{m`MGSxAYpH4y--{(QmHP13Pa{XTkA+tiAA@*UP}vU=?`)A7SEPYo7T}G z=N`TXBvl9V(Nr&`q{eYYSDtR{7FUA9q6}X%{qM8l&)2_5ny>G{;EywxQVSfl$2OxT zq)eJzIkgTUi@_{`wr}2C6^To&n9pI01;&zr^AskGpM|%(xU6x8CSa+O@9*7vICdu} zqLZ-+HUA3`$ij;`ulvg?0_WDjL)pc1vB;NfS{{P$jYIXa5HEybxZi1z%Eg;qhgm+M zR%}`;!}Wv9S+cE)QilnSXy6g9ymX;%*@2O_j#5D!3*t7F|C{0)E zo{eucMq!R>_cchr#v&nAkWP3KT#)!ZjrZx0a8v4+jIT|!Oj#{WdvdS$h)%cI04-V% z>(-$Cp3mQa+5Bq`%tnnVYt)1vD9$i@rjxdQDwkth5jWk>0|&KJN1ZLV>=dzN#aGB^ z+*3!{EqtFJcliu_%pVC;W@=pFd6fI=zva1qu@olB_M_ya(G-#YNc#Nvd^@UtddBK; z3PuI^H`X0s2)M7Mwmq9C%M4+`&3JXh@MUYxHN!A1PRwI3i9lfyuoK7zXyR^W%a^jS z#qx9zT50}bJCGk3HYSA89Q{&}Y8KUb=l6{D)_n2~?np z7gf)2VURz_H%?&90E@35HK(q(Iep+l5B+f03ErY9Nm~@Cb#l$la;=p;W2QD43f}N~ zg%gudZ5JCk*3lwHv?#;;*2QSq;-f$WtK!AoI_(C5B~t&oGhuW_5ka)m6R{TCTxBes3L(edk3Y3M=4 z&dsx>Mo|pdwxF|8)~2?{4Zb(QUfw%I)F=D_o9As?HBI z$PzV8{_o@~LAV@CCv+TQZ;A~%6cV$R#677@u|}X09pK6R^i?06!hBQ%8-bPw4aBbb zE7^gVgVY~3`jqNvUG8EzG3Tk{`b$!b0FHT2zwbYR0AYuSc^iqzA4tKJhr zeK)Jf+r6A-8^7lClu-K*gQ`qU6_MR9QG?aC%Z`fXg2Bp})TtAR1pC;FOm;H|Ma>6} zOl*%?12#K?uI+g2;Ad{#Nt!>mjplM0mTP!h_AzK*=7jqNAnCvbtsh_Uh)leto-oP# z;^O{}lh+oN$Dg#w%dPxH?a^a5#<22(pL{2zqwO?ir4=Zf(O0SlmUWDmzE<$asZzTI z`ECZUUUBc8do?`L&l5RAZ|+X9IvG+Ed3E~iEUSH8JbuJ->hU|sW>SW99>6veKcNYZ zoYgoTgN!|`*!27#me$<)_J|)UDSuW&m1v0uU(8O&jPAR5uv zWBInk=-FaoCw(5Baz1?lkPaN{VNA#m?jP0h@}=>}h|iAX<3qhGE6>6ANNJ>J){`$( zCl~vE)2zqtcdUjlqfoyG$S=;j-C|?nXJ3h4JgX|MYWZDY`9f3z>X@YX%K~*Zx$O3o z63lV6J3naU^O^8#%=gZ%-!dD&%-6HDbcqY8p$PoPJVn^$w3gta!~1!A9jM^!Y}Mq6 z9S+e5L!S8g4HO?LltouK6ce!JKVCb=bqG>TM&jrIg~~INtMRNssP!+I{WSZe23qsO zhOc2b9WLMe6URv_0JpZNrt>PF+D_uLp5T?%FIUwh8835W-zx}x5v(cI#2MsUg*Ih7 z9oF;>j8IElP9o=~&Qx;W`Q1rHgf}_2--H_Tg2M8Jbn(*J>|P)lrAZk^xAyR`*`fY6 zS={v-=GcQhZdVp@gRhQ@+(y}}*wTrer6cq=XY6@|lI9YJoYS>~j z5-G7v2%w>fqiFvOSTV6cE8<*L(}iaStgG{3v^eL4vR`pBl-oz2%%i^#PpSE{(@93F-z z2o_@rG^Z1O>O)^yHhMSKQ}De+Bk9-x&t0Ej_~nx4RS6=a9sS~%g`Yybvq5^dV^YU! z9*;2G!9vvjkS#s?xNU|`&=|(Ozx8nXr#l${N{q~*d3+^)>f(mwg+l&7A8%9?UJ0`K zz9O2H&Ls2hyL;-H?mdFa$14@Sh*;`Xft3 zc=2a{Vd+ZTG(eUbYn_eVw+dk{!cBCu3uDP@4clv|u31&2{RT%8({z)Cv=1?F^n-{G z*L7>1Z@p6rd&1^+)3KK|SyTa^MTTI)bW=Nhm!q%uqY(Kvas8<3Dz4hcrQ%C?PzyVy z-!z4FoshPhyKuLu-171h_$^0y4^`7~e>9S4ZARKcLECd{v)AAXz|Ze)C3%i93R)97 zN~mKwBA?7H66)odTG_VyK<9<#r|OH1^rG19aQ?5IaB}}exN`=gC*+C`!TdlE$QHQxNy{vmpgJBf(oH(aUMR8 zHmC_PUwf}07;1%0KP-zupPPH}M!h7o*za+PalS^0T(o;?R{5-vQ?ZrD9)lOvKsgQk znY^}rOxgGF6*OnDNQJ)6vanMwtIsG1HC*o+sdpoXdzm)fQz|I}6*5|4^6w_CvAH#$ zR6}6PFt$Fb8oz)0;^G>31@4tw*@d*p{)p3i7#?Qv_Iv2a4aX3s@i51{pb- zntuFaIN%!^p`RCqE`Q9(Jqq^QDSgXTb(}e=2rug1>T%tPZzvaTJ5Cw}$>sDL3*E2= zSatJEiZyj9al4qkHqw*Fihi!TAmMfS0#OgGr_+{E(#*UYyz?4WHMkuQ=Xy+U>7+05 zM9xhGT608f=U>Ppe}GI1>!Jo$n=w~t+B?TT<@pG34&krPfsykExJ>uC-KD)>K zU$9$*)L*uS_S&kKgkKnmBhv;H2hKMwDzgCwb=8e|#b{xv2wS$F`IO{+azdpB4Nk2W za7Y%gB|y!x(l^73=3x1Jv7n-UV6*4h(&_2d!hec57JC$Y8Unmy{u0F+C5r|r4lGsx zV9KaT-rO*{TM`!oFLh8tZ3jMcK$Ss5m)EQ$U*}{oGYk&>4jOEdP@bZF2Udn-o&|){7>9qj?j!nltC;R`89*BnojM#yQyy8Un z_&nQiN4Mv21=T(*!`(;nd2!)~2%aPzLy!J;=VLi zX0)WdzNafCE%)nQsLSr*2nw1>oAQlP|BG+hENGzF-j#s>IK+xQw%uimTW){X3*Ou; z^L9L347|@Q%*;{glMYpI+%J9cUKHAv6*rPP^%ek))mRc`e>d4Zs%^6J*`SlNrwPF0BefP>kIdD8|1H-Wo@gCnLI7oZJmE*elMLsSm!gHxAzr9f7ZRDJ>ezUrx!D{O^r@1apr}5V3yXq#kIVd ziY=HBVGm+H_Q=fDo1G$d@~*#hQ$ZtO2ynlsXvg;k(4yfmpj%!45#+NypW6IgQ3T)G z9@AjyHzFy5u#!~DDDKPdSXTam<#T#AD8na%@vz|yuLbwW+d@1vsER9)M_06Fj2tcZ z_GgtgZlLYP7x7J!AyUx`1zP3Nf~hWav5grD5d4Wuwc zmEyRB*r1An^8DKRr%$})4(M)c0|s{IE56aB0-YRRhjIbuML7UimvQn!2C-Ol2c?w!Nxep*VvzdVfrUWAS|fg`4DTEN z^(9L}gW1Mt%+C)slv+~qa81>8Mc&po?so}sLN%B@oR>46y#YV;xDrcL@E%zhuw-M@G=21$+6^?G&qGjLed9jpbfbN0WuEQqF$pkBq*aenagU_0$G z*hd;^n3XDo@TEO;xJ65}90YX!;7x7+&L!2hZrUglh%<^(AEOY|*hz;0hxTH80?PUe zpN)58hOCz?7Hs*-Vng_D)EJax{0Szs9*P2>4-K{7ZH2$$}p zj+5~8|EWw5y`S2o-9OuDzbmQT|3@z(STnmOrzCu|ff97hjdF@wADj)RqRiDaC^>-Np1a!*l6$Qf*juR{xIYiAIQH9% zl&HR#urFRZ7bqt3Yw|db36iN2zj)N?Gmm>g{rIzdqPFpyKfg`%5)Mn)O^KYQN zwETiUwD?w9?`#x+JgF@iULUaW!AN67yB5hf+}Oyo@nJ8j=j?lG zqV~qEP@Mx+g47hZ)`zvT2Opg&A2>Ez3;^G+gov(}w8+g=dxP??U?6V9+1Y^CGD7g^ zjBg_j2KjQtg7O|2!BY=ew%FrCrUX@u9*^vsy%KI}JPF>8akmh7k4L3hpbGh)r(OXv zMoWj-l4u;d&gia9M}ZXUZXb`d-rqH0pg3gObU&_$QP5B^$PXRf*hR}&x}UKcy{9V} z~~>b*MTKknnN+XmCi!&Ts*> z{!|#X_%P9|smDuK-u0qNdL42=!RHrpOxi!o&x9=E|ejnqvN*bMQ!z2W`l)ZcYOwvBjyvn z8Li!b~YZ*8XRp?lOlrOQ!UmnANA!zn6ow%swYk zCqWKuUwxDLj%7m9;aEtK-U{Re4sg+5u92&kyuwp{9M05)kr!( z`n(jE7+*IPKwnpz(-V&taenk{?259@|8lrqMmjgfb_r%U+jc(iHEEQhpj49D=r+t| z5^JZcU_{->-xQ@X{9}te;JO1mSP*jc+aX`@^aL!`%8!-BG+8Lt?BLHhS{jYX432R6 zBVJIK7OC~+vFYM6w)I`JOhjOJu1-M60DeilA1#$*a;cYkL$&XS`;`pLv&)elR*bDIl)U)MA3}QU?+wn;=qRNkgQ!lCg zulA9We}Ndm+*AKNdqncFqc`~e1ZItQ#T!J56RxmqjCw(u3Pn007G_s?GNGv=_6yE_ z4K%>~=sn)G(`gRY7sL|@9Hx*b!vPne)I)VaHdtsr7zSsafVL)^kjX_i^hH{voT8a;~I49qdS5;d-8UbZRu z>alEgOr-6E-eWHligftip@*D)EbKdjPWE;T(9;9RyS@obzW}MbV*IKlqL#BW>p_g6 zm-mcrad-|37h}290Z8kF?-!Zy792!_*yYeH#Cax-CrtsIgJV)vD#V)67Q?OX^msV% z;1>xs=Em-PkP`mdc&)&jrP8?A@|zw};gQgCc&tZPhdi>jCO#b7BCq1|+VjhEBZ|&K zC#<90U!j@qlY{n_@!xe5MMpy?t=sOl7l4R1;j5Kdao_70OLH6XQYI<+w%Q2qqC$Pv zqnxQU(^Sk*{#QjB9)AB9TO;-j8x7&032#W+iC%fwsbZL&nt_8j-0nB$fmk_5fR?Yg zOiW->1L&fY1qLy3uh)v9$X59ai$+uLA^lccq6d4W<9TPGmDxhX26fHNR&tOBUCx=x z={x$Q<3l^kqgY3pM3hL!Shx<=csPobtF`gBOqzz7J&x?2zie+Uer7$b_kx*=d{rKB zm;70;_Ns;A^-}}04oAQ3-l9>( z=54*`Cx+Qv%D@#%2|IFdHLtV&BUl z&5XP59+(q#Y3cw2y*dHBZ+7^2c&tn%kAOIIZ$dc=nd#Z}NLCd9N;^+tDq&_K?*CbF z`jM-(CxpECmDL`cd|ke}=m!t8J)NsE;LhD-6}gO8I=&WDOE)Z->Qzf89iOWL<|c!i zxwiTj`g+{6Cd6C9h6WWzl^auhZiD;&VJwz4Wny<^R}TFv0{`#A#YLl{7+I64qCD|* zl()8~--FF?J(}-eOTM~t z&?T>bnJdpeS!yR4xlXjlk5dpX);k@Xe@i7lc^Z}~<3qToOq6?Ycqr;9SJznodHKgB zg&GvvF^rn8g!i`z@a;Kgpy?K&JggQMHqPAcZ{84*`*_?=NP__9d3+FEY=)Q4xCY-s z*!KLC{O~3Gay2dUF>%hA}CpUkY>Hw+2gX5yDOVcOAb*4zl* z9~iMJFTN9V8&Nn_l@@hn@S@$ev~5^@{*nsI8d8dmRkr0jD#@4R?$yb-R1H4BBK zJ!eXn>xGwP+=qd?BC-JsAmtzU&fz~dxwjN|FG^vCd?7FSzE1z<&2-S1v05HdYDtax z!Xt2LRVCkBtRdZ;m1tJ^Ih94fi0xmJ1oYATKud&kcJP6P&lqw={V|i2urrwt+}}5= zCmN`EUnrdQcgSeA9aR4Rqgb=G=rYWIMzM-_VJ2$MEftaB*RgT|J1qCiE^F<96H$7s zr5!!NY^2A;TT{1`@zQj3h3n;%L-A>uUDaNv>12C6)4H!LR9jCfSUH%4sI&YzYnprV z%6JF5VI`KfPzs#ZSqtr)z@>w@P@flO<~~ezEAr<3Z}MH_<^4EXC_MKzj``y39p|f5Ss#0PeEA~-h zTDp>5YG3MEW>lKCKOS_yNTuG(tw}B}P_)f|J1n5h<7h~2Ea!)=_J9<+XZAVh{^&eN1xHQFwu@1ci|sgjUBT%OoUqxb@wiEpR2NK0m)jjJ)R{O)zQZAX}g ztc3r;@yg*Ji+PeEkE)2Nr3pH=mb30)>gY`}HpwkGk7;S4xd%L^q&0O+qqwWqAo%}v zi4p2()?-b9H-o7^gN93Qd9rZodHl0yI=l`xRduCg$eIWo(c>Mbt!YO*ncv}E%l2Dm zPb*~RQvF^0zAh)3j{BM}_w23Fx$UCC2&|&LDh3iOi=8%Ww8SfY*T}1+!if9|;mU*7 zOnvC;u{kbKh;xAM-C1Yvb}4Qx&)NqO24<@fnPHV$ngX-htF9iFdu;{upqKovb2HT^ z@%vRr4%NqX4{2p6T|*6p;3)hRklRQpm~Hepj1;t*L>=>&4q!Lfx$TaY+0|k29Q~BK zU4$OW0bY{m&F60*mh}*868H(@ORImXiI#MpjpsHoNhQo;9j4*zUc`wHje3zhwHF|b;`@ek{4<&KpWG@$mU)pH5O z5I^EkpMlSUp+wdb>9*bL67|HM4Y)dOtrbDjU8IwV*v(XNBYc8)Rp|rKakg!AV>2=O zpy(_XF04+ctNZ@$I)C0S64Fx;nCN|O@J@mxK9Z7y8bu`9@5jcLlI!DRkWp3qLcpc|1ez^6=6yG8`0XCL6Y(|koXGbBkndfd zz%?7-s^^^iI?F*3U@Pa;MfbBgd3=Ip^Qg)lQo6NKTzzn#4MQ28-(4V9yzL%F03()R zn^Jo5sm$DWraF2Yv6D~}yEmcx3I-uy720v|*Y5Mp^IdwyruVS~z7siy&EJSgopLE5 zqPDNQdkPeAS@{|EG|-J)QCWdjJ7ZRQSW)0h0=J`dBcN-tcz z1#>|3qzwei7GqeYJraI8g|cETLXSlLyRDDd&@nhrhRN{tVoRYBRy?*c%-DCUFGl0 z?MfTd1D(4*5J4Yfs7i8_r1_px9eq2$hxZ0$QcYdqOUxVUGSL0~y<;4o`J;FfQi?S_ z(}xe&c~YuXv{ZQV-i1$6e2aZUjoh0Ga<3Y(*7bp_B6zxjiJ722blK?z8Rm9>g1}{+ zQj^929pe=r6RC3RuM)7ATsd=dQg?Y;Ksz)bhdz>;tEc}9xRQU%CF{6gzp?tDPqnI6 zT_CZfBF!m_P0s;Od8La<*DcaPW8?2ON8}Y1MUe5kU0&{tr(4UTJEU$c$yQ8ajakz4 zuS55?J>Bah(XHtJERt6w-(Sg87p%MH%RXvr3X46^_rEW1jK#fWGi%4CAb+?P3dpQ& zqrNgeZ;dVYz^xKM&k9HLzo2E2N5E|+g2Jv!I&uv$a%MSROsSr89&`kH4o%qcRES+~ zAoR})?)XDSrmF2<%w8?Oddu_~YHVDS+gZ>&VZ2n2gR?rnndk*IT{eEzBG;E=1arh< zr($(*FnWno(#JGu9J!_momx|hScPnEHbkIoICv|0>ozw-!xx>jW^1Ef_Tfe(U%>qp zvRR)w!$0r*2+HT{Vhny-WrAp2AoVrMr#h2-Rc}(2Ox=Q72t!#Pi$7W{9^sEdOBc7$ z9=A47_KtLY!~OK^)jL|#KyHKI5!Km2(TUo>YtyQH5e7MC}@2s-|-WwTOX4!M#A%M?d(3WJ!`@&{!~NrO8EAPaHRVU z?!TtS8UBcfo;RuT#=gZer^ybMs(ifV9>6`8qbyOsQ?=03RvL!SbN(hqM%JgfV5))P zTVNgU(8^F!7N4{%@=DrOBYLpE4BeHrpZF0#?Z|8M+vA^KHT&)G>c>GoGL8`bnJ9K~ zQ}&!4Jv-slcz@np#+_!%hGg>0(Z$8#!DikD`{9|W8iFA_;<-j=1oY@4K}B(L+z29Z zYV6|Nc-r2-^w4>5HUM9o;qP2pxbVA~^b=BkrraFjdJS+q05tXqj&B|AP}>+7P99$d zi0FN-G5uCsoIbI@T~}99<5tvGBLmAJWq8fX#m9ctY_Ah=dMTWH#Q{Yg6_D%P-g*Cx zfDydl(dr;mK)z*iyk6PT<-&>=aFu5xoe5`KIqUWI#{R~;V&*d(RZFYF<9plVZ=c+P zER1qF(&44}xqVBcX+$TsQMNB~n$4-9u9x`gNo;8l6IXO;I9f?N25hji)`~*ZCz;+E z78(+5yk|AvhZ7#qb1M(nXul=Wv0Gn)0FFhnynMcoau_IgN`+-DLvowf$06*30)5$R z+LOOPyGgnL#8q+%8Wz!{I~;mY!JmX!qSI|85`Tz0@)vKo9~^VrQ2}ty%W7HP-*<9s zk7h^FUUWR6-cX930){m_cm-Ku7-%zG0e#H?JF4Q;*Zp=&B89|uW_j(N>1i~+r_naGQV;b@kG6CX z_}79=8ejkEkg}pAIjDf_rqu$kcZ7OY2Fv^9eUFP!I5oo&rwr#;FKkpXUI0yO_Py+{ zkI#}xiFjFt7{uo$4N#>Thf7JB7Gy`ZwoE0nMs)Q8#@R!|)AY_Jk4x1T%ULh3OD<5u zLd~N$tl$#Ra+1ehvCSlF{M7jTw92Mu0M)ygrVQ(Hox}oOOJ$kXQ?N5w!Zqi&uxO6k zePkkBqdVqc>4E8~EF7V4U^@^~ah|4>YT%|x;5E{?$(=hIYABkUQ5HL(FP%kkXz}eZ z%XsQd3&eCKI#`d(J~YWlQG%~_g!y{K4-supKRt6rvLaLB7_grajXJx?{rhI1Q6UiZ z>i_P{6TmNUl~trmfspoEWM(CFKaG8QC`7WxX*vItr{3Rl$l-0yL_{RDmBSmN7y91q zKH?o3+*wS^{1Qp2Ptve?xJ%e0xj#qypT$Br6#VZPcsj4ODUnyI4Ima;6>+DW;@a|t z#wd6A8D!_~O82fvNTTk&_u8AREKlYOXBk&7UWVt?9m0SI1lZS>P_14-Kjsv zGOke6_NlHMQGyJK&3WXNfd12s+eV*spl;0NZK$1FmXyt;ibei#I;Hs?Ym5+njmWehPQX5-{gHebVej}YbA5#c7d(`+R{l>!~mQ-Y$5|7X+7j;3J_Pf^i8x2 z!;ZDfX*E)!pPLgQOBay!LUHl{I-Gpg26Rq z1Ln1Se+_Ceoq{@bZX1ByjBIRvp-=FS+cU$sk#ySTs8`jnkt-v^ys}xvX*taPAA~ogR3l+KKv$mx;ObR(D zb*=|hunG|g@m}-sKXtd@R1N8mcWoGr|E&dFs~!;}ef!@Cgbt`*|0JKroh*LlfibNe zloDjMkpFGGJ=gW--|P653FC9}E;bIft8eJIjs4)HePGIKeuo20ih&^9 zn1o0(qzB<6gFpn{`t|swk@8kL25IzWixbOwLQ)65=P9`LE^4J2mo$xaTU_$bOdRnU zG&PGrQVr}?s&4lW7X}X6TGQBuYGP9WX&!S!de>d4fSzfkZuBJ$*ZPpk+PJ!ccIYt~ z31wy&jh@p3sDS2z#=PDTi$c|`fWtC;Cn)_LHmFk?!19lvzp5@uvC(BQxtVsq-KcZK( zZo7IR3f+GMWnd3C%7fooOZ>D_b6*h~p6=s-0{k_tP{&UWtd~5eEn7?H`9;_2!@ZDe zAs;0aT1hg_IW6kNH3NVyf4*_3UJB$YZzc$cH;#4c*ua(|6uEI6&9{Ctn`q}}@FGs0 zW)W`Fy#05t|B#~D=?K#cO0}l<8D0%X=3kPnnbJdO_8-7BRUQ#9u2M+1skaY_4i8z$ z%NXW=IyE%c4&2KHv8BTGVYfVle_U(p336y^Y2D&en{bzxK9`Y=Rfbbi43{hSt()g= z1Z|b(*WX=UpsEy=$7j;-)*h@7$(D|e4%Odr8Xbd%Lj~a!G6~D-zn1*d?Pv4N zjB3ZmvhRM{v5FSn7rnIKmJLvEuTPaJ0$d|Iu!mL1+RFV#Sq1}^PX9Fh?)MU^R>b!d zCrrG7#%m1*@ah+{0%l*UQ8VfKk8Y?l=WBdpUf1?8 zfu3L+ZM@2CTOvAKzWTAWsK=~w>3?0OeR)fgTnH9yrhzTFit`jSMcX$S z*G&o3*&x#G%3=Cuj*b+ts`~IOvNq=T?uj9-B+9u}u*B+to@Fz;%i(R+81zhmz3lnh zM&9YfNQ7zaw($(V3Bh0hN}HH?lQ$I28H~e`OhK5EnrJ(P90j$Vk3%Z{7BeHTeY`tz zIaJ%&_*^y;TO|$z^~Rhn21G9Io7|F&(3}WM+1Du)IqcBA)+WWPLHq`}Q@iY? zx-M?%)bTcn<1^>5`6I~>*blD)=Op_i~pmyp%~S4a>iYr{o8htn?Fm*x|=Ajcmb` z#JH5QzKY_q(sE5(TBqeB=r{3p@gwN6c>6ahD0g@Xb-({8t57(R6N8gF^6If8WnO?Zi*w0lj}>vtTd+? zo@)SO$jJ+BY3Px+;@f7iA&ye57g%1-S$!veR!_c?#sRU?1SUL*{+V&y`p)1Xpe=oN z8YhBZXr0Kf*b3X|=={JSJN(?MzV=t_)J99m(f#_F{kbO3RUuKL`}VKAD3$ha-|kv3 z_kKRU5)eRs}2CN48uG<5+t}K@7ca8g$U9q-cC2D)1BCv`6OdQ=H)^ zk6~4c><@8ZiF8!|2jzY5=8U#nSQTH?9U?@pHh5{wr!E~$E?bVUbMaA z&S%sWX_U^^R!}&$w7#BWokD_vS}g~pxuBbwcP&Q`dDW@WZp+(HV(UcOLE_)~hM*>< z8J??EX7-Q1{t#9nb2YMt#p|6sI1xUv;Hzl04~!T%g)arBXqx4!6j+edfzI}H|EH`n z@+5SL;Y1YXm`x&6&mG;hH z`qNEc%l7k$hq;wM+A>!!FMc`cHQSie;LuKbW<0T{uLHHyO92hVm7txzvfd|u^Moj4_fy|w6Tu+pM?OT*jn~P1s;Ev(yTsTl*HyC zT;gbP^*u@P>&1CQ(lRh5iHOpLAIndkyisYjH-`Mdhe+bU{Gh=mF;bR}ZewEPdd(-8& zFRNZ!uC;}Jrcug>Ft%rDt-?81JhIW0xadv~yt&R+t9NTcBB{_IPxEyNuj3s83Z+ce!! zG`hRP0(l)r;)4R@nCM0IhB|J?DXCoqWjNA_Cxvcmo4v0~PzNa$@ZSj9j|nFsT44pH9lAOrd78qBn330r-*f9D2m<>3PdtwqE zF^yO-&R(UvveMe(W@#Iiw}>$P^&gR)I?Gix$9)yR%g-67y8D!^x+8e5hs#B*te?Gb z!icsIIH|r&DNkM2)}Oxr_}J%ZAuc%|*?z%QxE;qPaU;p{y1k~lf)_9&1HRWzb} zVnPbOUXn#|!SeJz;NV8AZ}6Z$bJ>QO=juQxw%Fy`mgO-WyBteXmoS%>#kIRgN3Q34TnU=C;RuyxLWPzkS4UB&Q?f`*hiV?|!gF zS354$mq&`h@(=fx`R!HwtdCO`badF|6;< zbDCd6KPJ>y{6k|Rw@v+FA6UH6Qn=#h(G(qE@yW9Nh$c*}X-gArHo@p=XeA4_9BB{Q z{QGAhYGAt_ah4GK17NsJMRi_(D&ezT8N1_DGKD8yeJ# z4=%F8g=3v9ZBf|l2v$x32_yGM}b2M+Ifr}-k8O{sS$6jh3Nz3j!OEAFZeV(C!# z*CGiC+QE@^bpw|Y4tgHbTYv8pNsYxf%>w2(GFf`^Fq*$%Hyt(=1|y#dXr<#4Q&l(A z;|R?4(^9jMJ6In~0`Lg*otRl0O{5ks&l#5zo!#tb`FTUziiA%e;Q0I$qUm|Bps#mW zTaeizJ0T06rRQV8by-eA-ZygD8ryJUF+{^P3{m1ezgoX2qAWd8k3VxH6cdw>q+T4~ zd?zg}B}-S@V%oj_3Op{ZD48gJITuD^FdaHvA_4n?cJ&7e*=Z7<%iV*{`r;545$>^D zmjdvFXDP~k@bu#1`_jF}1DadX8Q&`l?3ReSO6+#*go9Q5krl)ZjPVIiedrtk)a6SL z6PVWlMaS=|!Lumjc)mDghNBwgM19$jrqNwN!!oQzOenum175yQo3t<7e@~()BJU-j zGRipJ*CLriQ!t|EJ~Q?pcH@q=$M6T+TY+h#hvEZ(FX2>zT+mF3-~8Sv`V;tueiMd#?|crDfo4>s4Nnj^|Lq`?O=%zP3&r%}i)G@~s^i zB&K5&f!ayLnHBGN8mgE z9H25@+OCAlj+VH9BY{SXFxiu`qN-*+Q+#_F_;@oRo!hYCwOtHq{e(3$l&BZ44@m#e zP*Yr!I%V9dtm|`Njn76VeCC&mv>b;!*u~e|{CV>B26Sbs<2nwWET33G94y)f<;;-h z%c>!i4^L!^*YZV7*#pN&>BVI8H=OG!U>G^BR=Cb=@-bA2p; z*~xudZUbsmo)Xu7T#Ma((-qa5EwQ~3etDTYbX>Hjd!p2#Y@y{)qb^yWL+F@7Qf&Xm zG8Gq}&q^HdQc<-g+#->{YK-=$EM^MlVH7T9-Ut%K^y^J1qfQUE9vOi} zU%kO(IWg6oOp-zy2j*2~Cy>kON7FI$-o_s7wV1KUJ#3|w9$4FZ3&5I?umF{v*~_SU z3l1xHbnrhCP}HD3X%`Pdx2&@tV1nK36~QCR3Et&&hm$Y6dYv>*_4VNtWJsN1k)#?j zK<1EC7u?|}Nuiy_dwej|^2$-uxkvrR)!N<*v3RvDt(`dE;(%SCKNXS+o8NCIRd$?1 zO`$tLrr%IsfBdtNOmz%vu$TFV^1$g#DEnT*V6uc9M?ius#`zE>Jykqe?tQGM!RPMc z0?Iz}dH$*h4};)72Kn}V_Z^{YJDh6 zN<3~k^?eLN;~zLz$~8t0o=)~8)@-Ei{c=cW!fhN+6PU0*TNPbL%F~!ZLh?$6@|1Up zK5f-f9o8Qc(20pkJ0e_~H0ckkESR)TDY>_FL}SwS*6FZJa;%x*&|6A=zQMV*!97{0 zC{1X}IYm476PaX*?LiX^raI0B)p_0OS_w6%YC2Lr*jo7xhgZQwxlLz|1|NauS-Jb6 zkwZEDoqX6X44ap=!m;ivDtoX@+!-6fZp?IEooQJx_GTh$5**O6_CFos$d~P(&~;I_ z&;OXs*{ZIgJ~$B*BXEtFlmu$M16a8TlHmId7vW5AOHxH(Q>bPx8);7^$B4w+XrAm* z&h_JzDr{N8H{!tH%5Z@!l`TYxPCxZ`HBr~A%^vcLlescj{-s!lJyyAa z#6ci6k8mRrh1!no9y+4Ai?LTfdh5CM5jP7Qd&ZPN^< znP0{=7gtz=)V+RSV4;%X$bmZAPb<2GBS-ySnN5Xb-a~_&sVR?qSpGe-x*n&)h}?#^ zXODT&w|Rh1=3oi$yo?iK8atAxS;p!V!2^m$DLP+kFiwwTFC?+7fRz(WHn*{+hWN}= z-&gNis`jQ@4PkBgiPJ2VAZP#iM%k`Da88Y9yoOs}I#9dYH*YzPxyj}C7(oYOTff$& zEc=F=@$lHmV+?WF62x1(H95mVl#W*HG;(wMH12wE)X5aDdP0!olNpeKCr{0Uq-|)P zuGMu5;NS=#H5pWReFNZ|+SLaN@;0o>T=+rEA%Q`O@!QJfI8;cM2Nh2m<@7h`J1O%! z&-}T@v(rOSe7D9h^4&>kX8i7xza+D`HtGj&ai{sNMU~_HzBV6?J@fzExEJtc+^g0& z7D@Opll1AU>8>fneCtP%vw!F8UQt9ma(!}OeorPT+P&Kzwu7u;80&WlRa-huavKbV zahhDs{j_qrKXcqa)}$IyyM2;#G_N5|REh@@(MVE&nqRLna+M~0P4HclIhAQ`V%smg z%sq7)Vr-dQ6rp-_YIatHJm$vUw;5TJ8G0s$Ngvln0>*CSzyA6)QS)i}k=4h}NUOMc zMXfR~Dw^ixwipyRfi)OaY2#NtVHg{5vbLGtNP>)~w8R1dfHC=y!*uE)ub)<0ige%X z`gDf#+mze;!`foQ@b-`B%u3)@zhl?0$(F`nytpzT+&W}qUlaAhnXXVn+eLDbm~0IRP|;x&;%jizRxgEXKQ^oTUE3A&dSKSdi>MAeR=N8>veTp zQB~VBt?5=-C?-v}m}j-)iq~BO&gg{OmjX+3hsG;w8v&+!vvS!Y<^XtQtnj0FQ7}J(fT(|cP?flH3=Q!);^cL?&SA3ch z6>OugCn}0MJV5D~w`#BcCPkNSDleAINF&x~QZnO&m=Km*?zB2oIP51idbK zUjOwQDkndC#2WkMyd9mVX=^Cch%&(Mdf;HE+}$!X(NJRBo7UE9)Gx^K?v>kdanb(D z6j<2)XrY?v>c6R7WzL26UeA|Gu(e9UbxvY?K zXWec{IVhj2I_<7NI?Wy)Pc80w7X3hRtDC?0A_((!lxw}5(^0&mo>*ySb?X_?XCtJSkb&2&l=(?#G)E?S+-)bgy8M?W7Xl!UNMr#x` z665x(!Arj~f9kC&?{FcUP* z+xc8>VbPU2nrRCWru5|u`ACneoBfpr8%>SUT;8QN_2-_G4NMvIW+-?7d10j#SEfih zkjtHaT_V(|IQ2PKzhWg0Q+o7P5$iX9`?LNw!L!c;Te z$vRCd;ION_oKX<$n%7$od9pY0Dynge`*c!^q9IYIL_H zZQ}Oj?|CDfG0v;pqo#T5O>F@7G)(14`Z}D8%xj&eB=Tb_ZQrfwvPjd7K1uH9E?F3k z_p`$XP8KkGtsz)Ca^_^M4@}uQw)gogOi&&TDBG{H9*eA1Xbg=oSKN+~_v&9x{-Bl5 zrL%zJzAby?<87t4^nxeY_iODNXFZdWwFB8AeT*mmXR?dG_3>30`a!zK!gJ5AgyWr+ zd#j8_*{5hfOa1vi z8UkdWygpG&60&QYrw%;A7DwO$NA|a@5+40q_V#rs6ug1Iw|jPNo9s zT=afalZ_7Zc-Qh=Ytc*ir^U2ml%DQXxp^&E=aWuhOMCI! zWPa5Fcc$MB9nmu~I$Av&9RfXHhI2TNZYlk&F}Ss+_26?@J=RgfC0&u$P5racn6!bd z|EKNbpSusWCq;{8wj)nIPVJl?;kzD4-O4IkLYIy)`fkDJ9ps*kzbm_6TzeUG*kGWd zr=SFtC9&&h90r%|?h}u8O^7RPVBPLyO1c^D!8x{~!|cq539b}jUh5WY6AoKORi2X^ z&3hFT!nv&{t8Z3B<>h@oAjTuK)CP83f)ID8T zy8n+vzP`mAWrFs{VMK9?_WcXTWyE6a(z?=YQQ%6~=q7pod)xNZhaiakM2Gpvb=PWp ztGF`gv|qLsHU-d|Ice%V0sc;YP(ntG+=|{_L_!sZ)HV>6LFcYGBM_f>FhI&41x#B% zB^u|EeqL|feMe{3g8po;u=xROg0<~0Cp?z=)*)8GS^v;{PI`SBS{%Ceg3q_Zuwul! zo4r4JA7H*ue$RQ12UFDe2;7}2yoAT{A_wq8-m3&j^wQ(QUYj+}fVs|kLi5YSySan- zzp;Q?8aO&V*?XsA4T)G@c)T~MA$z)*smHc|FM-rOqHaG;RMu||Zku@xga}g} ztqDnouFWteQ`KF?!U&ByRG7ro#V3o+4`YLQ0|B%d_Gove6>cErJKo13OYAn%Qxt=s zMupw=HDXF{l9}R_4!6kkc`Aqu0^Y38V&E&jGZy+4YR`K<#6#1k)NZVV?e6 zxAtzyB4f9+(9_v?v>>z>HAZpTyXp06#ba4;oo!s}lTjWYtJMnQU51$}#*?{*`k<*Irxv!;6Mtgp_4kHgr-#fslc}twJIMk4v;8i9`fBNUiu5o_kP_KIQ_N=5ml2n)pfQ~ zU;iGDQrQ2>7Sz}vf%Gw$Gh1zSZh@%e}2LBUf4U$CxUopRx!(!w1E!tB8C zk@Q}h1yQX#(7n?lY+UMnSvegADp^S`Ve)YF=Ssi8HJmQn&Xtc>oMb6CybEOg71I@8 z@!Jipi!z+RT*j9E#9n@%+3u17d1~JMMTd17;kB{AcbZrJK>+b@W-&gv#rVK{fZVd$ zJw317-!Ijkn=CS&k?V4a9S8QqMQn63Q>MmnsyWY!dTpop0LR-x#Ri@BrfWjjs&NkX zNyY1AMwMTDxGK((ymosFG*KXl-`v}ZOr^IP7{TE%^NS3CEdL#^ub!nkn33=rmVK`P zRtx)I-9X?9cGlqg7fG&vXQSpL?$NE&39rezX2DTijiW%M?9ZUNJN8Mwa$;3C3%f!i zG2HA`wMYv5f#3bCyujt}Q{AFm0#Yo3VOXgq(A+h`j*3pmLgyC# zxdp)ElJN6WgMHrE&Bhl>Ub_DdwKh;%A9{F$QS#H5JX`yJAt7h?^G|95$>G4&Jm_DM zl`=y1xMhJLa=XQJXcHD*3&zwAUUaVG!L!&?IFl+ zY}P_oi{q5UyLpm%rI0*KI~zj#O2on1imlY6#KW`1+qI;!^0B_W$8lcDPvv8~+bHKZ z(*YlEKQ=xrdc>QX>p-ddv#qOMv_;q5dL?FtH32m9RcNXXWXV47) zAqBe-E5FnuR!NTTKcrY(yU@;S!pyH(F_bV*-w8Y~);I&Mw&zhoHTzse8>FuL6y)t! z@ND*e)E0k&&-XVGu&cMe zJTVmJLfVaTaFbSkCjp<5OnoO|(p!E})6w(UhK)(g-*IZX!Qv9q@bRi49oJo9X?t%w zZc0d``G*Q&an9DT;Qv)ECNZl48Xb|CeW&92vegAW6}BHoI|Qnzgd=yVIN!aKL5d6~ z)0y|+x};funnZkw4^Gk!lK_py7mhmy%T5m(f)Otv*JW%4^NYw8|x8D&3szJJ3O zj-vN5qehVKK^&r21Yz$WVXnsm5|;-#N2h81BRyyHGm|7aXaE zhX;k+Sxk{sG4m7>9Y73p7V3)J_}Jwof`jAQxUvySBWT}CRogB5w#lC%kjNdi$Rz8~ z)S$vz)iq{8#>66dZSe3{JEy@~p^~^(ddEUlQPy{HItz2Q4uc5iTs`53+#muvqd!Rk z6qkx?YBF80ap0@0iK@jfRWh%941T)bZA0)dL;@MasJpfps+0_OkwJNbDby$ zWb;Opd!?|wlap3fs+0kIXli)i^N^jC^o#!rdjQ9iK%)epc(ZZU($OFoW;0L)?LK#2 zKU2X-&vWKET4AqpoW&|20w)RrRYxxFwEo!X~ z^^i%@p0EK<0rr7r=Mge9JNvtoY7Eu$tArWH`TO{7--3Qfm_Qsjck8~XsKF?jv`YMQ zoX}EW&yc9C(YwzAZvSI&AAuveJwS)OeNWvX=roSpd;LAv!8y03ZRrL0o^7F?CB2NM zB|9uTeMCPRvgvA=Z5WZZko2=5b6H`TyElGHxdS0G602+F*3?@8fTrUjpWA=&FN?XP zaOqGi;=98aIA(^+>Cl9Oi7zoVvofK;LXaH92I(|^Sx~@ga)?9N{XccpM15A$`m)8Y zc>lbBVm4y3pZTkC8w_C6o$Rjfr_L0G6~L z?Bv<{z4`ky0K~D=l(&G==jCNoNVFg`SL5L#vc+)e6-&0QGTtnO2$^lA3>#@}Q&^jxch)Q{7$S1XH*@wojdV^L>WlmMh|lNYxPtQ9 zDUsit*;%d|KI#98&qc7YkB*R65eL71{I38Y$@ir{0(l+C6A)iyb3E`3(LCgn+%tuC zD!=KqUZp*C;7Uw-NprvM&@3M%(N5=JXxpIq2)|xw+U6lk zjlVfkyfYB@MOQivQYFcMJChNE+M(eFpGx}f+}?Lh;{_AUdFUmK*P9%l~V zzeC~HndIH?Om<*@jkGy^07zZ;gZmx#VcASeMW~EAivr2S5Ufg6Rtz#VIXxun*|%{TSwB?5A~idev9gt4!agiPMnNeWBJBqh6U zWSE?sBs3xS#IRMWHq~eOe@hi1dRG$&KlNSO>K1g8a{Xni7jQYVacHn!(|n`{0TM=5 zOvOpGs4BA?w5CXG%4Zl%jEr+Bgcg>|b>}9;frg79gm-ZRMHcWO;wGXgz9`Ws8PWck z0oJZy?jT%K{3>bXE+sj^JG-mS*~{e4s!5HcoJG#c-Fbvemu^1l+qd4|)?e!SQg?bD zGTE5={!gdR-mszpvY_V{7HBE&r9~$?jf{*8cpmMy8kEDYMld=^Ca0m(hOE_Wu5~Hf zTf`icmc$ysf48UL=0)RYLRu1cy){|a5VOx!ou0xVWKEzJQTRIYZus&Yo!TaQu5=iAr_0v$kOPSI)5@p`M>3=M6F90X5HE>@%6As ziMVrLOR;&44`xTVB3$?^^O2HTg?ox*XFn<`QF5WMO-0H+&eb`h`Z;(nM{N0txss7i zqcVPJ25;G)fyjG_pNw-rxPN4{PxCcRSxl_`^g1**Du?8l3Gg`{{_*l$nzqHDlGROc z_WGvJ0}&XB?ATc2U#g`sVv;Z3Da)Fr!;9eGMki3sO#W(4AebUad)K5U?$-BZ`p+du z^q}j@XOReHyH{d%VnV|URw<91d!``2(}19GzjMRZQ=T&eLlx2?G$;=uPYbZv;r0{R z?edD{5+c^y73JqTZeDTamv~dC2C{-MP@DqJCe)PdJPEN+f4e@Ry6-cwdbdun*y!H; zkPD@9Kjbs(X^AivftDW`@l#7JcSQ7mAAUR}Ohtb%)uftz(wKl8UL`A*97Z-j7SZJ( zj91UwJ}qs-4b~5Gd+kD0f>MES3k@3&E*O)9NRlXw>u8)PMc*%fo>~2SdUbKpc5`8} zjb$Na^xTaJa$HiyF{%6K!pQ7Pam4m>u+D-)^(m}|Xz`|wsc&v|dLf2cI-3V(KKppS zX8A{I@tX?BC5{_uj|MA-(v@ZB*QwT&Dep({tE8X*BQD>j65!J!yL0cUg`>Z~HfT?eiC_R6wQ>6Vl zA4l;+nMy{G_qnxoCJ%X7L#DVe@s1xxU32i)?0u%>_n8H&hgO-ymY(j%NoOzPh4Uoa+3335l8h=A$7093LEj2_mDM|)$lOBhhwa+o zhPS2di?}&oNF^nvWIoK-Bzt^2=WA6J0-wp2G(_x6BHq;Fm`t*0xK7}FatiF~Xl?>6TwXs&l(j+G zkvOcMoydHtO)f5$1AVKD8Cb@o#2&VfOQkv%78oYe;oI_QCAUNc0%aF>$?l8XC=y6+ zLp2Q9fqt~sS_z19zH;3chc<_^4k2mL5eBvM9EjqirCxcNoPa~OBI~B#E*RSSWuEL(1r9eupf>P$Kg|$hSo;n z#(FZ52ANU3pwH|~<9n$Z9hCW98owC`8_E^)d0_vN>L+`{1H4P{5Z#WF+(`!yyS83A zO7m@Udza=O;yDtU8$6Kr9JmY)-H(jt&P}>&Y8!s7zShH4@pEXhscb|sA2nNB5myK~ zUH@iegmLHXgiu0=Y=tTQ6NE^V)e7zHJM4uHNL0ecEpmmPPydR>n_q+EKqd5rZpq;a zx9&-~^EUC39=58VHlIH=>SgaOKyYc^>wK{B;R?lS!LFs#;_YATd0Z)ot|s^Q0t2m(h#Tpai;VYXr+hMUiw=_# zFh4BsD$xPVb7imh|dgn4nPI0w%zm_Vno+7-vE~9Op%R zhdRs>kvE^7OH(J^g%8;xl51Lx6}ieDVJka(t0pO{g@=iitT|r`nc)LZow*0% z2(6Sxiiw4-Z&AGG9`l~3F_42@`(xZL10C7`^7_|E2L)Z&+|sxwq6-?fw4Q&i@a}4g z`0bq6%Zp-E3zTs>13Mj4FX+d|&&!>!R)qJ*`YALjj9D&H#|I%J+j41nsFg}ppNq_s;^U|mgjreG@DZXv*bfTjC69T?cS7EW`K|LGe3JR$ z=+tKtf)_KTvjHd6HO-cLZ@4Gu#-lnXbToDE%-}gGEn*8*Tz?vOZ_Z?BTL}wuIYlvV z>E2EZ53fweVnga_$UmJUVXq+06+)}~^+YzS(B+G&I~2RFH`Uq@o0}E2q@wVlB@W?) zwec9KW0b*3=H6a`fq`k^s#Q{hpyLm%az0oxGSZ~fJ>=Jms9sv>(&Acax*-Yx*J}q& zcd@cEtj@_ETouI{a;qPeZM?Iqj#GL0DY>C6u`uPrPbnvP3q|4k+zjVZ9$fJIhpNGU zsG4h?E?EEYb+*US3+8hLx7ggmr2=1BPHpLrp@SQ$f|rW&gdCER29k8SB_*Mms@kG- z&98Iw&S@6nfnBs(c}b~vjMCTzr3ZqG@4gtIp&jB^-#~WMI)5tLbH|P@^EG@~1u-3R zx?M{v=N~D_E@kG+{#yG4E%$q1EsRRaogh(EdnGB|i?CnwnY|IhZ)kP4Z-lHh)V6 z*QSDNISxu$-60Pv_z9e|BO~(_U30P6nk}9hRNceMhgjp$hx9e)Va)y}1n5C{E=ZR; z;d4S9ce~EBHBRrH-`95`1mOz<`l)gG4|xlU?46^le45oFudTTxAlo*NeukCdgYni5 zdU|?+;B>R@2RhmI1uwXl!``+p8duwc3M`s6ZtDf2^8>%;%mwicQQ~DE+4FL8>2_{< zBh8S85RkmQfiJ`WX*eQY<>$8##b8?1ib7~o11FTa9<}fm>*Dqs-izDDL6#FC3?3q) zbBhb@RwB32ip?8dJlyYU^D6_#N;)9(z;|HI$6-7!@C>ra+x;ABnq(Up))e>(TPaNr zJ9;*8G^jlNi_7~H)JNeW8n>77Ew`q-mN2WcEE(ZE4}l1iKUDi zeuO=RNZ?2lJB`b~*mT)}E-zCRZ`)g=ivl`g-1sRV@=?7jzHsPvVG^e4qoIkGqFy5U zyteLb(q-1(EWTRg$T`o+x6xsnhHG)lb-2r76`vsmN52VA%NH6A*E#CNU3;UMyq9*l;;9Oulu7L!L(-aAbq0h z;1HQd?i4aNdULQsw(;wvwnt+TA9gC+s5N3f&^TK*dZPW>w6OOeQ~be;=O#po6>N); z)G2u@ohbVX0H<7}@b27nq5WQmbU|{>{z`&0_dBSDQMMc{0tM+D;WlG(h^{HjL9GRh zP-uVyc1|l^jvYeULv39g3t^x|4o^>hm0=fOiq+!|XnW?{8iJDo0(*|6N!B6f`$$XQ z@iM4W_&vuRGD*Y2xu@DW`${7)eN)a^QWb zD?`+CwDPYE!zfOUunMQ}tZ{+N0}$FlcH#ln!LFtz^BeE{;;@WLz6jP!&8_d_uyWga z6--mL`3PSTxgsm0Zy}Y5Fq=)dWUZ$OF<)^2TPHERDMG`|#yA`hMJAAmfj2hht8>l* z6JBF$qQW}D!eq0CFHt|X;-$qj^s`Ers-v(I+5EDaiO}uZ%KYMLnBCi)*7wnHd_!k) z?3myamMpukheO7Go?Au^9-UFd94=g?mzu#W!f4TVH@+CJ20E;B8k8Yb8Lp&HO;+CEi;*u4TxwGH-V`cQ@~7rANyC z`jS2&q+;>&0}jf9TlP{G6OpGw`lFaCU%%a529eN?PWdFgV!J|~hq+gmNbr;3X+{JJ1ZeC$mC|cXP1RV>2MiiFTit@o^-?enToMoK*Fkz@? zSJi~)<%MfI?Fe_NZmqEcfwl+@@=Ivq>BzxF`If$?jJEAuLP3rrh?3$Gxk_#OYGTwyitc((>LXHnNYZ&AdaS42sil%G>{%B(S~Wqc*BP(3hUUQV|8=-6XMjn zb&OBnU%-tm^nGS&O*Q$>>)O_beq|~^*J}*+be(D%Ky`{xI*l0#S!CDQgHBhN%qZ^G zrXPGnM!pQZKoa526FJBRBC1^9<(;gXEZhC6=wP7+Dlg0&Z~gK1SIBe5yK9=R8PE0< zA6JXDu<46@E&0$eG1XoAu-J59kwJPuLY!CP$teOZZ=x3yAY}m7e=$;Z$;Z$zh@Ph8 zfy_U;`9fnOC*VcA5VHx^BvDyXe4xtu`u(|5)YgYot}ykuINd>nP>z~SNeoUB?Mr&e z*H_{`6t}&LL0LW*PwPKL>xvaK;C@5G{ zz5*PWo1UzU?4Bd9hlh;D*OnGn5ke3#c!@%6ExNX1#DPrkbXTg{P2zs*xHDY_{&>T3Dk=YZ7kYS{ zlEU+HxphN`FYKTZ0CFuu3;(o(JbAPGWR@Bt-Rs|fi7j;u?_8g4qfLg zhwmnDp}cN;n5!*qr+rjOOJzK${@LjBist_w$BbVcAHL5%n=D>g{p4yi&QL{$=^q&6E$4 zK+5z#u>k+FohXW3oKe5XGjkP9Yv0+g=O3%TiRCjjG(qlKx|P$Mr#mGpg3g~8yu&Jh zPENHgt{)iJw+UeJdYbQS81qZ~*|6$mj7xA-o}zJ$_nvry*n34kTox5lZEyf2YWsfn zdp9}~w)ap-SaO)~i7z$v<~(95%wZr^Lq7D)Ju`~!Rf7xd`yEj}XToX0V-GUP4EQQI zF(sjBl;36TWKjzT3B|+VccH8o_w+8kn7Ms>#tm;N?_r^7?e@tixBqn*iGk$%j->T3 z326o|4ae%Z2?g}E%oZFWJ`(&VvtvcE#9Qd;P>G-O2t%;LBT=2c@rr->G*N&*3ILax( z#mpn!Us8(F&2tD1{dgCkC(V^myD~(Onykxg@?fNiFQ=qYzgU=>kp;9R$6)lU6P5Hr z9cIK-=28m*Hg@&p7=?qZM{ubL$Ipq5Ls!;KT1;KXEQi2kq8}hXjlmJ0 z;+mDVOLef+sHT{K^`TXXf}RiO%Jw|Vs979H4y}Eg*e5SPdu zyr^{OP$HaUTK%4tPAl`h)OW+t zwK$Idt9EbHEy(S9b`hgGvDV4?VUg_;)S&eWnD^dD-HzBI56c}v_Xq>bOq2&@ zD~Hyds0DWJh~1}&byl-bHNDC5ft*ug+!N^6o5A-U%?irKob5LzM==L}%I(TRg~Gl{ zA)667IG^VVP{}K1M>A*OM1EYA6tIV2Tuc!>#-_uZ27G@1E(K4OD4)rM=ZKitC}h&b z`DGHTSyCLiiIY1zDAIOWy+TM-Xg38smgT%^aa>jsSIHNd5dNxfT;Fs8Db4|=K%=)o z<*y8LqEu)j!%UK^Z>#tHzNAuYL_dCOQOmmK5FfvNqVRzJ>4gb^!;7HLJ+;X#Ubbb_ zLOt5H9m;o;q@0|;WYRne9k*!$Qphz;l+@jWGpcHsCG)-(SjZZMMX@h-?}RSzO*6Vu z+%+T{7U?*f>WTP%rx1HUx?k#Y)hlid zVM}+VI8UP5Sh^nR;1jb@7SS=YFgaYz)m^09++H0bc&jZXYI5XDfqnJ5;9_sJwJ$}% zJJ6nR7-kcrFl|v&fhx+)fATppXuN38fmUtsfHHMYdM2Ld${H6u8^jrB!w+M-T<0Rq z*xy&NNxXMC%Fg7~W_)U*(}gXA5U8s96%TsXL4GGl-;aS+S{PCg?J_r>D5GBY$^%xv zt548>s#qMBV;FLk7r7u3HI+Y#vZHS;8VBf2Ox zk#@e0$%}zo)4L??gwu> z=yHRCn63hSIv19vU|n=KTx626S4~+D%mNlQ+-1R}t@n3P=m)U%UK424iZflHm`_0# z)H*a=vC78V>SAaldOL1|NQ%2~DB=##;m@E^XBh4ph!mc2^yg z{i1y`NRLm+bW%kY$I;~r#Ji5#OT$yfK#>$I;~p6h#Dd%$lyBDJmRJ!*tELmySuZ!q z@H;Qhb?|A8wJ`_}uO4o;!@$b5=V3=uj&$5@TVxK`!{l0X_&#i}OS`g^dbpS0xkMlZ zd2H5-+QXauMjZ0XwaMD9tEhrl11pQHng*)GON%m?P`kTBn5idDhq{=xS(hz~;~*9Y znFP7euHXllM|euwujuOhTIcwgeIvfb{r5zlQ2c5~Sdq;~wO?}=!DByG3u{%3bN#1E zQNqHM1+c^NxqW^If(p`%=A{{?m$bqi;xIHTLfQa8- z)&;yC1C_my)m0){*G|et!09jb)BJW4$+;X4iSkS2_+sTkMu8qp81TqW)^+|d3E5tf)PtTsbEd*#yP-T=S$q(;vo zQPI~(ETSSCoU;PyI#MMyKIlczMsegAl}(iI?Uf_RnRZFO=a!-3KaNk4P4cC1YDtoD zO@BAPJym7Iv?)_T_|^6K?MxC?5a&?Qo2PE&HhigeUtpyhGJ9p!^FVr>y1MAYmvfY| zKpDPRg1XMCsuTaUwUI-0?3PS=?bwaA{zCAeMT```^~TJoM@GtJar%@7NE2e^$Q&Z+ zQ6qWWkIWQ^`rXoUuiOGoD!yA8F!rl4Vd7QgdPr(wW3W}W=*KMzLVcf`f z1W_Fj>$?4mLQ=z;4i?hW^uTtrBU@$G+}P<60R4iR$tsj!&shXVHU!XPkG`}N7qhF) z9Cia3^e<}^qobSDDJ$Z$bj^}=ZZ)%GFX-3xWuPMGfw=n&9)J7SIuw!HP*R$!uW6{e zMwmadm_3p?BiY?YPI-w}Zx5p@r=*9+nRj1>ryY$kzlyK5`B`Rlwf$WWBHteWIr69a z+T4Qun2LI1lcd*LYN?031VgvV47m%d==e29>={D zIhX^uMcAhm@rw-;qT}7QL!ljB;`VzJBFrR%hq*w-C7OSEP)~TQ9YhWeS`8+l`kkHD z@%VLn$)57wA2W=$niQo~K#c3pu(?Q$0L+OUaBvK`L_JZ@d^Tsosm|bo5i+5L$vHX6 z?*+#s{Z^dK@2@gUcvjxJGNv+D^ut~6@$+5vlUM2s>?2~nL@{#fN~cw6>7bB#AJS@< zm~&F97bcA8pnBcE&ZR)ehGf4ojuhWo`M1~N_m?j(+ygqkS*wPNI!Psc`G2q#5}K&o zv+$_8AiObAgi$h$^`FoD^YicDrl%hWD3WG2Rcsdk(eHmJa(2b{sxyE%e@9oM_+ak* z_HutS&HJ1GdjLP<x|fTg@Hr$6%_`IBXMx+*qCO3(5u&jyPsXzISFsM#}nEzxD? zv;U-loT0F3tT(jH9UTiIY45~p85$Y=59R_O5{smG_)v_WfX%Qhc|gg_HbgvyZuz+QD@I=dJw5k$188F90GH9Gv2D z|29$hUQ&I$kPi_(r}~1LCoRAFD)6*l5zKPrBs2jVZja%SKnpa$f%tFy`yqTX&SiEM z?(P6H{5Q<}|DY)M4**C0EaAi2dQn7J`0RmyN7JpdfZ45EOfJS3R{(mqa`Xd}q^Z9_ zFG0-VI3@ORpN?Xj68>IJLUsRaac^mNMn3bIVLj_x&F^*md$!wq{J5n_ZA`|bc6TpI z1~!PSw6dQ64+-4E@#ig%g=K~+9*ZZQP1HgsG-JEf=EQjHsv z`27L2FvA<0)suu;b07aHsBSSBm0XFesGt=_RaE%zx8n1QstGO8v;X0dzx?4J=GE2J zpU;9KX;s>}xbhKaFau~oK>;*b=cT^q%6h=*F!odhSn|KqdZKZ*7A$OkNN7Hz1bjqO z+I&0jXK{(`$Mt#+bzrxhPVDL=$KM-XpQAe$5f*kq@}2M-S+kJ}#GrXcGL`H>^^Pp9 zaM}xXld}D@Gyf(UZ_x|4mrz{ABZ=|gC8PDaM&Nl03iPwS@?++B{Ax7V9EH|eSLjvHr5vd$7^=hPxmz-YHBX!PNT&36FEi3 zgJWsEN7(0ozmI=fGaBf6uf_aR&WYd{NgJyUN1;%FQsCD$!FaY3k?D`Hs~Iv2dHLGM z2crqjhZ9U;DtG_B|8*!(@o=;z8 ztbyLIrjTQTOU-e#Lgs%cNmYK4^EA$0C@&z-?SkR+!Xew_sYYtCNMVuygkHCz{v`pN8zvv4;kZQXx zy?Iy-sXWKC*#QQhEL2CH|6XuB#=ZE9Lb^GZ)9XtSdjbk3gB4DF-MQX7CRG+ zE0zEbPxBNgtQ@^|E!-KiQwJ_IJn1>c#)?b)Cmie~GDnw5A__QTBNZP2X|X_hETBMs zIo_eA-5%v|ks<~p8~~+FViVHT)WlTj2*$qJ z>a|c(0E%%Wu{JEr#Dhfh{@;7gRBy|*>%U{$8RFXf>YE010Ifiha;ffMEhMn^#M!B^ zY|nswr?jd#s`Bpy&1%Ylx*icFoUxgahDifzX!IrELn1&x9aPat|8BuRSfe8m@0naT z%nww|SxgJi`Fkh-?9Za)>D2CT(hZNh+Vr&w_2&_K^Oc;4`bw|U*B8@k z?lJvu!D-h2BofS+b0{W>z_V?Py;^TSR%eim)-uYu3*Zd0vaGH~u;tk$?wy{5mTg`W z?3Zvb-C?NRboGWV#-7ebnsV$h)YvBnObBzYWB0rgGlPI9y}lZUn2zQ!1+D~saRYKW zo8FPGp$V0uV7_40{;ckGmH(satK*vN-v3PuP!vQ;Kt(`7x^pT5(h{R#DBUf+p(si} zNVjw|>26^((vqW_O>)54_Pd9V&*%F)e+aLahjX8}@_nxBI@?Jqn3qUKPYWbP_?2Yj$>WYvAxEB5D#P;IZtxRQNZ^;>KY`p12ZSOjQv#~)xWF-l6_o25y(9_dd{|p7a(}*{aY*}GSvCyYex+*0`696*|59tXI{q)Dzxl@*071u!co_$r zs2Lzm9RrqDs7?>v!?m==XeuyK0Xe7`YL%vNr}C^ zJ+G}&&pq<1n;RPv-(=oxO~-mj=$-?j*hDeMSy z*+BdQU>!emrtE8;gF)Y8_0iDZy3*3X*&3HS;x3=_n}0#HdguTZ^^m#MHT8f_C?;0> ztfIowg6%u|R;3xcXTedi+6*8kAo<@h0C>^}cj(>(hco@M@CFjZgpvCN-l%$AYYw6? zTpOquFT8Vce|^INc>es10q1N@SO6hR#XA@QNaU_OA`hi+But8fKQz9;(x$wLEcuk49Dn8%zu7gxV*DD~60 zGW%`B|0x7;?p(w7+rR{a;&L5_JNAvFE&?gLZ72x5sm=~zUly5iG%P({tjFta-&*7@ zkqOjw)vIri!0&hB6z`^T|NZ>4RANA-Y+yr!@2l6(j9Q+w$Y8a~B5Pg@e1kc! zy!zPK@Uhcn5>W_=>ZcUI%q^c z?MBazmsd5wImW!ia~)iqpGR#yg%^S8gtE;{D=I640!^29vtkWAicw)j4Z(3V0L=4Y zk)er*E2J}^MKlaGu)mp7dBxY;nN{8GC-AJl`3#^Ez6@Plf^h5U(M((d$ zNI&iZ(5$Z(QGyDv2zBJj6xjW1hd6rg4#41rg+g{s8$o8IE2;lkX3dUg7Akl!Vm>?6 ziZ&iNYck*!$x76ucXK&#o48s+sDu37albcbjF; zzP{BHp&w@*1i%2N)6ZRM8d=Yd`M)5wI+4)ZPpmHA{`L^M%L{Tvw= zK3&}oSDx~}fJ2{dhyK?&LUZZ?VE(g=U`=zp{+ zd+tnb27kFK2=wg-EE?mhfDNPtJ4m*|UM5}XO%PT7I|;vtmNUX%S=d@y+_oBtOV-F8 zv;?@1rV!S6sA6MNpazt#Jy;Bvy8ri2n!gb5lteBn$vu$pJ#n>%rUC!s@65;IJ9Rjd zy)8WBSn+?Xh;z)u@US_uFG1J&?RvZW0c+FvwUO5{C_A*6tG}rZC;Y!PJ|eC$H#m$; z7??LvVK*CZfjstq?U)W9dZTMc2M3@3orGTmQ0nxYBgspdcv54tRicsZ2yes}TV*t* z4vcDZ{p$w;ANxr3Tj$8GUb`9fDnz>3idK|Jmp=!l-eqT+$5kR>19pHI|KB8dh^NBO z%*x)Hn^mj=RwGt4@M~8|mBMxCGsRcRef(?$7$pE1{Asd`#LbSsZAQ^NBtOsz7z)jA zLAX;z75hdjESjlW<8MblHKJGLX#P?zAQ2E5D*(|L#}DVIHXWnbvbO=3JsE^H_}{Rf zLIFcm>Cdipb_0f3Gatxd%Bu2ow2gcQP$+*|v*p=Y@B>_=)!Yq8?&b?X0dYJaxDqQ< z`E5={j&%MvBq>pn_LE)lz5)X=Wpos7NrcJ*dj1)#(jI9B3_RnGm;F1ipW4~}cD#J~ zk**?1%2$j{m}qiHt>fP|?u^N0Z|^uoQkpCO=Q|kR0lfBH`@-86{_~RqWy>C)N&f|y zj-+P4+#*2U(FvzW#NVS^dcsO_vUk)0SWNfOM?#0;?&BzF^B%~I+f;@Buj3NO`UKW2 zU~AH?$ArSLvHJBX_lWT5zbNVd_oJk&a^8%YXpKB<^;f3nFWUtq#M-bo6U=OqhMHaw zn?zK$e_BX!*tr-$G>XuZ`GTf~9W8oxZZiBk!(p6mkHgEIOIU7KDwSo*t`f0y*%kDD_9utt=i{EtLL zo;~C(ZjXzJ08Z{1TI%$Obhx8yFv6u@T<-4-{nX@$^w9x$elj0F(+Q4^vG;Vd11sWT z?e!C9uz zzguvZ5xu25jBeaB2k0o^%pi-uLq%mj85f`IjjwlU*G&hUmk4IW<#WbI20XAZKT(T!01Y+m@>F( zs+8Tx+C1vraTYbe)PI^|0MQsqk_S|*GW{?=@!rw6uvq1|&m9z336yZ$1aP8u5%6oB za;M=lcs)I<;*X!^7f39>GN%MYCGGAyb8{33*!K2tj98Wvv7vK*Dv)B`oT^^Rt-n}+ zmQ4uVG9Lo0%|*R9-gr^YRroLaG}D*^XD5=8&ipV$50D1Q@zF%yVJp(YuCLkv(!CmQ zEiSJAxBI#NduQu=%E#CHS6`_h3E#77v&V7=J33az$02}`@uxM@wiAUg|5LGUgL52< zTQ`jx@ZB03izb(ViXl_52|_Q@hGk z`L~y9PJM8~ec&+vi0HQThphl#@VWe$P-2e|`!z)hF#98lz-O>`sEJgFV|O{OQos0Y z9B-wBN7V+vfXu)XX8?WE$P<;HNZSi(|3?&y&v=NUK$}9W zFdvr^v3-U!$Nj=%YT|dbBW}L|!WAeNzr@C_Qo?**ll|O)H};%)<{_Su07C$cX5j3b z^_`tO!yUbaNkGaQAn}!qf#Rj~!ghjoO?#E}?MDj>Tdv!J9PQ_}e^m&x3IAt%ILZ?J z(jHE|L}_;NW_S$9C{hDnQ`v@V z5gFl6GwwaJnJCj+rT>ssbl+ou6aqIF2~exa>bJ-3VPQn0`A-X>Ju^ut0eOen0$Brl z7f&fJst0&@aX^Z2l^p^#o86xp>A1Sxba!#4r48Y{cU@`9p=Mq04_)$@)&`R|)oj0? z%|7w)ZrDGK1cuf*i{DoI5^|N-uD27=V19u))jDeG>1%e@eyQPc=Ky4JtlSA-KXLv~ zb#Gujoz~XYl;La`A;HH#iQ5`{=I~!rTe+ZP6;DKwCk;6h&38aWL!F6LKPGZ6ag(lGxKLo5&y{URy7ThJyGa`QJ0RUSFZKVljz`1-48l4$iiUA8W)QH5pu!V9{sas$$!4Ed>;_c%^u@` z|I|*$zetW>2>}0{?zxaR#JeM2W-)faE|>HtdPS#Gl_u_BaD`fie*-RPIrcxejVIu2 zZ4Y>fWU@X1{N0ZI!8kHr2&gg=2z36tvU4UhM0wM3i<_I=!lFja#NG%BSfap=op$$k zi90VlLZprmJ_59LdQbUHOj&LpA9=_`-})uto-=#lBNR4C;?Je zh+LG+-N>hbx1L~8OiCu^^ZjWe^k>L{XrxJR$cwnIF;4(#3Aqzg%20QhSP6k`NB?(p zNi1gzO!|W{5M3g_ofQ|CWjFx*C-gnM`0s#zC0&47$kXv70Xrfp$SDo{9%d=( zVmevGY7;&{MZy6$H-!vYHn!s-;1Vr_HeSE-E`vOP`jvh5eklU*D6#Q$p8 zo>)gKCnyLwv&D8c0=Bz+rxs%P#>C`*s&(RwB%cNa3h`fmqvsU^9Jhgp^OA?X(R;PU zO!-J4b|dR-Wa2*-RA($DQ^Fg8j1vH0U88D`_kfkPg;|fyhOPaOQRH%Tcc-K&FvqWW zT@ls>+;;EMO0?!Z)iZz)7~m>VmgwA}q~`&G!aW92tD#339gA!#fP>HvP4zwQ1N43&p;3cH63?=0@6c`0WFJLPCJmX0<4OTh&%imqqHTa1{(QmyeY--w z5yY-M7X*&IL3mraEVTjwbttMn5A_G;lRhg)!fU;|^mw=-vK!A!bvd>8_y{wxG#_ug z;L1gvjf70YE0OASUb-f@S%@L;_)JOcWy-#%Yb#G;BO1l%b8W(GXIwH@nI9*Yyq?9a zjjnD_mvCoQBh;TikPV0Pfw-TrsFJUio;&VY)fj{3ix|3?LP*LHh?{Xh#b0=KtXRe>B0AjSlY-U zOBIjFvNCbiJlfCGI9oc|J?ZSDc$@LN7Gsb5lwWHvXK4q{TJ5RC#6w<#V=U~SH~qRr zuK&IS{*a%y>*)4=aO%S7!CGDcUpNhUD(aS-{Z$j1>^z~>Qez8~Y$9~ERuh74E=x=-sDq&~w6dV325s=moaXczL?y$eiO zkbj?9_CEVa$2?0dQFUbc8)}}FgP+nR^dvyQ)IM{9!iG55$53F6g+a--*%!1FtZRWI zx7}+WN9K)oFxl1A)Eap05HT^pKdDjGET8J27@pK*@$_WBZEcX4-*{x2l0QIoT}Huc zwA2z}HJ}w*npr*MYEDcTHAyX?v^J|yAAPM0hTo;okI$$}3^w3m7)#KlGc)saibiQ3 zVUzQo#?>~a=gY!E)aA&9somWX^IX@3mP|Bb*|P4?R&+fy$(X5w=Bt2=hlD?*FZgBx zy98V(o=LH-ps?$I#ioQj(Oaw>#BpMg7B)&#FyBS?8Hm_xz{{&*CHl9 zFKL;Q!VZtg$v*m(65e=Xg(z0yFX4RPlp68fXStNVKYYXF=!IKXi6Am7il+RPuZgyy zSR35swi&)fFcts%IQ)rWq9cfq;-0E|N33~25-P2)q0k#rzeWL@Z_SEmVC{{HM~pd4 zmn=VftNz@7@Db+M$4}J#`i+m?{0L(5u;_Hc|+JT=cDvbLbCk`Wm( zoKBEXW>>9##M)9<NEw^4++sZEe5ll-x}y7+_FDJ9V_rAXT~Gw0P4A^Y(tHt<+*XNoq7ri?s` zq=raI4~ksss5?9$c-+)9CeLDD=_q$Im^1`}yzYA~EB9992PG=Ds;0feRh#hm_@>Cp z-CwieZo;YS!V-=>=(|UWoEw>4NbHV z&Q}Ms8BdsvMAIUhaI-z2m4g_rd?nFxEqCuJhj~MHsYYi72hpGAT*rTo2iDUc(g?MrEa0_c`N$7dG}c zW9KILSCYid#-gf4HR?NBukrEqyKr_dqldH!|qSa@PA;=XDk zfVqvdn%$L}0uK25p99t>NMxKH@FhOrfS=JL{-MQ}y0x=jBpr*oGm<;tBIpCp)4a3% z#H)9So|Ybz_3AKVeqnMtvRxr#kW_G)mL!WK*#zQ3oPWtg933x` z+d-TMKuo%K>?SC|Gly`#y&Ob}unOz?D?66Aeo<_~jb#9>-_VbsF3ca<~62 z;uA;Wc=LT35^!k&TL+MKF2nFkJoB%%FE$j)=pwET7aO5U2XQ#K!oU#}X4atHysUV^ z_h|Scvqdc|<%tBhpA$k-WJuMk^HE*~XMdsE^+9I8=+ud_M-kGUcep__3k$CUo)@t5 zGpmXYwDGWJu=fZ?3h#ylx$^43aDl48BgNZM?{A7~mmc(C1zOY7E(+RhYEj!ku?3IH z9&qQG8bj>WSBpKp6$2(DW7=n(hh51dB_Gco?=XZDMjV`;NatTCkOnOKyGA_1%}q`3#r&LJZIte}khW*{O4iU-{rujeeF^(zFtdNwYu{g) zeSY+-qttWCIFUBop3=PM`4CX0{ewaK*8RF?(LrysyaHP0+z;C0Wh9v;xzbv=7^UzJ zq$H`emzKd+{Soejpw}CEN^Q^V@vBF!F#p`Q1dH$+627A)1$DFDTQ&rYeNgFA(f8W9 zxdf9Hc+jgXbzVPLbzC4E1VxXw6p;1f{&sa31DyPWQk z5$q_;DQc})t6^`_n+eUQp{PX1i@{%U&=qWpZbeecN*~u^FgSz3hmT5u7n+(_W{t?A zDy?U1rpBn&#*CR@eGAA`@4I1dipSIQ!O>vLk7ZAr+fv2>^eaT}%s13~sq>{Mxu{m& z?ib)drHxv%d@V72>+7n?h{ZAw*gd}Guil70SY<5q+CaKTw9x%?^0jcc#qc>}RKO_{ zvZg?N?~uO-p+;g8HSoZZ6^O<{&A4|0C>VY5?dBhk4sIGiUFMvfSS{&DRRphpbGIn{ zL!vvt6GBW|zZ+T~!8N{Wr}NP$yf%zX8RchMTwy{l=0c{mOJ&lw?Z@vh^}``%JSQoo zne{`iby6BHrp3_OjNT0x)2~n8o`^=IIIBgv@0zN=pD>6JT5JQ#-0$>W)Ho@=Ry1n4 zu@}bBy|&DdU8MQevlHNd?ExQ4DPGo4!^0zm$nR0LwtG{l=uNYw=T@(`rk~(zO_V?} zac?~M#4!?!n*(D8M?Z=paHu&)#zo85)OLj5i>n|-fnn*7$ksZ0&!q%fs41M4rZ4Y9 zzJYJJ<-Wb;ZU1khQChj(8~X+9HkN~u)B$7bf^xHj!*$7q27Jl89jD+_+J@bQpEMRW z!FNll$ZuepelHv;E++SF$FZ(HTUZ4xF+Q;sI8rv=rdol`rvI|(KX6*65C(j{vtmVY z=4NJ8!YnI5ELuQlUmqs`=s9Vmw_nQPn%X^?@>hmb44eBLo`7;ramRaxm!D>GDT?W` z%EoTY$*I;;{yt>p?~vC-DJ4nIDS2Z2Rh8+PTcyqW5?Fc0T%oR`25$Khf(?b=01XYv zKw&Rl2<78wcq-ViMxJBayI)qCt3VsYpi`& ztg0AvY5d&4xBeOWAtS4(Cvtlj6n7W9%}=rJi*Dj~Hv~Pi9bO}t)3h?RqpcGPhE(yw zHQTpGb4$OHe!^cgq?h9!xavUP2EOr|m*D{e7MSXo_+-yFu87+E4f(YuU;h!Q-CrZg z?~a-ds0@EcHeebXP>74yKa8bXQ;GWScEeFoMA&-pDRl;;<&n|v1CWpOwupOMoK4Tfe`hhU28TKek9)ja-`=kXCNM z8(}gesx5YI!I<4rO46mT)aj8NbaC-C{;GZ1_nYF%Zpc(Yw=+1Q29>O80o*sArnKBo z?YtEa-C4V;>eTD#*qRU$xx8`NkR^COt4wL}S!2Y50DH1|*sWd*{3o>IESz#a=!}kT(oCm61%Q#~W@W@mXzqd3#kL!hVND=&>*K zaKc9EN{>eN>HbbJoeO}(3ZW6WU14OF!sILWQ;Y`uQGQYNi10nK(gc9y`JefOZ2xrdbp-pN>l_2>A7)i!pcn ztwhSrv5ExFJk+3vbx$@W;rI1wEa9#JgWVk;t6~LJ@4ZoJ@u1qs^N8>j2c0?ucKV|n zI-akuf`C~~(6EAGn{Wtm*rxy>IXN0-sfaznpx5w@#ivK_lN@GaRqDK22!>2=X^>q; zP22P6gH&g+euLlr{AyC8p9%;iIDb>+RikB}J*4z7)Mc=&z$cWSC84{|!QR_Vzo^B* z30vb1*>K5kRiX$8Z2ZK55SCs-9b)tj0EN>P}f&v-WOvJtl>s#vR_RNf`EB#**W(eUXS8AUkEL z3X|bL?S>2XFvphB9YeWsoX)F-72o|{Q(aU`PO7L=J&sJAjiMy-h0k&S}L1k0TdQr&XL1*y^A{z73y=iPHNdbqTWGe-HpL7Z% z7*y}0tK-1JgHu^PmySu?Zp)bb0}W{!mD`jFBis*v7%tJq-XDsxh#Cp*f1#xyV{SA+ zdKuO70Ckd6n*qse(!4{@N)l8S#6GIVp!?X(*Xh00@6HEa0Tu{xHn+i*)V}DiUu9>J z%)VI;*9{#v>7CaZ$YB0m`L*F`eBk)i@HQj$=UHMc& z5~-J?*kDJ)CQONNGph36@zf|CYjdW2477@Qz!hq>g@a3YZAPUaJmxv!me%%A{6A{F zg-fQgC0+`yFbA{G%QSUQf-&ZFAfHB!?9cBSw!yJ%@tK&+2#xfoM^goP-90=6ndbwy znEE2njVjKVeHRxeclT?7SFc6y1=D3n>sK7$l+jt6Mx#0if_ZycFd5=m`QdJa=)2Kx zLzSA^xz>|=tD!+ah4B#l@D`0CWT9dS^$Taz8^l&8`^xq)1T)6H>7rL#MYTe?%~8ay z_GsBvt4-asVLt{(NfP2h9^O!*E|#T)e?`GhZ>xw~TAzcU>{+_8z?PJ6130 z^$L_Q5sUWq{hsN4MZ&_!W!txkONePHzZZw!j|DP{ zRVFLE+fXK5@t&TS1z8c>w2RgRy;jZKFjXsI3Rs|r~?jE=} zs1{R}xv4!Re2+r|V+%Z3UF)-4V=i1&tTg+3u|#4Kx=aCEUkLUW7mMa|!Ausly}KD5 zj}SVow(EIs(#3iaT#e{&H10a>iR}$@NAF$0ee-Ia8hCp1P1Gf_pTJXkzP_9C1qK{Re4GRiMMMJZqP- zSVob3NBaqaWSOOvpE8Kvmtd(~EaM?F+t{i&MmP~fjN?!1_CHcHrrxzVNz>I|u{}US z@%J;@J@N?nUFcfSXLDMgo8Q#z0nZ9lNt-oHhiR->2O|_?yeF!tEon$D{^D@!PGDlh zJAk=MW;+_tg#G-*AUTW5{U)1>P$juIO(D@)UDJ_D+)o{kd^O-cC@ard;W`<4bv9o` zp{J3YpQ=Ve@kWv*NK|EPRQg0jO>bsUa%X$zN~~nfkM@0ZPIlvOoVIwRs@IsWZ?`x_ zgX=f!&xT*$4}aZ8Ke#9Xk_L&{^*BnJ_0jx9$2{j6MbO&x&+P4?t-ZI^nt$BKgx-kQ z&K1B{fFrekev^XZNYxU80GyCfgXMun{Uq{pmp=~oBTd5JD&e$OqAxv6SUG{w^V>~z z!jW<8Vco`uC}!Q(sI6%;MgLy1CxFJDu=q5_^A`(1$832X$wi{$^7!5*73$7!vUp{U z=5{+zDcaGFj}4)7)t{VTg_IkPcQx%YOJW;-Z-uS|3xQYcKvvnO;LdJH5g)&8y1@P^ zhgLh|iAa`(%X5XVZ*_sn9kB5LXo%!vQob-6i%f%s0l}k5HSp}L&m#4W2mB8|3)veV zN4U|4(^S;Ra2Yrcatik)Zb%h>)#1##yli%Zv$aK3;1%X=VODLX1ZQ2nk`2=>6cdvv za;hX>;g0;}3NtG9`K`ORmQ3tbl&zkF#}i~eJ$#ZU(eRuOX8)tGaf6L^J3O1G=yAid zEGNg{z@2NGIzWuVvxq1|FsJDoUa7&x6?b*8T1twcTmFlx!l%vS{H)VO<0X4^`Cq@@ z5v6J!9OsnkKa+_k0pN0O_9R+{ zTo6C!rz>GTv+<2zk6xbQ-%pT=!4fU4gQZ}0Qwf!MlS%u6av4m?+{{4gMe{^B5y37D z7N%|%i%4Q;tv&H^@vlBM+N(np^UxbOg*0vzSNSray_WPNe`%?yn^$-#xF6hCZb7L^ zP1(oIMf4vw9i6_ef!ZTEe>U}qE`XgU?yF< zV+OI4UpOQ@LgCjFKsc;Bzr@K+Du!RbG2>D*|okvd60dM?mYtdgP1}~m6<`hi} zzH|=`N%6MV1n$akY=cT9iji*j^Q_fgaU*d>A~o9$RUjN|Nj4_bl7kZClU{eM)P#DE zob`nI)j-u#s}2%XmEd03r$PEaU~rb&J$+n%)p1APO_tlM??}E2M6(H#_lRn0XFGVT zCd2S?<5T(U8aJUCbq$NK7Bm!-6YbP+Q1JZVqt|(|*I@9$xJ{F=UDI(kL|0e$T8Ls)Go2zVVRz#HmMdO9y7%HkmYX2%#1{hHT|4 z>CDf@$H`+c*qJBi$6q6@()iRgJWnmbawfX?M6`v9w~sUWKyT#9Cd_lsa-5rOwFj;8 zYAwXe(yJIEBBq+Sj^d;vjZ{#d!gGzVG5OkPG;$#O zZZ9aNXUp`M9^{YgpKd2AC>#`pic3v^xW%}5COJ=|vcFmWVjQ0T3HXRw%WL;c8;|0P zRFmp3larMhw}STRq8;6bkznd~A=ojQ=;+~GD?(YG)l-qt3Hf_7Ic{Mf9k->QAhKcY zT1JXhRDO|YiY9nwaIzZH2WccXeCHN7*k7*6q&#J!WAtiBi=)5(_D0sl$^C{Wo#ega%CN^@Q{0+cj|YqvllHqR zwaMvx%=n(3bX_qYa7JBstI*8JYQlOh6u-#GsSxm*Efg;f4095)9L^83oxs5bulpAO z@3X_h@2)`XbbPjAhL&uR7&zR;rB72}YW@WOZu^dz%)`t%2XD&zKY9pRx(PZ{^;00|^GOJ3u2Pn=w$ zt|4>$z>2>SisOC`J4R5QU*TTLQ?>KNg<6c8>6{##evpJUad026=ipt_vb0I2#tT8k ztHN>ivXCy=lETF}mheV=1O#2T$-vL13r39BQtqGDKw1Iav ztl8yzU-IgqdR5F97wY;wnA-F0)TgZ5-be9~6EjRyBUUo{;48}8I~NUg?@qLbb9=AV z9}YasSZSXEox<3rI+m=vCL!t7KN{@7-O@JU?l);HB#$@DxTmMpD0A7ZkS58AxvB#o zBZl=wyF)`SmzJ?@Rrf2FnXA(cM~5i5)#h00wBUWixdj>SY0BB#OX|_}gdgOPr40<5 zo5t>rBG)S(ukB1biN+!&FO(;IhLUpAIs?A57p_cLyZ63C3a9zbElSnfPgbMOx8eEN z1J`a2SR#Lson69ENNkE|t)v``B7n`$Y{w|inw?Z!m=@Y~>Mf?aRnOw2-8cP?uqO&O zO4;itKYX`uu>S)*UT_`aYq=M2cv`F7%K`6C9WzwQb@mHie{H3b)wtiA!eA%9=!^eA z%^1RGSfu~#S+$9Z)zta1STMW5{E2FXVCdy-rpKZuM(*-^lNHN)gI|-4@@h4U!!1U~ zP2HtlQ4svP7{U$vQqcUQ;t6h&zrR{I9J<*M-fGWhzBVEvIz|d=RcI88E0%Gk8@KmOI z&;3|FNs}OMq$&;?Iltkxsn-1R0VauR{n#^Rr^+S@K%;B76<=?Zx@I`7nc{b(!VEDM zmlfa9jH!%7u`Q1WcUF!IfoWM-(V(oBk_QZ`BFbjZYick#MuW*LrAnTc`lNO~q zpVD7FNk8#6S*CT{(%m6Xa`X6Ytwk$L zFeX-GWEp+r&mhoIQh9!6fy68Z%TFG7&)(zq5fmW z4a5V5*6(R=zGLUHd@whcV;@*PH_RQH?JKQ6i|3oza*}7sX}6U7ZtH7qAT=|;WD9m|7%ft<@;YRGsw& zfE(T9$G_ONa!R3U_f&D1D4$8smOXP4RsPh*Ayc~M}6 z;-FtULG`OEzfpX8d;`JUI|f*-DY^S*W{BVf%5-nrbo^Qx`MrxN9)etzxM8Y1Y7$<>+!IvL+G%YuQ-Cmf>FxV+%4U1aZ;g4L^rHv1+~0)BO`@8)-rt1;;4k-@Z0 z)4N-$GcGB%YOV+oU;9-~kJY`)eDP-5k|ZZ_H^FUbqjK)9&@s*0&7ZIVa*th!$lDVJ zGdah>4cmebd-4?uWD2WLTbhiZ(J{6L3Cab%!$MB(5x4h1AM*#b1ImJ<(Vvo;vam1f zs$gI9pNLm5=atcWf5NR;BT-+Xdoi`*Z*RISeSWz<*gLXutL^!IdCjrDN`(@>FNj*p z^MLj(ZHtlhj@15i+WdA2^Nci!2UK?NSVSZS6nv#7DxTk5C!k|Oc4Pw-Ei|6tckKtS zX;98S!Z`!;HsNbMMf6X|g#F`tNetw;o0l&i_5C39Rws9Id5_Kh=AUe-Y-T@ z#W@9*N}z;BlyMPWR6D+I#nR{Wp#=_tC26l65T|$*_GS<>dyAfFX)C{I@(8c$z&MP{ z(!d?Lb!YLGwT8HehUV*jRRk~9h^_ITY)YY(AQN?lUVZzDi;@gBRN_^SptJ8p zCX=F6-^odMJ0a!^kJI>lmzP}DOMiNh+ko{@C)+aS3vc};4ri@wFb?Qfkec4cx#&9S zsQ@Y4C)2oLW~Q|!Pp^$USQ-``5 z`ua0YBV+G*?ae7dD5`8yYL|&+4>~Bho&#Z37lMs_6-{ab;YnnmYm@#AKXk#IZRD33fN$Z2q7;Vtd+C>4XC-jZ4)-k~lK`c~> zOA1z!YIo2$&?ssxuf_^q*l?qz2HhHWaTi$MoIPIk8`j?DkT~wLxWWy!ZA>+LaK){q z#i94cG27%OV@24HZ%=)PB&(fg{B3F#i%-zuhnh}vFjX?(GPT$lBd+SRO*m9$hL9-o zXS(F}BXOA6D>025zihh$U{E3KcHL9SW~kSNcv`X_zL6AhREyp`Tr`wyH{ z&t$ROnru3FnEPGY3cExtAl-u6b1J{PyL8G3Uqj;Id8Pc4Sjq_KB&VCSAkn13 zPBX_^T>9~0q=i@&th0!{XzRphhPPSx`>Gt0kMK{I5nI_5QyrbzbGNcL=M+cA&Fel& z)@7#dk9Hoa?_s~Gc%rASGz$A>SxFwFI3P76W|zFSt{(pLJ|$HRDs?mL)4)Otrx_~b z2^mQqT1(owVIR2ik4&p0#xv@e^agU;DX+fV`5Oh7QHdB`DV$ozNW=~pFcDaiw1?sI@QVZ0} zs#)(0Y}ZHDE$Xa}k|--1#q6}@h#ihHp-$8G*yKCSswBK+)}A3BdcW+iI|%eF;KFW& zeQx6C6gMnAsAFhK7#gq)v0#w@wJ(Pba|plH7r(x!x6|e=TC45LDJn-bx|iI*@&f#| zu4!?{v2j0Og0yuiU6Z1U6dpK};QA8OYW!v};_LdG=d8Cm=&Qi6+NGG^?~}RZ_qW(k zVWZ(;Y|}3j7P3t2!u>-B7|0DA;~S+kUi($DulrY$Y4~URezOKYN8!U1@=F!bjky_* zqUF{E-Mzhn54agdiqcrtpS9%4oV00D-FXCl*YAV9%)mbo5=U9KjUOisN#XSe9nM+` zW>sus3fmSW#UZbCygy^FT$QOs*=OXJSIW>Vk8X6sPEY0{Pz81grR5)RhW?xf}Vnr+^$tp?D8qP;*at1i65+E4=HOaX;^6WMdVp z#^MahhE@ni9m@JmA>uInvZDDwJcFHU+D{?powkU#uk*w49=%`RzY=&-7p~doPK{`s zFthSuCp4qT%nZU6WTFvL$F<`I5hHBu_CZalni@%-yOuHW!72HMiLe^C!vh?oX4qQb zz!P!vP^crVsOTl%>GJO@CIG!t>8~F{&+>5cX4)zRD=2W)W8Gm1nz_mM)al zJhbbOL;LV8)?l-B1frU6kaIF7TD%aqkX{yI=pT6r9O^N0Av{fO?Ja}xMLFjAuryuuUOmtNu zH~4Qt0sLe!u((Lm)$qqp>-->o*D4e5ai}0>#{z9YWwl>iQ}xfpGkIH1IyCeZB3EmL zg`sC_(JM?g!Xlx2<2g7fVh9)Mtl%$8wkxi%OFJ&%t=-WF&!WLx2CuA`Ecf{15Tn|u z^(H>`wJD?&x~?HgOKZ$84yY<>$0=qh(u!WY-7T^et-yBHT@kae&)s;MG7_hecKdk; z?fqDn9Qc+Bq(YO8;dp?2nFOv{*v6IOf~e`2O=h*t4tGc3Ul|@y9)y2*^!Rb3iLsPT zUfD>A$(_ZRL$-{{^hy)*?HTjKUpKI89>QHMl^ncO;L%!lg?>B4sHjVQOKk^9)88qd3WO6!vCaRWsfd2&Gq?1@#WluI^P;A%MCoxnSTJ7U}t zVO@Okhq*yvcSjJ~D5)Xdp*NkEIbw}?+SMXc>)*`8n3ho=T<1Ha8X4gGa-NFm*J*H# z=!^w&e^rhvb#u!1M&80N0;A88{^E29s|li{wBW-5XZSm2J#sq}#%794AO`t>a<_p+1;R5Lt*kg#8bwB_}d z^1q8ajz@B#8=n<+O&zS44LI=dSEsN#Y>y*~gT9yR{rdJwI>7N}^pix;gdHlly)?Ig zu?ff-y(m^!4xr@ZOoK(!)oe+pD~ON+=EalsV;ttvlgr| zOHp4C4x5UX(|b3bvGq9d-YQ@qkdjk+ONlyFW1O%0_h*b*EcrwZ)>Qob?#-A~RbH?X znw{lc!POsm(KU4e2lB?8A0Mf~oIgGPm6C$us{GYASIoL7esH2@YVEg6y}4~--*M~f z*X0<#o3cS38X<}@+{)cX8g89eyJgnN_~0KNzN={YRwb1eI^9RUpyS7)92OFyw5zd% zwznEw9i*xIP5W+lDoJH(eo{XQxU3JEbD6KxH6Y_rwvA>^%jG!Eu#+1Pq7lD*&3lEC z;CZFr7B5bCsRyn*SEgQq?Ln&EqsNYs4nANg^c@ek8vjDU3O_cd@jXaDb6+nv%u`%K zgGRQ8BMvKd4osT~OnRnR&!0n^wF;;ekI-;#(h-l@0rr>!?DwS&)F*9eBpT+o-sK z&GrEs&(GQ*)k`)p{fB9ja2k!yr%MUZknxa=wNpP1^{6emo4D;^aBk=vPhl?zso!Bb}6 zGumNFYU{T&Wp$_6{t;3wU@+OS_)G9bd(3wuMS0Wgm4ThF&vy@%A2i z`TtS%)p1RKU)(k-0$)T@WGdYP(hL+tq&uWWcXw<|L_|TlJEf5x9nw9z*`ygAqXuKo z@ca8dujjdc5PR+Ox%ZxX&b{ZJdXED4zsEDfBivpJF+L&z`5-r(2P@0GpR^QgeZJ=- z6>*;cpKL{Y?NI;Z*E%u5@1nCYgC2n0p5U+kNLeZM^ZTq6I*gq5^BACE(yb%j+mt^@}D0|{$%PaIS+yL4WN0qcCwas zugs1w_fL>rvPDW`ds1LicK#GdjZ?aFh8SpeNQ3uP&1d zK2VXvsthoFLLbBr(v-C;0ToJxBSb6HY3jiv&EidGz<=_#F3Jvj;+pCv0}Aqg?n!uR zYh2i;IKwY=R~H{zXRsL)%;2s^m_f(be4BG1A@n3*3E80ni48ouEVw3Mb3{6OR-{~c*tq3 z9U)sJ;##G##YsoOVb-@-|0r2vF^noIJ_&j@{gW@(hKzqE9-$|DmabsC$u=~2HrVWa zjxg?djI6>H4PsX3b3TqplX2N9P@~gZvxeM&3UZ z(b2js>&CA1w@T9axF9Q{5Bmog<~k?cPKfQ-VGG3ZpE^2zQhGM~e!DXKkenKqnmQ?> zLY zt~2~L04Anc9Jh3;R`@IlTyKOl@Bia^nhhr}^}#yI_+sdTQ?O4bM+Mo;(2aqW@e1Mb zqG#1cha@D7PjMs-i@yoFs!+GSE=%mC%_N86N+IuZPkIQ7!l-cj4Gx|5m9tE~;3YsU zzYUIH5|8$&N{5x!lQ-;h?F;57*c%de7qQ3vUX}?<9UOd+y{q43=X^m&&hKrV zS3yP50RTE`J(p@#YE99|ZrSUdhiVBUW#lj!!6^_+YB{+(oQ>8zQZ?UWz)LL@j>64D zOF_}z*4CXzZK50D6?;{8Hcp$di#V05!WK4Rr<37Z=`&Y@>Cx4jd<3%|6^tjZAl#++ zdP7rSo5M8m&1TeS+)BX9i={*u@GwF577|4B$K$Gkl{KS(s6~gGd}#rrs;`%HF~%$d zr^?IuiuBykBkZ7PQ9pfjO3knJ%EoSbX)6eP*#?>$gzT_#LmHf?_b(u4xwPRwAO#(0 z9maw>%88rpbMJZBxN#M`l0RGE`&8-1za>@kxQwgbdI#VBvXbT9ZY$^cla<-4=F0@y zTXvmfruasV_KJ4Fg4p)B_a=qCJ!Yh;z3S z)iu2lGJ#Jw(#vx=65HA4X>wt?Dsv)*_8d9dtAt|ki5w|9$v5G zm4>xfV+dwf9a?--=6!+YK2iIyMHB zy@cfCS_&u2X)kL9$FG67nxYy-+0WokhM7VsP4i8Z zCySuB)%XWjCOeLgPRrGvxTEE4cHDsR+3M(5eu5?Ol##>ViH`~7IvEx!KrKsz1)Z~z zFc)Of=XL^@*W!l>|47Jk=l1==$7&#+HPm38mz?GRBdd&g6!5B3YJENR9-To*EBSri zZU8sJd&PNVtHE{rmlWH|9yFW1fQd@X>H zT~6BM?j5_Es$GmVzl=*}r{PAnP%zPEVSR4)(3;QTwjv(Th~3tnyH0No5r-N73Isgk z73BH_y@93c4&3{FN`ihEexG)3ZO2*EZa3n7OPt@mcHp>-dU&#ryaw*b_oVGx&x^@S z7SGZ;!5>P0+{*->C_mwPQIPAHB&S}|tLOXPDAJc*^lOVGX>7O*fZq$P8o7#cg4d5PC%{mYvrf;<*jG{G4z_3~W;4 zmzpKG{l0LiN@2^#T4TDg&UnCUkra_U#OI$xohCLrr^5tTfMK^^VeYv(s@0C ztcW|YAEA$MUF*1&UV!)#g;D1y8te3W<)JqU8aQ>25(V%{7u97mD`4JV5nT}xTxWR(f ztb=0xsEIuaUD+Hh(ggK!+Om`2&JA1jC0R_CFD5m8$8SF6#)@T`?aQ)HrxkfeXwK#SX40?)8*SPhb_bR#MR}8&{iS0HcC6*8vh<&96w2?t(F$1rkobuQT}-O96!N4 zI+$_{p!%nQ`(_p)N;vcGYFzj-xx)y=*hr&xBm8C28_f}a2(x2IlFhk)h*mTv+}$W- zE&iWX_q*A0F?%@;;$Fpey%)V&>9E*~Nn9fD+T`?F6T|)BN6UMnCeCY9hSO)=f#=~e zJSVPNewZ3vY>wZXr1B$Mqy!29=V$*@( z!USuebesQ4Z^LU=cK1k-dR0Dbdv84VK&cF$`F?hnStC;MkI4e={ zoCPeWvIWtU_OtM@SeqG8w}jKl!jAiWmk()|`S&vCa=*!f+QZ!p8vrM^vH;uT;tWFU$S$D=_Co<|7@{wjq50b zR5L4QYG+XaoHsGl_#Fk#^0wz4D-J8a2Xsc6z5uYGo2aC+Jz-f#_SHNC>~S%BPZnUym69qluM=9-f0hA z@Obl@_dSQJZEcEMLA5~mE32#N-JE~!WK+mI5=|O?g34#b9HMY@+uaj!h3u#o4sQhs zjy&{+)O?rXb0CB4DlUIIl*zClnz?Gj-h>0--NA)K7C*>=L_-_+eIP!q`4|2I^c?3h zra_neKH;9){6A2x-q-O*FJi4Jn-v<~;1+?FCrQGoz+COH9wan>?q{@{=twLUW3Nv7 z_~VZ6{B<%=0h@Kdeo^ZvqI?@RDkSdkzTRkc}1Pfj+oVrU5ypZWRnxf|hwCs9i| z@Uiz09E!mdg zx<@UJRySA~wF%qlmc8m=KODF!Mxr?%W~49mtQhk~*&N(#6gtOEl7&=7JLYR%_38A; z?aD{F&JxQ@RB7p%o+qj?C`VMoiN8r%b`zdr?T@A39yFh}1QUXIelKZDB@qVsO{;z= z4(w2T(AcJI%I3AiUpd9>SgCa1oN;eR_sLUa)lXX0@og4Fq-mTxBv=64#I)H-X`us& z&^C9xI9KUs)-QRmHolLFW3# zHW&i!&Tl;ycI`C9Rm`M6N5+@){m$Y9iR~SA*b2V;h!azAH$E*&-!;YLz@xi;iE7iu zH7%i6jGDsz;ULX@_-=8wVoC7RXcoSpU6%A#6lpKDHTr_*ILz_1fipOTjy|jJ2xj*| zfH7jj8u<8D>6$z>d<95ppPWGdoq97oE7x+0nJ*j-TwoRmb;prR#&4Q_LL|h?3JB5~YDq>d=q>EF6 zrG@tM2_GYQ2*UGd$V}aFmvPmvURr#hv{hFnRVSmMy>Kl;naZLL0!Y%~2||)9vQ`Tp}x>AJKw#Cd4EDGi!92<#KhfX{%i%>`;2?^f;&U)@e55Az_%602CN z1WAIL*$bCJy60ao->H3Xy!IQjfo$g(|3JP_b9gI|#dqTIGnW@%|6;VfO5KugY}A!@ zRs=H|5g_`)X`rzdfo-y0?_p}Jz+|2n{NWj9v!VKWxB%YP7g?2uZ0x-+xuv69t+b`{ z=DRh}t!eW3ht|)0~*n7lM!nz*u#_gwhkSC2P z&vse9lls$0(st2DE&I~h zCbh|gNj`T(T{Eh@qTakG38TJPT6Tv+qinfE-^Bz+6L4^S@gDFs+y1hfx^yP>RiZIx z<2)U2V<^hZX$8l5P{X{=SoYM4T4e&X=8W8Hm~ZQhM||tv8vtF|M_R|yu<;#vTCV9S zECcO3+!ZN_yxS??k%&6A)pMv&hM{7YS?N|~lwe!mlb`;X0tbFw3VLL%2v4~CMc#ae z`;W>Msj`^i!KN=IS5nQwxTF#+RdNHjKbHOexF_U~rv1+yt(-B=$z1l~zqc3sE+NkS z?SNHaWv$%0l+McDh+l5(sRSw_nb~JQJgTV2CfvFhGvkr&y8ew+Y0cFyxe{n?mp?Jd zdeN3A1V#X+2j=Fct)6Sgx3V*Rt6_z|WXKt);xO4Er+Zm-^^PSXuuqyHZ{a+Dz{XJW zu5>RavePRJ^cu0gCs7=tOTKe2?sA_p-kHkO@h3@M4+tIeQy1fZh@NAc}2$|6~7pfpSa<%^cG$ z)S1_L_(x$a`k1_CUE#f&p7W4gh3`>8L1z!f&8h2zsk=|GwJ+*EJF8~Z-c?oO-DlzB zZSLu@67Ap=TZ@a!lCuk^uK6L3If=}%kz{));HOY1jRBrHMKX6fwA#KN8!stdXtLy?-|mBNCqiRn4-&H^!Q6G<4wy~sh&x}W@N{MuC}3@mf(Zb${L6?rx{Lkxbk9BI zp@I!mW4JOHtdow3H9yFl+N*<%qX`@hn%4r5O6&Pl%%cTWKSbLUjn-vcp8(^G%e3^y zj^$N!?Mv^LeS_7?Al`4$>UWAyJHx3^K(rKAEzoZ zJ1W!Dd<>i#LT;wfzlW5$%sVNEeVBF0orv4eU6#p87y*DQ|Dc2oCl4O)k*abx3tGz7 z(oZ3Tq+~wm#i>L)qx8+kbnhvp*PSfO9<@;!-k$#)unTb?qBm{8{S;?D#3VW{tnV%O zS+#bQD|1I2QX9_SZC}50x9V3{#5oVpFK+b`x>THyKdP-fuDetpzAUQxJyVwU?mT-1{S_YR4wgG0ij*%l zl)e)=B`z=(;*1G4#noI+HgqXA;AXe>0LflcOXN7uF9lK}Y*$^~Oc;aTR|I;sz1{D4 zwB8R1$Ei=RwO2B@OfHyGff`~8^qRfVlLTFi2*Nt zzg!+nXvxQ+<`4r0C)3esdZP;B2qdY<_H0kF$$A4>i4S0Y?^IHncZ#^x(5lz#Ko~%|LKofd63hE0n>|=r`z&;?SIZygD`o|Jp0>&mc zCM+rmoz26-(KUc?*#hwMZC%H9Qn?f{XRj zV)=y)=R`!8rD6IJz6M5-S7wOYH{@sE^~;Nr-e?AGto^mtkA9MOxDDOF3F*H;`I^Wm zRZ>N{xR9XeTxoteIygGE{ynJuiRNr~^3nwxaU<~*xA*F?U3qlSZ~8zq+X%Y_kJC-0tx*(3*WefQeiXW z<)zi7;IX-IJ?s|+f#c}XxKPK+TLQ2U!KL#WiKg6TB`W^92D%1I`zZ4m8NYRHM+q8p z`XdczOkj`}_9K4Vb82H}FCjFUNWq}%f&`xE;B6Yds~6g9bC^lBXxy@Czl3tl>`!M; zM=xmed@Ui{X^G!1>Y0|P`s>A}y?DNpLG82dIl)HvdA2_t$(Nf!d+@SLpMvO%vbFsS zWngosjk!qZv!?BUQU^O=$AaC1;Ytr2WWO^Op!>+qjzLLDz%o(Lvujh6I|4DXI)xQ8we=HT@e~*K(|JN zxye&yi$?3c9}%5CI9L~&wfq7TV6WaV@z;Hu-)e*!HX@;JVQ#1z(_*XS($se_#9JZD z^Sga3RkZOpmO-`&HMJ4E{qv2o?^Y=n>mWNL7#fM~^|=LkCkHoY4`w%br#ANXz+xs% zO;ycwTkDI8n4?7b4c;+TR@nq(9CS(kP>1zDr4AbUIbW=tM8E(1+3RWP$;o<=ED8dRf5N zDWreN&dE^yN$ygVN4izV%W#PrKlxwJCq_z3^$sipvEOf%p8M=9&{KaF;15FjVn&F? zV;bF3-LlYqMk>IAZr#wgmv8{L(>&23H5WVx#0SmvJOozrJ~H zc>$lzy>GA~ZaGJ1Qvgopo_S&#QU&9|NeG8qe)D~RG{ZlpxVzzL2Th_9io27_m6E^C zw*wlwLwY}{=v^Kcrs{h7m;b3Qpo{sn>td)$l)SgsUIU1l)G9dYpWqi+Vj?Aq zM0eI?&FZ?#dZ&gB#>$DzQY^<1fdM4S-7bp@3m&N^?Q?ol2E13);K7s{hlpxI#|aAX zRbRR&+kj~)Wb11YsqNT=K~}OqPqA{H?3vnwQ=4~N4@2bE7s|$6@)t^tZr#3w;r5U@ z60H|Sc!3rzGxjBb?K(O-=6v;mYo4VM;SXEerLTPNl9U*K{-d%A5HIt*2&)-yuEdw?C8xOfj(^(wWJL@LzdwXM1sc1utZ~0dSkGcf2E_~Q1B017 z;UVFWu6=iA;}H7n_wUXxf*bJ#&+h7=?rwcf2t6p9~@nzhh-Q( zH46*;4FBQn6(45ZBIzA0qgdaEzbCz8w=d%3oczT;E{TsY^l{muN~c`YD4o*{zE&y*W%l{)DCrcl2!`iW_ek3+0uuOIMKyp2pTHo-^H<$zNm$10Bp_$@^QT zr)EHm2lVkxC!eG-o57EJW%{ONo@fEm$tjiACI%>K9JsqJ+iU^*cS`*lS)1bA!kp$! z!#P4)Bv(tn75>@3+y6XOh)oIMGSA;PZYoScrjqH*^DHQ*U_US%tb<0VKjm^e=)XC0 z7F)U?qB5%eV|xBmCNa5rj@X%$*e{<>c3&1ciRZ(a{I3;dX{Q$KOEp1kg1FTsRC~2L z4MH@+*WH6BS=5Zh#uZ=4=#kfTImtIIq|~JEomon97fyw(QPfF3MBVNxxl9^A^N4BZ zt4#N~ij_)`vAvOw+NPT4bdmWHIgI zB&bcTt5tQc^OH{s98Cb#A+15cYDH`8l^>C#SL*t4b4E=O+yE+d1}}jw zjob(F0Huh?<^9vghU!{k*x9T<>?e#ra?iG9`*{r-6L|ch;(Rw#+9^5s%$g%J9|wu_ zp3vgXLbfEKhFDbkv(w#4Vm)!}Bqd$4u$tKOt+2M(j%Aq-`otQ7|BObU7xqYY)?iv-M3rtslopf{LxIJCgU=A^}VM5&w=4c@c-p{omwGzkzr ziIGSi)9%n&fI6zF67acvURv+ui^YteWL`!w?VYWe6#>Ezv(rozK#@sVXtl1=Gas*L z76EnR0-EI5B9TE(a2Ifo8!(7gRIwk8Yf3DHNwGLbNQ`N(^i_@BWouK2jqe>NVw_{{ zC}JFXcaZ4!sc2BSUsIQ4awQd!ibx#Xn-|ZiQSB=W?My7_goT$d;r2lUr>TS7Q~!Ko zU)wpI)FE;6q@0IzE8=ei_#iE|r4KUqyk}A+SxBDD2G86#GhTe@%Xbve2Hv?6`$`|? zvmFp#Q(9-M#Q;aR#b^(prP-&zR~JbQr7oDh9(_#6+aVNT;3Poct{f)S(egkd8W3?XsP38@F(`wBz!T}jj(0S z-~Flhm0c(QvZNQW0{5eAlG9nZV2cOmsP-_~bQa1pENvS7EDDBp(e&sq;fIhP4wz<> ziCq^WeN*}tUn%v*hVtVj)9_Rwd!GGqRE)E!Ow`-2OY+pschyr2Pd&S8=h}84Z*x^C zC?;NUmXWwOYegm-SUoF?3=HJK7HheCz%4~xsNzP)B6*2dnT)(HE5=O|S521~b7WKw zzvGf>7SBpQYMEAqIl}9?aurYSd~VHuHR7=qtR+owl2*^}^H9pEfa$o^I`>N&NK!+d z1QPRxX@$&$+^PszP0wk$^xAt)E306>#n?9~PH<^*>khOOx%VdTaq2=)m;K6K^)`o8 za-Fx#TH~-)7sN2_vSkW1r)P|yiWKAE72;sCn@<=~msAbB@6&(s3~FvLP)^s3&TR13 zN8?lP@l88^?u@eKgHNPeB>W3WjneZ4>GLko#mUrEb1vIpFIx{Q+dyqZbhLKiO9zug zkXODw9UFjp!g*5Wl?N)FAU=uzxCVOTYb_ql&j$4nlTw9ceVJyJiRzKMGUV z^*yYv$ja(^qH9XnA&G1)`%><_O#fLS34cWXb3^vsTMaA&WL-X=uIotmY(K@hn&?Yy z?fHHrSA)r!|02AZ+)K&^Q4pdTt0i%CL`NfJ{be9p@p`&G*PsYDhSPs2e4{r@?QB)g zQ}6lH9vR5^rEH8NgapMq<#$GdD_ExpFoTOnVGrMijT-R_5p_Q_ZW@~`5om3 zVWa#e*-SNHxPbN3(&MK*On!1p8>E?j&AftHEQ7uMy^RqtXg7l1TLIGaJy$Y0N{P-f#)5uCA5?@UoyAkOeBc-lhI)k8b@v=AmuQtB=A% z6+}{JJ&dHJ+7n|cAJBh|{v8;loFeGmJ|jWhPh3D4b(QNl_ucgK+EDV+m}g;@Iv7y! zDW#;0M%CE~yG$^p&MN^BFX&;jk|VpzB<|Fn&^C{=gWCYMMmDRu4Db6oGzhhoc|miR zRDcT42??Vk^|$Dg<2@KknKBjaIp3}{%B<7*I?E&;OR|NPLNT!8FOMqZ!+9(+;CY?LFZn3J#yoO%e%aSGyyjV(d;;B`l=ZVvL3clZ+)K*^VITNhcL|>! z-nVTFKvu;4ld1eoC8MNwfZbI87P+9Up_w*e)Xiy4eSOTa=T=D&2_Z=;zbWrEtNpKi z`u#f*|0kOL!9n@tZRHdNrho`|f(#&;jcI{`cdy`S*4-Qu(mP#2-|fOCC@5E8Ii=sG z1{&6-LuPLIySKg1+eC@qUo~ZV2lCILkq~+j(fvB>&p2Lhn7c`fDkcEbbA7RQRn;|* z+(DB^Dtb9|Zaejx?oj|ef6TFVzsXMdEc*?JzzUYP`@HJlU%_d5IjEOa6he%8O zaX(ccl)ad&6KoU!S*gz1>`F<*?@dI0DA<6XbaSk#GN zF#a5im+r&OnJazK8+rn?h5SN?B zqB5RYMElEYP3IH+p3*aNOm(5q{h9U0qBOBcbmwmnnI#J`|4XhelWWQPu5Q_5 z#CTs3>^GS73vLE+=g|V(FDs0A=#>a(eW+SSAp-4aVJvc z-)!U1E{*TP`FM6cH1asXq(Aad=p% z!4stW1-W#!{1&&bOuNHUxKvankr^AkV`O<>bmJVNgvy3e=eg#DMh^tJ04@7zDd zAqz@@shOqyz_)LI8^8GNlD;-U)?JDy^X5mr4ue-Idn(YUN}YGXK}{Z4MTMiZm<`HR zON-T&mAJxh$J8uuM);#k%P*bwDuG3(9f&W62#om>a?+INMow)aCZN~%C`;V-W0bW1; zFHhj%YMien9XnjhLtjF$E8+A7{~wx0`sd-#*gYK*%gPyU@gt%CS9;z)_}gbZlQOL( z&~6SA3=CMG7F z>-6z=+Tk$)+{wjaHgieP#rgR)-0+=yEP~&s+u`wO<P_mzmT5Ors^1HSvS) z|Cg_|DgLT>+z;+tsy=APYxWBs8A^E);FT4$H5tC}^_y@!!j1Pj7EUeGbkr4MME4T>^%#8Jgwy#t1_7q1e;@e!v-SBk^v?~nHeaq%1zj!fk1B>&7*Dzl;jJqy zl8@E0Bq%757-BMm{r|`#_?LfLqU0D!7c(ht$MzMDS&m*yDv7MDBJQlxu{fwwr}z}< z6Zrr3m2r*4bpvgx&KzDt_stLwE0B?=!NLac-DC{-Z3Nu>ZhqRodh`G52>7=nK(c`O z=i}DYhLeBr6LlCv9ft#tqq5Yjfo#U_ws&gu|5XV8*Vn(n_ncxJ9sTllJY}^x9E_aT zAUtkxJa$W5lE;vb5Hhp|b4;0%lJfs*FgN}#3vw_;V8-VUKvd-pKI*Q1H9QrHvs~8d zQUx?$B!w%;{XY}Bq42LVX+N}GFnQY(TUV2r>3`i(Z(y-JcgqRaa{f;e1iy#Et!rld zboEAC`xJg*dxb4!Y~hCTobL1BS?RhBi~jo}PA-8yB)r?iC@|~r?E4s=5*bJ1n@K;z zTQbH_#WIR1uh#r89Plua_(zntz1Cs}QlT~){L+?cIPmJA0e7)Re9Zp@?Ni#n3+o#~ zF;Go+);-j6%~{M7Q?_xOAk6`{k3)Dg&ZZheXBU^3f8Pg4!P@(x>VCkaxB$wtUVEeC zp?Wk#_x>B9>%UZW{M|S+c^fx-L}Gey-ILx1WmvaXv79`-if*-U5q#CFCq@^4{lYgA zs$L?UJ{ir1uG>X>SEu{YDwZxNU2*b%J$Gz&|FGo0++I zu*#h-9z7)Faa>6GdDZ_fM_@?s@-@lQ*}07aqC3i~ky~5*|6`Smxxa(+|5Sc|a*Y#; z=aCx(RFA%{8~PmmSzP{MiuXo*r58NF$`)frH&vmRFBPP`V!-tr|GsD5)~lioSM%LMHeFTUl{tJP@E_uZZFzCM;j96k zeZ`L)dc8SXkqR%JKb<5v?qmD!otpo?v(=W!)_-6_7+-=V}$#^{sL zcA!%~EY;oJ9Z!i*-+vqFoOfNcT%8JbmE$9{#IgH|6#aiC@C|g0oUZ*n_OS^!?X`Jq5XN5R`+l2M=Q(GR=EIcb%%w_l(S2iD*{ zC{*2F1BuUV21RL!rLUjLz_}V%lI=FGf@~#XzXVPv7Bg@{#(?t5rMU37rT;0D#8TG_ zelJXPueR6cPYcfRHl7Anr8!mGaiRcWFq&a`I^zw_vE{OLF!V1TqS1|mM-O$-=LDvJ z=J3_Na(=}CCC|<7bU}%IFKeHB*8+uaPLKW$`JeI035SbiWoF67MlsI!21DHQ4Y*g= zJ;Zr0E-BUk-B?BAor^Wtt{irPBr^%FpZ?_E-OK88@VOHS+2s@K8q(-%yetU^LN=ng ztE#L2<1Bg({E`gA$>U=dAh_kbft_fizV=jC7apT14@L$)oDo!K3azBVZzBO?ne@zQ z`Ui4N@7skY)i-S8pc4Pt0Coe45(%u0 zUt-4h)Kr7i)FeQ3t+y=V;!INlM2)%+iT%a9=aOxRL}zeurM>m1z1h-E%{evM+q(dd zXQm(}7>|i73J!iMJJwezM2mU*QGbfqvPwtKl0uK_N$X+*DS^f+<+v7$?(XOu3(e#0 zo*$c0YVOX~{j>EdF_uox=VG^d8p$aut2g7eGL+CGZl|cTjA4yV#?2_+(aQ{UyLaDh z10jtdJ{wybHdn5v==$6svvRT9w`qklUaziRib$LH8)%bwDtkxJ^G(KSYfw^DSG~v7 z4mW&Ta-Tmi4|uPa7@b#CYOAZqte))2!Jp5@^^-(#i)+b+(fCE!Kz>OH|L)6Fn6-Us z>Rq=na&Hmd(j_&IIl?)8<;YjV@JMP-kH}?vgbO7pb91;LXj7Wj=pdsjmz^ZX6F(|m zu?7gl>oJe>C*D(+RJ9qGc=jiw%4#we#YM$x7UWb(@VFs6AN$+1*3y`IpXFHr8P%N> zVd_a3B-!(_%E)`o?d-b+qQ+fw6^*nbZc(?j9a($kDD8HUsYkjHmoy3EQ+{o3_}0LD zD{<^#OcGKpC%c#R_1^MBtMMU@xYuG)F0LDH28q#nAk77rctg$lyCS^V^;HT68lyh- zYYDTBxAV4kQz$q*NEg=X)cooBbX%?--oCBBAeuJirQ0(&hR;bOaa!{AePD*@xJglJ z6_zf{sYi;<;AKVXZT2+w(bf-YSI@g&4p=+u*I!ZurfL4!pVdwKWndtE_*}*i%D+j2 zxifu;k*|3fegmrAT^KPMuA{DnaEaeqd=Wz+!b=Z~KOoXPnjIDO7>Nfw!&w-dTm`@^ zGLeMX%G4xJXF+23?-=r>wM#iF0^MY=596=G{rDjTB4f> zZw#-0snpNi*B2g&bAMRDd3>*CWv@UEzN>;|tJ z!t1BeQKa(aOirvY^^2*%`lMy<7D3}qshjlgXMEmaOg541$XGHBBkjF%hfqtVeQ{9Reh#P&T#?g&}RQ)>SwLA4SA zPMnVOM6dXhOpa@6={~Y-s1K{h4fO<0%_fy!wskf+HfzU%tblB8Yva9eaalY0b9eR8 z7U+cD1hZ?jT(0;xc0Gu{|Zbg1XkZnk13 zh;CjDCqrK5+cu>jzbr4N)oy40Vf_xX$l)%+o%bcj3VTmxZ`XLw9ymhf&fT+*ok*bg zJjmd$J+kWnw~g`d>L75^w8hqpA|O7o$gm>%`M`Yj`UWM^%GQSCtLq(yWjY9p8OBWS zT6(wrRznX5LMF3C?4&)0QuqwLhm=0QINdM54e-+{Ev}$asyQDyMUwMC!%C=8eCr6! zz13%;*1$f0K1<3e7{_uOLJ(nUchtTHPu)Lj?|@j~+h&P>r zu1*jKiK#r_4*KrXv^f8z`OvF~H0(%eSVrC3HL^yQ04_NnmS1l8q(Qz$vLXMO^eQ`Dk3v6D{ENrD)0S0$grk&KS=4gtOHPD`&x z3vN3dWp=3@8)H*w?R2uBN9FwVkhL77^{eKr87Q{e^-WXu*Fl|g+Nm*%gwj76hgYZK zzc{sZ9_{TP*RZwxDUEgHNn1w=NNk116Z@g1=jQ7I8|k}IPn(!mzc2q!!~I0AA9N3| zgO*fP4NS7QvT_`e=%KcDTlUUE@Oliubl8S^O#5};%y+uZ>IHwhx1-cuMTZ?ay7ah? z)2(E>Hls7+1*k*gv40#oHkm9l-Y&73S2aE^^>WS9WNt=rj_OF^wU5_qZVAkSLqW;7 zu}YR5-spuvwmhB-gW?>!(BwOpC#mPmQFen~Cs2VYfxYSPXwDY0IbsMV&>a&@74YOZ zXsWoRNMknXphHlC*KjCrBt8ozzlUeMQYQe;BQ>qSvnOfvkRAk)8mGQ^p1VmeBVIF=ID$&|Gj7yU{4w&Som$jm zE~%UW7{CLO{?X}psDwQuEZ^CfLh@PMx002>X4S^n+rIaKP;sx5xsGAY1Mr-8;_5Yc z?IWgXLK6Qv51#hGt0L6g5z%3@VCgk!{OouPJ-Nls7nF14Nqn65{N*o*r;|q9xJ>*! z#t0QKBWtXk(;DJ>wyDi8yLct|Wq$aU4ox;%UR5UI2cb~H;yOjA$48MOV!uA%eJJD0 z+!_FY7~{XZMs&V8=aZWkFjI%TGWzlHpRq(f$QH34U(!&Dz*L;LAfCK_=cy$E$^dRm z{!CJ&xFXM*#kbqaRZ{6-^P{T$nJEt?!h+oAO}R{J7LFYG_`GEhB40g;a>#}fu~jSn_^(kV8m9vGp+?suVn19 zLHfmb$D{ug2yZRh|1fXO!d`l0N-D0g7{YaudQ+}L6v^{hdVD% zcbWM4<FlQ63%6y!*6sGhD{7h+WbgK>aV2RlE3{B9Pb?;mQsdlDA1U|{Bp9Mwis-?mXK${C5t znQPokfsQ(umO;iJ+Lwk~zfUe_2r$&P^E{pX$aqnbzn?QmPz`;c0u}tc*A@ygi}!hnJe{Lp2j^i^{c_~M3)h?3p__qwPXTI z6+mW;#O379=@%~$RF9SINPOI;<*gr#&-po*Sks?(SM&n54Yq)2#+H<0e0=i)+tg2n zQalWOHU{lbi$l7aewl@}G0so0`Yiaq8jIJ|lU0s+kK%I&am2cT_!2?V|==Nf$nB(+>NX$1aS0#%jNwW9DhF)k{d3%@~{U zQ}I7 zbpI^34YwNhbb9l2$F5+(V0^g5!0e(`c*Zwr*81oKP!(`uefsw4(`_-?9r2Ovsyk9L z4LwSxuiUE1XCT8}_oSxiBB|1?>VEbbrrnbP-h-5q3W_{GT!5 z9{pb>4cTBZgvMdlFH|2md*t#^w7pbnv0pg22y5wW%`Pfs>1l~j*T-Yn-@s7D>6k%3 ztIb*8koEeubN+2Ku-fHV29Hggs>>A0ee6Q+GflHaZnO8c>1CT8TqwrP`aOnU;W6)* zX{HAx)pgUqew8$YlXNbV`X2F0Wd7I*+#7VT2w@)$I=f2Ka~qbZ5sZ8wI5O|9bDx*meJS+_6K`QWQr!G zgBPPj-=`C&PKePD|9*n&HK`f&y@Q!co47*FIwaR%GuqxjFYr>OY%FV~J=eHEc|R9RUf8nJy!6bRHb~Z{p>wd+ASmfU=sltC?s~yj zdGwCVbKim^=Cwc?*8ZZ)H*R)wV1&bI-s*q02&22#W>Bj8msm)P-o;{}cKUJIg(k<& zc|218V#i0c<>@ZSjnNu+zJl#egIrXTG#ebDv`d@Lt^~Vd@6;Cx*k1JE{WTWNO-+2= zBX9H|(r+4#%wwaCfr6YPTY1oPfZv#j9^e0A>#f70{I>UD z^`j^TNGqvycSsIMO83yHbW8W3Qqt1hU4wMaph$N&jKna&5W);y?|`23JLkOL{Rdpv zGc(Wg?6vn^d#!t|b?>GRMIkH+Go-?si^8^sBZYKgCl6%5Tad#Op8{+z4fJ{BUk7)> z+z3_}6Id+vc~qXx6^NYCruLbUW9a+jF=%p~d3fWJ@#wq$#Ndrbt3MqYdnUsPqXc>m z*-7isV$k~W$!RX!g`sLZQFLTvIAmtcFRF8K^ z0Mvl78xFFkLLKAAEsP2Og#}b@h*?%|j79|5GuL&}i|u_YcY95C)O~>_=8CL>tzF1= z1P-JOz6u33V5^8I+E$~opnH{Qqbk0Y;o{g;f-bJe9FkY|#RD1-Rj4Ujw;e ztxbF_b9b6!cL&mh(UBqK7;cz4^T)|zx*6Rqb!boXwyI#;fo?arvVnc|z!=+)g{iJd zozr64x@dx`E2(PdjKV3mmhIYnC7)jN)WRqd-*MZ4XBeaVN^Z?)a2qhP9-OQ4v95EJ z?Z=?i%O$YB-L|-ul+IQJKW>pYJq2nQ5}v!xbFx}UMN_Bb(Z4pv`Juo2cVo5ffLyE3 zv>;nJ=(oK-&n!&Zgc35RoJs{kM%3xyL69C+rxmWD#jeKLq;wdipI8cwYqb#i{-pF zA6cQp;x0sYBTj#T+~xiSY;Z)sGX~ahUMJ;1{PKVVbY?vO=+tH-b)Vu9=$=cA9swd~ zuNo1i>UxzW+i!vIRNK27QJpfOI(Od=PhTAUAOV6Ho!s_K8h@vy`caL_ZIY2aLO+2k zUd;BV3Y%bHSG^r#+H|JY50w(RJv|T~<_$$hD^FMdmq6HCmWd8WIIH(@#+*YVbh)i# zxyGfuslbfTw?mnfFM#E6>-PcU(XWx-`IZEugfv3~5&t{>_+2rP!#4WSt{7N~3+C$N z=5~b2u6Q!nm|?ALtmS)h%%r8}*!PqKM;0Gn(U^mlqEbhngjnGjx#b`&#s?~AXJ!}j zEhsP;;SQkr5ZfUYwoPo1J)xz&mMN!Blvwxpom!lw<+rOMygVBCbKTAr^)WAl`8=ab zn&qvjhMb86;A*D4c4CdEDiB!l`)TG97n`K5ox@JFN$*xm!eCySF8cP9D(9I7X)TF& zBO7sg7?9~H5EnOdT28n&J!MiVe-PkU(v(${@4!mJKj$o`igEt)XfVKgA9oe{K*?^t zr{~Qt8(Zl~iHBj^7$;^FclNO1Tcbfm_lM+dd?mL1$n>I?q_L{7l$4#M=tGwL5#}m( zqQU|z&GP$khz+K&Fk-CZxy$_HhOb{&|B9|0eIA%#C>4J3#`AtS`8$TW zcSs4{0(ryRCirr)dV|ty&2Eul_so=y)a3Qjt4hosOlhnWkg+{}_zjS&r>6OVJHJkI zX4E2irM&yST?o^jI6%@+_>{}Pa)6<5QoBZ&ZFq?LEPuB@9Ffz~l0ZpI;g^#KWsneBum)dvHjnL;4(()~x_ z%|)ofK&*(v{SXa|fxI^6>JMH_?uZEpoC*0_r)tp4q%5k?s$TQh5ru!&*@bPYe#SEi zWah|JR41xkkdC7@{X)aR-Z|EmKOeLPm7)N!>FiQB4TSytNA%9Q5a>eGrHosk-VTtL z^K$>A8aBIIb6(*{1#AAsb32ts=rHJ2w}+8z{Hy1KX(AaeIt}UjKsHuAJg)Sx_4{xy zuUX6_>K|m1e^XImEz0&=Dg#3=>NYyW#;QCO*(%KiX)v(#412A(Cw5vRUQvGW9CMq< z)=NsEg8l)Uv*qRIgikrWOoK7rJ9S`TtW7VoUeK${QybrmLRii)Vu`2Kwg$kVLQU!m zGpvgl-lohS_?xO;RcF%iCsiGzP{0jsIisEeI?|;ZQ&MvfBo@}O-4(3$jo(Kk(?4W? ziyjwXi;Hqk4k+(_!+K@A`?ua{!SQ@`BM}JI*ED!5p&I%_sgU9(P>210kkD)CZ_g`;u-s(QCIor-OyT!tAw2w*9 z+n%pw*mb(SA*ZO>uJ6zy=l99d=1$Ayb*<*VGRFBaZ#|hDDTm1xu1) zYvEtYm=o$JNx8wbeKC1%|6u>S&U0|n_%`b}5ChBKwnTRyvyu57?;|rorzfpXo?vxa zVS<=`m#(Nx?@Ld3U?IKWZ~tFVSz}<1?KY`|(0;b;nsigFm_i{lu&7%+4@kJ0%A;>k z+}ce(UBc3u#d#5)N*AXvm++>j^>?$`X()Vi_OSjqPs{?vaUZT=@k-O@(c7miY;0G) zT`<3`Q(+pOzUU1moP%zE_XXFvh{3`2U<%CWz5O;;G%nkEyw;ZKy{)qLoSfIH!^CJQ z`){n^JNvbf{PwMrtE=yR<^457o|VSL#g?z}b~fCnrx&>@Ett3x<{Sn?dg2hd)|{~^ zWa<_1anSM65}K$S%xf%xMe8b&Fp9pfAf)*oM`?oDNxt8T=Tm=$o!{rsGT$!!Q6Oup znlm_+f`s|~MoE~);nw*sKu{^9scqF)*JL+ziQtxHZhm3Xn8hG_!%k#ci%%DVf*9{~ zkc^|}Z&(!z@~HVrtm*K+Qfj1ddEf~*?m3#|TrA9?Idn?9IaJ3J^N0nekfT#XYX;rw z)z}KF2#z&9TRrX35R`6yueFw_BfL3`_UE!}nbui2SA6^+a@-;1W2(>i}N2`;$V<15GvSMPY(B$h7k*J_k8IV)eJV?^Tx z^j?`i?EI^3lfqzefQd|x{`)sV(v#39bMB^$2isIMd?%~MWQP7S1C3T{X@02>qQL_( z!3#dH^3k+`)*VG=yJTU6!EBFR2Hw;H)G~B_f;SKPYd69sl5vi`1>JsFp*(!`hyk!a zy!>=(jZd)kAaVPQqlB%Ff1~@ym0WH`v!~Jfu{+Hz^lY1`D2FRY6Fp=NNwvBP`GVuC zy~#b|<{x!U`x?Qq&YE88z%+F$$~+FiN(YUUJ=~5HAJk`;5|rPCc7ME#%5#SjRqYD< z4Zawb_>kG4p=qDu@%UAKJ5aiOMQ$+OM~NSVwbnR#$w{BJ*$`U^SHdJ;YXE$uQt=Q>E67IX6F|7M#7jp&yrimkNjJysgT$Q%j1TMoKKRuvBsMw3a;-FQk`XA79L+<7ib7=xc?GC%<@cDWd zHCD-^@PiWuH&Ak_*4uR55|P#un6PvdN2qH9uzfbo-x#z91V8{I`+GoY+Km;xk{&F7` zSFcW37)j;&04~XlPF7P*jhQ2B^mx4NEv&FLlv5PZi^or}LBAH~%mnlj%UC6LA7#d++0T^!pOy`-XR`8B*0 zxMl&bG5^FIdxc7!-#vU)sU=81(ZQ-Umgy5cu2skgtZh$Z9=jAte`Q-0prm5}xO!on zkW(|D1I*T3?KM|uX)LIxb`=!nuTwJK^Uuj8$hl~AJz=VnIEzd$eDNUp=}X#M{*t~^e4d|vmATEKf(_$%%OqnDz=mrMbe;fp{j%+t&2HG2YGiE0P$GCDjnz1S5to8`aTbbmUQ zOjJnp)$vKi8w|-9^B7LLN;tV4P4f6&$}=rfx=fAQewuLIhV`aBCyhExeMS?ik4l+4 zc%rP*EVqzBo~So%%&M_q06(y+!-EW{U(qcTr)@v0I`c)%>2~V3{BmEhx_p*bC#cR? zIw6R1KC4|wxTg%vt{PJZa%qnYvH4fHzv)XK~bLktQVyM-J6tkJs{N|Yj`Cb`$_t#`-O%=&RpU* zbwu7;4Q+J85)U{mu?8c?G*O5B6Ne*#j_6*Es@}rmp0d|cZP@PfEe-kl;^3<bOCMkj+>BzVo7VntIC>d7)u5?JV zsRD&vw$90x+v^gu$KnwP;`#_5$DatcW|KTX;;qM&;Iv2;f;w8-$C&{4)BX*OyLZHH z!kv~JCp)-Y;{U<|5LeSG#&6mr>Dl}gd8L&93aIJ&OzT_Eq)JLOcm2>E3ERzED#TP& zl30W@NgH?x6c08x733!d+;v7D`1R!x0=Ze`8d}E?P_;c&r}%+&+kGuA{^ z%IzWfJ||m_-6(~_=-Y3< z_x9{GHO;86)dNtp!wh3yHg-3Ec{PW?BBdPrb>y}vgOLdZ>t7 z*8S>uf?@M4fjyK6@8gf}@scA~{4p{N?_1Djp*qi#EByKDGN;qVNQG!E`-|{yw*2qx zx~5W#OD2;)|LF`@^jXFe_>=a5>FoN;8?EvksI6UTeQ^H_hrAXrQ??PCi8;_mN95Og z4(%n{Ag1ZZ4%pLWlcO3^6%0+_du3j6$`v#V81jOG@|_RBWX;SgBw|~4Ldv+1(Sf-t z-uri-iZUr;qEsQ}lu#2B?meUcmmVE^)<-6CaU`&yaq&j4WINjj3^YMoFu-M+`9lXS zdWQ>P_Ejvws9TR8+VvSHU-1()aWBx_jE5m1zx!J9ldCaJt@_7HUS5&!0hO)u8o<8MDcR_>F~c(8o>Zr|tNL95XqbkfBA&9BF|*2so4Urs z@5SeVHFs5ABG^KnQ&Gz+@F$Kli)L!8GT?3xcK*@RkL|&fJ1BsU&5=R{kR-lAb^M#B zv&_8fqmG{{hJRi-3vuIY{G$y5Emd8hwJ-QdPrG-!$_T4+7k%*l1Cc`cD>Qp1xU6|JYc(x_Y%gUIi7B|Pvp{Rg}1SmgMGkXI|gpO;$a zey939;_9*h6Q2pLzzsvnV}V!4Q5Ndl4;~YWarcyM5RhkH4v7Xo3hTvOspNXMSNRij z1#x9Rg(SsX;VFzj9YkMa^Xm>b*81%tOwwF%Cg@^w&d-KVPdN4-)^R!L;$%8Wj{8C7 z$^HxMMjhLxTn@QD_3%WG`X~Ha2nMs-yWfuYvtt(Ji*(E0Sc482LmCEBG)Xe%Pra*p zeYz3>=9*uGH<f*!#YM2&WGNgu$2+IaW-`+1r*k@GKALPzUEtrpM! z(f|Af38vcUGQ)##Tb&(B26mbqN9gkcjJo;Fr%aWaBeyr%vs!lqb|+3_@N3!IY7Ngz zIgLl0qL96iO$xEUgZf}O`-f?kuopEPxhwGW8KX?6tSj@!T2G{LmvRCAGsc}}XUD=j zIyTT#K>1{J$<8n!0O!fgw9EoHUFQY=({~bHMrQB%Yiy{sIu`oA(txtzsBYlu1oX9_ zB3x&2&rxFFjqOE;@6}Aa*2uoJlHuMDSRGY`EtKVJ)Pv>Nop;rG?_pjY9n02+b9^C( zHk7)FdsAy;@NydD{?WF)1|N*zUeDcyP`bub+2cogJMokkdxfo$X7Oae0`{w49Arl% zLN$bS%~F)qN9Gf$Q?W=g3IYOrQfk~M_%Q|cPv~CeQd4ideCgz;))&<56)dOc<~h0B zYA0&AIL}uArFDh*)gfxyTXm@YxSaO?%$dJ`FK&akFxv8a_Jk$iZ1XG=v}nosY)g?? zkk$48IfRRiL+wvC$6>#hx2kePnED!UpzA%^>ZZd;)L3+%#5qxMG^(kVl(TUdR{;h23j0|UdIb}g~9(^PrCbHVb)(jQ)C#b(%y+QT(S2MV?=H{xzBQVQye z!j>AdTNgTwdF?tJp&)@}Qm^d;frPRv2br;ybuIWkW+b03FQ6^ng#ajqI<3?-reb@$ zX`@OiV4|}Dy$HlXOX?K1s9GB7XJ;CZT^6$sF!2ra#+-UPCsXE`q?W$W5(y zVuH%gU~n7Hs{7srEyGlbLWIz8Hqty3S2p+#>FZckWY~!dnk!H-+E0U&FA!X^tx_ zMvKk6U(>NgMCLOM_tsk)dAdKg?Pp#N7mg)&Rl<3UPKSSN!-FG@YkbaIHb1}D5G`4w zdtmD1Fg3_NBYL^!zHAraazF)HpE+haX_PmS53yk-pJ1QuAeK>viaG7)gnsssRtWaN zX#C9fbf(Dh2)>shtqpUOQAWd6J$96IPCcP2iC=Ll7bL;nzt~r-s{%kd1+E7fE*DiE zCQ2OOaeM25hJ4gI*~?SCKy|fp{_?=mZWa2qzd_Q!OLX1%Phfhxz1Z|@qtw&cLZtb9 za&Yj+kD{#Z06+K9v+37YPtVML>%IpxCK=pbm@@?*1bv%_Tr5D|SbX!K#`gVJ((Io* z4mm+?lM={n`&3-;y&iKzX{@WXj4=H(5gnYEqbRh%8 zK@?&PkT$22sn3n|^Joqr5ct2hKKp|o{aPa<85Q`S4?mdQbmd?7{`-Ge`r0u1-{yjM z-Jkz$hBq$j|6CO(`M+%ZzM z2Dj~Od9SMtKw+SlNaKoM9)5h+bQm|_X7$`$;t?hD|Mk{t8n?nQv7*bYGo+RBPYgh^ zE%t8^_ag97m?Wk){h^a%w%6|e`Pj8>SErj@?1K2>^`x=UkG)Ilzf^di$_47`Id(<= zM`PZ|Vx9ULd;H(lbW>{cA6K7g*1!cff|cQF`wiG5pLWQv$7h)0f5b1&fH5R}PO2e+ zq^By(9(%E+Yxj*BAH)`T^xfYr8yY92(kDM^?>&>n77~W=dyn;DxNctCka9dbYCk<9 zxl7}ZNn6HvdDpcb|8D3z5z9X28S-;NA8Bwx;!$>rF2Ti@afmCq2yox==?`8L)jXJ>5wEYxI=Uv6djoqnew_xm)Wv|0SU;gjFaG*z5Po zIRB?-_%+APREgY^B_o8CJ|~p|I9fG0c#5EnRw_l5O)I(eJdgfl|DFj*q$3hH<5Y0W zCzt|+IU9^E%*6rl<_C1hLBFzeL*h6$L${gv^LMRZ>s;HLaY{|&i*G;tl2v^37zSbb znMCgVD3kl;Yw>H5H2u6T3H^67xZQZaK94y`#`2NMwvd`clQjO(?|r%@rF-Qr&r_i@ zDcN=c5+{2cQ@n*0U*-bL6dWGNo|N7$2hNe9on!VC1h(dyOje?V5AEl^`P2kl-`C~Q zf32PmW^v@R=|=XVIvPHf-c<=MiQa877734SnsUWZa0}sTJ~o;OhDqOQzWR;GxBH_O zYxi<~R(=AzW1isW1MtkEvcA>};ws`KH@E-$>ch=<#q3LRNKd0eX}nbk=?4$&1hiwF zeH2Ot(S?Oc#=6<`h8q9?v7j#ec!NK66u>B0;*&dZ5neb|Car4{pDfJK7J zBg`3s48>lj-!J^&%p)Zu8@F#46S61{FfCD}9;UY~xQQ`eU^E2rj-*o6T{+>;NF{+N9%v z@U*Jcko)+l85AkY1akuAFyHgl;yz|*p~LaIH>IJM$z#}Qw%{ETuEx#qbYl?2#zD{P z-90*-uryAGOFI-ndDG!oLgCJd{a7~6RJGaiLF?9RY)ZU_Cd~s<;#AIq$!22OP$ryr z1Ubk-G~3yaX?hYsqcuo_y4v+BV8GAwB<*b^|JiSCuRxX2ISWtC(|=(B`FMd{?E|_k z$3aXli5F7wBdE<_gI7BR^(-7+qV#v)J8q2);T%%1Mf_xF9Nojjo7kHg7C%~B+25mdAh*+h#V^cAjg9b2@} z26*ewjmK@i?=5!j#Cayzp%?#tD1{+&+NaSKrywWA;E z(kwP)pp)s}Z?nZI58)scf{LkuOJP8QovK`ChV{oH#7~{#}iu`7ABsgZgNMPP^60_qJ z)p!Y)4Hlpi3w85t+4&W+4fFBC9B@OEQIAe$;?`{4bc|Uqrs)`PcT&zHtle0y5J9S% zJ0Es1*y|;IL07TNEP|n(GJP+7h9=*uO9w9+?u|sg4x%L4Kzw07NQ}Znj0PUoUsZ`| zIfCX8R8++^X$39tXJBRKtb-{P|REvzw-a@ zeS*0>Ax0@lV3xdYIAhkEv5?PPExpm*qF|#UYvd9Bao&AZys%84_XZjTFA<5r9bcrB zM%GKq!@el`)0dzLGTLZoe38^kZ3~KC)td0)K~cx5q74Vfws9To4KrW^#Lz4vB4|=r zq8fP7!e@rbnhfR#`?=Q5r=1i}pTP^ttb=q}hIWNBB=0He?01(4WO|-Ay;E{zw3`gk z=&IjRo(c^fXhbEuFZwKJ=-wWvUROV=#jP1R=7zOFf_Z;B_#XUZ6RzJmUOMQ~FJvHU z7TD(9zS^u&%CzNh^bvg&X*;JoUiEAj+Z|d1LQab61H2J~YOCX5+sgi&uC_sC7?+*d z;f`If=538kuelmSm40mN=Rl|T#!XB*1%uOQg}#*_j*}04R}j=ruSM&*@XC)235`7)o*p5%C12UvIKw46Rn2J894_?rdRDYG+IXZ^ls_gmafqo^-cRumH-2cavKtev@c&)V!oICUBF~87Qpvf6{qvQZqSQUMz-5vkx&b73a_#%Q5ZY7|eaIhLav$sAX zSKB(zA7ZtC_!_=7m-y5WHvbnj|*~k~s2cP0d6Gbe-RTc^C*vSeME>2NT1V7b- zL5Kx2=e0qj-|g3mh<7N#nf)gL*5xWojt+KYn<-+lO6hw zA_NfP-Df%G#^zyuf&q`Mh>{TxH|VU#XOcUV{ELL_Z^;t zk1|j6U$L}95X+q8ncY@PTWDz z?UIo=Qa!I1E^av5X6diu=Y9Ir)pFLLud2D3%j#=okiy$_dsPMAx9`FWmPnaz_;FkTl+_kNCTbo8{0drB6hnkHFga(+qSHnLebB*PcVwoM}Ep)X9Vot znn8NjW;y3j&HYW9GSH|28af9f~`#lqu4#23UvP!y-X2Y$` z%S>xMHJf+hCv{wkMIEO4|&hfh3&S>XJ33cpO|SE2k`UF;gHt!)Q4WeRmYec zPXa?y{&|;!jXMuuIEYAZTx*VhekK5E_9M976|?GVHUd>s zYZ=Ou#FF6241)q4s$k74e_#t{>FOcdlx<&EGC@y{e+ZR<-ucO&bxzm z?||mA2`8MQV!jmDMKdcS8H2c0(Bb4r2{u2Q8q)Gv>pGlCjj_W@roKRZvmPN)2+gba zcoQH&{NDY&^8!|1#E9D3s6z=2OswkdVE0UztTF@%-&P`OEefE+Rf(=;A9J5=27n;e zJ&eX;D1VS|Ue*|d4&$$gQ+<5+pN`>hUw_CmiS9JrpU@!@*(+TS)2R^Nj>(?|)^Ege zN$9&%3df@DF{h#*e^}@#7*_fI8LlEmBX->GPn8-Jw{`w2wfNo8-~q+ciKtUb4UPcs zBT^0JrnH|MaFo;O5RdLDhUShd$IMInAw2*E!(DUvmO%~;Q2ElhQA9{&%@$pwOBJ8f zlH)xl9F()=P=Asw9pVnDc6Z#%r~r!Yg;iEsv7`t|65f51$?3QEGmOCh-nIo^0IR{} zU>kgPKMhS#sJyk{e!fdXqEGt1x4_P(P*E3e_qq%s%lL*e?0Plbv~|Oj0xvGj-*GcE z1FpuQIH1OY;xKlYqlPR$P)6PA&5`Ah!|C~ohLYK*_Uav2n!(P4zS=vREdrvPOt(9e`x8zM0~Uq0XS7t^Gq2EzkSz z6e7{N>)VG$(M?BE?^4fFTGHoYQ{CC-1Qh9?cE9(XR$__w1g{+AnN~tEsJq>1f}QS! z9lgVbe2k0dX-1yh%;89R{rpM0*o*?eK}oQ&5bkC9{ z?>K60UQ4|FuU3{TJi6;uBDe__t4}+ct`NO@=gVpww!h7+T(N~ zz$nD3EpnN&B#Z%v>(d$?#_=^cm`N_k|H2@$C!Uk(_?{Va4paTM_x>5L zNqlnXLS`!_bh2>riDIhmlQPf2>jv+N%2PG9(~O@vTjZz@a~HU7E{ehcv!U90Xw(W~ zbxVqrG{TRpW)}qMayY5^RbZH>e6p7=JNev(8}IUW*A&#*!DE*Ygi7`rYFJiToL#TS zoF8m5E$6$OIMgaKp+3U&K{Gu3=Su=}X;8Acmp!qa_Yrc5qKrY7P?$Y{T+k#%JaMm2 z)|kEpwdG-QuNLFAltKqn56905sOU`0^ZGyv}7-521u0BzERy05$cPX4`U~^L8SZ30f1vfe%%UfP z+|zkk>Z#{k)je;@`8bmnjmD$uq5Af+Cbh`O2i``hMh`eJEUSqH*2<$v{wA~e zOx`7sgZpvS)jU>~oUopm^Uk)5Rz^NkIu) zK6_wEq>Pf>gY5d_{*AoJ4c$c=$-wJ4;3?S+f5^W@1}Re-l*SkJ{;bj`df0su@}s=N zsAd+-@P3>rRZIYrp{h2bkdZ_+S{E8l`>uyz>K^Yx@vFU_`@+>DeLx4xu}AM7!5j&| zI#>QOz#PcIbLDN0LoEZ58t=IwB){FXUuENW;P=mF?(~PtmxZ+NsvXD?GUM02+H66^ z1HWmzs6ngQrI?YSG9S~t22V{{WY1pu;qgBNvzoD^7$>T)n{@<#2wTBiR_6dYHM8Lz z3foF_-#I@G6QI(4IzZn(AYiNGQFFpSKGjmW872(y+05YU=`_N-_fmj8dA15yGnv8d za=&i~3F_&!+Wz4<-IhEROihMak;7LvFz^49gu+~1g}|HDHl_gy#=Z5x6;sjC8Ew~1 zj#8-Dl#j#ulRn~hJsEbXv7IJAWoKep*VR<5)T*aHHt%70zdE*qJfhO@mzBY3EW?YZ zFWBc!OQ`hIIIr@l#QaP`zo3~Vy6s9~o>q(B>V^cFZ8sHcV4l-$=89JTFqiOlJA$>!YK=IPOXfhdP?Q`#Zzk;5w=zx!jy^x+y$F&g|2cObOcZm9XL zSE6ZepSKFs*5ZzAl+}>pOyl9CpX2=9tMlEyt+e;?f*_fY&p+?bX*=>9?G5dSp0KXq zpRct#JBh743-YzBP3tBDf}p=g%EAq5xu&`k1B3g@G~CJG%-fYtE&TRz3`SZ}l8AV3 z3~3~!4fXRVDZcsq`ZHaDlt%xBzRDsjeZy}1zVKP_hu1q7jzbVTEX#-P(3am++@ zl^V;R+3H`fvaF5$3`gc@7p-5s57ln$BgoWIf?+U5-5H|JXyheiBhx?lI?BVKmYOiXR@k;mIWwjF`CCWz&C3~t~K$xJd zmE>*-1E*s$&iYq;gHH8`>vCxKKOFFi`ESb;LFH{GV-|T>b_A(^108UQzUg zO)%6VW7vBz#tQGBT9g9jdz?2!LjUoD;paOeVVSzz;Z=a>p(5`^KZ}X2Qr+aon6H3u z{U`AH@l&x>v6$OS4jtxD?6!CZ%)_n7XKLV?ySgzg?njG~#h&+w^z6nS$S<>?4xW<= zgU|u%;@CGR+8?GoOfY>!3sGp6Fhxe1&ID3zS50QIw)^w?-?f~78|=(o%~@P}f2m)O zZZx6sfnek)9_4ftq3;2le|`Nuf6^O_Z4_r-RGxXfG-Whq;etJb<0Sr;e`dWP6D50-rps8k)HI>|vc(>>7)}j&1 zjRZfIzAUV%qVtt**38k_TcJYXQ|}}ReU@D!!2Kg(H0=!poAu~1Uw&c?cj=S%H_!M* z1``plvAzU(b>x?cNZWXo8T1iIDKM@TAGp$;NW=9;@#;RvN!h-29`8{#Rs(YmNeg=4 zJ8O!j9L*|c>gFin@A%;%waSq9EHX?&Bii^w;} zgEr5zQH=_SNHQ{F@joBk@_O(`i1)C{^-K&8dy*L*{tgvmI$T8GB+7g(f0b~7&$WFXZwW&na+9TkF_g$|mdbi2x>n0NV)0l1 za-(NMmCJz^(Zl$rjE2y>%Qm;?RECg}rY3L!RJqimdHD;X@IuVGlq_iVi$ zi(D+T%drmh;&mDmw3)1HCabtyZK{NMe9b^Dw@$D8!^Afo186f#^FX_Zy8zfZ^}xjy z^=rgIgb+zR>nXHagr5Z%m#ix_az z3%Yzoh1LXxHksq8G!>>Iej@vv8vDi9z2jV&*~*TBqSdOpFiXK|9w4`=TtKU=vtDEbjelk3|}p z$8doPBGJ8Jyb&j;Gtw*cwMcX&_p-0u%Y0hH(X2Jv`t+ezE-2#42`zEeU zLQ*_d=j}br`9o%op1xz&o*s`x(>(KLle(aO7XzTXbsS{qfCTMd#V08Y0jJ63WWGC0%TFAqJ!;pOr9gNncO%gx5Bd{mW1KSnIAf_CytY9=pJ+klH`AuQ?u}H z&%LYOdQ00?w{p~gHPng3yHR5h+=j!f{+#y!%gY)n#k{vn;OhBC!ZTekQq$N-{rUQg zrkjF(d}=PQMS9xF;!kRSC0!PO8Qr9e42yYMf1#t8HetQq+KUfQW=mq!Svec4wo}r`(F8Pu5SYpAT@Z!;VZ>MD??~ObnYjPlnquG<(d;!5 zSl}CHluv!F7_dU{d?c+ZMREX?Yc=bNqZ#3!l-YE0`vjOJ#6mtT8QOVbMiDeD1Y>!j zj48J-62`_AcH4JGlXv0Uk@ZG@`T`vL@)Bx8bvYH~t$Yz3_s%AzWcKu^p8Qyt&Sr*h zC7;sguhBA$N2rGR$@=cA4158%x!YLjP0%0b3>_=_@rq7s%>wYSFGd=L0$*7IR({w} z9XPq!>((a3e<(0WrRN@xW7;Iuqvsz7CX}R9>9X>7HCn)au9t@O*$nupQ;JbfCEyuk zyE|K!RW>y!+F3X}cd4y)wR5l=1EP&^abi7xrIXye?)Qn#-07(P=VzDcQ^#Hpw-nuk zn%9?V_4z53Y&Z=naR>i&YS=jYF-ALx`yrNltuv8a>$L;vi-H=grTtOWJn&Hi`I2lS zhlbamY`ds6Y@zL4kl#6#&$M#6SwyADRyka(ihzkEi3yql2Oqj)djs4`-6{hmyObaD zUu`dUuUMCc_VTR(45b${>`0WceQ48SUXSV2<`m5iNltIns|<=bTCS@4fq;wdu3q|o zUg-O6JdjZ);M;nle@|*;9^W(4=Y=Ed2Y@ZXT84w={d)w#9$^nK8G3eRt`QR|?bGA} z0s@qC9iK2HSb_%bK~@8#)}`?nf=52mw&nc=E1$^n)5#B7S+^gXrwg6leRBPfQ_h=x z`&1CA8p-pHk*%Y@r?Ddbr;8iv(Ic>Yv!?x4s%l|(pLz{)GU{8vzFgz|=G+b!=2Fc< z%+b007ly_@J8vrouvtR2NV6G&9rH?uW!vpMbylthZ7HKzo_cvSxFIQk40ZVfOT5p* zTSv@Kg?ffhDYP<$&u{v85hPPM#G)~|fu77wOHVU>{i!|O!VkMcUdt-J$!IET49SPa z)Jy64`fFPZ^Q|vp_4QXSoZ^2L&XVO|$;ms61TfRSwbl7pJ`-iw;3LA)DyL>=Y5Ij7yCLm? z@kUvL>g~jnDIN=_&`5KmL6!tG`TgEmC3PV8{lNYlC$02OFILEv zuZXbDm0?e^l_#Cq1RomqLtPD;(_~ay&F5&JykEInHxPo`?A|HI-u!bYclch@rb15_ z`xIAH-d!=tTpBnVRp+*nV#w+U*All8s&ENQNmm)eM*SxJhf~wb%F1jbkpRw9EZ&bl z$~0s%v#yn>xbY1M;`fg$ifRKDzp)^$B>q^U7ZISbxH6nv2`pcl#+b*oM1k-Ly&$eZ z5qF2HtZeSvg%*;ML9tUPC6nBoW4jHu@?X}Krrev6%qc=pD^{(;NeNXI#SBLUb^aw&jR&deO2ANA| zrg6hJeVBzX0Yh@(3$bb0_0C$pvyAaDt3(A1kSNpk?G8FT`qGFeD0o)>h25*PQxz{a zH>><|IPM;?QV6Nj!Zc%NCG665xqfA%oxO1!HRGp*^VC}>O31kA*aZQj)3QEOXn5z- zpROUJgUbP&U_EmqGdV0`^O^%-e4vW%1+Q@efb9@-$v2+e=jtsiCv7Vz`u^}Im6XlG zf~YY-C*;rqkZyR~YGN@ZhUNei9QbWqxy=c>)x;UB zoN}b|xb1xTfUL~Cbo6lFzZdq&MbgRIg<6?Eu@U8BZm$bRDwtFIPxz+9@Z67iSR=)0 z+IIZl$PMp7??Ach)#j#k4OWg8QM!Vg8yR4}Z=c6~E+n-;Yn7HkFp8Amr8!yYB4Gig z0z3CTzBDF*>_XvEiE>up2vulpsKOftG!2HT!*F3}&!3R=``gKXk3Z}t0D~F-{=;9D zA+yIThIX}8+8k>&-an?anXFUp=zt9bN|-*5%`;_7I9SdxWowtK3O-Zb#6B%a>#q*1 z?F4u(R#yHn0z)!zt&6k>wS>vJ$FzwVtsy>Vn;fe;2DrdNr*cPU^aPyf0N*d81BZ8< zMZPo@r!}@zm}K|f^(+KayF%#e6|D?!%N43s8>UJAv#^B_8C)YWW>nXlcuu3J2QezO z;RJGy@~yji`H%Yp3X3RK|Ahqzwfp%k1*E6w8-Pp9(mu*o{=8_hR}PPO z_`8m>w5-_Mc?{}&lHU$?T$12m&uDwKSxf~QVr6Ea@H{#cj8 z@s{tG4c~RdjxC=zmSg|UShBkcl@-_TEjT_VH;%ZK!A$4=H(ntKh5%|!DvdPaD&?~B zlNg!|w{SsBFhiVyZylqQADYz;CUX!rZ=l!-tu`tsMSOf0X8l^+5~RY(w_ybyY093L z>y--3l7?@L0{6xmAR(^t&t;>A$4d5{dp~zT$(1~1nwY$Q#7IR%JaTdn%HAwNsYhnA zjgVUJ^Xy+pNs6ldQ+Sk((ew}}G&bqAszpmTtYcyDf#0e=As@9$II#*h6{;W7Requz zXhm?3$CIok2qf(`Je2ks^S*7-o4-Nk=wFhh1LtcOfnevV7 zxBH4gwcS2R-92j-FfQk^WO>AqZ^4JCNLy%;1|KI}XXIlwK9VLXY4R#xeEXPl{q%fbGT-e)0-|4{ERl_OVr2^bY*5fxl1E<* zdEN(!aRH(o$H~Z!C4$t}(*8 z78^6-i;B$2 zjWmm?Bbih+&l(?woG93Noo~t9x(khG7{RGM_ry>Af-!lh z-dYaz@$m@l>&&bY&^8wGuF+hD4?4Q>uiKRN5xNeXkU=vmf=|xw6A#4>m2lG4kxqD8 z;3n=%@5E^vY8lRbKfg7|s;mKYkC^b(MOfb>RC)FbIkO-@Jji-GQ-QD`^Vd`YEfxE_ zl)`rsCa+ysa-;l3i|6{sYrqU#bsk#cM0CL~T`|UQNk-~& ziWR|i$830mG{7Fr^@-arB)Hr&%5;@-h?xv-{8WTKeh5v-GT+X3r1yJ#Cvfu z%b8QWZPLtH=kztJL%9^V-cl&F`IZT_&Mqrf8DM!%wCuojuBeRXsJNg18hOyFx@vig zF*mE<>n&z;P1lwsm#pVP-b!ebS>UiiQ+}h=LH^DyMLsVu-QpQZ=-jzj#Za|Y6K5=d zU(P9Q^!hZ;MMxGSYuj&!-x28w^@&6oR56S;RlsDq1M{r}i|>!`T8rcacFC>nwW zcY*{BZUKV3I|K;s9<&Jrw-79NaJqrU-Q6{~yL98wxZF-c-g&;6^{x5V+ADa_%0o*KW{w? z>~kPJ`!LMz72~{7?R1SXF+6+}HliM{PO$0Za)#mfhB#=IHWE)*l|ZN36W%OQ*9^gL z8e8DrdIa;-*T89x7o2q#ouN)2!T?z5i~E9tY6d=CTcJC_Ey2!~oqU1< zgB<1LqcFC<6I8R;tLadOGJwAiG3K$$7OBQ#fu0X7Q5~;E+Z0vtG3cZurkfS>hnEaB zmr1f;Vnp4}52Blcr0KOPRjA9Sm($Dgb7i`fRKB~I>S~tIP0dHkUQ(*tn|;yOl@fA> zs>>y6 zN^tg}?OR3?i~lY~r8>cbFPIU_5{o>#MG$0Ej*6mOTS`>O(Xz^1{!X5g(f<2)TE^8k zSRb)?RDAo?cWn5F_1EMGF-dW>(4AHl7KrP=fg&Owb~rpgd8vs`LWfDi8}}I*hOLbr zhWrhqixTC~h)}>VkK~4sl4!Y3DZTOKC0oq;j)*4{m)gT{Z&K)I#oL9(RA1qzbczEZ z$W$nn9j|bSKT)=^woYM%l@T;KY}O7EaEJM<;zQ_=Vo$o1O-G8qU}iP)EMZ-xD3cVS(SR~W z6I_sXrWynI?CV#%1nJL54mA@iiG)=6tnEb{W$)&L(`;P(74+jAw+$WP1rd6)XQ(nz zX57OUlIcg)^%=p3#N=CQNWBX30L-!S1_pGJ^oHB>YGL4a&}$Z1U41=JjE6h-WzsHv zI0Xtm`$B#}F=>+uPz5@3qX%!+g^SeSb#qNwFMv-9_Yv_5EGl@6U^zJm(L$NIuikrt zi{_KXYd{+^E+gw979vJ)cZK;ZCD(cw8U}!JGpKzp`e@_%{i(86wp_k;gZbk2j_?&d z2QBM)f@n0JI6th71gWCok)&koWn@;ER=7<^Yw(0o)0}g8+X=SyjbIP5xVYBlnrn*v z>$~@1*xy465H)DgHq(&k@Z8_g7b^Yi3gQY#o#OFP&9!%O36C(43RySv31+$3*d-%H zJ3jhwo+9cqMbT6r!bRF2NQUrMg_=4dxC#Ry1od0?nv3TP3QsBrU@q)H(?0z!%hZ*N zH`0)an82Ff)7`4Olfslq-Dxg-sW5s4FHn$`?KY*bi?<*PHoU_UCVr69u5sDp0nChe zc0)y6CHGO|^9zz5qsfT7le4x{Qxf*EyFK&gVVQf0JQ>2Oj@P~oJQl*7lcqSbE}IT- zo#Td1k~+I^=vA+VTPnRj-T4UB{lKYZp!9Get+&=3@-8-%&<> z2{^wRJ4BA}ldTw*8KI}4F|0bmCi5n5q30YtII^P{rp@^oTUO?Llla8XS>UHaYUy1h z7Sx{$84*#`s*Bp?^nhfggUngB!Y(-_Me+K=LKqRR-+g}z^X)Hg+zS1JzQj*u3U2(otw z@DHUBi$KZ)11B*}L)tczYIBOU=y{<%F6j82RC&8!YvBP8e2!jrdJ0w_gN7keevOdp zp0dX|iK;nhuv*+^2Nzg4(JFMQJ;wCp{4j1(@Acuq;i{ymvo%=G;IRIoNW;V65*-gO zuZGH&rJWTBq}PC3f1m;mxoovT;Cwmh?5@o8sTRKRJ9&=o$6Py`oU)M1!52sFH|$jo)lH5Q=)R5O$o}pmPLmiRo`W!Ve!vtqxwOh(eLI~+Q*=Af3GDMCPGnECw@RMdyns^M^13Zd>{#K( zzYBiaa5OQGo9JbC#@>|1H>9zc4^#C0sQ9_RA}0fpQeU7t$lE1GmE|t4<7YU5WgW5{ zamVe+!Ud7ou%WejZ6Z6LlhoSRxzb*qoG)pim{D%Eq1I;=o1N);H3mG}bRoM(C#{kv z;ZCvdmrh4v%983MG^gSlVcE9r8Uu{SfekdTctfpP3`%U$2%F4gC_fK6EQd7s4|ST` zHEu@OTkx}zyD{9NV75NO2gxEJIkL-qvMl%{4Zy0PEZ z^z`_#@hgAh-o(qxORd7jmUkK_O*y17diiXh+zwXO8h;uFunubHj^A9%AHfB6_soc$ z!ek5c81t1l|6$u{0l5c%OZFNL{M*>;=uqzRf2tKWx^-e}y&zu1U6Nleb+bI-C&*aW z;yxB&M09p%tS9VX6Xi_fZjKi3!=tZZLexyBqJr6_GRbO{Ik=3rV}M+iQ(o#Tz78a^1Rb?lJ~cCMxo5<9U+H?~Oj6aSoX z*;e?JXD5tRU~2Qwl15@q@f>8SGlxhnA{}A!`lPqdG@6MM)fwker`QFv)4L6Z;1sTz z#p~?%BG%qV%P~rY_GC7MIoDrEnW%PXP>1;T|TQSy;p5t1uvim*3!6xUQ^hY- zH~c3q0Lsqe2%~POx#LZ-=YA3#ID0^vFy?1#xp1=`|BVS7n|?f8M%DKmoScWS-M^gf zslMT?(crefd`yHj@(L5@V5hYQD`|PO`>8hij%*#=>*oi?mxj<2fA3LvYz$?al+kWxujc<_LEO@0Clh+-S#ees4CpC8pEw#B5G*OsTWv;gT z5~tIx=q?)aX2^1oFul94Z$iF%$}{R~FK`4ObaWJRySrhNry8e15Xm1pt8+!*{`418 zmH#iK@=s#h3C19pJRu0g$O&s!E@GfBc9R_G@RI0O!1&c{!`-)5&!Z-}<{{>62bBcf z+VlX64g;7gO&?s4Uq-zCJ}CaFAculU{!@B*ef!;$Y+>yO)R*ZZp6z9;tyVBZ7<{kt z+jMl0q7SyF2sq~l%ztG3sb zQqDy78YyvP3{-3OF%|eL2(H8;cKiw}QE}?RgfDtk9$IiI7X^ixpqPT+xlERSAvsGf z<;^d_t${25*L< z5@HjzJ=61q^KF?2Ec@TE8-LeeP_IPh&rCTn}F4{L4JA?YQVG)6uY4 z=V#Ju`|D=U?Y5A9+3}#iT0;Hty_Tr1mLfNj6!JC?&X6iJu70*FV?Qst@cFY66Y)YD z0?J#4&vlN{PxHca=A_ER3dPiQ%>k!8DU3xjz!4T#1DrLgwYr)7MD?Qa^;mVdgFc{3 zS}~Dj&+6>(ar0rlra!2%y0rH5@|frx*G@WX$<1^=^lf}!Y#6%|Qsj+8 zTxj%vF!|G2es6nyG8p;%+cYFAc;Og&MnFFtqdLwk*KAxS>`58gB^C*Htdc*B1MURk zV&!)dxnyp4nyA?#E)6%M1i!ILimNdsb5U6Y!471_e|m4d_Hp2&Pg@WT;~(CYY^c6X z`)jVR90VfR`352R;vHR=-OvwR z>4$(L@%p`r-5Cu$s?U?&phc482)KiFiqtoOnCsU2isKcqe9nDnF}w$1@jh~B zE;IOUlQ(it0>bs^cog=tGvKiXx}Zb)!OboWPqNqdrb!Q19V7m0VZ{z*!CM=^LYDO7 zq==Lo?`^|Q!dcMWp5y?1Wu4DAeju)+H2o_hoW*PBm8BX&Ube~YlanL1klseXN0!NH zhaHmKL034AM8p*@vd<5FAz1_J{3npGJG`?Fni1J_ETB*b+_CY{rseLKcu;WS83-a7 zap;R-$1f!4Dn$w^oJ(SNe*pi_jC-M9nWb|-tS6`qPdccF?=jjb8D4V?0O>6HMOWCf z(+Xm|S4%{sIC|i%YJwKt+;%p!CJ%=&g7_+$b;FeK>u7EtxMgC-gdXrLN9HxN%V*cq zQMj%Y7MOI$wV&R3jv?d%sT#>uyA-Q!kzbQre1`CDKU068X~C<$v{Mfz6c>7ox@u%Y zNYdwGPd$u@{}uw_Q5KA302AaI7yUjPQBrqcY*1jO>$agHtW;o(UrT!fXBM5;C|Hl{ zTtP??p>=#@BX4L{h4my=b(3zXQ|7bMquSQ=_)Y{S>V^@Xl;wNfW7|4z_px}*s+sI> z-8&=U{hQ$B422a_ODLd74yamvQg7fuJ}N>&iDD#n=q@pIFtH>59P4zhL2^bhfYu z1*qagi8ngl(>K0p-jhEpEIFp$!XjQ#~m2MD-Yv%9s_tq431L8-?IU z=vMPl=~%E_?-+~IGgF!3!Z#Skcy){}EH#jdv(|(7QsTX(mkzx#nB)vk=`09$+vHT$ zZTX9%oP0y0qhaqF>9uu>96!25@rvB`$nOXdwrSj%2UTUJ6rskm#dE=3ld&j-xT>cB zIZt?t_wx_%3dQ6A-Z&dmJ@)~c_mAe2@%p$<=Xp2w_)_c}iwop3Gi6d16Kb0s(=}u( zeq`<0D1KhJ<#54jxFbpBk82#s#fU5Mj4{{4#SM{VYU$u%HVtnx8Hhm6w0(r|VF^x- zi?uV6+_F%3MW;P_H;b0{S;8JMa#68*3P!&SO4*T>iGeWB^=qzAFIYs@+x>GT^>R!RdB^fa~;?n0?8?o>M7{cd*?1Bm<;T=3NB&_^u zC2)|QD!033T3(;780~{DopU*IB{GK+bRxd>?B?bU7wkCr6>k@eVph7acj`=X7W4ShdtSV zA2Xf?^5g%Tmsg(S3KujIogLe&4m`CnLpGq)d zha(+URa{@?MAjOO2JNtliuirD4wK*F$nF18eXw# z?cib#y^=Uh2^FBnBIrsEg4Z$2$ra*@#REgBR(jUE<6z4fRi>*0Quw* zF4|(s&)_{k`qeLi6L9`3aIgNGz_B=GJc?vfR>>b+MhFMV8SxLx+4$KTrjEO|SHL0| z)8lAcehzPK5f8KN(r?PG0ct}xdyDVpb-3GE_;L*Sgd6GYzdH+>3IBpSV}QsUfRe7B z(8ui$ZVdwvl#9nkkYv2c9~g0yKSh)&Ulu}PPc}hA=XMgP%G3;>jJ5p$dB6tu;WlH= zov-9()8@rrYq$)2>QqGUCzQ3F1cp9_9~^f+zi!`C&J55=Vh*z};2nZ>iQ7&^yB{uH z5T-1=tKlL*<|+yI|M6P+QAn2o{BV2K&&!%BVI$&Mu5G*&R;RL%NJ`00bkeX2Z-{_F zwH;BnYXsUMi8ZhO`ZIU`LJ##vaK2n}a#@B9TVi3!Wzpcb5Lvc~B33+BJO`3}Z(b|K>@wM@{{%bvQ6S zQsEupYR~4;ckgs4jZ2ftioz=pLXKR!@W&N~E;>B_3yFfKPozcLS(Y1Lb}oz5`^QVg zv_G(SAc1~{>m%ZxEiQzQiK9dAABy8?8jR@2iEv93iFsq}&nuoUH>E+U3Enr-!>daA zyy&wz#|e+Pq+b^fWY2q~Gw2(IfRIZoH zIRP+aSM^@`uA~6A!nGw^oNe3q+3vS_*_~XH3*h_AHUdNP5F=#2H>^ z{_Os_+_uM8pBfJ_FBbeoduyKYQW=1OD$LJge1|WHGs~DM$G0C~D+6=nvDE8&X z8&Y4m!f!p#V*F_JK%1tcaHcrWiIQiS_cQHhr`fFP^7?PrA#_(|@W6%nx1ZB}qcoa5 z>?XD<>LTInmtx6Abv#@?#aT&M1TL&%tD{d69U@-qP}nth(b2F;aDWCo!hI69F1%y; zI4-M+(9qn*n%tyC*k%gj---i zfahca-(HaX>eh#6_lYoR(R+x5dnW>TsB;>-gJAh}rTXeg-PKFH0=ST~5qiP-63^;B z?%XF|(jM|PCu7WimO6D*LgFeFh7W;SlqMkKJ*yteJ!3pyRS^@({5-|MZ;}@VoWhPY z1b7Zz3w|4fqIOQh14DcwkY`Kv;Z>E*J(-VjyNrYp7wT3Mv#dPkr)uo;ME7qsy8dDT znG~eBkDND)nWx718=la-%060i|@`8`bg2GTHS|8BQZy zD|qkNO5_GkKT2X^4xWiubjR06w~uSqIWRxygb+a}s4MxI9Z%k1t{(Nq%C!fx#w!@s zDKIi{^d;=Sd+HAluM!k+N8AqZ$O)&MC$--kfy&pCz1wNgSoj?u4>lhWxs$6gV`1)= zuJ&ws?+AMjrwx4gr25|Hc=?o48mt^HX(t+eWmo4MfVCimU z1~9>Gq!7cpFl2OmT)A3??!M>}zF0rLx#|**6-~62I;E&vrC;%K!|^EbS2{^A4zGYY zjqcnG6YzE8gumKYnG4n{{uOosAB3O(nOu(Y4tZ<_ZV`{q8|}L-^a%{QDThc6I$C zxHa-5ze~Md=0mX(&;s7oOXzn5qB`P#xb^-m|CN$E!;w9RUNltH@Oo{n0lmCz+`-|` zH>;H%JwbB&s-K&a#%GWvmP^H~5cGcUR{;7Y+3!HQziYZI;bwceP4Kw|vg~0?&K%TG z2BH14AcFtZ1>H|QEc)*nK>wjC$RE1=Kh2Q$)c*hHv_7d-w}RR(w=nbv2tE;5SaH81 zE8vIdLb}Rl8y=SMK%Hi6b!BaMTO@=ZoWBj2;bSyH9?5&6AMvb!2{_$*0tV8{U*DxfJ)(FJqwp^EQPz*wQlc6o6Hj;O_RM$-E9)w5B5%gc zdA$X0Ty%!EuQHY;K<)4pjeRyeIz^`|^+RU%e224LAc|#uv!s zHXiAd>~&Aa!!w7T9P|Iu*41$h7nh+tDTBZwzLP1o!opvvP;LIV@Sx;5e*}0=ug*(+ z2fy1cV9{$|YR`R-xvAe8Vrjr8ED|%BGD>2L<=2XQEHNJZGdmmsf%N77!3!u=0L{2p zxR^P2fsszhc3xW(E=X&W9#Q)H97xB!w5-Cv)?7crCJ1=%m<=y8ZwFYL?=EtsXj&L} za{-IL`o%m3kKlF2nCDDn$nle~<&=|C97UiBKEEQs+ckSg~-8zil$|^z?@! zUchYH5*85>_W7&&;aE@6WBjfBxsx zDk!|pHCHAVd4w<-3b^*3oct}GaPdRB7e9?UdqX=h!*YbB%Y!q(waERf?fT^-+#3G6+~S=EcP^B^|$;Cot-8?buC<*bpG!j^h+GQ8)m;= zzz$#0BUJBs?RzYe9EL<)PXCtFe=X?GefLXsBX7!d+h_Te)kV1UANTpq#k2c2TY8I%qjfVDvDZ z^L;pWefM~riK=d(+VK91G#2mAobBNx6!(kJ3%qBKB{6kvNM9B9Td2GD^KBQ(XXfoS z)=A^Nz32{fjIirjCIfGX!gd}L*3Io|NbQ2Y=dS{Jq!v3BI2lf&%Cn%oEn>D7c>y8D zSS=TJ;1KT3UVhF35&iQ7Sno}b{fX!wBfc3ry#1R;o7qM z{v-&$T3xnNLU`?Hc$i>gE~fmG!;iUiTQ5L8IkOxF7Q6!_7j5gUozBe6(UJAVdKUo? z4?`32tN7}TiLtOoBOYoxGJ84ipUbKKPVv1>^)K$GKK?D|_bGn=eru#yu{gdBQ7hbS z0!m9kn2;A_W_6Co{pmXwA8zzlw+xgRt2@VQX-m=3E!?CHO?rO?eLry)$qgf7A5ZQv4RU zul38chL7gi7k{K0K}^QYTy-8>8&ExWKK?jmX#mWTQpB*$dYcoJT6D0a5>AAQO&}=n zX{(fiMMF$|ws9`{*aA4i`b1jUfz)+1xKgNTs`TpE_O=S>s$1HgRs3RApAlNNGq33U z=TX&+ERV^4|G7|*4cr(8CcezBT@PUU%-&A)-#EQv)3>2O%o~v^~Z`Ql?=*(EGEA>CM`h~t!7D1$%htwWt@84Er)Su<) zT3@bdcy)$sJ@+L=50^dVt!i>DE&%UK#Ux&=Ciwq01`!bS{%X?t=*pT6zvSzAb`wMy z_P5WCHwAhQlgbV@x<=cmHBOLpthrr}#`$w2lZNOfhn>m`4D-D&v_0agByKHsf!E>s zsrjq~%I2OrcC~}Df3bj>raNZ)Rt=!aizg%vbR1+->WbQ?=ccpiqx)*IA;m#R5H%7( zSPGbF;CtSsg-46V;D+KFM?OJQ_r=CMQkkyu&*HoG!^06}67AGPmW2c#wrKi7QP~{q z2-&dczJaQ;>$Qm@=}5Xeo4c!Taa-qMAL^oHMoW(JBrns7CDNMsv`4Gfonf%}+P1Y* z55JSekR-x8h4(*BGVgBS`9R`CF#S~Zp#*HRXY5cL{uq5D{1yw()_QesJX;he$!jA;ANm};j+kdF1CdSU5?g0k(7V)OMY18&ReDWq0gKqn3eZK-%%cDvKpwnoj8RG^Afi`=Q-?l zTpR~?o7w2wp4=O0U_|H>Q>9Er@UJK;0@b~GGYZ)m-MMVgrIs$!FwIl)1$~w^n9jIb>G{( zvi{)a>p@U^b3UB%eSLD?+_Wt1>UjiC4}}Uzi2^W9}FGR&EC^PRI* zy$DEN3hJ|%BxxVLwn?f})%Vu?h!am89EK)+f~gg=NYpfjc|JaV-J;(Yy)PZ%SQ?tW zS_UL8e*kx7{Pax0Gpq6FRC+Gf%pIzBjyVGz&2fWqTQotz=%6lMRD> zQOFI9`gZ;}!qtqos~hnu@X=3aa}VKul%L9pE%*f*Z=k)Z4mInf+D6o` zA9f+XX%I1WE(dAV`YX6+{h{EJ=+@qEj%n`GDJ*<_AvJv#O8F7ebf@j^u=wo#%(teE zYw(Y3n&ORps>f7RY&coHMQ~NgZ5#G6L?-p_y0kGW+(s5D=|gge+@y-K12-+*r!ca2 zkoVgow;I_#vwH*XW=itsDpi;HJIU7BAc>pFx`3(Puhr)t7XwnAz=TZZRtbc&oVrWc zQuEVd}~r z%s8mdgIN5XFD@SA=Sq#K#ZHXDysmH7M3GaaINwJ;KMI?yrhtAq*`_HKdeG)8&>vZ zrl#YE-rbx84l$)w?Br-hzF|#L9~8%-&* z*#k%10QfaCDr2{>VHb&)-yFRJMb~3VJX6a`3Yd(@lz&cAa^qAwXTv@Ld#l|RBSW6; zBs@L>T}s{g7-2rljzDjwg2i53-~+ZNT%7c=@1>Ke_1Pbn^HV%MRH*IqZ*DpVqG%f71!5EC=y^AxKWdP_V@LLP=zK z4T^lLlosFU1qLcZs^;)Evle%I={r>|tmFylR*f+#8=sve4 z0k&UG0Qx`q%aA*rbe=ouj95Hwd7<~SkXv8zvIE$8|%HR<=9&K;W0LL8eV7J<<+z@yDZzcOt7bGUIyn23oD4M0VWE410;YiWY*-S z3jE!Bn@dm0MWBTnYBoi zVtz}m$jH@QjOg`|4C!h_y>K)2=9rro5Z$jC!;@<}SH$9NbaJ;fD@jz?`R&))Dtzdi zJ^SMcG}nD!gz|Dt&aNkN3UZ&XQz_4lpHZ~6y%a&D!2aBNkx{r=_%OP&J8(-5&pq3# zM90W)gXSsVsKVeYy{^dkNY4A3+R{AETPchN2H}aM@8KVqMXH73)qAbjo93LezF)GqC_uR~!yU{J}ZvL^RNt+-2(4+MV4C#Nkl;tFTZ=u%> zum15OT@R!D!`S7{CbGfMYP#-$HXE>>nuUzLC|r1ygrWhYi+EnJo2j+1T%3+hn3t$_ z1_KMK`2|lz7-vPb+~dpkxdCgOYIH1n(-Dk?7c=V^!I6cv2693p zbzMzAk!t$_P|t7Ao1627f3#Om8B1cM8|_l!<6TWPm=L(F*D(oPK76TtO;mL8LKpDud2Q< zGw~9y2}FY+JzKaknkV(u3ym#Wi%lX61-Np}x31wi>o z>_o;T+RqI8bk^I;|{@(lBiG{COEG<>kFx#>T7~kbH!%DXTK9+;4 zD_(cupIQW=F*7PPZc}kNYZ7pF|J-A55R4(D=#c9h(kXHC(c5>JpnuF>vv3cG`ou=Q z>T)=~#x}0pDJ-FQqfymWw3@kZaf%?$IGb=@X&7^ndds4>sUQ+yHtO>Tkw&noGkuD+ zhMatQe{Oj}*Q3I1_H(ah{>wKMwcrU=mAI3~{a%7^%WVPXyjn>zicUtxm|GElOgkUf zi^ktXLI88Ck8N5)qz5%`b$#)%(!w)%kFIGkMby#n4@c0E) z;FTFF=5Kl?Kv(3(k6n(zy-fD@Pr zy7~bETnb03HK2}VXGuS!cv+5(cV@g=ASGl*JQA--l(IqrzzHtNIzY8(n^#y}>ll6Q zb`KiYC>Z%XkqyX%ke{h)jT97p{`lp+V~4;b68WcG!{iTePGK4+>NOX~s=l74Q`?s> zW8)*?wy;5dS9sH$t~8>Mi0>I+kqX20Q*7A=T{f#2=&3XrpUodh2nua&NuL(F3QVWH zh}z2M)|*kFiF9+&Y_A-mcW+GqTNIu6lgI{1*YMYSTZ-%#_}6|bx^vwP_IW-J;c5Gz z_>J3K2vhEPMA8eQG<=PQrty={A1A_sZkK^OUVT8mkhu*O#pkqrNT+*hr=FWi(`B}* zRAPZ0v{@F$HzSE?t6H6d3D`uCkGP>@`%i9ciXij5M1`B$MLDNo>Fc<7c&s_;y~Vjj zG4ZM~v*!?dgZ?kZg8iP%ihIpl8lzLyq>`e_`#E|bAh7Nwhj9Vv#P{m3B$9%m?W75X zre##@BJ09W)?t(%3+#YxhB#e^(U{?iPBYT5!W@Kx;o*0JwV8i7ZAlmJ!5^o+XchAl z^e8EwR>t+F7L*8Kl2`GZCP{efGEs;;Arl8<3DzxU0lJTYFvv>B@AxGJ^xD#CGoWtLoyvx@=DID22 zUe+Cd&bX;~!qL~Zcek9Z{ZO_^pLSofZ~1nMo{W`yHT_Y-QbCCj@W!J7_Nk%Pb+MNw zyXmlur18A1d^Q8sUdj9OgAy{#Fu@084YPoX^}eB|3lJPa$bZE~b`l`*)3&$sNKL$v zead99kq$cYNH7$=2vU$yL`n8s-nirRakl2*vjjY%t!AshKXhoW_@e$R{SN`*Kp3t{ zA`2U=Czg%0BiY9i0_VbBC2?`PXo~d~V&p7GcfKE}y?AB3*NOS&M(C|~I{K|$%Wm{X zUoC+a^%u1$X%*%84~yNKs3Rr3Hx|Eeo)HSpSj2t$&~MbIw9*M0LaT7&34Kj!z7sYz zg_)c0?j0->B>y&Yc-uvqxWl&W?OXf#83I3fkt+ma2lMdT@3@aUe_cUOz)YfXasi2nPLT9_OEjy3^tZ_3UkHK@7 zv!+^h*~JVuL5bK0o^89fB|qEr8~;NqpOIaU0oA~hyG?omrZ53E6H_fO2N1KmooxI@ zDYMow^!a~^0M3^-9h5btY~P&S-7NbiMH@caA1>quq#xER&69VpP>WIq_U=4h%wqzm z>mBRPW1CB~qxPhKb`bXssb#Og#g~q;NQvZQBxe}$UP)cIBlf_>sj(lAxBQC*3|umC z03k7j&K3r~!sV~PW39{t(0&U4x5Qm<5;_uLG(N)shNmdvE*7sJssK z$&_&$9K=r^8H`L~ny4 zG>MRa({1bV`l_wO_}#O?gD6qih;Ro|`ymIe$Oh!lIg=Y)3i1;{-z14HafVO=kNN`F z7r;u9+{QX~@aS6}_3=a7g%jCS5<2$BZ-Ls-CO#dPdZxJTtMm{IBv&?0H38$&+pgP( z@dQmgw)kBAwu1%5ig;eXEf@ItQ_wxC>4QNgor+DMsiro0+?PE%I=bca+;ov}FS(()wh2D(cIzAIJn%Q|yFZpSuG)S?j=bkx^?+rr)yjMq(v_nO~jZScI z6_;fDlfRX$L)*UeOu%`f?Ez?g`xOP>)en)i=+V0yy~5$bp?SQfW`c~Zp!?mBGh^|O zX8MpNAwtnAJm4Lvb?7%B$=+qz$U8gc4pg0Ys#!f!U0w9Nc}XMMFBy~){Q`$wtr{mR z;e|V*Bif&NnxXeFKLWG6L?(3?6kzjb^MD(%1b^51XYB|GQU6;nG;7Vqg7>v};l~Dn zO?Rs#e-?~Diu3o>z=M7>@9)xl5dX_-YbJH7_q6yJwDblJ?AzP`n$B*Z>z!%W?p4eE zMXiA3`zg8n?!R@ExLmoByJrg-(vF&U`ufwKmM`DI0ee`S{+~N_wExSG{=3@$&6NE2 zWB*X#e=q<3-=ccsq>i%$Ug4-s%}+?QZ(G`Z7q$C2>nAI)+1u}-g}=`sRnkX@`*i~R z#?Fhk?rprkB~23;eq8*YYuf+We`-hS&;DxRzfbue57ho&Ii>seq<;zi`}e=8et&j$ zPnCbu8sA#*Ok}S;Px8GkqTVcYS>TwnCLz6=(aQjizMic}r-1a#%}g(wN}g-ULDW7sr|( z`5S$@9`dvUf=8jU1NB}u-#b#RZ2453N`K@P)Tftd7Q$i!{R722sK0bjQf5;H=vJ-2 zh;S(Wj+$L#b|x;KSj!+7CZXp(*=lCQ<70G?@fQo20qvlk@uy8LoaKXWR%+^Z36SYj z_V4T~s@iF|6T2#p=2ET(*|oj7xeP{hrB@LjRrh`#eU@9IzDx!WI-JBz$?t>TfXS!gB=^+_fk^w`$G!(qAXrRH5#DIG99GggTe>8`mYKS<6EbG< zr%F>nU_{(teyiP+55WA$WUJy!8RE2rIo!69aXYJ3riuN6VbsLV8bp9QJJ;bn4A{o^ z2^Wklg2meJeCypyK*mXyHYhJ^va8!dwVK0W<-8;VHR1f=QDPnVor?l7cU5 zYZVb0X_b5n5Hcv~DPuFqrg}6DwHvOOnB*{bpJ`B`-BVTD zrv%c1d2k*i&K1U>et2O`H9KM3Yq^E*AOvh8f31NL_npsCPplVfQW(G|qN}PjJ$E$` zrqYyRpvR_R?|4x%=k?SfM5-^}a&{aCaFoWyO-)AY*fUz-FB#F$lUvBt>juzv8EkRf zt-NJO;c{O1hHbgulWcJJLOcG|-T6~**i`9(q6Q^AWHG+lWq-Sx;PH}d-cO$qX&xaQ z%4bA~cT1d%-9t4`Jx-&)$>MQ};fd|C*IY~(g-b+l4k(Qq}&)XX2wE1N|VKUiUC%^&^3tAGKKXof^T`lV1NbA1YZyI;K3WHe%q8L%XOr^gt^Ne5+uSo!0 z&s(ZtHGbE%xd^a6_hm=Va(BNK8sBCTn5n&USj6X7&k$pcOzqTs-c0eVxt7>D_0Hlr zhaVe15NAS#$jx>&yRvJNF(O*?Ms#{@q^m4d?(Q-jcr#(OAtZ#uYkIgcjq#w=hq;4& zCSQ0OVNO=t59}-Cv?44NoO+NLy zL)%}fD^4wL_j#qK$^?93e6%DH}FFff> z^PeP3lq-(ASv+Mn@orKTD|k)LYNBxr(?$O=ZT2+$KJ&prT+rK1mZk zB(od-NkJVumS@V8S|a=Qc2et+T0BHWBorZ%08Xuxw=Ubtyqn!-+}wc%_YFZ)k}i&1@;^b>4*eq91UQ3bx83^69rsM?J8@ltBEcV_(nVBF7p=ni zg{o;YmCI4G9dcc{x?IJ#xV|U5&N56vmde^KfW~t@dwqgt8gtmOe3t&z>I(O3vl(6V zySXW;_-ewD8n= z6GOxd&ch^#`NPXz`k21(ah^A4Fg5oe{$-gy)}cMPZzl}U)H2D zP7x7F!04J?{$y)8Bp_tw z!LsZdw|%RjK8<*8I&%CmRh!!29gB05V|ZVt#cTsML$MFf@-DA+xv}F&Br{EzB-5f^ zVCO$OI>W7v0R=!D-L4#vm=e*e7Glcy&l_WO3kKUR|Rx+`8km0c520IEB=si{Q40~Xt)9~Xu^Tx6JhMIP>yVqIvxYn{SlsyqQGQO-2C9N&7> zQ~mDF$H!;+0?fOmMF1t3{nFeWR4k-c@Uxsw^^!kRu;B6y3JMm%Vzzv4p=_Ea8S`5i z77cRpzJMu;*M`W2PL&+5^j;I@F7jU*G6gY{Q?(qHjSJcI`Zv}Gb0hf@IIIo7BCE}+ z(UaTxCg-(;UIdmUJdca4g-8rqGBr9kU-L(X(m2ZHv*OO+5rGQROY3V(G=YoZ5ECJM z@UtWJM;)E*T>*VWyN6p4GaGYD8+FSHC14Yu1_l;ix7qS;crGY~@IiIMp_Qnysvk%Q zh;!}NYP4cX`TmMNvf*|vf5R;%hjDkeqNFJWRF*5BN~eBBlanPIiB=iKZ|&&3J{=&| z1|BYww&)He=X00x9q{)6%B{z1y|1U;u>Rz6vx`{yaaV-C>67>TrPoc#=*+kJ*3c?h zmN(Ph4t?}>pu$ED^AX!LkDa}b$B6OM(ea(Ol})wY%AYS9vMSA>o&4KL>Jd8%rnPd} zppps?oey7ymj|0aaE|F3T6(m+nZ&0F|KQE9Vfm^ORH!+gkx$cX&AKl`BHXR-wDRyc*srr$O?ZJPr=-w^R9OlK`GcjC32NxLExH2K5QOi9D@Zo^kWb!$nc?!eYs)_~ZyxuZiDoO4T#WnW zc=PGsZM{N<30CoI&cB|D%-y!IWniM^S5MJ9YgHaES_frMbc$kL+mGvLPV?CmA*;|)zq z+dKva*1j()#HlYxMw*NR8oTJSVej2$G^W)iTJ|#K z)$UN|>6@D`MENRXhoj;WB-O2zNbk9sIQNn=LC{L`O(|)pc;?LKk_M+k9M@6lyfz)n zw4nbNRqy@Jb|3x!U%hCxRa~kling>?QMLEgs=Zo!MbzG#AV^D9RZ)BIReL5jwRg-| z5wTaqmLS3>*ZX~Ye{Z+%ACOyqdOlu{bI$!d4ibrC7Bp zHgA=TPjh)`a}9v`5u2POKSz+cBk3Yq6&Z%L+K?)cej-BXEw1*!B$%L{;I93 zh}K`k%G{F5Zt6=FrIx<$DH@OU>r!8S6Fu_&pCV z+|2S?#ECUy$GfFzKl@}(I3)1ao%0#@)N_MH6FHqXTOA;hruKofrFC1+jB?)VwqrNs zSz$lQD{uB_I<7_zX|m_^I|2hde*05=(K7b6AG;1ML}(hsvs|9@b%k_UP9@(wl@%C2 zr7E79g%loXnB#!a8Ra_0V_DpkYAKmM;m?BJK04w7i6yNc$ToIV>quHCks`*<>qmNx znRL7Rnr>Qby|+N^S33JI%Ctp$+2u|K!et$LtFA}8DIdhsmPWT89nVZ=g{%ng?4xDc zjV6LU$6%~YK`c+zO^Bu#o?!2aE$#KI4U12TLFrLZ83}K0X6PF@kqaurKC&n0tK6!X z3oEePnA~YaZ(E{9_SDAk!N7MRb%hfwe0FJ|A#G*>Ytx;`Uq5XOtQO~zZYUTGMAiOs zXee(4?>y?fm8oeLv%x0om$v|oTPEikGYD@ik-;=n9FN;&+b|;1X`OYG2-RYpbcBV;FGK% z(8>LihAe~~-W7GOrL_`9{fYa!gbzaSHgI+$G?aPt3$g`1>o_o7!8ssD zyDUBNkRd6Kp=c;l93=m^=w8&Kg9KNz zS|aam&4xmg+I>7;*`aHM=^4|?x!xTe(KZtdx9MVQ0-$n~WIFC+J{=X8>&cJ{-h+&V zH@CDC$os1#Y}J{WWkKZ-$1fF3g`&_fpv{4-Z&78?vc9;8!_tqI+R3hOQWM^q1YnSV zIL{ov?}b}-Gs4milSbWsbPRLTbUGaKyGq7u*Zsbb8ne6UA0`lC#C8i!d zCMtz=eh&W*3%LorXtT7s&GzeR4OpdF*R;`mnPCVdj`%6DsC8*ihVoOx-4lFI5R#A% zwXgZtT&>IqEALh< zT!Z0EL3RV{rncAT%2Si?!UK8=T?FBS$a8Z*eB13)B+U5NR}HhHHGlamj#{fOXkv5& zF)PO)6K?>=6j%6mrKH9z9 z9}9stT<-a^r{e=I!h3n|xMqt;yfmpUY4h&qb8}tPwe_k4;jav{Y6FRa>?TRu1KZ~d zn|Uaf>%;w9>8Q`LzODh9zm!K!FP7cdo9EwrP%wo$p#Xvm$oVht2%QaCGWOS<>lD|R zQLgx?RIJWMdo6t0!GTlt`l98a3U;JqW2%S~zOe&C5yf=L?X4FpVLz@l9S0yMU$!Vm z#CeX6KyuCUOTR3Dx$G*YKtu85E!X+e{r(GI z?Zg&IAWP`i&64`Xd|5na_=|`eGnA>S6Cq`@$6iu_J*1#IzCqya(BFjU zq;?4UpRJWK8p%8RZnW%=qn3U<3yz^{{ef*L8z~G;aHEZB4QfTRMW6A$JX)ub>$w+p zL0UJ26PH}(qMM2CI2p^H8>mv!_uuZ;gl1~Xwn@CRj)}q8bcgXzi?R`xqg=wsKyjOs zsX?aZC8oj3)(xcK8o_&65~ip`N+Qf9$Z0F0=oXDgBrED*h@@f`NbMZ<}tYY>Z zRhz5(y+eU@%Wk#+TsKBJ*&Jhaol>$HoWe9&0bZWd55nM_w!pjBTd3Bnwj5igWyv3N#R*n z5UY*F*AhY38>{`KwX|o9d%vp)r)SM=KrI5+ZE|uiYNenpaQQ3)&2B)Gn1Wke2?4TJ znGzN&@SF=&e~pV}f)6$nvJK2wc}3rXc2~ z!`15gv!Q9BPI}8yCjez$?}TbY&!G{d-hVQi-}PrQ-9J8h2jGLjASB&EpmR4}s1ks? z44+vO@CEX(5XSg`V~pLtI$focAP<ts zfU~URtcZI^Pe|Mm+rMX{d>A&AE1@X@=Z$OT(pwOB*0^KY-W{Tz(r+?J#(a&@A|@l>}<(>7;)=qkOHw8kHxm^SczYvY#%E6Pa9F zB^@dU%c=*eM0|EybAjx!Tmr^v_D8BNov~?RpCn4OT#={jp_z%LT{ILOj&sXJ!T@KN zgC^hatQ;9Y!z$NMd%igw&Y*0Yk3ID{w~8$)6uDG>6g&U;FN^n8gneVe2g8icoiu9K zi}CAtt0^^sKwP$IIB(R$-YDI7njJpknI+Hx6FXg-Oq;XZ^5${dSZgr0fhhFwlkgg0 zt!i;T?onI+bYb=l9yWQMK<_nkUEr@KT!_x(U9VW zU+pG`Q_q#Zfnq6^JUK6i5@ve&JC=wGxx-@8UiCgUf>+0%Ilg>QG93104d!n^ByOGL z83b6YB_}JZ|BeRJ*mrFHXw_w`kW2psd%d*lqJd6HpM_MP<=Yv;)0gNdiusq0`}TE5 z74}(Oh)ag-ewN$et4f-ZWAbO~q=SR9Rsd1pHlsV~p9jX-&sX<(Ph91&qW?e;Ep4M0 zNbxOXiqRiKUixx^4hmL=90J|o63L=pT8)3a-sx*$51uL|TQKw5Kx>(f{jmUyEc}4U zo#YV*e#$|mfr0qpI~5SWly~kl;_I`h7eg2Mf+U*#H3=T@W}_0g|?OTR8{nn zX2i&~Z$aDe1=|9(wVX@b$CKC!jEgexY%K~t2J%A`;=6z%p4W~H|vjHo4st8Bjt9g`d1goUhumj zn}RWoAK6L&(8=@|&C9Jl=jv4&l0v49gdZo}4&&}qeXzsF*~j&&^hfs#Tl!e-?Gb-U zTQ2Pup?5{3)EM~iyjLDP<#X-%GLy}fS$8*6{}f|dB!2HT*!MYkPw`lE%)MB?5c)T{ zEj8z5W7NQHLfz(GbSBqoa2(s_+^{8?E<7dv3Ckw?+Tz7QWu}8}| ztTiaj^XR1&wY$E|{`|dh?rzw)fvKsnom^>K(VLr}ye`29pSk5~zYnOcSXKQ_S4^N% zj>%Csol~DRYP5r7DoV-Sdgc*J`{S>Oc8ghkvk%n4^1Et04^^*KP+i|<-a27uqiscIRM8apc-%EfD=jtYNyR^U`Y*WD zUlm_Dou7`*ZoX`*9yek}@^#F)`WrGm5(c{}+4fDlzeTy%yb_l#u7*6AOKPR@%``br z(2ZXe-ExDr%AcmU*%H5nPlCfqwVecRp=uTqxX}0`l47Pq!fS}#EfNaC7}bD^Eg%48 znCn&j%xZpwFr!UDVWARVed%!?nKV9S^VY+K6Lio~kUxXCKO=SP z^%U6w*T}GmoSV62pyfO7uQ&hX>H&t-hSy$1XhzY1mTl)=PtS8k=E4&YqS$wn?$l@F zw&fpf-o~$5Dv*7J^)+8EygX4g7r@?JoiB22O8V9KTA^N)!azu9-zScAv@1yoaXton zV*h14jWTMJ#p~JI=2L=V2~YM=QXo6Yg`991w$Yedwts> zj)nin1xT|sl|kCGNFvJSw&P!UwyMUzzNy3eKqnMX;*0SwJu$;>O)?1yyySSniI!Mv zEn1yp+R&V#^Y9U)e}vaotx{3behn^m{JJZ^wU2j!K*)FP@Xk&4t0p|84(uNG$y@)+ zGVS52?9?oC9W?x6WDzJ=6`~o5*2wfLDhbtsKd>xptxt zxMdPrt$Y1GYQx`G>Y1LeOpw*N0+^*HV=#+>|BmWb4#_(ZCCp;$xjX%er_3&U+v7(= zre_WJ1rybXc1u1t>xUq7lyR;uTOg)A^nmxJB1JkV?cy8i`ostS0v9xCa6Bk1Xlp3V zXgm>zA4-=l?x+{@umEOUPD=|`pl3NHN{Gudv$#Sxvro)HHztq&Dg*F&i*DLLtc3Ia z-Q|PsdV3=)_eiVL-_85I-0_0hqvxFZx0NKTsd8X{iM`)=E zeEtSHdI!VEr!8tf^{}v}3@8gV*)Clo4(!g}*W_P_ zP)A=@`6S_IN_BzBX>Q>2OpLf(tgA_nxD0V{E2~tDWzopg`{{wD!QQVmMb-Q`-8GJv zcX+N8>^M*1AoKV-grysIlFZ3=DuQfajJg zvUsc%?%Dxk*p1{n_Pyq*62}e8e=?sLDg^PXWLw&~pZVHUY?wlh=e|A-cl}1}w(U8` zibSS5br_X7Gd`E`Q*erO9ztnX7(0u-2-h;MU5nx`%3C60J2{{`1~sC-O&1!V+N=rr zSkM-}!+m&{rGH-JE}>p@&d}SrH>-zj(%cSuw~3)~RRhK-ne_cljKFnf z!%~#e?WO!%aTG@pp^6q^TIzR)MCHo$4H9{Lj@B;4n%ZEiac9Su;b~DWUi3j5xpg(i ztvopbBEe}wNpN9))y@*xnE=}{itKbMAgvU>--+>5e&9(`zT{7S+3p45BITa`pV?1Y z4!+4W>rgf|B}kcCwM^Td=wY{ZnVvC8!UEwdOP*T za&ameI@&2bu|f!9QGx~g+6)7>tW!a!_voveN>d;{!7ehis_CrVI8g7;E_&qeu5h&yi>5rP*Mwml#=GDd|3_ z(=ngZJB=7k#6~|b_&^#-wyRn|Myc^ltBFF3L{eE&+cK$Y>!OL%y{`4V{i?P6%1a7YAc8TNd z_f5Nhq|p*mw&Oe4^bUAnh5V0+Q8QGyYlVwasR*?uA8C#eoFH#KVnzx-)D{yQoDOay zMpA~5r5)GCAndFajSQPw#9OWG8sGM=&zBsncvjTri5%2dq?0R*FX3$5WbE%RoJ^bh z;KS~i$B-uUzc`(*V}f~|k*%oBoW-=FNe*e!RvlzJO30%|*Vb<2P1L{3>jBn322^9? zmR$cfp5A+0RvG^1cgZI?uAc|SI|~aUQqTSzwFCmJHU<~2gVX0m${&Rtf=(J3UcBWQ zX6#`||3+|>(k(Q3B6`~LO-vUyl15A*omDNx?Q1C%k)MdFb=jgkl1p={@I71-B@mUr zlG`x`ay7${%fpHYGO61>QFfFgxSy{adUr2o!u9i1+Se-mH>%h~%pV~6?hDozaUwCF zewx)@?$2*B#VXpA8wH-1^H^$3%~Yakzrd{#g&f<*47&-5b|WT#|Ck>()TrBIrOXKE zk6}peXhpjguO23oYiCrrj)qNb^i)XJLX^G>I$V0{*y65CG$cI|VNJQU7PB)^rj)Pt zr3+M3r>H;@0LF^__B>axFEQSW-~7W}Ex_5NmnL4ewQM25e(+1{F0$zK{V51qhU904 zn_!*=hw8HVT}>SRNLuo+3?k;+N+8Cj3tb!S<%|`!_X<_X`uMd6)Ao}=<0-T*R*5=% z512m}8Xa!;0V~w%G=5`NYXsFt)at!A7erH&qHG=Zy1@JC*Q1!C^U^e7prP*w6%)GS zJ7kgrVl;c$Ln(=KQ+d#7B)AJ9FR?uS`BM>(f|%O=+!)Tm8h}6UA7GZX;x7r9_&3r; z0LS93KyT7978-+*m|AFA5m$J z+M94-W+7zDUVQCSrVccr|0nkGrmpj1q6@TC>sgl`UkkArvZc9}IVbE2XldhvFMCJX z+_nu><+erF$n@qlCvg{9k7nzgH`U-i0)Ff4KJ2H`d2I7hHU4kvvP)`>wXhR4fZjn4 zLxm=NR1lSGcFxm?Qcwy4cT#vs5!IG&y|b_@kh1^gQO#h{;Y;m$&ddd?<~}QSh+j3N zRDC`H;dJmi2FFsAFZqgJc50zTgh5-`Y$%^JBLu6LOo^sAxrosqf-AFW*Ya$My64w> z?RSs%PZVE7ohXcEr8~hHv#=c-G;KT_ldt{?cr*=wGtXc8TQ>#I%4FC@NnDC%m3EMH z60D?vj^rBpOo5CKN?)%DR-rL#k@ZOU@~ODJ(XOGwkEbIdMh1i#daqlhYInp%VSawN z`Z+wVtBKfqyMcI;cdxAj${2Wzl{77>AfDxc69uI@)C{63Ua-&5Wui5&sVn7gNyabyZey8{82IX9PI6n$lvO55G9^RmcR?9Eoii?kc&; zjT))p4r2?tCA>fJ53?kM8ON|?kcpDg`}Jg_@l3@ zPtnF4=dLc8RDKT$WCzSv%@Ca(>USm)`k5yqbSxDQAMxJ5FIbsoxeZQuS!vc_Q`u~X zK+d}HR_`(-zI@dz%P~@K8Pu`e8o}XydWM2x5xsWUpVzB=(C?idYn85*0#4P`jVaaJ zc(?ZU=|{@`C)jUQh`IFP1l?9?*9vEqrO|+u9d^d=;6>D~McrVn-KuV0JPv zbjl>;iS11baaPD~((yn3_d+((^5tE!P~MF7=&K==d0vLp<@2!qen)ljWR^57A$7Ju z&(Tp~%aDShp{}Yr@s7HN#lBMMb`hxN6}&bLICv+Q8$ZS;o zOsNGmy=_rK>1%_$@0|AfV2j1v1SKET|GDO`pITdidi9eM)k(#0kN*rckz}aln3lW( zf=vQ&uuQPJzHgF)+a>J%O?-!&b{7qMu(iX_DK~)nJI1T?lG~xLme;IOZjGI@C2MUh zN~~4**hCT0cU8Aj7khS%YdN1j=l%Eh?yx1u3h_dDNwhw-cG+ks?(@@LiQJ*{$2B~l zsXD0Ffl;uNTF8#9g|S)R4+&EY{Slyyg4gw-2tNG88JE={rk{poGJe;WU||YSTeFccsuPhZL}(otajv6&BMk|C9DpPT@N^D=AK z_v&0Sq%Xr?cv;b;Y2S8ia#>Z6RbeAdWQ?;E6X-yj1K0o4NJyru+bm7-TqSpz$KY+- z_UCGUgz+()^>B$MGa8W&xx1gB2E{hz@29k%;&xgH`O)=>{>3Z@>OW^1@`C8!e*4U> zi)j)%5T}{GIfyf@OR^PEqrx~y4Hv%xh}x7qO{(7#8=qig4)CF3^Ahpcxx6)D81l*O zQF@F3pQgEpo5iDKHzUh;DeeoWa_GV-G|YPx+L(JtHMV%@JNHwwsvWkqxSWr0Ox(JU z&){1GEmz7r``PBstIL~|{pRU+3!`{df?&q=xkrEApoXN6tP5?xznB<#PQm||BEqXH36V^7^0f_xbd|M;Mf14Md!&_PZO zkY1*Y>om)mCJPBEvdLfE&lSkXr(TJSDbmHo zxY8&6lBqnmIIy4@Dy4L3PRZJuNL$dE;_$wiOQkK!TfI&y&;^%Q98d8}{~&}C-@Tcw zG`EoK>ppxk`pIF(zu^&?a{c_=eNXt^*)fixvzE;jS*ML+N>8=1X&# zk9TC|vi6Wqwa*j+@;cyT*yk2=C0f~|Eh2|j7stR?ZzyLbSj4>$6@qGBS&t=}5gU=< zmUe$v^zNBkE^ZA`NaJfMRrrf28_c2*@eY3zby?w(f z*y9e4^*#t>e^Vs#@M-q8sVzUTV7~*yPuJpiWY+I(XO(LPaA-?OY>oxf{J!mycw`(b z@1SE_Dt)JEr&RKAN1TiTdauKaxLNEbc92M^We#1TS)gDGuIdU@ueXsT@E)k`LHY!} z9gu}@g^s+n8FW>i*Dsgox1$cdUy5>952{3a)^~Nt-h8B}D zeVRZ)J+>z@7FQxS+cq&(vtlx^Ym3O7lxF+E$rc0Kr6*QiDWVEOWh9PTG zX`-&m)r=O^V68D(%VnMxd%4RYKMF@RLx-uj3c<&iFAp~kCm1W*j`Kn7JbLPJ>=2(5 zA}O##hR4fia&LW5TkgnU<*eO(#4iq%Ou?=7C?4-V4}Ttu!H%`GC-swgvW zXXTHD?e{YL#=ebe<4477omDqm8M93q`rq8aSUSK3RuWcveNQ;s zK9c-W?%Hg==xFi=1!Mn2-Rn=4$4%eMM61Ui=xQFb8=|=GwmO zX`>v8%O%Ef7cE`%9Mm!o-=}7m)HBkVwRL-|*}Ii)?qEuoYAI@-qbn;0^S3~c1y zKr9O~qhiSw`|jNH>D#}Cy)0DvUN$+uR&>2`w8@nx=`;pEJ?x zhpGEJk3BTQ(gy$ubZFzbqwd&vChGLZi8ZS! z_|IKyos!oh9Oygm-!BAq2b!s5bQO=E^EMZ`{9)Okrh^no1IE0a1WHi86|_!%C*XN2 zXp%a~=Jdmu zauheg#F&0IRXiyjMR{9^8k+a#d%cmd2f8&Tty%aivyQB9#|d^lQ4z9gs}+l2E#LNk zKKH*p(3Gy%KxC@P+U9_zjwRV;!xWZZH5_l$P^jNTEAm=5C%@HZ)!~Klh-Q&gM@(Ru#E8L$h=3)Ibsa=JlS`7;Brz zAy2XO<*9T$1%nRye8gBCMDsmuQa(W@KovSCud;2B2;^ox{{h&G0>PPor3J1YBSUbl zvtE`e*1C3@CXEuGmdbyueVuXE=&53dVUHbQ8q^b^ zoLRo7&rZ;j@)vlwSC-bpk!mc>aer%267JZ@9qDY+Eo|;iwsQD45!wf(QR^+sSrj@h zttq=zkLSutT|^S?l_&oOqdJfB1($k5$QW<;#-UwX`=u^9pI*T1_T}C2MQ<=w_D=Wt z^qlCrzkckiDE@Odsqfj28mvR#lAB(B>Ci9WCjTM2dW3MGx`a20P?VHwXyLl6FBLR; zNf@=9J?%ZePK=;H^EGIhA;xnUbO@vt{EyaBHHp{wcpL@!cldR=0OMLzq z5O!L1M^kA#vlhz!Q!p)(g0-4Op1$pzUa)fD?@s&=tz3VDpP0C!V^UrNQN!F zak~NHH{_RJ=;GAim-q)QKLBS<5s_HTtI7qG>^glVDA`psxhQ|UArE`G5ZOzC+lBU5y94($)^KE(KI?Fyj3S9wP}pj_(A2c86iZKgIfoq*9*s`s&~HPvr%-FqVbbLMMs+7Zc`rCg2jR8#3Rlv;EH{G_avHI01#To_Cb==l>~nZ5Jeksvlo zUk60ZQG5IuSfBNRCcSdpeAK(b5ya(WIvAbl|4BY=R$fQlj@apvO^=^#e0y(dq~GH4 z5vHABQs7%9q2L;NF%f%vK}KpVB=2DeIoG-J3&rzC_+B6GGF}$x;=ZUU5d)F43JnI* z9>=o?k?4%vk}7-W^s>9-q6Ra}?6gCc@m6G`4O651s$4iDRE^m%O&V!?>Q#hxjbDfL zTy)WvX#g9^t9Udsc-rx>E}cw!xrpKa6QOfRnhdPA^&oCqFEG!;L1QfR0`?<6vFQ0G zUAi2yGhv9qo;^^u=f ziV)LtKOBB$i3*L__D9NCPrSOwdV=xbliC#!E#;yZgl7HDhWM)3oPKOF)(5 zTA9K;{Qg%KRcEUGip4LtA^Q_^XZ@)PNdbYm?0J z+IUuxC8m;2SGSA*P85y0MkeoLR{`iK{nj1dSXC~qXi-SXq`32s#X~2sq6gwLt#_BO zbGwDZ-Ac7ap7+%c{yY>&60N5s+mw#K(_gSyd1FjW88395QKJ}1`duM7h-xkK`y=Iz zWrUIe2cAF|P8vF@Zsh$Jggcm@U-!)eGEpZen$G%EE9o?HxmV^ zCpz)Aq{~woSqVw@elJ9yzTB|=beH;(unSm0vo5MiE2Ux5wxiyAC@wJ-8R;c(ztRgF zjOsedKX|w^O_-Vct~+t_%y;F-JbMDwWAd_mHFPKE*C^OZ$YQ{KzakM|MBwRRD2q0Y zwG_&{qt{zu^_&BlE z=!WC*CC`xhSk|i%X+3ALzglx7Pm(i)g=Y&SMDHh6V!ij_tYEdYMRa5doi|)l8lkZE z2V|yc{4nWyyDt7dE*NLLZ(LaM6I8}a6Qtw6ee(Un3E~afYv|jFQ}Vzb?ne|$`6ZVBp7M3AFyTYt(4Y=y90h!aR$$C^ zC^p~jYkq)w8h9vzh3C!w{9&1>I38_3|CoQ!oG?D?Xxk!C!SkHkZ{O+`CRVBc-paY5hu~PeZ>1sMyZ$<7QSX)KpV&sQ!`bB?_Cq$IljkBpQc`c@^qF z(pg?xxJeYLOMBnDcUOC8`lj3nss-uzXr9($H>=!oTv}mJEZ_$*beNg?maI~6{q_>f zRJYQoN5!ZQ2npPt0VkL4<|(!SciA`OTgi|}`Sie7H{;lzl8w%!zUAgKL-9@r0ictT zYA_1m6s2xdEwa-Cn{0vPGW#06I>&pmLk@sF0#MSOV5=tB)mq4+Hi=&x|K zlHXA2p(z(J@l4vdqoW<6+T*$)gthMcU|)HoMC!PS@dq16!GVXuMJ7GKs2|P4Q%bm9 z_X?KlHc-%v*8gB~B*hqmaS{B=p?jpZzf)m2)#IG=A#mw$@VEpKLg9n1+%ACwTfSsmbKNi7M$vR^mwbdQIhwju&o;* z8q)u<;P3_Bnrea%b6UCIEHPJ!i)y;4oCXN4BU4^!fOF43Ul-#U9qcVV1D(63{h4-7 zbk;(#pwCh|BMAszmVPBRdFf8Ar*`gFt>hgUk)c|^<;M+Mvr$hZE-Oe=8Ip^7RW%z{?_Q>Of$xy#3I7 z`P{C6(3V%`w6h2LEM#b2%RDUo^=DOKI{Fso@|drr#V`4rjjZ`HSGoVO5n;Fb!d}8> zZbkrso?F>Vax=h+di|{OsRDAT>*r}jS0@eVfr^Ink$xa8L&2{>>sz%R?+B@RLt<~A zL|9CWvWQ@ltcm&eFg$lF+UcZ3EwjgMCHXLQ{Un;n{IT9yyX>7@jIS)oakc4C6r@cMcqOepHJQ$KxIa&@J?YHx5Sx0%0 z1B_cNnv(Zk6=R+_E)zU*ONI~B0Rb1t`2Gw5MXi&$c`3_{`TR3&K|{4P!cc_5OHRSD zj89IxvcQ{rD?&%YMO>OSu2-ssGi3bpLH=dxPO_D87nYra#uzj_px`Q9Hv52h*%Psr zm>HtUNL_y8^J{f12KD5XghCc>{BpLQC2QVXOb@NJodKHGX0BYRXqf*pc?RjRa zb=8lq()!7MppkCEoRQ+$aJ`i~B)H_~&|SPW;x+w2XJI_?;c{7rVCEA6M&?WDN8b91 zqFBEC@K*MNX205;h1K|F)rlV~L71-Vy1&ah&Lzy~A9Sa>#>6aJE&X82avJ@g6zh6GahUTzKvgqvZ~Gj$AH-Jy$p=(HrqM9;5E zOwDr;Rw{imBqy?URL&iN~m0qe9+eX9W5d{f(pmmEkf0@WX?)|90ipERjQZr}<1G z%d8p=%j>(DqxQK%o*hEC_GeUym~>9EEKghk#{|q5O6kUtC(HDp+4 zm;MDb``}r7nWrFB4d@bO{4DA}@#8!<1hCjv7L&`{_K_yT+Fr3a@Y^CmK~To%h_mPj^vXKvS{b4R)yhL zi1)@SeUs9YkKQHs9p-%_#k)Pt!Y1_oBG=4@G+ZlpVA9468Q4dqjr?GP&pWWaP5w|} z!=5O*3)E$gspU7fkT?C|vIqO#-rPRg?r7RaHW?Qa(>tIrt-DWBf{0tuK3KKBNI^yF zlf!wbzc=RBB;UA~6f`wuJkVlJ@*fLzqRaeIm3-%Xc$8%di^VN<)wzDY_JfY|`93bX0c*+240-24xO+=O$oi6lagk6-S7L8E)Xnm5qOZorbjXf=W8}VsX{S1C@)rr!C=rA60{R!BrdNSUTI0D7JmZ2r2yv zsb|g>s4T7wP!&|7f;!<&;N>3wrK z(YXL92N=b^VxUYDL^-jV1I0QJnGno?Ds66ZbjMfu&nf4U1bXz3U^ zr_UAzy!rT?5}(mS`{c%|*U2O4f_qHkL78huk<)P zL;y`d((h%OKT!IYCrDX)!{!%3O5{D(oVV%Ouq`fbuLPW0U;M5+eFwQhMD5(0xnL!s z?Zr{YGQJJbyF$+2AP0c|>nm{B{OQIHt~m0y6ho}$qK~<&T-;siSa~_2Xz6E(bC>zJ z(oa9d0mT(>`TA4}D{oscy(CCqFX7yVNvQ*Nq*r;zRW`?Vj5S@ICc%gzC#)D=cShQ( zbpf|vCbYzK$Hf>oT5)h33P*9q4yYISnJp$H=d$|i$enEt(Yb>`pVQoIi3ggk8)-yb zn!&S!vyyMtX0rg9wus_gHKF(+ES&IEC8%(prKT_H!iOU~y4|S&2g-9h2z*3l>HVID zqxfv`-;k^_myrji4o~84$kTQ0&M#TSx_DXI?>BW+jN;CqvG0>Ngld`IykOm%RU#$j1eQRdm3uE@>`l}G`SueOF>Dys{_ifH9v1ov$k&reo}?3 z>518lj?KdLjkLfy4hN6Wluz&7rHs3md@1!zq;$bRGtJTh{jKt1BvzrYbm>gxxtmD_ zt?)GGk1Q2qh9}7+OI>GjQ;|_&iZe`+(qg6aYiAB}=)Ac3vtfIrz zr~PA{#E3!3I23_MZPR)D5A*9l79PJ^XNfL5vtRSZQz*T{J@eP@f=s@#YVOf|h4$`Q za;BqH+Wu@=PnvWlWel|z6-!m_o-o2wm!9Yt7>a=kk&iE->iXsr_P7=jW^4cJZt3Tg z1e?q3#IoDnje}se`;_|98&dKlGzRexdm1nv72v_;ff6sFsigyWpr2)|t(qdhz zDAjS?q+YzUaU_3pt*}O?eDkv(5_D7lOt3kwI`t@*unVR%cd`nq=sEYB;PN)n8=LJ9 z<9t6zT@oPY>Qwr>v|JQ+OCf%hUwwN3#!1R%mv3`&%*kmH72cd?M6|W2>!!7BjU*5V z?;MqJ_5qcJcJRdiSR4P!1S!k^9nDqgyS_jY?sg)mw+4b zpVLGP*XhmbO>dS!KxGttkiWd!}syZ~E<;NAaw8^*nU6-U$}^gwIw7m~Ony>O6>J8;3z# zP7a&-I`-GcCqNM}RPYDAWb8P@iGpY#SJyv_;QK!V9iC)N&%$--O~G$@6Omd9j{{`I z`kN=I({$4zd#F*85oRM%m1m@}Y5PygyZ2wCmN4vC1~$ofrg+N+(EBN)!Z+XpjiL_{ zi^=&;PH!+z5AUTj+sXdZRCzRVWWs1J@ZdqzhsSm@LK@HyiwX3MRz$6zmBf*yy*4y6 z*SjVm>+8LgXJmuUMKbW*1knwo`E|U*NcH$_Ojc9(@uvt*Ka&}h`dIs6(@v&LX3+t^ zxCa``2CtKN{btE|tlp4*7F0z3AnZg5w%XMzX=$z#@s{iV#nRPUzNxuQBG&3?Q;8km>L+n7Hgd1FSkwA>8xyNP|Q zoOv03X6_@??hfWnB}D(D7;0Tm@-a)zz6n^oq<~!0L)9tUM%k(X_K<}6mvcRqo zyPNZVQ!bs!6Lf!2cRmoHxD9uOfHY4Q0u08&$yPb>1Th0?~V^@7k0xiT_ZsPmh!{zF7n(bp!7SsT+(?GJZ3~hjH|tF@2gTR&_!g z-9NFrAN!12wwa|;7?t(;*Xytkl7X{nwD77J$b0vk57;imXeIoF0+foJG;{j{c*%VA|q%6RNuT>|=C)?CLW+DB

y9zHyfB{GM8tvh=@ z7-vQ)qctHpx&R5q(Td6F@y z<+$+n@9O$Y-U<;F*DVcQVBi5BNcH{ywfEgYO>W=XdOTKCR0Ko>R75(6G^sg?fQW#A zfOG}vCDKbEQ7lN8-ih>HLk~$rq&Ml1kVJY7A(Vt9gxs9_yXW4S``7)xzrOkAzO(0@ z%)IYR-r4i)XRo#PTF-i$ppS6q+ZpI}>0Qom3Rvdj7i<>da%FAxQd?@@?ykdZQ!P%1 zi6zWZ^uNh{?-C^Z!W<7 zRhgBY{1LtrfYwy>HJ%dd(nfjN%(9SQ)Z?EmQU%$R5aoP#p>=!hQ>p6H6IST~+kb6M z3A2fd@g{xHd?wEBiOqr{S$^PEC5wSWPMX1~dRer|1u`yyU?eWx${;vT?QWQM+#9q7 zc_2({urAdmCi2r~9`7dehD%%@;&^?=Q~jitwoCo+CB?<`Pm(AXdD4?=|H8gy4DpnO zA3UbzumGv1-7O26@M#cen_*Rj8Qgi@E1K(wDvw3b*Bd`@it(mWSQU>L1w_Dj#roMa zImvMwKw_1N;!6Rorn=VJF+1wAEJ*#}FpL^-O7FWC?MkRqqT z&$W}EKPlfYk7pC5N6W?;j!_ci(5oxI(uwCfzP$;xSN%fUs_AM81(`~}*RvokW;`r@KCZCr z3854!o?_0BN8+`dR-~#iFj-7p$wrb8NFDH+Z1~*nQOOtDw2xD(TEefu)GGP>yej`6 z&0hnjsyzMI&@UX%Hkd660Eml^MgPNmxX!I0KN7+lyPV}-Kl^!zrU6~q570iUxm(@y<*mdD?OJIbmf;*3RnZZ1kB0X{)!lI$<$Q?3)rY-z&#cGycpiXYkMKvNjvEA&+XJpjO+Gy zn_tIFnG)U%JFkLBk9oGOK;Ac-8dKC*ro-p_%_DS|zDI?da-GixoVd>j2`NFBs&EHN zw}g1Fey1L9Xjqx>$5n2aPLz76E@j<>{&rGq@L3`oal8=H7O-KoS5h`ZXcW9V-qsrZ z2KR8pdbep|=ERT8yVVCP+>msn+(h1P@1^CXoRxjSIwOh~@wmKeb+3vCLt+*}Yvb_* z2|q9U62ZT%U}qfkX^hYGE)6$C0o24IuUy%hIqYf`HY<2m8~hRIbA!K9!$;4rf!**- zh{uK0Q7gmz8);3;eO7S>$*V`!9-WF0)P0sdNVkz3hj57E9e)Uw1XWN)$_3>OHfkE< zkco^Fd@K$rIGTIdh_kL7Q1cMxCL{+<#OiUcs* zm)rB^Im4vzA+NY8@b=T_zg1f(15(-Es~txe;#QG8#zw7iP`l_%)ETx5Mo)YzpC%iz zq^H`(9fQXBl$;)hncK#`$)f)GFg230)QP>qag)Tuef=`0$OuJVBB+q1qj0k59D;PfKzZU}f9 z;Yw-~T?oApdTQS(c6sxl?PrBpY?_+zkKCr?9)%JWNc$tX4&Rs$MG}TT^^MN#heFn? z@N_cQAUoFpRH@J@)336DyG$%LG{Y@eNPqimRNVRo_{uK!413%~S3O&>XS(kyhkaZ! zSkRWCF(KJB1m(H%Do~8u3Y-H-(se=8wBJU~88y(RD!wyZB(ca(!utx1iR zi3tr#2{6btr&MQ+1l%Z;@iDe*%qdqm(hJ=rfFR!UWK7C{u3Y^Gc{;){I zSIf1&RD#|ITE3`QSS^VQWZDO@8QWdWl}&jgshdRa9*B+?5Lv^&OGz-mTTn{xUDHIf zswzO}9Jx=?HwP$P)(27c7D^ky}iJ*Pv# zqC_Y`yYI{YbQq~(;vcbP z63|#$H}0C<(^pxK@E4V)-?Tjb%e^ipEpB8aLGIbNz_B0E9m$~`C(mL{KrlAPs%+l7 zJSq8WeT*=bFGKlJ4PQ^|mJ0hwHscxv-juW4rYAoVM(rsf3O8ME-5$hsIuvYFUDJ64B9T*VMV>)j(ppTWOhJv?nD~r0vpgNPkGk||#VxOcATGYa zF;l*OvtqMdaNhaA3N@GC?UQv*EDP#w681Uf!3*AiGBN!e5jkv&bCFcGfb98|<63~3 z#kKc8yf(Z|=}nP!CLCjT&s*6T`_FdWl+LZ?=qOMt9XywGqY13nw&rY3QsY>ec~V(U zsvrF&Iyz!;sK^*L?{K>1HH^xdfz=dkZWkUmc<~`c0{;3phm(gf*2YrylxucQo$q5) zv_@!4{h^Tu!y(z<>iHK{F;6?Af-es}WDpIyHYwz{y?XqA0Q+S~M+7FaA zb*j71tZTT&gfk#ClQCvXG}OE^kftsxsif`|oM7_I#774wy7BU|&dyF@#0lduiiab# zp7YMDP7hL5K;^H-9`-?~>5up9rN0#G|9P4I3sL;e>1lpQSLB53N4JQ5ISGVR$gXCunoA@9 zyjVSbiVw$&dS{%;qh^Aj253w>l}ZA(rgtk6Pw{ zAL0tR0;aBLK1)mB8)VcnycK`XukoKaaR{Nl8-6^o89z_Sbm!!Da`?kDD)q z{o6u+$R$ypukk9}+DcKmzpJ>EA%!est$xv3%C_2G%2L$AjSzzotv(fn4IM;*KFtfP{Lh3lMVpIjD1PSnMLk z5(inf7B6Z69sW(3?JUZy`&no%%htq6p%DDUb>l)!g9k!x{hi$Y)B)x;l5BJ*vfQa$ zPtYPqEkV%IO4Ox}ms|RoIw`1^a}1v?F5GeRf!`pIs6<^Q`!EDt+}J4NTxfIk!=dgR z8Q$|suV{S@x=3@@5%?(`%OLj^XzUDI#aDe>FWWNnax?&J6NWzy(>Zl=zokuWW#>=S zdRk_xP+zEh-6SBXiEMDv-zL09(|r3#dNKe}``Wq;Cd2Biw#2U9QtfMR$RvwI)hTj0 zV9asS3pj}Nx0+|KJMDSPc%icv%Wuq{jC&XW(p>h3Qh<8|fURxe>^9gi$|pic7{4FB zA6F2&sUpQHF0ltmMzzK>Hw7g84{qFOl*RHx=Ma<8f#B}eKtGuFl@u_AUDt;Cv%P5- zj>#BW_6#CD>qHlu3G-%N`wv4dq0_<9;i_k^qF;*JIvaplv+ZNqe`*jRI2PgL_};L$ zwvMB7IlP+{_enD6`a`ftv+tblVCvV-qGxpz$8K;+_U2rOwrW#Vf<%!%j%HzDl+Aj1 z+7*@{iO;lUu<7HKkxq-A4~65$r5a{bU(j19(O*C6^?unerA^LD;e{2{ga&NZc;;IB z--DDDD}HhW`7V0OxuYvS4ojz9un-hMsin{>gYK(GK2NXfjd0wo;XS%?J0d-^;oI*I zA5ycd7v!z~&XBE2W625FtC{AXRROJgw+y)zZf92J99I|iFEPQUs^uuZvu3>3oAV&NYQm#auvGjf+27fjN zZ^FGq*kU}3Cl5dsXH%y$H9Y8=$$~N_G=rh7Aisr&ybN!Ji}_1!(Ooei#9muA036_B zQ+I2!pt7~z;yOTZw*K1-TMatfN`<19_4Y_rV!Ah;uKYiI69v}KZ$aDN_ej!Wj)<+WMGVkD2z!wdSaQ(SRYnMrb-1aGlWjY9WKA? zzYq|x)D$3kS;+OxM?94{eBc@C4F@~*DFVPgkb(4X>hO&TZt~3fOp4a<7F2!IUN~&` zZ&LgIC8f)-AYeY;dt+h3_Z3~yq_oNWG@&7NYZmPOip<&TVtfcP-tDpR3m}AV{9$~_ zCj5^g{Qd#0!N;Bt9B(1B8(S=ZnkbWfl$UlNviyp@=7W<|&zBpEL>ew#pb`{V^<&63 z{2Bh0)nQ~qz%!qlf|27BaX9&jh#`>?QC6iuwKb?aCU#GiHf+^Ew%JwsW<}kq;o*ka z5wI}6bJp7t)^#-8%tKFNHP2k#+_8xHA{Db@cJhwP>g;KVw^;__*?Ex8TatT&$K!sO zw)dMvHX*$$a*Rf}(a+Sxxtk~|=|VMDcH;f#+&hj`pd!G7%Jm*NO+cY*UQ^#_vs4{Q z#TAcUCwKCmK)z0=u%a?|S|E1l&4@6cdRB#N-A2+E%vJmWJjXh~EKJY@{c$D^wMQ4Y z7=pdGx-6{V4ptqEsKqVH5*fR?8{FvY4q^|MdI2aZIzz5Is+JX%;62(f!IWNLgHCq> zj$XGhp0*3`q`>)X)vieYO+4UaD;=R%SuNQrt+S;7J`=PVKsVTpCi z+T<18AUQY?2Yvk@LlC#jovCf+c567z^7|de7d zJU;98?(<*AJO9KGFM?9e-Vv77xdui#@jKjg8p>kQkt^Abk7*zFjeR5cM#HCL;+l!ZMkcHP1=6gZ}ZkckkhTr7}rh_OAQlv9pS z@EqKWfUwyU&;PTJKqP^P^25a8LaIr1*x*>gR1g#01<4~xZa*qtQI%aH3q`HtUcdEO zs#qHy#!bKNtwKb$65+Vb4=b^gZfksttAN@BldR#MDOR(^5Sy&(CimBdc)4;zjv3o=`3-@V`9qi#k}!vu zc>?I@Q<*yAuJ7y2P?o&9mro=G)n*|2~|>0I)#y-}|Y{WF@dwbIOC+Tn!a;nAn2 z^0F!9?v93{jhfd~==p@;3*v*WjK5sZFJL8KM!5>~x(I#Hg*{RccLX!T#39rkxAL;G zNt6tQf^Rwav76Ca(dvG%t!!-B>7sqc=(Ei1$wz~(F+_KC+ZGn`vsvC>ZmrNMM&Q@M zk_Q_&Xrj^wSA}`pSQ&gcnF&S|kjA61{le!5a7~TPn+vs-!LjEZzboG_T+H#tjUQ+2 z{rfFSKDHM1xe*(E%7xr1`zrRE&9@w^^I{f!c^;6&R|p4K!Kih;XP6*nUbZfCV+ZKR z+<@|@qw3mE`>q?wEy7l^c1!0M9u?Y3*wVkZDYM=2d+NGOqo@+p*ZVf2Y-k&sGrOtU zEu@Vq@!zMiNi$wf=iNyTmhDp^R%^{}NZZC&Lz^wqD4qs)_|lV#(wip>zUOKM~a@~q($LR1r8IpwVjA@?+1Y(An2lIV*?>-vly+`83x2+G1ir5dt zpIGIEC>;(cQI7m47uZa-d8xf?@{T`$o^^^X_t|Y{)P;xc;sib+eKt7uSCX+lp-&0u z761cHS*|LlyJ!k8Dz=J;3>KMiChtQUq(Ic($XzCptFD>}|)nH#9?Z1zW)DFj84_VRA!3qLzRf6Ns+ zO+9|8G+P{O1Rg2l&dH{y&PWe{JoekWCL_CWa*S3DY|uSitFJT@Jzk;24xr7}(E`NT z3a@7QlevfKFW87yW1m66iJ8Ar@bUx!U^xn#S88$4gO#;WzGr4ryq8shrC65&SHdk7 zO@QSvoJYTnHMROZUAWy9Ph|KX?IAGF{g;z?b)n3$P2ua<*G^4)b@5I)0pwp>lG)9I zqKK7u^ZL4}MCrqdgn61M-#)P)CZ1VmJ6vR-;{Q&t7pu&jMVbM#IUd^E0_#as0P4Z! zxhKs5wOL zBWCcr6C`*!?o&P>K84&KYa*_IY`wj{7M?VQy8r6?kwn4UuGO<@%xxMxW|B;t*kvj> z9N~@Fc0TVtEyXswj3hubdG&g4&|BI{3%6!}y==i8 z%1HIMeIGwg8P}CQ52#s$UMMO+o70IsH+6M~QW`tLO;Ni?mWJBUM-4!Rhy6YePfKpf z7mrV$UByz7&CPQ!?Rs5K=-Ordb!kh@@ey@E)8~5h_ym%&+Q2<1%6mWL?OGKK*MOOl z1XI(6c(0=X%)JrD#`)C36{pV{MEM)9%R?N|E4D`yE*jxiohB$f@}zImak#Z$2RfYgZQ$@%pxUpc7ebYOso8R6V9edCHEu z7Fj;c>F4c8tegE6c1r17rfqi6Zjh|TTpLF)Cq+bbC^~w_v;5`3fDch#e@s*glPNp$ z!!czNa+qGM>NUk;=|nMZ#?nwv{*d^b`qBrVMDa(JobW)(x$!)-&VzFIgRgBZ@s6;y z26IJQ)s1Bh38-?&Bdkm0e52r>q8-y@1;Y4sl`QK*Hao;D|ZT|7#*ijX>DBV$w0?o z0{ox$ucc%gf90bB&uiscz?S(iitZ)on%c5pjCO8P^nmBgNo*agRZf~zD^K(g6aZpt zif$XJ@HV+))shnH%fr0cqP%^K&~_>ql!u&?VENQzg}PQ+es*^3;DD$q>EkRH#PfSf z%@R>|i=`YAHyR!?*SZT*3|Rg+NvQPtk~cBWZD?QV>>}eVOlqq1?yOr9PpPzmds0Yl z-WFOmo;owVLJmQtgV&Wxsoz`~84l$~=H@M4mLh_v5=+Clu=%#Ca4>LiIkK>yt9rIs zZMH!~T2KvEU~A|)*wn??}>64UDLCA)I=C;!Nb8_@8OXjm7!#;3h@Y! zebhdo_0*n}$=N>%oUK<&O1At8o!Wf6UPgU|rNty~C{~q6S2#HpJb~4sw}3r5`>Z#t6quzaH#QhbvrZueNuL|m-mrmG^3)8JIb+~kb;beNSu-u6 zgP?xA1rqGG_~MD<8%0qG$wAKvYDMGhSmu?Q)2pvS{a~GCQ)qL<_WLym*bTxfih8v( zJ%87%!H=o|0Ie>CjwK2>8(uI&=iupt!!=OTYdY(6POx5{m+V;K*ViYlR%+{|KR^tb zwjhMu33B!@&Ln^a1SGm9DUS%4rz)#5mhi31gsA{|ilX&Kr*hY_J9Vhi4QF9wSnCB6 zQ4=iwR@1z&2b7#3OD^_oW@n5aN6IYxIyFC?-$i#;lEPyv%a$f73LaCuK7l2Ssl+iS z{I358OBa5XoiY-hXZD-+>`@aFyrV50a@DmZY$V^4;TlJS{Y3~Qa+~^cK4BECHKR6g@yWZ>ukZtJ(5Yx%CWBWjtm)ID2$58KhKC>z7R5Akrra)|c8_ zdfIwq*$J3VBM39inVr#wfM$DIFp!HNlOpgLXHH z9(-{w^4G1xY}I|F3y1gA(R`mMyHw?F@YX7Z2%q0OA^6@x>{=#@Ph9JS+rHzP$Up;4qYb&mX`_|wwnhhNtZ{X)uSbaz zEo*mU&34NS_(pYEeQ`zo!t0na;V&KHZawE;eoQbL^*(ny?m6`NrEH%_ z`pQ)S_lTtEdEAL8kieVwn;fS#`5#W9v`1^rI}78xe?PBia2$-UlZ;i3)Hq(hjPNKh zPIa8r52=S>oIekxe`3h8X;faW5ZXL&?pU@4q!^EmS==+7ZAI*VbMq`F-#~r2?xv|5 z^-v`Owbz!=9^c*b{IA~98BGN16Lf_FB>DGx=V^ZRYlA51QXf7I+_%dgGIPGGG8S_e zPBWk<4S;7mpFL2ko~r2)99QlwrD_VrYexqczx6lNS(G_Qgy)PuyRo<62DuNKcn_G8 z%4QHYG(2)XaOxRZye7W?gwT)~O92HNhwUHZtE~2He7SF7X@<0}g}iAp-unIU(`gk_ zN`iIP%-Yf*s2XpeOrBod<*eVJfK?zgoW~P!2?AfxN99(3^#s-W#?nEu+A=$r{RSwP z>`}s>s*T&W=S>}uv)-Y>+9OZV{Vw%1R!U9~4P9JndYw&)tkGoFH zUuq5sXevvw#AAH7err4UzH%uByDWa}D|SLqpj?Q`S3i3Mz@^&xz8OjFE7_#%Yy&*` zS-*FU5s01Uk7(dCK8WC;9j#4h=zAF5qb1sILa$Wm^{(DHH!=LF3~Pk>!Bc}i(umVh zlM<4@8mTuu5x1=7KVzoB+hy6Q1 zqtg(VQSKwJF@Y5RJVVaVWt5g?Udh=`D*wmaYg~TbxG$xORV8g?gTY?E^6Fh-ByC9s zWXf5PP7t413g2`%rw;%<2YZ)@T|H47L3#H|vJ6phQGFw8Ea#{*w@IOL^M~#bAr-eu z@S3?Cmi$@*lu34t;d@T*(E&6EE+ot1+-HtzS27MiJ1NheOc_XwFPm{Di!&ysv8x&3B-(} z!OOKdyNwZn%(1n!{EMn8%0gKQ@mZJ5jJnIf?5Rj)FhMEg#p?I!xwt9mARxVGbiSxa z14EswQ=H;!20h)<5$1;`T#h?c2+?<+*ZXhdLBm{uzP5bueu=$N0HXwW;H@=M2vBNb z4sa7QOlGoXptqf-!$U*In=Z~u)x2LEF^d~knxhJ+))WP~sVo<;a|ZXbrvkJ!pY_tmodmp>}wj%7OkA^DvX2NKefKGM2I&26gfMy!Y4OX12-_O>^Xe zH(8Jl#Un7u`m{>7jT~7RfDG}Ss|p?vQ&YcDR+1{W-98i-TV)vszeBmfWQY>M1dz{^ ze9Ne_Oi^^&&ca6H<5Lt<1#08=?`r01-Q|<}psaLh=>f)vaUB`GW|^ z;swlUKhDIE`#gMpN;a~{8e*nfl+??^z`7{vt-=D%QJn`Rh>N-5&W@i1i)NJw9d1Bc ze|saKbn26FQ4UMqC~$2~%f6?(;I6>x`Ao|)nJ-PTYLc#_lw}1(-6_d3`3C#|Maz71 zy2>2hS4L?8w1Srk+@K|AG7`e!wuOu4W}u9i{bq(zjOV+K8AZvrpwyOo#_c zT*}^S>Bre_wr^Gh@3!EXTWj+18e{6Ls53yduKJtMkF_ARQH`jJ(H|v(evi7D#fe8L z1e;khx?n!sQ&gGj{Hp#|Gnc&vA9{OHFO0dlRUs<3*y(5?7}C!}gkR}p{a0jg>X4F> zd2W;9W+<6d2f+L7<~!kEr*;Yi*Fp&#UpEnnMy;Mlk_5q|Pz1VS6ZA9SWihooz2cZN$KxTKf zR5jGYK%Di5`>V=k-_qg43F*SEr4E$*eYJyMX6MD~U1!9Ba?f7%DJ|{sTR-v|FT}5S z(nG0*!$(WFbOV;VtYPfHvnBI(-VAlrAQPI=njqV9;1j^OOUY#w9c~okFq6N5=E5;5 z%tKw_XptGvBp@p%nTP6xc-iEEzq$nLIB8#fWD-olfMqt)SpfIIz+|XPO@pg4hwXG? z_I_XM=kmd|fTP7sX=&*h?kC>jb@wa^S~RWY-pp%q^B0r|Zct|g!jbnj+Cff3mR3Q( zK7M{nnd2CO`o5-q-8>&f6nll_Q*ifxjSUp((DrW?jc!+oUp3ecaIJCk<+Z`<<+4lg;hQ z$MY#duZ=YZNpp0A!vk`$V7PGhxo|z?V%B@}@WC~gzZv-kvGEh26S z)VDq!BC`!XQ{zbkRia9t%nLtdTY~4C9T_TaHs)fHd3xRZ8i}Z;9gP4&xvBIHw;#?*G z=hG8-!eo`q?U^1jW4~_zU5P<%D30}Ewp_oX%OaP6W)=AQ(pFc@a+!69UFl4x+Nmrn zqm)#1VfwzMF@u~?NPJOIU3S)KWhEtPe=Qk0Nvo}|+(<#TzMA9RP%oKbWkc(jyAW*( z5s|X&tvduaZt2i`J4UM>CK2Yn8>LH6-G8uH%1~IV&g0(y+J3_Fg_Y%$B*cDZ$8V!- z$^&hCAAYBJ3DF=e8yfV`WOl8w(dGl$}+9`OsK zJAwJr#whvBsgtgTgf|$PBA2wap*g}1yZk)Z6>Q@|1P@|i{@wJ46FUu(*vl71S8ZCf z-TmKpRn``0gR9%NWmSo=X#{T*_UVa|3+4X@-D3g0-;E-I+aq#6sENJ@`t zYWgj?7CPMK+PAn%>r|tQ6@V!T?^$;B zU-$)rZPw)FYP!ywbYX+GgN#Z6Vo*ZTC}BH2L3(R%;`EJx(Uz%M->&t}YGiBM`oi7R zr|YAVio(TCI)+4l#;P7F}hz^}B#=@$f+%e2FN98(DZyj@Xx>9rO zU59b73srL}+4gE)hq6@DLG$dT8>3!WdiDsymepGH*n{W^)3*AWymmL2P%ZYM^A86e z10C~E@v+*z9{Uxq^ZgIjJFLq>aF!WK$2Z^plXMl4PH5@mJXSA#_T|4GvhK(K^UY(& z&OG^p>z@aI{l)r-f380g-ktpC_oc7@)i3?syF35Yq5aQ0$JzdQ>E_M<{fqydsQ)RV ze!Wh4H^EHWx%3Zw{_1{p>OXs||6hP?_3!YY_EBn9!e!sX|8m3s9}M@4dHUEfCbN`kgkoXl)Zb3#iDUNw6%ou5>WRNmq@$_-xa!fX Guzv%}FsI7^ literal 328769 zcmeFYcT`i`+BXUaf)qiD2q;DARXS1ylwLy!y$D1IU8HwWK)Un}(n|=Tm(Y=3rGzRV zy@RyS;l_Q=KKtzR-f_oxzx(}p!Wvm*u9>VkpZa^s3eixLC&H)3$H2fKQdD>az`!8% z$H2hF!^1+4yu%tWMW5)+6#%Lj7@o`+82+CyFfPzT{u>w=F3&MAw$Rr_lQ1wS9n%n+ z;^+%^&6MR|Vch=y``T0xiypysR#bhByLA_fgrDo^yP!V$%NUBUq=D`;J9BRCK=4fW zfoz{jeqO8`X$%P-rp%XYa_La4(okq!X(1c|ZK%j-+xn$BZh-s#$PXg zciN8CZ}EAb2LXT4*#nUuBC&C__}|) zfhh~te+T}JBLAhwf5qYdTO8nRv@iXb)zYP=WdCg<2FCNANZKDlQ?eUm6lH(A(5i<0 ze__BFZ5iI7@38ihmheJOnk1L+wk@L%<4yBk#5qEEZF@cm?q&YJ`@k{LQFMgsMfy|& zli+VF-g$-b-+_Ol$p0032$a;AuA4?>sgJ!iv8!QpnMAbFlKHdGC;A~GrD@PN%Xv*MZZoDh!1JRm{3dg&tjjg-ly$I z%DicnEz*NYJPhJCKE5&1R2mWEe|-C^!Asa`Xvgv>qmow4>%PQM%NII1$JK!!o{f9dF*jT^8Qb#Su3 zS~csEFJ3V3HmFQ+akHFaLU?_E(vb^3V;Q3In_DV;ZlGRhL`=e)LrPwuS-v>-kkMFD z?8k&ALoLMIa!CKtxv`&XHE7zNDzZW(an2I!S1$#{j&AJB+ zU=K8(T?d<8%z6wTLHJv?L53~{hAsPqwj#Up_#ps>ut=iJ29bC3ar6i3NkZG(ypm=+ z+i>HgAG05}iR~KJr?1$-WmAW@1!pUc-;D!#Z{Wr|D{*L zFEI$z%vtA?QG1r%_d%zJ4Io&v5){sEqqN|+B;l|+&NQ~CFT_5R@d;=E?kbOxR9et9 ztvb_gsL_PljVG+eI~c_-Nj!Dhw5zmF$*vF%0{6_tnw^7a+)%AU%l?QX9` zf=&g<1n`xIt#G<>dqpSUrxd&WbY_7`^orG2$JQz#>qBZmj~96~EkrGQ%`fhY-%VG7 z4)+hBk6UZ)GZ^upK>{P)4rYBN2v#m_V_RsC-jrEh?ZS^#o<^H@b<6bz5Bk^v{WCFS zoC`~jltzo8J5yOWs=gEQ`}uYQBd2`VAamL3@+|$>M7=81*g`gUlJ}(DazUWz$@N6C z?1~?g0x6|>%YHuZ0k;YQ^Iv4$pyR?@Bqw5mW}c&l;U^i*e3Qj5>Rc}~_h+hvWLXZM zBgVC*=5JnEJ8W`*e%_~Frh@DH}ZD&Ka*U!q`B%v;3S(}%`=s7gNm z-03q-{R@qTGiqfieIsj20W#N-wKVsOWS>iF+`Z5Bm&F`hw4GgBp~rx3lnt<;@p5^L z=2a8vy@2mk%T07f$!(;cx4D0J|0IBw^VmRGz4k-u`B+%uaPTPx6!aI`bj7`#w_SdkZ9aO)^13x>ED zOZ#kNqE1s^FaUT3&;mGLXnv)kbfN2cvO0DfL~eNi1id~ zwnqO?cfn^;0#Q-#pDw$YisXycmj}P5;bUP@r}IznYl`YCgs3bep!SNdNMl>x$;$bH zWCy;YNaMSBGR~JC<jyrkD-BUP+qTk7OqtUs2tA}G&5>12tk!-Tw^lRL z&|NCAqg<|{em6YCK!AmhSI7!jl37t*B+|kS|N0bcGv8KmKejp?S8Aq-VX*?dWFjWB z&f{t00X)qdp8B4YYLXtWhcLJC1?6m}ZMR&o+RYFMHRYnn-M2%TTnP#PB_^$qANC*3 z1%kD?HbLF zfLK}=V5=xJ+a31$HBP3R6)jXi*Fx)lF;G|*4~ryw=36AL*D=LXzP0yrncUQiqp%yw zND~KR$UL_s4iVtwX6NL#o(Nb;<-6bbYiZ<|pODvpuIF&QyRV&uJ#&0ABgA0( z+ysj$>sxGY0KEeEhFKYUY{SIziT_UgE2dY>nX0{y;H_HfXJiT79Nwp$Cc-Hl@1bsV zbD~nH+kAwzXee*=8=Sm?C7T&U9M)W;#u$r)kFoyTrWS`{8fU+f4XGQ3YNrm=E1#w5 z4%~MmW;Q=HNbqBq>OE><@BqamBi)(zB!NxwkobkZP4fk(-~6(9`{OWIC9JD%)dM>8 zoKbMBF#N~y7I$po&j2#08xynf)_%+{)_?KdUveaXIJDc4*27-~+d8&Nc?Vum@-K)@2rVnSjMSdh;|6lFV= z$-l6`VOI6s7>btYW6)lAP#ddXG3|0yjiMHhDN|5%fZT&z9c$iW)=B^VbtKf*OGqS+ z;eElWJH~|Y7VhQZZzeu^i+K4|lYJ-~IHspaO8tIPb}UvE{w8a2-H3v_@ll*_Nc5c4 zMkVUR_Iy$!J;p~gwir*5@IljMUzPc6zY3+o+j+qj%Zt7^&dtMV;=%1`@$r1HXk}ty zhu=s#r4gkKqVW1B^Q5cBU&*xZG+4ThpYquxWE~Ln&ZrfSYf2l)>tEVw);Q^Wcm5pLcshT{)QT&T&wFkHKdZETzV8htr%lQO1CuUAOi@gO!ewsxXNIeshIjPdk%0bf&u{46#~ zV+#t#GO7i6+s9v=<2ts<}lAoAWyTB~^EQQ*sUiE;(xYJW#KB{&5grwQ3!gjc3ti=vC zT>*!T2ugo6mMdW3;aDAjxiO&CW@U`RH13}H+X#{R(#u+$a|8UCZYIUov~AjiDTf1d zY{6=lXN+AQeoby739-t0)u^NfX(OPNVA8bC3=lr$m#!!dauj zfx3WPB@4EqsXv$40We6R5hu%QYyyM zt`-(9N=+R%a}%?jdY&i4o!I}I_GnI^`C?e~g=593-Y_4?jy}oEQ)~fP$@v0RN;Ud^ zbB%#c$acy!lLBK%9)lp#YS$xia(1l7t1sj2bx-&V>V}4y%4^M(5tEypU>ua35bw#% z`ZKxC6nK@q@>$$N+G3+-F90AjF_HG8lmBkR*EnW#12|IywloR$>aTi&s|lY8Bk$L- z71c}mxsM7c>)#QRw(Vfq3nNORbxcRk;Vm@PZ;$=KJ?>pKrQG2)?y z;k@=B3K_|=JDGl6Utl7S#T*`vPonf{Z-%w17UeBd1}o1`d^LKMo^VvK&lY_=AV4#l zPIK^Q+VH1sJ`(a|AtY~TNUd59I$?#A1=^|MqoWH1WX1Kb7=7RhR4Mw>zv_f0?rC(- zswBIKU!Y;>m;Rwo2y!Qr_|b3Q%*)hS{3Rc>20Ux@x|^jxYu67=TGwa9=ji1Maf_le z(w_gOiw3%DDii%;7K;{AV;|Vey&lG~bIin~i`(~^K1>vth;xv5<0ND*&#m&DsKLEc zoi4UcRB-#agc_Bp0uN256#XnqiwF;x#@T@ww6+^#F6wZ$T!hY2kDadD&g(?$e2>$a zyS!6a{c{C+ii{VpJgdpAgvCG?JpECITi6dSL!$87Qvd;?~AS1O^pNH)31-xR{>{(F)>*zzF#R+H` zWurioasGrZ|8>3Eb+94#2^BjI8HTVN+L3-?1uIF5Xti_~j#o{+CDTzT<`e`Tr-eD4wc~u)mW1xlKLs0fs%ps*8A}(TMyndb z`S%}*lTn8A5i}^s$Z@ME^!JJTpsMmx$EYohwuW~-2<*D#e4<_k))tJ;&ZkeSZDuz9 zyjsCQ%UwV|uaDovnh!eV8~U=>=Y8Q=JuG!HRLV2tj^!YiM4RxexL)wM0ETVnu8_#frG2_Bt~lof z1AC-}HWmDHm8&*{W=vVt!B<(v9DF^;ozRml%NXF8FYMsstJTD-G`%3SwY$G}#eR8g zQ{?u2wSQfgCW+g|CK{qjZZ|VT1D~G01yYZy<%1#&1sos5W(b^!xgk@2%s32RY&0zR zjNc0FZW@GzQl51af51)rY@JhS9Z1*m}xwvTV-#u zEWn}oibu2QNPp08dnRdQT1HFX2y#(rNU^%V1eKu@;ZUqERT(?c(soq@CkyO5=Z=WE zTA2AKN?>vZva`P=p>FUKN+915d=H!$1$Mx9pQn@C$P|bW@ykz z{7={EUm2CN*VTRn=OoM!N@$kLE)n$Lbt} zE7_1+hZSwecbr$b0+sTP52D#u3r3Bao7u%8ang`Mt%L8P zvYl%_OD0Cu6b2}pH7=^O06iScCELJtvT7gTdj#Sct=WjLIsgu zPOp4s-2^gjGz|`rwG@>J@#ASurYEj%5KSVLn(vGJ{JzC1V3Frb(QMPq zUY0glmh4YU12%b>u%F=}8hwxwef5FCQBEFNq1%>h9Za&4wN^fa>vlzf`(J$n z2uSrBW{DfFB{FW~{bCH|XqD}PUA#)mQ%VXLGP>z3RDU#U7vRtEEy2`#x@KXfm^mch88a`A>-YGO;V4xsu6AFY1sUKvy&_PyLAXLTp9u zqh=1$+Qk#iFyt2EMz&xKDkz$63`bg3i>8C1`%3mM*HaWn)^>I@JC%GS$6Q<&2?+>< zXE}`$|=(jvKeqx1sLIn)>yD~vkTQ(5CBoygc1bUr;Hlhg=R@s)_rrH zZB;0c6HXi8@O3?GJZZ^*98Kz1+S`pZN`8#FqpX!U^NgNJp%GRCi49MAoIB;RmMoo` zaIrV>g@uQR&u@lT$o`hqMLnmNbiWmHk8knE+Y)x~Q>4$PL?}NFPtsnfVNRg`7rlEK zUTQ5C0sW^C5{y5n}(Yl;q zzC?aYKHh!V;MDp<`>KrrWP=R62 zRROc(ifN(a{n)c%eIZw)nR4;i=mNi*&5VJ%z@z1u<*OW7pi+pbZQ=C!@(9ml+-V5` zQ>u_u#VyU~AB6>G*9^I9nx>ouRN@a2~~zYp@9NPFM7Uz)|&A_@O0 z>Y_8WR+T?N+Y&R-$jm#>^x^@$ELU^6(Pk(YP$g(0acMU}G3H>z=VZ81bn4j^v=s;2 zG=Xt#ox2~TrkJ`NWMDoZ@t7N3oLvyp!iK)lndxGRt|fA6QI-X}j&vc5XrNx45o%>K z_lD5=BtQV_F~T)?v;Rp_j30KAkj?5Bpd0Vv7>if*o@)Mg~yprXUP7iwJjh| z=8|dsmc34RZPXtL<`)dC)}Jyfw6)Sq5_L3EDg^gU_2i=ln}SW6oO72|Uqlc^)Sn|g zlJg8|)+_~4S9707lzB9WHZAY{X;$vWfozOFeVcUqm71iGOBw<+o4Gq8x4XMLXsxP~ z&J`i&g+NOiY-H0J?I_?ARxYAVpd93u3w4B=8~Vz~6gx`SG6#i1t-V!!Un#c}^eu=; zs|V~9sPfz0qoJ^Gu}Aijv%50GK6@V$DXz&N7QY8zr$A4m+3_$h6cKhHSq?{)5Q)*v ze&1)G|F&=FF+D07=Gs%hmFnVxT62*Ib$oP$%5-{3bt))E^ zavW6XVU|5OQp=2}Y<>qsdLBtU>l;NAmRsXU<;puXz!IxzF6g~3rO&ZtvhPJ+-tjgGR zqGVTJpI@OF5*{NLgEYgh=u?GChD^@RS`8sx5^x`%1DhJywuJYhv()xS{LhULJ&UbA zz}S`*(`A%oQ_BJ_{OMIq>T(c}BKsdXlLQJTt^J0rJe=(8WT8ydgLiI5G!hyvr;Y8T zW=nx7B^i~*K!$9hmzfhI1X=_D386jV><=tJ`pp)d0p1+-@5ftNdp6%TW5P(hV73JmR1bcq9 z{OJ*yBDEbdTVlk2KU%#3a+6Nf(pYQKvHY6%%?z2_{Rm^3;OTZDn!SnUrAd{N|y$BeW((mNI<2;3$E&uvMa`P^!D~-byG{9!Q zcTYW%qCzFMZC<+a@jd~Y=3v7%Ux?W(2aRnrBewJ}RI1nH6*?JXm}GcJ+;WoT(SNso z=JB7(DjOkZ{-70*mNU49g>|(HtobQ4=%=Hc*I%U@Lr1obMXTlDXdp+O4uYd$ zoe|X9Sv04O6|JOfvSDrC%?cf?st{{DQa-!zL)x8#_P{!M!UDX{`ltcHO4t0R%jxou z$O#Y%q-Ml-T?*!13R~)lh1c)2#8ygr3cHv`RRPs@AC<=cztX%S9k8+(bnkuf~>UAc?mGgu5Rc1aPQrij9SKK`-FZA zSm!#sjzlTh`RJdO_kub}g#tpoiw-)=C!(kkONbc$46I~F`0CK$?T;$@j)Niaopz3k zfZi~r|0KHkn|i?+K@l| z%m=KrDC{;J=*D%QIPTGL#2t6TyoIWJeQ^AtxZ=aeC&`a~hD$}f65Qs|xp9-8u>4(r zFQ?8q4o8?C8r4Q*KHNe;Wn${wFMNa}-l zqT!9aynLm8&e)a!LaVyED*qNXW?h8_X$50jf}ZI$P=jnaC~}*KE2u4_3fi;kBrU?? zDVTA2lBZ@*TEEOgOEbZm)403I`A* zHGfokpL`7f5K*L_CJVL3Fb&r*q~{Q(2ujx+s8GrTM!!JTuOz%)E0`XxXofwE z%6PjixJ+*RMk^*WjnCHu^4eM-V(qG>U4*F9ANy)YiLD;k!pS-62i*D9@#!<0`L|~> zcP}4@WIy&jspuJmCyR?Z@wF(?T(2BU`|xjbg@qC+WzY&cxEfY$u!f^878Q?brlI%7 zN`g_|=Cn&a_+lc%i#DF*#s26><d7|yIGibZu|%*W7v(s3)d<$Sw^;d) z&XnU9w*vS7$;y{>Ria@S2{9NGFMs{|RVG&Ng1}uW5;f3IH_kz36j$bnA*D16kVP&R zrG`2>RW=(ewLqoL)`y&}FKIvqWm+WIMFW0i@}prPTI;D?FFYMnKVI%2z6P&(8-4gc zYR(I-fb`b{)A`Li_qO9uHlYs3K1y(jEEKIIs1LGtX+)eNUN!h z`jZ3oeZsQ97(pzFy{O%b=x(@fsRB{`{l7J!`ZtW|0b64@3jJ3z#Kpz0u7JM2?Bz7B z&dyc>`Y<&((pr7;hp4utGE~CI8_F@1w;SrXDS(a_p<}h573|+FrW4?q{+W*XO+B}q zk*&c5*50lwSh?gXE5`n5Z$nwX%I z+&Zi)WU*a5#v%?E=drk6J7sj3od0&)pZ1&t56}hRJFSQ9+Z$HD>G`1J^ zIAZBq3P$~pg;sl3GAW@8pb`-vn;MIx=L1_H_B^LD-21`*g9rTRm91cx{dCe#_c(-) z1;K#qouORrQF^{_dcaUJ#jj zRvuE>ZE#35)v?Uc#c(M&5$*Od*qAc&w!oSlj z6l7@}sKAg&g!=mPhLcA;IZ7QmqJ*Kmw96Gg8t`k!wHPbmkLNmm_x0{5eM{#$a(Vo} zeWK%!r$DNW!m{SjpiLTwvn~ykBWUn6wjUk(UpwRh*yi@%?S`&Hv%C`;|3kBeET-?; zf8j8t5~y}Lsdk~R9%0oEdfo8}N9$M56FpGtVR=?nJdmAg=fgM6zcsM?8;t1PEh9ht zK>I1{W+PJ7##dRwbHn{j`fD;#u$796^G?#rc^G$^Fw>0NcQuMMMVohvih1*(+QS50 z6I|KHf0u9&TEf{@?#u*~2`$gA)B3r2yv0-i~OgFZh510LO0# zPjFszYw=f9yl3y<-m*c;g#f7@(>~;7J|C^8R15!FI{I>KYrufS(@T$7=bXLsc9XZ` z?=o+nm-z5gD5j0H73b8i!}MW$o?4%eX@lx>jux}#>JqiiX%Ts4DV1)1m7crnG-KPI zx7R^DH(c1NNp7nZdBQ_a_N2$@8x2QkRBcQfANlPSs$U&8#HT-NZnTC{&zQPVJS(}(sK4xDd-D{a z8`PJ%;JcnGlLO2K7BP~>1DNjN-H$2ERVo4$0dTsQBf^;Y#%UrZ8nF__XM->vQj`t( z{wI&D?-pStT=aC)+3r2_S%u*f69DdQwZ|alziG{UbZ9vkj7xObEueg}`(d5ddn?ReJ-;^wRewyaW(z;)J?Kpaz3F6Njq~aE7wG?S)5B+ z1b7;)luPP1{~p`%XO!NYzc{3Pl}!_&O=yIZnsMb$?<)heb`?0&x2_3^=+hZ1w$_d-K8 zsJBCl$D3{?_fyqtjl|w4mLSK0y9acxD|*S6YJDS!&Sy_cG9P@L;p=763{g@?f8TxW zCVs7)fB$RrNLzs$W3&Y&KB=Z?ns9?)4Q87`N zJ=4j95js*?vj0#m=id-VX7A#gbsziIU}9%@>Z1{_pFP5F4t#_i)UVwUb>iX13a?@FYeK6z+RHwOFG_T!TVmWJY7N2q<#gyq9=xTL@ zdTK)k>z^~$^@F_nT7433o%0$)Ih!!yv!W@Lpmh#-1U`YtZWN_CCGqWSFq!vT^cRq* zA~bbzcGaCzVn3k$+}0NV+iN(gxo?!Or=+IZncS{YW*(l>BspLaLRKlXhb3=QcDI9r zg07C}h>1{zm>B;3)xgO#_amd($#pCUmTUNYd~8j?UvoH(8IU zPFQsy^+)<+GCw}@y_>eBmk)Gqk0dvPqe{4WjdXokt2HDb+3^*CCQxfk;VbP{zT<~W zq_0WoxBMf&>w;P<7y|MDL;z8eA=$q@6jT=#8F@b^Oa84T8WZqMV_=uXmGgJuz_@vN zwDWo|mf8%O@s!eP9Y^95O++{teyL!ypIxm>Ni59RN2~?|@Hu4_7Q+%2#M$bi>;huf z2C3^8Q)yq9VFi4=kov;x)xSn~bm#S;fq;FssBtwSR~}r_{!G3 zd;SP17oiE$wDH_Lcp6Qpi&g!U|oAJ95(&j)KfdGM-{I99}x~S&y+NysTgs%r}U_4XlPghsjPa06UdEm#QX{E?b zyXoAl@^bX?_3PR9tYe-QV|27LBz4=UH&$RIvVHa7!Qw{=hQD^{-%w82*0XvCh^b71 z4H;NUTo8~v`5C%PF#!?;J^69A*A}r~%sg;;#3g+D#(w_;yR6Z>`qfg#5BH?iVA~dj zD$|aw;zG=dhT_lWbx5$akH_z2Wv&2|Zvg4BF-Bs3ucWl=2vdwN_7$1fHSY*N;&OW$ z&l>2e2$Uij<#CT{602v?5<3fe5&Xg@J3!Yt$Y@Ueu z&Q>nhN(9gw3Rrd-`8Bv(M@n>)6q>q(+xZ1DoOq3Xw;OLC;gX>W4=6Q%z4N6pgac7dKn5aCv_G4**7*2nlE zLQ^VyH+cC*8mnT8Fh~A2Yu^;xL9vUr1irrZV|BLC0ph&|ZyosD9c|3)QMtRSvOL;V zKv{tb3yKgqyr9A3Fd^<;~-@!%@V(sk-^w#%ulu5W`p{}#-I6is7b?q0M&=Q zojY%@jN&#H@hdC1->RLGihRwK_fg{a!|?xj)xC)L9nS&@<-&wzePASyI4Q7;oOTA7 z%q_VeU}x1|x%WlxrqYGWz4CbpXgSu|_J z2snW1dY=PFw8*fU*|uKnw2$XdOrmR)5=3vG3mpS6l`UNRzV3nZU4n&5+&NaqH#Kpv zp>gbaub8xzfJ!ELukrxJjA)dyDOOg`+&^j~$w%j-tbA*rdDACB(kjD)dZKW7DSP`} zS6r|TXk#<`Oug~@RAKYp3)(r08SLnu_i%tzLGXKo&3@!fMl~E?G^F-NmE8VJr=U3+AXDo45jW6BL#9%*RXvx1 zXjbFGW9P_$l)@;1A3#nDL49#V4ch~W?D<@20@>X`edSu&Oa|1Rgu1Z}5;-?BNoA61 zay*^TTz1X5I}vg@`H9n(_BT(*wrVhsjL#fbvtK@te;*F+lVOe*(w$Xef08ms&Tt=- zGwJeR9i}dE2{16YY^6g~RvvYtGino)x4V&2L?{TxPj9MjHA2*1N{kK%(J@c_9g^13 zT;Ges_fRc-)ohj3z>~WSh{>~9hmKsDOpj*_8_{Aq;1dJ+rpEXk=x;5UJ4jRMGYoucnRCvR*6$84$ z!$?L6yg0`*rcw>OUh6tKbPH2^vKcc)e=hY0lUFuxW&w*a2Zi*QZ*4cZHnpCrHkTOT)ON1z(f!sDjB`4?kF(v-?mjHdL$BU$cur=X>K=k(4G;P z+gDpMXFRX&t?|*-;?GvieoeQW|VpqIaX{a}`Xa*Q6qsesf^pF(;*@ z9cqe>;&`dKqCoG&^Az5{DFcW0SP7thE0;FAdF35U&Um787M4+K&O5pjP=g|Gm|#rD zv_Hl}wHL{$x2M4wv_EEa@{K)q(?iHem*qR}qh4Z&bfXqf;}a7vn(6+vm#==J=^ppx zvVe6rWqngru}?vXai@mX1BPCcne6*sYkM>WpH51;ElMP+Zt5u~AExZH2)L@44h!C4 z;TD&&TE2UfYS(pqdedaRMBtY^VS`ZixjIsJgAP9IVr0fX(be#>I;zPwt|^^uSPW|t zQfpi>mCJjh)CjIFnn&O&x?v58tah8bEY#0&@zxp29^0n#+^Of$ zX?jNUnUN1;l8-tZ-gK~M8ue$tBL>DgHmoZfE5RE>p9H^6D`8&5e3rwDYp+o!x1>=5 zoX5w`#wIDmticqG*LBe@iwkg~_MBEe%k4DB8az#_+UVn@k?iF)nRK^0ft^lBowp9^ zq~tXac1ktg9I_P|JrkuZlFWvRTDyJ2Vx7WF0rR0}A&OaBBf@tHr4G|4CVOwaOS#Kgh1h>o%E;(MOv zgQfRT#Hg1+Ul11;K|%RJUHRzuuCDxx3+g~aA#p}d`c|pd1)kOgzq4QT-BEm>t3=T? zH*;CpN_|+Vjg_sn_2{+%P$>@%A6;0G>RGU~IR4y5yUa#J6rpNetq+6P*qmw=0ohll zeGiSc(odzh*tp!hrS=N)^GoaX=1NA~Jv@qkTX4ST&rQp40~w0z^FbM$d4q}h`~9PY93cfv zb|=Z*WP*S^Qm8i(o3uL=88eJLPZA1EurdsBlszR zz46++$)`arEVE<5p)B~1GPlU<9AR`)GycwlR?^%NKlh(Jv-W!LG{$Bf1evJ3l=({H z;t$pIYn&6nuQ`x)G`Fsu`Ch`uPW@b#M@O{|zTlOr^Y@m6sie%>Iu9-K9cC&d8WIh) z$4lxDnwh0&*zaFC7S@s^DRk|O=eQ`FJsTj^l#@j}4MtM5+!M*b#iB#mvIR|@A5vyK zAGiPkTb+op%jmDFAlq%S@;mcCCUwrcZA!xv@1x?Tn1dzAveVS~F_p5*%d_7q1PP9FNlh z7N=H!_;AC((6;y{%bZM3{ZZ!6o}o&$l^uyMpf4ssePc~sAV6pGa(4Rjud>3PUk{!= zU6PMT^JuH8tBd~V0}3i_&yGoNIHgF818Gl!bS6PSkg$+Y#AiH$U#A54_a9TxkkODy zKt|7^HKRXD0N&~A>jO}^Z)t~13ro;pcu`3S8@zDGW)=PO8-3m+Whn6ap*e0k37 zTNdoF_X^c(QoCDRTwGKHg~N)V#YMYT4(P&07#i6|+rX0-hJ-B+oU!Y!@C*R-&6?3oSJxXe>n9ykcRAtT=)dr0kjAfEYU@jh*~b`ET&Spz2>%5J>fVjxnNZCd8y=!Dv zO=(_!mxk8D^gQ>yU$NLYeeLa6q+}M0Z?aKu3!yd&%5W*q65bHVeb@IcVM!!Xz)&X1teHsJ0Sb5K--`I;G~u z&p1z9w;yV9(FVb3sKWrlFr~Sw;+AwMx4!E#_LyG}O{JUdo&*PT0<)Vz&{`kf5!{w) zyvd^A)hZmjtq^3eCz~BZSbwM!KySs#-8dAf1~pLh39?8beEOhObP`g9pm6XpU8tF# zH}-2>>zj3G=;hoXd;FOEv3D`~_X;WFcvnXA&Q!XoCoN}AC!$cy5|f&ngiI4gJQ)Ln zFnT;=W67pcAzsLj@J+4D1!SA-=63V#JG92@-xQaXbqLVr0@(A_(w)!qC`~GUKE2F; zi;R+TxOTZ5g3Y>4H!GqM^iZ6Sm=;Rn=!Z?ErE%G>nEEiDWHF0O_}G2AxvOo5mfat- z>Se6}HG5w*`-D3b`+g&7R0L}#?jwQxkE!HNDdtcRyQC$FSK^W#OZgfdy^9+H~bYf*>4m z=!P~T=$u+^mG`oG%Wo5x9&2+xTXEP|ai}Cs;ydfH#lg6)s)V>24NEkvaqN+#a~;`q zX|bQHcImd)oL(&@c)ReSs+7*hCmyY{?~WoSXP!XxPtH>w`h+2o54qAP`T2@(U3{-R zwuQGR3Z4v=UBzEm7!UO}?$HGk6Q@YnT%1fAt!8gGW@Wv*jxz9_tyTwaG!_+kTz7G8 z3)^e5QM_LtFl@BCu}y4R5;3wd*VSYk=2?ehr%&O=)FS zJM=Nxw6WySLTvEdZH2D1f+q;+`2VnU6<|%a-y1(g{6qw#6bS_tWrUJSO$0Sp8qW-`@)_xj?q(;yurK&biNh-_N~V4^+(d zLTdoFw7kf`_^rT|7h9oOt*eJiwFP0q>6PqjL`+_O0o<{-;#?st4GOi|ib9kand>fm z3)<#MIwO2BdCI$u(|@(YZ)d+^uFmjz22%;E0Y$?-1}bg9E>9-X!UCngmY1`a!>byg z5t;_C_Uj&v>I)vQ8aAx8Ed)Qs$1u7RIxZ%^&(A*R{Wl_&XN*CB6U~?zsuBWZv%t@o&(XaWY z39-^rSUp_Pr25Ublfa(qg-Mi_{M|oGL694W_wC8jt}uTAQ$MBI9(B4~Dd#m;wLG_+ zo!o|poQ8*(BwPUj1S>1MjHmy^o-j^tO!>cZ{Bo*|CG8h&+v&`mr8KW0C?%S*j+Hc< zwQ41Pj`&a<4Bbw!i3=gLq-Cd{P8aGGWf>uaGzVQZX;V-3Wsjg<|Gwp>SSkp+S0tyQ z$;3=|L`bz^xmKyTBdn+Bi3IR3P*{%^P13CaGqG9_W?O7q)Gs$Q7wYN6mVrLnjnQ(! z3kpYw7);n*c^N%tzxYb)n`-Dga!C?xWo~Wfw%(X0S?$m6N2;kt*P*!+iOc^CzD|8g z#@lg6$J8v+yB_A{jmY_7db8@~PA6y~9QFfK=txyng0NXlpt5S^#~Y&c^T)kG%a7L9 z5@O{}tbDwB_+c-b1C{%M5{7Lgi!`Hvld31p$CyI(5hs>rLD{_0voJeKg)9Mlb{4ED zrL(wY zbZrE&-z3Oq$h&SRDp_#d43WS5qvo%W6UYKUUsZ|Nt|_e~W_W7;c`o%m zaI6PBHF#?S$I@m52tB9uiG^9|>G_??BGrg|`*baMC`|kqP>eo0!?;P< z&csi+jc@S>&FUQtz>7e_QO)?vg&Cy<>p5;nVS;M&IzV>{aM?Z3k)}k z)V*xh_)$E0SV2K?=uW`x`NczhP3|-+vkWYD5_(F@IR&2IV)Z$SmZ)$jDhCnVb=G1` z;mnt|!yX{t#F-nMoy>huQDICHx|S)?9g{IbZu=e^zA64;(o!tb0B<&AhPibh;Y1yg z0m7|lo+M0K)nAngE-5;3!WFJpo@<*MuJygSEgBeN5bgBNCGeeV&cnLD2 zg76k;;I820WD4cnN8h48*o7Umw_%*dPODd0KHro(Zj}GpcoS5rIz@UzUMbjDqRpvn zN)P$Z9q;isU{LeuFT`9LC1y1iEnH!AIL(_j<$7k@ySo!P6{L@gNv>WUMhZ2H{F)n5 zbeJLbnvq~edtE~pRl~#YEO>H$nY#c=Zz3&Zk-Jg{uNp~lA z{rpggUbPn{Z1Px(M(8C-mliCY|C3L5Bhhx?_W&-rosHWCF2w}W^t!2@ubXK zCBTl4eiq?jAEaFgYe}(hrD7GlvT*4_nWvGlF)GQt9A@3=9$~#`h!;*s7P^>Z+H~h} z+nyT^yf`=a;PkzDg&A(9lLxWAP>86_CtS~yC7yYnE~JvYQ5B?OhY=b4PaRcD7S^N| z;vDMwdeG~H_(Zh(2rn(0$#EOWCzSpp{OQRF-CkgoBNhjS!=5dz><9ta*Zh7Vav_kA z<=V8w4iwfjxWpv=vks7()L7EEwu51JGk2L$>v~x8Uc(x_TIEfN!B7n(jw-aYaS0#WVHYl}aH#SoW5 zk@IK0AXhvxf9xZnBA(V8a*FpUBib0B60*Wo-L8auqinIgu(O*;NJ7=rNJ)Z^@~5Lu zaxhd0#C>a1qw@xuT`fiBh=r-w2leQrfco~HZz)**;90X~yWn|2$xz{B%)D&YST8!i z)1uGlz!e(=T~vWP*3Iq(zJ)>e&Xjj?>@%@Bp?4_m|K_M(_&4OI;3L?R5BctYR8->3 zHzi%m1f8}A_!VV?WCQaM5;6x6czr@ZeYG!^K2ZEtoo46vuc5rR$Sb|kjf6@e=e1ds zsF_pk`6ifRv%=l2BlSLC3=_)%;|;uj<;5BCnM6e~2n5EH`Jz*LGsJBbOas@WDler)3zD%Dcg&r zxt136Z<>$R@ua?L#`%fh-$V1cc^xbF`6K2}`^8CagLzKee-ZhRpA|CSbxhuvnd|GD zmrg+_D?04V;!o}dEq^|=JhX!lw%(aVOl$5ovMvwn^9Xn=BEg<@`yh)?9LX_lh@WH+3 zhv)?X1>w6w*+RH9UG9Q(Go=#kOKJdtK=Hnos_u=1w~qkmTRUIzk(NpVg2$Lks-xx~ zy+XSBP+k5=yXL=Qi#kChSv(aWBiH1`Sr2*Pz1 zNV_4s(IeCC_*3tT%j1-CzoQI#&R-XcC+Cziea(0hp3XSz)-OQ0{Z|`VKb`bwTDi=g zqdqDqlaHp ztl;uye8xBnGc#rDln{(u9aH8aroGr@z3^?(AkkW!{cCV5A$!G8;^ZQ>cgm&NLzA`T zAl36w_PV2!6Z9k+rx#4hz+6B0RWeyjLr>Yr(AOPy4H$Q#8nAyXcQ81R7CgJ#ml+kQ zy4SF=L(YV~w@DsVjV0k2^pjnAjsdu_k(4uSZHAgti13i)@_JN0?cta(xxAuu-)v8+Ma>Sa@{Kt3f083?h>bimrJ zLBfA)&$K#tK))h!dplPXSyWjgz9HsJ#m#1OJE+DQBy6txd*TP#vnaa%lTELIGV- z9K}OkQVaMj*%#kTpCsefB&{S54i5uJlXg3Z1z!L-`FpO+lp9?WbVA_U5Adf(E0znm z+mzNraE!h+5JkcJqrnAIZzLQ@JwpTBXJMc*eN*<^X}b*@qbr8X{>4v6f$q2!8-r}- z37o|K$>xtlkuK!TmDw*e`lRLPC;FLS(VXv(rM>#md(uPao}WrQ`2q?Qr5&p zfv*eEb9(Gq+4y&|~|7!sscGy~_wt&Hqw zn0q@&-W17%=IfSMTcg%@;)(~jJx$L`j3~nY*+ngE6X?=R+;8;vgUrk=?PBk#g|c=Y(qxuYm1=hNW(DC_ zMTMiv=^jkov8`J4$F1~c`nShij<=eeoJ4!?<e|fzCu}WKYP{FmL6SDu-?2I z`0gZi&ouDo$CeIhsq{dnS?WQL%V6jE^o%%=BRvV{ZhVA22cyZj{*u&>DH;HhLnnz2 zd@Ac{ab2=)aLfaPKsPM)nx{8VfR{rqP<&m=H$}b;=SUzVZ*kPRJ^hvKmpmk3sW)hY~WJPZ<(s6NG z`(d|ErjU*gS@7O(uBl(D8a9QV=0`3l7Jowrq10oz{bylFwq_LvL#2pL^G!Fmy#qrR zJG%{?5kv{R=)&d|*h|8?G zXRYrI>u;`01>9S~C+SJKhhb&8fr4LsgyQtYvo{@xHc2PR@HxfY$pyc?Toil{vis_M z`SPtY%gjt$HIGF?QRn98GBXALm!7A;%z_h0!!+1oDBM{p)3h1)0gI`Z5Vu9_Z+}>O z)f7)^Xtcsyx7)3*RraV@vbV(-ln--y;uFi~tQeJe7Gk3_$W2@QAZZu8gpzU?e1^Oc z?BZT*fl3q7y72e?5dT`ApJNa77mHNf{k%1g4w4c%{g$)J*`ccQC(qR5eAs}t)XzLX z5JJy2_VMh_Yn?X+K7W34-hN4R&DN)6FiS~m2JO2q>7V8`@BE3pmgqoM%}6tci7M@I z2^ceQ1)hINWS5R|oh#SP9SYbwxqh4de-r+l+{sk*ow$J!09a}J3q@-pOkfV!ZEV_f z!OqP4_apFJntIL!;(R{i15k0#!uhfq$c(TtAS{EQA0NmH*r_0pA*d`jGe~C*DA5FA zf6(DqYx_D)>i#p!j-ur6TJlR5`19)lV-LkgPXw>zOGur$*+(codqZ92-+1aJL0xrO z;O=F%L-w5u7Qc5Ef?w@Cs3&jvgTZ_g2%Bi9$pnvtAdk?5D^16 zz4%OCo*~B8mPA^zLmM9C6m|#Nh>DAQ4{&=~ZuF8{&-Z+8yzEt%bISoiit8@jx7?7`~Lc zGI7axt=0-78n;5h*Ylq)WU^aj2>-|zTz|%w(RP`04>v2n=P~u;d`Wp;NogCE^yIS5JxN`+E zTMD0fs4rx9Lg;501t-OO%?-a32Up<76SWa4RHe=<1#sI@G~)Q%^)4FAXM2$ZRZTX{ zVlj_jHl#`&A}Lmb-s!Yby%Kyr7&X4=uYTjtUY=R)&p#kZlKKX#e7yzY z_oNW;$ta4S^Kn$`k*KJ^1} zUyf-2)tiD+0@r`hm<0aJP3Z-nSC$TYQcic&O>rd6U;}?8Fvbf4kvm+aU1Rxl^s)KQ z6@bwhD|0|>ssw8l$H(lsysv!EhI{OOeO$y81(}JXjeiAglr%I)@9$fWz+jE=FZSSm zP)Vl$I?&$`r!DB(Sfn)2rlqBo=9CrIY0k}ZoLuAk9=y9dFO9RKGz3dJ7KQGzNp<(v zgT3^q29BQ<=G52@=p`{{l(^XwN5ZmXiERt;G9oEd9bodUL|Pv20iY!v*wuaCw4Xr> zVUxy$oAQ!D&CJ|8z=@g~)GXN*(iY4W2u#aD{rn@Zs%@$Em(f%m(sFVn zKS2I=q*60CTzIse5rLjr955H>7!n|#LTPAr!g~f+x-nDA=10)NouN8v%jo2OzXcgn zih5VSi!>IR8i`m}b+ZZ18sCakqt&PHkOs!=LX=?hm{yz+u$Jb0Uto&=&3!ONHrbX= z8Z~zn$@SCUqP1kEM*Q%wTQ5uUZ$wc+ejC2ME5&ET(DcXwpf2sDOi-jQf9m-uQ4-Cb zJ69y(m^%3~_b>N@?q#%*%iHJ(pBE{&Q1*4}E~)=<_B0v{zsf7qc6d%KlEMV6k)*T$ zvmZ<}kD*i4R+7)sT}KK_3tofYqiZE$H6=X}q9O?yam{ud3^f#emR0{S}vSSHU1721sug zJA2xgXbh_QqM;j0RVI1Esp27#w;w>LB^yNY#rB&jH!6_}f5tXc6rz?ga1`lpI#3)q z>-y@_V0?UH@Zqe!#%#it=dcgP)E6Ikw9u2tit)bc1o*}Tk~pX7a)UvO*20i3&`_2-U6^=lv-+xO{3L53(_Dq=GHc5Ac$+Quz#+#c<9-%AI`>Tym~na zTXEa7nD!q}J6ie(MRC=6pChk)8@*{QX^yrU8~P?~chl>EP)z+y!DObv6Z?_Ug{#wL zER!15+zTh-#%ZbI)L$Sw?VY_apfw9rSmeu^tGn&hWv|2^yX=IrHo8h~jzpZ2!o znC?0UBO_-T1Ne!<*cZ`}EU*tbL02Yvz_{Iz2@J zga#wija*8Q-;QsLTG%Ku$qb6hCr~zMDMF9Dhlg$um{}8v@X`^mqL#{*@m_#PcqazF zOW?*88foqriZ|$=-FblRdMe(BMdr`cw@&290rRvVyi5H_FM-N8usfI|;0K3x6QEv9 zOcX~G|7=^6np~zo$Qoi*XZN=EOLfteJ{svcgaSKd<=%5{)0vUJ-&Qk?P~_xfX2AM= z*h{BQvPn)s!S1J6Fr}C-!*s!XSvJpLcO6v=hz9j(f)?I;Vjl4BriM7xd2%M zQ?}DNGlsu!_V;J7IBn{BYV6)AWV)O%BlNkc=4vU%{%RBlM}i>qcVf7o>1~O@;DDU9 z12V~t zBq~$Iy4kvRx2=1DbokCkAfBzB|eLhafz>U>zAPGHX zltACnCSl>Tk^Uny@8*QpbX4o0KgE$md?Qz*`G zA@We)^%LvojiRv{)#|i+eA1#H?qBKX$~$QE`;sG}GV)Z;XZUqSll@4ca(?w}_3mp`nbIsS9Dw>_3xI((;-qpjy!=}8-8fGmEqN}A~C_yM?#>OA0+ewq|BtP@+r ziNqF{x_NE+5nQG-$Wi zNJodyQJ*{bd~Fg*F@3vgIHOclt8-K0HN)q_EJ^7=G~HIp0%c*YtYXo|jy%~E6Vs#+ zrXaLaYiL&bQm1-0Gw)}ZBDr{=yJfiMES z*G)HZmk%~syYrs2vBQ+ol;T#n*)4qMO}+Wvn(lOn-kz4q%Qsz-DQkjLby?{~eaEvSz z#Ih76r1$mf59QATyvlVGQ@{xbQ|i`j#%(Gg%yl=`^j!&p&WO&yZ_VV<(OE+~H>!cU ze-hzEiWBe^ZX+(4m4MnWa(6oeUxX53V(5|Yf`DJmK7N|TxT|$ujN=)*M>dJu>%Wxg zpAWN_e^>|zlt)DH8`cUApkg)TN!SHNDh!9Nnb7XIcH`?rv7oXd^|I9D_=a65BTn>^ z{6hhcVIRn}+x+~T@y9Tr=+>$pfLG-$B2*XGO5LVW z)hOhm|B9fbX`u{>-1gJ$Yvi?9?u@5C!-FWYS0}*E0<;1vZ=c-yIcZ+*sYQhus6^CS zm*(r6lK0Vj@hO)+e){Jk@WIp@(|C?c%I8!CdK}59$a3`THhk~-9ME6_{H-)W;YWgK zUQ76L$3QL**^Q9$#VAV>XA8N3O zpwa;7S?+WL_F6@j6}$-lFWrxE)w+Z(^MtZ3j?ERp)wWTYR9H z)}uYvY&kR%CEtmtRT)0RC)mSgj0Tu)-Hpxf6=G+ zxTSi|yWPZV>8r6@BN@PZ76Ud`=8PB}ve6K%x6Fr0Y?$03H_$Ov^X6Jx<;L#~#{DzNiF0!jFp}?du{=8|H_d4Va811f>G{Gz!VU!WpF9|WX1WoQK znjR5;z|cwI|8CaR-`u7zq%nUywCj+%1sZ=qOCJYBzOrIn$>e#-_Je~px8_N%SF&~& z4aiJA|8|D{E0Kh+V{aQvNI)e$7Xb#Yz&W+XdmaUi+@9S4Fp;&6Wp6aUcXo6>8qb_vQl!Cq651R-mB5ZtFaY zgWkfsRmoW@sKarx6XBal2Jl9VQmfGGfb;4Pt9j(<13muUd3UGHbi)*if4RC7Cd$_H zRONpf&p&`9lJ)cd#PL1Eix(p3miXNwaDYq-Fu%rd+?ROr_{pQufq`H8QJk`LX=$8} zAI3=0b_2_j^70=wWt##3ZqrcWM*1KWs2X-aCGPh~2Id#afV#m=@p4uCReabU2^!$$ zquitpP${r1!tl#$Ur@*2>$Kp;Ui+n{Rb zJ`P9tv!LJmmCNS0(t!FmN)wm_xj#toKl*uy~`dNY7*e{Q~k z(-(-jLJz?6G|p(3xut6;{oV1cjdZN^;1M?fMaUgZe9B!}8BE-pO*>x(xv?gN_l-gv z93AB{6J#r5%`8%SYq~R0TJBFMr1xKtS#DsAK+8MVj+ry${K0cz2S+#mb>FTP0ji9U zJ7P8`+K~IsWF;*%p(Bn8=1cPYN7JlqY(cFRkje(0kZoc|1(n{)fxrOzr2yhxS3d2U z7H?C~L_$Imv8wuTniKLmTS%}K`7f8WkfWO!zs(pEJwE2v!idzOs?z+Br0=VHO6c&u z3`tuqpBGf6wK@!+0Zvh(c}}xS;{x$OmlJZ97h6*XIaLaN@y^8eAVc z#+r0N|+!TkQ z^Rp*^Pq+_)Vt@dh8E}#Cmudx?1MS;#Zw~XRvzzzd2e3WG+@$NN@o*QslBrcthCumZi+(-T#F&0Ocqwm4t zLy^;27;xaE(u26xlOEbczuX{a)DqG5xq0wW>x?>=-@dEyaP12r=+dmRBDS$>StA$< zC4_Ry++u*=Yot$YlcI98FK#F)AW#X3KDW#rfNIfw1)P=3HL^0@A3V44=dIe;0B!Qwq7|xNosIpc#V~Z0&OIqavbZTPee(dgJ=0!&&}|0JvvM z$ONbh4$c+*zmqC@a(iRa(2!Q#t})W!q~gCQETVHt(UDnQ;3MX0rl+uF_| zKq~LsR+F;!fOiwLo#hT#DF;{kc;?;Itp_Nv@(dGx)CIJy%!dc`=&Umwv_(+JO_Mu| z5os)jUkmn+jE$|keN9mOdaKg2Qv3-cX(y@z{k8bzsx%GZ)Y{peS$CT6j$h8~kN$F` zH@rTI{fB^n7%-^zL+xv6?H@b*>qDB1G1PZOS#IE^$}tNbY6&DaYFX3YA#cnLGswrt zoxaa1pFZ8-Zl3N$^?&QHcdl(^)=By|7BF)Q#g%5t$*E|9`Ps&y^)KIju4l{Wyd?j4 zVuqr;qcOVGb+1Dg-pb>p)1??oT+zO*KlT5XEo@ape#8q6Dq5jJ*u&Nb2?oNRxgJsL z5)4-?GSVC|x4T@IBNeN{!op^p<#pmnrVgLSxf|G66N)D7ct%9+UPDjJHBW-sPOE7s zwrUm~uY=^ZDhH&cr1VId^X1FXJp^pJO}_G-$Sh|)DGMu&PT@I6VM*hd6DK4^W#Yjv zl@-^9;i=d{fIdq?pG?mnj(DQzr|3CkcqIHDyX^{~bW~Np5c4;(0PiVg?Q(N3DQRiIvOMj%xQ)>TCmh$P^}xR=pZ$U_zd8U1fuwB-%a|LXKe#VPS?>ht5$jk%iG8d z^g4TQJmA*V^D@eR6t5mIlq^4O>CAFTK2T)ZYw2?@_qA4gNF^rBOiUWUl;As!QPtU6 z43_}FLTxC;$f{FDmQRl0YyRbDNTFs_(^%&Fs)|^i%JY6lEykRgH1b1QT0m#NRI5}M zGset{8XX+{yR_8I3uRq2yL6e0<}QNWeVe28l-Zb1vu6MFlg^@cIv0c4UPQy{HCMPA za4o^IPA3mZd&j|D^$X%%b-+3-78j@F0S(;m>5(~xAG3*Ctoj95O z@RR0g+mcd@>OeoQa(L159aRTIKX31hXb{ciGxT{OMhOd+6MA_!X?D>}R8ly2QHh6# zH?hgzp9}21Ui_!SaDy@#dg=XZuL*$Fe=-`3WcF^DCx)0Mxsh#@3jyJ0lu$}ix|qg9 zwi_M^t}q1HTlJa%s~${?`T&94j4pu1=OmfkdhGrJeEaM9k-9exJqF~$|J=&i8Mv`d zu2q@3AsJJt8y$(UD{V+~1MZzOcV4<5sASWYXXbHH{Z7jtulOXgY$ z!qU<(yiyPo$9CS7ZpJI?!-nuy@?4wBR2Zj}4+Ma<%6dKw+DAoxD$szuYqJxk_s7Bb7~h62+Jc|fQr4w`?8OhcTUU*Sgo2(LPc55fp4Oe zB(O?NZf=@bS5;$!^n(u2&K53oJQFem2k}$bfltq?r?EUMe!8BHA#030f^AA&{mj*Sy$D3wHxgS9tH4_V=q5o zS#mYRf?cf%IG$a$p5MFDQI@h1tHxo??owUS{d)}4N<7Z(Riq>AO1pl4)eUR936$;P z`lhGAfYljbkCe-T2zgi*c$t*dn8+aOzt+p$T(S-UR)BuXGW$dr2LK#kw;yFLC> z^z@)Z16pCkt9)jhE2^`zv$wbRlu9rEVm`0N)(*p^rLG83$*nB^ojt(twbo{w(oANr zzEBn-XT(qt6)@VU|87;E;qy}GC)UA?6F}w;FAKP5s`%S3d~;{>@zB9p3>QoD;5o05 z)xmwgtRU9`a`Qh1>b=9q3rAUxh0aSrWv(~j7OqSB@b&WM&3@?KGP8(J43ET1oi{~~ z9{y%#cA||UIQ(W6af9Fa5th`Dwt1e)4sOq;O8ot(Zp&-o)>A`+SI6SWdiU|~(M4uV#_T6d;)UDZn+&W~Q_4~y1LGK%LK)E%K?&0F9VO#=Ep}Q6GjXEHr4Wrz zdx(B=Fgi_K6yTpr=-LKM>H&8IztX}TFRFockK~X$UhsNu=vs96mm@2;hT(zr5llk$ zlhh6?^yg8;zqzEI7yt;Km39BGsyARajhemj<%O5s<>vzBHn@woUDIB<5Q=CNxB)`3 zR@B*5gR6~dFOsa9QzRrLqKwe5U4xbg+(Aak3#Oe)txd} z902pL>$xlX&afm2)>+paSZ$+^bs75Plw;n78wFRG-Zyxlp8i{(3KaBq&@b-C19c8C zz9kQcLR(~cX(gMnrv3Wy5%#9#t5^Fc!7b+lmD3S(+cVWWyIrvc|Eqwcq^RW|3hyzt zrHZ)|)7)At#EV&(Z~6|X+i9yScZWf3hteNFnB*CR4Bw9?kLG%4tj{8TT?jKCpc4n= z2w5|knFD+H0h&pbD4?yOQc;wDD;R7jbo#tEX2YcEl?g8xMR!}m8K}I0t@kBq5No7u zsO!nYhg<#C0`SO~o;7wqz+WC%Ez+f??)J7)wSTK00TOuUVAl`OGR|{T3{Am8O{*R6 z(~RNNm(Da(egtg3X~Kqqc6ll8XW`*(<1EPLd;DXWcI9VU)$mFaGqYQa?h*@@g6Hef z91v`aV(nhG}U=H_`@Pcm_ zB$)d}#+Ua$G<&h^)|iH!IY)xd?#Hk&w(*y4V)VCLLdf+A%Ae_vj)Cr)OFV}{r8m8J ze$665?~23`ryWH@u1|QF%uF6rG-jOt|ctbPg#(*O+y2ZKL!g$`0;d8+XPiDrw3*@Dvjkzbh zHiNHF`mV(=*d6=IP;UraLOhjvx?_Dvqk#Y^6*i09ECO(WF;Z?Uu;u(WHy@(wU>>O5 zwnt0~!s%OJhn%Frej40&wIH^}RUbH(^KYR4d1n-*@!sPWaEGdgQPfp~5W4$+?gIlZ zY9PwLx6klHn5w;>I9}ONUWeZqlyVmt$kX{TW99}x@wD71uiWaimzA|9$q&}` z4&sk?h#v1>goNuo)t$UPE&t@M1n~(pQTCvqjVgGYyEpc>=vvDvAT=q}P`Q3>XD@2& z3-uJnHYFulOmF9bsZcT`bB0P!OC@6ALfKu9FZ4D9T;Wz-(~Y>By+l7Y1aY1TxDH^( zl!Y}a>c9^QjlS(<*B-jHp~=buT*8uFRdmnxT|ASddEOr6CBz889 z-w|ZIz5PtfU~|9f_mZdG$mV%R`oYdmk zMF%x5&PTB^%tBHp)@t6-fCwZb%dN~U{?b{5)JOH+nybL^}izM^t6d9eoPl&EknbF&}EK%;H!k7x;K zJs@r|VCK>9Cl1Qz=7wqJH)YCkuJ&-SBcN7qR7yrvs(J)!b;sU-1xc!WKmtoO7Fh1q zH@DTvYDS0^aJx)nu02?SxBdFH!i_-pIBK_6J-g!V=xv#d$T^%T1=W4&&XHigba%fc z!g`lnT2$c|($|-2o|8e@*6Qk>LDaNYQ|V1kWr#1zI`;Pv-M0X>8q98nH@B5UG*Sxc z`)w&Uv8!Ibkj8%{#PO|3TV7t93KOYET5sY%iqW5cjLPwRMnc(s48Bi|$ zn$|Z#KfVT-U=6D%{7h12tFH7kTjXiJd#t|AYkE!?P;wE0eBCsrO;k4nCQI90@eEH# zD%|2|LwcOldum@XPPgvCZK+|zokSB3&+TX7?5Kq~F<+fYxIamc~mYR|bb z+L-e1%1#T*6O{nEv<{w)Rq$oZRqka-!H%fQ^IJ?zC)-g#6MxdBeQ(`1`_7e(` z-RS9TZg`8yMU86Ub)OOsJMmX8k0N!|T=%pec9}_Jp6{J604)F>{m0P~uDZ}aZvbt_ z_H|BHXahWvwL0yX*J`wzOWR$=O3C?a165+&<@r(mUFaRo%egu+m^+kzmvdN}erThg zw6-91*_cm|**yF;81QN#C6_x8TdrOyIhX$WB<=CAN4A+W)j);hk!b)E6R-fl;=skI z9833SP>f^9`KkZ|h@M1xQtd*M|M5aD*=uFrP_!#X z1JH*XEspRb4+&$riZv?Un#< zv!H`H?v_;#%IX22>v*KxEj4zZ>AzH`mOre}||nr9dvsK2VqmC;V33Pz3|RP%Y#I-~Qd z47yy_gRV`n1VQ|kzY}(viI?Obvf`6clYw69yr4iMPbVS*lPjC^)$LzLg{}^%a6Jm1{_vToEzl zk*M=u<92mlE+c4fO*H+uC}8k<+7x)YoMBfoczG0lhDVmL#O3WU;1{(8TrAy>ewnDt ze}8~{Zv2))o33*xz?+s4Yu7Vdv=`eD@TW z_w1l0D*8r?PE=~f_TnGB0dLo_P-bs)=y;@`Rb83ygrzgA^~d~H#tgUQ!ENc6&TKDC z4fhJCujlSHOlBV^`07ckr2;yj7=y#+swg3jb%&9s-uIUmE7yb09CpL9n$GXt+)hvQ z!5x2(XnCb4M)mk2mJd&z$cKP#BQ1oR!Ao~Aym*?m?2`N;u) z!i#cNt+Q;v->tlSXxGT50;sOvym@2tR$JTPtfG0?suRQbv8%yy=$0Jcy;k2aHQ}|?ajK+lt2p2W&nxb+z$~` zhm?vk6MenxU!a}GQiZ%`%|{_tTsIX-E-x|g60jHSKe@Oaas3taZAMu zIf4SCwgp(3&sCmO~g+UmRB^OXFciR^Una$~vB18O7=TK4X!H z7W7s))}4HIE8}A!yP(jk4MD`adhK`nb2>u1Y7UU3yzQ})QL(J2v+o5I;`ED#K#rcX zo(_e58Uds?-KaPl9IK%oe8GZ2O6n3leeD_-PgYKMqMl>%Eq?cxNOxUH|MaHqF1U`3f)GYgP?hYGwlNog{JMG{>H+cw3du>U^xstC`7 z2j23;nS15u8-2I=;W8bcF&>|xvyT)e^+eNUNb&`=o3L?S`THC#RCAvQ|C96f8=~X+ zZ=*|9*CgiCAp1dLq1OQ+84p<6)lX{6+EsM>0jyMcF$vH{V`o%(zK0v34_m~q|GH|R z5il}3A+YP_=yUOD^Olm?pa^%Ehp3fg{mv_wtoEjk5mqy^Oxks6ug;f(H6OC(t7(5J zKX5&&mAY-`sQ8(rBo16gfn!=|fGeSMw6DL&o(>J|!Aj$HG-i=LXn8y8Cp52@lYQOX zpo`ii@mIC9oJodm$`g1`M*7guS5}I88Wdy$p~V320R= zMv1}kT7t6SySx)By&%BhUd_&vYQW7o#HOf1As}+JvpP0RCSP0Yf`BxSRV&>bukSjw z#!T#yq7dE`kUFOu-z$dsJw&Mb_opBE<%mG?IKSK!^!xRWDbx4tw}NI}1Tm=$Ko&H` zIKf(HYpH)F#^wRllIG6;p{)92+k@ce-?CubsO6oU>MItr9@4ryf=2%pS4u#HlgU9j zyM}tr1tjITzRWAb{0K=YxvAcY$DiLWZ-#X%9O+dvskugy(p2*P zdl>eSn!e$}?4oOG0pMp~{APlpcUZ(I8+#K_s98|bj+-=14ZlRkZHwzp72N&}bV_%8 z&(6aA1pzM{Dv(9TQ{e+~qKWGA20$4t16-cQ?V|x_Cr9;CBkZF%{3UADPZ#fh{Mest z-`!`gMG%W5BF5Vo(#o)E`^QHa;`g$a+T+a6?beZI(-B*L9JqlyuG*8FRD))c-%p!| z*j+1UzVmmhG%yYL)7b6x!k1wZ3Ct{jCy?G0=91wX1B*BB0$#-LJx}U-s=p;5H#RZ* z-fb?)&9RXd*h3G4%@d05y)iS(28C*w0{o{y={c4Yejb5A1YN6)><+5Rn8ZM5@l5b| zLEun_3=ov*_TNeWA6;J^*5v!eEn)%!1|ls|(gF&SQ$RYTn}Kw9Glq(!bhk)Jj_#7~ zNjH;j24gU4+xz(az3-pzuKn}udba1f?w$LbbD#4$pU(&Fx{7A39%ZG+$L_O@{i)~1 zD^?LiZWBM|Kz%TbpJ7?5KWHHr%mmb@)qG`axCte)o;6) ziDM9?4lxf_eUkmr?=s=RiSsCDTk`A=AH5HPT-r3;0@-QYqq57Xug;nQfK)sl(YC1yk5 z8(gnO3Etj!&|j+5(Fx{G5xwNCpFRHcA3J5mVAWWVYbb#mfOvms**9h zx4nJjJYJT@W3fT;Ns5Fc(e0)#(Z&yo)Ys3-KwYF_jh!}Z-Ca_P$~3hRG~&9C{J@4t zT^2@ORTtH20vs+1?m`Ybs_qF5b(R&Z&wX6I9DE0kJwiW8Mu_;GFj(&QHwjyt;2uo> z`E!mwfgPMXL_fw*fj9~PB%tG~aptXFkl}}qZ+WN1I$HdtqZc(e94IF$l{|RC6=Jln$U`ck{9f&_FMA{4dM;FvvUjOwxJ&<_k~CKe2{P)|BjD0& z$V3S5M1*vUu4IXRDacC3U(66{P*u9_kwPq{fUX>MpxHmp>5fJjK+8gL<&2_2za-!a zz5u32Vteh}erIXHENNRdqeLmcZ*v>wk0NSaMqFB-->X_r&uTSug*(;HAbs|xwE7Gz z+S=46VC^Tx-V+*g^9jvXVAJ$uCD!Bt3vpb#VF1r&128Q^?QkxUQ+*j|`UfCTl8yWx zC6>m%D!LK;RLVtOZ&+@xp1?lYlUZY*wX$-QD$xJPk>aX2c(u9^aWvBg%ttW@pW zL8$wJt;CHhaEjUWi$eZmp3(O$Kwx8%dF$KN3y0Si$xC-BXHMs^Dva2=+$LEpNcM>^ z`(1nI_`_XnH|5A9JAe>U1Kwh&w~?{jmb+$Fm<_TR>udx_v<+~7KmbGR!S1mEb@%uP zM}8;!H^R)JFjz6+@WiPaJ@R(|ljikO?-x8|r?j<5rss}w`XdpHWOhu<(h!cBBVv=w zxa#+{Z31%#c%L=0uiBo(K7eRJk0PGcSC?gfRK>?UT&bF^pn_kQW^l`XpOSs zg1b=jYnP+qTL7joaoP!Y5;J`AyZl}C1J+aOg-Q;OD`YgaU-a zH`+mW$Qwt5L#Xcb!LO$FC`h+ZVj%)ss0`#qfYZrY_&*2!*MCC%o($-vQ@nUzfXN6d zYMWDQI3;7aCNEDev$d(gdFHHBp_yEAk^)-epTE<#H_^QD+x0DNR2}NMZCTTm(}8CQ zxk_`7q1lf9hi2A)H>M9IeF${lzkfgXxJs;Q_amIeKaO+JBu1ig z?Ab+^9c@Xrehc%juTe!3-lUi2n;ao z9b8?fOL+d#s*8KsKNJ$xVG?;{Euz)f2HA3=02oc6Nk8~xf+MMIp1m255#g|(>>u^W zz1b2|su)giFMUs{;(9pq&4O1i*du>opjgn+bJE}&c~irRd<3PI!{YkMjnayB=^b`N z(=ExV=CUpNXTKuomQUkXx=Lj^FSB5{1s($3zM$7DW-^#_pO|arw2`xbZ}cJ-#_KPw zO}*$GDVQPA32*7lB-5o=^|WV&ShW3zUlg$VI~WS@a2s+dPX%k)Ce#2TaGz^%~r}~tu)0BNf(`wI!;~J5A0^PiJ6x%J6%)%=EiH*^!m3A=qHBOAI z&UEJRh2BIojp)2ap-k(j72oK?6@Jnm^W-BA^~;lew<6CbB=1EEcIqtc@EBTp3|$hy z4POY|*gFr4$LeJ+PICFGcB@=ClYw-eF)>{=-f9$8KRu?_oA6?+*DV|^gs+MJDlP_x z-BlU7ANN>N`omypK8wb{Zzt%*b~TZu65SjAlP{H2Yva9GQlH)b`frCD28I>+HnzawBMXyvhZElRSU5(H);v zIf{L3P%`0K{}ujEZ`voG!JXrJ4q*b_&%b{IhIY{;rT$I&!-Ed}U~>zSfkE=sI*o51 z0ISSeOhRmD>ceGmxQ7J)G4A};o2FMFzdWuMkkv8`2;-DiNaXjzo%4hhxF2gIb8$f| zFL)xze>S~@*^h3ctMBU4dK&mWNmMA=r4HP1o~*-bm#BMPICzg|7&xYzR4zD>HZM8y z@SZpQ=$v{|#;DGj;TbYw>LpUkF?B(nGA@3amN2PJtz5BS<6Qoh;mi$fqNz~zo7}dt z-+SRL^lDV2&@RN=`q!m#>*}#FH3Yo~7v$23jBW?M%jJQ3aUoX4^xzH)BB{2f<`#!S>csrKzUvRoX|61h zzSDkt2@fQkz9d=WJ3k*EE|?(z+RLJmn@2HMRDE9X3>?~P)t7N}kIHJw454*ZoVgYB z!vuy{yM1wpcHd1GXK`xbwtL!@GTFenMUooB&c{=h4`tCvyiaoG^2rt=`Q=OFpoDIs z@71^j>o46ahP{>C@@ylY9DQ0+cEz*0qe?U8J0>0j(iYrqZd8B61@Nd|`H8ZE+3kH*rzWO{@Q z+RF|(^c9QM`pdSSHp2x;n*BH5dH$D|UZ+p_8e4P}bMo5wajGUJ@XgP5#cv#`#g9K| zMsZ2rbMyH&7fbJVCOsPUT3f^jGk0*%u2uA4DVVg=Ovb9mUCF0IUGt;qQ_0YVR1EHu z_H`pVVS%s->HYinq0ndj&ev~sGHAZq9R-KYtqIvLzJT}^R$mO^1Ati*424wQ`ApNP zt5u_O=k2UrzmCAD=`5BPhP@Nm&CShK2#X$NzlzSLl&LAom3ewHMNxq)zkr$3U@6Mz zL#EeS#Vd&Gr!SoJoTk!->Z_p-T`GTD$v%iyio@hpl#lvepAT$Xe?#grNa*v}U9L-$ zJ-I}$&@VH%W}z>P6!`{0VY$Kddc%+B8W+7d2d5%L=# zBO6qHFC-nn&AlUWy$!Cq8OYOWzRQpWHM|sMR4H`3ijqXuwv!RGiEc~x-?v+CMh32W zOJwYk;;~G66Ksp#4y#}6@>Idi?Roh{@|WnW)f0iXdGZ228Nulki;>eME1FL{qxTt- zr`McRDG!8OK3VjEPMKDN$je6G#}FunhNAf%71}%6+XGI-iX_DM%BT=Pffj9QCgYSDk+RKXEU5O@!1Pu52yWBa!s{;({`1lLo%| zKW49@M@{(enLd=f=a<;SI7-^#I_uy-{ILk2p;vgDQ5QuNBq9_hMOzo8Tjs{m6B?rH zVB{snPE|pY@Epwq#G7`0fpLr5CifZtE&d6t=_g*52apI^zSvXHtG;#nwuNdx^(^-~ z#r^vz`e*y1GdC+_Scg=CD%`AMS8tT^oc8SQmVW!-M;vkHiF?IvBmGJ8`orTsUfl0D zlA*EeVO0hVSLwpZ@Q;7Q3MU%vX6W^*nOQF67r#f8R(wyRhH0XY9yK;!)`yUCMy)SY zV**EbV1jjv@49~5clB})l778d@EXWu5i>M?sHKN%Jk!{DY-JRK>qCIJ?Jjz*6sW0b z@>CJNjy+;$FW!%ud6$eBUpnaR{)%mLd~+x5bXzSV!#}BDh?h!GcG;)h6K=z@Ninn(XSKQ144y_enewd~M&)YJbOsx~D(?f=3JPWS55WwtoQ8foZSb7A zk1fw*Sys%{oBaHzA_u=;Gmt{i6!(!q}Hda$WZ_ouNiLB2!H zgIk`9E0<{t)2f6`KwCYoJV3h7T~>JjoRB31dACb>?D)9LEQBl~l1}i2r?kqiLxU;~NfBDv*0%QJ8BAdG`nq&uwLQ*up>}3itAeM|)W5f% zQN}DUTJZZ8cB}e9{gSW7uWK(2T(d>VqTc#bwM0k2qN>KhOOUPT4cMpnWNJ`~g=JLy z8?27O3B_HLgIT?EU}ifiG&&-J#xXo9l#V_=;WdY;bm?b<-1o>?Xj_bg%g0dGL;bnF z-8Jw~)H@W76px$^dEnIe`b*q_B9QMq)T5HTXJrKG3VU3L*+T1Gc3QmF7wrxY33-wR*s& zG8?76{e-2UIFTwaKTBbRrWDTRdyqs;|5#R#E0Zoyy^5r`!34#JY_QQ z#}Dm+NMJsEQ~)mY zj~P0@za$;d`Y8x)J!mdo>rxscn)ayqiCP~>i)XWIo2!PMZ24viN=WcXIO^K(R{+uK zKU+*+HE*L|O7H0C_!l4pE)vmtf=&aX9=kPP8bZYVFp4nVr^4oPeek)*JmbhL zmzr9RH?tTYKFm#EC8Wu4=Z~fc0eXKg67~-WdH24cD?YZC5_zQyZm|=0094w`QLiNw zjpYWPdtG~vWG@FGtw60t(}oKuufusGW-lWqSyta7$_r6X#cbVtqBt66hx!E&KpSE{ zGQOTbO(d3LVf0FS8f^!-42p_S^K+@5b!!LK9(ZfGl-P*&;ob;`@mhSyq>&t*U|p7vMr~SGu;I5*?^WVWzpQZ(*WP3f4bQ& zx^=NdJGvZ=Gw@%+Y>c|DN{>Lu#HN|c4rRPl3QuzSvcz2^jW0`bq$1tgy&6>oV};D4 z;xRSnneBs7I&O1y4p!0G1`cP~yQBdCepM6+IiGanUH0vRw|kvFsFEzW6g=)!(whKo z!TVv5MA~SCGJkM&h`Vp<`D%uIm9}~1B;M>)q;bT~*0uE9IkHr*zz7(2`kKAi?Xic$ z=EKsty4E9j5&$>ma}g5VUoM9_s1Iqs6ai0jQL47~*BkV%En^poZw|PW@Hn+%v%XLm ziS-yDeiy4mA1_i%3_p3Yj!O(3mB4Mab0_)gRlj~C=u=cp$j;2ntf=y+6v+Q^`zLq~ zgPFSPEGb6S?DD1BX6q$roA-$-61O{m;L-C%eitU@P7(n-(PWHycXB~i$uFpG4bAko z$;D*FSRN6F(Nm`-Z0CRLTQ}anG(EEC+qpfzpNMD-*nSi}p6Qvw{v)|7AUgkk^R{kV z_YJOtSMg2oNyv&>->(`J&k6?!+oBZ^W4KY6Ei^Qsz!?;pw(q1fB(BhmRMAS=r(|ef zI7b{ZP@5+`5xX3twr~(c?p~;uuJ+fQ(h4e(n#4h4%l zKUb?Tpv8;7W?+Q#&@!k6wRFk5Zw0Q?!d(g6j?fDER<#=F z5HUxi1x$(_xX3%?rk%yXi0zs{ef7y(8_Bf~mb<%q`b&2IR-`Y~7fk;;5N(gH&Lv5h z=e0E$usATLl9v>VVtqS1^B|;;JkY~9-EEGgvP;=@Ymj#q3xuAUnwtL1u^o4bqoa5* zPDkZEF=G(9(Dl(g&;tpJr2`ooyQYyz7#i&!_F(8CZSQ4-i-8m?YxnPm$87zX8%ynF zVgZKB&ec8Y7>pK>6K7|*TkeZ`%|B`622>r8^ZO_Txofi#HdjT5!AKIC>-9F(BTTf! zg(ch=T(4hT#E&-~85!YiLnSTNwp^Tn0du!G)MRjcs>;_xk|}2MMWuv7vZk(1;;j2b zmRU#J7l{SfumhO9BG-DOqN23pQ18QwIj9J96Z?wIZ;LQ+!~C%$UQcQU`Q9q{o@!`Y zHkIyxeISirYIV6h!1ANP&qNocjXHmKsoCuqPnDCMnQ2uefm?S<(&$jACq#@-f{IBj zyLg%>X(AqsTL*8}e0x&=5`yhod(j)2sH-}nJ@=G!qYgl`m(RT9G-|opj*Qw^n3rn1 z+>{9Bdj1^M$)%T`Kj2@y3eN6N%kNn!Juht0tLakG{}A>WF8rBV`C4o)|Ark3)f~bx zIAV3Sv_-Q~fOZpRWV5P}U<5b{i=fo1)BY!uPIcEuvfrXXerN`K>DAdsCk<_fMRReE zD;A<;M#}yM1sj_L%IN?4soPQ*1!o`&W1|}H1*eQ9InMj6PiPsCWiDOKuc{&tvx1uW z3e+IJwmmbi+9oE0#QcH+#e1<`>c)Btby;%pIdIuCQdx(qS6SNmXW4~UaUSqMmNbK< zQ3|CSNeqlP_F4+mi^|&%sc#DeRJsf~c!}%!D2TU}-A zE{}PKzi%f*%7;@L`x_nVdqOo@j8u9`g1qNGNcqO|^V_It{sHNo4$$|$b6bN=IvkED^Gc+j!gV@8cRMOv8E41 zlkL^pxAu8?NbiXq{lcYX=j)(}&CTsG*|d_6_dT!~f(F3JPV$~zW8SleUAh?d*iV-0 zgG>Cf68dX`Gdfl*Fnr^Vds(U0v+BMfhp9$R$7KrFcoX5>vcje*a8&;@Hlggf_ZK0x z^N_n-cS#kpjVVPJHE0wn-$iat9=-^B*hEwp7BRuIMk>U7*%He8PoOODv+03gM#;>O z-zN{pF&21JfXo6U-vYUdkT~n)M3!GWPwkTFlzMN>fpMSa3c~9sC3D@%GoBzI_eRLm z>5&Q!8(hZqzIIco$&7Sb7n~r`)ieVNq26^%&iMs9s^ZDuHaCk4HoN*sJA(Izgsvoa z=8FFCSp$cHHUsC=#KE?4{6d@E*eOWwG`cqOa8PE_zyXh0M!;Gwd*SQM=DSB#* zFoB;cC0D;YODsm>Fzs|+c=W@aV|j|izJTMQ!aQCO=F=Qi&ZTG9svS_}bki9#0zq^? zG1Mbxv1jT{7Ua*#nLNCPSi67v}rj0N-*U^?eZU1wZudTml zmTVtCed;^aBJ8^5yPArCwrZ)%|Ay6Xf^cYJ*JS8v%Us*?t1LC&UrHI7u=30Efj!42 z>_K}AqR$X{?Xdx82vDD+c59&uWWfX;dD+}nQX~y%_8!P=CdM@2E~4Apv4<#~JXn6E zONDvDPV{{}D#}4F$kk=`GQ>!QWO@@;hjp1-7%yZ1G3K6t$QalWZD#e|_-iKaxgV=< zxseMqF?HDNk;NhUWeBRM5DdX&Qg9kWj~nl18)_{Ic2Xz3Vd|R1Tt$;w`Q4Vx(auh$ zswh=hP9h^dt-ss7Rei#DM4q|hG5-P4zI)K#A8N7lQO62$Da6})-X~;b=zE;ss39Av zaG3F-Xy5r_rAsy<98-4bow;}yo@rEX-3Aki+u}BUjfX)EEaFc@_BUbm+!x2woL3Hs>2ltaltR2*_15PnNG5 zLpj9}0?a1zBy#pwKhDKUxyHRbAM^mF;D3hz`)t)+5aELC!Yp2_ zrI)8K^qJ%J+E;R9vlADxR=SiB;8!|_hF>Ru9b{*>*EF-?zfrr9U5YMFO?_=H z7eu}~Su|mJvJ;MM^E<7@$NSChkUxAFuyV`iy;zkrY1pf55pKw3Egt7TJ-n#sb~Si_ z#v$;>eLBfrv?THIk7V+h73CjI13)^pd(dAA0Ihz6=YGs_pvJx9&FtkR=-0DW{{2m@AewZ%a2T0rbttcCmZ=imN&jn%Y9l_D<$ zj0AqpP^HsQE8wq_i+SNe(bCFIAMZVU?jvV<^kniRA;=DfJB)}_QOqoS`j(l~Ol~^$ zWn?h5yhY|$NwMtElIEjI8;U)NME;FKumqz8KQ`1jxAe7I@AJrlIV01mRZ8Qe+4H}l z&cK;?Fu_GrRN`ENoE`xHbgN^J#|DirLO|*Yaf&Zz#p5jwUX`|OJukw_s z%TU6iTx5` zg(SA&Rt~x3JrAYP12R@5&iBA7ooH~BzCfV6z-n_iOWdZWvYtT6V>02JA_td~lh`Lp z$wAE{tz zt_Far2#>2QZB$SoTRP8eZf`d^Toe}Y8$NT%Ygu(dLYcAbP9cV=r|EeOrpFj4b{)As zTI&TK$OK_G5(egk5a(YL*1L10;Mb?oNji>*Ec|l#ltqXIvP5S%YYf}5okOHBPx5S&vsge!%r`kvuo-%RXP`r0MahJXs z%chvLjFuXiJH7=}%lzA1hCmX=I_kc^PPURq1}P-1H*PV!7QS9@PW3xZ7%O521PiAf z+%J#?WefGlri)&N;S9dAKBglfzq7W>c@*9O!D3fwVL00eSH_sr2$<=&(zcp`pnS9-g{rLhb_2691N2fN3)>B9~Q}WIWjYde9gB%ct)!*KS&PZ9(zhwD)j+Bccp1IXw?_N<2ZRAV8RtFIg$N zUk+*dWzcNHZd%LX9JUznL7ARQYbNA-`(MqR$s1gZ(o6Lr!0poOLBrcN!czRa+uWbE z53(R{nmrltwsvdx&ca)j@%!>A^A+R%M$DfV9y82&ArMS zT#jp92`db7mA*?LZ5Z%G*+?aO8y6zRn}k}Dp~yDTa;|Vd=jOiA;E7N7wU66$`IS8o zRfsl67PS9{*Z6<`-F@dFzB-6KLg`j`PLKx;Vft;^5NgZ5JI zD_+h{$fVhxI+uI@ZCjwY1=RXU zX}JJepZ$Ct{3BuJ7b3pap{xz)g{nh&mwZLKH>f5CuA|elqr7pc5Z;zo0$%g$K+Bz; zlAPT6L^-0RDSSI6=M$2k(XAE8vu4T-va_1MG z4EDNfdr%`nw7O?WUurWrxJ0ycnY(FrkDMc2lj|RjXoAcV`fu zyJ(vp#ftfTEWDr&EjxobD*=Pfs%fC^s&*ycW!EyNrVRGXEpKm+@iMd&x?QM%1h}_e z;7L0aLZh$dTI*ZS!{hqmc{Vi)H#TKks*)auY^$eM*Z_x`_(|uDn@sBDQ(tk~EK219A@RJ*jF9?=~VwvMO;W zyh0A68iIa1^&yJw;){4oAbmMs`C0A~!yPAJvU7*ob6qTtkF-+$oZiC!VtM@*>)G#$B2aZU;r2`g(RWf5Kvl|{w3@Z?rLsjNE7-OVF#4{<3BC)XvXOJtc0F(;~Ksx2io0!uSigj8=a~I+LoA2WmBY zK`}QRdLjn3OEsrSWLs;s189AsA3yv0GcCOM2uAr$Ej(%7L?AA`FF4GDsIXWi3%(Gjs2QpGT?Of z>4+HZLHJ6StI>3fr7;U8RjBmK7=uD)jbC1w^)n#QU5z@!C1wc{bC!d7^cK^AfcD z9tnv*F11f4|NJdz5J0j#0fyl>x{{vCr}_nvGq!B50)%ZX9z{$+e|ma)jh3DU=8wxm zVpSYv%H2M^`}k)z8$lxgDej!pdT++i;MHAw((E6=4=wCwvA$F6>EoR?DW>kYBilgS zfietr{unBg{Z?|C%j~dGTyAYmN@E`4<-ud&ZG4|0p2O>@A}>J%Q+X!Co1VT}{P%dC z{mQf)vIprNMX>beLF9am6OunID2M;UXQO-Px&6Kv!!RDX&c@qWkg~ZQ9Kq+$kAeb# zsE~0yyhfHENwu-3eUGu*O%)WEz>J*h5Z(>z3png*h$L6;$6|2ZB_%0>r$5u347W>2 zgTL)&2P`VznqBf*9G4ABs%K&CZ9huy^RGSo%x5mw{e3Y4vP6U=A|krPI%PkTPzvDD zAxm}H=~0B%%+93$K8d1tP!w~+ddxhF%z`ECx-N{ckNIRMzNVD?HJmy#t!n$~qzSsrRpy@d{Kk;oYWZar z=BdPGdA9pxw8pn@r%t8;jL%JmAzt~**vES zGP(Y?-50R`iagk2-+NVu49MWt4r;IW3%&fEMoF^WkLuwPNg4Z_v0Ba*mm3IBVxq9) z-*h4Wl$|mQ-e*j$6x#gWivdHC0pM2e6!yyzi}q9Fn2!tv0CyiU3#%1_%=+%Dx=~pi z9DIi{NTsA1`xN#>QCV5OEg!>?pXJfzG2?Ak38a*b>%ku~L1FijsXv=nFR-Zi0LVQW z+|?G8?&wE0gXJQ=KPelS%#zAstCw#BaD$dQD+Sa^WqlKa#4E z!nWK%#lrZteJ@MA@dY*%2`(dY!=sWsQ*73w9W&6q3|+FAnEgUeL_o-TI6Ox_jBw(; zvU#2$>pW8gkq71L0>Pl7t_SpbI^DV~Va+dihyRt;8%^KQ^Q~U#^npD#I$&N$_a?2; z_fefh2I=YC`60f>`V}H;v3F}Fe660JMj70-iYu(iUe1rCJM6l{%x(PkZT)|0Fd5CI z)8+49iDYIc8+ATc10y^TCH0LA3nWa0|4Ny}R%3fLL&m=99@8s_%ANN7%t{8o5Qpz9^&|6vwEC&pd_t`*!ABfptm76n4n*F7O4YG2U6__%^Y)LSS3`DTOkyoXS5J#)TF+2U>N01vDhDwcHe4biw$IZc##G1H67ja-6h$6- zpOXifpx8u$lEyKpZQz-l!#%|&++7U>R^w6%6Mc@Giq)gsUj1ZbQ~A2L|7fg8s?hKW z2W+Dc8_YU;{BnLc)u)g=V!uys43#hzP`?xd}+x4LhS%Gu>%Z=%DGoDnz z4$j1vay0yh$z?E`CE18By=)(pI@}0F-<)LH9{XJsbR_ zMBnNjU~ITXId7TdlmPwJY%DsekRJa{zl^&7bhP!;G^*a1T4Zv2@4;U})11p0{P}fs zvZ0WP-Sq|5%g}FR@$>$gxEBxPY#`+8o8N>LXfD}gc`Cb0F5x_x#+Ro9D5iq)x zG8cKrQPC}BbmGt{=o;5eTTx@Iv5nx3YbP+j)mL+Dmq~ zk#4HuacOb;MJyq^9!_nS{{D-BtYY4$`BqFJ(%!Z;aAn zADi%c6$bU;wi@jItY;)GeGXUyhnXRaqKcZu{Fx_}CCXQS@I>2My>s^N;ql*_xjboa zHxCz@{eMl{EjXJO^NGeG*sdtKsgIhfJwra(aSAKScZ1h18V1~ARRVI%6$gGx1eA>L31s^Zm(=Pt%QYQ0&2sWER7|n| zppme@cFG?2c{#4uG%4yB*6#ZbIruN1{3~W!>j}Z=s@~!929w1?A9;!?AH5S3?+XOB zrg%EYX%qzVwZduX{L(b$wR_{ZzdeIHPv6Q4=#GwWkaIzZ*W)&KWLxPJV^f~taiTDU zdn;L7#z->?1>(NStl#5|R8sNBx6V*#R+0eG{>aLU$M2$X+nQH~fRxVGfA<)CeT-4H ztKFK(H-7i7Bck2U)#OFIKc|@3e%Xw11ExHmF^wEq#sab56?MWNvn>}53jLx51^isI zYE`*HF=P>8q^VeQ`B17bys5kG;0-S}7wPLEsP{b6m5i+dD%%*!__MI=Ay4D2oGGLN zx##>nQ&{Wu;)59oVu3R)P2hN~y&W>veIKHv0oLJlvEc(i2?aTTm^tEas=WyCAJSf( z_L^~m_UHocp+W(oeVaeuHwR4wmO5E=jE zD91~u;G_Ci*+`;$yEDXsfXkiZ#R|?3>6t}7W;1U`Ndo8(SosF+PdZl78zqN->m3WJ z0Pm2CY!vSp61rj>lj;NE*F$fgv@|`9{u3Pv$91^@`~O+Ro2a&}Gt=aeBaim~YC|Bs zW(o-LR6GH(v5wOobs)<|BlH!4E-v5UR^3>@{0w(&x5j~(&gCBa;HBtmb{{I+|tO^ z^fE<9LbvfOJKNv1+6Py;_(lgDfLIa&NJafmDdZRgR=Y69xT|6hhO&#A@=?fufu2nF zgJxWa&Pb-4O$Dp2%U#|k=iv}!%XQxj!l3%l4KApfPo&@QH$40LJR~QlsUTQ?M?i31 z^{S7a{t=fkZixwwUnzq^s@}g}7)6iH{jtbbj(n=8Xzk!|dZzQYm%&&VVmqw?=f1Lp z6To?VZ-)G0uM55{LdE?8rZ%!EU_SnV#?`btE++^q8)FZSRP zm)#L}Nt4(erEC?(k=cIMvlZ51G#}4Mr^gp7*AGkNXeh3xs|ul~>x#0RQS5e5z(Q*Q z5$Dr*n$!6#)lA>aNo&iYoizI7=UzH4ucz-nWLgJ1^@C%zra1z9w@w9ItMO;QLG6pz z?L1-7E7g(*jaO_!K%DV&6b)$7-sM9)%fb8|D=Ml~Ipl)CTJa8c4h~7p`Y?LL26+DLH73ye%zmI+&sJuQ}O0!Zd`!7(fMr~$xR09j<&a0 zu!mx`)M4?In)+jdtR17s#HqC92uS2Gu%CS zO7n->?JZ>wsgU^PA39LePFnl5$7ymp%9cBe%8KEs02%<;MUfVx-DpPO9Ywn;YT_nFQY zR_m2hb?9{iQkToAF0YXWbCkglzRjuqO6LS_bN{%~sU2tdd-NwX0M(P6j+C!9q}k zS>Jbmrwd~E^=2(Eu377_WkCKoTHM`A#uw}X>tv>^hSgs>+$klc+R24p4fB#FCbr{O z+HogytkKf(WC9<+kd5IB9V5%{-(&b&FD7lm6a`a*FYOgGg?{p8QVX)?ulEQY^c8Q6 z(nvTC_Zv8oMD21uVU=^e!`06Ies+AlB8kq%!BO~V)nH7ZkY(A~;Rjn(CEE#6ROxvL zdPho0O6t=ckr2b}?I&)yavaxt8oN0DRL7vZ3gp50K3qsM>FI{j|w15r39bO=Xn;P|)JKB7MeY)tsp$FqwkR-CHehL^Z3nm*PIT zRgVx--g!?Q=`wh0B7>^woS=+om%ZtHb}}2+ZG2(W1CkJ~t?sYRu^DR=28@sHQP&Rr zzoe(7#E+%n&&Qh;e?0{KX6SgOCGT+Uz}=lytcg3ut(p9n9?4%KRCHc1HjX23Cz{as zIHoM$>lFZ0z0{2HZ*{nucDU__N$fY0^KLNppKds&G*?QA;d9)#j_w|8xOIC#aimb) zF45s}BPWr%O+uMS= z5%sjIdFZw5vX|+#m2(UwFNblde4d-%caq_QbP`-LdNd+!?E!Tm@CNtI?}}FCSj<)luF}(Mg43}QG^pamQ(#M0TW%UlxLx*W z@Vbh+9M;(Qw&qDj7UjyE%}O>lhwDF{9^>^{P&f-m51tLwR6c}@r5^u*LoeG8M2S89 zbSod_?GIDY}gwA&_ zwy>Oy@aC6w9_jbIUusv1B?z_N3_OfaYSlc$g=m2t9r?3eo8NgfFk0%XpwSUs>Tydd zv{$Wj548I7YbIi5yD0yBh1RzDICrIC4jmmFhb+DPyOQRO{DR?)_Ols7li{icWo1Cg zL=6X5H6MAB2+8SIUip3XyZ`(5fIBANqG`E{SoCwk_yw~{)!GUI19SA|KGP3dl~cI5 zm4~LgeLJx332VVm?a#7s8l{rIcquL5KKMaRgO#e9tv{`@nc!*_4W0V@-gd5JTILst zsY)vH{Tvyzuu*e2{PldOnh-CQ$4MM8{-i>2AgraU3Ei$(RL%hgxT2K8p9xL7@3<#k zSCJN*0G|8vdk&Z}+)n+)3wi}Rm$xI$#CAlUXg}p#N2QjJ+35;LhenC&BtuRQZHV7 z?+txvX726ny&iJ}K>2cISir9w_hUkx!SzjGy4Tso;zoX#=f`OPBs4bmp;|Aasjovf z`B!#JU48xCpkvBy55ly&);cR$PT^;VOFC7^j^AA&L!uJVq4%Pr=>&(L2>5%WU&Y(Z zI!`!jxZCd^;e_DPn61&XCk$^tx*l0yR1OY?$Fq%S_1Q&1W|9(_Jsa3rEw@9@8xmr} zcFffK@}pzbxqM#PSzEuHRcr79%a1K%KYZ{L-O`B&iwchb@Sx%V-R;DY=@>9TPGcFa zqI3kA7EyIMfZO3urZnKKFCpbX!O!<_NA@ynvjm2(ck@!ynmS%%Q_p6;r2t z8+WP{*R}#w9f@d%K7S1XU2g*zo5qoTe^hMfcNfAx2)EI|(;1hhv&pR0rmqNn(f|YK zuhM*tL)DRwX8nJ17^OmjyGf>`Wp{p>y&h9z95a@`*^QWCqCGt=^rW0hrt_TCr}glN z@-697-zqKf(&ggFRH1YHC>vDLB{{Vc9Mj+bC+}9*wx&lz-e2VJIk0?*gqj&QZ@sqW zJO>9&o9JQ1bp309y5j-&g9kve2Yfq!rbKc9rx6ld{J_+dDZZ;Ozevi0GM1JEdwTzJ z53vn{=C?*ifnWJ-EhNVq5zHT;;&=#5t$?1}(Xe+UALp-Q$ypm84Wn--oh zSMxTbzag#%XB;p;aSX7Qhi}%!6P^c2@{LJR20*i51}^mXiHTe7Ee|aO+(e<IJdaJ$9e)}Eh z5&TWY+cD?W?y#~2L;U13rZ-Z#uVZ5w$JA8PTZ?)ZH4PS=fA*>waN0mxL$g7ua?ghT zJrtOBzxd%t`&~u#zNEKNYuR2DCHda~9%bhW7!9fFiyOqi9Y^6Pl-Z-gj3j}9naJDb zCEUwwc9*@>Zf22l)eax2I+dz`7xCZ-2Z@8-@t47?57qraFLAK&#XH@R7H zP5j@;VxUffzX7X7RVnd!?$7USj$F?dcEtUz+G7I3ua@`Uz45C3-_KZ^)y_5hqXwXD z=a#CKDuu*TduWkN3C;h*-g`$iwRL@?94m?lq7+5oU;*h(dQ%XPF1>^z(n|=vg`xtY zAfWUXkPd;+Lk}njNbdv^5(ES!kRUA}2z(p$ocB4;`;PJ5KkgX!e)m3mY)G=pDs#`a z=UVf(%F&M70VseFHkb_E!LwWT_7q7w)--^b8eLeBka!i5_+o)6po_@T!T z2MMkaB`JjAaaE>9PxxRKj0NZwFJiW7KoDpEF%>gA)g`NJOhn2|cP@>pFVX2em^ytvMc-HbST;agG|FV5LgOjNW17Incm_yq+o zP|EwwjauRx;VA&;ItZtG@7&A;+XYRvhkLWg7W|oCwJ@0x2#5pAmvts!pOXIIxd+kH zSGUWgUNw^SLD@jBq|6L)vo#_qCdE0lP)ohAeM|e5M6LSg+8{t$=@3`t&~MG%Xp! zmQ_80b@2hA#$Bs1rDDgQNQ{eN`nYRJd7TNuuw>j0IhC*fa`Pk*s@ygN&U1)+s8e2o zqqmtF=U$?Bg@+fC~>bj`yTz50^WHTNJvS~ML+ga}Xd}pAb2-Sc1@Ka53{C;LZ zg~M93d?R+-6L3g!0M-w4xJ!79G|e|WefrGl)2Cliu!lK-9l?%{?h-Jq!qHi`yLa!3 z-W9_TsePJo=88YUsy-o}K7INKrBx0okDN@hS1&}b03vcMpN&p82?~ndQV1JNhiW;`7le~RuA!Ef91X>{= zytfS-M9!Q!L-00*X_Ww7kpiTBG=i`#1y(^m07!st0C;CB09>S1KI&|4WdTHVTZrC4 zL1>m@=tD!_4NXo?0{0C$RAZT$4_Y5TQCC-oewk2h*d)}L0(t~K z3Me+rZ$|7k zX8n9X6yGj;Z*R+)HMP%rNf3CGq)uEdh0TsM&rTvwi`jwF=sJ9Wf5}!;Nkw4J9pFRqWbdDlp zN@9D>wH{Y+@N%)cc&UvkU2Y#$-fsl`ZoIg+H{hC4(`|3plDtWIL(a%+FpdY8bH)KK zBfR@F(UhKviRlV0EiDt%(Dca2H1IdF>L$T*QN`Li9+jQ_0sj~YV|)vWrj7=7U`7Fb zA2Ta!0eaa>3jDIb`34U{YekHLCj~GaipRph^e5UvWCw6X_8RFaCHB$ohoSoI3)uXCA zhF4vjJnl-S0Xc3&YHFW!+_DSVyYQ?67;Fv(59%1|R)NA_0A}Z;AJZ=g0*b-W3!-Uu z=7bf)-?iEs^DdcFBt41N)C6{gu^yNX!q5?E3V!&K_(9nF4s4Ba0V@k=-Vu888tjDN}> z&YyvF&xqWCxw12sv4G;i&;k`HU%p;?(eP~X^RfGIVC;v2-SniAv!}DVdQ|Szve)rL zQdHDaw9duh9~&ji^+NUUlN8%^?Y2*H?PI(&t8+0+M+i7)GGHK#Nd#IkA{D0mvng(&`~`Ej{hpYBV)9wHQ

k+o+T4^tXMKwEsQ*7-Utp)mKl(yT&hH zS%`lIlkX$5Z+~M?=Fa~i;icFBSrkzRl1`0+$Rot-Wn6;!$9NX$=(6Va3YDJxN zyt!$T4+uenn`~`X6V#=~xTI>^WN$=4_%@bLcWHb~jXRECA1r;5c5M#&+}`TY5ctcv zT@=|hkmSR{MV6Q|pu1HJicd!(BPKl82|{iQ$J2AK+gi{F(a_8)BARGM;U@WWyo|*+ zXO2qyyryN^Oly6OP*+MggwZ<-R0ph{zdyT0C!9J$g>|xmUmmQ*fU#@K9awVE!IE-Qgykr^iVXF<*Y+$1Is(@+P^q ztnReo;sAx_)8ms?Mruso)>v?1vOuMBrb*MGXg|8B`L3Nd3SE4ZVLT?V^K1IP;JnPO zD{Ee^PYyaKs!@=-x>FQCAsiN4yZuEK?fTt$gOai@`^c+mBfc>k|0KPtN)cZn8CAfP zz47k|T~8CAdQ{v)D)lg!_B%zRK6H27@a7O}LZXnI$fOkErVD)Q$HUqqZf$MUx5fkU z!(~L1!`h{uv;HOoNAfJ{_ZX`K#;kP5{&aSDpZ3J2=+Nrgtvck2o)l(&!(?n^|Dj7Y zqX!8&s{YGz|AE7U@uCMa?fEkzV6&M?iry6hJy2AytzB0%UzN2CDt1h}BoQ~tTt5}* z%$XcKRHnM;s%*Rd%?EH!t12a#g~WIM3I!|JmbmDHws&)Kb+iefl{7K5*`aToO&QVo zp<769HpmFEs{W*92fg)tlkiFE?IgROcA=`Uy=|3^ls6vgu0}&;DIaFmIg%##e>}%j z_sLRpMKm83H+T0rZ+5K|!HEd-dYqSbggxU-dAOAA-faa|Cm+vZ+|bCq)BEUGx;1Nw zPC7GR-HKjj#|0j_qdNpO(QYW#WMtRJ?GH1=oDQ=WI6!18)(vg@W7+d-xJn@@d+eJP zVXdK-Tpv1SkIf~&ZLrDMLCThkIZHp&kk^9S@WLLXBy&+ZYHxuz*$eI&OQk}m1+_jUPirRTKjikmasft;?!Kc-> z%K;w;_u@u~US}r0I4JAvdqM{xP3Rrhc7-VXxK@k4Kp^`&$wP%1Bs|C9jlRwGF#O^3 zch5;5ug(8;3#W@iOIj#QB6pfT4&TT=h(&RzQ1_P~85V{WQzMriAsu4uMXRVoH}W>#+_mWs zT}f3HLggD>LVcYou4|}xkPjVM-);*(tCRi(BK4Y~J8Q855Ix$^e9>0Pt*tDFqv1T6 zc2{j}4?C~d{}_3qhUEfmbx-gb)J1?{{cMmcr9fL_?wL~`Lq!}s}BzO?j!P;K2Wqvg7R1hi;O zT_nQ@LpP;sr@bR#qL5c;FW0T>7IGgkKW}VlYQ~#rh)C~yTL)uRlCR%TdabO2r#$WuR=+b8$A zGzSfeM#<;Z3qJyd$NujjgRY10sxVBrxZ8N*pXbc_EVSA)JN-&-8oNVPo7X1FqxD%e z4y7X;V@ZjAAtB7D;bQ@I)`y)(>t9~l%|#^!wwc>ScshP&T1eky6b7PrU0$bQpjv&W z34tpxpsWe;iRmqzI9X>5sWm9Y>ULCv8BMb~VrvD)AjsV8rN2zSN(842f*E=D_OZ5s z%O~%zh1mcKxZTXk$KiF!su_HiM(geNcsn;E@wr7MwJ}%2WTZkmbx%vwX<6~mHjlm& z)s>}KOHnt#XH}2S*Q5eTIYL@ifo2xly;$@gw;CSyu9GF5C!L&_hv6#X{&vg3-rEai zBt`8p&kytK&?Rx;6jb!Ei-qSlJVz~S?Z8gNv~kU~O=9>{co?U+1n zjo$MM3)aSvqsnQ0V`Jms>DKF&=IZYl7$t`c@Ad!1mw!8pp#JjOBlY`>Kym8p)8_5r zYhk4Xw^24Eypv1a2~Jz~KBtwAl)EwX3NweJmP&l8Yq1ujQLG<;K$7wt9{4pD$@cni zLR+{if0p??&cA#0vXwww{r)Op2%r*PLJO^ymU>G;t3BvqQl?}+&8Z~2_2#})v_rfB zT(X{1H&FWca%o<65cX_>4+u^E*{G6&MY3k~Cl@>yxFaFVGTxdzdDZQz>h9td5*g`? zxcfZ!^;lMxE~M4(*>^W|mo*RkQ#9Tp_#n9*Y@Id`oKxwSg#i+3EO6Ctxp@d5-mg<$ z?wJ0;%SzbGPb@dTEUXI4v)x~D>;w(v{Ks9|{a+BySI$4MAsNxHUR+mC5tM%{cpsOu z0|Q{a?-#4T_63m)@?T%^72nUxI(^}SzWDJFa&!B}H(fCJn~D?$t>5uT<^6g_3oCQP zycfOuk}{VB%LLcqoM&ahyz>z4wG(u|$@`~%aL{f3s{_GC#y~yeztlO8YbUTXn|F~q ztRG}`iG22ND7TZ9DR4r=68V?ovROL&r@_FiQbK~_FL$ha!wPJ{E#r4jvZIQK;lR1vV&@-DTMboGL?I$^UHWY*5#2f_k6_You_{gM{Uuh!J}&bv(4y((n!3 zUy>G?-9RKSWB<7Nn5mt^zhDge(~Zc)9=0{^G;1u(A==R##Z%M$arC!031W9l5%v#K zn4wPsBPmRTw!OB5$`L9IwP&<%PK85WexpGs+U(#zXg8_7i)R`>Vfqpmsj{W2G>; zA*r97g_3Znt&RQUj*=bG(=Aew6c;K!-Er-EdCj!9*)#w%1om(e^JVQfUdgc^@4) zlL}!wQ#sS!)4c^(rs-_eO~jgc+vZufLBv0z2>Ji^UH`u@`H{*=%(*zyWjFnyOX%kOM@oGD}BJG;Zk#h#-B^;LTC z=Z(4THt*^6bP?YxuaiSj=k?UbgXcQ^`{KZXwO|v<1M$9xmT*lUYVl>!KdLcdLd%s* z>eo+9$KEgTN~vHrtn#U(v1jgoyJ7ospgU2~sUU6}A0M9`t>nlss$S(YY0wIxYtSsh_RO4yH4ch$oqMf;d@i2*GBmLa8d|G z^Fnf3n`#8&7~>Krqh3xkwEt)q{}QimOsjw_RGr$Z4klQ#TT;R}9hcqXQSrie1u|v? zww%VF5?Mm5tpxqog5O@A>`&KSTEdeU%!C%>6E$2e8VpJSv9v+=qy5``*t%>j0h=~= zzfUi9c)%Ari$?JXi$r%T*thWF2+SZ zjMz`eq3SPEeCK9fcx2{d3O&V2yY2aw!h-9v7&*BV+dJ&U?LtOw3cSNNJBx1dk*^zjp$BZVzo59Ik%0o#PgwKWR&X#qq^M1ou16BqdGU$#ooT_JAFh&y!_+ogu(*# z!e4Il$|s@fQ=xSB>!njmD6*xOq)`XiYu$e$2Mon(bPk}4FA3&a@Yexi%4Z{83`FJ| z@8o0oc=xY4-r*uxCq&5KHp=uto2+@tG??{=XRl0Nwufa(zvV`S@0iG=l}9x*x{Qi5w@M9y!8~=C zTiF3}QcgcO^~D@r8NWP1=Dt3Hq;Hn4O&!c&%B@YX(<2M{C zO9xUox{4-T2{{g4l&|f1;UWpfzh%`o&i3meN3&P8xn9o%0Wc$*N1g^)YP|`>h=Wtk zD2pls*qD4qa;>tCZIBLk;9o`!m3Z5<5S^p7Nk zh-}g$_VD!L#Jq!$Iv_O=WjM+{@S8MR9_Z1gaVTTz^ge*q!?v2Z+Jm}R>xw|arj3RXV9 z_Ca>eyHGxH==}DM7{t1-;-bYM6x#n>N z#Q5e>nHWmpZCH;F_lTxhSH(LD?#uUBV#+akxJy|AvXcEq0(udqowN+*MfMC4bFL`PkZozk`J$eyOWah%^*m^e;b&uEVo7)HJ||7PUi-_t zs;bt5HeO|drw6|6uVi>Py^9sHH}mSPupLDr?b=lKa#Z9X%>7u@rP=&z7P`{8r6nzw zc^KTfRUi}O0KXbHL*6uWD`H2j8@AoGo)OA|Ycn?w_03cPn|uq(QO@&%LiIVtFINYF(1t+kT9MXA>>NE?_7 z;!dZ01uOyVaa)0PQjSx9BR|y0uG!6(-Ft#zKBLEmO}F|*SMMUU;M^+P?C4%j)^ORv zpL=0p2Hp~R@@Ig-rIPc5MvAtyX-`|0q|KEiY^2%kknIh4YeQk7v3fS8>`I{Y;<(GY zT%C!9eBO7za)i1!lSFvuoI$GU^0-;dw4k}+LkyAvHTZ^*r;RKHu&T8N?EV-kHtib? z@X?9|V7lV5?BjGYJHW{uIjwkdqQJvLpp@kZ^IDtJr6T__79cX)yyR_jTSb2~XR!?gR1sAAc-Hh~dCEnak;qmt@o}+SfecMd#{1P>4K`LH;ibfsGO#zM z4%ljp3Nvl3>hB+%%x>Z;?#lTL&bu&qW2jV1KqlMlY`VlJncX6P)hNDOWD}&4MwF$) z*ZnOaBYX0MAkNtws_-GD`+=9JrJ@vCx;4Qg+-)V0f>`n)i^hY}g2dk77CT7-gAu3JC36` z@L9fVFrleyRa^#g=&4;l>vf~?AD^KPO5+7x1GwxI-N!lSb2OzK#BMP0ibKm$dyypkMUdGFCkKg}X2kao9_~v=Rw#-* zLO7>_g4k++hBacj&HYyXNC3dF*AjCq+ee6pN9wN@d_FmQHxCTo>}`s1GHHop(PvPB zW||0?ZE(7;Jv#c3h)|{7KyRP!NXcr-q`Gv9LywJayRNyhc@0`l!{s~#gMg)^hAu|x zmghqIoeD)XLz3t;w0W5ch~X!r17ByD^L*{6)zaX-^3qRttAh6v4aMi*nKX_H-AMuI zz9;`!;*XCxUcGzvTP|;e=@kd9i9zf*ur?#Gp7%-+O(~jiKC;VT$P| z%V)Sgf)x4_W6#~lV5bVDB)+(or1U@MQT)`ykfEqIF*#BbH%CBilt`Y4|G3rkOugUGHO_ZvzjQK={T~(@9HhCwuWfclA^NSWa?4 zjTWK(q06X3h(jq4{IR8?IbgC3uYr{PSDy6Ka8L`U%5q)MK}c905kxc2a~%#q_6F8F zj2aXZEw^at=0xpcO9~`+mT6Hy{YLax777!xNX*|3_G}oZJd#X;BfZQXwtb*OH=eg` zm454F>;IT$4)5m9>@?Lfvp_PPayXaYptb&nZOy&A))4Vgn*Ajv`IEB>Co^pBEp^}e zAw~FK{_7xAX_Xc|xTA(+-EtVpTSW84=9E63mr=QpbrYjJ?Hy5NvC^TJ?dSUASRkJDBzBVLz#BF*{nC`HRVjmkIWm z={>sXB}p4jyG=TaKNK-r+t*Jd6=%6&l$)dW@&q*=x zXW#Wup*MIeyuGDc9TeE43ZN+<$o}ta?sA5o$lUTW`z2?herZ#({>o6%8nJF~en!or zh6E!mPoo#6x0}-z@IPyTEUf=TA!GbEz94~rKI3T(FW(RgeU+m>yuK4<3;5pqnT}<4_9rPfyQh zvJGjp=cJ|%o%;7tK5g})UvD5ynwuXLBWI5E8<%%yJN<#F+jvR|bgrKmPJ9D*f?EHS zfBCJv+u$_)w`VrECI9k!oSb=U!XnqySsIliCShNu@Q_<1BOM|D*r{IxMv`7#eeeH8 zzHwmQH`3|OueQH4tt+pdn;2flTLw5-;}>xnw1cAAI?^N+FctSQB>m%TUSGb0_M4o7 zE~M=xxNRpm#?1iFVt(#Eb!=c{bT_0RvcJAKCuVW=TW63&bQQ;_3P1Pu$N3vhLkUZH zWTU**hZx>kDqdAV{1!XJv!+<**?kBuj=006Sz-)M@Ksi%U!uJH)@i7^sdJ+i0Uq9f z3+2Op^G1EX$jQd~)P|j-QNDOx}d>c_B<`2(3=MX0EJk{P|0&WiyWS|UHW`KlkU6z47}B`^iP6>nwd7Rg$}M2Cb1!k(O6 zc0~dv2D!Hz_=~a-txjsbRWlyodlX8uet$p_x_>6jxRQKmiG6RCLccT|ha=#;IrM|} z=wVcHL#_SMa3tAN%szn|>F4;eeS?YVq)`)=Y`luu>7~$Lfr{ z5Dy9vO+XQY)kDq8SuZCVP*LL*^gL(#_m}Zz&Q1)y$1A}VGVtb~3bML!ft%V=&wF`| zKC_nROOJ03f4(gAceKTPD4wMrt2Xi2Qolr~IMq=R*#)B}uwz^QQ<#|kfM`HWK{))W zBtR~gg+%vOPPLSqY@MgN?8n=5NtZC=_GsGs(Nanu^`q6fJ9WcP*VA}79BsJ<@obEF zR2xLm<%s!(P8i8L2`5!X1w#mo$9W;=Or8^M(Uhgs04YY%N}b z?ERoAs1im^P7rvzKSF-GWtQ-e`ceZxXNz~9iPI~;p^%d|su;zPM*w1eB#S?V{ z)5eHJ-p`uwuf-tmbP(-2G>!KlgNT#;B&%uvviFhfB20U;hiRLKeVNWm-n?&hP5 zbMzEF3&6vRI~s$`bP^d|M;HSIB~h1Y&RaBJt?FL_(o}`~0SFxlk(R5;^WfW210lcT ze~UgSxoYtB>BZbpK^nCKhZZM95Fpj zbpd@%`ZB1Un^y2f4_X5%$H+l0?ptvA~_w_CbX$g{7=f&sI3f_b`q z$Kn(c4~Q@O2rhS!?{!`AZU`(Dw04q4$~j3QDm+lveVr}pZSLnpjfaPR`0Bx={r*if z_2%r~a$e`2bJN@9blNwBz0^+*DnI7x2fl7a*tU_B4(V;0nM1Ek{5#&vRZSucB<5t# zeTBr>P_f1=9(lOWeykl`E8r?BiG)qz#lkpb+}UARxsqz-XkmNqHahoDSENO+Y(aA%?G@Ph&u0}J z7HV_hfP071ndHUClcd^EANqYshsfd93`x28@SRP81r9$)X+pe0Q}Cm$T@0R$zw>gc znMQEa*X}pZy6?60{tf`mS>CZ{7f{(&D<@~PJR|$_r;*kE$6bx$IXVBc`^UsYF|yEh z$^7qq?KRp#*PSTvNt_pcD(T`begdP;h_1J2r=r!i>N|X5f0EyHJV21zApS1ujCSex z^B>Z{3H?HZ>X1F3mRcGps098kFU3d9Pe%KcDwgZpYlTaz7nq_WmOz@R*T1W+Ro`i) z#(D4TpPE01eQEY6W=?55iJS`wIRq9sSGxJ`ov4$Xg@bf6s+J0Cymm+z}!wz=u8FZit} zH3A(Kt~%G}zzS6F6Gi*VskFPfW9xt5z}?L*r#~DGx9}#pyudso!`obhHsxC%Whq9d zJ^z`(p)Viq0NsXC5hQu$WwLOSuFzN7APHES-A#ICw{?Sh`X?o2MVA3BJ{KiiQ zpZ|vWbo`X!bx8 zd+{hf{JQ9^E_*bT9Qb6Cr>_mBZ*p0}yN~+NaBAYNDwpgOZx@KG_aS94Oazf4ie`tG zLz)0mBhg7s?Wy~;W0)%W_}1J>{whe{v!UQg*81;Sx3<#bitc~N(~PT!(98?(5Li#T zO0w*sc;;`y_J)0H$n>WKO9a=Lmx^|`&9*Ln=TouyPgjoX@@JoNKIi9{dl%k-X0o?f z*IvFe!4)19Z%2ZFn#1YkMwWQW7SZxMp@wDVX%uXd8X#q5*Xl_2ZP+D~$*cpGz5(6N z_5c*=7~rBb;cBi3rG#7@@66Ux7{T?%QJN$Qp3 zG{;-415MLKaRKC4TwYAuQ$;Uo3L+hwEcImN?9Bsd$TvAwS*%N0)?e3%_*#txmsCc% z5C|V0|B!KxaT;K!p>_XT>}oCId$~K1%+RBJRt^O4<>W+uU77iuuiqt~`! zs4z}CF@J@l$4o}LgOUU0RKmEY=*npG9!?^LsL%L}Fc0vI;ZwD+)!Y@jOe~{nmabLJ z#Xxj>hAAAiY9SjOE)C`AWd=?yRzk(aDDl^0EJvd#z~y5h8v&R@c^l z5+vTLtqLhc9d=^WrLCmimsha_mhvGHd*2Wjr}Dx)7%{1j!anAEu_kl6GYbvZAUtWan5nRL zStC+S8Pz&3Pkm$kP)o#L`Wd7~K|D6ohe;W7bh)q(DFX3dw)YQ#!Gk14m=zr4&`XBC#q;~4l1|mboRCDOAu-38Uc3N;|e=KG+Vpu zlOzK{~kOfg60^?At^vtSj`LAr?6 z0A@C~QXQRQX|tkX-&6(~+D5yc_<0rSr77xk52gKEaTNWEIqGy+D}wObe2}tjh|wZ?yndLSKm-IjrRJ0q z=AfFt3=;RXo|wh@Hix(|uCe=hlf>ARrOuq?&t$OP*|uX$8TY><1tFnM2c#bzE_kPh z3tg$DWSg%9>xNJGF=?MEKJa)o7ylRwD_{^#+p9SZZ^}aodXFgY5^pZ-agNrLmbW#f zvFn?hDxRYYHf9Shcjawk;=>weC;Z|eK^h=gS-CfWZhkJ{Zc7w;Py>S3`LRzHo`?7v zY^z@(YN#oXYNm-&(u%m7cK)mies|OXQowQp%{zfEAP27v1BdZ< z87|1~oi|usZeePQI(S-ooy}`PyY{7Te5!2wIwkg_g4%7MwCVoW>AR_o;z=dq>7Ru| zaEuY+XQ4*}=uv#l{2rd5jik;O&JPk;{~fHAz0g$5wDJy=jS4#U4_RT0W^V0pK?>wK zHmrQ0IA!tnA?z)I>oS;vVhIq+4=CLfS!!dfDm323Y8oSzm7Q3spWAhWX~>yH54Z?; z$AcV(1Un9&c}s19f|%GGpv%v#@?G2Kkzw@{pD51J@N|-YS@uxiYBQ4yCt24p;U0?+ z2GTKh$lR&AnaJFy`LvE|6hz){C4@uY!)JJZnov$XODP>i3eJdOv+`c2tQE`wi6714lmv^z0z(ic1prTYsv5 zKN4|yQeZ&@`d5|A%UK(S1M2W0*&k`wTkfOLDaC|)hK@^-6JcD)fVN|WdG>En`N5iF z1#z8E&URsi?6waD>*$dOf!Zuoh;T(~|6(FFBVh%)Ks_K>+BWE6b$oH5l+Jn~>7=4X zJzDKTd~m65?UgoO+fD&L+X`ky+dvngP_$|7-}t&O^VxhM(`OS^1!7d}zKnu$^rZl5 zo`xILj&bY}b&*5{bK1Qu^;8i1rsoeH zt)O4S4=dk0oVNDpkBYgt*kt6ZynJ?}#ARSHYrPKyNk!u!Wjm1!I(!p=>7N~;s@^ek z57kn2H84`ct+PMJ&JVA*Nifo08tCe?I@-`Xd>xf5tk}h(hEeXfNdDWm5qP#4+{*d$ zh87^$HDIE|8-adQ@e2#tOTng5^dygca(F+-{df!Kr*3uY`?f1-Njk&T(t+H$y$^D! zdZ5aMXkbiesdV?{$<^MJ7z}8#4W(OV;+^eGC+sa8*vdPZ5Z1(n(d~8tK78Q!$3N<3 z^lNNPv0vW-2}$u`7Z1!JqZ*&^S@>9fLMNevT%JYRdQxSFABz`p;8F#0cq0mj>oF|IF<;)bsF5EWu9)(|@@)ykHWgf(*lY=^ zdWh&Z2M2X!3v7>fKRLG||GC}RxHa9q=a@oHLSI*E#mGP zNWNs4?I^F2{Qz+c{O(if;NC7|M!^+*-8^Gw5hW9xS`ml*%ylBSQnNL z*gkmY#2KnqAg+tmoi|YsVGddL=1kiIR5$pFAHE)p9F7;rW{+!dWO8xm8j6Hwk_P)B zsGjGM0S$j3avc2WVq~15lQWL9QH}laL`j~SI9lpp?}b{t;(4aD$;qM70?X)EXutV| zZ0G>(GMSa;S&EK+Z}qM|*&w@9J0!{4A}Jay#@|2B2=q84D#s`J25GF{inA}$@JgC- z_muMsQhfkp^fkHTgqk8v4U-A8dQX6E(eTwYsz@J&*83>(Ll19=x0vF?N0mG`r&{+E zWhr`$tlBV|G^<@n;9z74@`}gPlT}%n#Tlen!e3w*?yfrW$4N&1!j_^qW=F^myt8t7 zY6MBnYX-@w&9V=%(b8(|4$F0rXWMp8t}1nKJ?|vXoNKd3!`Y}=N>m(gL!+;7Db)(t zm6dLGTu?z8Ke5!utooSLwX%(i0s*ReiGZ??Ljv!reTPTbM6De)P`m5iHPJM0wzDs&3zs$i2cA9SK*T-{>%@zoeFPZmSD{Nk~FiD?FCgz zh}dxDk8I9Bop2q_c)n|m{k`4DpQRcqDE`c|dI>Q1-fJheUHCrPq44M8%a3Bgwp60g|N|}8< z_8FOrU39g4^X`0l@k%6MzI##fkT~UiT+NX&@^wu0v@bV{b><@TG+Poneza$JLEciFqu-F+(Yd%#UTAB6Km7Xo631m@0acU6}z7i9EcX4@s4fG=L*i?d<1wY0#H#R7*rycD2*f2ij|1foQjb$4I z^GpyVO!B01MWi6+TTaP$Tsz5f=PSzNm@`!8W$l!zG(QQXt+_=yXuL|{?k?2qCycD5 zAM{A56D<%<6XERBjJT}eK(tRD14@KL0vx%y~sgk8^ZhFwjh$!ESYRk)mp>uA}ejS(ke&WQ>h3*%jlRc2&Mojkc7gxGLh|>tY>bO^j261s~}spY^CR$?R^?IUGvje`|v@MesWz-sOe;~s?K^X8>_us{ zXT`fd1Nit6iU);@E^2%)vhj!(fAa=V_{5H$2X4$$yEeHzPk0H-nh?0csXQf%2Y&gZ z5TQ>&R&H;us%XK^JjQgM?)GB2-R`ONHyKOuO+JhbXHjWKBBzx|;{AFC{Mnwi6n?p4 z#}VeRD9+d`!AfB}x*Sqh=dH)vwY!rGeX0Vtn+XW9zeG8*%X)9zs2dY(^4-btxmka+ z2gWD%Iy*2{1sEKN`KTY%XmGbS_b!Qx%#T&veT0C4*{I1cLC{?Gq!u`21J zn|O1?JyFJ3Upezr+So#ewE7QBF~Ds(A0dgrOY@#`K99ZpMF1V#w3&#ilnUpMn>C4W-XT@|+ON=$j{ zw}5%z+CF|PKR7CSg#{Dwyvi&!Y6{zGac>KHcq(>eFk(wId9sE(Lj`|0AW48xKc(qt zD;XH82B4R0HgoQKsz`TH}(yTNC)N?I<-LM#mYcVG0^e@qQC zhW^lEHcH}|nlOYOyg>6LlSl32?SaB-k4_=BM;s2DTr1?pW3K>FB| zzZs~VedgPO7F!n6I1V(K&jQc$3%Tyo_V0{h;*v1W|A`d7PQcizdT70iTtBKOw_wj4PKq|Ba|Qm&)RBcEZsxbUh3ohe(m3JjaOn*?+ikMU z51_#hUK`VyHpEphfs?G5mDI-y|F2!=%J3zWlsH}t6+N{%g(TMC=9&roJG65AA1w^+ z-*^9cMxnK05k+k6Z20lA8Ufc6m^6-idT@3T3bWJVmf(wS1p7^uRPg=TBJsM%TWnM* zSQ>pdn#pl?T#oKzcn z3Wh;(ktJC{wSx^8=}=VVJ7s#;kWGh=SPBH%jHNC_X2oUI37az8H>c9yH#W1&_WRyy z$B|Q~jDA|~U#U8t5D0ByG6=HJu0b`^pAskTQ$|-@M_%x#vT@<3aC41tpW@fVM39^0 zb1Ai_aZp+<^s@j8au79-o#81tj{kB2Ks3jLFaHJfaeHE8d0=CutDwkvq`3Hyp9(0& z>HxcoOK)w@$U#htQ|df4spGNvr3%-#Lcr&-$N{Zq!+Y4kRLB5{(#>9iPW!eru@C2>{6MDs zdbM9>A1=7t=3x%NJecvw*;D6U@3blRca4-Mi>Xk0AOLB?87(FUpD-T>%s12JU|#Jq?eq!AZW7kzwTSq0$b~THD0X*z3T2mB`l`tq{|k{2`S8DyZqnA<#ASvS(Y?XFWSd|rTF*am zFe$aIR>6{D%^P*ry>>pIwUeG=lIbgIc;LWZ9GA1HywT{|rr+~vqx5*2d;KJ;d7<(n zy*n(Q8oTCj-i1kXm*4g3PhF1b>|K)c?UBy-UU&Cl`E^gt847%qtqii+OVMB$J-@Id zDV0k1e$W-`HpFXwJB2OsXDvcCRm6-LnF&^8RK_HYqJ`QWLUkNZbh zUaaT~0k4Z$OvVw>@4#PKj>psRZx&?H!HJ-?)%m?D9n#R$c-v5_*};cmDpq z#sB(9&?Pn}aHsh@F4~xU=COCjtDP&H%riM^O!*N zVs`_!$*GIq@nF_CHf`--J%Ozb6Nmk{sTL0G>kywom2|Fd{l}~2&1F0RR5y-EriSqP z!W)xY@jKb;aFJ4S;iERt`y&5GK*!RT%~!&T$L z;^|i&-#73oRBLq|*dqQ!1BQ4dol+aiFG4Z4cfbva}NPh5X0JQ>bCH!Uk) z6}_hX40`x%)7|S4Gm~3SR*}fV_{^; z3M|3*vlArJn=R+L1b*?3pkc(+zb7l%qtjq1%>qm~OAnwZZ27}S9E zqrIx)Vt}4rp)mKhOS^KRMU7bnE`L?ltdMzv=&W-6_6L)dn}MyYDwz;miV1jE+4g9H zNpcPerQd_D)mzILT4OkxR&lFRuWxY8<{8(bH@bPodNCC{gnez(9N{EJ&P2C%B3!pS ztU-*i&jm?DsSu7x**E^7_(@^mP+%VKyEb-B>_CNn7Y!N|N2d(A;As!2{Bjk{JgIM!Ujk;3`j;qv3 zVIXGs|0MJ9G!X0A9Nv4cvp7P6s*=whn)`h(OIL;N{m>&_-s>bJyls|GPG*#SgH^|4 ziSvXHtlo~xvUBm`9{qCu9Uf5S#cIVZO4ONs)pxKV9k^U^^B)rUlbkr(^=_l}D$q37 z+#FKe-~9qbVH{j~=MMOrKZ*c{!?p47geSt@D?s|vKb*4P$7IY)(%ZooF3DZ^RsZz& z7|PZDxqCI7WV;mg%tU(^AgkIwe6)QJRKsXK~l!^pCf{!{VGGTvE^ZYMeHi zJT6+l6;A6Pvu*^Ig)d6hb{cB=N)42v#0|k`CPg&?enoxl(N954Dvy36Asx8neNb5$ zMy@o}1g*P5%DeheMhiSaTg25Q_H7NgWb$fNF2VhAyAU#4&blQ_-$@+2HuS9ed~O@9 z#dP=P|KgI+(EgA9fC8h> z0U?xky9>E7>5B==J;HTj2Z)YU(Vtg!5~*n;?cdo&kEOj$mMt4+-6TO&%l6~W>U1D; zz)IYA8yIUvH$QF~x6^&Y2I)c_59GOi|4u`eLJGGt?{~>~)Iv-}&Zk#sy9HM02l@d% zLgi0AQ_bgw`R!s@8Afroyh1(B=anRtQ_;0H%6{vYVck1i68#&E&@ zv!!+qe=-Ug8cQ@4ZDze#>ydZwW_7KYeiyWdL~Kb6$1^*zlS-Oft?y209}Q4o##z_p zDD#v2|0K7;ZgBGbJLQ8|4XexL)*Vi`5(E|E(=YP`Y2qh$>M+I=FukT ztOz-YFviC*zqvZe9*NdGXLo;t^D5o0pN6epu}aAtZ(a?>>zYjD`ZUpsTNW!I+^1=L zQVEN?3SfNvAf(S(75PJc;8+1>zhbuu>S*;SLIF1KC=hSVImU39Nd2O4E*;j~G49ll zX2G*DNr>wAi!+1+*BQ6=30z*k3c@c4TSm=p%9w(vanVd)z+z!1d)K=s=6K;FuK`2A zIt)?^8F_x}*bgxqxRhcPar)39-QJN6z&aVC7S(t?y^!e-_gyw#0EAmQ*$!T`K`h)m z#<1W;IZ@xPv}@b&0JW*)%3CX~0l!#0tadh-?K&I5yHK30VXHZh@nuY?!Y4`|(nVqP zgJ09aZv%>IUh?MWtsBSj*!*d7zpNe0t^ni4C|j&}_+6ZmD`8E_WmEmddHN2>c4t!q z**SiTii%ycG)}_{I)s3*WEfaKC_5Z9kMo2#hwbtIZQ6}~Yk&q1g~imesgT(S5~ycQ zdi`I51{xan|Gj=#mS_j^o0O)rg#;8$O2~&BPg3@wHHfpXZE9dK*c=aUmc~=vO(i69 z<7V{M^IHxnp1EC~*L&q(6%}t5?FgeEPttV!YjT2QM%EYod18*~mD=M>ASwSKfJ=aK z=_8}yjJO@qTS>>hHg0~&jaDpUvGLY7i>Qeuwcwxluok?32y%DY=U*A$=JB6ppN=Nu zhBmH|{OV2#Z1_{od+GBE&5lB04l3CF#}{l2N~v4`au(Lu=ngI&yql#AC=i$VOjAY` z6EnFy5mGui-91TNAxX{jC6z9P{-N_}b;KgXP>ft*h^R9L*t$6HX`9N4XZwMTpZFZZ z!^$^?OJ(UK=Let%`<(x5xMT#1uqa7C`b4WaHJn1GGb<$_%^H0RZR+3q>Fe$5%QDZwa<0YI}Hz7ixw7SCUaZ$qyQox%=;ky$yu1 zh6bdpOc}V|F?YX3xE1o2GBBZ(fCu%Xc2OsTIuRZZ@-}Qp)9?UW4)dG+-`QEb85aC@ zYr!k*W}Vzc;hgyGotQx;yekE6y*g`H(#znC$O$r5 zTQ__+SIebRpEL7)E@OTosEAC10_HGwU>xLmV?Wfj!Tp9~vGp?6Cpl1Lpz_of!MxjW zE!qFLp5dtGtSR)+pix{dIC19m0Twj;7s8{mvP_CdcOjN{v0Pc&BX0M4Og@U^5R;dI z?70Tr89~)kxL%{#CFZJxL5^}K(~NknaJd3nH_vQx{@~GcJ_&?OBU`|zow0uRq$J@Q zoZ4t}C_E-_(6;pvFpgexWt&28fh^YoJV?Kb+_SVCup9zM@MvpqXlv)YiaTu5&Hdr+ z?}|4H`;R!O88c=}|xFm!hgQn2z=Cjg%s+o=6tfgTeupK2YSctQqo)iAh&=f%HL zIbDx(mrcXm_4T_eC*`3=`VVW)r=gBi^WdWT3n#sUm0*+KM_ZT+T(3@BD$lj^h({kx{>}R;SKn@(cegnlN?R5t=9WGPQFgA7< z%u*&iwZ1o(q=9HMFD{@RxKGFw<=aMxz2RC#yg`F=#GK|MAE2h#j1ofrgZxg-RsSVp z{`V>Q&jcOKlbeDQhL{!I_m14|DJv^0S@a2gjXUP$)hf(z^<(>K&c{T0v*8C5S5`;z z%3-sh*iL6I0c?c)$a=c;2aEg)uf&|a&qD_FwVmyd5G~vt!eJutH&;NiXDbcUPxcbm zjFBylpaJwZ63OHANObTj7^QJ7h|>!3Wju0&B{=SH*{gByEdY7e34ohsE|)nHyzpub z!H_`BMDLBig&V{!W}OH{Wu;9KtFbW_f*(H))5()!7J>WeTPp zt1J4Z)(}<-0WD0OuSHT2$jYAI!#xLx3>}9ILk`PltR*>UsB=+hYdcL%G}=6x#qPnT zGWX!5+HevUbvd+U2m>$5H{aX^TXtPGdQvevHoU zC#WYn6zkQWIBkl+bbdr)=0GX5 zvD0%GH#vLRQeUzpA5I9dK)dENx7tvd{3&22whuqJ7 zIzrQzlD6>s5|?YJa>w}p7l#<;|Be8&1bn|WQ0oM5x!rF^4*N46sz^*ZkKN$Di+YY7 z^oXBWkDtKjRpE2Bc7L<&oxFWT>!@N3tlM^ho!)o&-R);^tO&w_`iQl7jm#Y2DFVlS zr6}F)zTg%tOQ)a(y=GvZw1vSVIy|YcC9AGHL$e$aZ+#L|{C&|L%JI=NqxCGJ)HQi4 zjC#w4s9WjCS7&tbQJ(rRKit%cVW z_R{8CS{P6=;KDy)aKVQ92A>9A=iVcOD!;Kz4RwtG&wHDPsLcRjycAm(CY~mbnrnw@ z%BtHbq_?WGTp`81lui%@pPX4@4(I4-e?L5KlXxHuB|Dn$LckN&SgH)i%MdpWk9`FH z9q93U8RX;`c$83h-_%x%f9u_~huxXnsC{?%8#G^eAsfq-;TZVcsoVafbLi10+X3I; z5us0T_v1V~K|q$e!?~TStkq_#mE7ZqK&b1+zO}nl#q@&m!$%33?QOj#mPXV5NFO9v zQhc@%Cn1k}$X2z{S!!F}Fp-|yfScD2&JjYSGn|1#DLMHgH!HqSW9)N=F;iXkdVoOL z{i<9YQ=yYkW|_m#0#PTDlX~rr{r^$*6<|?rUE3%}P!JK3E(HN;X=xRZ2I(A-mX;VA z21MzQ?v$Zx=$xTJy1S&ihlcqd)N{V~-`89y%oBUBwPW=?rAIP#{`&r-fOt&87tzcz zKf)5i*D`8!W0IJ?9q;PoMlom&%a-|aF$uGV>GSsw4$G_0BGA&%iLvfEHeGWTT@Qb( zEl7KO8|Zl3)&*~CQ+pMezCn25BzjuBI%w~mt}V)sd{t_90DQ1U!4y0GHXDRS8-$r# z9K@eP3bto}&s_(QBz{FjOV*hX`upRD#Dna{SiQ4fSFKa5G@=bn)=W&spR3+MTaRe8 zXiK0C`kel5lNO8kp`&$PVclCmVUSe@`tXqMs>0Gdo zM?BY7y|mrU<&#(z;DwE4j(KV`@j1VoTi_Y>pnr7GJjFyaTn9>`Nl>r3B%&KvYpJbO zMl)~g{@$a$GwA#T-e|SkBdk6)zv)k>@#3>K%v^p@0;1b%u@Xp_!lc$qH66jM<@Wf5 zl6{!o{HVdi$DkRH$s`PB9N9b((LtSusL#+Di($VXO#&pjFz}(Hg6B*uTKM0J6IRN7oIJP@szXLAiV5487jQtU*lr2fyV_%#25V26y=LjxY_XLzFs#7m=b`$eje4OrZvRwbp z{x-B(4I|B{4T^OSufuLzf=a`6Tk~|BB@1-_ny%m3rby48W)i%I_|)cjdL&{g;K&(t zz{l}etNgkdTYZ7&^at)5T$`8(5ByqiQ@WLA)mqy!-Sopkrmc^}@z~paf2PAr=N9w+ zUICs|nPJBM3FFQj+92O!qEW|Xj`F7@_@g#rXjpm#3)_(fN1;i{HHm+IxoH8JW_QZ9@x3H>F$cAtv3Zc(*B-j@r;e5fCzFoo5-6vIPc@mZXcUsdRbRwBC5Xlg_l8D>q=qHtr?Eb4#nt(I zwf#knsE7Rsi)4p@BSBL01)-@C!Rl%Nr;+9Jejq>k+0zMU*SLekS^sDI2*119722MA z@tFO6mV+6Bi9h!|U%Rv|1!Lv{B`=?l9Q^Z6hKx*sp3}2M9dFuVkRC56Va6t?Ps!zK z^5u>S6<#-I>v)k*Lc|@fq5wf5u@uVMKhU3mXY@C;-vBMnqutYDAZr_fx=dYdg=AQ( ztEdnYm%FOmqBVN;1v8Z#Khbc#SIPQjSWwQkOzosG|UqGrj6|pfy|By4Ii7 zx3e>P{Km`On^y1RhV+hF<`>dd$BB2^RZW!&b(%$uJ*pp9r(jKJl^={~o=ys0`||&j zT?eueshjIJ(u)|0#0w+NBI66)@HxQzjYI;x`wiyJA6Sp6Fa2xxFu3X)eyHS8Th9#r zSz~DdR^z7oAF+K&BK)yYMvU4GLe7H>_BNoJh>lmH1E!J??O>7rAhh{!k%rM6R7FKG z%oQFYo}~XV1EdLij|amZJsj>{MDfScj)K7hZJni$D1yVy6iiRaIopPn#z+6{P-D=8 z-%;~*m#R+R6x z=55@W-yy0y3cf9U|ED3-&~FE%FY5P?nL_JJ0Q>QGWBZa3RemNa(Pn!J2K^m<{rc~d zkJVD6^UY2`cE8TZe=7HY_fzAQj`^GY0ShMUtK;c|Dq;VLv!J3sU@~Z4M5MmY{sG+H zJ-wqf61e(r--fS>+3GemR9_0~HaPqN^jv?`VYhmbYPxxgM7jRVRODRAoeh5S>zan1 z*`uD^_n~{B+;40eG_-Px-*HjnJa!Y0cLL}YfH6+w?f$P!tJWk5wWVgJk}W0}ZX-{|}*ThQ1C?;VY!T)Bf!UTAJut zKSa|lBcp(Mci$HD2eugSSy2D>S#&49VwH7UJP%Tjl7{}zi#j;>esLN`oE%?W;)4Go z4F2~P1*5?qdTDw-0mdr?ztpK6{{0@!;r@KAX=%tqGTEp60-JvU)X<#qew#=7(c9@X zFvF7mhlkH!3;uY%S2U1hu)4U;?ee7vr~`i6yom5_x72;X`xL^h1C41-ty_4C@tTwW zQ3{R!tH-{VH&YC;wx(khZ1;4kSt=3oii39;{0A`&ZJ*lUH&~IEW8>(WZz!?d*~A9M zu+t9T@lBU;ZPBAY+ykD|aW+V~B(e1uj1-YoHqm!^EF-^_$a6{Ao==AqGG{LIhTv_X=ML;4V&TEC^E$<(V{-K;z=1ZnJHHqzVvF@( zoJ6h+&JpTB5C6=VH~9oFUhKplX<1za`UgfRhr!RSTif-XuB`mWpA_CdE8;|9IXQE= z90R`&ypZSHF z_GzjnvZnqRc2aODA-}Cx%|)}ARb+9S_6vN!hV}dKu2@;)or)1HBTzHfY7bCe1OIuh zs)k1E>xP1QgIOn${*)>%c0Sc{04)#L4B_Pzy--FUa1QB_J&JTeM#3-5XhD4bM^IF zrQ2IgRDQ0Ai76y#(Jf7hToN&Rd_I05IoCxbd)vh&f(PX z3rpD~S{HFV{tSwpv5tyZ-sc?OeGya^#1x=$mwujsE#wey;izFnB`nlN(+T$B+KcM- z+K<>I954tU%}XAbmsSVOd~dJO)ghIve6)qIw$d_tJ(ug`1H<=4SEqlHK5h~I8XHIE zicY(ScW)X}{Zj@atrZfm3<(4!1&^ofQc0ArR(pn)O2DAMg1&-R|!D)dCH zT(Sv!Al#gmvSP13Bh?XCl|KN(-Z*WQ-_WuUV)r#+bP7mEN=%GVO4?N^S9;v|wy@Fa zWVhA)JOW;ErCKCh70?!%EXZqn@#RNH|9Y)1&>=Jv55gWWvEX_E=x-Y2?sAa3V zYu#7NdTy3ODegD_a~w?2n_rKx7>;>Kf=5y<=eozOmtKTzj@cDqfv}!R%%C=K>Cvjp zEPj9&Eg}ft9oK$B&9R3%2~<46K#l{h)&T#S$cUSlbz*tcG4O!K=GJDNWxNSQYS{x% zeH^spz?77^W>g}#xl!jP_g2C>tEST7*%J3WC7&L)rreUM}96S&#K1-1B|x_RKm_75ZfI-uUDe|dDfW9zVUbDwV>;XPDwtNxl;ka~CkR?>&w#U7q_jFMM;@pG;&(X%r=?DkhfK6>$}O zJdp!Axw&dlpSE66UBB=L7N~}1;4S-$g;4_|t(gIB8P*-LDHZFdA}CM!S{Yy^H69rQ zqdVPR(z_h!oGCc%VQ?jv$IpL}Qd45n6tRpQ`+rl4_I4L)w9m1jLeN~IH1;87$H%1C znQn*6Kl^4e>1C54D^nu1MO9WRjK;>j2+b28AIuBBQ#-;3I4LHH6loc+Kp9)4Q`#<` zc9l^;f}qgOJ!WcZJD=PzDB>;JzH)039S`=2^5!J z>IfEe{doD+HeHUHrZ~!G;&}QvRxbpRnzrS7E8uMoF4?{JD5Lv)AeachGdD~pf4-mE+12Tiummn=EBTDGu`lkme-zNI(_VfsZ9v zfteGxuGX8eYZ%2xs;YG4Py2a`({rj^olrJ`y=Q!~+&U;?-Qx66g=LZ#t+8ypK>*dw zO?6edB4AOh=UMU-d#=zAGy+7OLQNdjTiN=x|5>_R87-r|QB$Mub>GzJUqN47uSP?xh25i)*_6YD=XI8!OhB$a zUaF-+pv>3lwUY$-*>#<9{{CI|LuI3UoNpD}mZtX98O`KWg8a|mk-&6hLmn>gOtY1& zt>Sb}xocepBSKD+^9Id_4o+?)8J#6Iui4y=YEA-}$4M7;^J3)t$w zAP#^#ZqlIRjDX3`E6TmBG@iVOPlhO#9e(bEv9;NyYz8o3(G3?!X?Bv^7>Z zEwNYpif0vdF&#=*s;Xo*xn_ZwzuGz>uz_>vTgDaTTk{e`G#_{Wnhtwmf#aRc7>F}p zgVHK3pKHZ3ld3F#Awau(AzsW-)_!k#i_=nt)-8%6)^l|qnN*u4N6@yoY+!4rV|O@n zUUn_3H5ZyMcs({Zo4vPeaRO0NmBi+Jg2nm7Kpu3QT=P*!B+E$YTjs0t!wMk@uFyBM z-Tjh%{gQBeqWY^*yc1HAf}!Oj?J7`r&Q_Z$L$3nDp88r>Cv|X@g-Jz|_~t0fhXg6# z>N6|CJP(4vt{`yKdg^9yf|q}9S>=;lfB)m;>+KM3LL;)E?GT2VtQXNwC`uOYoh<*v zM~ZOGWu)EU;UY+Y2+RCq! zv#E+79`6-yiQdXLpQ7MWr|J1>wO=3A)mw{x%dCdA1rBpOYPg9xEf2WX)yv$9_`4fK zx+?B60h=QBZZ9qNq^qmQ^&!2YGD{H_R*>+`u{I##c5~8g)h?eXh9C69jMzml7%%ou zA#GA*dWz9hRi|qafl&MR97ZKrG^bR>-#IXRrFY4N_x8AYJL0r51h1!7Q~OQ1D20?( zI=0V)Ob^~SaH%5GpmEEgJ*jOc3+z8#)> zV33j>B`96I6WYdM&ogebp>`)lK%sm-E^%A@W%yGWQiKpjzt?-CqQPuNPENPMeALZR z70wNOuC@pOyGBc65V}0WCf}wakjnX4%nQ2=TNaTez1}fpW>x=h@@4<_*3`e$g@99P zY9tamcp%E<{B^)tUc|DW?df9cXqifw{0?HZnba*VmJCt=ndP2 zI7STwiU$rR8Wh*JG;x$!W7Lo3HIViQ1d$=9fFfC6%zCA7YY65c!w>c;;?ZeQ9gy%P3 zEn%I#H9b!!XH@E6Dv~T#pT)Sh{0yoC@Z}ixHXUQdgA27 z^6LGyam|<1nFsx$NAx4*HCXpIsh-rJI%aW4k5u_=dCPoul1UOc<1yv61$$+ZKT`)B zjp1Ax=2vMl{LELWJ(|EUIq9Rrm=q)=$n$w`3GbYn)zv8bcDP+_QE|WozublnoBNV3 zBS|@=Buj0=GV^IPHrRhgJmbXPvBg2|bRhVxGKCZx<~=)5S2r0NLK3QstV-ec68)tk zeo&33Jc$AAN8F>)F(-8(5Z|+@#n+&y)a(r8ey$QN?g^Mv=m9+4pe$Vxx=sW|Cwjs4@StD&jIHT&)X ze>R{5e!y<0CTHi;7>~YH=eYXaE?&e4pRUGOG1k-vv`t)GuSJ>%cvDGoTpnEm5xJ{FFuUlUQP|JF2A4?<2lRn=OacbQa89FwYb<+yDa<2#|1b0{d^+vD}&a9yW zO1R$Jw}V6$2yDcK!f?>HM^uNu+-$JE-35(L!xLl}7__UCA7$AXunyR84I4iz{bX#K z%fG#Ps7lalBN~6&A2y%dH}Ei+pu_U~W!8;SUNBAK9N#-5-pwD-YnFujIWXk6vA(8A zS(S6`M_D=k{*1%L5&@SXTj%AuJYGdo0v$)o54gj3fXt89DK<6m2lm6jC)A6iaqAZa zf;5Ffb0Z_c3c`9$Y`brVgBk?wyGo(hF7JI=rWqZq+X4_bpLyP1KYvNOKeu&<(CVyO ziBNB%^e7V$tTe{T<`T$1ZI#8EOY^9}#bcCGvSYrjc ze6On!MOz+p8W$KCVr`q4(F6~za#RSEGa_KPdEw)H$Q>(N}KQyriSP@p384smZxTe>2~$=jL7OP~tLn z*P}}@J5(i139I`uVMM?g$}I4M)vJw{*UbJXmk#!IRgf?`$7U&ZH3%y0Cfb|m{6~KP zQ9Lz7CY2g^`hyeURXOOUn*ynM5yw~3r3+@;(EYd4QhwxAc&br?)pjgO4HhvP3%izs zAHTmf3&Pa@2k6$><%4|TbAA?~vH;@=%P}Tw6GZ8Pv17-wnN+5q&@Te zj=iyq=iBpP8w^0tlf*%|E@c-Iw9(2jx2zmpG~5JQOuPaSO;fYOaQEh92EYUZ*39B2 z=`EO*?&tPIG7zKw1#)?@37x z#I)R6mgWwp^{{FHjo;F$VBGU2JG=3Spaq2(>TO1mlq%u1`KSFnvFDa(TR){9@ zxY3#Y=tjr7YqA4&D!cjQ#|%$y@S@E1bNK+B!BRL#*w7^kVeIvH`=C?D9A+~x#>RqS z!g^QD5_0qylsJzuoq+;sAV$70Il5@UWeoVMvNcgFL8(wX zZeXT|^O}38)H>S+&=K=l#bmLB6D8qE+2hgVo}uIYPqwqAck$6*5Td0WF%-X?KO3$@ zoRYf&2NQmfi|w73qAzFpiCM4JE01ZuVugLT_vmafGW;*;R5cS6gUQ^0l4a3EQbnE! zhOO1YEF$}dB;@6mFu0Txu;(eE#AYwS)50M2J3ConsiygN0lRq$!i7ogmIW@xb0cc^ zN_ec#wbMUTy1T3|>m+U88k*781OM2Ydt1IPtERdqCq<04axlltHM_7i)P7?1JGyJ4 zt3kNyA$Ya{8*O@aaGqL~`TxtR6@{c4YP#12$m#lQjHQQp9M<0?4&DD@Mn6gs3eof( z(`|A@krzC2BhW=b>dbULi#WMiz!?QDrUqN8UT`y!PSi{vO>5nqZlxKRdqz`}QR{m| z260{9u{3z25`MZb|4ynq@k8{QNSE@wBZ0pwX&F$#wX}Y_d9!TOa;!v^c)F*hcnwC5 zeCCt4Ors4CJVKrV0_!uW_lG;{)xotX#B`LN8AMqR$j}@}!mBdWJr2R1p8ELR#3ZE5 zdX>&&#{obkC7RcJXZR^s1*x>iBXrkPB8Z}*m0$h}O_(90S z&!9^n&HE9M5TNe#&#p3^3lv45o{UnAejx#p2_4c39<@I%|9F{QS8Qg@DN%-+XCcIR(Vhia zh3evcEuBd`mJ+l-3G+GiF;<1{Z4N6b#zp;@D+p*HP{@C;*59}Q$E%~U*nJ_RWYCx0 z`bzLQoaVj2!F{x}VdpE2+TA+24*3)dSJ<8FyYQ=RA4-#Vz>2Pa>A8P|OQfVC)=E|W zY3xP!;d&6B`mX6()Jv5McWp8YL>e4jCuH(Pf1T5%k(l{=>i`vK2QNM!^6(+Wws|@- zyf7+oX;;YK<$G+hwvLXbcxDGLJ9kz2{^7ws@-c~phc!bGXGYOXL-@-+2F1tS12gT8 zjU{)`6D+4PnfpQm+#e^?^)Y;xX3F<(W9Qz|Ut51NF+STMvBmaUh0yfr2$zO|W>_)- zp-LTIWiI#p$kbqhUX6VR`aL{FDg8=W<8hsW+>B=e7kxfoXBMx%EM7gi2YV>jE8EM` zdv$SvIYny5{IdTw?6TuN1Kvj6Lx^eR4=xUORP+eBCP6$>mn+9%EuAZ5SAB?Vx@ztj zb=h3E9+goCBfP|TP0AKLd3)sNq{1wQVcN8$iLta+*K`5?vX}CnrWqA?@NN&tpdcB> z9)_2FY|QH6l=D3!^?vcm3VZs(LUC|NwDF7^F_?{;wn{@O;{zV-80Vf)&&x8-GoJwu2w78Xzs4X=@4xyJFi zU_Lx5_Dp_>z34gjG$l7&yB|@y=EEe$92C$^2Y{p{L_vSTq!dq;IV}Q5)(-9oiOEy4 zdf$YOB}k~An)8wzPaVUqw(&8G-b1CBwxaVGdu2lcdV-F3u14 zoywhkCk4=)g`-vj*$P2=`_Evx-vFlsx4!JcBFb=U2?W4t-Zm}FTjzzd{7fkP^dP~e ztjs=9qkW)pp_i8s4#-y6^H3k!W3sC1a=WAOP51}dpg5c)0hZPqvne2e8di82ydN@2 z*)qe@G$8DJGZWDZneDPEz>S7q`=b(helc{vdi<@2H1=lem&VSWkYhltTTX;;eYP|% z)6VVb&Vi?zS*y=(OEVCd<||vehc|fTN*bOrVAJQ`@Ox$t(ip{Gh9-#2ZZSMDK||Zw z9*M`EFG5Kv=?t}U+*OXmodfYOI>Z$PdFV#3y%3rJFSfJT%8}+OCBz@U;$+}7ZvL!y z5HcJ7jXRY|b*Onej%gCNtP6nVsmQtHQe2s>t6h5IF5l3f%4jB)UY_}Cx8A9q2;yqYutKkT_CqUIjPu|ETWCb$P0{sH`>YS*L*#jOoLfCL?JQW=xbVR6 zvbqF24C`g(WQqm>?ZzlF{RX_H8-83+((LRtdV{N`pj8(RT3Umou`yNQ4dd0Oh#VV_tuan7kQMbYFDA??cJuBD%p;Vt)`)$GK;xR z;vU<^+F}#LhYN&`DChy@=J-B;=}#04c>a0#HF{YioKVs9NJUH~-f>YZsp&B$y$nsS zf0TD_=VZA!neuA;j&OcFd}x@L-KaZrXmdEDGr0P5qsi)BszR(b+($tK6tVD(!0j@L zO=z)(ps=2*u8t-JD1vVGY^z{W0N;XIqiZn-9cNfQA&}}T%C`CQIhTZ?IW%xPi2nAD z3tx=9!vtk9#)H{wQ_ENdizdmip>57OwpK3okGag3iz(Qv#bFODCrITE_7_!ETZRj} zLhA>fd8k4x;w4VLCc>Yr&Xy9Pqt$l;mGIZpAeu=jlg9>kj&x5F1!TtCDih(}H_CLO zgMhGXt{G-GMeD_F-6QEHKn9}1DW6jO!KpyixVUMrm%=AL?swNJJSr*lc1FEX_xl|P z0tjzK@TZ;)0rvnL3Qx#NsqGKIzdv+hJhT@QcFEqy{yC-0wKs{-vUy<2V;NNwc7=%9 z4mD;sx;YgkP0ZipxGtp1=H(b}ESQ%mK6c9;lq}ZH;o&K}-py!_TR~=m$Oqn*2J`WCNmLx06>*#!=-G;MeHcf$x?)5fCSd0v;qU+t zk0D|&K*l3DBtTM=+=HjovHW#RC#{2jY@H8@9-X|TyYqoT1b_(5$a2_&tJ9?`MQysLr!#9%?6+KPDF%BQ^IG#K z-R+QrZFrwMA)czy+zs-SY?Hp|H9NzP=Dt}k{~WHJllaVpV}kBhCyn{0hV33?ciEoA zu5d!;?w%&On}CV`&H`ly45dq`^|%VG>>L>A|5w>@jpbE<;d_DR@zby{D@oTs1*ZjUqv;BMv(8cXKj zMPsW4he@rOOzOKYaunVfuLfYpTbGeyw5f3aYMfGz~KzGwtze z3N=|33Z5Xihld$T;ru~fOeMmBr?ko~?mkHDPTB)#&EgW%99!&DURZFA-}Hi$dfddx z2pIPJa7P=c_>&5yJ+c#*HLMQ!F5UN%jn{_&d*wiRWOg5KIQ(K9af5`EWwUx%dBcS- zPK3zld`uMLmOE*w?Hs69?N0kA?2r#VV!yX>j)s#KjX&0#p7;zl$}ZK=N`GO? zSZe-+g>DynH7tknU8oJwC$qIN(Y?!LTmT+PmBERFc24)MqvpwV{giv`=)zB#7O~CO zk*SEF;hSp+&wW9~22ct47n5(UBVz8sNj~q|>2`#_s{ttD1?5ps*n%hvYY#sz3mP@s zb^8|z>~Re?<=@l55YoZGjAsb&Z*$tTV9F`_Zv1Q0ptnmlm(NUzmV{Hx`L(qnT@S@e z^WhTD-odIz%Y=8b^s9 zcbTvZcSqZcdwhEnUsbw(enYe5DU?Jmhnlc^o*n^IC5(xB4n7{cF|c0cgLuxy#X(Pp zBP`mBE3vTR$BOBUje$%H%g^toCaa|ZXouF3D}8E<7bR$EwlFtb9>9x1M*I)_5Yj*w zoNhhq0Al#VT?dyx;W%o{3}%)w(9Xpm?!9yw!1Dh_=4q^;wPj3MWXn{)h=Y^;7&wIO z38#QacLIh?EBC6s|HvI?Aq(8C1ZF%K5+_7cY%?uFS~r*4Kv~@Sz1-e24luGLpURQl z@v|+bi*x>0e@WHv$)F9|w8Ne+n}7v@`R;Wumy*Q$J`alCAA3B+<|P@=>KuuHGT}%| zSbN(&RZlBj8Px=u!VEw)&Rq1zUkhuE5mY^s_Q(v?HKfSj%6ZA!Cm9u^)&(Uv#IgmM zGE?&{)m~6OALa)j)mwQg$jD8yZ>p5E5|R_1hyP?^>XMNO%oh;PHO*BKV^)eW84vx@ z$#G=2V$m8>y-LYy9ltrLBODYZR)|GI5*J6R43@}WT#wWfdUbSoO1iMOU}R>rL-lcX zap$G+w+f*KNuw?8(;-@FgVPcmhlLk{A0FfVY-PA-XCqv7gCXlHhm$9#hxhX+vb7@M zjcSt*K0Q}ydY+xs&M-YwjLI#B$iUaG4roYk>7#v$db=7$Tu#vV-yq5U2YiiZMHOBdBfs5p|q7-zc;uit@fgiNO_-JZRA{sg*O#thXu}~l2`HN>ARDN!H=_^)hL00M{f5Y96JF7ba6A0Fwag`8)wxOx#Z0A`RIO_Tb=Q`9# z-Q_lp!q~3?723GH42-$uti-XXXTqtZin%%N7nzi3g5#E#m|38JPp=f`Gu;MC)hR?V zHhzbT0km>1x{?{cjo}g`pzL4og$8kD1J--X>BH+f8#eI^z;K=)9AT$ILM)Y9X$c z*Hf`NfSnf^0kCdBM?!OSF*YRc-M4=T(YV@Z9jd=&`>j5$Tewm7yvExAoWHm2lhg0; zDbU z-#~GT42;lH)>(r_Aoy~$LRCqXfvg}sFK=K9Lnc+5^DsSSCr(oaGoeyKM@lszkv`Yu z&W8x>Ksu1JNH2?}kcYsvYZDjOR;BIP@&{n{Bcwp8Yx{M4BL^mBj%MWAw)pX)w#n%Z9jpkN>Xra&$&q zYAPHZPpudiY<*Gl5oWH3mM8r!@7D)^)h_AY56?u?%H~K+w@B8v@N(7JZ&hBYN*3!3 zz;sHxy`_LUKd%b5;FYD=X~>o@C7VzfC-~zsZ#?=PRP9XJlfZ5}798+9J{<^*T&g&? zsprufS8W{rcyR7}A-3rrUY@2i`s|@HLvQZReW*>(xGA1(h($4uN(`ZTp1+<`VOmD@ zhoM);EgY{FM*bm^mkMFQHm$84?= zd{xN7h|G(5K|jsnulG)J06){S;sN9F%%QIWB-|TJo~e_=x|_x-*E#Q)a|brBS{I2& za7Y-UM<9+-_i!DlY7+I{is|G(Z^=F{hv~%uvU*)=XPw^TaOsG65+aXN10zRp_%|~} z&9{9;azX*j=*phGucH~wU@GEBOx7qOqVdYX#e~Czh9^)yBYhWJAkp<+$?{Lrz9dOz zbs&8Jyx#M6jzUQ51)75`xoArhQ|xfpN-gW3&`Y!wd*Uy*b)SK@aIXK+%oK!g8=gfw z>gzEgU8ZE=w`_>+!qNl@o_^f+kawx}9-b_tLCw=VDQgoWqcNC>Y&U*|Bs`X3P;i#uo9!tl z=O|mrm%~+_Ke40$f%b!+dHwxm|LS9+#^uyV@S)n_ewawVPIYWDeRM3Lk%XRb8MwbC z;X_nPB%;tuPVBv7gII-TRuYrxiJrWBTpXSU=VCF7+fA@Z=LcBt_7#^S3?-@K2>v9K z;;}XrK9Qgi98ghb%!CofS?Tb(sPM-Nvkn{j;>;Me$KgsTvGI7|)z;I*W(&VZ<%b6) z*5*3(wQmn3$(>F@tH1a&22$zb&Qqj^+eHwpb>|?vj_iY&rb;NeiTBwr7r$Y;5Qx3# zhZDgrJ6mcC67#sSYb~O}lf$g!!sx}Df8qGu`8dCnA0KDDw$@@=T+!IP@{eWaB-xlm{=QbGXlkt)W-QMUpmm@QXa3v?6X(oor*8H zORc%PlL8iYlBGQ~2o!=QSSF$R>j?oj%!;SKr|YYddFadC^_#FT4iNRnWn-9(q2f(e^p+RkM4 zxsJ|P-$?7!!5flyK{B~2>UNGGhw9w;bZIr}LboRYv~~yiQiDK>-Gi4>*O}^c--@;I z<^R|ecbiI2sJPbq(l=v27|(@7(%XDOtR7o+JgVZ(B76mYNMUka!mpbp`Pyy0tjyIN zUo=7rS}|L#Ve-~zN2~n*=pX8L8&LYY^%I~^&y{7lYH?FOG?;Ymi=V73@`>^WPqf~t zl%=_dzr;yh=@>BXY=rG4<+84lDywkx9uv%4^nAT{ElYyYW`QL$z+`kFtDt?IML1No zA%u$lHJ2rqdbg}el$^MZCf@a*ev;v1O48ik!P#77zS%E+oMY3u(D=)d_?yb9e$r{K zPv#TP$m9*Rm)*WH6K`n{iF3R<+A$uAP@0H=+nbz>IVZmi#NGm`dbAo(6~G<2i}5St ztZkI4Jen5*9eK!)$leBdXfe|m}w_zA4i>@xDu>36d_ z{o9tb`TX;$`l}L;FDwymxIcEGS<8enBMOf<4tfhb{Elyp2J- z>4rF;3f124ffzBA_OY6{{&In}sR^3OnZjXg>(A{DKR-BjDz{b@;Hre3wwh6$L*I+j z(`FKy#7d5`p%3xe9^obsZ+;r=`b_mdJiLvuy=`guuaFuS@tDI8V?Yy0d(CmH`xHb4 zx?>Y36Yk;jZ^GvrV|G)m%!#-XAu%EZxS!pw@@WG!CTg#X|y zNjcdqHItpzPt?K;sd!{CK>=`iKq>XDEECD(`9Blq|CDe4Lt2MRa%o%EA4H1q)0@yh zlmL)Kq=FV>3z{Lj>*yk9hb{LfwT@lxp^}!bib_@%)jU?a5UMX z=k+gMf?s)&>(9G-`&Hksg91fDNeRGk{Cu0e0JjK_i8Lv#svgav*H`Gu3cE11nE-zE+)2YGN4_X|| z4#WKmvghkz(_cr6G3@fFpxZOmTPg3+i-x}b5ZRW_^=#Z|SWbQ*><&tOye>I3ZE6kW zTrQBtr%~u&0RKr*6jXop&+~sD4g4!e);1p2W`s*@0)mv+G*6ornU>oIA#G=};Rkae zyzqL(_hL`0P=1D@y!joKk*1WhP0aVAh^gA= zvYG7a?nWz%qnq^5-0(ZoiOeQ8U!9{Gtgd4dnSX6V{{f|zgnm%e<+)smee&e!sUWr+ zuW9NX(KT6$)uMO~=K5-*sn~I2v1af2-7ACgc_Isrd>m|IJ3ON@cVNDgy?1X!`PJ7EJR_$dI*?!gapzlkq zdes_}+mf{XEB=Q--}_k^S^0jJrTbPhI&SA*L-n3cb<0{jmUj7dJv?27qVyCIQg5o- z8m0fZaD0yc<(r^ztXwO3I+I*=_0BlDgtf@a2og<~^OFY;w7Iw^yjzy+14tcOBTaWT z;Lgde14?s^Hm4BwsbK%7r#x_WQ#UL;O}TT=Jw`o4008vNo5Iv^w=GNZTweH&2R)n% zoKZD~t&!lq*$5l1;O@Sc|N< zRBQjk$boQY?_p8e%STy{!;eqmw`!U=xA{oRXcbLqf`A3(IV1C^$_dov5$IuSMGrq} zhmMbd*r=&j3iYaha(xEALeS*d+8TZ|v7|dkY6{@?1)rS&3jyXCXLAkviNWM z8MW-QnLml}^A&_OFG3GDCyog^EUv_M+dQhe{i9%}#ZWnqNf75Gjhrw=deN}Kx(wW` zt}k=zVp*t=USR~qrk-Pf4=isZmyN0;0K`bdb)hG)Vvp_6MDSN&vwxvX3{{S>yXIKo z5@J@G?b2gu@eQ1Mr;*4W4%eL(k&m(4a_%bJ&q)f=V>a+-A&q~)_w5%7RDLKck9=yZ zI0Lm}VoRnxJy^x{IECVJw)+RvKdcVjWXuF~@zdH;~$8{Xyo<#v@e%M6@rb((j6K6lbb zPEn^Qe8N>_@PZ#u_S5hiAAkK{-le~`ey;B^@)Jk1C;UE4$1p{Z{&0Nx~S+yjAcqWd+#-HHeGKxp(8(Uqx(*c3%(2tksc<$ zVy9tDr%R-MNW5-1Rx4K(^TmSGL1%uG=*zU<_cTw4MN|Nrskt0zDP0Z+EZV!0A-tuC3RAa%=WcviMOSU}ycw{LC^3dhxak zTJ{r6@+?A(qS(4>mB%xOEcL0?gr}(zu}AeLgd4YZLk_in)F2V2D{#HGN9C)TOHT;8 z@EE^VDkOlyOy%kAX5Q&toDH86`tpxn(WlE+d!WW(hT~8|kF5H`gKU{>+>#uSIG!5= zqx#m@1F@@P&i_Z%cYw3ihJWkZx7AXkRJA44C@nQwd#1Hpvo=Afy=%wrFp8F<_9}`J z#Eu!Ys%WSYd+!~D1d;fT_xt|;>-vsd;^K00&U5bh``yodj}^4ohn|w+hgG#n`zO&l z8Ix%nly&TyMhMO(zo4!D&dvFo-C{0Xr>h8;CTWI!x5N0d&Z~6 z%~&_)9Zr7M<8U1qXN$^`_>vQ{!KTXXq7z{jw1G`GzZqXa@b;0D0Yh&(WiW;qaoGHJ z5mWWt8n0#T`x(9!+5&StpG6clVo+W-N^GCuf?~XNr=U+Ccf{R?yR5;k7C3U_XRx{S z>B+xcKwpxO*&xG3{7|bzslU`0ZEkckjH@7k%L1`o@4b3jxl=U2nj!70vtxl<+^wxi zy9j)|c`k^8|2>{jE`7Wji-lyR{dIZC%?t&z)gT{A&K(|?$0*e-_d(jOINR=0x!DS1lj+DcDY<6SdPA5sA|?Y z_N_qo)d9*pKGOp(fiRb~KKp?j4IwI{+lM}0&KQVpNeQm>$MlK((uTBTNTnr?+b9`kYnZr_@&gm?1u`9e2= zIsC9{758U5P&3GDUrFZZQ?t6gHC2?FV1pl5gD)LksO5YoqXTIDVQ1K)0?g!YERJ@a z%W+KAN#KEgg5O?xwPo?yqLX3xgnj{NgtsF$|PWeIJx9U1ewGOB~Z$_C}by z?*uMIQeDKojjR_G!v7xM6dx^9J;hP67=x;RIKSl9ZuR9he4%7B;rIIO4^aLqGx&VC zp4_IkSGqiS{L(%=w!NBe!x>6)w^dN`^w}(4Cm)3YgTve3=7QwNmSVI60-}F3Nm#Sb zcFM?s9~n&hlwmZr%k+k`r{G7$LcS1j{C2Bl#}Vg`8P#68|(y3d}B3;pFI94?7#l^H)cOeHGHHzPa}p& zJ!)C1hYsHKH$GU+xFzM%oWH{}pBw?l(+{kV;uSK1?FxmS_wH<(r8seBL)J&$nJTLG z#ys_MQnX7(F^YPHm`zy;T|sgTe*X>ASr!lZ>bKJ`iNA!x*RIK{K7IfUf4*{~)a3i| zA_78~z1{cXRVtg;@>ottt2wE6%~KMCu0GCthTZ&N(79tO>a=<27jjC{(0E~MuoLom z;sW|KzxjU&hW}abpI(|-B;hTwC!HJX_*r|Ya@Glkec1q#V z_?Ml(y~n91od)HfFvN1@y@6{87_xHEO4y3L{wgjB)Gb z{EcM~{qhl*WOe0k(^c&o0?~xLoO1dVwnICP!YrDG`Pp)mP zl+`OhB={S%josxMN{vmcgJY8teMqZzLHu<5{b_lI;3*ZiOa18xW5K!i1H~bViG8=I zfM#O#Zj7VB-4kbPmFh^n?@{)spq)P{G~(70-d*sHQn+@!0DanGo5p%i6ylLZ=eW4VU)h=o4?o&>a$sHmlze_HE zpVIk_^5yOmLogO8WIkXpQVXf3&ZfWrE#1bFxlpH0PT?0)cv#KGLrP^jJu)cwE#l^n zMbE9|MD@Fbv7npLhPNm6)v6l&qVR#HQg+F*=$G}Zu^#9iospeLYmo3l-)D$6=^)x~ zruZpCBcAP!v`^P5REXRX5CNIFQI4tb2|YjUQor2N;u6Lh(Gr3L=Y9ch_1o6-|3Q=f z6T5XCss>(i8DA|F3W&RQ{=ODS>lI>mW^<=i>eW)WnDv)8vA z{$j>SP0gMW>Lee$nkdr*@9dFZ$4g*0d@p^1xjjeWr}T!RK6WBscX=8$vSk!xchlXu zbL{eq{VROxhX?CF?N_Pt1(|O4ypC0xM)H*akqkS-YX0t*?+%F0oY*%v&@d%2blxY; zMyO2akAf}nn;s<~A%rc}aF?#s-7#04SE;Ec`b|$zZ`C^&sE?HZ)XD{m44uSRdFkEc ze-t+Ww@9~1BY4{8mx+=;BtlxC-=-SV>!bbFYSm>lb7G z!{%V`nP28bOSfimsMg~`cbt$H3fJj}O8sFWK?>3DJYpwHW<$(Kt0{zm0&sqJ*`bJ& zV%Nn|D*nfi{kM!4-v<62^Ifk;ZLY^0{wjA5ZAn#BvbiNGST8Lw$mX94qEQcMo80`nI#^PMeppFH@z?mkLE=vclc>HlALrj)uk5jUk;;NA z8_Y(C(-UvAC;OR~FaMiqC-U91r$vO=Is<#6sinj!5;!*~)Ozz)o|Lc63m4t%_l|&g zz+3A7Upml>+#jI&KLO9ipY?4Oc}3kasoZpkwVk_uS*f;9TtYYPF)s(tOMZ>PnK3c)zi$Gmdyrxxepd`g>oIy>U5?q_@RX4lT?;@D4DtSnCUZ_HbFna4lv z>|fsoTwi|qoX&T6j!rK2A@K27mcxA$#A&*yncKvMc*fgx7pJ;qH#Q1lLGhZy#SA_ivuY z{f{4)(~2a>H8{^FAaLDh7rXO=Vd5nazNO7+cuZ12{s;E-aO=-Z9bO_ZOY`@R^DPoj zl6>yByW1aDm}+FCr}=d8LX z`T_w_N@_eZNP2y=eH|Vyj#p6D0_dbRkAHq?A8yt^k?_YqSDc|edvi!^Qf+mY)V^~% z^iaZQ`ZKd+=sL{t7~=Lo^HXvH>wH3kYaYk ztaJANF>03jV_Nk+CfRcMG1`xfgDUEpy?TTX{bdy*h-&KFno^Y7^Obz-vyBo|Fxe=c5wi+H--o~+@s>NmaL_khrEA?>++Sl~T1krNWSIn))1bRzq046v_WDe*JE z6*3>t_`W_biX8=>f%{%lwtc1YQepZ-cU7a&VweTWi3W>RspfXMLOoeCr`zA}A6v-PFRo#shd$d|y4KCED&&k2Rr zT8AGSlZ~Exbj8?1_g5^)(c4s?Q*L&1Jp5C%`rtnYH|VChAN22Dqru$vlIM7EvkL$P z2Gso(doxlMMZ#UeeMO z;qt$qJ9{K|^^CB`wi`M|M^n@p*vY_=W`gh8LCA-vlZj@Y`%Q$o$&g1J`Kjy&Lkat+ z(ezhBsGXntiWqB!1dRlp??9fm)pVM&(;9j@^n-$zL(e={3p7@K16VU@`imB8|9iZ} z+uyD`4H$QdxNEki9GY#?Swl6pvl4t$$XRps3S!d7-2fM87CgVWS(q{(%@g8hnO8pW z2N%nH+>Z~$idO673&maC=5>il{hN>_j^|vMjGMeIAMMB#+cDzqu8BFCZ~|YAh?pWz z&yFzIb%;mOS!WLa4h>LJvg3dw=A4c-?ozgdZj2J|F-nd;A4c6s9}P+}i`O3SleZF* zXyv{+CVyOpeLpvpy-PYkK|Vp|jVU#z_Rc4oaK|P6M6ENrAq*zkmxJSarq^tYrdUQfn_83DW~4d1&_dXYVm-5aD{krVgnNh2pc<$zcJ-h$n4h^AHDv3_U-@Ci6==_Ku>YwZG z-Myl&y;kPgCxId^vCNV7hnr6RV9!s+$Rx-miu#1>|C4`bBC4H?K|#yIVDfL9-1cZ> z3(=R6`k$Yfc2o-g_u0M0=?5-_lESA_G?+_Eg159uyFzoiDKh!y|4^=K6El&i;7gWny=7*E!E@AZe)S9ug{AMsr7~+aZ`H2(vc`qeS4AfyT76%{ z_rwAbL#miIR{mHOsOA_96@m+%Hv~$Qm6WCLXaRo{ro8yiH6ahrIeb>+RfXShQ@;)a zCa;w%UGtT41K7=%nE|Orfk&VD`#gtS?_xtpCn=MWJaY!z=};Zh#tPqs_MztOVRh>) zq1Fl{*KLJsgI7LFl$Y1fPcB9E11(#;y(PowISY*DDjLOwX6j>i146X@IK${Ynai8p zd>pyM=yes#WGtqqee(39Zu=rGP{kpDVmTDk_y}+GK$hJ)6 zS+1j64rrwERwCEkWk$|p0a>?mlGaKjtl>fFJF^$-Q1sB7f+7JxL4J#js zjn54F)+{Nxpa0a6ji=&BNK8qA>RL2Z*gfx>SSep3m~j z5PTmfUbFgEG9hGkwzHqB&gXsTF=Z~hI6yA$Gs<84_bva_ z&&ul?@GqN=Gk$KTd@=$3Ho+Y`M7)>}ubuZ>FI(kqWL~UL}NNcYbAxVE5-<6Y?_?{Qc&yC zVwK+>C1 zUMd?9V$a94wXq3KdeR4ilVm2iJ|n<{&Y{VL4XJ*sLz&`gIbWCOqXbq~U>adcj_j-t=w2-7ch zhm{ijy9|`=TMVOa8z!yR(cFlne|`BwrT*`%@7Q8vKjOlYB1hPlL{l9J^MFT~IiJIk zyZk@yWlRXe9iH31+H&~fT8~@rSvYusvtaa-lznpm+iDw=(K0yVd-zCZ02h?42;8jY zy{;?%t*5A|U9+Y|A=JrX6uGTOoY)^r8o;`fj%NS|h_0I0y=Z%7bP-?H53LB98R~hi zY6raj*5v+J^OiulAX>(6Np1iAVspm_;F3Bm{h)dBSei3k_wO!%*{HVT`vY{!LsRYR zU*ky95RtGN4I6pI$0XP|@^?R6c#I&iFYsw%PId zi$NE^yTB9Tf9pg=@g_XxcG5~#*6xc3uZwhF4mLMmZ^$WiSSX&@W9X>V_dLC9)>*!~ znmv`{Eu>VHr+i`!q48!1Cq1ABu4EuykMW8M^}zs;J-&EAj71V+!{`-Kjm-3X3rF+~ zEims-inI&{;>&YC^<$dfr^5{~*&rDp9=8-{Qt}Gf&DIP(t@ImBM(4b)X3mHWTExvA zPkJG^g}VIS0gIXn(YFRJ(~O%zN+$a2%Pz(uHyZmDjT?__?QNB-6x71$Rdk~yL_?YE zZj<^EX2#}!F@SNuBYQX$BbQWw!*OyjZDBXJV?VnkKwgQ2Mqj_=^)h7+=tL*7^(v2G zY&2)QMg{M~4Ltg5U}2JpnaZN#_;kp&bMMEaRPgY~ShJc%h=-%C&(8bJMTe^!aNOPY zPsjDZb^<&+3){H%&Jp9NI&jP!o_*;@Q+UB}E!OADz|-%zx*;zxZ= ziCa%XI_F$+BWtG&)c=$iIrN9>UQZ6x%(Tg7B_aL&_J(*Ya6&=z`eu(825t55bXNJn z*-6i37`A6N@k3OpT8I_UNkW-4WLw)MRW0YOaII%n4w5bZ#&a>c)WOQbWF-M!F1__q zv(Ce?rlO|Q>6NwhPFT(+3_DntT!&`_F6Kj&KDVi{zL{C$o1TNysz+`D|4`Vd00k`N zu(h!FRVtSad*~I^3Qm&Up85oAG?)IaLEcj>oSJ9&`1LWg_WOC#>Mmg4W~V%w37HsN z^gYAJLak~dygXWL8Egkt8hx0PR2*QYs@YS`VP%IwU&J09sP8*W(;wN7dvs!GPPk@}0h0MQ@ zZyesZ5lk!?!-F;?r6grNTt}Pzsrw2y&bfqk0t*UAF!#`FRvYgRM$2b&jf1K_)3IU6 zU7=^Ia$|_Io+h{712-q?-|WwJ2$aoqG!z`s=-FE`eBhOt^!4*5%vEemmN>RE6d`Ah zldJ1A@wL0tbxUgJX*;@=ZHeX|c$+5aYDdfFV6Hyz=to)H3bo&M&i9nJwU}qkI7=Yl zIjaf}RD_%fJwEOxdVv8`%RdKarl3wz{;LEVz9+CM`#l?NRmfI+t-FblP#EEtIvyl9 zG9>oJz^thmpAJMuDM5_R!bE5fLI&w19Rj;#4IPpUn3uF^L6z zFO>Ltx{Rpga{uP6Wlt_6WyVyXHRITyK7K(2qzd-xL|I)12v)XO(--PK?5Gl%nN!T{ z{OqbYuSl6#Nir#@(fH!KHlukmGe>l^3B?24RaH2_(do}XRnNxJjn25k8-+A~m8#YR z_=FyQ)bhc4*gN}q*`MpkYMRfM3cEk|n4LTwN_hVQ`W9S^9pB|@Q4NexQBhI2R%sm6 zR)&>qsk06te}MSS_{MizolB?%51sSD%{Uurm=SkU@T8+q zx3trBAi|cdE;<C1EZap6~;aG z{ySAqp!iG^%42b|l{{eIN*X$LpZgRR@6Uy6Z+<~I*^434vVr%(@}9^WqgP(ivWB`6 zX7R=kZ(wL@?GZhE&^llLlj7lwd%JS8liol0Z=XVZB``_l(h__g{l9t^P_cDmkjB@! zt+v`;ivCLu|8UJVoOb~mWWHBaH8z$sOG@1E^Q)<2lS;cra+UG@>NZ<%{O{Ulu@gqi zkdAZ%Oi{a6&+#*(sduZfZS>BeuAxSlNspV&)e>62h)t#$88TNp=Hpu^H%rRPdVydZ zNP7B#lN(z3``G&UJG%M4uE443L5v=EPD1@RYTJ>e3~l}&X#B*mvolZcwb@nrqHrP{GcxWvdR5CM9A zJ4#CVHDq-1fEr%Xr;hVV_+Y*hX)1kHG0&M0+oCM-S(@zztdp6xynTwhcTCh;$xWp2Yu z*3Rc6P7SEldCGV0emp-x`eEe)o@zKQ4Mju!yGv@|AvkBD){tiRNxj3lxh6YIyXZ-V zJ8A?kG$0lSNi{O?I-UE#aYpj1L&4^MV}p+SqtLC$t)Ha18DJ1HU5y{e0c&b39_nS? zaV#0yEpzR(4;GgDgQ-Q z>2~vav#>COf|@rdfMop2TJ!mxO{v+*QDDCBi%X??5`P)@7nJyv1T6}ApNz{0X1QjQ+uFHG!rQI_B3tQTZ_Qu`i*4mzG2tNEy=ZvuEWop461U=bbt`T7 zvRR{e6Dmj49%sKSRJna?%>n{7H5A_HYvsk)v`&r=Po&6G>V0K4sjWDUg*zV%hyRFR z>}fs5XQ!Ap7VXYLSm5#=#NHY>d3!+)l2O$2H^>iLnbYigG6Q$!DL*YLqHP|3Y7k~C zoIO1_8C(HBN61%My{6rU$e0#sLc{6n#C<~i1}c$drowI~bwGLNcZ9;V^SYhC?#=wb z7FC3+jA-I21nTaJ-GvJZ+(Vvi%uw17$%gGMmlsA|Dm5~V?XU2keLopRy{63Rzmg;} zt*EX_*gW0mttf7BjzAMYyQ%eiwfQ11fe)kgJ1R5}S?)YE3e|@gxtjyv_o_(aDM#2S zqoed2?_a9!S}DWQqaquFYiMeYKP-3el2ijk*kvx1CTPuoYQJ;6EPH{`wOe&bq7UIF zXDP!)?1^1o zQ)%x}QsY@H-l#)oLFupaGA@h^gf=5u^1)&BR4j%?=Qb?;E_Rinsf~AUJfp%Z-2Hn%iNewbpWL3oqwNgA1>cN@ln5hD|WT8Di0TP zpg>sPsaU|v3DsiLP5}h4*?~I@K-~sg0*Nt{`3+2chUJf2@AQ0Ev#<~iyTBSiGeqyn z0Wr>#m0EQ}fk%C<-=>Y6zHJ#}$4ME}(dh{+^}<4g(T4 zt_LUaaA)<+UY;`by&%2r7)tFL0xjKxg#Atjxc*})25s(02H-XgsH1nyA%VluB{k?) ze0wee_4lbOUFgv)T;j1{PW>6#KAmmSAGTQs$!N?DE?Xji-^_FxXm7jaTp01=+1Vu{ zp~qQwES+}Of!~O2RvN`=-LULpC@vm0Dgq)qKlY6Vl-1c?SV65l4_fh>45Wz{9~!X% zr*C)Oacmcyxm;Pp$Xtk-oDg~P7USb-8G&vh4v+lH_OZ);T$Vq5H~81BjqK|xy2HWi zvIS~}>3%|28bpGzhF?-p!Wvd?HSP{7n6P23OD2*_w12dbwkKz4$^69v>Uwsir4K`^ zc2pHoC7z;>iqrQ8uXAt{W7#C1-cfjHR^u^@F=&|Rd51SbwYIocv>qMdZC7Z9ud+uF zRbyIgS5{RDr^Mx90aK;h^a?>0O<*R4YvD{?j3ErvCHEvy1srTup23gi>!mmjsK7OR|TyTQA z{XEQTt4IIZVECA;DFpHr6cL9F2-#{&th7e3Zy+ozWYG; z*+GDkw5z#T?oC6M8j|>p1O?~S))I{NVP7speJdYJM|l+k2B(XpHJ>k7Cn0YBb*#Lq7>pippiDh~ho zGyGeS)&6&(XwPUqc=RMK9=P{B;`n}UZ^ZjPzIRMjF;3+z;+4+Y>SN0|rW4tW#0x*tjkYG?R)$@ejc8@+k&_>n^+=Y;~ zFkaTs1c}fHi}~TT&BjF}aZ&Y(-94fYP!WkDnpe6K9A_Uc_t!_bJr_lbRMgDNqKL2J z1qJoYr9<5thjbV*hTqdDDpAca9#_R{mD#&d)T)d7FsQrM;Gs;EN)=1VkSr;AOuu;~ z$c6`0Um}iX{knB~?9X@CYgQdqjWvB^LWn>I;^FYM+bhb5lCm3CZ-Q{PwFpq!;0d%< zR+F;47*Q^e5emDS^kWvaTjJf)J2O3AVT(PUqu{dQ4s%@e=sNMoHZE-5&6ZQRR@}Dx zPXAhNMF~n&LMH5sxsC-iX>Rms>b-4{M!3gGM(3Nc{iOvBL(bW$sx@QIFd)G+mzGx) zb6bMB8+TzS#7Pk1XL2w9E!NH9NRy63)c5 zy`nWB|GN=ePzrKE#?e&Y==x-v1Mxinqul%-z0QX;WwfLai{Eg3d8~` zb3Q+tXjKs?eDl|p3Wne;P&cJEv~=Vd(HTtw6ao6mT$F5oJpDQG!wy0$2Ov}0OEw2J z44g`x3@7&tece2|4w;St$|nQZ8T3cl#9s_PnmpoN1KN=soB|8q(*CaSad~-VCFQG; zV5y1D6G4Sr0+5CfYxC~G50NrJUB>I;FG)uF22BqB!6t@;p5MmGCDlqVlMxolb{~#kdE%c1OJ{_LS|` zrVU=huj znzYI;|3orzwGTeT&6;coY#lwCR4#B#f$ABZIZYihdljCQ}C%Np1N3li`ws^_I50) z%;m{t3d7X{L8NfRg0%GMjF+cAsg?R3aIVsI_MREgg?Z`mUP^; zVl;DFCknmS|EQz^|0~@iM|t`C1T4DTj%gIpLdv`h>yevT1?rx-G25yde2 zrEA8+v2t1YdbjY0T?qxlS_)q*v^Iv&?hxr}*Xq?cto?66^AOxrat`5DKGWEYx1I_q z>xBf4&uNq2Y11AROlm}n%-*9|t*VG1DU7cq_x0in|n`HLp zO>0l17SkJtqTm4CL+a^hDqUPd4dE2q8Su1J$xxS z^!!=(MM`R+<@eRPMG?>v^mqSjxuoU2#pr!x1+Cw}LlUFH`nI~>&qr)I8@!%eT#j{* zd%$s28Y|(B;^hM9oSqXpj*+B->k9&9>5enWJ~6$+f}xP5_t@Pm`A$QVuT710Hj4-B z(-jWRdSf+ciN}wc{9JcBSDd%{cIz8Y_h)wWBwEXJ4A zTRgIMzHt7D9C$n)ztDfkD>dqI_&XmufGF=1e0YPyC8UpLt6YYcPv0=LwJx_1RmJU! zd+*Q>Jzs9}W6dnQF*Vu4Z{9*`gwNImOtdZWUR`oTkrU*EM!qr`X~>URSh*b{!QS@2 z^#MIGnhTslpa^xJ2s%=&dq{a$;4d`J%kzSH0Q(IT(zkO7?Go2`E!J7m`RLMFqRxIp z5TSDrL;IjkiL%o6=tVV%>g&C|ccYDY1-1?QjAaO9H&8b)bdc8dz3OGkOof$!a{mqS z&wrpWOs$cS0Lzqk0t@;&$8Fsx7~kFY`uqo!0Zqh6MMJk&jvnPdM<00J%%%vZ*>bOM zwBzY2siFe7FqH2LO!2X1zwttTr=_ge{wW>7mq1gb;AZ7m`E#})!CdDRtK2wyOf9>E zyhU?f$^Ao62;Ep=|M#k)xSbf%bU(D@a}B>iTF%4!)wL#+T25C z=P;-r){;+&DyEOh?qKQ)g{2*!f^Yj&oPHGLhn6Y?JvT8K3>&y!ou^!2dnLaA!rd}O z*)+Vk`jh2;wp!+TT1Yn6Q;|E`Y#VH`MME}bRn-dSjv?f)Cq;#^=9}B2Y`991(8|}w~TwA({g;Wf#7N+?8jN1j~N0D*O zqeEf_H`XzA*^yAG0}4YmVAhNLdY+_L2tU;APY9kK)fgK?RBnFmm z_#M&!)YkfB<32}43Pk$9oU}K$oMp4yUWb--C))~_sJI^mehz98xyW+^h*!0get{LE zaaD2M_WDXC!n_*W&tB~Q)|iZ-zn#MLoka83*t{Ou5<0mW(|ge8HS9f~Z2LgF)fZqI z(>e5St-6%vfGhpmn)haF#pcGJ7?2CA9ySInocSS&UrwFVArL+HUbMh|tN{;@H_$@M z(2)D75HCMJ@H4lwg=m7o*Nm2$yuH;G+smZOml@MjR>S7}*Ls;v8+l~{EVS@PG12@~ zUc=&a96=7|M>I#ul{Tl;@nY^4D`yTSAzDGOJ*D)2%|C|i4SuEuH|}eQ?l}aGX_L>e zIUz8dCJXoom6N%6Hr?#hgN7%EX#l{my!UW=R3BqhXUHF z)eSUVcRtxOXz-^e1_RzMe0$;L*eHg$Mkg<}WAk-P$ov=@I=|ZcIvnvd-n`-s#vn3h zY9@Ytyk%uyBzNH~9G;w_U0Tj;4(os**!u>!12d5FO-=ksn_D0atv}jXO&+Ho){<{4 z)8ct`gS(p|Q*LLmEJXRW){%&=A2_NkXCM+IjmS4qnLxw__uMIy7>%>|vHWaHeE=BI zM*X4XgCmqbtHtBg8t8nm5}Dm5JU%0r`dEf=WOHCM+&923dAF(;(O_%c@o7=c=038@ z1Jk|>QoY(l=R(>|3-F2Wg&P(;(wAu6uS55B3;x>MPeHo4vfTL4($Z1%Ex38HaOlKphG9}8nc}B1*V`);X7W{4oF*LHir1oI z3qef*tE%0>?1Dg`IVcyfC7^3;5X2TvNpatLzVfL8Jj%@Rb>+Lwvg?9%4sqp(e$(>L_+?k=1L(mX>ff6j9xXKX*KF)#4ot&0?P%}@T#x|GSEs$Uf z^duUcpCNt;JT`02d)DRPlL!P+CeT_+|N0gn4@LfypdEsW8D1u)Jc>ps&g(F z?+%vuE8$^)LHn&4jcLuujt-NS^XcUACyb0nwkvBX7Qw;0@5808Z18h;1qPnd?t({Z z2$W=xgkEWriSU@@)iH4!Tic?9sCa!(?ljgHJgVpM(Oruzyrz>=A?xQFlP#HxL;i!! z{#?cJHm|vpUOq_VD(+O{luFQu(64$Bw!;zR$YbyCd5&vO?^t=1tr2`Q0Xhc=8YRz? z*D%}g6EF$S;@%R_<3b=%C2z|NdDlzaRQR~cysA{?=@qjI#NtLqMz=JLym^{n3kTt7 zbJq`1v6s;^7`VOuYJMzOX^BsjX7)62LM-{TLvl2;sin@ zdGzuQ%~pST9uwBpbq9a~IIncQ8e^xZPX&^43PBbVe}mc;%BEO4AUSR(7zbj%XRI0` zjO!{aRaSa0SvmzPRw{zz_Nx~`s*~DE{LVmyRNAK;*(V99B`UA9UiiED8yH$xOxE_i zBj;MHCVWUt?&(255}dhd7kgqCil=t|*Y%ZvyB3~7CTL?`0C{riT@+*}oS=YSE>tmC^wqleVIL6Z&J zXov!kA%y|BP(*zES@@BZ@&lmiu&i8Ha&rStD z2fdXjkaGn_5VaN2=A;$^L-Iz=kCyzVb-u1SgPA#JF=wU_?Z--d2>-r_t~H3=LuAlK z!Vp(elZck34YNMfK;oQ8sU4Aso(ru)E|fgLoJ??Z5+@-R;1MQj4yE3d5vD$<*;D4j ze}AXVRWw)Gssi!M?{^*QCCS<6;Xq|3s-`*E-N>oJl}I?1KdP!%1Z$`evNzdxsX3OHvtRK2q0Virad-}F)6)fNp7MDEhPCqbQ!}cY?b5%hVCVO8 zV|Q)%G>K`4=STTjfu=Hzl*9jeSG6~94&~LVJMpb)OS@~SVt@9bBjb(^ixkpnm)Dj{ zjg6mlajOz({erNV%K~dxQ?+7LIMv}Vv1Uzqxh0~mxOc{9Q$j6uKJLTsy$jx2Xflyb zByudO>sNPc`FUh;(9_u%A&{(RGjDMFBQS5(YpWGXar3~A=*AkBv)xvCJc(4sm@+o) zTL>ma(t5iE^SR=xwq~D(o|Fh@#g@&nNRG&QoSeZ8Q_3cQ%8B>kG10f7KF14`L!~89 z(*X3*Fq>R!VY!3NWXgC1m z5FDl!Vjfmnjwd%V?3F#M4i0!DfH^h?_NYhz8qzJkE3Qpj)B}FN^WsWD(zCTn>KEz;w@50O>UGOd=SZhCx72~mT$^!d`%Xlc1md1#j7xEW_;pzE_I?SHhF zlqzHK#;U~d?MwCV4{5r{C;QB705@AK0T;+P2l87V19QwDeDAF)Du7m{qoL)Iwg}#g z_pH=DzJe+Df}(GYmYGA` z@eS(h?b#F)S1^5>)H3NJLhqBzn{Ch$Z97w@T7?X9u1!!k8GF~7(@h_vEPK9>xcDr{ zEBV%w2}<;6WGL1Oy&yo z%OuXcQ}bVH{Ka=+gX@DluTz*Aa^jrJ1Juz5KwR6NAMJ6g|eJM2stG zgMk8Y!RM=baadXU!kQ418N~TK1efo)iy*Hcvw;F}QH<&)Uo`k=c4}z#$7Ex``T87c z%G;&nqp@g0@Lbtg@Ecru6g+@3f;>NmBEIYsWj}|u*5D=N_v&g$=NXYX0H`-BoHchk z56UnUOs@~>bB;Vegg}a1LwtfUIcMuKyn>-SYHLEF``|*iCepL1G1U1cIQZbQqnAr7 z5fO)Vk-Dd#28k+?2)n3!I_@8%W0Tp}^DBs=3I>WCw6Sj4i|ZfuJNuLK1LIiFi*kt1 z92bsj(4lTqi2zcswUB4~LDiHuLapimXP*Gjr(ge{)Nb^qiN@W!1^9i9PtTY}g)VSV z1&ft-fOwitTg!RbyQ<8|$p21L9A}!&vCOP_!_rpSEoF z66!wba`&e)VPto;^&F4mJ*6xf=Sj#q0*W`z{%SgVRV6`C4}qb_I2@PNn?>mMh~-!t z2yu(a)gkpM1t2;$Jy#wGEBL-5O=i4gt`kSHZfx&v#eMdDARYRgWboMtqXsoMWj_7) z_apPF?^Q>b*v|R*%VmHi&k4O*y8XVySz^v`W^^dvdrOnw8)2aQIZVyUcILpq2%{}S z$HvUVud4%4QM~;T+0j18mQ&AH`I*<&&Q7j zdW8_sQJ8_^#L0m3bqZk`pi+UtaxsOfPv?PKYc(4M`*B{3fd3J7P@dW z%91=52O!7V)MOhYd z-YyX&1f;gHBIxX2rpmJuk;hhF_4KQJXC_NWjybK&l^=*~|IAC`r{?PhhX(ZaC5kv( z=$o1%@#!Aakukle_M@_46|3n6CR|I--?P+sRI2Q~t*sn`eY94NYet_vpZ%+GFH$Fg zr8w*Fxea%by&anCvfLJrC>_U*F5=a807vhSp%HrYyZ0qBlDwPWQJ{xjOc9g4wSzTH z&bYCWUL>6nv1AZ+)*kOaVZ|*bB=*#un@L3%r@;4JW%cv-s-?>9+{6`sF2{^WfKdsR zJoc1rb+B;a{!|CLd!4J;zgU0txAD93$@Bvq5!>1dG9D<)JkF5h4TC5D*=FsqwePA* zQ_)YuNe2)Cbc(6FJi)DQp&*3Ik z=B$@m2y|D&n;i>sd&qZ<1L@l_N%+aV16q6}&GM!!rkaI&sB-23~US|FHMoQB7@I-!Mmy2Ne;I1yq_1L1`)= zQgaSPRGJ79IttRH_Y#uW009970jW83sS!dCBoqMw2|e^gYUm*Z2qYwVH+b)T?j7Ir zeBT}8`QG=BcRYLS!AMB5_g-twIoDcq&fjm^ZxkPq$_yYmEBOJgm8Z@0C@>hIr+<|* znoOXpX~PS!_0AsmJq`T)CdvPrwY0{4!$}spOB5muCagGKaESm* z3y$x}xR+nc7YYr4)n9K}LBGux#{3Z@-X2%G+($fCkgOO4f7RMUhi*|-_GXf- z`)3K}7;$?kyps*|EsQ*v>y#p0AjU)!x3j`K0RRGwg20S?SnW$KuD(E)2JM8GpBWGu z#n2GK#M>uQHUx}$`HIZz*0S8uurkv-2qKZrgi31ZrM`RSKmt;FM@D*6ms6LQ9j_>f zDxt-s9ok>5M~N>s&)86&^;1fKyvI}F8kbFbQ}4+tDT&6W@dJ~-=g^@i2`XsP4}{A0 zhKEf-X857rR7BlUX!xJ5QV)SRr91$y`OvgGj_MennVBQm1DgzEzl5Vk-k(GpQp~-! zpV5HOy|_3#bqSG{V*V-ajM{aLOX&c;rXEPnuOT>IBY&rEO2tOGD#MRyl2Q`yYQ;)oKSs9Fzjiq%$KKu zOq~Nd7vC)L=3PVi(0s-1Pc~eCI~OcAaqLs%^#`h3l}lvyYNVmOwTU8|j}T}$`WOVI zMTOM#x+1G8XxYd^#$bbBhFmA?N%Nr75HH^hltKb+0P`Y(&@Z5$ojGEv@L}0e&rk2l z$u3;<-7+pOZ(eY`6}H^3>`H(A5GuG4dV;x&hlP~#< z_3Q{9EYDOJiK=+aSH{U9^?@(tIJ*%G^ zj_C23)O7q!)nQ@nkQ3|9i#osDP7FLyS7S%QNq6x~F!SReSoSmL)i+DR#Rby-%wJyA z?4|hva4Zpr)n+~EUPP|hqm_4g;ig7O_9ayzIaELufVj2|(5(-F=Kck}YxkM=arQz( zL~;Z(ZXox(!)#MWI9H;S7~=le2{l_+yV2Rs0&O5!tNR(xQi4Qk^;+m`h4~il^MQWu z&ZWK$Yyl7_XKN5QLW+Q7d*zjN;o+Wb;jvuAZ>3_=wF?EZGB+^fR%f9J%4oUVX8?hL zA%)8Ro;mfZ&br#4$_{qy9O&Z81bn=>l>QQW654%PQQ?lJV8w3gxAl-SWO%1s+WhN_&f$YSe5{MgI{eMn4*!^X22S$G$Y`WXzZD&caLEmUG2DV zjdicGLhZ-`?W@e2*RFkI)Ie_QLRfiz020E#$P{)FVq&b)FZ?S9!HDI}z+A%{fA41R zK-@aH(=T&D($JvS6!*viK55gVDBkzgJ4EU@Mt4wwBi2Qq}D zck|KoltKVU;f#6;5{OczZ;G>R$I{y)XsIhb45H!-dgWY{LfDaCs1Vf{tR0}EMI z#Xz!geFe&4ajvtV6BO*7eDv-aDi>^SMIOc)>(-yBy94k0_`RD@x<$iKg2@d%r@N?L zm-gJ|EsvcFRO)RfI$pP-g~w|li^qdEfW6QCM0iJ6IAHtS*jaG)>S^$ovg#jmtu4Zk zvCwj75G7i-Mgf50DCOP>0s?An3p1#z1l)732vokfIH=QL{ z!)6yl7xQJF`OdZg57t+5UFsX7IRbPl!(MReg7=z2o_3iHY$$3(%###~y8wLkcMSc{z%S#tI5DtgoDJkGOlOMq%XQ zcXJEVN4MI)Dt0<|uvVwZ3d1*@4oHBKy$72juHB{e`x66*#Y<+2wWN^1-rkcB6h?ZN z@9AABbo}kq>5_s%`cyaEk?4$suU!^KRBts42Y0)(cH43Z&HCe>&E;B2YIdHD(5e#j zdliWSvHRwPV;r9>{VmYDrGm&9^Xlq?4KTsGa4{UfK3G`nrHM93EV;v0mcuSzeoCmL zI$EV77}jp4DU@CkNedKAHK9&6uAWeHgDnKEOfJ$33YQ9Rt^Oe^3h|zv>WVr?z93O@ zUE4hUdFX?q)FHi--N_l5b@Pk#LwgqOMdWI&2p9ZGa{+t!_HD4c+Z8XMX#`qAAqs`F z<#hp`;nO}7JMd!eFa5e{$*79>?p0UQvcHx&@$!1N^KR@t=7qhr>fTp!mGBZ++phJI zRX~S0yi;vD_y525=iJGksMWpK-`@WRh36lk|HO}U>&l+;{XE`Z{M%K5w=a%q+N1ZN z{?vHXW`~!RauaBzp z=g-0r;4dQF6;%^uPb(pfnkcWGSn4QjCAoFHZr+8K9BJP^RZAUZ70VNE2*h;xA!{5v zIwKv>TpWK~{o8~5IO^AJSvBrGB#D~fvhNawtv9144%zn$vsb0N{eWkZHYk@z1 zuBE?Zxb0w}+n@O8`unY0Y ziIKnU<5h{~;6PpdUARtwC1Cx?8}`DfP3cx_@=7$vA9Hu_J_6&d*Tf5}TAk~F(_M(q zzb=tFV)zR%Tm3vPsN0_T;ziYVK#R(FO)>>^n=P+R*L&>Wb=*UhC3fpti(sv(%vJC}DD4AclRc;fxERu1I`VdOKxj zvo+=A&t_OgP^@qc-AaG_{oXH*?HiS!Z=c=!531J->U&!Ax8uJlcK-)%u`k&-IH3z{ z==7u(B6Y=z!{V3G<5l}|@PoT=2(-1%g!PkJGaD18uWa7sZWHE`)9*`?Q zD`xEi?rc@KD=Y5VTi_#^e-fa7Ps9GXsbaUr1OYJESj_GkCo!;FkTf`hVAHUwyDM!D z=xL6`nn2XV9`5x5oC=1YjcMyHY}bdc8>*<(qb58#2pZ#X09~R942jge%~u7BhaQs# zvoq^l2+Hm*z|WpYp`W*uQC(x>;->e2=ne7TPQUs4fcf5Z zN|4Mya#14_fLbTY-@A1CNuBp6K*$m&1Ner9y7tuXN9^{1G~C9+F`yFSDJ0cQTfUM$rKi}!~(R~B#LYw8jr9ABV^Lp-;lmmeAuB8e3Q8b<=m6N(vdcB735xvNznjnT+n#kpC2pzv#0VUG8he)Jg?JwihHOcyV6Hv! zKYURh>r<)3`t2s6rYnGSdpd25;4%%g^o9IE!@n0g;NbXI8?`^jyf~p203d@pyR?A$ zh)?&{O#!uU(K9$dt#?BK$S)LL5(zegU$TdM-RlU8`}-5Yg*PVvm&$UA^m002Do{nW z*+0en1=Z&7tP{YU|HU}|RIeKZPJ&lg0UZ!~t{>bo>kMngstawemtTm-@5{qG|G}n? zt|;#PFT|cdm-KTmJP!T$tjWKsVK4g_k6*W+zWt2NCiTHNa*Iu!cgjEd+WsGJbZ-(g zY#b{goZm4YJ`C8cM^bJrgi_8U_fN8me`7$@(BDqd9iYplRs*C-sJsLJAP4^=wDtOv z*||jA-o2U_3dy19N3D>_cm8fN{}`u#ys30*;}4(d@)LeJBJXa6FJjwlFarOxFYb*l z@GJD`>H5^}^rQbY(?opt10a(?4sH&4KFi$v_!YIo8AX|sC7W_~sEI76EA&cNBDz@g zoZ4I71Y#wu<t<9!{0J*|yhfSm0881O~`N}~r8GFl|7o9ObbxrD30K5ot=Zt>tRP$?Q zxYO-u-*RQ2C-VbYQ)suM6z9D#U3PGPly8VF&m)NSINE>y)z)5%ITG_acf~&gFB+~* zFyIVow~Iai$r3YVCB;mXWjI!%eS~ZvLP2Y6NGW?+lM6q-IYche2EEl70%Nzk7%^!T zR{$Oy*jzgLfdfQ1wAl+`V?k*|Nit;4$aL1KnRQ{ zTRDIdajuWe49bc!gGb78{7XvAWfzRzm8v7ja7Qh3LkD6;I$?G4p~<7lqS`vo*$4=D z$Oz$Em0z3TX*ikDc2{Y*e|^5u*ZDlvf%V_!A&)qCE?UgjkV+&{7oNaY6RtLE^bY%z5JC zBstg*(w%SIYgg#}K)-ldbsYbkcb~pyPGFy)=1EC};4qrqYbmM3<6p7$b^2oAmuyE< z->%$2aW`|*_{y`7tRejeyH_Xts!1lqhwxi~7ithZPbx83O z;$Q5>nVW$t>^B!_CB@e|aZ(+}FDkgw)DBZtl%gw<7B^uebxZx#rL;e(GvSp9qW-cf zLteXo&>X|KmI@2;MHWHi(NLd$l8Ao!3VZz&y0U!JPZY%~@foO?GgY6GSQ7D!PcQIw zHnM&;gG6(8Co2w&5YxV2FBY?|{A;mt)7i~D6=CX8fThl7^po;3?R6Jws0Tx5!nYDc zs}SdgG&YB@)B5CMg(^Q2d@`CqP2$($G8WPI@ey=%qPH5Ns}RWu5Av34cX17)q+^-F zP;^%89DZo+Jl3mKrOf_e^7Z+AK3N%CGg({}wRNMHHlt2Nvpp7l`ujp>$(ELeg5fJ8 zRqS@CXsy0AKCjZ%+^=n}573sv&*t$Ax_1^;a!Y{NA7>FKpTPr+I){hIzI}M1AcmjV znnl9NFPj~}*B@-sHddDxCTiS?iF`%7IkbuDom2_E<%TUu1hjf*xz>;}nUR|?2b`Rh zQab}-?lmjRsSh*B?e40(BNKUjD(%3>>bjC)>El%k@bP%-^GUZ}#G6AMe1~i$DJRr` z!1-8hB8EC{)DvFG918*r(6hveTIH<)3`GeQI#x$0-Ho0o(Fg;wlW!pB27#-Tm=Rul3{B%aq&?)+4EN3O0TqUeJ z0JXngfJWIu9NM-fAXrbPB-WSKM@6XIz!Um>gTlB9oOx60!!kurb zf`*-R^IEF95AfaSy7Lkue{Ev726w+27#aR>?iNgS|K`pnv{#g^}=0<4p@w9cT)S0*f5|~L*2U8X<4Wp zcOkdu`l%zV1}c#e>2on-_23vL-~zh#S;wkorn9lU#M!ZaO55>`RGD_mv*V~NSJ_>| z>VTHH*z&G%qCr*|gT*>3NdkjXzpVZz^*(oNc#$!61Eh=Yl@c`r3#W$&~ z@sYRXjHHUiB)V|z;om>MH7d`X#Mk!A!(*B1rNcVOv+=~5N=4GZ4W5e44IudOW2;y< z4{+^me^cvQdmEkM?*P)|AN3MD~X_d4sxdhY&&{QYLT zx>98@WkcDs#;&TwvNS^qG&)q_YTa)uX*-#|T{;0r*)0eYtRA0Mn|Efc8;0rUz`XD- zbrjp7p+PR~r^6w>;-Zf%N$WWa;}DnhPw>dX2gf{aWIAf)8b$2lXSz@v@AKFgg`JFKhZ zr&?9ISkgyLGL*g7Jv^o?LSIAt%Tx6X2d&w7VqKM+ zY!52XbsEBtV8tV2a&o6683<>03!_BAB`3)0aMG5xJb@YJKXe*^KlP!Fhjv@A%7N^J zaHx*ZXa%)B-SD-`@OEC7X~~JE#WPidP%_`$m|MXc?FV1`7lROg7+mt3(zzpPhVWNB z6~F8bf(2P{8J7@zg6vx}YiT7?5SPyIyh>H-0458uGc3t(@7G}B&~NCP*JKqUm*VV@ zfIZrdNpI}00MKAry(etcp~ZRHV7tVk{ClWEN2jj2zVJv?WW>6DTMa$zH`^or*?W5O z$?&IZ$@@VOX&bj{$a8dQ_4yr$SN$$`%8+Y^%I0U%b|nq~6`IvgVVT2W*v(Gi?S7td zRz8{^SK#t-c{Mz(!>dK#Cr0l{qKoU2qUF(ikC9{^DUkQuve_lon$mz;)vM}n@kp!Q zI|!1B`zIpNx`S9{BGs9+2Jc(paMl_ zIY7*)SRiC%8)|-~*(07Z;%;ImOhjo{mJ+yr6hnom-WR?{iGKa*3w2J^! zpqYWc!$f7hB(RWPddY+^p5HzlV&Xq$%c&J(;zt@S$b5e+JiLojKUGh8mRC|-x$3mx z6>MpVn1E36dc0ztMp<|5z574u2tV1513pt=8zb^TIgK2iekvFsayZd0CElSRCus2&alG%? zrprj#N~h9sxlKg9ud$PxG0kIfoEbLUvtVzk88U6F(p$INk`2$=T`^4L0rC;(LG z$|*c*3$YsCqA~QM{>rvD)MV+zJT9HVnDYwLdu6G^3 zsBw8GWF#-sI=u0edi#4nggP`V6o^B0UnBauJrRe5QZMYSz#HP#_T7wg2`DviDXPiD z_M~l6wt+&{@5z`->`tPVmN&pV&=;{N$L-6XFj!=2A*8Zak6_X>Se}gT&f`yI>aH4Gyi2$xAS0oa)@H$045x4y?t;>2l$osHDR0?_f z`-+2Z-JMhW`e#mdmN*Yp)Z~ifoe8PWO1wRSVh;aM5*J;S4s>jz!6?1a0RTc)TXkpB z{I00VI|Nhtd-|uOy^Yih@f0PieNEKh(k?(Xmp%hH#URnb{6`~kxb`dbEx+^<$H(PG zvgg&-drA_81huVaU4n)nsuh*&dZ! zV0LQ6^y$IzilVjrGyovD`kGNCC>FTk0c5xwbzZf&jbD)6%(=8PsF`;_r&$FOWK$KN zZuqF69=bMwMe!QV@P#Y~SZ(;kOsBbWhMq6%RumN-HQ&;E_f8YwJ-YZ#O8?DbZLJ)s zHHuEBPX%g{&efk^VkfDHa*lrSyds$+JR$Y}H6hpna!n8E!5n zvJ1bXfDLYX_qfSP?u%D`8&g92tq$+B0JIH&8W9=0RG#A9mpp^IR-8*4o)0?CI~U%T zu&#CT)!iIv>kubC{IF@no{tm7tn4Hl6bDQ56O$Tq6ViLB<3v6sxsqo^@U@L2QW72D zeJ}hAh5gtu$J|FpgU7a-!wShS7u^G&v{7193iRy4Vpll*To)87oHPfMW0z6tG8bp$ zjZ@p(zA7eT*1k^m_a^lQnf0EqdvG}+P4R0!%iFpa9=2%sM#bs&jj0OtN^mqaaaDnk85D5!^mJf6JRImR_O>G zZ2~N5A8{)Rs^W}KOuc`cTadYqRbGGMASRJ-feM-sbuU(mxng_;i@72GuH}bKTaeA< z_tZD9_twf)RX$(U@yT7}ZBP~d*~zt|7d6%k3zrJWQquhN7~;r?Fu==oipLcc1fIw@ zKomdQaZvT^40S5B)m$GcD!%4&4x;*%$DO&Xu8F9rnp}R553g6k2HO{WU8k2quj>13 zrIp+_HhE+;?{^vyZ4U3qeG&IKbbe8(r;oQw#sBFQ^q1`CrcOJqh2mJxqRT=6{xglp z?6!8Pp|{;eS{a^H`r{!+pgC;8)t% zoXhQ<@5_H~GRQ5C=M#cGmq5yH49jCr(eV?;vC3cE+3M&~2A?}UCjygyz@+`zo%n+5 z1?HZlR}(a0fG>_U*{k!2l3S5?$~G^{%t1-K@{Ae{LxNF+$5zq4U9Q|aqNB+e%KTPZ zo}bllPj&t!r8g&vKci7@V}yj#KI>v>RTgBjm8axgx}#H|ix!peWyY||LHI0R!_l3n zr9`QTod<@1PFI0;g|P(8WZUiAvc_{-jA+>AQVeJqSm`oV1dG*2Nb-4BS86|B0+qiz zlw>wWr{j5}qUG@zcBHwUWwm6??ze;^uHKPDNSEE7;wa|_c}Tl6YOv|iT6!AeuKqyj zK?ZGO&{Y-dbhtagvfgW=p6fWsqX1*NgcF73ILtx;OF0{QB6dZRAq>nin*(_5L||8? zA{jpX67@WG5L~z)QT$t2p9B2YbV}%LDSPBNpLxcxNxce+Otab|x ztGIQhj&2HDTMGfiq>fNcgTtbC6RR-W^D%V+&$D7@v4PIb-mYD`faN*CA05?_@qmAI z%z`Ve1J`UG ze}co(RLre_CGyxn5Zk+(J4SNeZG=`O$>?)2E#1W|b++ksKI}F{SJH=7)r+6L?f0@I?Yxaw};Jy}=6($y&c;|iuZf+AJ)Y2op@ zz`HGv~w&27|1s7W}37fv14>Gx)C&G6vtKf6Z{>;f?wi~-)+0Lf9$EmUjSwT?d*x$f&%WHZ^bp<7BKM- zI@p+d?UrO%cX9(;wHat6g$jG09`C#r6I)#eeL ze-5x+yhm*;%7OeCi`+RQ(6qSpZ;#~(6s)lj)Mtr68m%=`g2Y7e!mmq=l?Qb}tEAFy zgiU`h(YrsG9P!29M+!SRtzUVJ@YwXUS{ZbxA4}bw!=F%V=>Y8x&&iQigS)2I9lKw{ z11|`vIl7V+KYSKX-oofz!FrnB$Gt2)VB2PvytOsx`Yo8Z>BL=x8EC5!VohZKz%$K{ zr_QUQt~&qe+6<9LqN_@0ip0-4j}H1h?`wK!GW2D!Df;-{Xe9p3E~r-fSh6jQt(;P> zjF~~2L8jRzkZGODs02I^CrtLw9l77D1fpLi>k!4zE}qk(eDW}_rggh2EZe*)2Gmpl z0c{sU`d9PjD|rb1=JboFCpLS9Yn=P69Kh06%=VT-&^i0-W@a=3ZK&Sgh9`lzu8;NX zG!iAPCi{ox+exFDDyQzMgx9T)B_yhNxcH2=4d4VN!|)|mvOXJ~L?<`E1igM<(cDqR(Za9=Y2l;77M5NG7Dmet z8I;_FN-h1tSYQ`qXd33NTwHvU+DNaAGE%sp0{XiC+BA2{Sqa- zzp|dazrV8n5tWGDVG6VGjBcygU~;i`I>VS+nm=|14_k&-1%H-O9AhG0-rEU-E!qkv z1Mx55Xvn#`Z{s}_zLi)(IuVbN!&^aPgspTu4zrbxv2D&bboO54U%XvxO>G>U?YOtM zQ*5x?3r_<(MgVkaZ0Qg7ov=hr?E71v%$5B;*JSVN;OqaKhVuVQz_EIH<2bi4c>w`D zzx(@vZ$Q}eXB6(!&nO(NiU%Y=X1RBFjvpOQ{@|cwED%z2p9v73-P&98xV>G*En$m-j@ z$lBgu+-Unh5S#v&py0n1uKl;tVE%gp^nb@K=6q#U6vm*R23pPG9^`E zKdRr96b-!pm%!yamgrst8*=M1AE6t_CKGq8T`nR~}e-b+Ve||!1vcx&) zANrXT2Hf;tqtX8}@*MxWG8+Exkl+7l#s98^pnpsCdg`KcMIF#QaialCJkjQ5oB8mR zpSdsl_9L+;KK}fpf6f8?(953rk2DMAVd$YKTGKf)kO26fCU*RPe_~bLxwD-AU!{Wn zkBR>`b7udiqyDv{PA9GGul)frp8i^k*y-?Q)ADT@F)dR-Sf$)%T7@6JNI0a)_Y^e5 zT$?MP5_oRt6)m7?KoJxK>?tuLt~7Oriyx0!If&R^fZ%b66_}-RJw%vQh-J^oAeTYo z=VFW{%z{=VK@=`wpvw>o@CLg1*Z=R!yhys1nBETi!)p?OkLjZLxzo zqTPmq?`BvBXM2zLwFqywNyEIRlSt2t7E<85rx1Zt1NM|SEi3q(IAWp}xrAVCU}`=z zE>18Sq?zA|m>&P<4QXLpFtRvy>p^FGH=G`pprP-M-|Di4kR2*pajyImyV^vc*z>@S zY5l`eLHHIW1~%JMopXjv7q(`St5Szh$*8Vn2Z4Q#w|zTg8w+i#JzgQwCaLUmDQRSN z?$vff??IKY+HT9?iJ>#v7Pb1L^khFak=%AwLT?qe7N1D2o1QzziM=|a|D1doZC#VY z+UnB|7`=MlEpRu|dOJ^&Ns_SG`Jklg(YBzqH?YeNthq)g$*tdiY7KKLj8-pzVSWRH z_^#iE&-lYh)0v#vp4&U{-Eokzr&~eh7AKO?gO+YA)mvo`A(eX-o_e|f7~b-g^lnUX zaAkK($kG_40-2^~oz}baJrPs6lx}SbRvu#>tR6r7gfiEUVniaCy9p(wT7&NF9DOYO zKF-B|vG1)gbYUb+7(O?cYg-RQ4YH>S_=#{LLmvOVchbslDoltmPFC^oST0W32e^^Guv%fY-8r!Sv$n-QfI;%(zL52eZqSCbvL$0Y3nn61xsI`4egZH zeLD_fg$uLm9Te3MD>%b;xY$0{uaP4+>g9XI--suJkbK}eCWO7#Ms(DDr`ln?X^5qw zaHL-?yII|!8u-wU`Pbk2XmC2VevoTX+<1517QvX{JRXGePL~?AMo?ki13>fOYdew- zga;NErNJdVZu$;C8nEnML@;?^Rvf!Rf^UWqUC6G#R1esQ(&>FA^PrJI#9=!b<{S4u z=CKXL+3GjHl=*}z`Ky$o;Nwjq{I{hP9=m1xlM(@qV}C)pCo_HA0_oweg==#9t~kCM zDs^9qrhNFkSRr3reZ?SSvIH0B1m5uV+2Rfp$LfuFSd%_%fTGgUEK8~5=NeL8a^O?U z>u5cNN>!_)@C`?>gTqj7NnH4ec4=wkXE9`czIjbhm)wv&XD-88IpNmN6l#`Y;BzFloptNMWQo&fmWCO9GUkTkc~?sg6jU=&k%IH+RJVpaljZ;D zhS!vez^HU%WL4phOX*wA(z-Vb#@8cuE(#=7XMQfroPR__397D7ff3_Gzdg#9M3>!p z^pNFltVfOJ8LmjRc|^56OmioG>(m_sVzib}a4>P+CEgAw- z+A+HftSTl?=BAsb&6gz;=b=*k&{o~rfGxnj-6cq7XZRQ2PaM9#p%c4=5u38Ktmnfd z%9n0ukl0Mh>u@e&?{c)wmn#1!l}f)PX5buHi`_yNj67ir-`|PNY2)lDcXE1l_0EW> z-|j}OxztYm3zL3-;q7I19XsjusRNIoD`D17Ab=^VJDBkSzGLyWq>gzC#HPd$jAUke z7|E5(!jP1Dk4neE#n2T@2^x9?C_08l=M7B{#Z?OY-WwGA7nn1-NR;a|!)&KJ z9V+u24QAsLtj@c^SO`^GJP_o&7-YFJ=4L8qJvwcpRKpH;U71b*c!K9D5*;f$cEn-x zmjRc55(RbQu6*#1EPkoY-Pff!J|OyufH*Cag6t+$+^xZZ6tWFpF|&cwZF`FtjR^TN zHHX-4SQN+DG_^B{0H9%eahRN1)+|o0wrv;qBzN+Y-2oH#2P$Ybk;I}<@aNum1DyXb zhA^B~p#-AHkTHH1zRc|>q+V^l4-qs!KW^N*Q*CC85|C4&D)!`h0OkD>9 zamF!4S8&Y>>oUpM>d7DAACi*%!tAMqW^iSm4`KPP#i;{S?{!BFul!P zNK0ARiH5$?Q15IEvW8mqe$y)jT5`F#LjNV;nwV6m#GtCYFKLW=)~u&g3qRZ#EHnADAg< zyP0`5s<^|qyRjaOWPPGJBww#_Mg)6u6CK)=u3c_u?yap-NS%G@E|YBcBP0!mwfaDI8& zh+|3ZirQ3BfCc51c;VXMHA*8cE}ip6{_rZCj$*iPFX>yoaWQHu39AxFcCjN>YE=qh zW>dEf${iOvmdA*Vb0uRWr~|vv;k%1b11Bwmd!&h`TGAzRf_oPV5rsRd7v7?1?&kt=B7g99P+JbBYZE8nZBTKkuuCTOX~-(eqNfy%zKzcJPN!5wS(sLCbbb*OtE@%DZN z8adZMr(N(Lk;cT*xs9q{vZ(!!D+-`P?#yW<(ZGMPb=;$k)rR$A6J4*>HrZbgP_4Dq z?sW0DDdBe;t99MIAAOrp=Q$Z4KbVqKGAAiX;UadWm{?O$=;MGfAe{B4Ts!6+56n1S zKVBww*ADVu(k8M{h=8GpSeePxT|1G%&Jt6ATY*4dBdOBJx_Ro#mH-jE@|X3+SB5du z^_KOCtI~`dNn}erP&4Jdn&XWes5PS=OWe*OR#OgKHW5TBuYWhSR!za~Ovnqf=9=@I z<@_-lKiJ)bv^F{4*{EqyT6^DxBgt#;zlfgwWh6eQ%;+@Z4Be<4zxb}RYIL;A*c{;K ziE?7h5A;S48k;*?3Q_WI#hfl~3g!$r`u3;^@1dsiSx3r5j)liwa7yrl!T*Sb#HGCF zF5RNSHe=4mLdXiVaqn&wnm5krz3M_((%JCya7CZB$wgP8v{r(((uK&D+dpgkz++yASHndidfYq?Jn z+`1h<)HryzSI*ZwHa9}vV8#9Z;q+1OKY!u4$}s>8JAg!&xdD+;3?bgT9XVvnEqNvTi|#H*%qSHcX-VT2y(C zg=2bq-OL@3v$?^2r8iYdB2}Dnr`@rCefj*>!`;x~R*2%uo6=HOP5Z%+AMS0}&-euo zx^<`;pw|4X5#{>Pb?Vb)bD9C@44KqdU-)~)zDl}wom}|qfp(6JDJFHT=WseL`ztTE z|7qCfQ-Us7KFh$T{Owohqncn7bxJns6Abp}&5S>fwW0JB0?aS%wdnfw{%(1(R5B^7 z(EO_S0q1IP4_=~XLR8C#_}AK(w`a|LU-><}^nCU7=BD%Sk#t9kt@8t3%YCpbtK&7p^Ww*NR$r0I_}Ml`8_&mHzw7hubCFByuHkCkXfefezt9LJ5u%(e3s7v;pIp5>E zo2?b}lz@u@$iOrrO*S)HI@dqA{n<4;dbd=k#EAQvR+T=@=7vJu{LH(7uYZVgo>~gM z5(O??GJ`;#Mf7%EIvLgY#OuK8>A7stwo;WMmXV}ukt8~}*s$Q?^~YJ9U#GUchglzv z+&}VxW%bNlsaPCYBod1MS8OBfSmlH9AUJE2cd0fv4<6X|jL8!XJ)u)7=T*3dM8pFD#=|$~>ekywlZ0Az2P40#)O*Q= zw{Gx+r>e+bzIHrCF>ob4MxTpvGhi|IlrToFwm4gtjErc)=6KYd6)T{@6OXD6250k9XsyyB(#rX?-~HL6Iip5j zE}x2xwRO!NI8WkIUi~t&!q=T>vGD%75oqD}@YZKx8dsC}j&)jASrwM>9d4+M~Ko1cEk==D5;I;Eae*?GkoCoWu)ws5o0oEbBv9d)TxE$Y2~E247u;;%}Bh)fiw zAlJ%=%W&Q4*wx6cUgm-?p&{%6-$-`F+tvV)d5a+wns2;xp0~-&0@2o6Pb9a0ZoM`IQRPR?9PQG))$hPu zu~9hMhE)5!>?|QAmhV_2xf*x8gX~yw*dVvZR3|~$;qJHP5beHO&3U7vF^hhIPbYUe zT5+l^!`;6pHQ5BA?%ZroayGXA`b1cj(-{=Zn&6s?D_{B*fz;j=xcKJbT3J?`wZ5!2 z@@}t%cTQbxzNy$r?dEA*$DfP2aB=MW#pdg$@z31$3Hz(4?l1Z8>EwYzp zt*=F&*4MSRWl>&H<)YGP(*0cPo?_j#yHb*G!l=X%$G$x$~Ra z;%e)tHW=v}0kM0aa;f*M1iNSWTWKO?XSgF z=Ur)qL4Lx$SXa z_p)u=)TOriihTE3^2B4ePiGOln61}fLz7|Y`P%N4F@sU(U3n#2D!Q|#y6bp_FcJFM ze)-z7a^21bCKF$g+&N?p(Q)AX;CSq}w3c;J%hepw?wJz;IkQL9UKbB)ic}O&S>>c} zFT8Ge$jGk`lS*Yci8vU%{;u{-v_?(Ssdl5aCnPEulvc2il88(Kn7hx5M~% z0_d-P#T4Tgz9y>kAEp|M@;y!j4^~CEwHv){J$5Wg4|h+$QFc147CHEF==M?c*Q0Xb zlg->mpf0PaGf9T76^&{6M|A>c7AM-`+c?4RHt_MM69=`kAH^a3`T`MKb791u zsT`DubUwNH?U`4Oihtm;yXN5RW8EpM{9swcec7vV%}>1#G)6^;iy8y(k$;P(J9|AMJqhd&1)~8AbwiorN3_tXrR31c5)rG^JwbSejd8 z<#$(rbjx7}r-(Y3eZ}=VTlTesA~s?@#=6ZnAhTgM4AT)#1FDt8&+oG8(cP7{S;Xq= zu*KZNuY&vTEkzuSKJyWLgnZ&kl$zg?f4@F6;_wGz2ld!G!$3+5{ zcCdCkhQF(g5*EpR%k9OBO*5em*TrFsIHM`zr8g&>StW)F`q4wSY7T+XIcmN@8E6P? zR?c*WR6g%u)jjg@&+#a70$1b5M=c;C+{rw@A{A>$X++%=exiC9$D(g1$WGC?nx_`0 zvI0fU$jKkUpUL@>MY2PhqPN>`C@tRI>nP;b89CE|8_$U4pcxm-xH+wJUH6ZwjZ)-E zCn6OaWqgPH$GG!+t81Q+!x-XMwrB5%Wl*ayRdykPES^YhFOc%X3pWZW%6ZZ-$Qa&5 z^r=1a&W#PwzqEZa}tlGQ9zhQ?p%3-sou`}hf zKxHERQqqM+b*=+nU}GB{ZZ+pkP0265{}8?V+tM*;vk*=q@~>ryPS9rkAE7bXy@h9P zm8xA`{d`R_Iyh)>^j(r3=Aht*k7;+Hl}-&VWfJvfBH+TAvJ!k@ONh5|UFG%g>EW$- zNs&B#IPs|!X>ss@Ch}3QxJ!Z5Ob*f4A=PeomQT}BOYI{;7E$} z6{{6}=u%(OG%si9dn=@IJH%)`y(G8LjRg zr2Q+bN^}BN<9*v*9Qx*hZuS&rKd6tsdR;jJD>>zXYR8L zS$X_lC~t#uGOH?feU@@M-feb^jHb8giWYW)Ja@g2I|{<$_7$`9y+3A(*o@`6KjaTB z&u0(eQt$ZRcd=5{J*0HOY13QVaCueAKcm4mA1$4x`bqS?t$fD3YkhFZrkCBiH?rue z*xA}f{8WElr&*WHpWLGL3tk2W%YA$ZZJyF}-^&jMzrD}iZg?3h)I`nZcaczx1z)=z z<1kPde^904Wjg==0s}$(zJl_9@RzQCE-RFv9W~Tic<^YMSdIOeN+!O3?$GtJ*yV3L z?cS!Bta#_))X;59{G5!|*Ywq1sKz{XP=Yd|f?`+^th3$5lcH^?sOrz+aos-5LaS8a zo6~Y>->?=DJ)Bcnaf=rw4hoE|S_)x`HF8nCWVBUH4vXhIJKn%EEIx=*glX%m6n!eK zX`qR!?c~nA`$}(`KMZK^srv7!`nx3!b^n`xY;XG7?e9ICNRS+@_~PLYcNPfD?=(8C zT-GO-^$DAb^9r)w+xDS74<}agwzuKC9iJYl>N5NIPM`2jpRfs&*6H(?t=zEA%2O^H zmdl5gk}Elzc2^L07!7w69{zAoq0sz!L#OKRR`qvF8mfwO4=1eMw*BQKf=&mG;!57G zoC9^{Ibq7lic`l^w|=~3O*|7hKqr^0M}`!aP8>bghOOAuN|~s>R8(K?le~8Q>IBE` zJ8q~&)k>vKH*%}`_|d{91?G0B91u4Rh?}Y|J-&9s@@Rs$vlG?kv2LWdI{);UtG#An z@oMEjPuoCG+x7B)1ZOQtyK@+ByO@udVNa}HnHI&u ze{mAq8!j|Wjj126y=N=ur1klSPuKRC7X?L+SkWWysJ~oxhPPnDD}+aGMB_y8S>*#` zNl^C+rOFf*rh3zAOoB$_qFctZM+=U_bRp~Hg0s12%5PY7dFjFKB`uU+xV+67ia zd|dpwElxkb*k9YQA{MQE?ujt_<4W~P>5zJ4Z){t z+t7U|hmPj0vb+5Bk?%W9N2FGWn<~XkwN1~y^5O!n$B3lt#ECOargZ9+1ER(OQRCH0 zEB8u=sse(i1Xu4qZpdWk$c2T+zcG9CQ1slWJ#(YBIAZCB^+eA^uH5ByipK9xA1@bK z+!hAJiUDz(pr-U(fVK9F)iems9K}RmA)mSHS7D;BT)z)GU00}YmE|7%4pUH(UaeG! zo63$fRbP5~{c4`=giTQW3;+NRrs09YgXLde9=1Kd&%NIMq9gr9dCWx{GG3cIPW+HU z(6sOCYt_9!+xU)dp#SU_7fSva`Qe`u6Rpq2G~C&;`?IUn1IQ-r&C9KyUA}o~^}_f5 z;N$wdRCe#Q`7f7mRJ{0qwk~$VPTgh#PAJlfYpTp+I4Jy#D14L&J#NF>E9*DqnZBt~ zSnDk$pplk;yLWqTosAdjDq&sz&bs_>W0$|TW$U~k8(}@l&f?vle04#157}~0Qkj=i zS$uwD&hC|QxM8Ilt-GTo+|F=-s6(O}9C12%yUf5|Pi{^xj-qh`>` z1f@r|mF9l2@TJ}FJsV^F6@{|%LeBg9D!K^&sO!IS^0oHPtzYJ>3G<};F{$nkz_LM6 z{^_>OX9;Eq(+VqgY*J_s?OJ0To~@Dob5G7TqA#Q2n-@NLy;->Z*xRrvlpGAr%=zd( z6lw5)`rTw70Jy)%Bo;1?YM?y#Oyw;r9qJzb3trul^TD4+|9gAwY{v#`>J)Xk`#%3p zAhXp?scQZH(4NzRK_@0&)%BKrbKs1{YcM>$?CR++PYZ?|*si)W|ImKRu069ai7O9& zQ`9+vKO*_Ho${UpjcONb5cJB)iy|PDA zlYh=?^i|2xJ>wnN*s*&Ux_Ut{^X=BIAUSVmzwHHW3jrpG+vV47lrxcyrPjE?&2+!>~fO`dk`%zQ= zmnW*E?l`BI)K=`?Q!Lgv_hW5e4Q>%YN_1u_*V8)`JNq|6+O~gQyzS&3Y@vAj;oG3v zrhOlO^-sb!FO~D;+q?3Qe`WY@?~ZJOE)tcnu6gv=6n<)y!!z6(4~$Cp#Qj#QHHC_) z8uxA5vfDEH%ABsR^2qu(4qq8G`+T_hbk3T03Iw+AG}@cRpS-?)Pwk*H`K}cgZeQ_= zMf4S6Qp4u2w;Fwg5W3eZY|G#Cm6hl#ZAZcO^`BTqUzxd6pZo5HJ+`;!p|HE5#kfM-SnmuW)}Ks5yG*RHFk?7VfD*CEv+#KY%;cGJRdq{< zzqyadTJg`}s zpUrnL`U*&oP~Ip$yx;nx zN87nQ-&kA$F5TsoE&1OZG>^VA&##`#M?cTKW^X@_uD2}rbd4!@X`#=qT@h)#XSn4) z=v-Z)>n=OG=erIoXH_ly{+q)k66@!D68m`o0O0<+Bl-G+e|#}su#U++MbY0by!YLJ z_09zS?Mvlve^6&T`pUFDVe5OZ{iEO(VZVW1da3-CKaCp*W%SdJr!Ktxwb_jAj&gLT z>gpq|@6>(Ve?k#LNF%MU5t)A&nwnrS;FA%*cK4?%6O28R+P3qBZG*@TD$&XJw`|TG zucK*{yea>kZCeXO*6!wm)nBcDe)o9MSEfv>Kk(L?o#pr*TJ8M|mvDi$qww%ag5Vf3 zg<^Jg6m`e9t^dGg^p%OM>cG1%>@4pkx-ylW7dF4Kd%RF26IDar=I6H^ZLnF$6vF=% znYRf2;#0ZUZcokaGu9(Bj7MI)bNjxAVZ`F_Sjb)6kYue}L;6OG=V0T(Zz~ASr^UH< z$`pSt6KO;zKiO_2`pTfIB+3TPH|;xMd?BT{a==RT zl_4J_dE0Xeto7|{drS7duxZ@rD@>Nu2Vees*%R|NvFerp(kk$l{kKE?IZpMCb_IYGC*(EucmfM8yLKbwIYPeNn*1O)g8 z1aJdcaVUBBr88%5;QO&?%;2!7q{OI59us?T@~h{HTJ3$4G9oNKRo;vHh(u@e!V~5s z#m?}@CAz2b46ff+*?R6AzAK!>;)N&7NlK0n4-KGWe=56tuBg@2zozT1I#b?-OANz5 zGW;Jd*Q7QYDl{J+&p#l*2R~sMiN*F02=EW!1#>Kx3Y0e$7v3Q5 z5gfTyb*e~Y@i!8U$z%|$v2*=QzV%o-S@oGx+r6}^`VvFZNb~@ne*l|_PXyh^kK^yh z;qm+p(;*NS175_eh zb&wcre(ani^BPRwWqwKOdTWZRY?daZ6nD7e)fGOQDi3cj5IGa5qmfpXHe%YUAbLg zzhznwg~j%>n1MiY_N#-{xU}?^9^PGQak+T1{esyR@nGG>pC2xEA7=BoZZZ5_FD}aeLGqK8LsSb(i<=z0fClT& z9Vs<6<4FMvUVJXeG-}0_76q2|MvFV2BP=*sY#gL)ICrkec0hke$>pw*iTSxL4n4n& za$9uBS-}s>r7s^I-muL9fd6yp%mYun5@gkWES>BL;+O82+ z%9ihQ&tOhBYtE_{7x)|NQFq-3ov15x^6Q2F5ZTV%p0+M)YipiG|G%p3;3r?zJG$ge2MR6@*q?Q&3D807Nl6 z`_AETxt@a;^N!Zo5w6rqCV<3-4NSRVF!HHGJe!w4Z|L&5I6o8xWvUe)ouDxx0q!RoIzd{8KH z^tAL>Er1SAK}UHk4_G}M^8fLZ4dy#$5d3 z4nrNqm{%Q$6rB0$`s_%?FY3DpAu4U^$gLj^9w;)rC{@&zcVvbpjX#5zPSIFWi)o?L zB1e}$FzvVfe-0cqc2@RWtrTW(<3>j<4|UaHI%H`PtKNMTw+78nUzsbE=bG zJG}RV@#TtC+$+~{U0Ret+1uJ}$|_pK(l@Y6Q4F=8+Iz^jbdFkhO)O9Hw%wR>zQ7Y= zMrJHXj$-rKo$iD1CQBhkP#EO4g~A zhAbg5d_rdM16clXED}Px?yEo4VII|3@vAnhN-`}3sZ-RQK5)bkV5+}SRw`l zm1VdJmKnQnK}sZ_7r-RzRPvtovix&3;`=xczZ?M`ve66g1eYN}E$4K7Hx6v&K_Tj(+R#u~lX>foVGn3#zbvN5Pr9_HTL85RUd_ z_TnY++t=?jMri3eOAG5)Mr2r<1z2uv*>y#EZXRwFhiB~a_g;D-CBmPM5JH;4#)BW^ z9BVLUMfdr9Qen7rnl9T}Im2tl-+ghVQn%eWc#FeR*5CaCnKI zLkkfi5_?v5W>P56Kahds(%xG)YKx07^APmZ7PqCpmSD#=&1%uGtPRi231dzj`?tfj zD2W}Lp7r?bP#%-4l!;rb3;%(cVMs4&606cVhNE)#&2D@;rBIA{x5}Kg`uWVq{2v{= z|LSCQxrdGz;{p}!^*!n+FI&rd-N)Ojl?2J#YcSDsq!~+dHeuE^N=VIm>WSh5d(VyC zJoKVVS0zb{qKs*p%=U&0O;a;g+k3WhuKVf__1NrYgfD$=>4Hc$86l)o-l{J?b=nkA zD6g+<%nVPr+FqfD#QKxk#@=M>dqmwM>Ba^vc6KWR#jPGp<6%V6*<-9Qd_s9+j};@g zstOyheZM5$<4adOl@v18GEhq!tIp;ZnPLz0y;UV0sV~I&5tUiB(pXMtEM9PE0Gku) zZzGdNx+`-}V?r(%(W}>IVVm5pv?%bbHd@Agjwtzh|Jj>kF=w`)BZzXuiDKg{TJYSb z(i0;2fegJ|EUGRqF2_tFB^L^$BYYc6OB;XgLs2NYB=uJ=Irei~?0bGe2svKc2+J#P zC1W4c?8S;m&wPAV7?17wQ*UpdRNP!weAV<&hQAo^LpgL7EqLzJPt1+vvyee{e&5h%8EWaQX2<7R4hRk)MD!B9y!Q593b+xN^A2(*QlUd2jSh{o$KafL5 z%7JdNsQ%IqmrM_9s-}{v?uhKMEBEmuDpg(hx$r zq2AW=bH`0#i|W?$DrrJ?pshBEPFj)w%p?vDL7>39g?30N_C~(v)}Q??Oj|bw+9HbFUOeJ+&+Z z*QC(O+V_2Z!?EZqgb+eH;kAR819m>iG7ZPm4ix5gjQd`g_}DJAv_GB}>qI};r0mpf z+Y7oOa|3b#g>Mtc6++TIx~(CS1`lxcpxo|<8WTuv0SBd^+byf*dYG@@QiMayN2 zIr@DgGD0yI)L`S$<8^ihkZBtV^Mz?yTqd$7cxXKCNwUVBA$?k;MOsB&X&do5rFyy{ z{)!b8Lzfnxy<%tJnYJPKi_)}x*-kHNG5#K03cnhx-Ti5vqj8xCAysAW!HaRbGdb7~ zm4f19c2`-QMFn-)E5m%r#q#?wZev(nSzt$`(arLUw_gZlR-C>dv1hCOSMqDtrkb7H z!7ICW7TWc5f(=&e*pnFZX_h~BixYH>YozBt-*v&E=&P}VEZLnqKgQOkXa0W?}%oE7rq5A%kCdL!dTf?TA{3yj*#+N`S^oU-#UiIP5{o zMcDTTh%xKMx7H?N@{sJsO9@$6!}VUeo`nwVDz!33d_>u;H2s{xi{{}=KbbRgK`gV$kTZI< zwZI>wGkt&)%E5X)vyx$_0y(pbS53!r39&7tEcBy&611)3ai04;o&;=!axOI%F$$jS;@|xfQqyO4r+3k9pC~n6+X#=Jg3e zh|ElUdQn}QF+xt!+SEG|I$G@2-Ay9Q;{+{s{YFe|2h+nVbv1`JvipRi6Kt5H20pnd zRWAH9ZiIxk{fj+$G5cRibRg4??YSmoXIMS1l_dB}5m>cY(B%Q8~KnAV=bH2rN=V-v5pM=DtJQOrcBMhzxE-MskG5%Ws{%jmf$~W7ZA#Nw7n) zrlmf!a!I@?0Ep_BwEpFc;x|m8Zz}n)gO7Czu>?C~@08SaYoEkT21W^Aw03!6ttsxU zzpbu+IK$g^2j7&ig)7%AN=l9Ads7iYNYmGNp#hsq{>wkj$w|Sa#XD}pM_a_t?=X&+ z(Kb}n^=I)$%a~T$*@tLE;O*$L^h{U0^`o6L^OObKWzGQ8_l%jIf>tiDCQ zueVog2~zgTOZ0drQYGrRQ@8frgx6uUg1PsVf0Q?Plf8*rgTLBz7rf|TPvN6enf z{pZaNSDqsrD>lxd@mFnmCEZZEJR+0+5bE!>`!i!p5-AmO9SX7I=Nk1d9^rA*qh>hp zb6XsHesQ?AQTFyIbzvS3o{`v9Z>=AT9MVv{xeV{HS@ArNy`LMSSoD&5p(2Gd8Y_7p z%As?7GG2Yv5Rbx*%Us16F{|F0bJxGJdTbp!Zd~Sb476|8_l7U%RZ^wi(8_nJDiGHS z<+HpwWaRqY$L*QyAa=(3H{wIeYh>F(#^~qg9?2I9k#y?(&I&n0mwJ zU0x?|ug7)~1f;+5>QZAyk-hxmmOi&y_Sq5RoCs2pP_7HKUVDt(syu!gGq>@ysMRay zV6)E6_d$oMnh{xJ8zxeX#9y)T71K?Sp-oQmr3LI^SE9=0|3ASK`ZOZ zD<_-^mohVB_8*@NnCVYLcNOgd;n{*4O_uBN>HhWif}Z(*;$j{nMw$wms_znPaaz=j zKW2p{Mte~P2I{VMoW0OvvC{OOqxRbowSw8L=#ld@BHnSc{&plGgXGk+-_9ES^zam8Tmu{@$VoIIz`U$$)d z!nhc&r%EEacC|R~RB4OZBuOm=CmXVM&odpgQdoYnl3)TAdi=7jYcpd)Iev7c>KD}& z7JQLcWj+H-e`9frB4c!GRQ}?ZS90YDX%)?-B^Cn?rX?)O2*U&n`bRt6ndu7cprEaf z_)ioH-EepsX~pRaM4>0sq8BY&lbx0r%0VhoM?>QU{J>O%5E`yKR?wF9wy~8qEN*DC zXhcm(Si5cO%A{~_V}=ZOl;m#zyud8q*=O|V`7xNlhu(%l ztINHyX3k$YKO>$)QAwL?O7cr@YEgGp$?fIwd^+lOd07beB$>)hUBA||7a^n)6`!?O zYSVLOR(2ewKurmc8mAvsqY~9yTS|{I#SLo^`cC&()gBui{_UH%d|7G&mx<)Jt1gxl z*K`y0c;BlkC>JkW6+S_gAe9-jXzdG8_cTZ=ZzwKqF;BZ+diH#-sZyf)#qux_y&a`_ zgdsSr`0Vv7GgG7ZEF_h*w3Qd;9W!60PF9>({=z(Lx`vwz3$4X+Q9@IfEKH5_dqgAe zZM^u83k`RWys@}*<&sb)YH=dW)G->BwC>_qSRN%fHZ?kkDsQW*xoLS7^a*dv`)?N; zMqk_hGD7#7YuZ$}04!}sQ6BLeV8My2*DYHTAK^#W$|Y^(g$2jX63o4)sw_C!lC~}0 zbjl{^8dtUy94#f9RVz4k^{VV8$q_udRxE6*zgT!Q&-}vdD=a8q7Po@yjwL<8uLlHR z{U&+MT)p*AyzkFo=Ptt2%f;7A#Mes@Eg&`}`{^vlJKPy@FK$2dqV>O%Y2F?rgvNUH zbZWID@g-_$TbHRTi^iXuU^%l6nH3q#D(xMs>D0YKsWOE%PT0Umce~V-BlO6}!!2ho z@LjSeJP<-;66U-5){-9@Lzl*3U%rz4vp(3DMVJ?4%-3YKMK!Z< zV>%J?id(<$po3??A^0xlQTTa=P(a2LaRjdt$$@j8h$}aBWXpu@avh)WB?2uXYiX=m zPdzO#Hk@_2+V~=TNUgK+5*N?C#eg~Y=_kx`fXw12@JK?#Y)059sF9uy%uFbx(8n#` z($PXP7A0RjR6`hNLndQ;R+UXZlnH~M3B?{5Eo{}Fzg&73VQ2hu0XZ=J-Tmn{zTm-N zcpzjrSly6ryhu3b3%ogF!N82(uvIX;lVANR+1B+7xyYhi@i=E6l0#?nX?u|nD;l3S<%@GgImAra&Z1Qtig`RBZs9YndbfWj>w8<7Z;h@^nDjAn%Bmq zxjS5+5xwfzY5u;QyH44j8+Q0+)!DDB&VEgg%6#+fmsUg*^yLh8HjRy?qXZ>{j`M^m z7Ze`c-(=SE)vBaoVPk!Z*^!VF(vx^pgfzG78g66%5VkBk!7@XbE(cp&G3U7qaE)=V#HGMP>3>o(LI} zIV@M%jPhb7Zv8T6O&D%FDq&p*w){oRUmkCMQmB4$S@F{2UKV=W`-pZhDx-_3IUy63ebNg@AX9q$9Jv#(M>7R*P!X zcg?=wKrPCc^~_T<&5p}!Ml_Fh-8jpFTCLQf(VN;twRvpu;Qry`eSg4wV3uAogXu*^ zhF7zCoz6yx?ft#IT0O_MQzoxdeMdWW+v^ zU_OlrnH4%Kgd{RVkEw^0HVOkGv-qJLBp!Rut+*vt=tBL;2%!p>Jdo|CxyX_Hq7Iq5DGCp{*36XIb4GDd4*?X1lNCW=z#f2+T8ck zTrTlJvgWLRW9`iGhlS{5)p@5e16pX2E7v{~i96{owJ0nRHCpU>j$lOpao6WAv?m=b zHqPN?E=@5@8rdf_!XNb*;$3WpCb;->8ytRqaiX?yc$ft`{*fO}cpZadRd+oR!hp85XfzWdGR5`GsQRq~kJSk18vWIeH{!cX{)lTEEMWNJaRJ zO0mc&MrZ<;)?^4V#Kp;3=9}Qm(IQji8!0&Z|1OHhFYN5z`=CR0g-+Qk#@z(dEpCFy zD>Q3Sgdn$#uwQNB>t_G}crfU1Uv94_czI7tdGo0c7fc=N0P*xq4e(8z6a4l2KNML6 zG~c<}S||we@^Xt?>D!bHK%&}t9muD zVWa&GBke!-SiY)eXUyO8nqgB2aY6z)AwGeW3vU#d1s)Bzi-)z*R0rRgPRdT*#r-GL zXt1rg!Mxv(6`$(-?DaTWab;n-XX@v*DN}` z`Tc^pI&l&*X9Ft6+5?gQYSq63TYM1H6>!f zJ%q^7Yq#%yIcYR$O2K!VH+^k3FQ8UZcfLSTH;fRPlCW|6?saja#aSshy@6=Fy-Lv3 zp~^@zR{N^9{K9sEV+hXPcW8@o#5*H8ZGCi7M*Q9vJ}9v$FgJ?}8`sAhUsz4hHLj>C zD3^@4&G7iN@c5+pQM=Zd73|^0;s)swZqTHKR)fg^0a(9P9{wq-wxm2QYpJikbop9S zkG1MW-P2Tdut|VlK+ru%&X+#?P(m6_*WKPJCnJP(nqhQjT6CVT&^{a#clm=-#0HVzC@{i=vYMO|8W64Zjs1NEf4pQ$b+56{9Y<^^9bX;tBWGH)r zUvd#5IaJVGDDdX{b$7wAdkQs#Sjh{LExOV1Qx}jtNXlD6k;RwPYD6}Mkm`qe2S*Uc z3!xDOu2cpRP9jKV@%Sv%Jv}$11|KYx`?2v?$W%IkpCzJ*a+3+xF+J5nRbHC_G=ii_&7kn6Az-M)hRjgR2Gy zhY`ZZU}+Ti$9@XkX5Sun(b~mfW>H+)LCJ0GvxKK761Oy3v?sK3gx&Hzu~4DOm7+SdsnR# z*R*41xq6gAGbV9T+Z!%4O)U_%_iQn{A#y@}EPqXAuz8`uBV!>2Bo57H(ePkKhB8Zf z#v+xC6I|((y~f$Cym--}N>Tmj6{_rQm5(lcrIreN4X2zI80kND4UH(eu^ALFGk`d9 zip-k9Wt;B2KPh^Li37z*{LD zx@a%*eJO`JWq0?$2w@hJXq-TQqR2N=NG#WE4`do0lQ2?XY_Qo6$z(d2@T59f6{zLi zDqRAD?DE~my_xJNQb0J5sO{(#Fq1>-FvjE!_9#aX)6T2eN>#2?$_6m$XGan>v^?11 zp*+RdyG1h7>Kc{m-b=YfHKw5}w8&Mf=5ft>q}_WTbgZth=O&2A^pFVC>biK7-p>O7 z0QZM>sOhT0^5bcb&w0%ltYbi##(i#WNY$se%!6=r@~RefCVTXStX|RC)n*x@J|*C} z=Yl!-EqaE!}y?H}z@}LTGC6 ziz}yd%=k+X z?Dpqx+_K!fI}1^R(>7%Pus#pBNTjNvs&g=7YzAgTiCMP&OlVtuOLroI=r*er@g(BR%hQLJ84hkA8I`1Ha5G8Q_%P(zMgqe^@Gproz zQPlR;R9Vi}oU&2gAbMHYOD|rrx_-yp<2$Z)ZbRza62{Ad~4LO zv<)wJm!sGx=Ox+f>>ZX=H=4)bc}ApSNARj{*LM<){`HLA{NCC`Tv?>}r@VgfWbu+jd%H@gSQ9pJs}MR#DXbt^sWWtC&Zabf^J8Tuu3VW~xEmiKceAQN zx*?iB`dv+5TQlLzPoXP!m~W7oxGra(e9QX0n|-#|MOBIZSy<6L-g1vrdSJ$)tQc#R zulDAJ12+*utcBZ*(N_o|M)cyXFITPj93O(#UvVrCA%y&%$uUM>jedD^c7<6rix5Kc z+kJ8^q8r0H>*@%uufWyszm#Eqcog2ewVTtg`@W* z91UDLdX%(j;~zrp530b>siorf)+Rx#xOZ?!tyb$tbZWgTbHC`+N;T&9itDEjT_5i^ z`VqyrFY5Foa_n~oE}b}ZY20H`s>Vi0lKqpPO07OniD@sWoGEu!uR9}t|jmZn;hepgwi1KASb5>y}iZR_jJa)XrRx zE);mPtZ<=V=!UvQt!;wV)~;^3TCG=)j5w4O8iz*^>!uq~Vix7li%U=JFCA}7hQg>* zE3vDTd9aAX7CpTDy-hFYT*1tEdCTgSw@zV$s8IKXJzpj6Uu(zDhnq{TVCR>nM5m_u z;vVWORw0yCl{OB}%W(G`SY!M5kIvW}ow4~nMMq6}eql**J9`$pT}bm8-+3!fu|ruaq0&CkwWu`tP*vq_={FbJy`niq$GnI})B9Bzqo?a%N+)VjvKO6bXC&ePu8C^%*lk4C*(PdkzJ8W#vZ?qU9?VMhAM45z-Cb5?m@RF2e-Vkbe;-d(ZaZSdA_4Uxtv0Lv_2n#yr}C5|)~X>ABU&Z;OdSURWv1 zvWVte@=jj1CcgZO28U-YCNcp@c3oG=7r6)_a@fl42hEq%pfKp}iv0>)vNnTgUM^

3o2Ibm653{kHMl@*a1My|paoK9 z2&(!i2`jV12sY>$vvQe5M3)8C?F--Ljq@FZ5GB?-IU~l~_W7NjwKB=yEM_ice)NGR zftFx{YnH{~%a3Q+!X$3qO^JCfDmvv_G**{t(yW9*2*mUEcJ@6Rz+89L**HoPi*GW6&d)k)MR%0bL-V7 za?cS=j;NO15Xf!_E`5KRF(YN^)7dE@ZV$cYLEd{RkQ@S$b@# zq%o|djYP(*J>wehxDD^USpN$CL8ZT9-?uZ@jN4f=*m$7^JH!OP{@~d5%~v5*HB?mf zr7fAz!8cT9_`GLx<~^IE8Sa&|Rh5>UFRc~yTXdrgHyt~ECF|2HW6M?3-&Q9X{og6! z@opMI$BJG3-nPxgNxw8&4fd}TPr9eIteAp-O4Vw#qXAe5dB?r^(dNN-_EiuLb3r;) zS5t9rQ*rKYzvLG`eDB#bhjW|amzr{uJR%?0+WJEu^2kAC&YFfA?NrOVz2Cs#80sMX!8)$IBTG zhGEgHRWD=`ZmNS3yYD7$sYOX72PW#9xHixDdzK3dbI+CBP`I*@Nw~cdV0GepAE() zxI*$9y<-mxB%hD~CX2~6%oC}X-sl}!!1hp@miHnBhKHCUCe3cUG#kwg?44Jq(-Y^C z2h+I~!Sv$F!lv1;CE#8jxYnXDL)>UV$gbyzs(*a%>~n;}#l{(A*J7ny{JHIIJDhyS zYMYeI*&Ijr;9+IXEHcT%gN(Vkja|U^sT?|P+`M(t_G}cra-1$Ypme zJ|$0$4Y1P1;5jumj8ZgU-kqT686MUmPYOa>_3a+Z&W7IyMX?A+s4?Ty_s83PxD*e5 z^mM-I=rz)wf%eY+rfdCm*Zw6Vyn$%7#?BlkP0CK4<^B_Dpb}LybenyP9+eow!hJ#w zTa{nEYIGCjAHQxfHO{p8e>waG(duBz{@Y^9Nng4OUz=ZOciN?$@?nIC0>Y@Cyy!HZ z>EFY>ouX!8TYXJaT|=FOc(NIM|KYE|wE1ssSz%+*H?5+%&$77QJobFnF?)A$uUDwh zpwXxrihuI5EZ@2d=3S#Yc|%2hLq-1BKgq1%h?w}qjLfw8Q9-t6W770BR!Pi0BZs9& zVEei?@@~SAm+)jO-N@9SjD-;g8=Bk^66yXVyG_*EoR^a(X1Fy%2%+G(tZ3p;OR8U7 z0vB}(%|9QMOH~LxU~|X~_X?#JAL{CNu1nl$?gTq~E{9<=yWRk|aibcen9DZkyPLTKVg zID!WU1YiL(G)`pd>d4fkQbBS4xso{B&i$;T zapn4#ES5NN$wa=i05*68roQ%Z^sV}`qLQjs;@9~<^)?oN-YHtTaebDn9kS}~vfMAf z>u_r{qzeV!6*^gU?xEZUmtq3O5Oq7@A9`n3s z`-i;^6;-%LK8m)baNmBjwz5`b)=HK)o^McO`Q4W>C{$1Xu%s1XNh>xXWnb;Fox6_J z;GQv5Emh4bQvlXrXPsm;7byJHXxE0&QKoUDBNEfI*JL|2_tJP6VT->w9amnMcQCi4 z)B3c)eKkj3d*$Z#{o9ss9qV_a`lW4sE3*?D*d%YSI@?}#mgw7|%eHS!v-y}!p}6sU zYY?Urio(m>c0M{Y|9I}vA{$Fj-Vx;OSR+v!{q)&zM^1-Z-wRGeE&AXZ5*8l(rcCU< ziL@r4*UZ&h*6Vg3tr_R;L63a)x$N*qC;M<+VdbttyV_Rokd~$P}~xaq`f0=7xdR% z>uKs7Xz#TnsKOm_7n8D6ClA%_Cv*(aI#cAZln8HkbvLPGDpxn5#e@Y|ZeQBm&{)?{ zSJlwgER>CZhP3yz+WS5Omd1sWpVcJ@C@fzN z9W}X}^odBayO5(7CWTFCT!{B{t0|x`d33U6i&{H8?ED;Zu3QMVHajerk2|&0c(iv^ zhMGaEBH9|A!(v&>EKh$f6Wy->EP5DG6cmNYV@*NG{5I_u%4b=gEGpC66CsO>M`OAO ziECY|_3`dh7Kg_|<^fnpt1_PSiMhsAeZ#JnDy>#ES)tWnl0bjzZ_7ycwM+F~L!-Su zBzj0fYGNR9pRWfe`k7b#Ip2JiYnm@zEmx`$vagSLq_@m0`ic>pJSQoV$7Zo;9v*ZC zNmX-b?-{XE;aOxdc0M4|jOCwfN*$eFN29)r5TeqiI+s7VVYrOw4ezgyb0TEK!#^%_ zZCu8RZlOTfChF|&6^nb7_=M`kmAU!+4_}PNZ=UQOn!5U})D?0`YpbYLDC!n>iOrX= zQ8eYBDdTO-3Y_q+Ib3e&)Falf+|G`(e5TV2<+UB^mscZXud9g4fVrntKVPg~sGtp$op zaSap+?(Xgm0RjYhbKTGP<=B5ivN^KW+-uG;&e1P!jo=MYXJ15Yrq$n1pV~~PnlFSv zwR)~hx34#PwmwWQq&8!5R9t4&RY-$IUlzJ9MLbkLzhAB?)J8YoW9XmU%B5t?U9-vw zx&9Tcz09fONR%9&`QCB#=j~qVlJD^S6r9-$l$Cj*9vY-Mqw%;8t`8{i==tIL!S?=7 zXX6tX5-}F-mY+_?=zTMG-YC2$)$;2BQt2b+Gzv$QKF3kk{D;t@y$?Tq0Ru_m zS%P#d@CJciUMtn^1Kl8On%PS~Qf~jKh;d4M5t@8r4NL0XE&B&)8{e_24U}@MYza~h z=SV_6yLw%>HvcHJAAWRJ@}!Y+767UKeq3wV*5XOw z*)?ufUxz~jMzRMNO3V9C&Ts@9_2nfCdQZZV!=t^M(9g97@v8cbFt5*SaYqTzV)yWM zk5L5(j0N)K~fmR%h}MgiybV8o8p& zY>47#Z7x@nB!${P<<}*O5ufV)TkAF2?g^^uU;D96{$JCM=&+QP<@C#ScY?3pr0Wmf zJ;nYGN^#`FijKkhm4s$&k^t$fnu8eKynn2K63s)8vD<+p+m9bc`K#2cb@UGJrqR8C zgXkcdx<^U9#x`+z69+*QA{n~alCT*;sru{XaHA>!ep!Qe0pd_E^OKBqGrWGZfIdlms;QGL%fy}`QnJnvshmwY7I zEqBT4gVb^?*AHcd3xk{@=_W9SH}?6FKa4ZBf2@X^P?1eh{ahr}a%;&kcOEeLG0B9u z5K7;|q{cM79A*c?FUC}y;sL5U>C;blg8I{RujyI-G59={m*cqXddZ#K9erxXYEO?KHJ>slAi+6MN5hR8b ztBJrdz>~lP!M$lbkfhU5^vDR6?9MN&G)srp4v~Cm5%IRX-YwTvkuDC23QH`>#HzR{ zkho3kPlA`%7>ZXIioR}(pMygo=nek~kRq2jW=pC%{x&+R$Ah~-zS?mbamaIC8pD6m zi4$b3V@Q5jsz=d3t!snQd1i5NE>7)O?M7)=PCQn5W+kSbPq%$hWe{?Ma%^)w_n7(o4gU z98;`{8w(a=9JFBbmO?eHEs2c&P zk--VN?w;D33vTF}i$yfz_Qx|J?!7O`Ug>5(=uZTGmMBaN?k4f|jMKW7Hr6B=P|g4T z`ZIv}PdW!CEykXWcq_(k{D^!25Onl3Xu9>*rSy-uVuU7P;Z9_*! zW6>AdB=beTme_%LII(ypw%YQ;+y1e??|Cy|;+#QeKRQv4;&-W&1^;MS?GI!QmLZG- z-+rnGs!q)QM);t=3B@b%3Lhjz@P-GHckFw7OnSbZML@S#Cq3qe){DKzqOX)Q$S{Lj zJ4C$|_Xdh6{^qm>Evqsj;K`$>tx?p_^WY--yHzJDsWsgM@OBI6CQqw9zeH5tRS z`l)k#ac!;DGr2ZypN%b?!{>ym?9&BvA6LrV(0z+=fg1`FepDJS>}ab5+ORG6)f3RT zt>9u6{C@8>F&F|a>U+z?@2Bk>u8&co;Wd1Z5buzFA|oo^OY~YzJbuHv{x{Ym#ov7+ zanotoq*=h4Y5o?C>OA{1=y5kcEV9Uf4H^$6eTYS z4h>-L6+tx)haC4&>D%*N8XDwisp-%VXGO(s=G}Q?-@c)8x3U5c#O;th90U61#u-7|fahG{9+Z6P&b!G6DfZw2rB82(a70>N64UPEJQZBi{OH;G zemz(^1XP#a$8?LfbyS_&GunCyx&W3ohit z%4C{L=wsR|^*J9pHO&%(=t{eBiSkEGn!xSwKI&&9Is#OO4z$*s%tZ1pM`a!Q0pgi@l1K z#(m*8d;|~y{;;`b#x)~Rv<8y#wTqZzvY?Dqd!kz%P{U@>=k26u*Da1eI|_lOZKkhp znA(%0>56LNa@+5sZ!T^12ixZS)l5HAmYtcf*81J@NL!aX&?eglq9B;JkaQ&G4}OpQ zL{SFU8Ty*W<OK$E;Ft##_yUe>?}CUlT_B2 zac_y2l#kTxo8C`<&D768N$D;(2da7Qr)YSBgT^ zV3^++_g2*#>7Lm*(bI#*1;ED1#!UZDO)I#}-~PaoQz9#BSg!=4skYkv#DM#sSKCn= zU-Dhgo^-RdTL8k)rIpICYF5hJ8DbdQL1sPY)ad>m@6SeC0p&u+_dWf z%5^1qZVU*~+z*jx_7787VrZ&e3Pp@sT0k|HT5w6kxmK{nXy?G+03RjAFt5jf`|GHi zP#^yUN=qL~oV!6^k#XBT2i7Z{N2gkhS1Y>Y*)AuEK}SXI`*s#{ee4LbUR3pM=Ckq@ zqRxQcHQ|9(;!ZPZrw6QZ8J3WQPb$< zB9um^aV6Dac`I6YPugFZ<{2Kg|>F@9JL`lA=)-pSB0GYQI`vzlTE=Lievy5{Gu zIL5B23?pMzT7y3B&xER>;rs|>;|moZTlmtUSAe#ATYLRzO74%bsYsP3;ee?i>=w{kD9y3Ise-U!_8bv7+Rt z`%y^HJqTVh5Jfj6Mw++uRq)rsy&d!J ze)Q zcJ~d8y)hiq?f%w|PCSyFV-JZoF$h*P=P^Hubmd3qsGkkVt9Ind_?Y?HvK^Dr;op@(W%^E%?rky{lkUP1aSQ z$iP3DYx+f9mov0iaWs-F$`Tf<_l57@b#d;nbdga&hMC&t2RAeT7Ns;%r7K7yld36}ZaI1R2v0Dluj$_Xz5&m&I#;(hhfG*y| zBh6csBU0-#(WId}z z`^uancOrm>-T$+R&@=;0LOBZ@#k!f~O+m}`=#!rHtG2Xujk95OeyGCJLD}HHtqn$C zNgO2t+0tBH^~z{aVW?D6QuE||TNDc>{4-Hh$GT5H`m z*#e2V&kSroNR_*mIy|N;P-Pk&spNhCc2T3Dr%mSp2sOy}o?^A$J#;28q_C(UO1>>S zIS(xIcHd5|Sa!QmbCUM-afbl+F#_)PxFQ0Bsc^3PpgbB%V@!g~B%?xQ0o!{>Sug1d z7__JuGcr#9v2)m=VXj)L>yD!RBjNiT3^&zvtX9V=N?Fe#GcAG z9|r^6tnr-bE%#dDIzkwk1>f1rQ&p7z8Wc6Z!sqEU};O}K07dG^Zy*R)DGA}|dWr6JkleEO>?m9a+2p)rP2?Dx+ z1k5Yx=h4-63psryPk`DUC2;;DK>N27OC{J+aQEm;vBmF`)^=!245LLXTe9?RMD$I6 zp^D2!Odv1E#&W2<}>H2@0gNetQq+1Y*6dY7w#pGZ+Nqw~6BBx?BMdK={$)a#t zdU~!(Lht*~&9 z;>s_Iwd{Rhp8~c~OAR#{YRHj8c zb$R*zc#gL@&gu*OvHRg9p1SXgF7wWOV=1Tkl}?&-=gw8LKij9pM(?#0UJd*R(mmg= zf5AZ3wI>legh&I(pMFD{T^s7zUb2FA(`Avc`MSJoR>Iu)@3YHLz3U{=G4QL_0_X*j zZo1Dq1iV!!d1$?*KJ&vz`K-TS`7lxnJ9(%}&!1gc;EVkTlebxJ*)Zt5EV<1 z@qi@vdXma(1%DuWF=Ma@Q!qevv4y_JXc$o;uD|t5KYXjz_po}I+|n^`(NH8o!ysqG zHYCy8HhU7LW`cj14iw@ufup~AML(KcTJxNmSC+*#96=HU{7{Bmgqp7ucg5YlQmCA# z{N+c-iJc$LWg$HcABeR1KUu$%={Tf&kNsKa>S!+#6_L*r)r?~Nzs=;dwpeHZ#q z&Tp|Y3qSA3V_=dkNq+IL#B}lN>Yv;0b2}(oeVWl}XF&fY;{Pc2Tdu&S!-fNu8!>ZL zB(&P14Lw&fawb7^Eh>NY+MVxyJB`)Vu`IMIWb=Cpfx*t089S-tKcgFK5c`gh{lvNVrhGEv zzoXsx`R5Kj|7Z4$z`f4ad4{g*kWZ13@Avkj@N$TqION4r@Oq(i{CcyHcIca*`J4-C z>FZln$NFNK5 zR7f4EWwfisT23K)@qQ?sarI>4JX7)9r) z^9GWP-f~PJmX#ha0-8DB3m?eY6DBUvXZvgj6`|r_o|JtkERDPT*s$3Tqk` zI4L>h^j_+pDYR~mP1zy=D`U`%(&8?+cC!~rd4GX46xf^JplIz&72MukhyQ^u(E4Oa zvmvm_`B=>VZaQG$S{6jqy1IbyM4-{!M!fUe`^CVu&6yOrtmK@ygI)~e#{Dp|@xd*9 zCt+=lK7kaz(Vg2sX;x7%!R+BhSlPq>pdL_g0L3WxN$@9+3XL~l!*7VhTga<5vkr;A zMf1HwER*Enk0N^yKQeKINEEwiBo{w&PGJl>tsWe7HA3W8b_U?s-$T+9&tgTS17iJi$FteXo ziXCJ;^eBT6AK~L{Z%@Fsz(Psf^zDbrVnD(CU4EKy;oKrQ+wmq&H~lP0Q^^_s)1lvf zx+n`qtqM~T6DvjEtg@__U*`Y=rO#oj65D3^YlH{VURbZ|HMa$ex3BAD6>wP3fj&k_ zjS}JfU-z_vjVYw6?o5|IW7_V|2<-e5P^C2yG%!EVpMYsmJQk*6{X<8;xTGv^HC%R$ zpm{$Yu#&83@>`W<17$43cMIvLQ%LP4hh)l$z_&NE$)wuXGH-P6?r&lJn|Yz`$l=KS zg_SDng?3selcwHld7yTL91+F-gol(A<1-py$+w2J82crAW!*%3#>5g{+@*g^K%6SB zC4cnE1r=aq5ZUX>=z?N=0{^4A?$fn|pOFmY+rxwR zBxw&1hf{-zLknG!QhWDNl{WQ(WuMyNJFhvJ`NLyFcb`(t8;fX1-2FeZhir_PbP?Hu z(B%dr{}$jH&g5I)J~p^y^4X)#9vaNNu39^p(Qws``z?{grL`@hGbQj^X~ybAL~Vv* zbx{sxH6A5R*&EA6^1PVt6`|ody`ds zo99a`xnjfVj^rblpTI_;=Yty;lKOi@$5@G(WB|Wj+__&R0yTB?d4pClJi4QwuoE6N zxq9@S9OeCFg=;9kx{OVjrOp$iiHw2hgZIp$y4jLoq?`X|r)U7qfO*!*%37S!bCHs^ z8iUrR*dlUNL-FD{~z%7izcV_ zGEl5A#F7QMavbrCa&=&R_|0V|ojkv@hGa_>Cx)f(zQhel$=_c2k+XmD*bsG7r?R?w zs`kIShz4Qy`h=rD;z-S0Ey%fFI_f@BgfU-cvM7^P*sqEp+!;Uc}N;b>Qnf&;0DzIcW zA>*nStSu99Vq~AE4>T=xwcR@xtBW9oP}Z$#%zUIjGIh{4mRhalXUN#fUv_|M&!fzu z57mBBx9Ivw00k2H>U9H&+iZZmj7`0i{-c&3bYTpI$%& zhFZ#Vbd;zvmf&{D>CXu?fyb2f^}yW>0|I?4D}b zN+-)6t*HwRP7suKFT(}8liNP?pMBE{y?p5kYR5W#MvzBrUa`ZlM+}S2%_BDQ?mFRb zLj{6_#2!!ip=B~OmHftQ2o4-l&99-0Su?}#I4aB4;e+xqyK`3-3twv3n zc3UhOC2N3)^r7QCls}tIJs3Pu_gbGVL=v4^RKnts*qNqIT;-}0Z?13C1$Pb-E;ko| zxxTw!$+*__R-gel>;bymd78>E#$m6kxT-mG`8i6GcT3ZtneG`-AUlw$S`qk5o18mqBj z^q@wSJ)HItby4ho(czKqPugm8U}-zzvs#N#C<3If#*Ln(!kI;`Evziw%%5cHGJo*( z^^I|TTlifcrCFjRb-1;6RR`{4%y>~{dWl0bLY_J6MpX5+*4%pYN~bdZCOn*)wgMVy z(khGs=s8Rg$$u!HO=Cm5h$xU21)Dh`TY<)k3S9+U>V^ZuTN`g$Fu}m5@dn{Xc`?=5hgB6`&J*{h9^Fukav^1jQm2@=GeFU zfs;@i$J-qB79w_*dgBkQ16i@)I!6S^wk}1j2Y=szn$fxJ*(n0SJ4YRRyg2=sdwG~Z zT)%V5s^4V=m>O(zQfNiC8nI&a4}eN6C;arL&6^-QwfJ?nDNR zWWaCWoq4{1(d7AGN*3EzMId^B{Hxra7&_OH#g|Qf=9&ml@PCjkM^H0W5Q(kmV-`7f zB>Knv_ha8O1A@o|lNenhGnzf5YZu+a45*H&_~nA-+NFo{Hd^bSDPLo}>=b`#DLe;X zWAr3{L9p&lIBmzVcK+!{VLQkO#6Kbup1#ADi7$3t95vm{*5PevLS&hXp@SF=WG3C= zCdq^Svl2KapD5lTz_R*x7OlRpj0z?C_tv5!QI>G#_Vt7mv!x59-n9Fqv$f;6ud!?BF3uN1@0S6>%VpqgcPBPIXqm4CKJSe z6Sz;Bb4YacQ|eX;d+*sc`d_Q}=qM;91QBhEG>niAs_D%;beQ1^(L+uHfFm=OL`K6E z4tKs!@UUT5`(RtTG}S$Sdn_L5%`7ufWSTNN)Y#3r?<8l8D-BkfxeI)*fyt287=z)4 z_e{@$Qm1#Dw`fA^28j*)uFqw7&L?ZejNQQmUwXEJ!cFFfBgGbP_b#`tj+Q|9xofRX zr;#~$ZovnWr5v$R%L3uuihxv8frjYz&(_mkBSWtcJMRh08kk|+%(9V*V-U++Bs)mA zvC5$79@<@_MjIS|yH}%ZD9Ykcr$?Z%ysV~SNe&2U*Ut@Q>)afRiV3g+-9`_D)BD?* z^^HW)I0>VnnfQ5~hN`OOF_nU+)cl_BZh6Mf%M)pQJy^z*wnAUg1L=LPgI&A>r7C3+ zVHkWSfo_31OIvC4pugu9Jr?6_k~h~;`*rbET<5Di&i?PL8W%kAr58JJ|Woe zF8k_S3tm@HFagWYa=pojB5yT8F=k7_&4e4gQ}3%qR*JJuwFh&>n>exDNAKZG)z1%@ z0xY84SywL77ZWt8*5mb$dI(U-W{k1*->AS6r@Zx+C&DfV?diJOnu2mfZ7G(mpb}J( zHyH~R*(F4CoQpM`;dV$4z6W;tzAC4-^6#u%fiLn zh~D%eUz+ogpuH9A|Kh0nG~Y3MO)jOV1`Aw}N8{gKGiS*ayDJJgoMiXrHf*)v9X2~@ zax&bvVCbi`v`AO-0!)(@b!YO z!5iL)hm|)XE-sB%btA^=X_^gdHv9w4Q_2lg2#;7fs%CXkw-|Bn$8<-W$~psf(|~sy zmyu(a#Zx)>Pa{jC?ACO?SxXf`-9me7nMLPy z!}7Y{oPYC&oX)-J9@F=LUDA?+x_du`ncCI7j^rON0x4OSdqrijIofXaVg+uTCV>Gp z8{?^QK8-fcxl5WCJHOM+NJ)a^PymgmjJp?MeyEP@3Wtb03Ue0A3e7Xj7#dv&5N0$T zUt5(vZFEm;vl%8hx4a}hx}fCRuNqQe6UdEzWneoqxF_vY8B=Q4<#~73V7zJSp|El`x`i7*_wSl-fT3COx^4vwG*I zk1S*rLyo;E*^Z=hebh8J-RG#8;E$ba(%}0V20Lmy8>8F7i^zJ>Sk*D2{sectO%m&l_#Hz9B zd0=?M0e8P=8xJ?U8H(DOE@lJjcVcjv=b1DJCY{SM; z5>?m8kwyFXOAT{sjp(RD@fyz50rH7v0(gljX{it5#<4bMvOsrLW2S+Malz59RDa-^5&8)fsu1g~Ou=zTTJ=#NV zz<0qb@OyH%5sMQocc@)w${4crO~Y?XDi$~Z-g+PW35f84Hwo%t>8m?sxA=Udv0KdC z*&^fF^C>PHTz+&Q|L;s6^|OAdt0T)T|C!m>-wS`fIUFxbS{&clYxVxT;&YsW^2gnY zhN->GG8F*{8-uyw>Q2ga!^x>&c$L=z^$}#6J$HPl&9;FrE*yCMNH{L|GvrhKDjt%T z?uMvYF+bjA!w2{e1ATjo{wrlliSs}SJ^RD6hFQ(3Ry9Sn6% zx&B2c=2o*fUytZA)LwQoSm?OK!xnZ_476WHMSrrOMay%H92K@2uYEJoS3_iBSK#`d z-<{B@@Ae}Tkdd5K3HlWtRajQdQKP!KnX1w7(fi=05aAF96FcQ2kwJvGc|x@{sAR7~ zf0CiUM=+BVj%vB<$Tv`0F%MQ2T*ebW!{=`bK3G-`3+ii9*6>Gf<&Z?d5J|Fb2e^gm z@^r?*_In?!EX~)uz57^(R2b`P6XikEAuIGNk@&d?Mb>9 z76e3Nkf3Y&3GoNW&TzbNMEC3PD~-t`{fM{`%|<$ZtWDRoWf_G!DRNV{muP)o@4S6mc+ocbt`+4n=02* z-7en4is5k}ON+)V^`Z75mISR}*f{hK{7_YT`7kh3-%vezc3M*D7#iHGB9o{GVeI02*87elCLWC246f)i}y>3+Q)MAmcDDW^o;a)!6=GpA5_4hgEx6+nqNCWLAu0oj0v{XLc20q4x}I_mrO-PJII^zfvD2eh%b( z4v~UwzI*3x%`l!ZdQ-2IEK|_6cRjvDHB35fJG1ciNxFQdq=pNsEbn)TXLSRiu#+<1 zhfS7tQHj>ee%YUCG5enE>mYC^U8%i3n+}ZnbnId&9W*WzOrF7MeQM7o#Y(_F+XO`a zR1zq0px%mME#s}F;b!zME7&7wEti)2q3>#45bA%6?C;x?MT9-s&=*Sr4&sFT+8R?0 zbr+o#OBWJi+N>#g^YI>S!z2)>%X?t)RaNw-`rpcE8_)JZbAN{R=g3^RsUV0f`kL5IC+26rYtFK8I4 z&dI3y<+4?3W|m9mNw=84$kS$O}xW+upr5Fy8Otlc-6@SG9Nq@xGGcNr=ls``$c$w$2}cdav<^1w0j zl{%&B>(U1*Jz~{cGVfif@h?EuEE@e6iHfEMYW27j^J+rKTyL zQ-Ew-(X*-M5rs*7eIvJ#vm@rsHSDGQ<@US3URZn>gEes!lw#$ZiNDdziGG493#tSS zaf9o@Moy|hkE?5bPqEoc@Tk06fdRH%rr~NX@qD< z3n0q(`KksHdcF5vwTq%`^4u)dN${nYf;g~U!+<&21KnfwECnM@lrUZ-$on$x51TJ1 zt4|;I=gD!Om;2knjsE9cfZJv3U&Y^&$m;wnCC%UZ#(_L`g{|sWT|!}Yq`9N0RGWY5 ze!hIn;(=@lAKi$p&X!cJF?g#AgN`nXiHE9ijXT&dBiahh>-1~yUU!j#oGdIJw&VWj z4WlN)5*;q8p~@{=97*Tcr6{IB`&flabS zz14v+B!HLk+VXj)SYBzlYg^K@ir^P6zR2ObockO5@{kP)Ry*0Aw@=*B@Ud|i_fcCj zRj10@$w*UlAo=8y`bhwG8khqA#oXs9jx4|GM-7RXBQc+^(^3+XKfXQ`4OPEoQ~zi^ zs;o5-VPTXlzLI?2R(~>%d3fCn&Q!;EP6u6SZLeNpJE|;dtx=+8?mH7!b4Hw^Gh`Kc zHNq}rC9iq%nqM9%SxQ@OkK$Od<>!CdhI~%6cJW?mAF1~S<=0v#SzmnQJ>q2h8B+BA z?_e|Qw}Fv&>|Yj)Pr~%VTTe6P zByZWDE;?t)jDa5)FH3;eKP#;FuJm^Rfs-Pf5S-tW;V3K6aAUtoyBht+y60l5m`scP zah{{Lv8RxLp_rGK!S-;evCX3%S9o5ro>585@9-#&Ok**}6tTFA zeBjXgIJ#KTBfE1Sbu8k_XAp;2fXe*1Ri!I@$+;7Cf&%UmbS>&%Z?M|QD*6Z;ht=Qp z&#ucpSx;>hU7SMmP-Rw*aZ8NBNP9@l&yrUeU1rMHyd|lpz^2fvbpsKid_B6UX)f$+qrag>tv-jIH z*R{@Dvo9c!bn3}r@%;WAP%n80L~72lcJ$e;y@M+Jq@zqId$F<>tBu5dSXr`hHt$T5 z-8iA7#l-SgU(?WD`FU@=qG@j>q5xLn0RgL9fT{w&GJe|oOwedd2PuqU@0BO$r=8El zzxX@Y1uPnKvjq-ChOhrNePWj~eU}bZOs_du_e~LrL0_9@@6gVI`mm~ZmbV+4F*lFU z_{?ge6hX%Zd=D{Rs-YHW;s+5bYdk9H#B^s7bkjEl5mVvSbMhn4DzOSEruxUywD+ul z6tCwe)HdqPJ~z)^;o;}X$9lo^z8<_(FJAfQ1E{ws#85;%k#8TM8u{~sC$vFLt*x1vewU+v*_}g3~-xQ>B42HIn~cT^)KDP zK~Rn_!x5iI_Se9tB)&!hB0qOw$-KpStJ9y1V#7bg%5tM#9`nXw=KJ-x=}&8b&d2fZ zD5R`>HP^L6+ihbc^xwUcJ8xHcE^*o^cdPqXDKG-u%b*OtTNF)|A75B^hgA7G#A?^? zc++-ru07^nG9cDC%JCfe2UyAez_PQd&9~!)X?#^EK$x57tM6gcy|cXLcy|k-L3NE_ z-hEvmMZ8ytSDO5V+7milH9U4CGPw#LI)sd&yPAR)-g|6(lru2gK8Ee-6W z7rIQdcs|-W=C>vG@wGLOuFahdz-#z*spow3H2u6eptzS|3#fjKACR~yEshX0Y*f5NJ^qCEz&g#Xz7(*sy=yp*J%LAc_+zfUB%KNK#GSiU5p1_P^ZQz>8Jyem!6D zneeLx-E-2}nm-p4rJIO%M@x4N1D|0ere2;UL)GX0pz*~9r%vndV7x5oPdD$ z(+hkWzHe}n2SUy}Ezn=`xf-|sgyIUI9P|G;niB!{YP6Z?3$q0-cYYDqxN|?Wz{_ zyYBKAM{>a(BG|q;XzW(H5$h#2;~~x5nZ36HW|e`_RgM@?@O<=4c|1fHl0nmyn15!q z-ewuLd|nBpidJ7`^xvL+ajL=UwaO67{LwA4{BoRk3rQlnBnY@{BG6!&@6&7`_1>>; z*)~zjb#)H7KN})S?iz-tbpn2jewa&IR_S<2&*8Gep{l}>04%yY#Oxa21_K&3%I$mm z*S+E8ycYdw4T%(=z?bqNF^u~8EXH?>#q1+cy4ER%4PpwBv9WV8kh^N7Bn}bv_`v+j z?OE#p)2An7%m+_t^ccvwx-DX%$(?LHxW>x#JQ++K3(B4MpijYF>T*n0C&qj1g5&$O zoOx6BZt23vn-PVB66~~mb-G+>9<*Ep5hdSz`et!xk+Y77FHsoL6YLCOZ*O3U;9m;m z<#DfssgA}7r1Xil$_9oJYAtCoy!lHWeN|Bq&r#q)L6ty}qmYxnblD|WO(S!LzV?Jd zwZjp^mBKqaNdW8YxX#38vpwaG5w_}O^YId|Z@_y~`mAYAVb4}k&QsyCvw66Ci#r50=G>iLEIK{7pkrcuxd|EW-e7qt!UlGf+!*#eOvYuTpQV7iq|~GK&}Ey73*? zCNs?K>pK0CC9)JCM&D*G5YxQrE`l|tc)b`bDtFzEClfD5JgVyht?v7-FBvk3I*5gx zV`dHd_U|zcV>}yvT9v&*pCYm@JK#Jy7j$l^AL@2j?C`MXy>kYGnP4uF*c_%kZiOMT z+V>EKx;rm-@WP|wW1)bls1`}YZ)HTP@f7f(sxnXaM*j&rqwQ)nu2zyLxa9#kMXASA zK!Z_A#Ib;YX-^Ij{h3{g23Vv$csX7tsSG_Ambl{1(f=G%@_}-BHbCKSS(dR-wtp~8 zwy370zVyEvxqs?FKgU&YIHBfuhZk3~qd;jNw+})t9);c;j~OF(!oCO*m}kY@_W;(C z9PSFcW#qq%Q!LF`H&H|@2xZsR{hjW3NWe}3!>8x>adM>$mX-mh+vkfZnil3f@?LX( zhPr)sYFPQc8YRe^UT`?mnCBjeUMbFiNB*cmc?Qut9_aZ2Z9Y0qpa6@|e)9S?hNz&1 zBzMSfy3Ua5)4+56@~8*pe>|d}t#$#>0=AR2d+nMDulR-^)+cCwZYB{7CZ4dS>D+9@ zavNxobql~j)Ri-#93$_iRB|dA7o&eYFHVH7NJ>o0ugZ0mZ`Fk<5stJkjd1h7fY0-k zp$%bQQtAsR;Q!7Cw>d!#(}358GKKR7mdI3j0}Jab^-K%bpGj0erK%ZO|ExI%_U?a8 zGx^y?DONGZ^fqhw4j}f-hW!mV`cycJasy-9)Rbw4R*hp*@GZkRgN#zFy&V)vt_2 ze`hZ5qVFA_s&`}K1oVsDRQ>YpuUt=RDy(}vXnGIfIdv=FB%bb`;+m^4Y@^NmvJsSV z`Mj27>L<1WpyG(C`Sn**x1-}`m)l$lD`o8RR_abI+7xdyurxaw2Fs1;bUAU5_mOOZ z=ne!1Dh-FfIZvKf&5aegb#w|mREDNA%#A=0ndPH6UZ;x+w*i zc`o9uQ@DPYlasCWf7fIS&tNa&_Jb^L2I!^CyxEg6@|r?f0a;mxXl@-}?nlC}k34M3>- z(!i&*e3up`oA*)gX5T~NBykx}h;m!#Kx;$H?|u4FhmPAS<`$AS7NSp5oU4?sni9AL zL4*Ban!2fZBdi1Lj(m!l3}QD87>yMXR)X2J0Fv`NZ9uM)4I}e?UiVQXJ~LK5u2*;W z$?kA1XMTS`2#5N7Gw^zAwV2DK*?UvZ#MCEf6`J6IEIC-`mh{?Smf)bK>W2d2`^~`(4MTDJ*FP z?g?Do8MiL?Q={MdyKOG%{I$VK*Dfnf+y6UD>-fj&W9xPHTAVs|=R9BB*5U}YYl|mp zZ_k6uuzpu6}SRSkMe&6TKVv46ktg5A2EvKI2`^o>$DrQGm^| zpBF6bz8skFpJP*b^Q_x*|C;CXdcIdYpE=LbX8ugYme2V+zyGR!h`b*;2M@jyI#4wfF2K;OU+&>4krsei|m0{VTB1%4`n+>goFZ zE<@ncGqtoc-H&g+n{rvhD(aS%>Lo|fMZb$G?sUx9P`F6oopbHdK+mIxZa;Etjaer# z<^E*_m!+F4FAA;MltThBnOS-InZ!L-E)7MdYX2<7WYI5?~%zn%Lvm679Pv;iS z-{N2A-@NR@qsyCun(hRL-St+@72DGJTeMlKOMbe%+^Od`WfvMvyLbQZqZh3S^Ak5N zSnFK0+rKDXKv65Z>e!ww686@8zKaF8^SgoDL=GRWtt|(-%A{3XYNCXEV*Mqa*-~QM zuChXb^Sd6s6+iO$ck4!t{Rh9brF|$X_Kse@WOsP$iZ7b?O4RONp8Ql?L|E!b!80HK zJErSor*v4U1nrD5T(Qe|g%gMG$7gJ+ZpT{-kN4NMd|R?uO(`qm(E*+aVE0DbIHI`Z zI%lJhV%eXrQb~T#FQNO^(wqiaA$_=rVkmthbGET+>>tq_s##mxi0aH40n#~ymd@H@#n3p z0k=WL_Jf7ayj}C;L~QT&N8`Q0eAlu z+tx2#b-5;Q!h$0|7W}r6=Re#(|HX0hf0sXs9}xNfLIsGv%S+A`mOIbcU;acrB(0)m zRl=hvlHZXsuh58K!n8XiyI z_VDSEUvo?Qd=ENuo~g2ae%Oa^js1^+MIX~r)$BJaT)Y3;yz;!l=0(?QKg2otK3%Fd zF~VlSmFWLkz2}}49DX$UnafW$1CIhr1_qf&F1g0$iZ&m%N16i9PzJ7pXrJvCSRR-n zS1&3x{b1|S84a^}wuN%Mn^AkLt42qJZ8{SJ!;Bn(237`!{dy^WpKfiRto~brfq`Gc z>&K?I={t_H??1cyf`at()oUE$GF%Sr`ROgD)@sjlS2*QdlWx53j5mub4oqOa9DjR? z-A#AHn>IHBxx-v;#+ub_RzAbPunu_T@J4>))XV3lob;0fIf#LQZvt>@`@5O7&&21S zs+Fjke}FkLAR_d7{AGJ&}G2;foBB#c`W|kVoSyG6nS=#2n75% z`|-3zg<k#5vWA~ynhm=5|yr7tH8jGDg@)r0p4Q=W24d! zfM)=MU4tqN;;B0_h+&F?L@YR&Hei=I;0X?IY?6LW3_94v85jzLfG440m1>wg>LpOX zjK&fqd5k7JNH~noF0Od*R3)DT9cW>Di8Ag&68BrQxBeMWO^OYmrpB$6y7~+Z%=Vrxjv>gZM_4H@ zCn*v?>rFCp2q4%E@v1Wz(dz04|MQ>o6$)@H0#5Jwp0IjX?&)$0*htRg&Z@)6bD{}`G zM3vvcf@l~_L^K~1gb-=jg$Z77H+Ue@fB+|=4WPt0YB)4@sT3^o-3)6N#s* KpUXO@geCx}aoTJE diff --git a/pkg/natsauth/permissions_coverage_test.go b/pkg/natsauth/permissions_coverage_test.go index eed0ee5b07ef..e5b9db21ad38 100644 --- a/pkg/natsauth/permissions_coverage_test.go +++ b/pkg/natsauth/permissions_coverage_test.go @@ -1,8 +1,6 @@ package natsauth_test import ( - "os" - "regexp" "strings" "github.com/mudler/LocalAI/core/services/messaging" @@ -157,33 +155,10 @@ var _ = Describe("WorkerPermissions subject coverage", func() { }) }) -var allowPubRe = regexp.MustCompile(`--allow-pub "([^"]*)"`) - -var _ = Describe("Documented NATS service-user permissions", func() { - // scripts/nats-auth-setup.sh ships the recommended service (frontend) JWT - // permissions. They must cover every subject the frontend actually publishes, - // or prefix-cache sync (and friends) break once LOCALAI_NATS_REQUIRE_AUTH is on. - const scriptPath = "../../scripts/nats-auth-setup.sh" - - // Representative subjects the frontend publishes on the control plane. - // prefixcache.* is emitted by prefixcache.Sync in core/application/distributed.go. - frontendPublishes := []string{ - messaging.SubjectPrefixCacheObserve, - messaging.SubjectPrefixCacheInvalidate, - messaging.SubjectPrefixCachePressure, - messaging.SubjectGalleryProgress("op-1"), - } - - It("cover every subject the frontend publishes", func() { - raw, err := os.ReadFile(scriptPath) - Expect(err).ToNot(HaveOccurred(), "cannot read %s", scriptPath) - m := allowPubRe.FindStringSubmatch(string(raw)) - Expect(m).To(HaveLen(2), "no --allow-pub list found in %s", scriptPath) - allow := strings.Split(m[1], ",") - - for _, subject := range frontendPublishes { - Expect(anyAllows(allow, subject)).To(BeTrue(), - "service-user --allow-pub %v does not cover %s (frontend publishes it)", allow, subject) - } - }) -}) +// The "Documented NATS service-user permissions" suite that stood here read +// scripts/nats-auth-setup.sh and required its --allow-pub list to cover every +// subject the frontend publishes. Both the script and the frontend's bus +// connection are gone: cross-replica fan-out is a PostgreSQL NOTIFY, which has +// no allow-list to fall out of sync with. The property was retired with the +// artifact it guarded, not moved, and the surviving suites above still pin the +// MINTED credentials, which are Task 17's to remove. diff --git a/scripts/nats-auth-setup.sh b/scripts/nats-auth-setup.sh deleted file mode 100755 index fb29fda490f3..000000000000 --- a/scripts/nats-auth-setup.sh +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env bash -# Generate NATS JWT authentication material and server configuration -# for LocalAI distributed mode. -# -# Requires: nsc (https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nsc) -# -# Outputs: -# ./nats-keys/localai-nats.env -# ./nats-keys/localai-frontend.creds -# ./nats-keys/nats-auth.conf -# ./nats-keys/nats-server.conf -# -# Environment overrides: -# NATS_OPERATOR_NAME -# NATS_ACCOUNT_NAME -# NATS_SERVICE_USER -# NATS_KEYS_DIR - -# -# LocalAI workers receive their own JWT and user seed when registering -# with the frontend. - -set -euo pipefail - -# Ensure newly created secret files are private by default. -umask 077 - -if ! command -v nsc >/dev/null 2>&1; then - echo "nsc is required. Install from https://github.com/nats-io/nsc/releases" >&2 - exit 1 -fi - -OPERATOR="${NATS_OPERATOR_NAME:-localai-operator}" -ACCOUNT="${NATS_ACCOUNT_NAME:-localai}" -SYSTEM_ACCOUNT="${NATS_SYSTEM_ACCOUNT_NAME:-SYS}" -SERVICE_USER="${NATS_SERVICE_USER:-localai-frontend}" -OUTPUT_DIR="${NATS_KEYS_DIR:-./nats-keys}" - -CREDS_FILE="$OUTPUT_DIR/${SERVICE_USER}.creds" -ENV_FILE="$OUTPUT_DIR/localai-nats.env" -AUTH_CONFIG_FILE="$OUTPUT_DIR/nats-auth.conf" -SERVER_CONFIG_FILE="$OUTPUT_DIR/nats-server.conf" - -mkdir -p "$OUTPUT_DIR" - -echo "Configuring NATS operator: $OPERATOR" - -# Create the operator if it does not exist, otherwise select it. -if nsc select operator "$OPERATOR" >/dev/null 2>&1; then - echo "[ OK ] using existing operator '$OPERATOR'" -else - nsc add operator \ - -n "$OPERATOR" \ - --generate-signing-key - - nsc select operator "$OPERATOR" >/dev/null -fi - -# Create and assign the NATS system account. -if nsc describe account \ - -n "$SYSTEM_ACCOUNT" >/dev/null 2>&1; then - echo "[ OK ] using existing system account '$SYSTEM_ACCOUNT'" -else - nsc add account -n "$SYSTEM_ACCOUNT" -fi - -nsc edit operator \ - --system-account "$SYSTEM_ACCOUNT" - -# Create the LocalAI account if it does not exist. -if nsc describe account -n "$ACCOUNT" >/dev/null 2>&1; then - echo "[ OK ] using existing account '$ACCOUNT'" -else - nsc add account -n "$ACCOUNT" -fi - -nsc select account "$ACCOUNT" >/dev/null - -# Create the frontend service user if it does not exist. -if nsc describe user \ - -n "$SERVICE_USER" \ - --account "$ACCOUNT" >/dev/null 2>&1; then - echo "[ OK ] using existing user '$SERVICE_USER'" -else - nsc add user \ - -n "$SERVICE_USER" \ - --account "$ACCOUNT" -fi - -# Frontend control-plane permissions. -nsc edit user \ - -n "$SERVICE_USER" \ - --account "$ACCOUNT" \ - --allow-pub "nodes.>,gallery.>,agent.>,staging.>,state.>,jobs.>,mcp.>,cache.>,prefixcache.>,finetune.>" \ - --allow-sub "nodes.>,gallery.>,agent.>,staging.>,state.>,jobs.>,mcp.>,cache.>,prefixcache.>,_INBOX.>" - -# Generate a credentials file containing the frontend user JWT and seed. -rm -f "$CREDS_FILE" - -nsc generate creds \ - -a "$ACCOUNT" \ - -n "$SERVICE_USER" \ - -o "$CREDS_FILE" - -# Extract the frontend JWT from the credentials file. -SERVICE_JWT="$( - awk ' - /BEGIN NATS USER JWT/ { - capture = 1 - next - } - /END NATS USER JWT/ { - capture = 0 - } - capture - ' "$CREDS_FILE" | tr -d '\r\n' -)" - -# Extract the frontend user seed from the credentials file. -SERVICE_SEED="$( - awk ' - /BEGIN USER NKEY SEED/ { - capture = 1 - next - } - /END USER NKEY SEED/ { - capture = 0 - } - capture - ' "$CREDS_FILE" | tr -d '\r\n' -)" - -# Retrieve the seed belonging to this exact account rather than taking -# the first account key found in the keystore. -ACCOUNT_SEED="$( - nsc list keys \ - --account "$ACCOUNT" \ - --accounts \ - --show-seeds | - awk -F '|' -v expected="$ACCOUNT" ' - function trim(value) { - gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) - return value - } - - NF >= 3 { - entity = trim($2) - seed = trim($3) - - if (entity == expected && seed ~ /^SA[A-Z0-9]+$/) { - print seed - exit - } - } - ' -)" - -# Validate all extracted values before writing output files. -if [[ ! "$ACCOUNT_SEED" =~ ^SA[A-Z0-9]+$ ]]; then - echo "Unable to extract the account seed for '$ACCOUNT'." >&2 - exit 1 -fi - -if [[ ! "$SERVICE_JWT" =~ ^eyJ ]]; then - echo "Unable to extract the service JWT from '$CREDS_FILE'." >&2 - exit 1 -fi - -if [[ ! "$SERVICE_SEED" =~ ^SU[A-Z0-9]+$ ]]; then - echo "Unable to extract the service seed from '$CREDS_FILE'." >&2 - exit 1 -fi - -# Generate the trusted operator and memory resolver configuration. -# This contains public operator/account JWT claims, not the private seeds. -nsc generate config \ - --mem-resolver \ - --config-file "$AUTH_CONFIG_FILE" \ - --force - -# Generate the primary NATS server configuration. -# The include path matches the Docker Compose mounts shown below. -cat >"$SERVER_CONFIG_FILE" <<'NATS_CONFIG' -server_name: localai-nats -port: 4222 -http: 8222 - -jetstream { - store_dir: /data/jetstream -} - -include nats-auth.conf -NATS_CONFIG - -# Generate the environment file consumed by the LocalAI frontend. -cat >"$ENV_FILE" <=", 2)) - - eventMu.Lock() - defer eventMu.Unlock() - - var hasAgentMessage, hasCompleted bool - for _, evt := range receivedEvents { - if evt.EventType == "json_message" && evt.Sender == "agent" { - hasAgentMessage = true - Expect(evt.Content).To(ContainSubstring("systems operational")) - } - if evt.EventType == "json_message_status" && evt.Metadata != "" { - var meta map[string]string - json.Unmarshal([]byte(evt.Metadata), &meta) - if meta["status"] == "completed" { - hasCompleted = true + for _, evt := range receivedEvents { + if evt.EventType == "json_message" && evt.Sender == "agent" { + agentMessage = true + Expect(evt.Content).To(ContainSubstring("systems operational")) + } + if evt.EventType == "json_message_status" && evt.Metadata != "" { + var meta map[string]string + // A metadata blob this spec cannot read is not a + // completed status, and saying so beats failing the + // whole run on one malformed event. + if err := json.Unmarshal([]byte(evt.Metadata), &meta); err == nil && meta["status"] == "completed" { + completed = true + } } } + return agentMessage, completed } - Expect(hasAgentMessage).To(BeTrue(), "background agent should produce a response via EventBridge") - Expect(hasCompleted).To(BeTrue(), "background agent should complete") + + Eventually(func() bool { _, completed := seen(); return completed }, "15s"). + Should(BeTrue(), "background agent should complete") + agentMessage, _ := seen() + Expect(agentMessage).To(BeTrue(), "background agent should produce a response via EventBridge") }) }) diff --git a/tests/e2e/distributed/cluster_baseline_test.go b/tests/e2e/distributed/cluster_baseline_test.go index 6f69898c23a1..e6cb851c677a 100644 --- a/tests/e2e/distributed/cluster_baseline_test.go +++ b/tests/e2e/distributed/cluster_baseline_test.go @@ -288,10 +288,16 @@ func startClusterOnFreshDB(frontends, workers int, customise ...func(*cluster.Op Binary: binary, MockBackend: mockBackend, PGDSN: infra.PGURL, - NatsURL: infra.NatsURL, - LogDir: logDir, - Frontends: frontends, - Workers: workers, + // Deliberately a dead address. Frontends and agent workers are still + // handed LOCALAI_NATS_URL so this suite keeps covering the promise that + // an operator's existing command line starts unchanged after the broker + // is shut down. Pointing it at a running server would make a regression + // that dialled it invisible; pointing it at nothing makes such a + // regression a startup failure in every cluster spec. + NatsURL: staleBusURL, + LogDir: logDir, + Frontends: frontends, + Workers: workers, } for _, apply := range customise { apply(&options) diff --git a/tests/e2e/distributed/cluster_control_test.go b/tests/e2e/distributed/cluster_control_test.go index 9daad1b5c2bb..c6f213f83c15 100644 --- a/tests/e2e/distributed/cluster_control_test.go +++ b/tests/e2e/distributed/cluster_control_test.go @@ -413,8 +413,11 @@ var _ = Describe("Control plane over the worker tunnel", Label("Distributed"), L Expect(entry).ToNot(HavePrefix("LOCALAI_NATS_URL="), "the worker was handed a bus URL, so this spec is not about a worker that has none") } - // And the deployment it joined DOES have a bus, so "no NATS anywhere" - // is not what makes this pass. + // And the harness DOES still hand a LOCALAI_NATS_URL to the other + // processes in this cluster, so the worker's lack of one is a property + // of the worker and not of a harness that stopped setting the variable + // at all. There is no broker behind that URL any more, which is the + // point: nothing dials it, so nothing notices. Expect(c.NatsURL()).ToNot(BeEmpty()) probe := newRosterProbe(c, client, 0) diff --git a/tests/e2e/distributed/foundation_test.go b/tests/e2e/distributed/foundation_test.go index c96bd71e54e0..cca2e271f053 100644 --- a/tests/e2e/distributed/foundation_test.go +++ b/tests/e2e/distributed/foundation_test.go @@ -30,7 +30,6 @@ var _ = Describe("Phase 0: Foundation", Label("Distributed"), func() { It("should reject --distributed without PostgreSQL configured", func() { appCfg := config.NewApplicationConfig( config.EnableDistributed, - config.WithNatsURL(infra.NatsURL), // No auth/PostgreSQL configured ) Expect(appCfg.Distributed.Enabled).To(BeTrue()) @@ -38,26 +37,30 @@ var _ = Describe("Phase 0: Foundation", Label("Distributed"), func() { Expect(appCfg.Auth.Enabled).To(BeFalse()) }) - It("should reject --distributed without NATS configured", func() { + It("leaves the inert bus URL empty when nothing sets it", func() { appCfg := config.NewApplicationConfig( config.EnableDistributed, config.WithAuthEnabled(true), config.WithAuthDatabaseURL(infra.PGURL), - // No NATS URL ) Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) }) It("should accept valid distributed configuration", func() { + // staleBusURL points at nothing on purpose. It is the shape of an + // operator's existing command line after the broker was shut down, + // and the promise this spec holds is that such a command line still + // STARTS: the value is parsed, stored and never dialled. Pointing + // it at a live server would let a regression that dialled it pass. appCfg := config.NewApplicationConfig( config.EnableDistributed, config.WithAuthEnabled(true), config.WithAuthDatabaseURL(infra.PGURL), - config.WithNatsURL(infra.NatsURL), + config.WithNatsURL(staleBusURL), ) Expect(appCfg.Distributed.Enabled).To(BeTrue()) Expect(appCfg.Auth.Enabled).To(BeTrue()) - Expect(appCfg.Distributed.NatsURL).To(Equal(infra.NatsURL)) + Expect(appCfg.Distributed.NatsURL).To(Equal(staleBusURL)) }) It("should generate unique frontend ID on startup", func() { diff --git a/tests/e2e/distributed/gallery_distributed_test.go b/tests/e2e/distributed/gallery_distributed_test.go index b6bf9a6e2edc..b5b4294e4dcd 100644 --- a/tests/e2e/distributed/gallery_distributed_test.go +++ b/tests/e2e/distributed/gallery_distributed_test.go @@ -70,8 +70,9 @@ var _ = Describe("Gallery Distributed", Label("Distributed"), func() { // The gallery families ride the broadcast carrier, not NATS. // - // These used to publish and subscribe on infra.NC, which asserted that NATS - // delivers to itself and nothing about this deployment: they would have + // These used to publish and subscribe on a message-bus client, which + // asserted that the bus delivers to itself and nothing about this + // deployment: they would have // stayed green through the whole migration while the gallery service had // already moved. Two carriers on the deployment's own database is the shape // a fleet has, and it is the shape that fails when one end moves and the diff --git a/tests/e2e/distributed/mcp_ci_job_helper_test.go b/tests/e2e/distributed/mcp_ci_job_helper_test.go index 27370b4768dc..9210c1f621e2 100644 --- a/tests/e2e/distributed/mcp_ci_job_helper_test.go +++ b/tests/e2e/distributed/mcp_ci_job_helper_test.go @@ -16,10 +16,20 @@ import ( "github.com/mudler/xlog" ) +// publishTestEvent is the one place this helper publishes from, so a carrier +// that refuses a publish is reported rather than swallowed. A dropped progress +// line has no other symptom: the spec's subscriber simply never fires and the +// failure surfaces as a timeout that names the wrong thing. +func publishTestEvent(pub messaging.Publisher, subject string, payload any) { + if err := pub.Publish(subject, payload); err != nil { + xlog.Error("test worker failed to publish", "subject", subject, "error", err) + } +} + // processMCPCIJobForTest replicates the logic of handleMCPCIJob from agent_worker.go // for testing purposes. This allows e2e testing of the full MCP CI job execution path // without needing to start an actual agent worker binary. -func processMCPCIJobForTest(data []byte, apiURL, apiToken string, natsClient *messaging.Client) { +func processMCPCIJobForTest(data []byte, apiURL, apiToken string, pub messaging.Publisher) { var evt jobs.JobEvent if err := json.Unmarshal(data, &evt); err != nil { xlog.Error("Failed to unmarshal job event", "error", err) @@ -30,30 +40,30 @@ func processMCPCIJobForTest(data []byte, apiURL, apiToken string, natsClient *me task := evt.Task if job == nil || task == nil { xlog.Error("MCP CI job missing enriched data", "jobID", evt.JobID) - publishTestJobResult(natsClient, evt.JobID, "failed", "", "job or task data missing from NATS event") + publishTestJobResult(pub, evt.JobID, "failed", "", "job or task data missing from the job event") return } modelCfg := evt.ModelConfig if modelCfg == nil { - publishTestJobResult(natsClient, evt.JobID, "failed", "", "model config missing from job event") + publishTestJobResult(pub, evt.JobID, "failed", "", "model config missing from job event") return } // Publish running status - natsClient.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ + publishTestEvent(pub, messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ JobID: evt.JobID, Status: "running", Message: "Job started on test worker", }) // Parse MCP config if modelCfg.MCP.Servers == "" && modelCfg.MCP.Stdio == "" { - publishTestJobResult(natsClient, evt.JobID, "failed", "", "no MCP servers configured for model") + publishTestJobResult(pub, evt.JobID, "failed", "", "no MCP servers configured for model") return } remote, stdio, err := modelCfg.MCP.MCPConfigFromYAML() if err != nil { - publishTestJobResult(natsClient, evt.JobID, "failed", "", fmt.Sprintf("failed to parse MCP config: %v", err)) + publishTestJobResult(pub, evt.JobID, "failed", "", fmt.Sprintf("failed to parse MCP config: %v", err)) return } @@ -64,7 +74,7 @@ func processMCPCIJobForTest(data []byte, apiURL, apiToken string, natsClient *me if err != nil { errMsg = fmt.Sprintf("failed to create MCP sessions: %v", err) } - publishTestJobResult(natsClient, evt.JobID, "failed", "", errMsg) + publishTestJobResult(pub, evt.JobID, "failed", "", errMsg) return } @@ -92,7 +102,7 @@ func processMCPCIJobForTest(data []byte, apiURL, apiToken string, natsClient *me defer cancel() // Publish running status - publishTestJobStatus(natsClient, evt.JobID, "running", "") + publishTestJobStatus(pub, evt.JobID, "running", "") // Buffer stream tokens var reasoningBuf, contentBuf strings.Builder @@ -100,13 +110,13 @@ func processMCPCIJobForTest(data []byte, apiURL, apiToken string, natsClient *me flushStreamBuf := func() { if reasoningBuf.Len() > 0 { - natsClient.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ + publishTestEvent(pub, messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ JobID: evt.JobID, TraceType: "reasoning", TraceContent: reasoningBuf.String(), }) reasoningBuf.Reset() } if contentBuf.Len() > 0 { - natsClient.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ + publishTestEvent(pub, messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ JobID: evt.JobID, TraceType: "content", TraceContent: contentBuf.String(), }) contentBuf.Reset() @@ -119,13 +129,13 @@ func processMCPCIJobForTest(data []byte, apiURL, apiToken string, natsClient *me cogito.WithMCPs(sessions...), cogito.WithStatusCallback(func(status string) { flushStreamBuf() - natsClient.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ + publishTestEvent(pub, messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ JobID: evt.JobID, TraceType: "status", TraceContent: status, }) }), cogito.WithToolCallResultCallback(func(t cogito.ToolStatus) { flushStreamBuf() - natsClient.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ + publishTestEvent(pub, messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ JobID: evt.JobID, TraceType: "tool_result", TraceContent: fmt.Sprintf("%s: %s", t.Name, t.Result), }) }), @@ -140,7 +150,7 @@ func processMCPCIJobForTest(data []byte, apiURL, apiToken string, natsClient *me case cogito.StreamEventContent: contentBuf.WriteString(ev.Content) case cogito.StreamEventToolCall: - natsClient.Publish(messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ + publishTestEvent(pub, messaging.SubjectJobProgress(evt.JobID), jobs.ProgressEvent{ JobID: evt.JobID, TraceType: "tool_call", TraceContent: fmt.Sprintf("%s(%s)", ev.ToolName, ev.ToolArgs), }) } @@ -155,7 +165,7 @@ func processMCPCIJobForTest(data []byte, apiURL, apiToken string, natsClient *me flushStreamBuf() if err != nil { - publishTestJobResult(natsClient, evt.JobID, "failed", "", fmt.Sprintf("cogito execution failed: %v", err)) + publishTestJobResult(pub, evt.JobID, "failed", "", fmt.Sprintf("cogito execution failed: %v", err)) return } @@ -163,27 +173,27 @@ func processMCPCIJobForTest(data []byte, apiURL, apiToken string, natsClient *me if msg := f.LastMessage(); msg != nil { result = msg.Content } - publishTestJobResult(natsClient, evt.JobID, "completed", result, "") + publishTestJobResult(pub, evt.JobID, "completed", result, "") } -func publishTestJobStatus(nc *messaging.Client, jobID, status, message string) { - nc.Publish(messaging.SubjectJobResult(jobID), jobs.JobResultEvent{ +func publishTestJobStatus(pub messaging.Publisher, jobID, status, message string) { + publishTestEvent(pub, messaging.SubjectJobResult(jobID), jobs.JobResultEvent{ JobID: jobID, Status: status, }) - nc.Publish(messaging.SubjectJobProgress(jobID), jobs.ProgressEvent{ + publishTestEvent(pub, messaging.SubjectJobProgress(jobID), jobs.ProgressEvent{ JobID: jobID, Status: status, Message: message, }) } -func publishTestJobResult(nc *messaging.Client, jobID, status, result, errMsg string) { - nc.Publish(messaging.SubjectJobResult(jobID), jobs.JobResultEvent{ +func publishTestJobResult(pub messaging.Publisher, jobID, status, result, errMsg string) { + publishTestEvent(pub, messaging.SubjectJobResult(jobID), jobs.JobResultEvent{ JobID: jobID, Status: status, Result: result, Error: errMsg, }) - nc.Publish(messaging.SubjectJobProgress(jobID), jobs.ProgressEvent{ + publishTestEvent(pub, messaging.SubjectJobProgress(jobID), jobs.ProgressEvent{ JobID: jobID, Status: status, Message: errMsg, }) } diff --git a/tests/e2e/distributed/mcp_ci_job_test.go b/tests/e2e/distributed/mcp_ci_job_test.go index 62c696850c77..638d4397af43 100644 --- a/tests/e2e/distributed/mcp_ci_job_test.go +++ b/tests/e2e/distributed/mcp_ci_job_test.go @@ -280,10 +280,18 @@ func startMockLLMServer() (string, func()) { var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func() { var ( infra *TestInfra + bus messaging.Broadcaster ) + // One carrier per spec, on THIS spec's database. It is the PostgreSQL + // LISTEN/NOTIFY carrier the deployment runs, and it is what the SSE routes + // read: a spec that published its progress and result lines onto a message + // bus instead would keep passing after that carrier had gone silent, + // because a publish onto the wrong carrier succeeds and simply arrives + // nowhere. BeforeEach(func() { - infra = SetupNATSOnly() + infra = SetupInfra("mcp_ci_job") + bus = infra.Bus() }) Context("Full MCP CI Job Flow", func() { @@ -303,7 +311,7 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( var resultEvent *jobs.JobResultEvent var eventMu sync.Mutex - progressSub, err := infra.NC.Subscribe(messaging.SubjectJobProgress(jobID), func(data []byte) { + progressSub, err := bus.Subscribe(messaging.SubjectJobProgress(jobID), func(data []byte) { var evt jobs.ProgressEvent if json.Unmarshal(data, &evt) == nil { eventMu.Lock() @@ -314,7 +322,7 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( Expect(err).ToNot(HaveOccurred()) defer progressSub.Unsubscribe() - resultSub, err := infra.NC.Subscribe(messaging.SubjectJobResult(jobID), func(data []byte) { + resultSub, err := bus.Subscribe(messaging.SubjectJobResult(jobID), func(data []byte) { var evt jobs.JobResultEvent if json.Unmarshal(data, &evt) == nil { eventMu.Lock() @@ -325,8 +333,6 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( Expect(err).ToNot(HaveOccurred()) defer resultSub.Unsubscribe() - FlushNATS(infra.NC) - // Build MCP config YAML pointing to mock MCP server mcpRemoteJSON := fmt.Sprintf(`{"mcpServers":{"weather-api":{"url":"%s"}}}`, mcpURL) modelCfg := &config.ModelConfig{ @@ -364,9 +370,7 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( raw, err := json.Marshal(evt) Expect(err).ToNot(HaveOccurred()) - FlushNATS(infra.NC) - - go processMCPCIJobForTest(raw, llmURL, "test-token", infra.NC) + go processMCPCIJobForTest(raw, llmURL, "test-token", bus) // Wait for result Eventually(func() bool { @@ -404,7 +408,7 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( var resultEvent *jobs.JobResultEvent var eventMu sync.Mutex - resultSub, err := infra.NC.Subscribe(messaging.SubjectJobResult(jobID), func(data []byte) { + resultSub, err := bus.Subscribe(messaging.SubjectJobResult(jobID), func(data []byte) { var evt jobs.JobResultEvent if json.Unmarshal(data, &evt) == nil { eventMu.Lock() @@ -415,8 +419,6 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( Expect(err).ToNot(HaveOccurred()) defer resultSub.Unsubscribe() - FlushNATS(infra.NC) - // MCP config pointing to unreachable server mcpRemoteJSON := `{"mcpServers":{"bad-server":{"url":"http://127.0.0.1:1/mcp"}}}` modelCfg := &config.ModelConfig{ @@ -448,7 +450,7 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( // Process directly (no worker subscription needed) evtData, _ := json.Marshal(evt) - go processMCPCIJobForTest(evtData, "http://localhost:9999", "token", infra.NC) + go processMCPCIJobForTest(evtData, "http://localhost:9999", "token", bus) // Wait for failure result Eventually(func() bool { @@ -477,7 +479,7 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( var resultEvent *jobs.JobResultEvent var eventMu sync.Mutex - resultSub, err := infra.NC.Subscribe(messaging.SubjectJobResult(jobID), func(data []byte) { + resultSub, err := bus.Subscribe(messaging.SubjectJobResult(jobID), func(data []byte) { var evt jobs.JobResultEvent if json.Unmarshal(data, &evt) == nil { eventMu.Lock() @@ -488,8 +490,6 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( Expect(err).ToNot(HaveOccurred()) defer resultSub.Unsubscribe() - FlushNATS(infra.NC) - mcpRemoteJSON := fmt.Sprintf(`{"mcpServers":{"weather-api":{"url":"%s"}}}`, mcpURL) modelCfg := &config.ModelConfig{ MCP: config.MCPConfig{ @@ -523,7 +523,7 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( } evtData, _ := json.Marshal(evt) - go processMCPCIJobForTest(evtData, llmURL, "test-token", infra.NC) + go processMCPCIJobForTest(evtData, llmURL, "test-token", bus) Eventually(func() bool { eventMu.Lock() @@ -545,7 +545,7 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( var resultEvent *jobs.JobResultEvent var eventMu sync.Mutex - resultSub, err := infra.NC.Subscribe(messaging.SubjectJobResult(jobID), func(data []byte) { + resultSub, err := bus.Subscribe(messaging.SubjectJobResult(jobID), func(data []byte) { var evt jobs.JobResultEvent if json.Unmarshal(data, &evt) == nil { eventMu.Lock() @@ -556,8 +556,6 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( Expect(err).ToNot(HaveOccurred()) defer resultSub.Unsubscribe() - FlushNATS(infra.NC) - // Event with no Job or Task evt := jobs.JobEvent{ JobID: jobID, @@ -566,7 +564,7 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( } evtData, _ := json.Marshal(evt) - go processMCPCIJobForTest(evtData, "http://localhost:9999", "token", infra.NC) + go processMCPCIJobForTest(evtData, "http://localhost:9999", "token", bus) Eventually(func() bool { eventMu.Lock() @@ -586,7 +584,7 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( var resultEvent *jobs.JobResultEvent var eventMu sync.Mutex - resultSub, err := infra.NC.Subscribe(messaging.SubjectJobResult(jobID), func(data []byte) { + resultSub, err := bus.Subscribe(messaging.SubjectJobResult(jobID), func(data []byte) { var evt jobs.JobResultEvent if json.Unmarshal(data, &evt) == nil { eventMu.Lock() @@ -597,8 +595,6 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( Expect(err).ToNot(HaveOccurred()) defer resultSub.Unsubscribe() - FlushNATS(infra.NC) - // ModelConfig with empty MCP modelCfg := &config.ModelConfig{} modelCfg.Name = "no-mcp-model" @@ -624,7 +620,7 @@ var _ = Describe("MCP CI Job Execution", Label("Distributed", "MCPCIJob"), func( } evtData, _ := json.Marshal(evt) - go processMCPCIJobForTest(evtData, "http://localhost:9999", "token", infra.NC) + go processMCPCIJobForTest(evtData, "http://localhost:9999", "token", bus) Eventually(func() bool { eventMu.Lock() diff --git a/tests/e2e/distributed/nats_jwt_helpers_test.go b/tests/e2e/distributed/nats_jwt_helpers_test.go deleted file mode 100644 index 74f74355d098..000000000000 --- a/tests/e2e/distributed/nats_jwt_helpers_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package distributed_test - -import ( - "bytes" - "context" - "fmt" - "strings" - "time" - - "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/pkg/natsauth" - "github.com/nats-io/jwt/v2" - "github.com/nats-io/nkeys" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/testcontainers/testcontainers-go" - tcnats "github.com/testcontainers/testcontainers-go/modules/nats" -) - -// JWTTestInfra holds a NATS server configured with JWT auth and minted worker credentials. -type JWTTestInfra struct { - *TestInfra - AccountSeed string - NodeID string - WorkerJWT string - WorkerSeed string -} - -// SetupJWTInfra starts NATS with an in-memory JWT resolver and returns worker credentials -// minted the same way as node registration (pkg/natsauth). -func SetupJWTInfra() *JWTTestInfra { - GinkgoHelper() - - infra := &JWTTestInfra{TestInfra: &TestInfra{Ctx: context.Background()}} - - operatorJWT, accountJWT, accountSeed, err := jwtResolverMaterial() - Expect(err).ToNot(HaveOccurred()) - infra.AccountSeed = accountSeed - - conf := fmt.Sprintf(`listen: 0.0.0.0:4222 - -operator: %s - -resolver: MEMORY -resolver_preload: { - %s: %s -} -`, operatorJWT, accountPublicKeyFromSeed(accountSeed), accountJWT) - - var natsContainer *tcnats.NATSContainer - // Override default testcontainers -js: JetStream fails without a system account in JWT mode. - natsContainer, err = tcnats.Run(infra.Ctx, "nats:2-alpine", - tcnats.WithConfigFile(bytes.NewBufferString(conf)), - testcontainers.WithCmd("-c", "/etc/nats.conf"), - ) - Expect(err).ToNot(HaveOccurred()) - infra.NATSContainer = natsContainer - - infra.NatsURL, err = infra.NATSContainer.ConnectionString(infra.Ctx) - Expect(err).ToNot(HaveOccurred()) - - infra.NodeID = "550e8400-e29b-41d4-a716-446655440000" - cfg := natsauth.Config{AccountSeed: infra.AccountSeed, WorkerJWTTTL: time.Hour} - infra.WorkerJWT, infra.WorkerSeed, err = cfg.MintWorkerJWT(infra.NodeID, "backend") - Expect(err).ToNot(HaveOccurred()) - - infra.NC, err = messaging.New(infra.NatsURL, messaging.WithUserJWT(infra.WorkerJWT, infra.WorkerSeed)) - Expect(err).ToNot(HaveOccurred()) - - DeferCleanup(func() { - if infra.NC != nil { - infra.NC.Close() - } - if infra.NATSContainer != nil { - _ = infra.NATSContainer.Terminate(context.Background()) - } - }) - - return infra -} - -// jwtResolverMaterial builds operator + account JWTs for a MEMORY resolver. -// Follows the NATS JWT tutorial: self-signed account, then operator re-sign, with the -// account identity key listed as a signing key so MintWorkerJWT can use the account seed. -func jwtResolverMaterial() (operatorJWT, accountJWT, accountSeed string, err error) { - okp, err := nkeys.CreateOperator() - if err != nil { - return "", "", "", err - } - opk, err := okp.PublicKey() - if err != nil { - return "", "", "", err - } - oc := jwt.NewOperatorClaims(opk) - oc.Name = "localai-test-operator" - oskp, err := nkeys.CreateOperator() - if err != nil { - return "", "", "", err - } - ospk, err := oskp.PublicKey() - if err != nil { - return "", "", "", err - } - oc.SigningKeys.Add(ospk) - operatorJWT, err = oc.Encode(okp) - if err != nil { - return "", "", "", err - } - - akp, err := nkeys.CreateAccount() - if err != nil { - return "", "", "", err - } - seed, err := akp.Seed() - if err != nil { - return "", "", "", err - } - accountSeed = string(seed) - - apk, err := akp.PublicKey() - if err != nil { - return "", "", "", err - } - ac := jwt.NewAccountClaims(apk) - ac.Name = "localai-test-account" - ac.SigningKeys.Add(apk) - accountJWT, err = ac.Encode(akp) - if err != nil { - return "", "", "", err - } - ac, err = jwt.DecodeAccountClaims(accountJWT) - if err != nil { - return "", "", "", err - } - accountJWT, err = ac.Encode(oskp) - if err != nil { - return "", "", "", err - } - return operatorJWT, accountJWT, accountSeed, nil -} - -func accountPublicKeyFromSeed(accountSeed string) string { - akp, err := nkeys.FromSeed([]byte(accountSeed)) - Expect(err).ToNot(HaveOccurred()) - pk, err := akp.PublicKey() - Expect(err).ToNot(HaveOccurred()) - return pk -} - -// nodeSubjectPrefix returns the sanitized nodes.* prefix for a node ID. -func nodeSubjectPrefix(nodeID string) string { - tok := strings.NewReplacer(".", "-", "*", "-", ">", "-", " ", "-", "\t", "-", "\n", "-").Replace(nodeID) - return "nodes." + tok -} diff --git a/tests/e2e/distributed/nats_jwt_test.go b/tests/e2e/distributed/nats_jwt_test.go deleted file mode 100644 index 46444c197ce2..000000000000 --- a/tests/e2e/distributed/nats_jwt_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package distributed_test - -import ( - "time" - - "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/pkg/natsauth" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("NATS JWT Auth", Label("Distributed", "NatsJWT"), func() { - var infra *JWTTestInfra - - BeforeEach(func() { - infra = SetupJWTInfra() - }) - - It("connects with a minted backend worker JWT and publishes on its one remaining allowed subject", func() { - // A backend worker's whole grant is `_INBOX.>` now, on both sides. - // Every verb a frontend gives it, file staging included, is an HTTP - // route on its tunnel, and it no longer opens a bus connection at all; - // the JWT is minted and unused. See pkg/natsauth.WorkerPermissions. - Expect(infra.NC.Publish("_INBOX.probe", map[string]string{"path": "/tmp/model"})).To(Succeed()) - // ConfirmRoundTrip is the flush AND the server's verdict in one call. - // Read separately they were two assertions that could drift apart; the - // verdict is the one that matters, because a permission violation does - // not close the connection. - Expect(infra.NC.ConfirmRoundTrip(2 * time.Second)).To(Succeed()) - Expect(infra.NC.IsConnected()).To(BeTrue()) - }) - - It("denies a backend worker the file-staging subjects it no longer serves", func() { - // This spec used to assert the OPPOSITE, and kept passing after the - // grant was deleted. A NATS permission violation does not close the - // connection, so a spec that checks only FlushTimeout and IsConnected - // cannot tell an allowed publish from a denied one; LastError is what - // actually reads the server's verdict, which is why the sibling below - // has always used it. - subject := nodeSubjectPrefix(infra.NodeID) + ".files.stage" - Expect(infra.NC.Publish(subject, map[string]string{"path": "/tmp/model"})).To(Succeed()) - Eventually(func() error { - return infra.NC.ConfirmRoundTrip(500 * time.Millisecond) - }, "3s", "50ms").Should(HaveOccurred()) - }) - - It("denies backend subscribe on the node prefix it no longer listens to", func() { - // The node subtree was granted while a backend worker still held a - // connection with nothing under it subscribed. It does not hold one at - // all now, so the grant went too; asserting the denial is what would - // catch a subject quietly coming back to the bus. - wild := nodeSubjectPrefix(infra.NodeID) + ".>" - sub, err := infra.NC.Subscribe(wild, func(_ []byte) {}) - if err == nil { - defer func() { _ = sub.Unsubscribe() }() - Eventually(func() error { - return infra.NC.ConfirmRoundTrip(500 * time.Millisecond) - }, "3s", "50ms").Should(HaveOccurred()) - } - }) - - It("rejects anonymous publish on the JWT-enabled server", func() { - anon, err := messaging.New(infra.NatsURL) - Expect(err).ToNot(HaveOccurred()) - defer anon.Close() - - err = anon.Publish("nodes.any.files.x", map[string]string{"x": "1"}) - Expect(err).ToNot(HaveOccurred()) - Expect(anon.ConfirmRoundTrip(2 * time.Second)).To(HaveOccurred()) - }) - - It("denies backend publish to another node's subjects", func() { - other := nodeSubjectPrefix("other-node-id") + ".files.stage" - Expect(infra.NC.Publish(other, map[string]string{"stage": "nope"})).To(Succeed()) - Eventually(func() error { - return infra.NC.ConfirmRoundTrip(500 * time.Millisecond) - }, "3s", "50ms").Should(HaveOccurred()) - }) - - It("mints agent JWT without backend.install in claims", func() { - cfg := natsauth.Config{AccountSeed: infra.AccountSeed} - token, _, err := cfg.MintWorkerJWT("agent-node-1", "agent") - Expect(err).ToNot(HaveOccurred()) - - claims, err := natsauth.DecodeUserClaims(token) - Expect(err).ToNot(HaveOccurred()) - // agent.execute has left this list: agent execution is a streaming - // control verb on the tunnel now, driven by a claim a frontend replica - // took off the job store. What must still be here is the cancel - // broadcast, which cannot become an RPC. - Expect(claims.Permissions.Sub.Allow).To(ContainElement("agent.*.cancel")) - Expect(claims.Permissions.Sub.Allow).ToNot(ContainElement("agent.execute")) - for _, subj := range claims.Permissions.Sub.Allow { - Expect(subj).NotTo(ContainSubstring("backend.install")) - } - }) - - // Regression guard for the silent permission gaps: decoding the JWT claims - // (above) only proves the agent JWT is *restrictive*, not that it is - // *sufficient*. Stand a real agent connection up against the enforcing - // server and exercise every subscription core/cli/agent_worker.go actually - // makes — a denied SUB now surfaces synchronously via confirmSubscription, - // so a missing allow rule fails this test instead of silently dropping - // backend.stop / MCP-CI deliveries at runtime. - It("lets an agent-minted JWT establish all the subscriptions the agent worker uses", func() { - const nodeID = "agent-node-subs" - cfg := natsauth.Config{AccountSeed: infra.AccountSeed, WorkerJWTTTL: time.Hour} - token, seed, err := cfg.MintWorkerJWT(nodeID, "agent") - Expect(err).ToNot(HaveOccurred()) - - nc, err := messaging.New(infra.NatsURL, messaging.WithUserJWT(token, seed)) - Expect(err).ToNot(HaveOccurred()) - DeferCleanup(nc.Close) - - // Mirror core/cli/agent_worker.go exactly. MCP tool execution and - // discovery are absent, so is the per-node backend.stop, and so now are - // agent execution and MCP CI runs: none of them is a bus subject any - // more. The frontend selects an agent worker itself and reaches it with - // a control RPC over the tunnel that worker holds, and the work it - // hands over is a row it claimed on the job store. - // - // What is left is the cancel broadcast, which cannot become an RPC: the - // replica holding a run is not the one an API cancel lands on. - _, err = nc.Subscribe(messaging.SubjectAgentCancelWildcard, func([]byte) {}) - Expect(err).ToNot(HaveOccurred(), "agent JWT must allow %s (cancellation)", messaging.SubjectAgentCancelWildcard) - - _, err = nc.Subscribe(messaging.SubjectJobProgressWildcard, func([]byte) {}) - Expect(err).ToNot(HaveOccurred(), "agent JWT must allow %s (progress bridging)", messaging.SubjectJobProgressWildcard) - }) - - // The narrowing, proved against the enforcing server rather than against - // the allow list that feeds it. The subject is written out by hand because - // its builder is deleted; that literal is what a worker from an older - // release would still send, and this is what the server now answers it. - // - // It is a narrowing and not a lockout: the two subscriptions above are made - // on a JWT minted the same way and both succeed, so the list this trims is - // demonstrably not the empty one NATS would read as unrestricted. - It("refuses an agent-minted JWT the retired per-node backend.stop subject", func() { - const nodeID = "agent-node-stop" - cfg := natsauth.Config{AccountSeed: infra.AccountSeed, WorkerJWTTTL: time.Hour} - token, seed, err := cfg.MintWorkerJWT(nodeID, "agent") - Expect(err).ToNot(HaveOccurred()) - - nc, err := messaging.New(infra.NatsURL, messaging.WithUserJWT(token, seed)) - Expect(err).ToNot(HaveOccurred()) - DeferCleanup(nc.Close) - - _, err = nc.Subscribe("nodes."+nodeID+".backend.stop", func([]byte) {}) - Expect(err).To(HaveOccurred(), - "backend.stop is a control RPC on the worker's tunnel; the bus must not carry it") - }) - - // The same narrowing for the two queue subjects that became claim rows. - // Written out by hand for the same reason: those literals are what a worker - // from an older release would still subscribe to, and this is what the - // server now answers it. - DescribeTable("refuses an agent-minted JWT a retired queue subject", - func(subject string) { - cfg := natsauth.Config{AccountSeed: infra.AccountSeed, WorkerJWTTTL: time.Hour} - token, seed, err := cfg.MintWorkerJWT("agent-node-queues", "agent") - Expect(err).ToNot(HaveOccurred()) - - nc, err := messaging.New(infra.NatsURL, messaging.WithUserJWT(token, seed)) - Expect(err).ToNot(HaveOccurred()) - DeferCleanup(nc.Close) - - _, err = nc.Subscribe(subject, func([]byte) {}) - Expect(err).To(HaveOccurred(), - "%s became a claim on the job store; the bus must not carry it", subject) - }, - Entry("agent execution", "agent.execute"), - Entry("mcp ci jobs", "jobs.mcp-ci.new"), - ) -}) diff --git a/tests/e2e/distributed/testhelpers_test.go b/tests/e2e/distributed/testhelpers_test.go index 84b2f011281a..0508f067528c 100644 --- a/tests/e2e/distributed/testhelpers_test.go +++ b/tests/e2e/distributed/testhelpers_test.go @@ -8,14 +8,12 @@ import ( "sync/atomic" "time" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/pgbus" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/testcontainers/testcontainers-go" - tcnats "github.com/testcontainers/testcontainers-go/modules/nats" tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" "github.com/testcontainers/testcontainers-go/wait" "gorm.io/driver/postgres" @@ -23,37 +21,44 @@ import ( gormlogger "gorm.io/gorm/logger" ) -// TestInfra holds shared test containers and connection strings. +// TestInfra holds the shared test container and the connection strings derived +// from it. // -// PGContainer and NATSContainer are the SUITE-WIDE containers, shared by every -// spec. Never call Terminate or Stop on them from a spec: it ends the run for -// everything after it. They are exposed only because nats_jwt_helpers_test.go -// builds its own TestInfra around a dedicated NATS container. +// PGContainer is the SUITE-WIDE container, shared by every spec. Never call +// Terminate or Stop on it from a spec: it ends the run for everything after it. +// +// There is one container and not two. Every carrier these specs exercise is a +// PostgreSQL LISTEN/NOTIFY channel and every worker verb is an HTTP route on a +// tunnel, so a message broker in this suite would be infrastructure no spec and +// no production path can reach. type TestInfra struct { - Ctx context.Context - PGContainer *tcpostgres.PostgresContainer - NATSContainer *tcnats.NATSContainer - PGURL string - NatsURL string - NC *messaging.Client + Ctx context.Context + PGContainer *tcpostgres.PostgresContainer + PGURL string } -// Containers are suite-scoped, not spec-scoped. Starting a Postgres (~10s) and a -// NATS (~3.5s) per spec cost roughly 48 minutes of pure startup across the 213 -// specs behind SetupInfra, which is why this suite was never wired into CI. -// Isolation now comes from a database per spec (~67ms), which is what the dbName -// argument was always describing. +// staleBusURL is the address of a broker that is not running, and is not meant +// to be. LOCALAI_NATS_URL and --nats-url are still accepted and ignored so an +// operator's existing command line, unit file or Helm values file starts +// unchanged after the broker is shut down; specs that exercise that promise +// hand over THIS value, because a value pointing at a live server would let a +// regression that dialled it pass unnoticed. +const staleBusURL = "nats://127.0.0.1:1" + +// The container is suite-scoped, not spec-scoped. Starting a Postgres (~10s) per +// spec cost roughly 36 minutes of pure startup across the 213 specs behind +// SetupInfra, which is why this suite was never wired into CI. Isolation now +// comes from a database per spec (~67ms), which is what the dbName argument was +// always describing. // // Plain BeforeSuite rather than SynchronizedBeforeSuite is deliberate: under -// `ginkgo -p` each process gets its own container pair, which keeps NATS subjects -// isolated per process. A single shared NATS across parallel processes would let -// specs on different processes see each other's messages on the same subject. +// `ginkgo -p` each process gets its own container, and a database per spec on +// top of that keeps two processes from reading each other's notifications on a +// channel of the same name. var ( - suitePG *tcpostgres.PostgresContainer - suiteNATS *tcnats.NATSContainer - suitePGDSN string - suiteNatsURL string - dbCounter atomic.Int64 + suitePG *tcpostgres.PostgresContainer + suitePGDSN string + dbCounter atomic.Int64 ) var _ = BeforeSuite(func() { @@ -74,12 +79,6 @@ var _ = BeforeSuite(func() { suitePGDSN, err = suitePG.ConnectionString(ctx, "sslmode=disable") Expect(err).ToNot(HaveOccurred()) - - suiteNATS, err = tcnats.Run(ctx, "nats:2-alpine") - Expect(err).ToNot(HaveOccurred()) - - suiteNatsURL, err = suiteNATS.ConnectionString(ctx) - Expect(err).ToNot(HaveOccurred()) }) var _ = AfterSuite(func() { @@ -87,9 +86,6 @@ var _ = AfterSuite(func() { if suitePG != nil { _ = suitePG.Terminate(ctx) } - if suiteNATS != nil { - _ = suiteNATS.Terminate(ctx) - } }) // sanitizeDBName maps a spec-supplied label onto a legal unquoted Postgres @@ -156,18 +152,16 @@ func closeDB(db *gorm.DB) { } } -// SetupInfra provisions a dedicated database on the suite-scoped Postgres and -// returns a client connected to the suite-scoped NATS. Call in BeforeEach; -// cleanup is registered with DeferCleanup. +// SetupInfra provisions a dedicated database on the suite-scoped Postgres. Call +// in BeforeEach; cleanup is registered with DeferCleanup. A spec that needs a +// broadcast carrier opens one with Bus(). func SetupInfra(dbName string) *TestInfra { GinkgoHelper() - Expect(suitePG).ToNot(BeNil(), "SetupInfra called before BeforeSuite started the shared containers") + Expect(suitePG).ToNot(BeNil(), "SetupInfra called before BeforeSuite started the shared container") infra := &TestInfra{ - Ctx: context.Background(), - PGContainer: suitePG, - NATSContainer: suiteNATS, - NatsURL: suiteNatsURL, + Ctx: context.Background(), + PGContainer: suitePG, } db := fmt.Sprintf("%s_%d", sanitizeDBName(dbName), dbCounter.Add(1)) @@ -180,12 +174,9 @@ func SetupInfra(dbName string) *TestInfra { Expect(admin.Exec(fmt.Sprintf("CREATE DATABASE %q", db)).Error).To(Succeed()) }() - // Registered before anything else can fail: a NATS connect error below would - // otherwise leave the database behind for the rest of the suite. + // Registered immediately after the CREATE, so no later failure in this + // helper can leave the database behind for the rest of the suite. DeferCleanup(func() { - if infra.NC != nil { - infra.NC.Close() - } drop, err := tryAdminDB() if err != nil { AddReportEntry("drop database skipped", fmt.Sprintf("%s: %v", db, err)) @@ -200,63 +191,23 @@ func SetupInfra(dbName string) *TestInfra { infra.PGURL = replaceDBName(suitePGDSN, db) - var err error - infra.NC, err = messaging.New(infra.NatsURL) - Expect(err).ToNot(HaveOccurred()) - return infra } -// SetupNATSOnly returns a client on the suite-scoped NATS for specs that need no -// database. -func SetupNATSOnly() *TestInfra { - GinkgoHelper() - Expect(suiteNATS).ToNot(BeNil(), "SetupNATSOnly called before BeforeSuite started the shared containers") - - infra := &TestInfra{ - Ctx: context.Background(), - NATSContainer: suiteNATS, - NatsURL: suiteNatsURL, - } - - var err error - infra.NC, err = messaging.New(infra.NatsURL) - Expect(err).ToNot(HaveOccurred()) - - DeferCleanup(func() { - if infra.NC != nil { - infra.NC.Close() - } - }) - - return infra -} - -// FlushNATS ensures all subscriptions are registered server-side before publishing. -// -// It asserts the server's verdict too, not only that the round trip completed: -// on a permission-enforcing server a denied SUB leaves the connection open and -// the flush succeeding, so a helper that checked the flush alone would let a -// spec proceed to publish into a subscription the server had already refused. -func FlushNATS(nc *messaging.Client) { - GinkgoHelper() - Expect(nc.ConfirmRoundTrip(5 * time.Second)).To(Succeed()) -} - // Bus opens a broadcast carrier on THIS spec's database. // // It is the carrier the job, agent and response families travel on, and it is // what these specs must build their dispatchers and bridges with. Publishing on // one carrier while the subscriber reads another is a defect with no error // anywhere: the publish succeeds and the SSE stream is simply empty, so a spec -// that used the NATS client here would keep passing after production had gone -// silent. +// that reached for a message-bus client here would keep passing after +// production had gone silent. // // Every call returns a SEPARATE carrier on the same database, so a spec can // build two and assert across them, which is the shape a deployment has. func (i *TestInfra) Bus() *pgbus.Bus { GinkgoHelper() - Expect(i.PGURL).ToNot(BeEmpty(), "Bus needs a database; use SetupInfra rather than SetupNATSOnly") + Expect(i.PGURL).ToNot(BeEmpty(), "Bus needs a database; call SetupInfra first") db, err := gorm.Open(postgres.Open(i.PGURL), &gorm.Config{Logger: gormlogger.Default.LogMode(gormlogger.Silent)}) Expect(err).ToNot(HaveOccurred()) From 646fd69ff585292cedf4f9ec4fe2b6c46d69031c Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 5 Sep 2026 03:27:17 +0000 Subject: [PATCH 100/153] chore(distributed): take the nats-io modules out of the build Distributed mode has not dialled a message broker since the control plane moved onto the workers' own outward tunnels and every fan-out family moved onto PostgreSQL LISTEN/NOTIFY. What was left was the dependency itself, and the code that existed only to feed it. Dropped from go.mod: nats-io/jwt/v2, nats-io/nats.go, nats-io/nkeys, nats-io/nuid and testcontainers-go/modules/nats, along with the fourteen indirect requires that only the NATS testcontainer pulled in. go.sum carries no nats line either, so the removal is not the partial kind where the require goes and the checksum stays. Deleted with them: pkg/natsauth in full, the broker client's remaining options and TLS files, the per-node JWT minting on both the register and the approve path, and the natsauth.Config parameter threaded through the node routes. The credential manager is renamed and stripped rather than deleted, because it still holds the tunnel token that every re-registration rotates. The bus flags stay accepted and ignored, and are now hidden, on every command that had them, so an existing unit file, compose file or Helm values file still starts on the day of the upgrade. What is not kept is the validation that REQUIRED one: a distributed frontend started with no bus URL is no longer fatal. The TLS paths lose type:"existingfile" deliberately, so a certificate deleted along with the broker cannot fail a startup. One operator-visible behaviour change: --nats-require-auth no longer makes an agent worker wait through admin approval. Ask for that wait with --distributed-require-auth, which already implied it. It is documented in the migration section and pinned from both sides. A deployment now needs PostgreSQL and the frontends' own HTTP listener, and nothing else. coverage-baseline.txt moves from 54.2 to 62.0. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .agents/building-and-testing.md | 14 +- .agents/localai-assistant-mcp.md | 2 +- CONTRIBUTING.md | 6 +- Makefile | 14 +- core/application/cache_fanout_wiring.go | 29 +- core/application/cache_fanout_wiring_test.go | 10 +- core/application/distributed.go | 37 +-- core/application/distributed_test.go | 9 +- core/application/fanout_wiring.go | 13 +- core/application/startup.go | 6 +- core/cli/agent_worker.go | 76 +++-- core/cli/agent_worker_test.go | 73 +++++ core/cli/cli.go | 2 +- core/cli/run.go | 72 ++--- core/cli/run_bus_flags_test.go | 172 +++++++++++ core/cli/worker/worker_vllm.go | 2 +- core/cli/workerregistry/client.go | 34 +-- core/cli/workerregistry/client_test.go | 41 ++- core/cli/workerregistry/credentials.go | 190 ++++-------- core/cli/workerregistry/credentials_test.go | 152 ++++------ core/config/distributed_config.go | 139 +-------- core/config/distributed_config_test.go | 61 ++-- core/http/app.go | 11 +- core/http/auth/helpers_test.go | 2 - core/http/endpoints/cluster/connect.go | 15 +- core/http/endpoints/localai/nodes.go | 32 +- core/http/endpoints/localai/nodes_test.go | 77 +++-- .../nodes-per-node-backend-actions.spec.js | 2 +- core/http/routes/nodes.go | 9 +- core/http/routes/openresponses.go | 8 +- core/http/routes/openresponses_wiring_test.go | 9 +- core/services/finetune/service.go | 5 +- core/services/messaging/client.go | 282 ------------------ core/services/messaging/interfaces.go | 19 +- core/services/messaging/interfaces_test.go | 68 ++--- core/services/messaging/nats_absent_test.go | 67 +++++ core/services/messaging/options.go | 34 --- core/services/messaging/tls.go | 68 ----- core/services/messaging/tls_test.go | 25 -- core/services/pgbus/bus.go | 7 +- core/services/quantization/service.go | 5 +- coverage-baseline.txt | 2 +- docs/content/features/distributed-mode.md | 20 +- docs/content/reference/cli-reference.md | 20 +- go.mod | 21 +- go.sum | 47 +-- pkg/natsauth/config.go | 66 ---- pkg/natsauth/decode.go | 16 - pkg/natsauth/mint.go | 59 ---- pkg/natsauth/mint_test.go | 72 ----- pkg/natsauth/permissions.go | 67 ----- pkg/natsauth/permissions_coverage_test.go | 164 ---------- scripts/build/healthcheck_test.sh | 5 +- tests/e2e/distributed/cluster/cluster.go | 80 +++-- tests/e2e/distributed/cluster/cluster_test.go | 12 +- tests/e2e/distributed/cluster/failure.go | 2 +- .../e2e/distributed/cluster_baseline_test.go | 13 +- tests/e2e/distributed/cluster_control_test.go | 11 +- .../distributed/finetune_distributed_test.go | 8 +- tests/e2e/distributed/foundation_test.go | 32 +- .../distributed/gallery_distributed_test.go | 4 +- tests/e2e/distributed/job_dispatch_test.go | 4 +- tests/e2e/distributed/model_routing_test.go | 4 +- .../distributed/skills_distributed_test.go | 6 +- tests/e2e/distributed/sse_routes_test.go | 4 +- tests/e2e/distributed/testhelpers_test.go | 8 - website/layouts/index.html | 2 +- 67 files changed, 967 insertions(+), 1681 deletions(-) create mode 100644 core/cli/run_bus_flags_test.go delete mode 100644 core/services/messaging/client.go create mode 100644 core/services/messaging/nats_absent_test.go delete mode 100644 core/services/messaging/options.go delete mode 100644 core/services/messaging/tls.go delete mode 100644 core/services/messaging/tls_test.go delete mode 100644 pkg/natsauth/config.go delete mode 100644 pkg/natsauth/decode.go delete mode 100644 pkg/natsauth/mint.go delete mode 100644 pkg/natsauth/mint_test.go delete mode 100644 pkg/natsauth/permissions.go delete mode 100644 pkg/natsauth/permissions_coverage_test.go diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 3f614d039e85..ae1c9a4f629d 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -48,17 +48,17 @@ Rules (both gates): ## Distributed-mode test suites -Two suites cover distributed mode (frontend replicas, worker nodes, PostgreSQL, NATS), split by a Ginkgo label: +Two suites cover distributed mode (frontend replicas, worker nodes, PostgreSQL), split by a Ginkgo label. Neither starts a message broker: a distributed deployment needs PostgreSQL and the frontends' own HTTP listener, and the `nats-io` modules are not in the build. -- `make test-e2e-distributed` runs `Distributed && !VLLMMultinode && !Cluster` over `./tests/e2e/distributed` recursively. Services are wired directly into the test binary. ~240 specs, ~75s. -- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. It **builds `./local-ai` itself** (the `e2e-binary` target), so you do not run `make build` first and cannot forget to. 6 specs, about 8m30s measured over three consecutive runs (509.1s / 509.8s / 512.3s, so 8m29s to 8m32s). +- `make test-e2e-distributed` runs `Distributed && !VLLMMultinode && !Cluster` over `./tests/e2e/distributed` recursively. Services are wired directly into the test binary. 231 specs in about 2m20s (measured 133.3s for the main suite plus the 8 argument-validation specs in the subpackage). +- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. It **builds `./local-ai` itself** (the `e2e-binary` target), so you do not run `make build` first and cannot forget to. 21 specs, 802.5s of specs in a 13m26s target run (one measurement; the suite grew from the 6 specs and roughly 509s recorded earlier, so budget on the newer number). Both jobs live in `.github/workflows/tests-e2e-distributed.yml`, with `timeout-minutes: 45` each. They trigger on pull requests *and* on every push to `master`; the `paths-ignore` filter (see [.agents/ci-caching.md](ci-caching.md)) sits on the pull-request trigger only, so a master push always runs both. They are advisory only because `master` carries no branch protection, which is a repository setting and not a YAML key: `continue-on-error: true` would flip the run's *conclusion* to success and hide the failure, so it is not used. -- **Containers are suite-scoped, not spec-scoped.** `SetupInfra` used to start a PostgreSQL (~10s) and a NATS (~3.5s) per spec. Across the 213 specs behind it that was roughly **48 minutes of pure container startup per run**, which is why this suite was never in CI. (213 rather than the ~240 above: the larger number is everything the label filter selects, the smaller one is just the specs that call `SetupInfra`.) Containers now start once in `BeforeSuite` and each spec gets its own database via `CREATE DATABASE` (~67ms), which is what the `dbName` argument was always describing. Adding a spec needs no change: call `SetupInfra("some-name")` as before, the name is a prefix and a counter keeps it unique. -- **Consequence for new specs:** the NATS bus is now *shared* within a Ginkgo process, so a wildcard subscriber can observe another spec's traffic. Filter assertions on an identifier your spec owns (a node ID, a job ID) instead of counting everything on `jobs.*.progress`, and verify the spec with `--randomize-all`. -- **`BeforeSuite`, not `SynchronizedBeforeSuite`.** Under `ginkgo -p` each process then gets its own container pair, keeping NATS subjects isolated per process. A single shared NATS across parallel processes would let specs on different processes see each other's messages on the same subject. -- **The label split.** The 8 argument-validation specs under `tests/e2e/distributed/cluster/` carry `Label("Distributed")` only, on purpose: they need no binary, no PostgreSQL and no NATS, so they belong in the fast job. That is why `test-e2e-distributed` keeps `-r` (it must reach the subpackage) and `test-e2e-cluster` deliberately does **not** (the subpackage is out of its scope). +- **The container is suite-scoped, not spec-scoped.** `SetupInfra` used to start a PostgreSQL (~10s) and a broker (~3.5s) per spec. Across the 213 specs behind it that was roughly **48 minutes of pure container startup per run**, which is why this suite was never in CI. (213 rather than the ~240 above: the larger number is everything the label filter selects, the smaller one is just the specs that call `SetupInfra`.) The broker container is gone entirely, and the PostgreSQL one now starts once in `BeforeSuite` with each spec getting its own database via `CREATE DATABASE` (~67ms), which is what the `dbName` argument was always describing. Adding a spec needs no change: call `SetupInfra("some-name")` as before, the name is a prefix and a counter keeps it unique. +- **Consequence for new specs:** a fan-out carrier is *shared* within a Ginkgo process, so a wildcard subscriber can observe another spec's traffic. Filter assertions on an identifier your spec owns (a node ID, a job ID) instead of counting everything on `jobs.*.progress`, and verify the spec with `--randomize-all`. +- **`BeforeSuite`, not `SynchronizedBeforeSuite`.** Under `ginkgo -p` each process then gets its own container, keeping each process's carrier isolated. A single shared database across parallel processes would let specs on different processes see each other's messages on the same subject. +- **The label split.** The 8 argument-validation specs under `tests/e2e/distributed/cluster/` carry `Label("Distributed")` only, on purpose: they need no binary and no PostgreSQL, so they belong in the fast job. That is why `test-e2e-distributed` keeps `-r` (it must reach the subpackage) and `test-e2e-cluster` deliberately does **not** (the subpackage is out of its scope). - **The binary is built, and a stale one is refused.** `test-e2e-cluster` used to take `./local-ai` as given and only check that a file by that name existed. An edit that was never rebuilt therefore ran the entire suite against an older build while reporting on the working tree, which surfaced once as a spec failing on caller line numbers that matched no line in any source file. Two things close it. The target depends on `e2e-binary`, which is `go build -o local-ai ./cmd/local-ai` and not `make build` (that one also builds the React UI, which this suite never touches and CI stubs). And `localAIBinary` in the harness **fails**, on a developer box as well as under CI, when the binary is older than the newest non-test Go source in the tree, which covers a run started with `LOCALAI_E2E_BINARY` or by invoking ginkgo directly. `_test.go` files are excluded from that scan on purpose: they compile into the ginkgo suite and never into `local-ai`, so counting them would fire on every edit to the spec being run. Set `LOCALAI_E2E_BINARY` to opt out of the build; there is no opt-out of the staleness check, because a rebuild is the remedy. - **`--fail-on-empty` is load-bearing on both targets.** Ginkgo exits 0 when a label filter selects nothing, so without it a refactor that renames or drops `Label("Cluster")` leaves the target reporting "Test Suite Passed" having started no cluster at all. `LOCALAI_E2E_REQUIRE_BINARIES` does not cover this case: it only fires inside a spec that is actually running. - **The binary gate.** `localAIBinary()` and `mockBackendBinary()` **fail** rather than skip when `CI` is set, or when `LOCALAI_E2E_REQUIRE_BINARIES` is truthy; `LOCALAI_E2E_REQUIRE_BINARIES=0` (also `off`, `no`, `n`, `disabled`, and anything `strconv.ParseBool` reads as false) forces skipping even under CI. **Any value that parses as neither reads as ON**, not off: setting the variable to something meaningless means someone meant to turn the gate on, and reading it as false would quietly restore the silent skip the flag exists to remove. The whole polarity is deliberate, because in CI a skipped cluster spec is indistinguishable from a passing one: Ginkgo exits 0 on skips. Locally a missing binary still just skips, since `CI` is unset in an ordinary shell. diff --git a/.agents/localai-assistant-mcp.md b/.agents/localai-assistant-mcp.md index bd391164c781..57d51fbd6bf5 100644 --- a/.agents/localai-assistant-mcp.md +++ b/.agents/localai-assistant-mcp.md @@ -94,4 +94,4 @@ The user chose KISS. Every mutating tool has a safety rule (`prompts/10_safety.m ## Distributed mode -The in-memory MCP server runs only on the head node (where the chat handler runs). `inproc.Client` wraps services that are already distributed-aware (`GalleryService` coordinates with workers; `ListNodes` reads the NATS-populated registry). No NATS routing of MCP tools — the admin surface lives on the head, period. +The in-memory MCP server runs only on the head node (where the chat handler runs). `inproc.Client` wraps services that are already distributed-aware (`GalleryService` coordinates with workers; `ListNodes` reads the PostgreSQL-backed node registry). MCP tools are not routed anywhere: the admin surface lives on the head, period. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 69ddbd6f2fc7..cc7341d04dc0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -267,16 +267,16 @@ make test-e2e ### Running distributed-mode tests -Distributed mode (several frontend replicas, worker nodes, PostgreSQL and NATS) has two suites. Both bring up their PostgreSQL and NATS with testcontainers, so Docker has to be available: +Distributed mode (several frontend replicas, worker nodes and PostgreSQL) has two suites. Both bring up their PostgreSQL with testcontainers, so Docker has to be available. There is no message broker to bring up: a distributed deployment needs PostgreSQL and the frontends' own HTTP listener, and nothing else. ```bash make test-e2e-distributed # in-process: services wired directly into the test binary make test-e2e-cluster # process-level: real local-ai child processes ``` -`make test-e2e-distributed` is the fast one (around 240 specs in roughly 75 seconds). It starts one PostgreSQL and one NATS for the whole run and gives each spec its own database. It runs each spec exactly once, with no retry: `DISTRIBUTED_TEST_FLAKES` defaults to 1 and feeds ginkgo's `--flake-attempts`, which counts *total attempts*, not retries. That is deliberately below the repo-wide `TEST_FLAKES=5`, because this suite exists to catch nondeterministic cluster behaviour and a retry hides exactly the failure it is meant to catch. Raise it locally when bisecting something unrelated. +`make test-e2e-distributed` is the fast one (231 specs in roughly 2m20s). It starts one PostgreSQL for the whole run and gives each spec its own database. It runs each spec exactly once, with no retry: `DISTRIBUTED_TEST_FLAKES` defaults to 1 and feeds ginkgo's `--flake-attempts`, which counts *total attempts*, not retries. That is deliberately below the repo-wide `TEST_FLAKES=5`, because this suite exists to catch nondeterministic cluster behaviour and a retry hides exactly the failure it is meant to catch. Raise it locally when bisecting something unrelated. -`make test-e2e-cluster` runs `local-ai` as real child processes, one per frontend replica and one per worker, so a spec can kill a replica and assert what the survivors do. Budget about 8m30s (measured 509.1s / 509.8s / 512.3s over three consecutive runs): three of its six specs wait out real staleness and health-check windows. It needs a built binary and the mock backend: +`make test-e2e-cluster` runs `local-ai` as real child processes, one per frontend replica and one per worker, so a spec can kill a replica and assert what the survivors do. Budget about 13m30s (one measurement: 802.5s of specs in a 13m26s target run, which includes building the binary): several of its 21 specs wait out real staleness and health-check windows. An earlier record of 6 specs and roughly 509s is superseded. It needs a built binary and the mock backend: ```bash make build build-mock-backend diff --git a/Makefile b/Makefile index 7a114dc4a77c..fd7327005f26 100644 --- a/Makefile +++ b/Makefile @@ -349,8 +349,10 @@ run-e2e-aio: protogen-go # you are bisecting something unrelated. DISTRIBUTED_TEST_FLAKES?=1 -# Distributed architecture e2e (PostgreSQL + NATS via testcontainers). -# Includes NatsJWT specs (JWT-enabled NATS). Requires Docker. +# Distributed architecture e2e (PostgreSQL via testcontainers). Requires Docker. +# There is no broker container and no NatsJWT label any more: a distributed +# deployment needs PostgreSQL and the frontends' own HTTP listener, and this +# suite stands up exactly that. # VLLMMultinode is excluded here; use test-e2e-vllm-multinode for that. # Cluster is excluded too and runs in test-e2e-cluster below, which needs a # built binary. The argument-validation specs under tests/e2e/distributed/cluster @@ -366,7 +368,7 @@ DISTRIBUTED_TEST_FLAKES?=1 # in this file that does exec a prebuilt binary is test-e2e-cluster below, and # the reason it now BUILDS it is written there. test-e2e-distributed: protogen-go - @echo 'Running distributed e2e tests (label Distributed, incl. NatsJWT)' + @echo 'Running distributed e2e tests (label Distributed)' $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode && !Cluster' --fail-on-empty --flake-attempts $(DISTRIBUTED_TEST_FLAKES) --timeout=40m -v -r ./tests/e2e/distributed # The local-ai binary the cluster e2e execs as its frontend and worker @@ -392,7 +394,7 @@ e2e-binary: protogen-go fi # Cluster e2e: runs local-ai as real child processes (frontend replicas + -# workers) against PostgreSQL and NATS, and kills them to assert failover. +# workers) against PostgreSQL, and kills them to assert failover. # It BUILDS that binary rather than checking that a file by that name exists, # and that is a correctness fix rather than a convenience. This target used to # take ./local-ai as given, so an edit to core/ that was never rebuilt left the @@ -409,8 +411,8 @@ e2e-binary: protogen-go # the harness makes every other path loud. # # The argument-validation specs in tests/e2e/distributed/cluster deliberately -# stay in test-e2e-distributed above: they need no binary, no PostgreSQL and no -# NATS, so no -r here and that package is simply out of scope. +# stay in test-e2e-distributed above: they need no binary and no PostgreSQL, so +# no -r here and that package is simply out of scope. # # --fail-on-empty is load-bearing, not tidiness. Ginkgo exits 0 when a label # filter selects nothing, so without it a refactor that renames or drops diff --git a/core/application/cache_fanout_wiring.go b/core/application/cache_fanout_wiring.go index a351a8efcadd..bbe4d592ae7a 100644 --- a/core/application/cache_fanout_wiring.go +++ b/core/application/cache_fanout_wiring.go @@ -18,22 +18,21 @@ import ( // The four process-lifetime caches, each wired onto the broadcast carrier by // one function here, and none of them by a call site naming a carrier. // -// Two are methods on DistributedServices and take NO carrier at all, because -// their call sites hold the whole struct and could otherwise reach for the -// cancel carrier. -// The other two run inside initDistributed before that struct exists, so they -// take the CONCRETE *pgbus.Bus rather than messaging.Broadcaster. +// Two are methods on DistributedServices and take NO carrier at all. The other +// two run inside initDistributed before that struct exists, so they take the +// CONCRETE *pgbus.Bus rather than messaging.Broadcaster. // -// Concrete on purpose. *messaging.Client satisfies messaging.Broadcaster just -// as well as the carrier does, so an interface parameter at these sites lets a -// caller hand over the NATS client that is also in scope: it compiles, it -// starts, it publishes, it is delivered, onto a carrier the deployment is being -// taken off, and nothing fails until NATS goes away. With the concrete type -// that mistake is a build error rather than a deployment that looks healthy. +// Concrete on purpose, and the purpose has outlived what prompted it. It was +// written against a SECOND carrier that was in scope at these call sites and +// satisfied the same interface, so an interface parameter let a caller hand +// over the wrong one: it compiled, started, published and was delivered, to +// nobody the deployment would still be listening on. That carrier went with the +// message broker and there is one left. The narrowing stays because a second +// carrier is exactly what it guards against, and widening these parameters back +// would give the guard away silently on the day another one arrives. // // The adopters themselves still take the interface, so their own specs drive -// them with an in-memory double. The narrowing is only here, where the wrong -// carrier is in scope. +// them with an in-memory double. The narrowing is only here. // wireGalleryBroadcasts puts the gallery service's progress, cancel and // cache-invalidation traffic on the carrier, and opens the wildcard @@ -62,8 +61,8 @@ func (ds *DistributedServices) wireGallery(gs *galleryop.GalleryService) error { // // Exported, unlike its siblings, because the OpCache is built in the HTTP layer // rather than in initDistributed. It takes neither a carrier nor a store: both -// come off this struct, so the HTTP layer cannot pass the NATS client that -// hangs off it beside them. +// come off this struct, so the HTTP layer names no carrier and cannot pass one +// that is merely in scope beside them. // // A hydrate failure is the OpCache's own business and is logged there; a // subscribe failure is returned, because a cache that hydrated and did not diff --git a/core/application/cache_fanout_wiring_test.go b/core/application/cache_fanout_wiring_test.go index 0cc078230f8d..45cf857da318 100644 --- a/core/application/cache_fanout_wiring_test.go +++ b/core/application/cache_fanout_wiring_test.go @@ -25,10 +25,12 @@ import ( // // Every case here wires the cache on busA and drives it from busB. A cache // talking to itself would pass with the wiring pointed at any carrier at all, -// which is the defect these exist to catch: the NATS client is in scope at -// three of the four call sites and satisfies the same interface, so a site left -// holding it publishes successfully and is delivered, to nobody the deployment -// will still be listening on. +// which is the defect these exist to catch: a site holding a carrier other than +// the deployment's publishes successfully and is delivered, to nobody the +// deployment will still be listening on. The second carrier that made that +// concrete went with the message broker, and these stay because they are what +// would catch it on the day a third arrives: busB IS the other carrier, so the +// defect is exercised rather than argued about. var _ = Describe("wiring the process-lifetime caches onto the broadcast carrier", func() { var ( ctx context.Context diff --git a/core/application/distributed.go b/core/application/distributed.go index 09aa565299f5..7dc3823c9455 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -32,21 +32,21 @@ import ( // DistributedServices holds all services initialized for distributed mode. type DistributedServices struct { - Store storage.ObjectStore - Registry *nodes.NodeRegistry - Router *nodes.SmartRouter - Health *nodes.HealthMonitor - Reconciler *nodes.ReplicaReconciler - JobStore *jobs.JobStore - Dispatcher *jobs.Dispatcher - AgentStore *agents.AgentStore - AgentBridge *agents.EventBridge - DistStores *distributed.Stores - FileMgr *storage.FileManager - FileStager nodes.FileStager - ModelAdapter *nodes.ModelRouterAdapter - Unloader *nodes.RemoteUnloaderAdapter - ModelCleanup *nodes.ModelCleanupService + Store storage.ObjectStore + Registry *nodes.NodeRegistry + Router *nodes.SmartRouter + Health *nodes.HealthMonitor + Reconciler *nodes.ReplicaReconciler + JobStore *jobs.JobStore + Dispatcher *jobs.Dispatcher + AgentStore *agents.AgentStore + AgentBridge *agents.EventBridge + DistStores *distributed.Stores + FileMgr *storage.FileManager + FileStager nodes.FileStager + ModelAdapter *nodes.ModelRouterAdapter + Unloader *nodes.RemoteUnloaderAdapter + ModelCleanup *nodes.ModelCleanupService // Bus is the deployment's fan-out carrier, riding the auth database's // PostgreSQL rather than a message broker. Every cross-replica family the @@ -536,8 +536,11 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } idx := prefixcache.NewIndex(prefixCfg) // S4. One call puts this replica's observations and its peers' on the - // same carrier, and it takes the concrete carrier so the NATS client - // still in scope here cannot be handed to it by accident. + // same carrier, and it takes the CONCRETE carrier so that no other + // thing satisfying messaging.Broadcaster can be handed to it by + // accident. There is no second carrier in this scope to hand over any + // more; the type stays narrow so there is still none on the day one is + // added. See cache_fanout_wiring.go for the whole argument. prefixSync, err := wirePrefixCacheBroadcasts(bus, prefixCfg, idx) if err != nil { return nil, err diff --git a/core/application/distributed_test.go b/core/application/distributed_test.go index 731b86cf29bb..bedd6dfb81b8 100644 --- a/core/application/distributed_test.go +++ b/core/application/distributed_test.go @@ -119,11 +119,12 @@ var _ = Describe("shutting the distributed services down", func() { // It was five field reads before this: the fine-tune service, the quantization // service, the agent-task setter on two startup paths, the per-user services // manager and the Open Responses store. Every one of them takes a -// messaging.Broadcaster, which *messaging.Client satisfies too, so a site left -// holding the struct's NATS field compiled, started, published and was +// messaging.Broadcaster, which the broker client on the same struct satisfied +// too, so a site left holding that field compiled, started, published and was // delivered onto a carrier only agent workers read, and nothing failed until -// NATS did. Collapsing the choice into one function is what makes it a fact -// these specs can hold. +// the broker went away. That field and its type are now gone; collapsing the +// choice into one function is what keeps it a fact these specs can hold, rather +// than a property that lasted only as long as there was one carrier. var _ = Describe("handing the broadcast carrier to its adopters", func() { It("returns the carrier the deployment opened", func() { db, dsn := testutil.SetupTestDBWithDSN() diff --git a/core/application/fanout_wiring.go b/core/application/fanout_wiring.go index 15ef31ca907f..defcecf98208 100644 --- a/core/application/fanout_wiring.go +++ b/core/application/fanout_wiring.go @@ -20,12 +20,13 @@ import ( // parameter type is the reason. jobs.NewDispatcher and agents.NewEventBridge // both take a messaging.Broadcaster, which they must: neither may know which // carrier a deployment runs, and their specs publish through a double. But that -// also means *messaging.Client satisfies them, so wiring either of them to the -// NATS client instead of the carrier COMPILES, passes every unit spec in both -// packages, and presents only as an SSE stream that stays empty while the work -// it is watching runs to completion on the other side of a carrier nobody is -// subscribed to. Naming *pgbus.Bus here is what makes that a build failure -// rather than a silent one, and it is why this exists as a function instead of +// also means ANY type satisfying the interface can be wired to them, so while a +// second carrier existed, pointing either of them at it COMPILED, passed every +// unit spec in both packages, and presented only as an SSE stream that stayed +// empty while the work it was watching ran to completion on the other side of a +// carrier nobody subscribed to. The second carrier went with the message +// broker. Naming *pgbus.Bus here is what would make that a build failure rather +// than a silent one again, and it is why this exists as a function instead of // as two lines and a comment asking the reader to be careful. // // The observable persister is STARTED here for the reason startJobDispatchLoop diff --git a/core/application/startup.go b/core/application/startup.go index b9d371336bec..44962d841397 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -402,8 +402,10 @@ func New(opts ...config.AppOption) (*Application, error) { } } // S2. One call sets the carrier and opens the wildcard - // subscriptions, and it names no carrier, so the NATS client on - // distSvc cannot be passed here by accident. + // subscriptions, and it names no carrier at all, so no carrier that + // happens to hang off distSvc can be passed here by accident. See + // cache_fanout_wiring.go for why that shape is kept now that the + // broker's client is no longer one of them. if err := distSvc.wireGallery(application.galleryService); err != nil { xlog.Warn("Gallery service subscribe failed", "error", err) } diff --git a/core/cli/agent_worker.go b/core/cli/agent_worker.go index 6df16fc21854..7f7a6caf7a44 100644 --- a/core/cli/agent_worker.go +++ b/core/cli/agent_worker.go @@ -67,26 +67,54 @@ type AgentWorkerCMD struct { APIURL string `env:"LOCALAI_API_URL" help:"LocalAI API URL for inference (auto-derived from RegisterTo if not set)" group:"api"` APIToken string `env:"LOCALAI_API_TOKEN" help:"API token for LocalAI inference (auto-provisioned during registration if not set)" group:"api"` - NatsJWT string `env:"LOCALAI_NATS_JWT" help:"NATS user JWT override (defaults to nats_jwt from registration)" group:"distributed"` - NatsUserSeed string `env:"LOCALAI_NATS_USER_SEED" help:"NATS user seed override (defaults to nats_user_seed from registration)" group:"distributed"` - NatsServiceJWT string `env:"LOCALAI_NATS_SERVICE_JWT" help:"Fallback NATS service JWT when registration does not mint agent JWT" group:"distributed"` - NatsServiceSeed string `env:"LOCALAI_NATS_SERVICE_SEED" help:"Fallback NATS service seed paired with LOCALAI_NATS_SERVICE_JWT" group:"distributed"` - NatsRequireAuth bool `env:"LOCALAI_NATS_REQUIRE_AUTH" default:"false" help:"Require NATS JWT+seed to connect" group:"distributed"` + // The broker credential and TLS flags, accepted and ignored, hidden, on the + // same terms as NatsURL above. There is no connection left to present a + // credential on. + NatsJWT string `env:"LOCALAI_NATS_JWT" help:"Ignored. An agent worker opens no bus connection to present a credential on." group:"distributed" hidden:""` + NatsUserSeed string `env:"LOCALAI_NATS_USER_SEED" help:"Ignored. Paired with LOCALAI_NATS_JWT, which is itself ignored." group:"distributed" hidden:""` + NatsServiceJWT string `env:"LOCALAI_NATS_SERVICE_JWT" help:"Ignored. An agent worker opens no bus connection to present a credential on." group:"distributed" hidden:""` + NatsServiceSeed string `env:"LOCALAI_NATS_SERVICE_SEED" help:"Ignored. Paired with LOCALAI_NATS_SERVICE_JWT, which is itself ignored." group:"distributed" hidden:""` + NatsRequireAuth bool `env:"LOCALAI_NATS_REQUIRE_AUTH" default:"false" help:"Ignored. Use --distributed-require-auth to make this worker wait through admin approval." group:"distributed" hidden:""` // DistributedRequireAuth is the umbrella switch; for the agent worker (which - // has no file-transfer server) it implies NATS auth is required. - DistributedRequireAuth bool `env:"LOCALAI_DISTRIBUTED_REQUIRE_AUTH" default:"false" help:"Umbrella switch implying --nats-require-auth (agent workers have no file-transfer server)" group:"distributed"` - NatsTLSCA string `env:"LOCALAI_NATS_TLS_CA" type:"existingfile" help:"PEM file for NATS server CA (private PKI)" group:"distributed"` - NatsTLSCert string `env:"LOCALAI_NATS_TLS_CERT" type:"existingfile" help:"Client certificate for NATS mTLS" group:"distributed"` - NatsTLSKey string `env:"LOCALAI_NATS_TLS_KEY" type:"existingfile" help:"Client private key for NATS mTLS" group:"distributed"` + // has no file-transfer server) it makes registration WAIT THROUGH ADMIN + // APPROVAL rather than starting against a pending node. + // + // It used to imply --nats-require-auth as well, and the wait was a side + // effect of that: the worker was waiting for a broker credential to be + // minted. There is no credential and no broker, so the wait is now what the + // switch is FOR, and it is described that way rather than by what it used + // to imply. + DistributedRequireAuth bool `env:"LOCALAI_DISTRIBUTED_REQUIRE_AUTH" default:"false" help:"Wait through admin approval at registration instead of starting against a node an admin has not approved" group:"distributed"` + // type:"existingfile" is deliberately NOT kept: validating a path this + // process never opens would fail a worker at startup over a certificate for + // a broker the operator has already shut down. + NatsTLSCA string `env:"LOCALAI_NATS_TLS_CA" help:"Ignored. No bus connection is opened, so no server certificate is verified." group:"distributed" hidden:""` + NatsTLSCert string `env:"LOCALAI_NATS_TLS_CERT" help:"Ignored. No bus connection is opened, so no client certificate is presented." group:"distributed" hidden:""` + NatsTLSKey string `env:"LOCALAI_NATS_TLS_KEY" help:"Ignored. Paired with LOCALAI_NATS_TLS_CERT, which is itself ignored." group:"distributed" hidden:""` // Timeouts MCPCIJobTimeout string `env:"LOCALAI_MCP_CI_JOB_TIMEOUT" default:"10m" help:"Timeout for MCP CI job execution" group:"distributed"` } -// natsAuthRequired reports whether NATS JWT credentials must be present — the -// granular flag or the umbrella (LOCALAI_DISTRIBUTED_REQUIRE_AUTH). -func (cmd *AgentWorkerCMD) natsAuthRequired() bool { - return cmd.NatsRequireAuth || cmd.DistributedRequireAuth +// waitThroughApproval reports whether registration should block until an admin +// approves this node, instead of returning a pending response and starting. +// +// A method rather than the field read it wraps, because the answer CHANGED and +// the change is the one thing in this command an operator can be surprised by. +// It used to be --distributed-require-auth OR --nats-require-auth, narrowed +// further by whether the operator had supplied a broker JWT by hand. Every term +// but the first was about a credential that no longer exists, so the gate is +// now the first term alone. +// +// An operator who set ONLY --nats-require-auth therefore loses the wait and +// gets the historical default: register, start, and let the tunnel dialer be +// refused with 403 until an admin approves. That is a visible change, it is +// documented in docs/content/features/distributed-mode.md, and it is a seam so +// that it is also pinned: inlined at the call site it sat inside a Run that +// dials a frontend, where no spec could reach it and swapping the two flags +// back would have stayed green. +func (cmd *AgentWorkerCMD) waitThroughApproval() bool { + return cmd.DistributedRequireAuth } func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { @@ -121,17 +149,15 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { // Register, and obtain this node's identity and its tunnel credential. // - // The manager is still the NATS credential manager and still gated on the - // NATS auth flags, and that is deliberate rather than left over: what the - // gate decides is whether registration WAITS THROUGH ADMIN APPROVAL instead - // of returning a pending response, and that behaviour is unchanged by this - // worker no longer dialling a bus. What it no longer does is dial one: the - // only value read off it below is TunnelToken. - credMgr := workerregistry.NewNATSCredentialManager( + // The manager holds one thing now: the tunnel token, which every + // re-registration rotates. Its gate decides whether registration WAITS + // THROUGH ADMIN APPROVAL instead of returning a pending response; which + // flag decides that, and what changed about it, is on waitThroughApproval. + credMgr := workerregistry.NewCredentialManager( func(ctx context.Context) (*workerregistry.RegisterResponse, error) { return regClient.RegisterFull(ctx, registrationBody) }, - cmd.natsAuthRequired() && cmd.NatsJWT == "" && cmd.NatsServiceJWT == "", + cmd.waitThroughApproval(), ) res, err := credMgr.Acquire(shutdownCtx) if err != nil { @@ -208,8 +234,8 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { xlog.Info("Agent worker ready, serving agent execution and MCP CI runs on its tunnel", "node", nodeID) // Wait for an OS signal. There is no internal fatal condition left to wait - // on: the one that existed was a NATS credential this worker could no - // longer renew, and it renews none. + // on: the one that existed was a broker credential this worker could no + // longer renew, and there is no credential and no broker. sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) <-sigCh @@ -385,7 +411,7 @@ func handleMCPCIJob(ctx context.Context, data []byte, apiURL, apiToken string, p task := evt.Task if job == nil || task == nil { xlog.Error("MCP CI job missing enriched data", "jobID", evt.JobID) - return mcpCIAnswer(pub, evt.JobID, "failed", "", "job or task data missing from NATS event") + return mcpCIAnswer(pub, evt.JobID, "failed", "", "job or task data missing from the job event") } modelCfg := evt.ModelConfig diff --git a/core/cli/agent_worker_test.go b/core/cli/agent_worker_test.go index 1e537999ad50..39a52d47c1f4 100644 --- a/core/cli/agent_worker_test.go +++ b/core/cli/agent_worker_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" "strings" "time" @@ -229,4 +230,76 @@ var _ = Describe("The agent worker's bus requirement", func() { // edit every unit file in the same change. Expect(parse("--register-to", "http://frontend:8080", "--nats-url", "nats://bus:4222")).To(Succeed()) }) + + It("still accepts the broker credentials that came with it", func() { + // The whole set, because an operator's unit file carries the whole set: + // a command line that parses --nats-url and then dies on --nats-jwt has + // bought the fleet nothing. + Expect(parse("--register-to", "http://frontend:8080", + "--nats-url", "nats://bus:4222", + "--nats-jwt", "eyJ0", + "--nats-user-seed", "SUUSER", + "--nats-service-jwt", "eyJ0", + "--nats-service-seed", "SUSERVICE", + "--nats-require-auth")).To(Succeed()) + }) + + It("does not stat the TLS material it no longer presents", func() { + // These paths were validated as existing files while they were dialled + // with. Keeping that on an ignored flag would fail a worker at startup + // over a certificate for a broker the operator has already deleted, + // which is the exact upgrade the acceptance exists to survive. + missing := filepath.Join(GinkgoT().TempDir(), "a-broker-ca-that-was-deleted.pem") + Expect(parse("--register-to", "http://frontend:8080", + "--nats-tlsca", missing, + "--nats-tls-cert", missing, + "--nats-tls-key", missing)).To(Succeed()) + }) + + It("keeps every accepted bus flag hidden from --help", func() { + var cli struct { + AgentWorker AgentWorkerCMD `cmd:""` + } + parser, err := kong.New(&cli) + Expect(err).ToNot(HaveOccurred()) + var visible []string + for _, node := range parser.Model.Children { + for _, flag := range node.Flags { + if strings.HasPrefix(flag.Name, "nats-") && !flag.Hidden { + visible = append(visible, flag.Name) + } + } + } + Expect(visible).To(BeEmpty(), + "%v are still offered in --help while doing nothing", visible) + }) +}) + +// Which flag makes an agent worker wait through admin approval. +// +// The docs promise this specifically, and it is the one behavioural change in +// the broker removal that an operator can be surprised by, so it is asserted +// both ways round. The positive half alone would stay green if the gate were +// widened back to OR --nats-require-auth; the negative half is what says the +// change actually happened. +var _ = Describe("The agent worker's approval gate", func() { + It("waits when --distributed-require-auth is set", func() { + cmd := &AgentWorkerCMD{DistributedRequireAuth: true} + Expect(cmd.waitThroughApproval()).To(BeTrue()) + }) + + It("does not wait for the broker flag that used to imply it", func() { + // --nats-require-auth named a bus this worker does not dial. An + // operator who set only that one now gets the historical default: + // register, start, and be refused at every tunnel dial with 403 until + // an admin approves. Documented in the migration section of + // docs/content/features/distributed-mode.md. + cmd := &AgentWorkerCMD{NatsRequireAuth: true} + Expect(cmd.waitThroughApproval()).To(BeFalse(), + "an ignored flag is gating a real behaviour again") + }) + + It("does not wait when neither is set", func() { + Expect((&AgentWorkerCMD{}).waitThroughApproval()).To(BeFalse()) + }) }) diff --git a/core/cli/cli.go b/core/cli/cli.go index 2b17ff96d5d0..8643a04fa4b6 100644 --- a/core/cli/cli.go +++ b/core/cli/cli.go @@ -20,7 +20,7 @@ var CLI struct { Transcript TranscriptCMD `cmd:"" help:"Convert audio to text"` P2PWorker worker.Worker `cmd:"" name:"p2p-worker" help:"Run workers to distribute workload via p2p (llama.cpp-only)"` Worker WorkerCMD `cmd:"" help:"Start a worker for distributed mode (generic, backend-agnostic)"` - AgentWorker AgentWorkerCMD `cmd:"" name:"agent-worker" help:"Start an agent worker for distributed mode (executes agent chats via NATS)"` + AgentWorker AgentWorkerCMD `cmd:"" name:"agent-worker" help:"Start an agent worker for distributed mode (executes agent chats over the tunnel it dials out to a frontend)"` Util UtilCMD `cmd:"" help:"Utility commands"` Agent AgentCMD `cmd:"" help:"Run agents standalone without the full LocalAI server"` MCPServer MCPServerCMD `cmd:"" name:"mcp-server" help:"Run the LocalAI admin tool surface as a stdio MCP server (controls a remote LocalAI instance over HTTP)"` diff --git a/core/cli/run.go b/core/cli/run.go index 4ed84c3d43b7..bd4433fad165 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -164,7 +164,7 @@ type RunCMD struct { // Distributed / Horizontal Scaling Distributed bool `env:"LOCALAI_DISTRIBUTED" default:"false" help:"Enable distributed mode (requires PostgreSQL; no message bus)" group:"distributed"` InstanceID string `env:"LOCALAI_INSTANCE_ID" help:"Unique instance ID for distributed mode (auto-generated UUID if empty)" group:"distributed"` - NatsURL string `env:"LOCALAI_NATS_URL" help:"Ignored. No component of a distributed deployment connects to a message bus; state and fan-out ride PostgreSQL and workers are reached over their own tunnels. Accepted so an existing command line still starts." group:"distributed"` + NatsURL string `env:"LOCALAI_NATS_URL" help:"Ignored. No component of a distributed deployment connects to a message bus; state and fan-out ride PostgreSQL and workers are reached over their own tunnels. Accepted so an existing command line still starts." group:"distributed" hidden:""` DistributedAdvertiseAddr string `env:"LOCALAI_DISTRIBUTED_ADVERTISE_ADDR" help:"host:port other frontend replicas dial to reach this one (peer link). Empty = derived from the local address that routes to PostgreSQL, which only works when the database is on another host." group:"distributed"` StorageURL string `env:"LOCALAI_STORAGE_URL" help:"S3-compatible storage endpoint URL (e.g., http://minio:9000)" group:"distributed"` StorageBucket string `env:"LOCALAI_STORAGE_BUCKET" default:"localai" help:"S3 bucket name for object storage" group:"distributed"` @@ -173,30 +173,41 @@ type RunCMD struct { StorageSecretKey string `env:"LOCALAI_STORAGE_SECRET_KEY" help:"S3 secret access key" group:"distributed"` RegistrationToken string `env:"LOCALAI_REGISTRATION_TOKEN" help:"Token that backend nodes must provide to register (empty = no auth required)" group:"distributed"` RegistrationRequireAuth bool `env:"LOCALAI_REGISTRATION_REQUIRE_AUTH" default:"false" help:"Fail startup when distributed mode is enabled but LOCALAI_REGISTRATION_TOKEN is empty (node endpoints and worker file-transfer server would otherwise be unauthenticated)" group:"distributed"` - DistributedRequireAuth bool `env:"LOCALAI_DISTRIBUTED_REQUIRE_AUTH" default:"false" help:"Umbrella switch: require BOTH NATS JWT credentials and a registration token when distributed mode is enabled (implies --nats-require-auth and --registration-require-auth)" group:"distributed"` + DistributedRequireAuth bool `env:"LOCALAI_DISTRIBUTED_REQUIRE_AUTH" default:"false" help:"Umbrella switch: require a registration token when distributed mode is enabled (implies --registration-require-auth)" group:"distributed"` AutoApproveNodes bool `env:"LOCALAI_AUTO_APPROVE_NODES" default:"false" help:"Auto-approve new worker nodes (skip admin approval)" group:"distributed"` DistributedSharedModels bool `env:"LOCALAI_DISTRIBUTED_SHARED_MODELS" default:"false" help:"Assert that every node mounts the SAME models directory at the SAME path (shared volume). When true, the router skips staging model files to workers and loads them directly from the shared path, avoiding re-downloads." group:"distributed"` DistributedPrefixCache bool `env:"LOCALAI_DISTRIBUTED_PREFIX_CACHE" default:"true" help:"Enable prefix-cache-aware routing in distributed mode (default true). When false, routing falls back to round-robin." group:"distributed"` DistributedDiskHeadroomCheck bool `env:"LOCALAI_DISTRIBUTED_DISK_HEADROOM_CHECK" default:"true" help:"Reject worker nodes that lack free space to store the model, at scheduling time rather than partway through staging (default true). Free space is measured on the filesystem backing each worker's models directory, and compared against the model's own size plus a small margin. When false, node selection ignores free disk (pre-#11054 behaviour); the check still runs and warns when it would have rejected every node. Can also be toggled at runtime via the distributed_disk_headroom_check setting." group:"distributed"` DistributedPrefixCacheTTL string `env:"LOCALAI_DISTRIBUTED_PREFIX_CACHE_TTL" help:"Idle-timeout for prefix-cache index entries; also drives the background eviction cadence (every TTL/2). Default 5m." group:"distributed"` - BackendInstallTimeout string `env:"LOCALAI_NATS_BACKEND_INSTALL_TIMEOUT" help:"NATS round-trip timeout for backend.install requests sent to worker nodes (default 15m). Increase for slow links pulling multi-GB images." group:"distributed"` - BackendUpgradeTimeout string `env:"LOCALAI_NATS_BACKEND_UPGRADE_TIMEOUT" help:"NATS round-trip timeout for backend.upgrade requests (default 15m)." group:"distributed"` + BackendInstallTimeout string `env:"LOCALAI_NATS_BACKEND_INSTALL_TIMEOUT" help:"Timeout for a backend.install request the frontend sends a worker over its tunnel (default 15m). Increase for slow links pulling multi-GB images." group:"distributed"` + BackendUpgradeTimeout string `env:"LOCALAI_NATS_BACKEND_UPGRADE_TIMEOUT" help:"Timeout for a backend.upgrade request the frontend sends a worker over its tunnel (default 15m)." group:"distributed"` ModelLoadTimeout string `env:"LOCALAI_NATS_MODEL_LOAD_TIMEOUT" help:"Fixed gRPC deadline for the remote LoadModel call sent to a worker node once its backend is installed and model files are staged. Unset (the default), the deadline is derived from the checkpoint size instead: 5m plus 20s per GiB, capped at 6h, so multi-tens-of-GB diffusion/video checkpoints get the minutes they need without a fixed cliff. Set this only to pin a specific budget; the value is used verbatim, including when it is shorter than the derived one." group:"distributed"` ModelLoadWait string `env:"LOCALAI_MODEL_LOAD_WAIT" help:"How long an inference request waits for a model that is still cold-loading onto a worker before it is answered with 503, a Retry-After header and live staging progress (default 60s). The request is served the moment the model becomes ready, so a model already most of the way staged needs no client retry. Set to 0 to wait as long as the load takes — only safe when no ingress or load balancer with an idle timeout sits in front." group:"distributed"` StaleNodeThreshold string `env:"LOCALAI_STALE_NODE_THRESHOLD" help:"How long a worker node may go without a durable heartbeat before the health monitor marks it offline (default 5m). Because a beat that only carries a fresher timestamp is held back by --node-heartbeat-checkpoint, this must stay comfortably wider than that interval; raise both together. Dead-node detection through the per-model gRPC health check and through request-time failure is unaffected by this knob." group:"distributed"` NodeHeartbeatCheckpoint string `env:"LOCALAI_NODE_HEARTBEAT_CHECKPOINT" help:"Minimum gap between durable heartbeat writes for a worker node (default 60s). A beat that only carries a fresher timestamp is dropped until this interval elapses; every field is compared against the value last written, so a node's first beat, a changed total VRAM/total disk/GPU vendor, and a free VRAM/RAM/disk reading that has moved more than 256 MiB from the written value all still write immediately, and a node that is not active is never suppressed. Set below the worker heartbeat interval to write on every beat." group:"distributed"` WorkerReconnectGrace string `env:"LOCALAI_WORKER_RECONNECT_GRACE" help:"How long a worker whose tunnel was lost is treated as reconnecting rather than gone (default 90s, clear of two of the worker's own ceiling backoffs plus the dial between them). Only after this window may the scheduler stop placing work on that worker and clean up its rows, so a value below the worker's backoff condemns workers that are re-homing normally; raise it to make a rolling frontend restart safer, lower it to reap a genuinely dead worker sooner. Measured on the database clock, so every replica agrees." group:"distributed"` - NatsAccountSeed string `env:"LOCALAI_NATS_ACCOUNT_SEED" help:"NATS account signing seed (SU...) used to mint per-node worker JWTs at registration" group:"distributed"` - NatsServiceJWT string `env:"LOCALAI_NATS_SERVICE_JWT" help:"NATS user JWT for the frontend (and agent workers) to publish control-plane messages" group:"distributed"` - NatsServiceSeed string `env:"LOCALAI_NATS_SERVICE_SEED" help:"NATS user signing seed (SU...) paired with LOCALAI_NATS_SERVICE_JWT" group:"distributed"` - NatsWorkerJWTTTL string `env:"LOCALAI_NATS_WORKER_JWT_TTL" help:"Lifetime of minted per-node NATS JWTs (e.g. 24h, default 24h)" group:"distributed"` - NatsRequireAuth bool `env:"LOCALAI_NATS_REQUIRE_AUTH" default:"false" help:"Require NATS JWT credentials (service JWT + account seed) when distributed mode is enabled" group:"distributed"` - NatsTLSCA string `env:"LOCALAI_NATS_TLS_CA" type:"existingfile" help:"PEM file for NATS server CA (private PKI); use with tls:// in --nats-url" group:"distributed"` - NatsTLSCert string `env:"LOCALAI_NATS_TLS_CERT" type:"existingfile" help:"Client certificate for NATS mTLS" group:"distributed"` - NatsTLSKey string `env:"LOCALAI_NATS_TLS_KEY" type:"existingfile" help:"Client private key for NATS mTLS" group:"distributed"` - ExposeNodeHeader bool `env:"LOCALAI_EXPOSE_NODE_HEADER" default:"false" help:"Set the X-LocalAI-Node response header on inference responses (OpenAI chat/completions/embeddings, Anthropic /v1/messages, Ollama /api/chat,/api/generate,/api/embed) with the ID of the worker that served the request. Disabled by default: the node ID reveals internal topology and should not be exposed on a public endpoint. Best-effort: under heavy concurrency the header may reflect a recent routing decision rather than this exact request's." group:"distributed"` - ModelScheduling string `env:"LOCALAI_MODEL_SCHEDULING" help:"Declarative per-model scheduling config applied at startup (inline JSON list of {model_name,node_selector,min_replicas,max_replicas,replicas:\"all\"}). Authoritative: overwrites matching models on every boot. Distributed mode only." group:"distributed"` - ModelSchedulingConfig string `env:"LOCALAI_MODEL_SCHEDULING_CONFIG" help:"Path to a YAML file with the same per-model scheduling list as LOCALAI_MODEL_SCHEDULING. Distributed mode only." group:"distributed"` + // The broker credential and TLS flags, accepted and ignored, hidden, on the + // same terms as --nats-url above and for the same reason: kong fails on an + // unknown flag, so deleting them turns every existing unit file, compose + // file and Helm values file into a startup parse error at upgrade, in + // exchange for nothing. There is no bus connection to present a credential + // on and no minting left to do. + // + // type:"existingfile" is deliberately NOT kept on the TLS paths. Validating + // a path this process never opens would fail a deployment at startup over a + // certificate for a broker the operator has already shut down, which is + // exactly the upgrade the acceptance exists to survive. + NatsAccountSeed string `env:"LOCALAI_NATS_ACCOUNT_SEED" help:"Ignored. The frontend mints no per-node broker credential; nodes are authenticated by their registration and tunnel tokens." group:"distributed" hidden:""` + NatsServiceJWT string `env:"LOCALAI_NATS_SERVICE_JWT" help:"Ignored. The frontend opens no bus connection to present a credential on." group:"distributed" hidden:""` + NatsServiceSeed string `env:"LOCALAI_NATS_SERVICE_SEED" help:"Ignored. Paired with LOCALAI_NATS_SERVICE_JWT, which is itself ignored." group:"distributed" hidden:""` + NatsWorkerJWTTTL string `env:"LOCALAI_NATS_WORKER_JWT_TTL" help:"Ignored. No per-node broker credential is minted, so none has a lifetime." group:"distributed" hidden:""` + NatsRequireAuth bool `env:"LOCALAI_NATS_REQUIRE_AUTH" default:"false" help:"Ignored on the frontend. Use --registration-require-auth, or the --distributed-require-auth umbrella, to fail closed on a missing registration token." group:"distributed" hidden:""` + NatsTLSCA string `env:"LOCALAI_NATS_TLS_CA" help:"Ignored. No bus connection is opened, so no server certificate is verified." group:"distributed" hidden:""` + NatsTLSCert string `env:"LOCALAI_NATS_TLS_CERT" help:"Ignored. No bus connection is opened, so no client certificate is presented." group:"distributed" hidden:""` + NatsTLSKey string `env:"LOCALAI_NATS_TLS_KEY" help:"Ignored. Paired with LOCALAI_NATS_TLS_CERT, which is itself ignored." group:"distributed" hidden:""` + ExposeNodeHeader bool `env:"LOCALAI_EXPOSE_NODE_HEADER" default:"false" help:"Set the X-LocalAI-Node response header on inference responses (OpenAI chat/completions/embeddings, Anthropic /v1/messages, Ollama /api/chat,/api/generate,/api/embed) with the ID of the worker that served the request. Disabled by default: the node ID reveals internal topology and should not be exposed on a public endpoint. Best-effort: under heavy concurrency the header may reflect a recent routing decision rather than this exact request's." group:"distributed"` + ModelScheduling string `env:"LOCALAI_MODEL_SCHEDULING" help:"Declarative per-model scheduling config applied at startup (inline JSON list of {model_name,node_selector,min_replicas,max_replicas,replicas:\"all\"}). Authoritative: overwrites matching models on every boot. Distributed mode only." group:"distributed"` + ModelSchedulingConfig string `env:"LOCALAI_MODEL_SCHEDULING_CONFIG" help:"Path to a YAML file with the same per-model scheduling list as LOCALAI_MODEL_SCHEDULING. Distributed mode only." group:"distributed"` Version bool @@ -365,9 +376,6 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { if r.DistributedAdvertiseAddr != "" { opts = append(opts, config.WithDistributedAdvertiseAddr(r.DistributedAdvertiseAddr)) } - if r.NatsURL != "" { - opts = append(opts, config.WithNatsURL(r.NatsURL)) - } if r.StorageURL != "" { opts = append(opts, config.WithStorageURL(r.StorageURL)) } @@ -444,34 +452,6 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { if r.DistributedSharedModels { opts = append(opts, config.EnableDistributedSharedModels) } - if r.NatsAccountSeed != "" { - opts = append(opts, config.WithNatsAccountSeed(r.NatsAccountSeed)) - } - if r.NatsServiceJWT != "" { - opts = append(opts, config.WithNatsServiceJWT(r.NatsServiceJWT)) - } - if r.NatsServiceSeed != "" { - opts = append(opts, config.WithNatsServiceSeed(r.NatsServiceSeed)) - } - if r.NatsWorkerJWTTTL != "" { - d, err := time.ParseDuration(r.NatsWorkerJWTTTL) - if err != nil { - return fmt.Errorf("invalid LOCALAI_NATS_WORKER_JWT_TTL %q: %w", r.NatsWorkerJWTTTL, err) - } - opts = append(opts, config.WithNatsWorkerJWTTTL(d)) - } - if r.NatsRequireAuth { - opts = append(opts, config.EnableNatsRequireAuth) - } - if r.NatsTLSCA != "" { - opts = append(opts, config.WithNatsTLSCA(r.NatsTLSCA)) - } - if r.NatsTLSCert != "" { - opts = append(opts, config.WithNatsTLSCert(r.NatsTLSCert)) - } - if r.NatsTLSKey != "" { - opts = append(opts, config.WithNatsTLSKey(r.NatsTLSKey)) - } if r.AutoApproveNodes { opts = append(opts, config.EnableAutoApproveNodes) } diff --git a/core/cli/run_bus_flags_test.go b/core/cli/run_bus_flags_test.go new file mode 100644 index 000000000000..c0aefa3239fa --- /dev/null +++ b/core/cli/run_bus_flags_test.go @@ -0,0 +1,172 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + + "github.com/alecthomas/kong" + + "github.com/mudler/LocalAI/core/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The frontend's broker flags are ACCEPTED and IGNORED, and the two halves are +// separate Its on purpose. +// +// Accepted, because kong fails on an unknown flag: deleting --nats-url would +// turn every existing unit file, compose file and Helm chart into a startup +// parse error on the day of the upgrade, in exchange for nothing, since the +// flag has nothing left to do. Ignored, because a flag that parses and is then +// quietly honoured is the failure this spec exists to exclude, and a single +// "it starts" assertion cannot tell the two apart. +// runFlagVars supplies the kong variables cmd/local-ai/main.go supplies, so a +// RunCMD can be parsed here at all: its path defaults interpolate ${basepath}. +func runFlagVars() kong.Vars { + return kong.Vars{ + "basepath": GinkgoT().TempDir(), + "generatedcontentpath": DefaultGeneratedContentPath(), + "uploadpath": DefaultUploadPath(), + "galleries": config.DefaultGalleriesJSON, + "backends": config.DefaultBackendGalleriesJSON, + "version": "test", + } +} + +var _ = Describe("The frontend's broker flags", func() { + busFlags := []string{ + "--nats-url", "nats://bus:4222", + "--nats-account-seed", "SUACCOUNT", + "--nats-service-jwt", "eyJ0", + "--nats-service-seed", "SUSERVICE", + "--nats-worker-jwtttl", "24h", + "--nats-require-auth", + } + + parse := func(args ...string) (*RunCMD, error) { + // kong resolves env: tags from the process environment, so a + // LOCALAI_NATS_URL inherited from a developer's shell would let the + // first spec pass for the wrong reason. + for _, name := range []string{"LOCALAI_NATS_URL", "LOCALAI_NATS_ACCOUNT_SEED", "LOCALAI_NATS_REQUIRE_AUTH"} { + if prior, had := os.LookupEnv(name); had { + Expect(os.Unsetenv(name)).To(Succeed()) + DeferCleanup(func() { _ = os.Setenv(name, prior) }) + } + } + var cli struct { + Run RunCMD `cmd:""` + } + parser, err := kong.New(&cli, runFlagVars()) + Expect(err).ToNot(HaveOccurred()) + _, err = parser.Parse(append([]string{"run"}, args...)) + return &cli.Run, err + } + + It("starts with no bus named at all", func() { + _, err := parse("--distributed") + Expect(err).To(Succeed(), + "a distributed frontend dials no message bus and must not demand the URL of one") + }) + + It("still accepts a command line that names one", func() { + _, err := parse(append([]string{"--distributed"}, busFlags...)...) + Expect(err).To(Succeed()) + }) + + It("does not stat the TLS material it no longer presents", func() { + // The paths were validated as existing files while they were dialled + // with. Keeping that validation on an ignored flag would fail a + // deployment at startup over a certificate for a broker that is gone, + // which is the exact upgrade the acceptance is meant to survive. + missing := filepath.Join(GinkgoT().TempDir(), "a-broker-ca-that-was-deleted.pem") + _, err := parse("--distributed", + "--nats-tlsca", missing, + "--nats-tls-cert", missing, + "--nats-tls-key", missing) + Expect(err).To(Succeed()) + }) + + It("keeps every accepted bus flag hidden from --help", func() { + // Accepted for the upgrade, not offered to a new operator: a flag that + // does nothing must not appear in the list of things to configure. + var cli struct { + Run RunCMD `cmd:""` + } + parser, err := kong.New(&cli, runFlagVars()) + Expect(err).ToNot(HaveOccurred()) + var visible []string + for _, node := range parser.Model.Children { + for _, flag := range node.Flags { + if len(flag.Name) >= 5 && flag.Name[:5] == "nats-" && !flag.Hidden { + visible = append(visible, flag.Name) + } + } + } + Expect(visible).To(BeEmpty(), + "%v are still offered in --help while doing nothing", visible) + }) +}) + +// The serve-backend worker's half of the same promise, which nothing pinned. +// +// `local-ai worker` kept ONE broker flag and dropped the rest a release earlier, +// when it stopped connecting to a broker at all. That asymmetry is documented +// in docs/content/reference/cli-reference.md, and a documented promise with no +// spec is how the wrong half gets deleted: --nats-url is the one an operator's +// worker unit file actually carries, and it is the one whose removal would turn +// an upgrade into a parse error on every worker in the fleet at once. +// +// The negative It is here for the same reason the frontend's two halves are +// separate. Without it, "accepted" could be satisfied by quietly re-adding the +// credential flags, and the docs would be describing a surface nobody checked. +var _ = Describe("The serve-backend worker's broker flags", func() { + parse := func(args ...string) error { + for _, name := range []string{"LOCALAI_NATS_URL", "LOCALAI_ADDRESS"} { + if prior, had := os.LookupEnv(name); had { + Expect(os.Unsetenv(name)).To(Succeed()) + DeferCleanup(func() { _ = os.Setenv(name, prior) }) + } + } + var cli struct { + Worker WorkerCMD `cmd:""` + } + parser, err := kong.New(&cli, runFlagVars()) + Expect(err).ToNot(HaveOccurred()) + _, err = parser.Parse(append([]string{"worker"}, args...)) + return err + } + + It("still accepts the bus URL an existing worker unit file carries", func() { + Expect(parse("--register-to", "http://frontend:8080", "--nats-url", "nats://bus:4222")).To(Succeed(), + "a serve-backend worker dials no message bus, and an operator whose unit file still names one must still get a worker that starts") + }) + + It("keeps it hidden from --help", func() { + var cli struct { + Worker WorkerCMD `cmd:""` + } + parser, err := kong.New(&cli, runFlagVars()) + Expect(err).ToNot(HaveOccurred()) + var visible []string + for _, node := range parser.Model.Children { + for _, flag := range node.Flags { + if strings.HasPrefix(flag.Name, "nats-") && !flag.Hidden { + visible = append(visible, flag.Name) + } + } + } + Expect(visible).To(BeEmpty(), + "%v are still offered in --help while doing nothing", visible) + }) + + It("took no broker CREDENTIAL flag back", func() { + // The credential and TLS flags left this command in phase 3 and must + // stay gone: re-adding one would put a broker credential back on the + // surface of a process that opens no broker connection, and the docs + // say per-command which flags survive. + Expect(parse("--register-to", "http://frontend:8080", "--nats-jwt", "eyJ0")).To(HaveOccurred()) + Expect(parse("--register-to", "http://frontend:8080", "--nats-service-jwt", "eyJ0")).To(HaveOccurred()) + Expect(parse("--register-to", "http://frontend:8080", "--nats-tlsca", "/dev/null")).To(HaveOccurred()) + }) +}) diff --git a/core/cli/worker/worker_vllm.go b/core/cli/worker/worker_vllm.go index 8596546b5849..1471d780fb97 100644 --- a/core/cli/worker/worker_vllm.go +++ b/core/cli/worker/worker_vllm.go @@ -96,7 +96,7 @@ func (r *VLLMDistributed) Run(ctx *cliContext.Context) error { FrontendURL: r.RegisterTo, RegistrationToken: r.RegistrationToken, } - nodeID, _, _, _, regErr := regClient.RegisterWithRetry(context.Background(), r.registrationBody(), 10) + nodeID, _, regErr := regClient.RegisterWithRetry(context.Background(), r.registrationBody(), 10) if regErr != nil { return fmt.Errorf("registering with frontend: %w", regErr) } diff --git a/core/cli/workerregistry/client.go b/core/cli/workerregistry/client.go index fb00fb3f1d86..1073b46ae1a7 100644 --- a/core/cli/workerregistry/client.go +++ b/core/cli/workerregistry/client.go @@ -68,15 +68,19 @@ type RegisterResponse struct { // hash, so this is the ONLY time the plaintext exists anywhere but in this // worker's memory: a worker that discards it cannot get it back without // registering again. - TunnelToken string `json:"tunnel_token,omitempty"` - NatsJWT string `json:"nats_jwt,omitempty"` - NatsUserSeed string `json:"nats_user_seed,omitempty"` + TunnelToken string `json:"tunnel_token,omitempty"` + // There are no nats_jwt / nats_user_seed fields. A frontend that predates + // this release still sends them and this decodes fine: encoding/json + // ignores a key with no field, so an old frontend talking to a new worker + // is a no-op rather than a decode failure. } // RegisterFull sends a single registration request and returns the full -// response (node ID, approval status, and optional API token / NATS creds). +// response (node ID, approval status, and optional API and tunnel tokens). // Re-registration is idempotent: the frontend preserves the node row and mints -// a fresh NATS JWT each call, so this doubles as the credential-refresh call. +// a fresh TUNNEL token each call, so this doubles as the rotation call. It is +// the only credential a registration mints; the per-node broker JWT it used to +// carry went with the bus. func (c *RegistrationClient) RegisterFull(ctx context.Context, body map[string]any) (*RegisterResponse, error) { jsonBody, _ := json.Marshal(body) url := c.baseURL() + "/api/node/register" @@ -157,26 +161,20 @@ func isRegistrationRejection(status int) bool { return status >= 400 && status < 500 } -// Register sends a single registration request and returns the node ID and -// optional credentials (API token for agent workers, NATS JWT when configured). -func (c *RegistrationClient) Register(ctx context.Context, body map[string]any) (nodeID, apiToken, natsJWT, natsSeed string, err error) { - res, err := c.RegisterFull(ctx, body) - if err != nil { - return "", "", "", "", err - } - return res.ID, res.APIToken, res.NatsJWT, res.NatsUserSeed, nil -} - // RegisterWithRetry retries registration with exponential backoff. // // It drops every field of the response it does not name, the tunnel credential // among them. Callers that need one use RegisterFullWithRetry. -func (c *RegistrationClient) RegisterWithRetry(ctx context.Context, body map[string]any, maxRetries int) (nodeID, apiToken, natsJWT, natsSeed string, err error) { +// +// The two broker-credential returns it used to carry are gone with the bus, and +// so is the Register one-shot that existed only to carry them: it had no caller +// left once nothing dialled a broker. +func (c *RegistrationClient) RegisterWithRetry(ctx context.Context, body map[string]any, maxRetries int) (nodeID, apiToken string, err error) { res, err := c.RegisterFullWithRetry(ctx, body, maxRetries) if err != nil { - return "", "", "", "", err + return "", "", err } - return res.ID, res.APIToken, res.NatsJWT, res.NatsUserSeed, nil + return res.ID, res.APIToken, nil } // RegisterFullWithRetry retries registration with exponential backoff and diff --git a/core/cli/workerregistry/client_test.go b/core/cli/workerregistry/client_test.go index 5870d2524b5c..f666e5f2b529 100644 --- a/core/cli/workerregistry/client_test.go +++ b/core/cli/workerregistry/client_test.go @@ -5,6 +5,7 @@ import ( "errors" "net/http" "net/http/httptest" + "reflect" "sync/atomic" "time" @@ -128,7 +129,7 @@ var _ = Describe("Registration client refusals", func() { // The default worker path goes through Acquire, not the ladder above, // and its bound is 100 attempts rather than 10. A refusal there is the // same verdict and has to end the same way. - mgr := NewNATSCredentialManager(func(ctx context.Context) (*RegisterResponse, error) { + mgr := NewCredentialManager(func(ctx context.Context) (*RegisterResponse, error) { return client.RegisterFull(ctx, map[string]any{"name": "w1"}) }, true) _, err := mgr.Acquire(context.Background()) @@ -136,3 +137,41 @@ var _ = Describe("Registration client refusals", func() { Expect(attempts.Load()).To(Equal(int32(1))) }) }) + +// The other direction of the same upgrade: a worker of this release registering +// against a frontend that still mints a per-node broker credential. +// +// The fields are gone from RegisterResponse, so the only question is what +// happens to the keys still on the wire. encoding/json ignores a key with no +// field, and that is asserted rather than assumed, because a decoder switched +// to DisallowUnknownFields would turn every registration against an older +// frontend into a hard failure with no other symptom. +var _ = Describe("Registering against a frontend that still mints broker credentials", func() { + It("decodes the response and drops the keys it no longer has fields for", func() { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"node-7","status":"healthy","api_token":"key-7","tunnel_token":"tunnel-7","nats_jwt":"eyJ0","nats_user_seed":"SUUSER"}`)) + })) + DeferCleanup(server.Close) + + client := &RegistrationClient{FrontendURL: server.URL, HTTPTimeout: 2 * time.Second} + res, err := client.RegisterFull(context.Background(), map[string]any{"name": "w1"}) + Expect(err).ToNot(HaveOccurred()) + + // The fields it DOES have, so the assertion below is about the two + // unknown keys and not about a decode that produced nothing. + Expect(res.ID).To(Equal("node-7")) + Expect(res.APIToken).To(Equal("key-7")) + Expect(res.TunnelToken).To(Equal("tunnel-7")) + + // And nowhere for a broker credential to land: asserted on the struct's + // own type, because a value assertion would need a field to read and + // would stop compiling exactly when the field came back. + t := reflect.TypeOf(*res) + for _, gone := range []string{"NatsJWT", "NatsUserSeed"} { + _, found := t.FieldByName(gone) + Expect(found).To(BeFalse(), + "RegisterResponse.%s is back: a worker that stores a broker credential is a worker something expects to dial a broker", gone) + } + }) +}) diff --git a/core/cli/workerregistry/credentials.go b/core/cli/workerregistry/credentials.go index b023b9916918..2a45673bc910 100644 --- a/core/cli/workerregistry/credentials.go +++ b/core/cli/workerregistry/credentials.go @@ -7,7 +7,6 @@ import ( "sync" "time" - "github.com/mudler/LocalAI/pkg/natsauth" "github.com/mudler/xlog" ) @@ -16,40 +15,44 @@ import ( // package (and its gorm/DB dependencies). const statusPending = "pending" -// defaultMaxAttempts bounds how many times Acquire registers (and how many -// consecutive times RefreshLoop may fail) before giving up. It is high enough -// to ride out a slow admin approval or a transient frontend outage, but finite -// so an unauthorized/unapprovable worker exits and surfaces the problem (via a -// non-zero exit and the resulting restart) rather than waiting forever. +// defaultMaxAttempts bounds how many times Acquire registers before giving up. +// It is high enough to ride out a slow admin approval or a transient frontend +// outage, but finite so an unauthorized/unapprovable worker exits and surfaces +// the problem (via a non-zero exit and the resulting restart) rather than +// waiting forever. const defaultMaxAttempts = 100 // RegisterFunc performs one idempotent registration round-trip. type RegisterFunc func(ctx context.Context) (*RegisterResponse, error) -// NATSCredentialManager acquires NATS credentials at startup — waiting through -// admin approval when required — and refreshes them before the minted JWT -// expires, by re-registering (which mints a fresh JWT). The live NATS -// connection adopts a refreshed JWT on its next reconnect via Provider. Safe -// for concurrent use. +// CredentialManager acquires a node's own credentials at startup, waiting +// through admin approval when that is required, and holds the tunnel token the +// most recent registration minted. Safe for concurrent use. // -// It addresses two failure modes: a worker that needs credentials but registers -// while still pending approval (it would otherwise give up and never connect), -// and a long-running worker whose 24h JWT expires with no way to renew it. -type NATSCredentialManager struct { - register RegisterFunc - requireCreds bool // block until credentials are present (frontend minting in use) - - // Tunables; defaults set by NewNATSCredentialManager, overridable in tests. +// Renamed from NATSCredentialManager and stripped rather than deleted. The JWT +// half went with the message bus: nothing mints a broker credential and nothing +// opens a connection to present one on. The tunnel token did not go with it, +// and it is the reason a manager is still worth having: it is ROTATED by a +// registration rather than expiring on a clock, and the frontend keeps only its +// hash, so the dialer has to read the current value at dial time instead of +// being handed one at startup. +type CredentialManager struct { + register RegisterFunc + // requireApproval blocks Acquire until the node is out of pending. + // + // Narrower than the requireCreds it replaces: there is no credential left + // to wait for being MINTED, only an admin decision to wait through. A + // worker that proceeds while pending registers and heartbeats fine, and is + // then refused at every tunnel dial, so an operator who wants the wait + // rather than the refusal loop asks for it here. + requireApproval bool + + // Tunables; defaults set by NewCredentialManager, overridable in tests. initialBackoff time.Duration maxBackoff time.Duration - maxAttempts int // bound on Acquire attempts / consecutive refresh failures (<=0 = unlimited) - refreshLead float64 // refresh once this fraction of the JWT lifetime has elapsed - refreshRetry time.Duration - expiryOf func(jwt string) (time.Time, bool) + maxAttempts int // bound on Acquire attempts (<=0 = unlimited) mu sync.RWMutex - jwt string - seed string nodeID string // tunnelToken is the node's own tunnel credential from the most recent // registration. It is kept here because every re-registration this manager @@ -58,46 +61,28 @@ type NATSCredentialManager struct { tunnelToken string } -// NewNATSCredentialManager builds a manager over register. When requireCreds is -// true, Acquire blocks until the node is approved and credentials are minted. -func NewNATSCredentialManager(register RegisterFunc, requireCreds bool) *NATSCredentialManager { - return &NATSCredentialManager{ - register: register, - requireCreds: requireCreds, - initialBackoff: 2 * time.Second, - maxBackoff: 30 * time.Second, - maxAttempts: defaultMaxAttempts, - refreshLead: 0.75, - refreshRetry: 30 * time.Second, - expiryOf: jwtExpiry, - } -} - -// jwtExpiry decodes the expiry of a minted user JWT. ok is false when the token -// is empty/undecodable or carries no expiry (e.g. a non-expiring service JWT). -func jwtExpiry(token string) (time.Time, bool) { - if token == "" { - return time.Time{}, false +// NewCredentialManager builds a manager over register. When requireApproval is +// true, Acquire blocks through admin approval instead of returning a pending +// response. +func NewCredentialManager(register RegisterFunc, requireApproval bool) *CredentialManager { + return &CredentialManager{ + register: register, + requireApproval: requireApproval, + initialBackoff: 2 * time.Second, + maxBackoff: 30 * time.Second, + maxAttempts: defaultMaxAttempts, } - uc, err := natsauth.DecodeUserClaims(token) - if err != nil || uc.Expires == 0 { - return time.Time{}, false - } - return time.Unix(uc.Expires, 0), true } -func (m *NATSCredentialManager) store(res *RegisterResponse) { +func (m *CredentialManager) store(res *RegisterResponse) { m.mu.Lock() defer m.mu.Unlock() m.nodeID = res.ID - if res.NatsJWT != "" && res.NatsUserSeed != "" { - m.jwt, m.seed = res.NatsJWT, res.NatsUserSeed - } - // Guarded the same way the NATS pair is: a response that carries no tunnel - // token (a frontend that predates them, or one whose minting failed) must - // not wipe a working credential this worker already holds. Overwriting with - // "" would lock the tunnel out until the next registration that did carry - // one, which is the opposite of what an empty field means. + // A response that carries no tunnel token (a frontend that predates them, + // or one whose minting failed) must not wipe a working credential this + // worker already holds. Overwriting with "" would lock the tunnel out until + // the next registration that did carry one, which is the opposite of what + // an empty field means. if res.TunnelToken != "" { m.tunnelToken = res.TunnelToken } @@ -105,43 +90,23 @@ func (m *NATSCredentialManager) store(res *RegisterResponse) { // TunnelToken returns the node's current tunnel credential, empty until one has // been issued. It is the callback the tunnel client reads on every dial. -func (m *NATSCredentialManager) TunnelToken() string { +func (m *CredentialManager) TunnelToken() string { m.mu.RLock() defer m.mu.RUnlock() return m.tunnelToken } -// Current returns the latest NATS credentials (both empty until acquired). -func (m *NATSCredentialManager) Current() (jwt, seed string) { - m.mu.RLock() - defer m.mu.RUnlock() - return m.jwt, m.seed -} - // NodeID returns the node ID from the most recent registration. -func (m *NATSCredentialManager) NodeID() string { +func (m *CredentialManager) NodeID() string { m.mu.RLock() defer m.mu.RUnlock() return m.nodeID } -// Provider returns a callback compatible with messaging.WithUserJWTProvider, -// supplying the current credentials on each (re)connect. -func (m *NATSCredentialManager) Provider() func() (string, string) { - return m.Current -} - -// HasCredentials reports whether complete NATS credentials have been obtained. -func (m *NATSCredentialManager) HasCredentials() bool { - jwt, seed := m.Current() - return jwt != "" && seed != "" -} - -// Acquire registers and, when requireCreds is set, keeps re-registering with -// exponential backoff until the node is approved (status != pending) and -// credentials are minted. Without requireCreds it returns the first successful -// response (the historical one-shot behavior, preserved for anonymous NATS). -func (m *NATSCredentialManager) Acquire(ctx context.Context) (*RegisterResponse, error) { +// Acquire registers and, when requireApproval is set, keeps re-registering with +// exponential backoff until the node is approved (status != pending). Without +// requireApproval it returns the first successful response. +func (m *CredentialManager) Acquire(ctx context.Context) (*RegisterResponse, error) { backoff := m.initialBackoff var lastReason error for attempt := 1; m.maxAttempts <= 0 || attempt <= m.maxAttempts; attempt++ { @@ -155,15 +120,12 @@ func (m *NATSCredentialManager) Acquire(ctx context.Context) (*RegisterResponse, case err != nil: lastReason = err xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", backoff, "error", err) - case !m.requireCreds: + case !m.requireApproval: m.store(res) return res, nil case res.Status == statusPending: lastReason = fmt.Errorf("node %s still pending admin approval", res.ID) xlog.Info("Node pending admin approval; waiting", "node", res.ID, "attempt", attempt, "next_retry", backoff) - case res.NatsJWT == "" || res.NatsUserSeed == "": - lastReason = fmt.Errorf("node %s approved but NATS credentials not minted", res.ID) - xlog.Info("Node approved but NATS credentials not yet minted; waiting", "node", res.ID, "attempt", attempt, "next_retry", backoff) default: m.store(res) return res, nil @@ -175,53 +137,5 @@ func (m *NATSCredentialManager) Acquire(ctx context.Context) (*RegisterResponse, } backoff = min(backoff*2, m.maxBackoff) } - return nil, fmt.Errorf("giving up acquiring NATS credentials after %d attempts: %w", m.maxAttempts, lastReason) -} - -// RefreshLoop re-registers to mint a fresh JWT before the current one expires, -// updating the credentials returned by Current/Provider so the NATS connection -// adopts them on its next reconnect. It returns nil when ctx is cancelled or -// when the current credential has no expiry (nothing to refresh), and a non-nil -// error after maxAttempts consecutive refresh failures — letting the caller -// exit the worker so it restarts and re-acquires (or surfaces the outage) -// rather than silently drifting toward an expired, unrenewable JWT. -func (m *NATSCredentialManager) RefreshLoop(ctx context.Context) error { - failures := 0 - for { - jwt, _ := m.Current() - exp, ok := m.expiryOf(jwt) - if !ok { - xlog.Debug("NATS credential has no expiry; refresh loop exiting") - return nil - } - wait := max(time.Duration(float64(time.Until(exp))*m.refreshLead), 0) - select { - case <-ctx.Done(): - return nil - case <-time.After(wait): - } - - res, err := m.register(ctx) - if err == nil && res.NatsJWT != "" && res.NatsUserSeed != "" { - m.store(res) - failures = 0 - xlog.Info("Refreshed NATS credentials", "node", res.ID) - continue - } - failures++ - if err != nil { - xlog.Warn("NATS credential refresh failed; will retry", "attempt", failures, "error", err) - } else { - xlog.Warn("NATS credential refresh returned no credentials; will retry", "attempt", failures) - } - if m.maxAttempts > 0 && failures >= m.maxAttempts { - return fmt.Errorf("NATS credential refresh failed %d times in a row", failures) - } - // Back off before retrying so a persistent failure near expiry does not spin. - select { - case <-ctx.Done(): - return nil - case <-time.After(m.refreshRetry): - } - } + return nil, fmt.Errorf("giving up registering after %d attempts: %w", m.maxAttempts, lastReason) } diff --git a/core/cli/workerregistry/credentials_test.go b/core/cli/workerregistry/credentials_test.go index 02ed413c4e8f..cbc372d9db01 100644 --- a/core/cli/workerregistry/credentials_test.go +++ b/core/cli/workerregistry/credentials_test.go @@ -6,9 +6,6 @@ import ( "testing" "time" - "github.com/mudler/LocalAI/pkg/natsauth" - "github.com/nats-io/nkeys" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -50,20 +47,20 @@ func (f *fakeRegister) count() int { return f.calls } -var _ = Describe("NATSCredentialManager", func() { - approved := func(jwt, seed string) *RegisterResponse { - return &RegisterResponse{ID: "node-1", Status: "healthy", NatsJWT: jwt, NatsUserSeed: seed} +var _ = Describe("CredentialManager", func() { + approved := func(tunnelToken string) *RegisterResponse { + return &RegisterResponse{ID: "node-1", Status: "healthy", TunnelToken: tunnelToken} } pending := &RegisterResponse{ID: "node-1", Status: "pending"} - Describe("Acquire (#4 — wait through admin approval)", func() { - It("keeps re-registering until the node is approved and credentials are minted", func() { + Describe("Acquire (wait through admin approval)", func() { + It("keeps re-registering until the node is approved", func() { f := &fakeRegister{steps: []step{ - {res: pending}, // not approved yet - {res: approved("", "")}, // approved but JWT not minted yet - {res: approved("jwt-1", "seed-1")}, // finally minted + {res: pending}, // not approved yet + {res: pending}, // still not approved + {res: approved("tunnel-1")}, // approved, and handed its tunnel credential }} - m := NewNATSCredentialManager(f.fn(), true /* requireCreds */) + m := NewCredentialManager(f.fn(), true /* requireApproval */) m.initialBackoff = time.Millisecond m.maxBackoff = time.Millisecond @@ -71,28 +68,24 @@ var _ = Describe("NATSCredentialManager", func() { Expect(err).ToNot(HaveOccurred()) Expect(res.ID).To(Equal("node-1")) Expect(f.count()).To(Equal(3)) - - jwt, seed := m.Current() - Expect(jwt).To(Equal("jwt-1")) - Expect(seed).To(Equal("seed-1")) - Expect(m.HasCredentials()).To(BeTrue()) + Expect(m.TunnelToken()).To(Equal("tunnel-1")) Expect(m.NodeID()).To(Equal("node-1")) }) - It("returns immediately on the first success when credentials are not required (anonymous NATS)", func() { + It("returns immediately on the first success when approval is not required", func() { f := &fakeRegister{steps: []step{{res: pending}}} - m := NewNATSCredentialManager(f.fn(), false /* requireCreds */) + m := NewCredentialManager(f.fn(), false /* requireApproval */) res, err := m.Acquire(context.Background()) Expect(err).ToNot(HaveOccurred()) Expect(res.Status).To(Equal("pending")) Expect(f.count()).To(Equal(1)) - Expect(m.HasCredentials()).To(BeFalse()) + Expect(m.TunnelToken()).To(BeEmpty()) }) It("aborts when the context is cancelled while waiting for approval", func() { f := &fakeRegister{steps: []step{{res: pending}}} - m := NewNATSCredentialManager(f.fn(), true) + m := NewCredentialManager(f.fn(), true) m.initialBackoff = 10 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) @@ -103,7 +96,7 @@ var _ = Describe("NATSCredentialManager", func() { It("gives up after a bounded number of attempts so the worker exits and alerts", func() { f := &fakeRegister{steps: []step{{res: pending}}} // never approved - m := NewNATSCredentialManager(f.fn(), true) + m := NewCredentialManager(f.fn(), true) m.initialBackoff = time.Millisecond m.maxBackoff = time.Millisecond m.maxAttempts = 5 @@ -116,83 +109,48 @@ var _ = Describe("NATSCredentialManager", func() { }) }) - Describe("RefreshLoop (#5 — renew before the JWT expires)", func() { - It("re-registers before expiry and updates the credentials served to new connections", func() { - f := &fakeRegister{steps: []step{{res: approved("jwt-2", "seed-2")}}} - m := NewNATSCredentialManager(f.fn(), true) - m.refreshLead = 0.5 - m.refreshRetry = time.Millisecond - // jwt-1 expires soon; jwt-2 is long-lived so the loop then idles. - m.expiryOf = func(jwt string) (time.Time, bool) { - switch jwt { - case "jwt-1": - return time.Now().Add(40 * time.Millisecond), true - case "jwt-2": - return time.Now().Add(time.Hour), true - default: - return time.Time{}, false - } - } - m.store(approved("jwt-1", "seed-1")) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go func() { _ = m.RefreshLoop(ctx) }() - - Eventually(func() string { - jwt, _ := m.Current() - return jwt - }, "2s", "10ms").Should(Equal("jwt-2")) + // The behaviour that survived the rename, and which nothing pinned while + // the manager was about a JWT. + // + // The frontend mints a FRESH tunnel token on every registration and keeps + // only its hash, so the previous one stops working the moment a new one is + // issued. A manager that handed out the first value it saw would lock the + // worker out of its own tunnel after any re-registration, and the symptom + // would be a 401 on a dial rather than anything at registration time. + Describe("TunnelToken (rotation)", func() { + It("serves the token from the most recent registration, not the first", func() { + m := NewCredentialManager(nil, false) + m.store(approved("tunnel-1")) + Expect(m.TunnelToken()).To(Equal("tunnel-1")) + + m.store(approved("tunnel-2")) + Expect(m.TunnelToken()).To(Equal("tunnel-2"), + "the frontend keeps only the newest token's hash, so serving the first one locks this worker out of its own tunnel") }) - It("returns an error after the bounded number of consecutive failures so the caller can exit", func() { - f := &fakeRegister{steps: []step{{err: context.DeadlineExceeded}}} // refresh always fails - m := NewNATSCredentialManager(f.fn(), true) - m.refreshLead = 0.5 - m.refreshRetry = time.Millisecond - m.maxAttempts = 3 - m.expiryOf = func(string) (time.Time, bool) { return time.Now().Add(time.Millisecond), true } - m.store(approved("jwt-1", "seed-1")) - - errCh := make(chan error, 1) - go func() { errCh <- m.RefreshLoop(context.Background()) }() - Eventually(errCh, "2s").Should(Receive(MatchError(ContainSubstring("3 times in a row")))) - }) - - It("exits promptly when the current credential has no expiry (nothing to refresh)", func() { - f := &fakeRegister{steps: []step{{res: approved("x", "y")}}} - m := NewNATSCredentialManager(f.fn(), true) - m.expiryOf = func(string) (time.Time, bool) { return time.Time{}, false } - m.store(approved("static", "seed")) - - done := make(chan struct{}) - go func() { _ = m.RefreshLoop(context.Background()); close(done) }() - Eventually(done, "1s").Should(BeClosed()) - Expect(f.count()).To(Equal(0)) // never tried to re-register - }) - }) - - Describe("jwtExpiry default", func() { - It("decodes the expiry of a real minted worker JWT", func() { - akp, err := nkeys.CreateAccount() - Expect(err).ToNot(HaveOccurred()) - seed, err := akp.Seed() - Expect(err).ToNot(HaveOccurred()) - - cfg := natsauth.Config{AccountSeed: string(seed), WorkerJWTTTL: time.Hour} - token, _, err := cfg.MintWorkerJWT("node-1", "backend") - Expect(err).ToNot(HaveOccurred()) - - exp, ok := jwtExpiry(token) - Expect(ok).To(BeTrue()) - Expect(exp).To(BeTemporally("~", time.Now().Add(time.Hour), 2*time.Minute)) - }) - - It("reports no expiry for an empty or undecodable token", func() { - _, ok := jwtExpiry("") - Expect(ok).To(BeFalse()) - _, ok = jwtExpiry("not-a-jwt") - Expect(ok).To(BeFalse()) + It("keeps a working token when a registration carries none", func() { + // A frontend that predates tunnel tokens, or one whose minting + // failed, sends the field empty. Empty means "nothing new", not + // "revoked": overwriting would lock the tunnel out until the next + // registration that did carry one. + m := NewCredentialManager(nil, false) + m.store(approved("tunnel-1")) + m.store(approved("")) + Expect(m.TunnelToken()).To(Equal("tunnel-1")) }) }) }) + +// Deleted with the bus: the "RefreshLoop" Describe and the "jwtExpiry default" +// Describe. +// +// RefreshLoop pinned that a worker re-registers before its minted broker JWT +// expires and serves the new credential to the next connection. Every noun in +// that sentence is gone: there is no JWT, no expiry to read, and no connection +// to serve it to. It is retired rather than moved. The one credential a +// registration still mints, the tunnel token, is rotated by a REGISTRATION +// rather than by a clock, and the property that matters about it is that the +// dialer reads the current value, which the rotation Describe above pins. +// +// jwtExpiry pinned that a real minted worker JWT's expiry decoded, which was +// the only thing in this package that needed a broker library at all. diff --git a/core/config/distributed_config.go b/core/config/distributed_config.go index fdf90d477874..b34045c0fee4 100644 --- a/core/config/distributed_config.go +++ b/core/config/distributed_config.go @@ -5,13 +5,12 @@ import ( "fmt" "time" - "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/pkg/natsauth" "github.com/mudler/xlog" ) // DistributedConfig holds configuration for horizontal scaling mode. -// When Enabled is true, PostgreSQL and NATS are required. +// When Enabled is true, PostgreSQL is required. Nothing else is: fan-out rides +// PostgreSQL and every worker is reached over the tunnel it dials out. type DistributedConfig struct { Enabled bool // --distributed / LOCALAI_DISTRIBUTED InstanceID string // --instance-id / LOCALAI_INSTANCE_ID (auto-generated UUID if empty) @@ -22,22 +21,19 @@ type DistributedConfig struct { // answer is only usable when the database is remote, so a deployment with // a local or sidecar database has to set this. AdvertiseAddr string // LOCALAI_DISTRIBUTED_ADVERTISE_ADDR - // NatsURL is accepted and ignored. No component of a distributed - // deployment dials a message bus any more. - NatsURL string // --nats-url / LOCALAI_NATS_URL StorageURL string // --storage-url / LOCALAI_STORAGE_URL (S3 endpoint) RegistrationToken string // --registration-token / LOCALAI_REGISTRATION_TOKEN (required token for node registration) // RegistrationRequireAuth fails startup when distributed mode is enabled but // RegistrationToken is empty. The default (false) keeps the historical // fail-open behavior with a loud warning; production should set it so the // node-register endpoints and the worker file-transfer server cannot run - // unauthenticated. Mirrors NatsRequireAuth for the NATS bus. + // unauthenticated. RegistrationRequireAuth bool // LOCALAI_REGISTRATION_REQUIRE_AUTH // RequireAuth is the umbrella switch (LOCALAI_DISTRIBUTED_REQUIRE_AUTH) for - // distributed-mode auth: when true it implies BOTH NatsRequireAuth and - // RegistrationRequireAuth, so a single knob locks down the bus and the - // registration/file-transfer layer together. The granular flags remain - // available to enforce just one layer. + // distributed-mode auth. It implies RegistrationRequireAuth, and that is + // now ALL it implies: it used to imply a NatsRequireAuth as well, and there + // is no message bus left for that half to lock down. The granular flag + // remains available. RequireAuth bool // LOCALAI_DISTRIBUTED_REQUIRE_AUTH AutoApproveNodes bool // --auto-approve-nodes / LOCALAI_AUTO_APPROVE_NODES (skip admin approval for new workers) // SharedModels asserts that every node (frontend and workers) mounts the @@ -49,16 +45,6 @@ type DistributedConfig struct { // false preserves the historical per-node staging behavior. SharedModels bool // --distributed-shared-models / LOCALAI_DISTRIBUTED_SHARED_MODELS - // NATS JWT auth (optional; see pkg/natsauth and docs/features/distributed-mode.md) - NatsAccountSeed string // LOCALAI_NATS_ACCOUNT_SEED — account signing seed to mint per-node worker JWTs - NatsServiceJWT string // LOCALAI_NATS_SERVICE_JWT — user JWT for frontends / agent workers - NatsServiceSeed string // LOCALAI_NATS_SERVICE_SEED — signing seed paired with service JWT - NatsWorkerJWTTTL time.Duration // LOCALAI_NATS_WORKER_JWT_TTL — minted worker JWT lifetime (default 24h) - NatsRequireAuth bool // LOCALAI_NATS_REQUIRE_AUTH — fail startup if NATS credentials are missing - NatsTLSCA string // LOCALAI_NATS_TLS_CA — PEM file for private CA (server verify) - NatsTLSCert string // LOCALAI_NATS_TLS_CERT — client cert for NATS mTLS - NatsTLSKey string // LOCALAI_NATS_TLS_KEY — client key paired with NatsTLSCert - // S3 configuration (used when StorageURL is set) StorageBucket string // --storage-bucket / LOCALAI_STORAGE_BUCKET StorageRegion string // --storage-region / LOCALAI_STORAGE_REGION @@ -95,8 +81,8 @@ type DistributedConfig struct { // database clock, so every replica agrees on when the window ends. WorkerReconnectGrace time.Duration // LOCALAI_WORKER_RECONNECT_GRACE - BackendInstallTimeout time.Duration // NATS round-trip timeout for backend.install (default 15m) - BackendUpgradeTimeout time.Duration // NATS round-trip timeout for backend.upgrade (default 15m) + BackendInstallTimeout time.Duration // control round-trip timeout for backend.install (default 15m) + BackendUpgradeTimeout time.Duration // control round-trip timeout for backend.upgrade (default 15m) // ModelLoadTimeout is the gRPC deadline for the remote LoadModel call the // router issues once a worker has the backend installed and the model files // staged. It therefore covers only the backend's own checkpoint load and @@ -179,13 +165,6 @@ func (c DistributedConfig) Validate() error { } xlog.Warn("distributed mode running without registration token — node endpoints and the worker file-transfer server are unprotected; set LOCALAI_REGISTRATION_TOKEN, or LOCALAI_DISTRIBUTED_REQUIRE_AUTH=true to fail closed") } - if err := c.NatsAuthConfig().Validate(); err != nil { - return err - } - if err := c.NatsTLSFiles().Validate(); err != nil { - return err - } - c.NatsAuthConfig().WarnIfInsecure(true) // Check for negative durations for name, d := range map[string]time.Duration{ FlagMCPToolTimeout: c.MCPToolTimeout, @@ -228,54 +207,20 @@ func WithDistributedAdvertiseAddr(addr string) AppOption { } } -func WithNatsURL(url string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsURL = url - } -} - func WithRegistrationToken(token string) AppOption { return func(o *ApplicationConfig) { o.Distributed.RegistrationToken = token } } -func WithNatsAccountSeed(seed string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsAccountSeed = seed - } -} - -func WithNatsServiceJWT(jwt string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsServiceJWT = jwt - } -} - -func WithNatsServiceSeed(seed string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsServiceSeed = seed - } -} - -func WithNatsWorkerJWTTTL(d time.Duration) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsWorkerJWTTTL = d - } -} - -var EnableNatsRequireAuth = func(o *ApplicationConfig) { - o.Distributed.NatsRequireAuth = true -} - // EnableRegistrationRequireAuth makes an empty registration token a hard error // in distributed mode (see DistributedConfig.RegistrationRequireAuth). var EnableRegistrationRequireAuth = func(o *ApplicationConfig) { o.Distributed.RegistrationRequireAuth = true } -// EnableDistributedRequireAuth is the umbrella switch implying both -// NatsRequireAuth and RegistrationRequireAuth (see DistributedConfig.RequireAuth). +// EnableDistributedRequireAuth is the umbrella switch implying +// RegistrationRequireAuth (see DistributedConfig.RequireAuth). var EnableDistributedRequireAuth = func(o *ApplicationConfig) { o.Distributed.RequireAuth = true } @@ -286,30 +231,6 @@ func (c DistributedConfig) RegistrationAuthRequired() bool { return c.RegistrationRequireAuth || c.RequireAuth } -// NatsAuthRequired reports whether NATS JWT credentials must be present — the -// granular flag or the umbrella. -func (c DistributedConfig) NatsAuthRequired() bool { - return c.NatsRequireAuth || c.RequireAuth -} - -func WithNatsTLSCA(path string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsTLSCA = path - } -} - -func WithNatsTLSCert(path string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsTLSCert = path - } -} - -func WithNatsTLSKey(path string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsTLSKey = path - } -} - func WithStorageURL(url string) AppOption { return func(o *ApplicationConfig) { o.Distributed.StorageURL = url @@ -532,44 +453,6 @@ const ModelLoadWaitUnbounded = -1 * time.Second // DefaultMaxUploadSize is the default maximum upload body size (50 GB). const DefaultMaxUploadSize int64 = 50 << 30 -// NatsTLSFiles returns NATS TLS/mTLS PEM paths for the messaging client. -func (c DistributedConfig) NatsTLSFiles() messaging.TLSFiles { - return messaging.TLSFiles{ - CA: c.NatsTLSCA, - Cert: c.NatsTLSCert, - Key: c.NatsTLSKey, - } -} - -// NatsMessagingOptions builds messaging client options (JWT + TLS) for distributed components. -// Pass explicit userJWT/userSeed when set (e.g. worker overrides); empty uses service JWT from config. -func (c DistributedConfig) NatsMessagingOptions(userJWT, userSeed string) []messaging.Option { - var opts []messaging.Option - jwt, seed := userJWT, userSeed - if jwt == "" && seed == "" { - auth := c.NatsAuthConfig() - jwt, seed = auth.ServiceUserJWT, auth.ServiceUserSeed - } - if jwt != "" && seed != "" { - opts = append(opts, messaging.WithUserJWT(jwt, seed)) - } - if tls := c.NatsTLSFiles(); tls.Enabled() { - opts = append(opts, messaging.WithTLS(tls)) - } - return opts -} - -// NatsAuthConfig builds pkg/natsauth settings from distributed configuration. -func (c DistributedConfig) NatsAuthConfig() natsauth.Config { - return natsauth.Config{ - AccountSeed: c.NatsAccountSeed, - ServiceUserJWT: c.NatsServiceJWT, - ServiceUserSeed: c.NatsServiceSeed, - WorkerJWTTTL: c.NatsWorkerJWTTTL, - RequireAuth: c.NatsAuthRequired(), - } -} - // ReconnectGraceOrDefault returns the configured worker reconnect grace or the // default. // diff --git a/core/config/distributed_config_test.go b/core/config/distributed_config_test.go index 031d455ea6ca..6b5b7f91ece1 100644 --- a/core/config/distributed_config_test.go +++ b/core/config/distributed_config_test.go @@ -1,6 +1,8 @@ package config_test import ( + "reflect" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -95,7 +97,6 @@ var _ = Describe("DistributedConfig.Validate negative-duration errors", func() { It("rejects a negative BackendInstallTimeout with the flag name in the error", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", BackendInstallTimeout: -1 * time.Second, } err := c.Validate() @@ -107,7 +108,6 @@ var _ = Describe("DistributedConfig.Validate negative-duration errors", func() { It("rejects a negative BackendUpgradeTimeout with the flag name in the error", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", BackendUpgradeTimeout: -1 * time.Second, } err := c.Validate() @@ -118,7 +118,6 @@ var _ = Describe("DistributedConfig.Validate negative-duration errors", func() { It("rejects a negative ModelLoadTimeout with the flag name in the error", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", ModelLoadTimeout: -1 * time.Second, } err := c.Validate() @@ -130,7 +129,6 @@ var _ = Describe("DistributedConfig.Validate negative-duration errors", func() { It("accepts all-zero durations as valid (defaults apply)", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", } Expect(c.Validate()).To(Succeed()) }) @@ -140,7 +138,6 @@ var _ = Describe("DistributedConfig.Validate registration auth", func() { It("rejects an empty registration token when RequireAuth is set", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", RegistrationRequireAuth: true, } err := c.Validate() @@ -152,7 +149,6 @@ var _ = Describe("DistributedConfig.Validate registration auth", func() { It("accepts a set registration token when RequireAuth is set", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", RegistrationToken: "s3cret", RegistrationRequireAuth: true, } @@ -162,7 +158,6 @@ var _ = Describe("DistributedConfig.Validate registration auth", func() { It("warns but succeeds with an empty token when RequireAuth is unset", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", } Expect(c.Validate()).To(Succeed()) }) @@ -170,12 +165,7 @@ var _ = Describe("DistributedConfig.Validate registration auth", func() { It("rejects an empty token when the umbrella RequireAuth is set", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", RequireAuth: true, - // Provide NATS creds so only the registration-token gap remains. - NatsServiceJWT: "jwt", - NatsServiceSeed: "seed", - NatsAccountSeed: "acct", } err := c.Validate() Expect(err).To(HaveOccurred()) @@ -183,19 +173,25 @@ var _ = Describe("DistributedConfig.Validate registration auth", func() { Expect(err.Error()).To(ContainSubstring("LOCALAI_REGISTRATION_TOKEN")) }) - It("the umbrella implies NATS auth is required", func() { + // The umbrella used to imply two things, and now implies one. + // + // The It that stood here pinned "LOCALAI_DISTRIBUTED_REQUIRE_AUTH makes a + // missing broker service JWT fatal". That is retired, not moved: there is + // no broker connection to demand a credential for, so a startup that failed + // on a missing one would fail on the absence of something nothing uses. The + // half of the umbrella that survives is the registration layer, and the It + // above it pins exactly that: an empty registration token under the + // umbrella is still fatal, and still says which knob to set. + It("implies the registration layer and nothing else", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", - RegistrationToken: "tok", // registration layer satisfied - RequireAuth: true, // umbrella → NATS creds now required + RegistrationToken: "tok", + RequireAuth: true, } - Expect(c.NatsAuthRequired()).To(BeTrue()) Expect(c.RegistrationAuthRequired()).To(BeTrue()) - // Missing NATS service JWT/seed must now be fatal. - err := c.Validate() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("LOCALAI_NATS_REQUIRE_AUTH")) + // And with the registration layer satisfied there is nothing else left + // for the umbrella to demand, so startup succeeds. + Expect(c.Validate()).To(Succeed()) }) }) @@ -232,7 +228,6 @@ var _ = Describe("DistributedConfig worker reconnect grace", func() { // a caller may reap and evict on. c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", RegistrationToken: "tok", WorkerReconnectGrace: -1 * time.Second, } @@ -247,3 +242,25 @@ var _ = Describe("DistributedConfig worker reconnect grace", func() { Expect(o.Distributed.ReconnectGraceOrDefault()).To(Equal(90 * time.Second)) }) }) + +// The frontend's distributed configuration has nowhere to put a broker URL, and +// that absence is what makes the accepted-and-ignored CLI flags ignored. +// +// A help string saying "ignored" is a promise; a missing field is the mechanism. +// Asserted by reflection rather than by reading a value, because a value +// assertion needs a field to read and would therefore stop compiling exactly +// when the property it guards is restored, which is the failure mode this +// replaces: a spec that vanishes with the regression it was meant to catch. +var _ = Describe("the distributed configuration's broker surface", func() { + It("carries no NATS credential, TLS or URL field", func() { + t := reflect.TypeOf(config.DistributedConfig{}) + var carried []string + for i := range t.NumField() { + if strings.HasPrefix(t.Field(i).Name, "Nats") { + carried = append(carried, t.Field(i).Name) + } + } + Expect(carried).To(BeEmpty(), + "DistributedConfig grew %v back: a value the frontend can store is a value something can dial, and no component of a distributed deployment dials a message bus", carried) + }) +}) diff --git a/core/http/app.go b/core/http/app.go index f61b7f459e15..15db18427fb8 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -483,8 +483,10 @@ func API(application *application.Application) (*echo.Echo, error) { // "operation visible" and "operation gone" between replicas. // // S1. The carrier choice lives in core/application with the other three - // caches, and this call names no carrier at all, so the NATS client - // hanging off the same struct cannot be handed over here by accident. + // caches, and this call names no carrier at all, so nothing hanging off + // the same struct can be handed over here by accident. See + // core/application/cache_fanout_wiring.go for why that shape is kept + // now that the broker's client is no longer one of those things. if d := application.Distributed(); d != nil { if err := d.WireOpCache(application.ApplicationConfig().Context, opcache); err != nil { xlog.Warn("OpCache distributed subscribe failed; running standalone", "error", err) @@ -561,9 +563,8 @@ func API(application *application.Application) (*echo.Echo, error) { } } } - natsCfg := distCfg.NatsAuthConfig() - routes.RegisterNodeSelfServiceRoutes(e, registry, distCfg.RegistrationToken, distCfg.AutoApproveNodes, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret, natsCfg) - routes.RegisterNodeAdminRoutes(e, registry, remoteUnloader, application.GalleryService(), opcache, application.ApplicationConfig(), adminMiddleware, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret, application.ApplicationConfig().Distributed.RegistrationToken, natsCfg, workerHTTPDialFor) + routes.RegisterNodeSelfServiceRoutes(e, registry, distCfg.RegistrationToken, distCfg.AutoApproveNodes, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret) + routes.RegisterNodeAdminRoutes(e, registry, remoteUnloader, application.GalleryService(), opcache, application.ApplicationConfig(), adminMiddleware, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret, application.ApplicationConfig().Distributed.RegistrationToken, workerHTTPDialFor) // Replica-to-replica peer link. Registered only in distributed mode: in // single-node mode there are no peers, and the route authenticates with the diff --git a/core/http/auth/helpers_test.go b/core/http/auth/helpers_test.go index 1e31ac27f7d7..047a4e1880e9 100644 --- a/core/http/auth/helpers_test.go +++ b/core/http/auth/helpers_test.go @@ -12,7 +12,6 @@ import ( "github.com/mudler/LocalAI/core/http/auth" "github.com/mudler/LocalAI/core/http/routes" "github.com/mudler/LocalAI/core/services/nodes" - "github.com/mudler/LocalAI/pkg/natsauth" . "github.com/onsi/gomega" "gorm.io/gorm" ) @@ -114,7 +113,6 @@ func newNodeSelfServiceTestApp(db *gorm.DB, appConfig *config.ApplicationConfig, false, nil, "", - natsauth.Config{}, ) return e } diff --git a/core/http/endpoints/cluster/connect.go b/core/http/endpoints/cluster/connect.go index 8d175704669c..d660c9c21d83 100644 --- a/core/http/endpoints/cluster/connect.go +++ b/core/http/endpoints/cluster/connect.go @@ -128,13 +128,14 @@ func ConnectHandler(registry *nodes.NodeRegistry, tunnels *clustersvc.TunnelRegi // 401 would send an operator looking at tokens. // // Only StatusPending is refused. The rest of /api/node/ self-service - // gates on nothing at all, but the two places that hand a node something - // DURABLE both refuse a pending one: the agent worker's API key - // (provisionAgentWorkerKey, guarded at its call site in - // core/http/endpoints/localai/nodes.go) and its NATS credential - // (attachNatsJWT in the same file). Cited by NAME, not by line: the - // previous version of this comment cited line numbers into a file this - // same commit was editing, and both were stale before it landed. + // gates on nothing at all, but the one place that hands a node + // something DURABLE refuses a pending one too: the agent worker's API + // key (provisionAgentWorkerKey, guarded at its call site in + // core/http/endpoints/localai/nodes.go). Cited by NAME, not by line: + // an earlier version of this comment cited line numbers into a file the + // same commit was editing, and both were stale before it landed. The + // per-node broker credential this used to name alongside it is gone + // with the bus, and so is the function that minted it. // // A tunnel is that kind of grant, not a heartbeat: it is // a standing pipe into the worker recorded in node_connections and diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index 07e8aca21ca2..bf284c847e4d 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -31,7 +31,6 @@ import ( "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/nodes/prefixcache" "github.com/mudler/LocalAI/pkg/httpclient" - "github.com/mudler/LocalAI/pkg/natsauth" "github.com/mudler/LocalAI/pkg/vrambudget" ) @@ -120,7 +119,7 @@ type RegisterNodeRequest struct { // RegisterNodeEndpoint registers a new backend node. // expectedToken is the registration token configured on the frontend (may be empty to disable auth). // autoApprove controls whether new nodes go directly to "healthy" or require admin approval. -func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, autoApprove bool, authDB *gorm.DB, hmacSecret string, natsCfg natsauth.Config) echo.HandlerFunc { +func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, autoApprove bool, authDB *gorm.DB, hmacSecret string) echo.HandlerFunc { return func(c echo.Context) error { var req RegisterNodeRequest if err := c.Bind(&req); err != nil { @@ -248,7 +247,6 @@ func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, au } attachTunnelToken(ctx, response, registry, node) - attachNatsJWT(response, node, natsCfg) return c.JSON(http.StatusCreated, response) } @@ -256,7 +254,7 @@ func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, au // ApproveNodeEndpoint approves a pending node, setting its status to healthy. // For agent workers, it also provisions an API key so they can call the inference API. -func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecret string, natsCfg natsauth.Config) echo.HandlerFunc { +func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecret string) echo.HandlerFunc { return func(c echo.Context) error { ctx := c.Request().Context() id := c.Param("id") @@ -286,8 +284,6 @@ func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecr } } - attachNatsJWT(response, node, natsCfg) - return c.JSON(http.StatusOK, response) } } @@ -296,11 +292,11 @@ func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecr // hash, and puts the plaintext in the registration response. // // It is minted for EVERY node that registers, pending ones included, which is a -// deliberate divergence from the two other per-node credentials in this file: -// the agent worker's API key (provisionAgentWorkerKey) and its NATS JWT -// (attachNatsJWT) are both withheld from a node awaiting approval. Those two -// are bearer grants that WORK the moment they are issued, so issuing one to an -// unapproved node would route around the admin. A tunnel credential is not: +// deliberate divergence from the other per-node credential in this file: the +// agent worker's API key (provisionAgentWorkerKey) is withheld from a node +// awaiting approval. It is a bearer grant that WORKS the moment it is issued, +// so issuing one to an unapproved node would route around the admin. A tunnel +// credential is not: // core/http/endpoints/cluster/connect.go re-reads the node's status on every // dial and refuses a pending node with 403, so the credential is inert until an // admin approves and stays inert if approval is revoked. Withholding it would @@ -385,20 +381,6 @@ func tunnelEligible(nodeType string) bool { return nodeType == nodes.NodeTypeBackend || nodeType == nodes.NodeTypeAgent } -// attachNatsJWT adds a per-node NATS user JWT to a register/approve response when minting is enabled. -func attachNatsJWT(response map[string]any, node *nodes.BackendNode, natsCfg natsauth.Config) { - if !natsCfg.CanMintWorkers() || node == nil || node.Status == nodes.StatusPending { - return - } - jwt, seed, err := natsCfg.MintWorkerJWT(node.ID, node.NodeType) - if err != nil { - xlog.Warn("Failed to mint NATS JWT for node", "node", node.Name, "id", node.ID, "error", err) - return - } - response["nats_jwt"] = jwt - response["nats_user_seed"] = seed -} - // provisionAgentWorkerKey creates a dedicated user and API key for an agent worker node. // Returns the plaintext API key on success. func provisionAgentWorkerKey(ctx context.Context, authDB *gorm.DB, registry *nodes.NodeRegistry, node *nodes.BackendNode, hmacSecret string) (string, error) { diff --git a/core/http/endpoints/localai/nodes_test.go b/core/http/endpoints/localai/nodes_test.go index 26a3b26ed849..c3da844f3519 100644 --- a/core/http/endpoints/localai/nodes_test.go +++ b/core/http/endpoints/localai/nodes_test.go @@ -11,11 +11,10 @@ import ( "strings" "time" + "github.com/google/uuid" "github.com/labstack/echo/v4" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/testutil" - "github.com/mudler/LocalAI/pkg/natsauth" - "github.com/nats-io/nkeys" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -102,7 +101,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusCreated)) @@ -121,7 +120,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, expectedToken, autoApprove, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, expectedToken, autoApprove, nil, "") ExpectWithOffset(1, handler(c)).To(Succeed()) ExpectWithOffset(1, rec.Code).To(Equal(http.StatusCreated)) @@ -304,27 +303,61 @@ var _ = Describe("Node HTTP handlers", func() { "an ineligible node kept a usable tunnel credential, so the mint-site gate is not structural") }) - It("returns nats_jwt when account seed is configured", func() { - akp, err := nkeys.CreateAccount() - Expect(err).ToNot(HaveOccurred()) - seed, err := akp.Seed() - Expect(err).ToNot(HaveOccurred()) - + // The register path, and the approve path is asserted separately below: + // the helper that attached a per-node broker credential had two call + // sites, and one restored call site is exactly the regression a single + // spec would miss. + // + // Asserted on the DECODED MAP with ToNot(HaveKey(...)), not on a struct. + // A struct assertion would pass vacuously: the response is assembled as a + // map[string]any, so the only way a key can be present is if something put + // it there, and the only way to see that is to look at the keys. + It("mints no broker credential into a registration response", func() { e := echo.New() - body := `{"name":"worker-nats","address":"10.0.0.2:50051"}` + body := `{"name":"worker-no-bus","address":"10.0.0.2:50051"}` req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) rec := httptest.NewRecorder() c := e.NewContext(req, rec) - natsCfg := natsauth.Config{AccountSeed: string(seed)} - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsCfg) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusCreated)) var resp map[string]any Expect(json.Unmarshal(rec.Body.Bytes(), &resp)).To(Succeed()) - Expect(resp["nats_jwt"]).ToNot(BeEmpty()) + // The keys it DOES carry, so the two absences below are facts about + // the response and not about a decode that produced an empty map. + Expect(resp).To(HaveKey("id")) + Expect(resp).To(HaveKey("tunnel_token")) + Expect(resp).ToNot(HaveKey("nats_jwt")) + Expect(resp).ToNot(HaveKey("nats_user_seed")) + }) + + It("mints no broker credential into an approval response either", func() { + node := &nodes.BackendNode{ + ID: uuid.New().String(), + Name: "worker-approve-no-bus", + Address: "10.0.0.3:50051", + NodeType: nodes.NodeTypeBackend, + } + Expect(registry.Register(context.Background(), node, false)).To(Succeed()) + + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + c.SetParamNames("id") + c.SetParamValues(node.ID) + + Expect(ApproveNodeEndpoint(registry, nil, "")(c)).To(Succeed()) + Expect(rec.Code).To(Equal(http.StatusOK)) + + var resp map[string]any + Expect(json.Unmarshal(rec.Body.Bytes(), &resp)).To(Succeed()) + Expect(resp).To(HaveKey("id")) + Expect(resp).ToNot(HaveKey("nats_jwt")) + Expect(resp).ToNot(HaveKey("nats_user_seed")) }) It("returns 400 when name is missing", func() { @@ -335,7 +368,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusBadRequest)) @@ -355,7 +388,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusBadRequest)) @@ -378,7 +411,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusCreated)) @@ -401,7 +434,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusCreated)) @@ -419,7 +452,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusBadRequest)) @@ -438,7 +471,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "correct-token", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "correct-token", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusUnauthorized)) }) @@ -451,7 +484,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", false, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", false, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusCreated)) @@ -474,7 +507,7 @@ var _ = Describe("Node HTTP handlers", func() { req1 := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body1)) req1.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) rec1 := httptest.NewRecorder() - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(e.NewContext(req1, rec1))).To(Succeed()) Expect(rec1.Code).To(Equal(http.StatusCreated)) diff --git a/core/http/react-ui/e2e/nodes-per-node-backend-actions.spec.js b/core/http/react-ui/e2e/nodes-per-node-backend-actions.spec.js index 3feef9693387..7cbd2ff02c2a 100644 --- a/core/http/react-ui/e2e/nodes-per-node-backend-actions.spec.js +++ b/core/http/react-ui/e2e/nodes-per-node-backend-actions.spec.js @@ -5,7 +5,7 @@ import { test, expect } from './coverage-fixtures.js' // - a delete affordance is present and goes through ConfirmDialog // // We mock the distributed-mode API so the tests can run against the -// standalone ui-test-server without spinning up workers/NATS. +// standalone ui-test-server without spinning up workers. const NODE_ID = 'test-node-1' const NODE_NAME = 'worker-test' diff --git a/core/http/routes/nodes.go b/core/http/routes/nodes.go index a511c6b7c1fc..e9377ab50c95 100644 --- a/core/http/routes/nodes.go +++ b/core/http/routes/nodes.go @@ -10,7 +10,6 @@ import ( "github.com/mudler/LocalAI/core/http/endpoints/localai" "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/core/services/nodes" - "github.com/mudler/LocalAI/pkg/natsauth" "gorm.io/gorm" ) @@ -36,7 +35,7 @@ func nodeReadyMiddleware(registry *nodes.NodeRegistry) echo.MiddlewareFunc { // token but do not verify per-node identity. A compromised worker can heartbeat/drain/ // deregister other nodes. Future: issue per-node JWT at registration, validate node // identity on subsequent requests (compare :id param with token subject). -func RegisterNodeSelfServiceRoutes(e *echo.Echo, registry *nodes.NodeRegistry, registrationToken string, autoApprove bool, authDB *gorm.DB, hmacSecret string, natsCfg natsauth.Config) { +func RegisterNodeSelfServiceRoutes(e *echo.Echo, registry *nodes.NodeRegistry, registrationToken string, autoApprove bool, authDB *gorm.DB, hmacSecret string) { if registry == nil { return } @@ -45,7 +44,7 @@ func RegisterNodeSelfServiceRoutes(e *echo.Echo, registry *nodes.NodeRegistry, r tokenAuthMw := nodeTokenAuth(registrationToken) node := e.Group("/api/node", readyMw, tokenAuthMw) - node.POST("/register", localai.RegisterNodeEndpoint(registry, registrationToken, autoApprove, authDB, hmacSecret, natsCfg)) + node.POST("/register", localai.RegisterNodeEndpoint(registry, registrationToken, autoApprove, authDB, hmacSecret)) node.POST("/:id/heartbeat", localai.HeartbeatEndpoint(registry)) node.POST("/:id/drain", localai.DrainNodeEndpoint(registry)) node.POST("/:id/resume", localai.ResumeNodeEndpoint(registry)) @@ -67,7 +66,7 @@ func RegisterNodeSelfServiceRoutes(e *echo.Echo, registry *nodes.NodeRegistry, r // registered. It is nil outside distributed mode, and those two routes then // answer 502 rather than dialling, because a worker with no tunnel has nothing // for them to proxy to. -func RegisterNodeAdminRoutes(e *echo.Echo, registry *nodes.NodeRegistry, unloader nodes.NodeCommandSender, galleryService *galleryop.GalleryService, opcache *galleryop.OpCache, appConfig *config.ApplicationConfig, adminMw echo.MiddlewareFunc, authDB *gorm.DB, hmacSecret string, registrationToken string, natsCfg natsauth.Config, workerDialFor nodes.WorkerNetDialerFor) { +func RegisterNodeAdminRoutes(e *echo.Echo, registry *nodes.NodeRegistry, unloader nodes.NodeCommandSender, galleryService *galleryop.GalleryService, opcache *galleryop.OpCache, appConfig *config.ApplicationConfig, adminMw echo.MiddlewareFunc, authDB *gorm.DB, hmacSecret string, registrationToken string, workerDialFor nodes.WorkerNetDialerFor) { if registry == nil { return } @@ -91,7 +90,7 @@ func RegisterNodeAdminRoutes(e *echo.Echo, registry *nodes.NodeRegistry, unloade admin.DELETE("/:id", localai.DeregisterNodeEndpoint(registry)) admin.POST("/:id/drain", localai.DrainNodeEndpoint(registry)) admin.POST("/:id/resume", localai.ResumeNodeEndpoint(registry)) - admin.POST("/:id/approve", localai.ApproveNodeEndpoint(registry, authDB, hmacSecret, natsCfg)) + admin.POST("/:id/approve", localai.ApproveNodeEndpoint(registry, authDB, hmacSecret)) // Backend management on workers admin.GET("/:id/backends", localai.ListBackendsOnNodeEndpoint(unloader, registry)) diff --git a/core/http/routes/openresponses.go b/core/http/routes/openresponses.go index 235639a08b4d..085e022f6751 100644 --- a/core/http/routes/openresponses.go +++ b/core/http/routes/openresponses.go @@ -94,9 +94,11 @@ func setOpenResponsesRequestContext(re *middleware.RequestExtractor) echo.Middle // // A named function rather than a block inside route registration, and that is // the point of it. EnableDistributed takes a messaging.Broadcaster, as it must: -// its own specs publish through a double. So handing it the NATS client instead -// of the deployment's carrier COMPILES and reddens nothing anywhere, and the -// only symptom is a cancel that answers 404 on every replica but one. Registering +// its own specs publish through a double. So handing it any carrier other than +// the deployment's COMPILES and reddens nothing anywhere, and the only symptom +// is a cancel that answers 404 on every replica but one. The broker client that +// used to be the second carrier in scope is gone; what pins the choice is the +// spec beside this file, which drives it from the OTHER carrier. Registering // routes needs a whole Application and therefore has no spec; this needs a // DistributedServices and a store, and therefore has one. func enableDistributedResponses(ctx context.Context, d *application.DistributedServices, diff --git a/core/http/routes/openresponses_wiring_test.go b/core/http/routes/openresponses_wiring_test.go index 1492511398cf..9720b406118a 100644 --- a/core/http/routes/openresponses_wiring_test.go +++ b/core/http/routes/openresponses_wiring_test.go @@ -24,10 +24,11 @@ import ( // EnableDistributed takes a messaging.Broadcaster, which it must: its own specs // publish through a double, and it cannot be made to name a concrete carrier // without dragging that dependency through the whole endpoint package. The -// consequence is that handing it the NATS client instead of the deployment's -// carrier compiles and reddens nothing, and the only symptom is a cancel that -// answers 404 on every replica but the creator. So it is pinned here, by -// watching what actually arrives on the carrier. +// consequence is that handing it any carrier other than the deployment's +// compiles and reddens nothing, and the only symptom is a cancel that answers +// 404 on every replica but the creator. So it is pinned here, by watching what +// actually arrives on the carrier: busB below IS the other carrier, which is +// why this spec keeps its force now that the broker's client is gone. var _ = Describe("wiring the Open Responses store to a carrier", func() { var ( ctx context.Context diff --git a/core/services/finetune/service.go b/core/services/finetune/service.go index 8e3978a00a9a..65504a68824b 100644 --- a/core/services/finetune/service.go +++ b/core/services/finetune/service.go @@ -50,9 +50,10 @@ type FineTuneService struct { // across replicas; pass nil for both in standalone mode, where the disk Loader // hydrates the map and there is nothing to broadcast. // -// bus is messaging.Broadcaster and not the NATS client: this state.*.delta +// bus is messaging.Broadcaster and not a concrete carrier: this state.*.delta // family travels on whatever the deployment's fan-out carrier is, and in -// distributed mode that is PostgreSQL LISTEN/NOTIFY. +// distributed mode that is PostgreSQL LISTEN/NOTIFY. Which one it gets is +// decided in core/application, not here. func NewFineTuneService( appConfig *config.ApplicationConfig, modelLoader *model.ModelLoader, diff --git a/core/services/messaging/client.go b/core/services/messaging/client.go deleted file mode 100644 index 47a98862e64e..000000000000 --- a/core/services/messaging/client.go +++ /dev/null @@ -1,282 +0,0 @@ -package messaging - -import ( - "encoding/json" - "errors" - "fmt" - "strings" - "sync" - "time" - - "github.com/mudler/LocalAI/pkg/sanitize" - "github.com/mudler/xlog" - "github.com/nats-io/nats.go" - "github.com/nats-io/nkeys" -) - -// subscribeConfirmTimeout bounds the server round-trip used to detect whether a -// subscription was rejected (e.g. by JWT permissions) before returning to the caller. -const subscribeConfirmTimeout = 5 * time.Second - -// Client is a NATS connection, and NO PRODUCTION PATH CONSTRUCTS ONE. -// -// The last family that needed a bus was agent..cancel, whose subscriber is -// the agent WORKER: it has no database, so it could never join the PostgreSQL -// carrier the rest of the deployment fans out on. It does not need a bus either -// now, because it holds an outward tunnel and a cancel is a control verb on it -// (workerctl.PathAgentCancel). Nothing in core/ or pkg/ calls messaging.New. -// -// What survives here is this type, its connect options and its TLS plumbing, -// still exercised by the NATS JWT permission specs. Deleting them is a -// demolition of its own, together with the JWT minting at registration and -// pkg/natsauth's permission tables. -// -// The methods that carried everything else are already gone: queue -// subscriptions became a claim on the job store, and request/reply became a -// streaming control RPC on the tunnel each worker dials. Deleting the METHODS -// rather than only the call sites is what makes putting a family back on this -// carrier a build error, instead of a line that compiles, publishes -// successfully, and is delivered onto a carrier nothing reads. -type Client struct { - conn *nats.Conn - mu sync.RWMutex - - // reconnectCbs are invoked after the underlying connection is - // re-established. nats.go transparently resubscribes existing - // subscriptions on reconnect, but it cannot know that a consumer kept - // derived in-memory state (e.g. syncstate.SyncedMap) that may have drifted - // while the link was down — these callbacks let such consumers re-hydrate. - cbMu sync.Mutex - reconnectCbs []func() -} - -// New creates a new NATS client with auto-reconnect. -func New(url string, opts ...Option) (*Client, error) { - var cfg connectConfig - for _, o := range opts { - o(&cfg) - } - - // Allocate the client up front so the reconnect handler closure can reach - // it; conn is populated after nats.Connect succeeds below. - c := &Client{} - - natsOpts := []nats.Option{ - nats.RetryOnFailedConnect(true), - nats.MaxReconnects(-1), - nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { - if err != nil { - xlog.Warn("NATS disconnected", "error", err) - } - }), - nats.ReconnectHandler(func(_ *nats.Conn) { - xlog.Info("NATS reconnected") - c.runReconnectCallbacks() - }), - nats.ClosedHandler(func(_ *nats.Conn) { - xlog.Info("NATS connection closed") - }), - // Surface async errors (notably permission violations) that NATS would - // otherwise deliver silently. A subscription the server rejects for a - // JWT permission means the worker never receives those messages, so make - // it loud rather than letting the feature fail invisibly. - nats.ErrorHandler(func(_ *nats.Conn, sub *nats.Subscription, err error) { - subject := "" - if sub != nil { - subject = sub.Subject - } - if errors.Is(err, nats.ErrPermissionViolation) { - xlog.Error("NATS permission violation — check JWT pub/sub allow lists", "subject", subject, "error", err) - return - } - xlog.Warn("NATS async error", "subject", subject, "error", err) - }), - } - switch { - case cfg.jwtProvider != nil: - // Fetch creds on every (re)connect so a refresh loop can rotate the JWT - // before expiry; the server expiring the old JWT triggers a reconnect - // that transparently picks up the new one. - natsOpts = append(natsOpts, nats.UserJWT( - func() (string, error) { - jwt, _ := cfg.jwtProvider() - if jwt == "" { - return "", fmt.Errorf("no NATS user JWT available") - } - return jwt, nil - }, - func(nonce []byte) ([]byte, error) { - _, seed := cfg.jwtProvider() - kp, err := nkeys.FromSeed([]byte(seed)) - if err != nil { - return nil, fmt.Errorf("loading NATS user seed: %w", err) - } - defer kp.Wipe() - return kp.Sign(nonce) - }, - )) - case cfg.userJWT != "" && cfg.userSeed != "": - natsOpts = append(natsOpts, nats.UserJWTAndSeed(cfg.userJWT, cfg.userSeed)) - } - if cfg.tls.Enabled() { - if err := cfg.tls.Validate(); err != nil { - return nil, err - } - tlsOpts, err := cfg.tls.natsOptions() - if err != nil { - return nil, err - } - natsOpts = append(natsOpts, tlsOpts...) - } - - nc, err := nats.Connect(url, natsOpts...) - if err != nil { - return nil, fmt.Errorf("connecting to NATS at %s: %w", sanitize.URL(url), err) - } - - c.conn = nc - return c, nil -} - -// OnReconnect registers a callback invoked after the NATS connection is -// re-established. It is consumed via an optional interface type-assertion -// (interface{ OnReconnect(func()) }) rather than being added to Broadcaster, so -// the messaging abstraction stays minimal and standalone/test clients are not -// forced to implement reconnect semantics. A nil callback is ignored. -func (c *Client) OnReconnect(cb func()) { - if cb == nil { - return - } - c.cbMu.Lock() - c.reconnectCbs = append(c.reconnectCbs, cb) - c.cbMu.Unlock() -} - -// runReconnectCallbacks invokes registered reconnect callbacks. It copies the -// slice under the lock so a callback that (re)registers cannot deadlock. -func (c *Client) runReconnectCallbacks() { - c.cbMu.Lock() - cbs := append([]func(){}, c.reconnectCbs...) - c.cbMu.Unlock() - for _, cb := range cbs { - cb() - } -} - -// Publish marshals data as JSON and publishes it to the given subject. -func (c *Client) Publish(subject string, data any) error { - payload, err := json.Marshal(data) - if err != nil { - return fmt.Errorf("marshalling message for %s: %w", subject, err) - } - c.mu.RLock() - defer c.mu.RUnlock() - return c.conn.Publish(subject, payload) -} - -// Subscribe creates a subscription on the given subject. All subscribers receive every message. -func (c *Client) Subscribe(subject string, handler func([]byte)) (Subscription, error) { - return c.confirmSubscription(subject, func(conn *nats.Conn) (*nats.Subscription, error) { - return conn.Subscribe(subject, func(msg *nats.Msg) { - handler(msg.Data) - }) - }) -} - -// confirmSubscription creates a subscription via mk and forces a server -// round-trip so that a permissions violation — which NATS otherwise reports -// only asynchronously — is returned to the caller synchronously. The server -// emits the "-ERR Permissions Violation" for a rejected SUB before the PONG -// that satisfies the flush, so by the time FlushTimeout returns the violation -// is recorded as the connection's last error. Without this, a worker whose JWT -// lacks a subject gets a non-nil subscription that never receives a message, -// turning a permission misconfiguration into a silent failure. -func (c *Client) confirmSubscription(subject string, mk func(*nats.Conn) (*nats.Subscription, error)) (Subscription, error) { - c.mu.RLock() - conn := c.conn - c.mu.RUnlock() - if conn == nil { - return nil, fmt.Errorf("subscribe to %s: nil NATS connection", subject) - } - - sub, err := mk(conn) - if err != nil { - return nil, err - } - - // A failed flush here means we could not round-trip to the server (not yet - // connected, reconnecting, slow link). RetryOnFailedConnect intentionally - // buffers subscriptions across that gap, so do NOT fail — keep the - // subscription and let it replay on (re)connect; a later permission - // violation is still logged by the async error handler in New. - if err := conn.FlushTimeout(subscribeConfirmTimeout); err != nil { - xlog.Debug("Could not confirm NATS subscription (will replay on connect)", "subject", subject, "error", err) - return sub, nil - } - // Flush succeeded, so any permission violation for this SUB has already been - // recorded as the connection's last error (the server emits it before the - // PONG). LastError is per-connection; match the exact quoted subject the - // server echoes ("Subscription to \"\"") so a stale violation for - // another subject can't be mis-attributed here. - if lerr := conn.LastError(); lerr != nil && - errors.Is(lerr, nats.ErrPermissionViolation) && - strings.Contains(lerr.Error(), `Subscription to "`+subject+`"`) { - _ = sub.Unsubscribe() - return nil, fmt.Errorf("subscription to %s denied by NATS server (check JWT sub allow list): %w", subject, lerr) - } - return sub, nil -} - -// ConfirmRoundTrip forces a round trip to the server and returns whatever the -// server pushed back asynchronously, so that a refusal becomes an error a -// caller holds rather than a line in a log. -// -// It replaces a Conn() accessor that handed out the raw *nats.Conn. That -// accessor was the hole in this type's method set: every half deleted above is -// still one call away on a *nats.Conn, so a family could be put back on this -// carrier through it without a single build error, which is the whole thing the -// deletions are for. -// -// What it does is the publish-side twin of confirmSubscription. NATS reports a -// permission violation asynchronously and does NOT close the connection, so a -// denied publish is indistinguishable from an accepted one until something -// round-trips and reads the connection's last error. A flush that fails is -// returned as-is: the caller could not reach the server at all, which is a -// different fact from the server refusing it, and neither is evidence about any -// node. -// -// No production path calls it, and this carrier no longer has production users -// at all. What needs the verdict is pkg/natsauth's permission grants, which are -// asserted against a real enforcing server and would otherwise be asserted -// against nothing, since an allow list that is EMPTY means unrestricted in NATS -// and a spec that only checks IsConnected cannot tell a granted publish from a -// denied one. -func (c *Client) ConfirmRoundTrip(timeout time.Duration) error { - c.mu.RLock() - conn := c.conn - c.mu.RUnlock() - if conn == nil { - return fmt.Errorf("confirming a round trip: nil NATS connection") - } - if err := conn.FlushTimeout(timeout); err != nil { - return fmt.Errorf("round trip to the NATS server: %w", err) - } - return conn.LastError() -} - -// IsConnected returns true if the client is currently connected to a NATS server. -func (c *Client) IsConnected() bool { - c.mu.RLock() - defer c.mu.RUnlock() - return c.conn != nil && c.conn.IsConnected() -} - -// Close drains and closes the NATS connection, waiting for in-flight messages. -func (c *Client) Close() { - c.mu.Lock() - defer c.mu.Unlock() - if c.conn != nil { - c.conn.Drain() - c.conn.FlushTimeout(5 * time.Second) - } -} diff --git a/core/services/messaging/interfaces.go b/core/services/messaging/interfaces.go index 980927c02156..a70cf1b437de 100644 --- a/core/services/messaging/interfaces.go +++ b/core/services/messaging/interfaces.go @@ -35,13 +35,14 @@ type Broadcaster interface { Subscribe(subject string, handler func([]byte)) (Subscription, error) } -// One of the two carriers a deployment dials, asserted in the package that owns -// the interface rather than at whichever call site is migrated next. +// Broadcaster has one implementation a deployment runs on, *pgbus.Bus, and it +// cannot be named here, because pgbus imports this package. The conformance +// assertion lives in interfaces_test.go alongside the test double's, which is +// also where the method-set pin lives: a conformance assertion stays true +// however many methods grow back, so it cannot say that the retired halves are +// gone. // -// *pgbus.Bus is the other and cannot be named here, because it imports this -// package; it is asserted from interfaces_test.go, which also pins that -// *Client's method set is this interface plus its own lifecycle and nothing -// else. That pin is the guard, not this line: a conformance assertion stays -// true however many methods grow back, so it cannot say that the retired halves -// are gone. -var _ Broadcaster = (*Client)(nil) +// The NATS client that used to be asserted on this line is deleted. There is no +// second carrier: the last family that needed one, agent..cancel, is a +// control verb on the agent worker's own tunnel, and no component of a +// distributed deployment opens a connection to a message broker. diff --git a/core/services/messaging/interfaces_test.go b/core/services/messaging/interfaces_test.go index 5a11fdbdbe76..c58fcc53f3db 100644 --- a/core/services/messaging/interfaces_test.go +++ b/core/services/messaging/interfaces_test.go @@ -12,21 +12,24 @@ import ( "github.com/mudler/LocalAI/core/services/testutil" ) -// The three things that must answer one contract, asserted where the contract +// The two things that must answer one contract, asserted where the contract // lives rather than at whichever adopter is migrated next. // -// *pgbus.Bus is the carrier a deployment runs on, *messaging.Client is the one -// carrier left that a deployment still dials (see below), and *testutil.FakeBus -// is what every consumer's spec runs against. A double that drifts out of the -// interface makes each adopter's suite fail in turn, in packages that do not -// own the interface and cannot say why. +// *pgbus.Bus is the carrier a deployment runs on and *testutil.FakeBus is what +// every consumer's spec runs against. A double that drifts out of the interface +// makes each adopter's suite fail in turn, in packages that do not own the +// interface and cannot say why. +// +// There were three. *messaging.Client, the NATS connection, is deleted: it +// carried one family, agent..cancel, and that family is a control verb on +// the agent worker's own tunnel now, so the type had no caller and the module +// had no reason to require a broker client. // // This file is package messaging_test because messaging cannot import testutil // or pgbus: both import messaging. var ( _ messaging.Broadcaster = (*testutil.FakeBus)(nil) _ messaging.Broadcaster = (*pgbus.Bus)(nil) - _ messaging.Broadcaster = (*messaging.Client)(nil) ) // exportedMethods names the exported method set of t, sorted, so a spec can @@ -64,43 +67,16 @@ var _ = Describe("the messaging surface", func() { }) }) -// The NATS client survives this commit, and it survives for exactly one family. -// -// agent..cancel is the one fan-out family that did not move to the -// PostgreSQL carrier: its only subscriber is the agent WORKER, which has no -// database and cannot join that carrier at all, so a cancel published there -// would reach no worker and be reported as sent. Both ends of that family still -// dial NATS, which is why messaging.New, the connect options and the TLS files -// are all still here. +// The two Describes that stood here pinned the NATS client's method set: that +// it was a Broadcaster plus its own lifecycle, and that QueueSubscribe, +// QueueSubscribeReply, SubscribeReply, Request and Conn could not come back on +// it. They are retired rather than moved, because a method set is a property of +// a type and the type is gone. // -// What the client may no longer do is everything else. Its queue and -// request/reply halves are deleted, so a family being put BACK on NATS is now a -// build error at the call site rather than a line that compiles, publishes -// successfully, and is delivered onto a carrier nobody subscribes to. -// -// The method set is pinned by NAME and not by a conformance assertion, because -// a conformance assertion cannot express absence: *messaging.Client satisfying -// Broadcaster stays true no matter how many methods are added back. -var _ = Describe("the NATS client's method set", func() { - It("is a Broadcaster plus its own lifecycle, and nothing more", func() { - Expect(exportedMethods(reflect.TypeOf((*messaging.Client)(nil)))). - To(Equal([]string{"Close", "ConfirmRoundTrip", "IsConnected", "OnReconnect", "Publish", "Subscribe"}), - "the NATS client grew a method back: it is the cancel family's carrier and may carry nothing else") - }) - - It("cannot be handed a queue group or a request", func() { - // Named individually so the failure message says WHICH half came back. - // As a set assertion alone, re-adding Request reads as an off-by-one. - t := reflect.TypeOf((*messaging.Client)(nil)) - // Conn is in this list and is the reason the list is checkable at all: - // while it existed, every name above it was one c.Conn().X() away, so - // the deletions would have been a naming convention rather than a - // constraint. ConfirmRoundTrip is what replaced it, and it hands back - // an error rather than the connection. - for _, retired := range []string{"QueueSubscribe", "QueueSubscribeReply", "SubscribeReply", "Request", "Conn"} { - _, found := t.MethodByName(retired) - Expect(found).To(BeFalse(), - "%s is back on *messaging.Client; that half of the carrier was retired and its call sites moved to the worker's tunnel or to the claim queue", retired) - } - }) -}) +// What they were really guarding is the carrier, not the type, and that guard +// survives twice over. The retired halves cannot come back on the interface, +// which the "is fan-out and nothing else" spec above still asserts by count. +// And the connection itself cannot come back at all without a broker client in +// the module, which nats_absent_test.go asserts against go.mod and go.sum: a +// deleted method is one file away from being written again, while a deleted +// require has to be re-added on purpose and shows up in a diff. diff --git a/core/services/messaging/nats_absent_test.go b/core/services/messaging/nats_absent_test.go new file mode 100644 index 000000000000..9566710d0a65 --- /dev/null +++ b/core/services/messaging/nats_absent_test.go @@ -0,0 +1,67 @@ +package messaging_test + +import ( + "os" + "path/filepath" + "runtime" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// moduleFile reads a file from the repository root, located from this source +// file rather than from the working directory, which Ginkgo does not promise. +func moduleFile(name string) string { + _, thisFile, _, ok := runtime.Caller(0) + Expect(ok).To(BeTrue(), "could not locate this source file, so the assertion below would read nothing") + dir := filepath.Dir(thisFile) + for { + candidate := filepath.Join(dir, "go.mod") + if _, err := os.Stat(candidate); err == nil { + body, err := os.ReadFile(filepath.Join(dir, name)) + Expect(err).ToNot(HaveOccurred()) + return string(body) + } + parent := filepath.Dir(dir) + Expect(parent).ToNot(Equal(dir), "walked past the filesystem root without finding go.mod") + dir = parent + } +} + +// The point of the whole removal, asserted as a property of the module rather +// than as a property of any one package. +// +// Every other guard in this repository is a guard on a call site: a deleted +// method, a deleted parameter, a deleted field. None of them can say that the +// DEPENDENCY is gone, because a dependency survives on a single blank import in +// a single test file in a package nobody reads, and `go mod tidy` succeeding +// proves nothing about that: tidy keeps a require exactly when something still +// reaches it. +// +// This is the assertion that reddens when it comes back. It reads the module +// files themselves, so it is blind to build tags, to GOOS, and to whether a +// package can be loaded at all, which is what the source-level greps cannot +// promise and the dependency walks cannot promise either. +var _ = Describe("the module's dependency on a message broker", func() { + It("requires no nats-io module", func() { + Expect(moduleFile("go.mod")).ToNot(ContainSubstring("github.com/nats-io/"), + "a nats-io module is required again: nothing in this deployment dials a message bus, so a require is a family being invited back onto a carrier nothing reads") + }) + + It("requires no NATS testcontainer", func() { + // Named separately because it comes back by a different route: a spec + // standing a broker up for a suite, rather than production code + // dialling one. It is also the reference a dependency walk WITHOUT + // -test reports as absent while the suite is still starting a + // nats:2-alpine container. + Expect(moduleFile("go.mod")).ToNot(ContainSubstring("testcontainers-go/modules/nats"), + "a suite is standing a broker up again") + }) + + It("carries no nats-io checksum", func() { + // go.sum outlives go.mod by a release when a require is dropped by hand + // rather than by tidy, so it is asserted on its own: a checksum left + // behind is the evidence that the removal was partial. + Expect(moduleFile("go.sum")).ToNot(ContainSubstring("github.com/nats-io/")) + }) +}) diff --git a/core/services/messaging/options.go b/core/services/messaging/options.go deleted file mode 100644 index b4c67b16c8ae..000000000000 --- a/core/services/messaging/options.go +++ /dev/null @@ -1,34 +0,0 @@ -package messaging - -// Option configures NATS client connection behavior. -type Option func(*connectConfig) - -// CredentialProvider returns the NATS user JWT and signing seed to use for the -// next (re)connect. It is consulted on every connection attempt, so a refresh -// loop can rotate credentials before they expire and the connection picks them -// up automatically when the server expires the old JWT and triggers a reconnect. -type CredentialProvider func() (jwt, seed string) - -type connectConfig struct { - userJWT string - userSeed string - jwtProvider CredentialProvider - tls TLSFiles -} - -// WithUserJWT connects using a static NATS user JWT and signing seed (UserJWTAndSeed). -func WithUserJWT(jwt, seed string) Option { - return func(c *connectConfig) { - c.userJWT = jwt - c.userSeed = seed - } -} - -// WithUserJWTProvider connects using credentials fetched from provider on each -// (re)connect, enabling JWT rotation without dropping the client. Takes -// precedence over WithUserJWT when both are set. -func WithUserJWTProvider(provider CredentialProvider) Option { - return func(c *connectConfig) { - c.jwtProvider = provider - } -} diff --git a/core/services/messaging/tls.go b/core/services/messaging/tls.go deleted file mode 100644 index b594845cc197..000000000000 --- a/core/services/messaging/tls.go +++ /dev/null @@ -1,68 +0,0 @@ -package messaging - -import ( - "fmt" - "os" - - "github.com/nats-io/nats.go" -) - -// TLSFiles holds PEM paths for NATS TLS / mTLS. Cert and key must be set together. -// Use tls:// in LOCALAI_NATS_URL; CA and client cert paths are optional extras. -type TLSFiles struct { - CA string // LOCALAI_NATS_TLS_CA — private CA for server verification - Cert string // LOCALAI_NATS_TLS_CERT — client certificate (mTLS) - Key string // LOCALAI_NATS_TLS_KEY — client private key -} - -// Enabled reports whether any TLS file path is configured. -func (f TLSFiles) Enabled() bool { - return f.CA != "" || f.Cert != "" || f.Key != "" -} - -// Validate checks path pairing and that files exist. -func (f TLSFiles) Validate() error { - if f.Cert != "" && f.Key == "" { - return fmt.Errorf("LOCALAI_NATS_TLS_KEY is required when LOCALAI_NATS_TLS_CERT is set") - } - if f.Key != "" && f.Cert == "" { - return fmt.Errorf("LOCALAI_NATS_TLS_CERT is required when LOCALAI_NATS_TLS_KEY is set") - } - for _, path := range []struct { - name, path string - }{ - {"LOCALAI_NATS_TLS_CA", f.CA}, - {"LOCALAI_NATS_TLS_CERT", f.Cert}, - {"LOCALAI_NATS_TLS_KEY", f.Key}, - } { - if path.path == "" { - continue - } - if _, err := os.Stat(path.path); err != nil { - return fmt.Errorf("%s: %w", path.name, err) - } - } - return nil -} - -// natsOptions builds nats-go TLS options. Call Validate first. -func (f TLSFiles) natsOptions() ([]nats.Option, error) { - if !f.Enabled() { - return nil, nil - } - opts := []nats.Option{nats.Secure()} - if f.CA != "" { - opts = append(opts, nats.RootCAs(f.CA)) - } - if f.Cert != "" { - opts = append(opts, nats.ClientCert(f.Cert, f.Key)) - } - return opts, nil -} - -// WithTLS configures CA and/or client certificate paths for the NATS connection. -func WithTLS(files TLSFiles) Option { - return func(c *connectConfig) { - c.tls = files - } -} diff --git a/core/services/messaging/tls_test.go b/core/services/messaging/tls_test.go deleted file mode 100644 index 9fdb3e81686f..000000000000 --- a/core/services/messaging/tls_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package messaging_test - -import ( - "os" - "path/filepath" - - "github.com/mudler/LocalAI/core/services/messaging" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("TLSFiles", func() { - It("requires cert and key together", func() { - Expect((messaging.TLSFiles{Cert: "/tmp/c.pem"}).Validate()).To(HaveOccurred()) - Expect((messaging.TLSFiles{Key: "/tmp/k.pem"}).Validate()).To(HaveOccurred()) - }) - - It("validates files exist", func() { - dir := GinkgoT().TempDir() - ca := filepath.Join(dir, "ca.pem") - Expect(os.WriteFile(ca, []byte("x"), 0600)).To(Succeed()) - Expect((messaging.TLSFiles{CA: ca}).Validate()).To(Succeed()) - }) -}) diff --git a/core/services/pgbus/bus.go b/core/services/pgbus/bus.go index 1e0ff6ddabbf..fbe7ebbcc799 100644 --- a/core/services/pgbus/bus.go +++ b/core/services/pgbus/bus.go @@ -592,9 +592,10 @@ type subscription struct { // message, a job result rather than a progress tick, can type-assert to this // and go read the row. // -// It is not on messaging.Subscription: the NATS client's subscription cannot -// answer it, and widening that interface would make every existing consumer -// claim a guarantee it does not have. +// It is not on messaging.Subscription: a subscription only this carrier can +// answer for does not belong on the interface every carrier implements, and +// widening that interface would make every existing consumer claim a guarantee +// it does not have. type DropCounter interface { Dropped() uint64 } diff --git a/core/services/quantization/service.go b/core/services/quantization/service.go index fc5c4f4c5368..34a587a60e99 100644 --- a/core/services/quantization/service.go +++ b/core/services/quantization/service.go @@ -72,9 +72,10 @@ func isTerminalStatus(status string) bool { // consistent across replicas; pass nil for both in standalone mode, where the // disk Loader hydrates the map and there is nothing to broadcast. // -// bus is messaging.Broadcaster and not the NATS client: this state.*.delta +// bus is messaging.Broadcaster and not a concrete carrier: this state.*.delta // family travels on whatever the deployment's fan-out carrier is, and in -// distributed mode that is PostgreSQL LISTEN/NOTIFY. +// distributed mode that is PostgreSQL LISTEN/NOTIFY. Which one it gets is +// decided in core/application, not here. func NewQuantizationService( appConfig *config.ApplicationConfig, modelLoader *model.ModelLoader, diff --git a/coverage-baseline.txt b/coverage-baseline.txt index ac1025b8090e..2fb78f04e031 100644 --- a/coverage-baseline.txt +++ b/coverage-baseline.txt @@ -1 +1 @@ -54.2 +62.0 diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index ee4bdab330da..7497d0e558df 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -603,18 +603,26 @@ A frontend replica that dies mid-load does not wedge the model: the job row carr Earlier releases of distributed mode required a NATS cluster alongside PostgreSQL. **They no longer do. Shut the broker down.** Nothing in LocalAI opens a connection to one: the frontend's cross-replica fan-out is on PostgreSQL, queued work is a claim on a PostgreSQL table, a serve-backend worker takes every verb on its own tunnel, and an agent worker does too, including the cancel that was the last family on a bus. +A distributed deployment needs **PostgreSQL and the frontends' own HTTP listener, and nothing else.** Workers dial out to that listener and hold the tunnel open, so no worker needs an inbound port either. There is no broker client left in LocalAI at all: as of this release the `nats-io` modules are not in the build, so the binary cannot open a broker connection even if something asked it to. + There is no migration step and no cutover window. Stop the broker, delete its service from your compose file, chart or unit files, and delete the credentials you generated for it. A deployment that keeps running one is paying for infrastructure that carries nothing. -**Your existing command lines still start.** Every `LOCALAI_NATS_*` setting below is parsed and then ignored, so an unedited command line, unit file or Helm values file needs no change on the day you upgrade. Remove them at your convenience. +**Your existing command lines still start.** Every `LOCALAI_NATS_*` setting below is parsed and then ignored, so an unedited command line, unit file or Helm values file needs no change on the day you upgrade. They are hidden from `--help`, because there is nothing left to configure with them. **They are scheduled for removal in the release after next**; remove them from your own files at your convenience before then. | Flag | Env Var | Status | |------|---------|--------| | `--nats-url` | `LOCALAI_NATS_URL` | Accepted and ignored on the frontend, `local-ai worker` and `local-ai agent-worker`. The value is never dialled, so it may point at a broker that is already gone. | -| `--nats-account-seed` | `LOCALAI_NATS_ACCOUNT_SEED` | The frontend still mints a per-node user JWT at registration (`nats_jwt` in the register response). Nothing consumes it. | -| `--nats-service-jwt` / `--nats-service-seed` | `LOCALAI_NATS_SERVICE_JWT` / `LOCALAI_NATS_SERVICE_SEED` | Accepted, unused: the frontend opens no bus connection to present them on. | -| `--nats-worker-jwt-ttl` | `LOCALAI_NATS_WORKER_JWT_TTL` | Lifetime of the minted-but-unused worker JWTs. | -| `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | On an agent worker this still makes registration **wait through admin approval** rather than starting against a pending node. It gates no connection. | -| `--nats-tls-ca` / `--nats-tls-cert` / `--nats-tls-key` | `LOCALAI_NATS_TLS_*` | Accepted, unused. | +| `--nats-account-seed` | `LOCALAI_NATS_ACCOUNT_SEED` | Accepted and ignored. The frontend mints no per-node broker credential: a register or approve response carries no `nats_jwt` and no `nats_user_seed`, and a worker that reads those keys finds nothing. Nodes are authenticated by their registration token and their tunnel token. | +| `--nats-service-jwt` / `--nats-service-seed` | `LOCALAI_NATS_SERVICE_JWT` / `LOCALAI_NATS_SERVICE_SEED` | Accepted and ignored: the frontend opens no bus connection to present them on. | +| `--nats-worker-jwtttl` | `LOCALAI_NATS_WORKER_JWT_TTL` | Accepted and ignored. No per-node broker credential is minted, so none has a lifetime. | +| `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | Accepted and ignored. It used to make an agent worker wait through admin approval; use `--distributed-require-auth` for that (see below). | +| `--nats-tlsca` / `--nats-tls-cert` / `--nats-tls-key` | `LOCALAI_NATS_TLS_*` | Accepted and ignored. The paths are no longer checked for existence either, so a certificate deleted with the broker does not fail startup. | + +{{% notice warning %}} +**One behaviour changed, on the agent worker.** `--nats-require-auth` used to make `local-ai agent-worker` wait through admin approval at registration instead of starting against a pending node. That wait is now asked for with `--distributed-require-auth` / `LOCALAI_DISTRIBUTED_REQUIRE_AUTH`, which already implied it. An agent worker started with only `--nats-require-auth` no longer waits: it registers, starts, and its tunnel dials are refused with 403 until an admin approves it, which is the historical default behaviour. If you relied on the wait, set `--distributed-require-auth`. + +On the **frontend**, `--distributed-require-auth` now implies only `--registration-require-auth`. It used to also require broker credentials, and there are none to require. +{{% /notice %}} {{% notice warning %}} `LOCALAI_NATS_BACKEND_INSTALL_TIMEOUT`, `LOCALAI_NATS_BACKEND_UPGRADE_TIMEOUT` and `LOCALAI_NATS_MODEL_LOAD_TIMEOUT` are **not** in the table above and must **not** be removed. Despite their names they were never broker settings: each one is a control-RPC budget the frontend applies to a worker, and each is still read and still enforced. They are documented with the other frontend flags in [Frontend Configuration](#frontend-configuration). The names are kept because renaming them would break every existing deployment for cosmetics. diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md index 5478b6de0292..c825c1ca27cb 100644 --- a/docs/content/reference/cli-reference.md +++ b/docs/content/reference/cli-reference.md @@ -224,7 +224,7 @@ LocalAI supports several subcommands beyond `run`: - `local-ai mcp-server` - Run the LocalAI admin tool surface as a stdio MCP server (controls a remote LocalAI instance over HTTP) - `local-ai worker` - Start a worker for distributed mode (generic, backend-agnostic; needs only an outbound route to the frontend, no message bus) - `local-ai p2p-worker` - Run workers to distribute workload via p2p (llama.cpp-only) -- `local-ai agent-worker` - Start an agent worker for distributed mode (serves agent execution, MCP CI runs and cancellation as control verbs on the tunnel it holds; it connects to no message bus, so `--nats-url` is accepted and ignored) +- `local-ai agent-worker` - Start an agent worker for distributed mode (serves agent execution, MCP CI runs and cancellation as control verbs on the tunnel it holds; it connects to no message bus) - `local-ai util` - Utility commands - `local-ai explorer` - Run P2P explorer - `local-ai federated` - Run LocalAI in federated mode @@ -232,6 +232,24 @@ LocalAI supports several subcommands beyond `run`: Use `local-ai --help` for more information on each command. +### Flags accepted and ignored + +Distributed mode no longer uses a message broker, and the `nats-io` client is no longer part of the build. The flags that configured one are still **accepted and ignored**, so an existing command line, unit file or Helm values file starts unchanged on the day you upgrade. They are hidden from `--help`, and they are **scheduled for removal in the release after next**. + +Which command accepts which is listed per row, because the sets differ. `local-ai worker` kept only `--nats-url`: its credential and TLS flags were removed a release earlier, when a serve-backend worker stopped connecting to a broker at all. Passing one of the others to `local-ai worker` is a parse error, not an ignored flag. + +| Flag | Env Var | Accepted by | +|------|---------|-------------| +| `--nats-url` | `LOCALAI_NATS_URL` | `run`, `worker`, `agent-worker` | +| `--nats-account-seed` | `LOCALAI_NATS_ACCOUNT_SEED` | `run` | +| `--nats-worker-jwtttl` | `LOCALAI_NATS_WORKER_JWT_TTL` | `run` | +| `--nats-service-jwt` / `--nats-service-seed` | `LOCALAI_NATS_SERVICE_JWT` / `LOCALAI_NATS_SERVICE_SEED` | `run`, `agent-worker` | +| `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | `run`, `agent-worker` | +| `--nats-tlsca` / `--nats-tls-cert` / `--nats-tls-key` | `LOCALAI_NATS_TLS_CA` / `LOCALAI_NATS_TLS_CERT` / `LOCALAI_NATS_TLS_KEY` | `run`, `agent-worker` | +| `--nats-jwt` / `--nats-user-seed` | `LOCALAI_NATS_JWT` / `LOCALAI_NATS_USER_SEED` | `agent-worker` | + +`LOCALAI_NATS_BACKEND_INSTALL_TIMEOUT`, `LOCALAI_NATS_BACKEND_UPGRADE_TIMEOUT` and `LOCALAI_NATS_MODEL_LOAD_TIMEOUT` are **not** in that table and are still read and enforced. Despite their names they were never broker settings: each is a budget the frontend applies to a control request it sends a worker over that worker's tunnel. See [Migrating off the message broker]({{%relref "features/distributed-mode#migrating-off-the-message-broker" %}}). + ## Examples ### Basic Usage diff --git a/go.mod b/go.mod index c0abdec079e2..4af06a2de0f7 100644 --- a/go.mod +++ b/go.mod @@ -44,9 +44,6 @@ require ( github.com/mudler/memory v0.0.0-20260406210934-424c1ecf2cf8 github.com/mudler/nib v0.6.0 github.com/mudler/xlog v0.0.6 - github.com/nats-io/jwt/v2 v2.7.4 - github.com/nats-io/nats.go v1.52.0 - github.com/nats-io/nkeys v0.4.15 github.com/ollama/ollama v0.20.4 github.com/onsi/ginkgo/v2 v2.29.0 github.com/onsi/gomega v1.41.0 @@ -66,7 +63,6 @@ require ( github.com/swaggo/echo-swagger v1.5.2 github.com/swaggo/swag v1.16.6 github.com/testcontainers/testcontainers-go v0.42.0 - github.com/testcontainers/testcontainers-go/modules/nats v0.42.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 github.com/timbutler/zxcvbn v1.0.4 github.com/valkey-io/valkey-go v1.0.76 @@ -86,11 +82,8 @@ require ( ) require ( - cyphar.com/go-pathrs v0.2.1 // indirect filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5 // indirect filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b // indirect - github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect - github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9 // indirect @@ -117,15 +110,10 @@ require ( github.com/chromedp/cdproto v0.0.0-20260321001828-e3e3800016bc // indirect github.com/chromedp/chromedp v0.15.1 // indirect github.com/chromedp/sysutil v1.1.0 // indirect - github.com/containerd/containerd/api v1.8.0 // indirect - github.com/containerd/fifo v1.1.0 // indirect - github.com/containerd/ttrpc v1.2.7 // indirect - github.com/containerd/typeurl/v2 v2.2.0 // indirect github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect github.com/dave-gray101/v2keyauth v0.0.0-20240624150259-c45d584d25e2 // indirect github.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352 // indirect github.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7 // indirect - github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/dunglas/httpsfv v1.1.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/filecoin-project/go-clock v0.1.0 // indirect @@ -167,17 +155,11 @@ require ( github.com/klippa-app/go-pdfium v1.19.2 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-sqlite3 v1.14.28 // indirect - github.com/moby/locker v1.0.1 // indirect github.com/moby/moby/api v1.54.2 // indirect github.com/moby/moby/client v0.4.1 // indirect - github.com/moby/sys/mountinfo v0.7.2 // indirect - github.com/moby/sys/signal v0.7.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect - github.com/nats-io/nuid v1.0.1 // indirect github.com/oklog/ulid v1.3.1 // indirect - github.com/opencontainers/runtime-spec v1.2.0 // indirect - github.com/opencontainers/selinux v1.13.1 // indirect github.com/secure-systems-lab/go-securesystemslib v0.9.1 // indirect github.com/shibumi/go-pathspec v1.3.0 // indirect github.com/sigstore/protobuf-specs v0.5.1 // indirect @@ -198,7 +180,6 @@ require ( github.com/transparency-dev/merkle v0.0.2 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect go.mongodb.org/mongo-driver v1.17.6 // indirect - google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) @@ -536,7 +517,7 @@ require ( golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/mod v0.36.0 // indirect golang.org/x/sync v0.20.0 - golang.org/x/sys v0.45.0 // indirect + golang.org/x/sys v0.45.0 golang.org/x/term v0.43.0 golang.org/x/text v0.37.0 golang.org/x/tools v0.45.0 // indirect diff --git a/go.sum b/go.sum index 95fe9da7ed67..d5ca69ec5f2d 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8= -cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -67,8 +65,6 @@ fyne.io/systray v1.12.0 h1:CA1Kk0e2zwFlxtc02L3QFSiIbxJ/P0n582YrZHT7aTM= fyne.io/systray v1.12.0/go.mod h1:RVwqP9nYMo7h5zViCBHri2FgjXF7H2cub7MAq4NSoLs= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= -github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU= github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d h1:zjqpY4C7H15HjRPEenkS4SAn3Jy2eRRjkjZbGR30TOg= github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d/go.mod h1:XNqJ7hv2kY++g8XEHREpi+JqZo3+0l+CH2egBVN4yqM= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= @@ -304,31 +300,18 @@ github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUo github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= -github.com/containerd/containerd v1.7.31 h1:jn3IMuTV4Bb1Uwb0MFPW2ASJAD3W1lh6QqqZHIZwDh4= -github.com/containerd/containerd v1.7.31/go.mod h1:jdwD6s/BhV4XVJGrvtziNPVA+83n66TwptVaPKprq4E= -github.com/containerd/containerd v1.7.32 h1:S54xuVcPxeLaYgaRABtpJ2VyVUVsy0IGf7qHBs+sbY8= -github.com/containerd/containerd v1.7.32/go.mod h1:jdwD6s/BhV4XVJGrvtziNPVA+83n66TwptVaPKprq4E= github.com/containerd/containerd v1.7.33 h1:iAkYGC/ifR/V+0eR4iXWHNGYUF0DF2PmGV5iz4Irj5M= github.com/containerd/containerd v1.7.33/go.mod h1:gSbSCVjPCdkfJCjyrzz7aRC+xFlqVbatNpfHfVCYGUM= -github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= -github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= github.com/containerd/continuity v0.4.4 h1:/fNVfTJ7wIl/YPMHjf+5H32uFhl63JucB34PlCpMKII= github.com/containerd/continuity v0.4.4/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= -github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= -github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= -github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= -github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= -github.com/containerd/typeurl/v2 v2.2.0 h1:6NBDbQzr7I5LHgp34xAXYF5DOTQDn05X58lsPEmzLso= -github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= github.com/coreos/go-oidc/v3 v3.18.0 h1:V9orjXynvu5wiC9SemFTWnG4F45v403aIcjWo0d41+A= github.com/coreos/go-oidc/v3 v3.18.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -384,8 +367,6 @@ github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqI github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= @@ -993,8 +974,6 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= -github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= -github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg= github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= github.com/moby/moby/client v0.4.1 h1:DMQgisVoMkmMs7fp3ROSdiBnoAu8+vo3GggFl06M/wY= @@ -1005,8 +984,6 @@ github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9Kou github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= -github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= @@ -1038,8 +1015,6 @@ github.com/mudler/edgevpn v0.34.0 h1:qDrD/rCPFY/FdURbXudIZWihVKY4VOX3nMn3CcbeQEU github.com/mudler/edgevpn v0.34.0/go.mod h1:yki7uMi5LR9gSMrw8PdPieuxsrk8BLV2Ui7VBEmbbIA= github.com/mudler/go-piper v0.0.0-20241023091659-2494246fd9fc h1:RxwneJl1VgvikiX28EkpdAyL4yQVnJMrbquKospjHyA= github.com/mudler/go-piper v0.0.0-20241023091659-2494246fd9fc/go.mod h1:O7SwdSWMilAWhBZMK9N9Y/oBDyMMzshE3ju8Xkexwig= -github.com/mudler/go-processmanager v0.1.2-0.20260720195933-3d64f5c974fc h1:NEFmd7+JoImN5dZI81/vcBRjtMg+GfEa7nEjQ029hd0= -github.com/mudler/go-processmanager v0.1.2-0.20260720195933-3d64f5c974fc/go.mod h1:h6kmHUZeafr+k5hRYpGLMzJFH4hItHffgpRo2QIkP+o= github.com/mudler/go-processmanager v0.1.2-0.20260823202314-dfa0ed852db6 h1:/nFm1Ttf8g1BnWtEth986JR34pCh9rzae5A2vKBZosc= github.com/mudler/go-processmanager v0.1.2-0.20260823202314-dfa0ed852db6/go.mod h1:h6kmHUZeafr+k5hRYpGLMzJFH4hItHffgpRo2QIkP+o= github.com/mudler/localrecall v0.6.3 h1:uXOrP9JmetzxgVKzSrawviyBHZfAcvPBBIrvVUdZjDA= @@ -1090,14 +1065,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= -github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI= -github.com/nats-io/jwt/v2 v2.7.4/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA= -github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= -github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= -github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= -github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= @@ -1126,10 +1093,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= -github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE= -github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= @@ -1279,13 +1242,10 @@ github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/i github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/go-m1cpu v0.2.2 h1:4nc55oVv7nygGnfI9bhLCLzUEs4794y0Bkqx4q2zy7Y= github.com/shoenig/go-m1cpu v0.2.2/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= @@ -1386,8 +1346,6 @@ github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fx github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= -github.com/testcontainers/testcontainers-go/modules/nats v0.42.0 h1:WQR0+1r4GkM5QgOBoLxlP41empovt5PxtaiDpC0G7ow= -github.com/testcontainers/testcontainers-go/modules/nats v0.42.0/go.mod h1:ZAI9iisjDNJmcRcycQFKSLpiBN9u2g1v9AJRq1afriE= github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo= github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs= github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA= @@ -1666,8 +1624,6 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1756,7 +1712,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/natsauth/config.go b/pkg/natsauth/config.go deleted file mode 100644 index e9de534701a1..000000000000 --- a/pkg/natsauth/config.go +++ /dev/null @@ -1,66 +0,0 @@ -package natsauth - -import ( - "fmt" - "time" - - "github.com/mudler/xlog" -) - -// DefaultWorkerJWTTTL is how long a worker may use a minted NATS user JWT before re-registering. -const DefaultWorkerJWTTTL = 24 * time.Hour - -// Config holds NATS JWT authentication settings for distributed mode. -type Config struct { - // AccountSeed is the NATS account signing seed (SU...). Used to mint per-node worker JWTs. - AccountSeed string - // ServiceUserJWT is a pre-generated user JWT for frontends and agent workers (broad publish). - ServiceUserJWT string - // ServiceUserSeed is the signing seed (SU...) paired with ServiceUserJWT. - ServiceUserSeed string - // WorkerJWTTTL sets expiry on minted worker JWTs. Zero uses DefaultWorkerJWTTTL. - WorkerJWTTTL time.Duration - // RequireAuth rejects anonymous NATS when true (both ServiceUserJWT and AccountSeed expected). - RequireAuth bool -} - -// Enabled reports whether any NATS credential material is configured. -func (c Config) Enabled() bool { - return c.AccountSeed != "" || c.ServiceUserJWT != "" -} - -// CanMintWorkers reports whether per-node JWTs can be issued at registration. -func (c Config) CanMintWorkers() bool { - return c.AccountSeed != "" -} - -// WorkerTTL returns the configured worker JWT lifetime. -func (c Config) WorkerTTL() time.Duration { - if c.WorkerJWTTTL > 0 { - return c.WorkerJWTTTL - } - return DefaultWorkerJWTTTL -} - -// Validate checks consistency when distributed NATS auth is required. -func (c Config) Validate() error { - if !c.RequireAuth { - return nil - } - if c.ServiceUserJWT == "" || c.ServiceUserSeed == "" { - return fmt.Errorf("LOCALAI_NATS_REQUIRE_AUTH requires LOCALAI_NATS_SERVICE_JWT and LOCALAI_NATS_SERVICE_SEED") - } - if c.AccountSeed == "" { - return fmt.Errorf("LOCALAI_NATS_REQUIRE_AUTH is set but LOCALAI_NATS_ACCOUNT_SEED is empty") - } - return nil -} - -// WarnIfInsecure logs when distributed NATS is reachable without credentials. -func (c Config) WarnIfInsecure(distributed bool) { - if !distributed || c.Enabled() { - return - } - xlog.Warn("NATS is used without JWT credentials — any client on the bus can publish backend.install. " + - "Set LOCALAI_NATS_ACCOUNT_SEED + LOCALAI_NATS_SERVICE_JWT (see docs/features/distributed-mode.md).") -} diff --git a/pkg/natsauth/decode.go b/pkg/natsauth/decode.go deleted file mode 100644 index 1ae156f449c2..000000000000 --- a/pkg/natsauth/decode.go +++ /dev/null @@ -1,16 +0,0 @@ -package natsauth - -import ( - "fmt" - - "github.com/nats-io/jwt/v2" -) - -// DecodeUserClaims decodes a minted worker JWT for tests and diagnostics. -func DecodeUserClaims(token string) (*jwt.UserClaims, error) { - uc, err := jwt.DecodeUserClaims(token) - if err != nil { - return nil, fmt.Errorf("natsauth: decode user JWT: %w", err) - } - return uc, nil -} diff --git a/pkg/natsauth/mint.go b/pkg/natsauth/mint.go deleted file mode 100644 index 387e8d701ba0..000000000000 --- a/pkg/natsauth/mint.go +++ /dev/null @@ -1,59 +0,0 @@ -package natsauth - -import ( - "fmt" - "time" - - "github.com/nats-io/jwt/v2" - "github.com/nats-io/nkeys" -) - -// MintWorkerJWT creates a signed NATS user JWT and user seed scoped to nodeID and nodeType. -// The seed is returned once at registration so the worker can sign NATS connections. -func (c Config) MintWorkerJWT(nodeID, nodeType string) (userJWT, userSeed string, err error) { - if c.AccountSeed == "" { - return "", "", fmt.Errorf("natsauth: account seed not configured") - } - if nodeID == "" { - return "", "", fmt.Errorf("natsauth: node ID is required") - } - - accountKP, err := nkeys.FromSeed([]byte(c.AccountSeed)) - if err != nil { - return "", "", fmt.Errorf("natsauth: invalid account seed: %w", err) - } - - userKP, err := nkeys.CreateUser() - if err != nil { - return "", "", fmt.Errorf("natsauth: create user key: %w", err) - } - seedBytes, err := userKP.Seed() - if err != nil { - return "", "", fmt.Errorf("natsauth: user seed: %w", err) - } - - accountPub, err := accountKP.PublicKey() - if err != nil { - return "", "", fmt.Errorf("natsauth: account public key: %w", err) - } - userPub, err := userKP.PublicKey() - if err != nil { - return "", "", fmt.Errorf("natsauth: user public key: %w", err) - } - - pubAllow, subAllow := WorkerPermissions(nodeID, nodeType) - - uc := jwt.NewUserClaims(userPub) - uc.Name = fmt.Sprintf("localai-%s-%s", nodeType, workerSubjectToken(nodeID)) - uc.IssuerAccount = accountPub - uc.Expires = time.Now().Add(c.WorkerTTL()).Unix() - - uc.Permissions.Pub.Allow = pubAllow - uc.Permissions.Sub.Allow = subAllow - - token, err := uc.Encode(accountKP) - if err != nil { - return "", "", fmt.Errorf("natsauth: encode user JWT: %w", err) - } - return token, string(seedBytes), nil -} diff --git a/pkg/natsauth/mint_test.go b/pkg/natsauth/mint_test.go deleted file mode 100644 index f968613574c3..000000000000 --- a/pkg/natsauth/mint_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package natsauth_test - -import ( - "testing" - "time" - - "github.com/mudler/LocalAI/pkg/natsauth" - "github.com/nats-io/jwt/v2" - "github.com/nats-io/nkeys" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestNatsAuth(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "NatsAuth") -} - -var _ = Describe("MintWorkerJWT", func() { - var accountSeed string - - BeforeEach(func() { - akp, err := nkeys.CreateAccount() - Expect(err).NotTo(HaveOccurred()) - seed, err := akp.Seed() - Expect(err).NotTo(HaveOccurred()) - accountSeed = string(seed) - }) - - It("mints a JWT with backend worker permissions", func() { - cfg := natsauth.Config{AccountSeed: accountSeed, WorkerJWTTTL: time.Hour} - token, seed, err := cfg.MintWorkerJWT("550e8400-e29b-41d4-a716-446655440000", "backend") - Expect(err).NotTo(HaveOccurred()) - Expect(token).NotTo(BeEmpty()) - Expect(seed).NotTo(BeEmpty()) - - uc, err := jwt.DecodeUserClaims(token) - Expect(err).NotTo(HaveOccurred()) - // A backend worker opens no bus connection at all, so its node subtree - // went with it. The JWT is still minted at registration and simply - // unused; asserting BOTH lists are exactly the inbox is what keeps it - // from silently becoming an unrestricted credential, since NATS reads - // an empty allow list as no restriction. - Expect(uc.Permissions.Sub.Allow).To(ConsistOf("_INBOX.>")) - // The install-progress subject is gone with the carrier: progress is a - // line in the install response now, so a minted worker JWT must not - // still be granted a publish right for it. File staging went the same - // way, so a minted worker JWT publishes nowhere but its own inbox. - Expect(uc.Permissions.Pub.Allow).To(ConsistOf("_INBOX.>")) - for _, subj := range uc.Permissions.Pub.Allow { - Expect(subj).NotTo(ContainSubstring("backend.install")) - } - }) - - It("mints agent permissions without backend install subscribe", func() { - cfg := natsauth.Config{AccountSeed: accountSeed} - token, _, err := cfg.MintWorkerJWT("node-1", "agent") - Expect(err).NotTo(HaveOccurred()) - - uc, err := jwt.DecodeUserClaims(token) - Expect(err).NotTo(HaveOccurred()) - Expect(uc.Permissions.Sub.Allow).To(ContainElement("agent.*.cancel")) - for _, subj := range uc.Permissions.Sub.Allow { - Expect(subj).NotTo(ContainSubstring("backend.install")) - } - }) - - It("rejects mint without account seed", func() { - _, _, err := (natsauth.Config{}).MintWorkerJWT("id", "backend") - Expect(err).To(HaveOccurred()) - }) -}) diff --git a/pkg/natsauth/permissions.go b/pkg/natsauth/permissions.go deleted file mode 100644 index c64bdcd882df..000000000000 --- a/pkg/natsauth/permissions.go +++ /dev/null @@ -1,67 +0,0 @@ -package natsauth - -import "strings" - -// workerSubjectToken mirrors messaging.sanitizeSubjectToken without importing unexported logic. -func workerSubjectToken(nodeID string) string { - r := strings.NewReplacer(".", "-", "*", "-", ">", "-", " ", "-", "\t", "-", "\n", "-") - return r.Replace(nodeID) -} - -// WorkerPermissions returns NATS pub/sub allow lists for a registered node. -// -// It serves AGENT nodes. They are the only workers left that connect to the -// bus: an agent worker subscribes to the queue subjects listed below, while a -// backend worker connects to no bus at all, because every verb a frontend gives -// it is an HTTP route on its own server reached through its outbound tunnel -// (core/services/workerctl). -// -// The non-agent branch is therefore a grant of nothing, and it has to be -// spelled that way rather than deleted. NATS reads an EMPTY allow list as no -// restriction, so a function that returned nil here would upgrade every JWT the -// frontend still mints for a backend node from "its own inbox" to "the entire -// account". The inbox is self-scoped and reaches no cluster subject. -// -// nodeID no longer narrows anything: no allow list below is per-node, because -// the only per-node subject an agent worker ever subscribed to was its -// backend.stop, which is a control RPC on its tunnel now. It stays in the -// signature so a future per-node grant has somewhere to come from, and because -// mint.go names the JWT user after it. -func WorkerPermissions(nodeID, nodeType string) (pubAllow, subAllow []string) { - switch nodeType { - case "agent": - // Keep this list in sync with the subscriptions in core/cli/agent_worker.go. - // - // MCP tool execution, discovery, agent execution, MCP CI runs and the - // per-node backend.stop are all absent: every one of them is a control - // RPC on the tunnel the worker holds, addressed by the frontend rather - // than by a subject. The last two left when the queue groups became - // claim rows on the job store. - // - // Removing them narrowed this list; it must never be narrowed to - // nothing, because NATS reads an EMPTY allow list as no restriction at - // all, which would widen an agent JWT to the whole account. - subAllow = []string{ - "agent.*.cancel", - "gallery.*.cancel", - "gallery.*.progress", - "jobs.*.cancel", - "jobs.*.progress", - "jobs.*.result", - "staging.*.progress", - "_INBOX.>", - } - pubAllow = []string{ - "agent.>", - "jobs.>", - "_INBOX.>", - } - default: - // Backend worker: nothing, held open at its own inbox for the reason in - // the doc comment. The node subtree it used to subscribe on went with - // the connection itself, which this worker no longer opens. - subAllow = []string{"_INBOX.>"} - pubAllow = []string{"_INBOX.>"} - } - return pubAllow, subAllow -} diff --git a/pkg/natsauth/permissions_coverage_test.go b/pkg/natsauth/permissions_coverage_test.go deleted file mode 100644 index e5b9db21ad38..000000000000 --- a/pkg/natsauth/permissions_coverage_test.go +++ /dev/null @@ -1,164 +0,0 @@ -package natsauth_test - -import ( - "strings" - - "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/pkg/natsauth" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -// subjectMatches implements NATS subject-token matching: "*" matches exactly one -// token and ">" matches one or more trailing tokens. It lets these tests assert -// that a permission allow-list (which uses wildcards) actually covers a concrete -// subject a component publishes/subscribes — the same check the NATS server makes. -func subjectMatches(pattern, subject string) bool { - p := strings.Split(pattern, ".") - s := strings.Split(subject, ".") - for i, tok := range p { - if tok == ">" { - return i < len(s) // ">" must match at least one remaining token - } - if i >= len(s) { - return false - } - if tok != "*" && tok != s[i] { - return false - } - } - return len(p) == len(s) -} - -// workerSubjectTokenForTest mirrors the sanitizer both packages implement, so -// the negative assertion below names the exact prefix without reaching into -// either package's unexported copy. -func workerSubjectTokenForTest(nodeID string) string { - return strings.NewReplacer(".", "-", "*", "-", ">", "-", " ", "-", "\t", "-", "\n", "-").Replace(nodeID) -} - -func anyAllows(allow []string, subject string) bool { - for _, p := range allow { - if subjectMatches(p, subject) { - return true - } - } - return false -} - -var _ = Describe("WorkerPermissions subject coverage", func() { - // A node ID containing NATS-reserved characters exercises the (duplicated) - // sanitizer in pkg/natsauth against the canonical one in core/services/messaging. - // If the two ever diverge, the minted prefix stops matching the real subject - // and these assertions fail — guarding the copy noted in the review. - const nodeID = "host.a 1*b" - - Context("backend worker", func() { - pub, sub := natsauth.WorkerPermissions(nodeID, "backend") - - // A backend worker opens no connection at all on this build. Every verb - // a frontend gives it, the backend and model lifecycle ten plus the - // four file-staging verbs, is an HTTP route on its tunnelled control - // plane, so there is no subject left to cover. See - // core/services/workerctl. - It("no longer grants a backend worker its own node subtree to subscribe on", func() { - Expect(sub).ToNot(ContainElement("nodes."+workerSubjectTokenForTest(nodeID)+".>"), - "the node subtree went with the connection the worker no longer opens") - }) - - // The grant must be a grant of NOTHING and not an ABSENT grant: NATS - // treats an empty allow list as no restriction, so a branch that - // returned nil would silently widen every backend JWT the frontend - // still mints to the whole account. ConsistOf, not BeEmpty, is what - // tells those two apart. - It("grants a backend worker its own inbox and nothing else", func() { - Expect(sub).To(ConsistOf("_INBOX.>")) - Expect(sub).ToNot(BeEmpty(), - "an empty allow list is unrestricted in NATS, not restrictive") - }) - - // The negative half, and it is the one that would catch a verb quietly - // coming back to the bus: a backend worker is granted nothing to - // publish at all beyond its own inbox. File staging used to be the one - // exception and is not any more. - It("grants a backend worker no publish rights outside its inbox", func() { - Expect(pub).To(ConsistOf("_INBOX.>")) - Expect(pub).ToNot(BeEmpty(), - "an empty allow list is unrestricted in NATS, not restrictive") - }) - - It("no longer grants a backend worker the file-staging publish subtree", func() { - Expect(anyAllows(pub, "nodes."+workerSubjectTokenForTest(nodeID)+".files.stage")).To(BeFalse(), - "backend JWT pub allow-list %v still covers file staging", pub) - }) - }) - - Context("agent worker", func() { - // node_type "agent"; subjects from core/cli/agent_worker.go. - pub, sub := natsauth.WorkerPermissions(nodeID, "agent") - - // The subjects an agent worker still subscribes to. Both queue-group - // workloads have left this list: agent execution and MCP CI runs are - // streaming control verbs on the tunnel now, driven by a claim a - // frontend replica took off the job store. - subscribed := []string{ - messaging.SubjectAgentCancelWildcard, - } - - // The half that catches a narrowing going too far. NATS reads an EMPTY - // allow list as NO restriction, so a branch trimmed to nothing does not - // lock an agent worker down, it opens the whole account to it. - It("keeps the agent worker's allow lists non-empty", func() { - Expect(sub).ToNot(BeEmpty(), - "an empty allow list is unrestricted in NATS, not restrictive") - Expect(pub).ToNot(BeEmpty(), - "an empty allow list is unrestricted in NATS, not restrictive") - }) - - // MCP execution and discovery, backend.stop, and now agent execution - // and MCP CI runs, are all control RPCs on the worker's tunnel, - // addressed by the frontend rather than by a subject. An agent worker - // subscribes to none of them, so a JWT that still granted one would be - // granting a subscription nothing serves. Every entry is spelled out by - // hand rather than built from a deleted constant, which is the only way - // this can still name the subject that used to be granted. - for _, subject := range []string{ - "mcp.tools.execute", - "mcp.discovery", - "nodes." + workerSubjectTokenForTest(nodeID) + ".backend.stop", - "agent.execute", - "jobs.mcp-ci.new", - } { - It("no longer grants an agent worker "+subject, func() { - Expect(anyAllows(sub, subject)).To(BeFalse(), - "agent JWT sub allow-list %v still covers %s", sub, subject) - }) - } - - // The other half of the narrowing, and the one the phase's own trap - // makes necessary: an allow list trimmed to nothing is UNRESTRICTED in - // NATS. Naming what must survive is what tells a narrowing from an - // escalation, so the queue workloads an agent worker still consumes are - // asserted present just above, and the list is asserted non-empty here. - It("still grants the agent worker the broadcast subjects it lives on", func() { - Expect(sub).To(ContainElement("agent.*.cancel")) - Expect(sub).To(ContainElement("jobs.*.progress")) - Expect(sub).To(ContainElement("_INBOX.>")) - }) - for _, subject := range subscribed { - It("allows subscribing to "+subject, func() { - Expect(anyAllows(sub, subject)).To(BeTrue(), - "agent JWT sub allow-list %v does not cover %s — the agent worker subscribes to it", sub, subject) - }) - } - }) -}) - -// The "Documented NATS service-user permissions" suite that stood here read -// scripts/nats-auth-setup.sh and required its --allow-pub list to cover every -// subject the frontend publishes. Both the script and the frontend's bus -// connection are gone: cross-replica fan-out is a PostgreSQL NOTIFY, which has -// no allow-list to fall out of sync with. The property was retired with the -// artifact it guarded, not moved, and the surviving suites above still pin the -// MINTED credentials, which are Task 17's to remove. diff --git a/scripts/build/healthcheck_test.sh b/scripts/build/healthcheck_test.sh index 9dc811b8fbf4..bcba9a460c40 100644 --- a/scripts/build/healthcheck_test.sh +++ b/scripts/build/healthcheck_test.sh @@ -135,8 +135,9 @@ echo "== a 503 from a still-preloading frontend is unhealthy, not a crash" run_hc 1 "local-ai run" STUB_CURL_EXIT=22 echo "== modes with no HTTP surface report healthy rather than false-unhealthy" -# agent-worker is NATS-only. Reporting `unhealthy` forever for a process that -# was never going to bind a port is the same bug as #10987, one mode over. +# agent-worker binds no port: it dials a tunnel out to a frontend and is +# reached over that. Reporting `unhealthy` forever for a process that was never +# going to bind a port is the same bug as #10987, one mode over. run_hc 0 "local-ai agent-worker" expect_url "" diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index fe48d5672ceb..65672ac1267c 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -4,7 +4,8 @@ // process to kill, no second replica to race, and no real HTTP boundary between // a worker and the frontend it registered with. This package starts the same // binary an operator runs, one process per frontend replica and one per worker, -// against containerised Postgres and NATS. +// against containerised Postgres. Postgres is the only container: a distributed +// deployment needs it and the frontends' own HTTP listener, and nothing else. package cluster import ( @@ -23,6 +24,21 @@ import ( "github.com/phayes/freeport" ) +// StaleBusURL is the address of a broker that is not running, and is not meant +// to be. +// +// LOCALAI_NATS_URL is still accepted and ignored by every process this harness +// starts, so an operator's existing command line, unit file or Helm values file +// starts unchanged after the broker is shut down. This suite covers that +// promise by handing over THIS value: a value pointing at a live server would +// let a regression that dialled it pass unnoticed, while a dead one turns the +// same regression into a startup failure in every cluster spec. +// +// It is a constant here rather than an Options field, because there is nothing +// left for a caller to choose: no process reads the variable, so a per-cluster +// value would only be a knob that changes nothing. +const StaleBusURL = "nats://127.0.0.1:1" + // Options configures a cluster. Every field without a default is required. type Options struct { // Binary is the path to a built local-ai. @@ -31,9 +47,12 @@ type Options struct { // into each worker's backends directory as "mock-backend", which is the name // model YAML refers to (see tests/e2e/e2e_suite_test.go:75). MockBackend string - // PGDSN and NatsURL point at infrastructure the caller already started. - PGDSN string - NatsURL string + // PGDSN points at infrastructure the caller already started. + // + // There is no NatsURL beside it any more. Removing the FIELD rather than + // ignoring it means a spec that still sets one fails to compile, which is + // the only way a harness option stops being set by accident. + PGDSN string // LogDir receives one file per process. Never empty: a cluster failure is // unreadable without them. LogDir string @@ -275,7 +294,13 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { // the children need PATH, HOME and the Go/CI environment intact. cmd.Env = append(cmd.Environ(), "LOCALAI_DISTRIBUTED=true", - "LOCALAI_NATS_URL="+c.opts.NatsURL, + // Deliberately handed a broker URL, and deliberately a dead one. This + // is the shape of an operator's existing unit file on the day they shut + // the broker down, and the promise it covers is that such a command + // line still STARTS. A live address would let a regression that dialled + // it pass; this one turns such a regression into a startup failure in + // every cluster spec. + "LOCALAI_NATS_URL="+StaleBusURL, "LOCALAI_AUTH=true", "LOCALAI_AUTH_DATABASE_URL="+c.opts.PGDSN, "LOCALAI_ADMIN_EMAIL="+c.opts.AdminEmail, @@ -405,10 +430,11 @@ func (c *Cluster) startAgentWorker(i int) (*Process, error) { name := agentWorkerName(i) cmd := exec.Command(c.opts.Binary, "agent-worker") cmd.Env = append(cmd.Environ(), - // Still set, and still ignored. It is left here so this suite keeps - // covering the promise that an operator's existing --nats-url does not - // break an agent worker; nothing in the process reads it any more. - "LOCALAI_NATS_URL="+c.opts.NatsURL, + // Still set, and still ignored, on the same terms as the frontend's + // above: this suite keeps covering the promise that an operator's + // existing --nats-url does not break an agent worker. Nothing in the + // process reads it. + "LOCALAI_NATS_URL="+StaleBusURL, "LOCALAI_REGISTER_TO="+c.workerFrontendURL(i), "LOCALAI_NODE_NAME="+name, "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, @@ -434,9 +460,28 @@ func (c *Cluster) WorkerEnviron(i int) ([]string, error) { if err := c.checkWorkerIndex(i); err != nil { return nil, err } - p := c.workers[i] + return processEnviron(c.workers[i]) +} + +// FrontendEnviron is the environment of frontend i's RUNNING PROCESS, read the +// same way. +// +// It exists so a spec proving a WORKER runs with no broker URL can show that +// the deployment it joined was handed one: without that half, a harness which +// simply stopped setting the variable anywhere would satisfy the worker +// assertion just as well. Read from /proc rather than from the harness options +// for the same reason the worker's is: it is a fact about the process. +func (c *Cluster) FrontendEnviron(i int) ([]string, error) { + if err := c.checkFrontendIndex(i); err != nil { + return nil, err + } + return processEnviron(c.frontends[i]) +} + +// processEnviron reads a running child's environment out of /proc. +func processEnviron(p *Process) ([]string, error) { if p == nil || p.Cmd == nil || p.Cmd.Process == nil { - return nil, fmt.Errorf("worker %d is not running", i) + return nil, fmt.Errorf("process is not running") } raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/environ", p.Cmd.Process.Pid)) if err != nil { @@ -476,8 +521,8 @@ const ( // starts at 32768 by default. That is not tidiness. Ports the kernel hands // out for outbound connections are exactly the ports a long-lived process // full of outbound connections is liable to be holding when a backend tries - // to bind one, and this suite's workers hold a tunnel, a NATS connection - // and a registration client each. + // to bind one, and this suite's workers hold a tunnel and a registration + // client each. workerPortFloor = 20000 workerPortCeiling = 31000 @@ -622,15 +667,6 @@ func (c *Cluster) FrontendBackendsDir(i int) (string, error) { return filepath.Join(c.frontendDir(i), "backends"), nil } -// NatsURL is the bus this cluster's frontends were given. -// -// It is exported for one assertion: a spec proving a WORKER runs with no bus -// has to show the deployment it joined has one, or "no NATS anywhere" would -// satisfy it just as well. -func (c *Cluster) NatsURL() string { - return c.opts.NatsURL -} - // WorkerName is the node name worker i registered under. func (c *Cluster) WorkerName(i int) string { return c.workers[i].Name diff --git a/tests/e2e/distributed/cluster/cluster_test.go b/tests/e2e/distributed/cluster/cluster_test.go index 2f63d0a28e76..5089c09e4b35 100644 --- a/tests/e2e/distributed/cluster/cluster_test.go +++ b/tests/e2e/distributed/cluster/cluster_test.go @@ -17,7 +17,6 @@ var _ = Describe("Cluster options", Label("Distributed"), func() { _, err := cluster.Start(cluster.Options{ Binary: filepath.Join(os.TempDir(), "definitely-not-local-ai"), PGDSN: "postgres://test:test@127.0.0.1:5432/x?sslmode=disable", - NatsURL: "nats://127.0.0.1:4222", LogDir: GinkgoT().TempDir(), Frontends: 1, }) @@ -29,7 +28,6 @@ var _ = Describe("Cluster options", Label("Distributed"), func() { _, err := cluster.Start(cluster.Options{ Binary: "/bin/true", PGDSN: "postgres://test:test@127.0.0.1:5432/x?sslmode=disable", - NatsURL: "nats://127.0.0.1:4222", LogDir: GinkgoT().TempDir(), Frontends: 0, }) @@ -39,9 +37,9 @@ var _ = Describe("Cluster options", Label("Distributed"), func() { }) // The HTTP flow inside AdminSession and GetJSON cannot run here: it needs a -// built local-ai plus real Postgres and NATS, which arrive with the failover -// suites. These specs cover the argument validation that would otherwise panic -// on an out-of-range slice index inside a helper every later spec calls. +// built local-ai plus real Postgres, which arrives with the failover suites. +// These specs cover the argument validation that would otherwise panic on an +// out-of-range slice index inside a helper every later spec calls. var _ = Describe("Admin session", Label("Distributed"), func() { It("reports a clear error when the frontend index is out of range", func() { c := cluster.ForTestingEmpty() @@ -59,8 +57,8 @@ var _ = Describe("Admin session", Label("Distributed"), func() { }) // Like the admin specs above, these cover argument validation only. Killing, -// stopping and restarting a real replica needs a built local-ai plus Postgres -// and NATS, so those paths stay unexecuted until the failover suites land. +// stopping and restarting a real replica needs a built local-ai plus Postgres, +// so those paths stay unexecuted until the failover suites land. var _ = Describe("Failure primitives", Label("Distributed"), func() { It("rejects an out-of-range frontend index rather than panicking", func() { c := cluster.ForTestingEmpty() diff --git a/tests/e2e/distributed/cluster/failure.go b/tests/e2e/distributed/cluster/failure.go index fe808401cd56..49e6ecd3537f 100644 --- a/tests/e2e/distributed/cluster/failure.go +++ b/tests/e2e/distributed/cluster/failure.go @@ -51,7 +51,7 @@ func (c *Cluster) KillWorker(i int) error { // TIME_WAIT on an accepted connection does not block the bind. // // The data directory is wiped so the replica must rehydrate node, session and -// job state from the shared Postgres and NATS. Keeping it would model a pod +// job state from the shared Postgres. Keeping it would model a pod // with a persistent volume and would hide the very class of bug these tests // exist to find. This is only safe because startFrontend pins // LOCALAI_AUTH_HMAC_SECRET: the secret otherwise lives at diff --git a/tests/e2e/distributed/cluster_baseline_test.go b/tests/e2e/distributed/cluster_baseline_test.go index e6cb851c677a..e85797e8e0f1 100644 --- a/tests/e2e/distributed/cluster_baseline_test.go +++ b/tests/e2e/distributed/cluster_baseline_test.go @@ -288,13 +288,12 @@ func startClusterOnFreshDB(frontends, workers int, customise ...func(*cluster.Op Binary: binary, MockBackend: mockBackend, PGDSN: infra.PGURL, - // Deliberately a dead address. Frontends and agent workers are still - // handed LOCALAI_NATS_URL so this suite keeps covering the promise that - // an operator's existing command line starts unchanged after the broker - // is shut down. Pointing it at a running server would make a regression - // that dialled it invisible; pointing it at nothing makes such a - // regression a startup failure in every cluster spec. - NatsURL: staleBusURL, + // There is no bus URL to pass. Frontends and agent workers are still + // handed a dead LOCALAI_NATS_URL, so this suite keeps covering the + // promise that an operator's existing command line starts unchanged + // after the broker is shut down, but the value is the harness's own + // cluster.StaleBusURL rather than a caller's choice: no process reads + // it, so there was nothing left for a caller to choose. LogDir: logDir, Frontends: frontends, Workers: workers, diff --git a/tests/e2e/distributed/cluster_control_test.go b/tests/e2e/distributed/cluster_control_test.go index c6f213f83c15..fb363e9dfba6 100644 --- a/tests/e2e/distributed/cluster_control_test.go +++ b/tests/e2e/distributed/cluster_control_test.go @@ -416,9 +416,14 @@ var _ = Describe("Control plane over the worker tunnel", Label("Distributed"), L // And the harness DOES still hand a LOCALAI_NATS_URL to the other // processes in this cluster, so the worker's lack of one is a property // of the worker and not of a harness that stopped setting the variable - // at all. There is no broker behind that URL any more, which is the - // point: nothing dials it, so nothing notices. - Expect(c.NatsURL()).ToNot(BeEmpty()) + // at all. Read from the FRONTEND's running process for the same reason + // the worker's absence is: the harness options would only say what was + // assembled. There is no broker behind that URL, which is the point: + // nothing dials it, so nothing notices. + frontendEnviron, err := c.FrontendEnviron(0) + Expect(err).ToNot(HaveOccurred()) + Expect(frontendEnviron).To(ContainElement(HavePrefix("LOCALAI_NATS_URL=")), + "the harness stopped handing a bus URL to anything, so the worker's lack of one proves nothing") probe := newRosterProbe(c, client, 0) Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). diff --git a/tests/e2e/distributed/finetune_distributed_test.go b/tests/e2e/distributed/finetune_distributed_test.go index b1a1409e39a3..c3c56de2cfc5 100644 --- a/tests/e2e/distributed/finetune_distributed_test.go +++ b/tests/e2e/distributed/finetune_distributed_test.go @@ -103,8 +103,12 @@ var _ = Describe("Fine-Tune Distributed", Label("Distributed"), func() { Expect(appCfg.Distributed.Enabled).To(BeFalse()) // Without distributed mode, fine-tune jobs use local in-memory - // state tracking. No PostgreSQL or NATS needed. - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) + // state tracking. No PostgreSQL needed. + // + // The "and no bus URL" half of this assertion is gone with the + // field it read: DistributedConfig has nowhere to hold one, which + // core/config's "broker surface" spec pins for every config rather + // than for this one. }) }) }) diff --git a/tests/e2e/distributed/foundation_test.go b/tests/e2e/distributed/foundation_test.go index cca2e271f053..5623b0f3d226 100644 --- a/tests/e2e/distributed/foundation_test.go +++ b/tests/e2e/distributed/foundation_test.go @@ -37,30 +37,28 @@ var _ = Describe("Phase 0: Foundation", Label("Distributed"), func() { Expect(appCfg.Auth.Enabled).To(BeFalse()) }) - It("leaves the inert bus URL empty when nothing sets it", func() { + // Two Its stood here: "leaves the inert bus URL empty when nothing sets + // it" and "should accept valid distributed configuration", which passed + // config.WithNatsURL and read the value back. Both are retired with the + // field and the option they used, and neither property is lost. + // + // "The value is never dialled" is no longer a promise about a stored + // value; there is nowhere to store one, which core/config's "broker + // surface" spec asserts by reflection so it cannot silently stop + // compiling when the field returns. "An existing command line still + // starts" moved DOWN a level, to where it is actually at risk: kong is + // what rejects an unknown flag, so core/cli's "frontend's broker flags" + // specs parse the real command line, and the cluster suite starts real + // frontends with a dead LOCALAI_NATS_URL in their environment. + It("should accept a valid distributed configuration", func() { appCfg := config.NewApplicationConfig( config.EnableDistributed, config.WithAuthEnabled(true), config.WithAuthDatabaseURL(infra.PGURL), ) - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) - }) - - It("should accept valid distributed configuration", func() { - // staleBusURL points at nothing on purpose. It is the shape of an - // operator's existing command line after the broker was shut down, - // and the promise this spec holds is that such a command line still - // STARTS: the value is parsed, stored and never dialled. Pointing - // it at a live server would let a regression that dialled it pass. - appCfg := config.NewApplicationConfig( - config.EnableDistributed, - config.WithAuthEnabled(true), - config.WithAuthDatabaseURL(infra.PGURL), - config.WithNatsURL(staleBusURL), - ) Expect(appCfg.Distributed.Enabled).To(BeTrue()) Expect(appCfg.Auth.Enabled).To(BeTrue()) - Expect(appCfg.Distributed.NatsURL).To(Equal(staleBusURL)) + Expect(appCfg.Distributed.Validate()).To(Succeed()) }) It("should generate unique frontend ID on startup", func() { diff --git a/tests/e2e/distributed/gallery_distributed_test.go b/tests/e2e/distributed/gallery_distributed_test.go index b5b4294e4dcd..05babe2798d3 100644 --- a/tests/e2e/distributed/gallery_distributed_test.go +++ b/tests/e2e/distributed/gallery_distributed_test.go @@ -171,7 +171,9 @@ var _ = Describe("Gallery Distributed", Label("Distributed"), func() { // Without distributed mode, gallery operations use the existing // in-memory galleryApplier map. No PostgreSQL needed. - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) + // + // The bus-URL half of this assertion went with the field it read; + // core/config's "broker surface" spec pins its absence. }) }) }) diff --git a/tests/e2e/distributed/job_dispatch_test.go b/tests/e2e/distributed/job_dispatch_test.go index c1c74d8ee0a3..070a186d43cb 100644 --- a/tests/e2e/distributed/job_dispatch_test.go +++ b/tests/e2e/distributed/job_dispatch_test.go @@ -239,7 +239,9 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { // Without distributed mode, jobs use local in-process dispatch. // The JobStore can still be used standalone with SQLite or in-memory. - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) + // + // The bus-URL half of this assertion went with the field it read; + // core/config's "broker surface" spec pins its absence. }) }) }) diff --git a/tests/e2e/distributed/model_routing_test.go b/tests/e2e/distributed/model_routing_test.go index cb64e4622a2f..b70a2c137f02 100644 --- a/tests/e2e/distributed/model_routing_test.go +++ b/tests/e2e/distributed/model_routing_test.go @@ -117,7 +117,9 @@ var _ = Describe("Model Routing", Label("Distributed"), func() { // Without distributed mode, no SmartRouter is created. // The ModelLoader uses its local process management. // This test documents the design decision. - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) + // + // The bus-URL half of this assertion went with the field it read; + // core/config's "broker surface" spec pins its absence. }) }) }) diff --git a/tests/e2e/distributed/skills_distributed_test.go b/tests/e2e/distributed/skills_distributed_test.go index 14929e6400f2..0bc822c72217 100644 --- a/tests/e2e/distributed/skills_distributed_test.go +++ b/tests/e2e/distributed/skills_distributed_test.go @@ -109,8 +109,10 @@ var _ = Describe("Skills Distributed", Label("Distributed"), func() { Expect(appCfg.Distributed.Enabled).To(BeFalse()) // Without distributed mode, skills are stored on the local - // filesystem. No PostgreSQL metadata or NATS cache invalidation. - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) + // filesystem. No PostgreSQL metadata and no cache invalidation. + // + // The bus-URL half of this assertion went with the field it read; + // core/config's "broker surface" spec pins its absence. }) }) }) diff --git a/tests/e2e/distributed/sse_routes_test.go b/tests/e2e/distributed/sse_routes_test.go index 2e7b64a9a911..0accf54537c0 100644 --- a/tests/e2e/distributed/sse_routes_test.go +++ b/tests/e2e/distributed/sse_routes_test.go @@ -127,7 +127,9 @@ var _ = Describe("SSE Routes", Label("Distributed"), func() { // Without distributed mode, carrier-backed SSE routes are not registered. // Agent SSE events use the in-process LocalAGI SSE manager instead. // Job progress is tracked in-memory. - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) + // + // The bus-URL half of this assertion went with the field it read; + // core/config's "broker surface" spec pins its absence. }) }) }) diff --git a/tests/e2e/distributed/testhelpers_test.go b/tests/e2e/distributed/testhelpers_test.go index 0508f067528c..d52d3cb315c5 100644 --- a/tests/e2e/distributed/testhelpers_test.go +++ b/tests/e2e/distributed/testhelpers_test.go @@ -37,14 +37,6 @@ type TestInfra struct { PGURL string } -// staleBusURL is the address of a broker that is not running, and is not meant -// to be. LOCALAI_NATS_URL and --nats-url are still accepted and ignored so an -// operator's existing command line, unit file or Helm values file starts -// unchanged after the broker is shut down; specs that exercise that promise -// hand over THIS value, because a value pointing at a live server would let a -// regression that dialled it pass unnoticed. -const staleBusURL = "nats://127.0.0.1:1" - // The container is suite-scoped, not spec-scoped. Starting a Postgres (~10s) per // spec cost roughly 36 minutes of pure startup across the 213 specs behind // SetupInfra, which is why this suite was never wired into CI. Isolation now diff --git a/website/layouts/index.html b/website/layouts/index.html index 22137a23c16c..4c014bb578a4 100644 --- a/website/layouts/index.html +++ b/website/layouts/index.html @@ -111,7 +111,7 @@

You can talk to it, and it answers.

03 / DISTRIBUTED

Add a second machine.

Routing, VRAM-aware placement, prefix-cache affinity and failover are the runtime's problem. You add hardware, the cluster works out what to do with it.

-

Smart routing · autoscaling · P2P · NATS · federation

+

Smart routing · autoscaling · P2P · tunnels · federation

From cf62a728f4224a68f0348ca0aaaa448b3d52101c Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 5 Sep 2026 21:47:35 +0000 Subject: [PATCH 101/153] test(distributed): prove the busless cluster on two frontends and two workers Tasks 1 to 17 are proven by unit and integration specs and by two e2e passes taken mid-flight. This is the pass that boots the real binaries with every carrier in place and none of the old one, and it does so on the topology the feature was built for rather than on the one-worker shape the rest of the cluster suite uses. Two frontends and two workers is the configuration that matters. With each worker's tunnel landing on a different replica, the owner path and the relay path are live at the same instant against one roster, one scheduler and one health monitor, so a routing mistake has somewhere to show up instead of hiding. It is also the only shape in which "killing a replica re-homes only ITS worker" can be stated at all. Three scenarios, all 2x2: 1. Both workers served from both replicas. No broker as a property of the ARTIFACT (debug/buildinfo reports no github.com/nats-io module, with the module count asserted non-zero so a stripped binary cannot pass vacuously), no broker in either worker's live /proc environment, and no advertised address on either worker. One completion over the owner path and one over the relay, plus the mirror image through the other replica, plus four control-plane listings covering both paths for both workers. 2. The replica owning worker 0's tunnel is killed with that tunnel blocked. Leg 1 asserts nothing and only waits for the killed instance to leave the live set, because before that it still reads as a live owner and the scenario is not yet about absence. Leg 2 then holds a window inside the reconnect grace requiring that nothing acted on the absence. Leg 3 requires the re-home and inference again. Worker 1 keeps serving throughout. 3. The suite-wide negative control. Both tunnel dials refused while registration and heartbeats flow, both workers refused at both replicas naming the routing fact and not a departure, nothing reaped and both heartbeats fresh. Then ONE tunnel is restored and exactly one worker recovers while the other stays refused. Which worker served is read back from node_models rather than assumed: the two models are pinned to one worker each through PUT /api/nodes/:id/labels and POST /api/nodes/scheduling, and every assertion requires the model to be on the expected node AND absent from the other. That the relay hit a non-owner is read from the production Owner query before the request and re-read after it. Attacks run, each alone, each reverted, each behaving as predicted: hand a worker a broker URL reddens scenario 1's environment leg; point the module check at gorm.io reddens its artifact leg; start one worker instead of two reddens all three at the topology guard; delete the relay in WorkerDialer.Dial reddens scenario 1 on exactly the request sent to the non-owner while 2 and 3 stay green; a one-nanosecond reconnect grace reddens scenario 2's leg 2 on the demotion while 1 and 3 stay green; lifting both blocks at scenario 3's differential reddens its "still unreachable" half. The brief's "restore the NatsURL validation" attack cannot be applied: DistributedConfig has no such field left to validate. Label-orphan arithmetic, counting non-skipped It nodes from --dry-run: all 256, dist 231, cluster 24, vllm 1, and 231 + 24 + 1 = 256, so no spec is orphaned by the label filters. Three test-e2e-cluster runs: 897.0s, 897.8s and 906.6s of Ginkgo time, 24 specs, 15 minutes wall. The only failure across the three was a pre-existing spec dying at cluster.Start with frontend-1 exiting status 2, which passed in the other two and is reported as a port-allocation flake rather than a regression. test-e2e-distributed is 223 plus 8 specs in 131.7s. The budget comment and .agents/building-and-testing.md move from 21 specs at 800 to 830 seconds to 24 specs at 897 to 907. The harness gains ProcessEnviron, which reads /proc for any of the three process families; WorkerEnviron and FrontendEnviron become wrappers rather than being deleted, so the specs that call them are not re-aimed for a rename. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .agents/building-and-testing.md | 3 +- Makefile | 17 +- tests/e2e/distributed/cluster/cluster.go | 61 +- tests/e2e/distributed/cluster_busless_test.go | 784 ++++++++++++++++++ 4 files changed, 845 insertions(+), 20 deletions(-) create mode 100644 tests/e2e/distributed/cluster_busless_test.go diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index ae1c9a4f629d..228c02bc2098 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -51,7 +51,7 @@ Rules (both gates): Two suites cover distributed mode (frontend replicas, worker nodes, PostgreSQL), split by a Ginkgo label. Neither starts a message broker: a distributed deployment needs PostgreSQL and the frontends' own HTTP listener, and the `nats-io` modules are not in the build. - `make test-e2e-distributed` runs `Distributed && !VLLMMultinode && !Cluster` over `./tests/e2e/distributed` recursively. Services are wired directly into the test binary. 231 specs in about 2m20s (measured 133.3s for the main suite plus the 8 argument-validation specs in the subpackage). -- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. It **builds `./local-ai` itself** (the `e2e-binary` target), so you do not run `make build` first and cannot forget to. 21 specs, 802.5s of specs in a 13m26s target run (one measurement; the suite grew from the 6 specs and roughly 509s recorded earlier, so budget on the newer number). +- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. It **builds `./local-ai` itself** (the `e2e-binary` target), so you do not run `make build` first and cannot forget to. 24 specs, 897 to 907s of specs in a 15m target run (three measurements; the suite grew from 6 specs and roughly 509s, then 21 specs and 802.5s, so budget on the newest number). Both jobs live in `.github/workflows/tests-e2e-distributed.yml`, with `timeout-minutes: 45` each. They trigger on pull requests *and* on every push to `master`; the `paths-ignore` filter (see [.agents/ci-caching.md](ci-caching.md)) sits on the pull-request trigger only, so a master push always runs both. They are advisory only because `master` carries no branch protection, which is a repository setting and not a YAML key: `continue-on-error: true` would flip the run's *conclusion* to success and hide the failure, so it is not used. @@ -65,4 +65,5 @@ Both jobs live in `.github/workflows/tests-e2e-distributed.yml`, with `timeout-m - **Flake budget: no retries at all.** `--flake-attempts` is *total attempts*, not retries (ginkgo v2.29.0 `internal/group.go` sets `maxAttempts = FlakeAttempts` and loops `attempt < maxAttempts`; the flag's own usage string reads "0 - failed tests are not retried"). `DISTRIBUTED_TEST_FLAKES` defaults to **1**, so each spec runs once and a failure is a failure, and `test-e2e-cluster` pins `--flake-attempts 1` outright rather than reading the variable. The repo-wide `TEST_FLAKES=5` means up to five attempts, so up to four retries. These suites exist to surface nondeterminism, and a retry converts exactly that signal into a green run. Raise it locally when bisecting something unrelated, not in the Makefile. - **Coverage:** `tests/e2e/distributed` is excluded from the coverage roots (`COVERAGE_E2E_ROOTS = ./tests/e2e`, run non-recursively), and so is the `cluster` helper package beneath it. Neither suite moves the baseline, so production code that these suites are the only cover for reads as **uncovered**. Unit tests for such code belong under `./core/...` with `testutil.SetupTestDB()`. - **The cluster job builds against a stubbed React UI.** `core/http/react-ui/dist` is gitignored and built by Node, so the workflow writes a one-line `index.html` there to satisfy the `//go:embed react-ui/dist/*` in `core/http/app.go` and skips a full Node and Vite install. That holds only while the suite drives the HTTP API and never the UI, which has its own e2e suite. A spec that ever asserts on a UI asset would pass locally, where a real `dist/` exists, and be served the stub in CI: if you write one, the stub step has to go and the real build come back. +- **The topology that matters is two frontends and two workers.** `cluster_busless_test.go` runs the whole programme on it, and it is the only shape in which the OWNER path and the RELAY path are live at the same time: with the two workers' tunnels landing on different replicas, every request either goes to the replica holding that worker's tunnel or is relayed through the one that does. A one-worker cluster cannot tell those apart, and cannot state that killing a replica re-homes only ITS worker. Which worker served a request is read back from `node_models` rather than assumed, and the two models are pinned to one worker each through the admin API (`PUT /api/nodes/:id/labels` plus `POST /api/nodes/scheduling`), so a selector that failed open reddens the spec instead of hiding in it. - **Do not shorten the cluster suite's waits.** Three of its six specs sit at ~167s each because they wait out a 60s staleness threshold plus a 15s health-check tick. That wait is what stops the assertions from passing before the system could have reacted, which was a real false green earlier on. If the job has to get faster, the levers are CI concurrency or making the thresholds configurable, not shorter waits. diff --git a/Makefile b/Makefile index fd7327005f26..d5be94cfa003 100644 --- a/Makefile +++ b/Makefile @@ -424,17 +424,20 @@ e2e-binary: protogen-go # should stay there: this suite exists to catch nondeterministic cluster # behaviour, and a retry turns exactly that signal into a green run. # -# Budget: 20 specs, measured at 800 to 830 seconds of Ginkgo time (13 to 14 -# minutes wall including the compile) on a fast developer box. It was 591 to 612 -# seconds before the phase 3 control-plane specs; those five added roughly 200 -# seconds, most of it in the two that wait out real windows rather than poll for -# a state change (cluster.InstanceLiveness is 30s, and a departed worker cannot -# be demoted before its reconnect grace). +# Budget: 24 specs, measured at 897 to 907 seconds of Ginkgo time (15 minutes +# wall including the compile) on a fast developer box. It was 591 to 612 seconds +# before the phase 3 control-plane specs and 800 to 830 after them; the three +# two-frontend two-worker specs in cluster_busless_test.go added 118 to 127 +# seconds (3s, 70s and 46 to 53s), nearly all of it in the churn spec, which +# cannot be shortened: it waits for a killed replica to leave the live set +# (cluster.InstanceLiveness is 30s, measured at 27s) before it may assert +# anything, and then holds a window inside the reconnect grace. # # --timeout is 30m rather than 20m because of that. The margin is not slack: a # Ginkgo timeout kills the suite mid-spec and reports a spec name rather than a # cause, and 20m on a loaded CI runner was one slow health tick away from -# turning a green suite into an unreadable red one. +# turning a green suite into an unreadable red one. At 15 minutes measured, 30m +# is still twice the budget. test-e2e-cluster: protogen-go build-mock-backend e2e-binary @echo 'Running cluster e2e tests (label Cluster, real local-ai processes)' $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Cluster' --fail-on-empty --flake-attempts 1 --timeout=30m -v ./tests/e2e/distributed diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 65672ac1267c..d0045b458474 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -443,6 +443,50 @@ func (c *Cluster) startAgentWorker(i int) (*Process, error) { return c.spawn(name, cmd, 0) } +// ProcKind names which of a cluster's three process families an environment +// read is about. +type ProcKind string + +const ( + ProcFrontend ProcKind = "frontend" + ProcWorker ProcKind = "worker" + ProcAgentWorker ProcKind = "agent-worker" +) + +// ProcessEnviron reads the LIVE environment of process i of kind k from +// /proc//environ. +// +// Generalised from WorkerEnviron, which an earlier phase added for exactly this +// assertion and which could only see workers. Reading /proc rather than Cmd.Env +// is the whole point: Cmd.Env is what the harness INTENDED, and an assertion +// about intent cannot fail when a variable arrives from the parent environment, +// a profile or a wrapper. +// +// Linux only, like the rest of this package. A platform without /proc returns +// the read error rather than falling back, so the assertion fails loudly +// instead of quietly becoming the weaker one. +func (c *Cluster) ProcessEnviron(k ProcKind, i int) ([]string, error) { + switch k { + case ProcFrontend: + if err := c.checkFrontendIndex(i); err != nil { + return nil, err + } + return processEnviron(c.frontends[i]) + case ProcWorker: + if err := c.checkWorkerIndex(i); err != nil { + return nil, err + } + return processEnviron(c.workers[i]) + case ProcAgentWorker: + if i < 0 || i >= len(c.agentWorkers) { + return nil, fmt.Errorf("agent worker %d out of range (cluster has %d)", i, len(c.agentWorkers)) + } + return processEnviron(c.agentWorkers[i]) + default: + return nil, fmt.Errorf("unknown process kind %q", k) + } +} + // WorkerEnviron is the environment of worker i's RUNNING PROCESS, read from // /proc. // @@ -452,15 +496,11 @@ func (c *Cluster) startAgentWorker(i int) (*Process, error) { // reading it proves the harness consistent with itself and nothing about the // binary. /proc//environ is what the kernel handed the process. // -// Linux only, which this package already is (it signals with syscall.SIGKILL -// and reserves ports by binding loopback). A platform without /proc returns the -// read error rather than falling back to Cmd.Env, so the assertion fails loudly -// instead of quietly becoming the weaker one. +// Kept as a named wrapper rather than replaced by ProcessEnviron at its call +// sites: cluster_control_test.go calls it and re-aiming a passing spec for a +// rename is churn that hides the new coverage. func (c *Cluster) WorkerEnviron(i int) ([]string, error) { - if err := c.checkWorkerIndex(i); err != nil { - return nil, err - } - return processEnviron(c.workers[i]) + return c.ProcessEnviron(ProcWorker, i) } // FrontendEnviron is the environment of frontend i's RUNNING PROCESS, read the @@ -472,10 +512,7 @@ func (c *Cluster) WorkerEnviron(i int) ([]string, error) { // assertion just as well. Read from /proc rather than from the harness options // for the same reason the worker's is: it is a fact about the process. func (c *Cluster) FrontendEnviron(i int) ([]string, error) { - if err := c.checkFrontendIndex(i); err != nil { - return nil, err - } - return processEnviron(c.frontends[i]) + return c.ProcessEnviron(ProcFrontend, i) } // processEnviron reads a running child's environment out of /proc. diff --git a/tests/e2e/distributed/cluster_busless_test.go b/tests/e2e/distributed/cluster_busless_test.go new file mode 100644 index 000000000000..cfd9ef13f7a4 --- /dev/null +++ b/tests/e2e/distributed/cluster_busless_test.go @@ -0,0 +1,784 @@ +package distributed_test + +import ( + "bytes" + "debug/buildinfo" + "encoding/json" + "fmt" + "net/http" + "slices" + "strings" + "time" + + "github.com/mudler/LocalAI/core/config" + clustersvc "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/tests/e2e/distributed/cluster" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The whole programme, end to end, on the topology it was built for: TWO +// frontend replicas and TWO backend workers, with no message broker anywhere. +// +// Why two of each and not the one-worker shape almost every other cluster spec +// uses. With two workers spread across two replicas the OWNER path and the +// RELAY path are live AT THE SAME TIME, against one roster, one scheduler and +// one health monitor. A routing mistake then has somewhere to show up: a +// frontend that answered for the worker it owns and quietly failed for the +// other, or one that relayed everything to a single replica, is a green run on +// a one-worker cluster and a red one here. It is also the production shape: +// with N replicas behind round robin, (N-1)/N of traffic arrives at a replica +// that owns nothing about the request it is holding. +// +// Read the LAST spec in this file before trusting the first two. Frontends and +// workers share a host here, so every port a frontend names is one it could +// also have dialled directly; if it did, the positive specs would pass with the +// tunnels doing nothing at all. The last one takes BOTH tunnels away and +// requires BOTH workers to become unreachable, then puts back ONE of them and +// requires exactly one to recover. That differential is what makes the two +// positive specs mean anything. + +const ( + // alphaModel and bravoModel are pinned to one worker each, by label and + // node selector, so "which worker served this" is a fact of the + // arrangement and not a guess about what the scheduler felt like doing. + // Every spec here then VERIFIES the pin from node_models rather than + // trusting it: a selector that silently failed open would otherwise leave + // a spec claiming a worker served a request it never saw. + alphaModel = "alpha-model" + bravoModel = "bravo-model" + + // slotLabel is the label key the two node selectors discriminate on. A key + // nothing else in this deployment sets, so a node matching it matched + // because this spec labelled it. + slotLabel = "e2e.slot" + alphaSlot = "alpha" + bravoSlot = "bravo" + + // nodeModelSettleTimeout bounds the wait for a loaded model to appear in + // the node_models rows a frontend reports. The load itself has already + // answered by the time this is polled; this covers the write that records + // where it landed. + nodeModelSettleTimeout = "60s" + nodeModelSettlePoll = "500ms" + + // buslessOwnershipTimeout bounds the wait for both workers' tunnels to be + // claimed. Two workers dial two balancers at once, so it is the same wait + // the one-worker specs make, with room for the second claim. + buslessOwnershipTimeout = 90 * time.Second + buslessOwnershipPoll = 500 * time.Millisecond + + // deadInstanceTimeout bounds leg 1 of the churn spec: how long the killed + // replica may go on reading as LIVE in the instances table. + // + // It has to outlast cluster.InstanceLiveness (30s), because the ownership + // query joins against instances the DATABASE still considers live. Measured + // at 27.0s on the box this was written on, so the bound is a shade over + // twice that rather than a shade over the constant: a run that lands one + // sweep interval late is a slow runner, not a regression, and the spec has + // room for it because leg 2 checks that the two legs together still fit + // inside the grace. + // + // Nothing is asserted before this completes: before it, the scenario is not + // yet about absence at all, and a Consistently placed inside the liveness + // window is the vacuous spec this file exists not to repeat. + deadInstanceTimeout = "60s" + deadInstancePoll = "1s" + + // absenceProbeTimeout caps ONE leg 2 poll. + // + // The shared session's budget is three minutes, sized for a cold model load + // across a tunnel, and a single poll allowed to spend it would consume the + // whole grace and then report the model row as evicted when what actually + // happened is that one read never came back. Capping the poll turns that + // into a legible failure in seconds. Observed once, on a host-wide stall in + // which a LOOPBACK dial timed out. + absenceProbeTimeout = 20 * time.Second + + // absenceHoldWindow is leg 2: how long the fleet must survive with the + // worker's tunnel genuinely unowned and its departure inside the reconnect + // grace. + // + // It starts only AFTER leg 1 has observed the owner leave the live set, so + // every second of it is spent in the branch the assertion is about. It is + // well inside config.DefaultWorkerReconnectGrace (90s) and past the health + // monitor's 15s tick, so at least one sweep runs against a worker nothing + // can reach and finds nothing to reap. + // + // ONE window covering both facts, not one per fact. Two 20s windows back to + // back put leg 1 plus leg 2 at 40s plus whatever leg 1 cost, which on a slow + // runner reaches the 90s grace; the grace would then expire legitimately + // mid-assertion and the spec would report an eviction as though it were the + // defect. One window keeps the pair inside the grace with room to spare, and + // absenceViolations below is what lets a single window still name WHICH of + // the two facts broke. + absenceHoldWindow = "20s" + absenceHoldPoll = "1s" + + // buslessRefusalTimeout bounds a request to a worker that holds no tunnel. + // Resolving the route fails on a table read, so a request still in flight + // after this is not slow: it is parked on something that will never happen. + buslessRefusalTimeout = 60 * time.Second +) + +// putJSON performs an authenticated PUT and reports the status. +// +// The harness carries GetJSON and PostJSON; node labels are a PUT and there is +// no reason to widen the harness for one verb one file uses. +func putJSON(c *cluster.Cluster, client *http.Client, frontend int, path string, body any) (int, string, error) { + encoded, err := json.Marshal(body) + if err != nil { + return 0, "", err + } + req, err := http.NewRequest(http.MethodPut, c.FrontendURL(frontend)+path, bytes.NewReader(encoded)) + if err != nil { + return 0, "", err + } + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + return 0, "", err + } + defer func() { _ = resp.Body.Close() }() + var buf bytes.Buffer + _, _ = buf.ReadFrom(resp.Body) + return resp.StatusCode, buf.String(), nil +} + +// pinModelToNode labels a node and gives a model a selector that only that +// label matches, so the model can land on that worker and on no other. +// +// Both halves go through the ADMIN API rather than through the harness or a +// direct database write, which is deliberate: the pin is then the same +// arrangement an operator makes, and a regression in either endpoint fails +// here rather than being routed around. +func pinModelToNode(c *cluster.Cluster, client *http.Client, frontend int, model, nodeID, slot string) { + GinkgoHelper() + status, body, err := putJSON(c, client, frontend, "/api/nodes/"+nodeID+"/labels", + map[string]string{slotLabel: slot}) + Expect(err).ToNot(HaveOccurred()) + Expect(status).To(Equal(http.StatusOK), "labelling node %s failed: %s", nodeID, body) + + status, err = c.PostJSON(client, frontend, "/api/nodes/scheduling", map[string]any{ + "model_name": model, + "node_selector": map[string]string{slotLabel: slot}, + }, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(status).To(Equal(http.StatusOK), "pinning %s to node %s failed", model, nodeID) +} + +// servedBy asserts that model is loaded on wantNode and on no other node in +// nodes, reading the frontend's own node_models rows. +// +// This is how every spec in this file answers "which worker served that +// request" instead of assuming it. The exclusion half is the load-bearing one: +// "the model is on worker 0" is also true of a deployment that loaded it +// everywhere, which is precisely what a selector that failed open would do. +func servedBy(c *cluster.Cluster, client *http.Client, frontend int, model, wantNode string, otherNodes ...string) { + GinkgoHelper() + Eventually(func() ([]string, error) { + return nodeModelNames(c, client, frontend, wantNode) + }, nodeModelSettleTimeout, nodeModelSettlePoll).Should(ContainElement(model), + "frontend %d does not record %s as loaded on node %s, so nothing here says which worker served the request", + frontend, model, wantNode) + + for _, other := range otherNodes { + loaded, err := nodeModelNames(c, client, frontend, other) + Expect(err).ToNot(HaveOccurred()) + Expect(loaded).ToNot(ContainElement(model), + "node %s also holds %s, so the selector did not pin it and this spec cannot say which worker answered", other, model) + } +} + +// absenceViolations lists everything a deployment did in reaction to a worker +// whose tunnel is gone but whose departure is still inside the grace. An empty +// result is the invariant holding. +// +// It returns a LIST rather than a boolean so one Consistently window can cover +// both facts and still name which one broke. The alternative, one window per +// fact, doubles the time leg 2 spends and pushes the pair past the grace on a +// slow runner, at which point the spec reports a legitimate expiry as an early +// reap. +// +// A read that fails is a violation too, and deliberately so: this window is the +// only place the spec can observe the branch, and a poll that could not answer +// is not evidence that nothing happened. +func absenceViolations(c *cluster.Cluster, client *http.Client, probe *rosterProbe, frontend int, nodeID string, worker int) []string { + violations := []string{} + names, err := nodeModelNames(c, client, frontend, nodeID) + switch { + case err != nil: + violations = append(violations, "node_models unreadable: "+err.Error()) + case !slices.Contains(names, alphaModel): + violations = append(violations, fmt.Sprintf("%s's model row was evicted (rows now %v)", c.WorkerName(worker), names)) + } + if status := probe.statusOf(c.WorkerName(worker)); status != "healthy" { + violations = append(violations, fmt.Sprintf("%s was demoted to %q", c.WorkerName(worker), status)) + } + return violations +} + +// busSymbolsIn reports every module path in the binary's embedded build info +// that belongs to the message broker client, plus how many modules were read. +// +// The count is returned rather than discarded because it is the control: a +// binary whose build info could not be read, or one stripped of it, yields no +// module paths at all and would satisfy "no broker module" while proving +// nothing. Reading the ARTIFACT rather than a process environment is what +// turns "this cluster ran with no bus" from a statement about what the harness +// set into a property of the thing the harness executed. +func busSymbolsIn(path string) ([]string, int, error) { + info, err := buildinfo.ReadFile(path) + if err != nil { + return nil, 0, fmt.Errorf("reading build info from %s: %w", path, err) + } + found := []string{} + for _, dep := range info.Deps { + if strings.HasPrefix(dep.Path, "github.com/nats-io/") { + found = append(found, dep.Path+"@"+dep.Version) + } + } + return found, len(info.Deps), nil +} + +// rotatedFrom returns urls starting at index start, wrapping around. +func rotatedFrom(urls []string, start int) []string { + if len(urls) == 0 { + return nil + } + out := make([]string, 0, len(urls)) + for i := 0; i < len(urls); i++ { + out = append(out, urls[(start+i)%len(urls)]) + } + return out +} + +// withSpreadBalancers gives every worker its OWN balancer, preferring frontend +// i%Frontends and falling through to the rest. +// +// It replaces two things that cannot be combined. Options.SpreadWorkerRegistrations +// spreads workers across replicas but pins each one at a replica, so a worker +// whose replica dies has nowhere to reconnect to and the re-home this feature +// is built on cannot happen. withBalancer gives every worker somewhere to go +// but hands them ONE balancer that forwards to the first live target, so both +// workers land on the same replica and the relay path is never live at the same +// time as the owner path. One balancer per worker gives both: a spread that +// really is a spread, and a worker that survives its replica. +// +// Blocking is per worker too, which is what lets the negative control lift the +// block for ONE worker and require exactly one recovery. +func withSpreadBalancers(into *[]*frontendBalancer, arm ...func(int, *frontendBalancer)) func(*cluster.Options) { + return func(o *cluster.Options) { + o.WorkerFrontendURL = func(worker int, _ string, frontends []string) string { + for len(*into) <= worker { + *into = append(*into, nil) + } + if (*into)[worker] == nil { + b := newFrontendBalancer(rotatedFrom(frontends, worker%len(frontends))...) + for _, apply := range arm { + apply(worker, b) + } + (*into)[worker] = b + } + return (*into)[worker].URL() + } + } +} + +// twoByTwo starts the topology this file is about: two frontends, two backend +// workers, one balancer each, both mock models seeded. +func twoByTwo(into *[]*frontendBalancer, arm ...func(int, *frontendBalancer)) (*cluster.Cluster, string) { + GinkgoHelper() + return startClusterOnFreshDB(2, 2, + withMockModel(alphaModel), + withMockModel(bravoModel), + withSpreadBalancers(into, arm...)) +} + +// busless is the state every spec in this file establishes first: both workers +// registered and healthy at both replicas, their node ids, and which replica +// owns each tunnel. +type busless struct { + c *cluster.Cluster + client *http.Client + probe *rosterProbe + owners *tunnelOwners + nodeIDs [2]string + ownerIdx [2]int +} + +// awaitFleet waits for both workers to be healthy and reads their node ids. It +// does NOT read ownership: the negative control needs the fleet without it, +// because there is no owner to read. +func awaitFleet(c *cluster.Cluster, dsn string) *busless { + GinkgoHelper() + // The topology, before anything is read out of it. Every index-taking + // method on Cluster indexes blindly, so a cluster started with one worker + // panics somewhere further down with a runtime message that names a slice + // rather than the arrangement this whole file rests on. WorkerRegistrar is + // the one indexed method that reports an out-of-range index as an error. + _, err := c.WorkerRegistrar(1) + Expect(err).ToNot(HaveOccurred(), + "this file is about TWO workers on TWO replicas; with fewer, the owner path and the relay path cannot both be live and nothing below means what it says") + + client := inferenceClient(c) + probe := newRosterProbe(c, client, 0) + Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(And(ContainElement(c.WorkerName(0)), ContainElement(c.WorkerName(1))), probe.describe) + + b := &busless{c: c, client: client, probe: probe, owners: newTunnelOwners(openClusterDB(dsn))} + for i := 0; i < 2; i++ { + b.nodeIDs[i] = probe.idOf(c.WorkerName(i)) + Expect(b.nodeIDs[i]).ToNot(BeEmpty(), "the roster reported %s without a registration id", c.WorkerName(i)) + + // Neither worker publishes an endpoint of any kind. Both halves are + // needed: an empty value alone is also what a spec sees after the keys + // are renamed out of the payload, which would leave this reporting + // "advertises nothing" about a node whose advertisement it can no + // longer read. + advertised, carriedKeys := probe.advertisementOf(c.WorkerName(i)) + Expect(carriedKeys).To(BeTrue(), + "the roster payload no longer carries the address keys, so this spec cannot tell a worker that advertises nothing from one it cannot read") + Expect(advertised).To(BeEmpty(), + "%s advertised %q, so no spec here can tell a tunnelled request from a direct dial", c.WorkerName(i), advertised) + } + return b +} + +// awaitOwners reads which replica holds each worker's tunnel and requires the +// two to differ. +// +// The difference is the precondition of this whole file, asserted rather than +// arranged for: if both tunnels landed on one replica, every request below +// would take the owner path or the relay path but never both at once, and the +// spread this topology exists to create would be a spread in name only. +func (b *busless) awaitOwners() { + GinkgoHelper() + for i := 0; i < 2; i++ { + idx := i + Eventually(func() int { + b.ownerIdx[idx] = b.owners.ownerIndexOf(b.c, 2, b.nodeIDs[idx]) + return b.ownerIdx[idx] + }, buslessOwnershipTimeout, buslessOwnershipPoll).Should(BeNumerically(">=", 0), b.owners.describe) + } + Expect(b.ownerIdx[0]).ToNot(Equal(b.ownerIdx[1]), + "both tunnels landed on frontend %d, so the owner path and the relay path are not live at the same time and this topology proves nothing the one-worker specs do not", + b.ownerIdx[0]) +} + +// pinBoth labels each node and pins one model to each, through frontend 0. +// +// Through frontend 0 for BOTH, deliberately: the requests below are made at +// both replicas, so a scheduling rule written at one replica and not visible at +// the other fails here. That rule reaches the other replica over the same +// PostgreSQL fan-out every other broadcast in this deployment travels on, and +// nothing else carries it. +func (b *busless) pinBoth() { + GinkgoHelper() + pinModelToNode(b.c, b.client, 0, alphaModel, b.nodeIDs[0], alphaSlot) + pinModelToNode(b.c, b.client, 0, bravoModel, b.nodeIDs[1], bravoSlot) +} + +var _ = Describe("Two frontends and two workers, with no bus", Label("Distributed"), Label("Cluster"), func() { + + // Scenario 1. The headline, on the topology that matters. + // + // A WRONG IMPLEMENTATION does one of three things and each is caught by a + // different assertion here: it still needs a broker, and either refuses to + // boot or leaves both workers inert; it reaches workers by dialling an + // address they advertised, which no worker publishes any more; or it serves + // only the worker whose tunnel the replica taking the request happens to + // hold, which is the (N-1)/N failure that a one-worker cluster cannot see. + // + // ATTACKS THAT REDDEN IT: restoring the "requires --nats-url" validation + // reddens it at cluster.Start; never starting a worker reddens it at the + // roster wait; blocking either tunnel reddens the request that worker + // serves, which is what the last spec in this file does deliberately. + It("serves both workers from both replicas, over the owner path and the relay path at once", func() { + var balancers []*frontendBalancer + c, dsn := twoByTwo(&balancers) + fleet := awaitFleet(c, dsn) + + // THE ARTIFACT, not the environment. A spec asserting that no process + // was handed a broker URL is asserting about the harness; this asserts + // about the binary the harness executed, which cannot dial a broker + // whatever anyone hands it. + busModules, depCount, err := busSymbolsIn(localAIBinary()) + Expect(err).ToNot(HaveOccurred()) + Expect(depCount).To(BeNumerically(">", 0), + "the binary carries no module list at all, so finding no broker module in it proves nothing") + Expect(busModules).To(BeEmpty(), + "the binary links %v, so this cluster is not running without a bus by construction", busModules) + + // And neither worker was handed one either, read from the LIVE process + // rather than from what the harness meant to set. LOCALAI_REGISTER_TO + // is asserted alongside so an absent broker variable is a fact about + // the worker rather than a read that returned nothing. + for i := 0; i < 2; i++ { + environ, err := c.ProcessEnviron(cluster.ProcWorker, i) + Expect(err).ToNot(HaveOccurred()) + Expect(environ).To(ContainElement(HavePrefix("LOCALAI_REGISTER_TO=")), + "worker %d's environment could not be read, so the absence below proves nothing", i) + for _, entry := range environ { + for _, banned := range []string{"LOCALAI_NATS_URL=", "LOCALAI_NATS_JWT=", "LOCALAI_NATS_USER_SEED="} { + Expect(entry).ToNot(HavePrefix(banned), + "worker %d was handed %s, so this spec is not about a worker that has none", i, banned) + } + } + } + // The frontends ARE still handed a (dead) broker URL, which is the + // control on the reads above: a harness that had simply stopped setting + // the variable anywhere would satisfy them just as well. Nothing dials + // it, and after the module assertion above nothing could. + for i := 0; i < 2; i++ { + frontendEnviron, err := c.ProcessEnviron(cluster.ProcFrontend, i) + Expect(err).ToNot(HaveOccurred()) + Expect(frontendEnviron).To(ContainElement(HavePrefix("LOCALAI_NATS_URL=")), + "the harness stopped handing a broker URL to anything, so the workers' lack of one proves nothing") + } + + // One roster, read at both replicas, by identity and not by name. + at1 := newRosterProbe(c, fleet.client, 1) + Eventually(at1.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(And(ContainElement(c.WorkerName(0)), ContainElement(c.WorkerName(1))), at1.describe) + for i := 0; i < 2; i++ { + Expect(at1.idOf(c.WorkerName(i))).To(Equal(fleet.nodeIDs[i]), + "frontend 1 resolved a different row for %s, so the replicas are not reading one roster", c.WorkerName(i)) + } + + fleet.awaitOwners() + fleet.pinBoth() + owner0, owner1 := fleet.ownerIdx[0], fleet.ownerIdx[1] + AddReportEntry("2x2 tunnel ownership", + fmt.Sprintf("%s owned by frontend %d, %s owned by frontend %d", + c.WorkerName(0), owner0, c.WorkerName(1), owner1)) + + // THE OWNER PATH. The request goes to the replica the database says + // holds worker 0's tunnel, so it is served without a relay. + expectMockedInference(fleet.client, c.FrontendURL(owner0), alphaModel, + fmt.Sprintf("frontend %d owns %s's tunnel and must serve it directly", owner0, c.WorkerName(0))) + servedBy(c, fleet.client, owner0, alphaModel, fleet.nodeIDs[0], fleet.nodeIDs[1]) + + // THE RELAY PATH, at the same instant, on the same cluster, through the + // same replica. owner0 does not hold worker 1's tunnel, so this request + // can only be answered by relaying it to owner1. + // + // It is the FIRST request for this model, so the backend install, the + // model file staging over the http tag and the gRPC load and predict + // all cross the relay. Warming it at its owner first would leave only + // the predict on the relayed path. + Expect(fleet.owners.ownerIndexOf(c, 2, fleet.nodeIDs[1])).ToNot(Equal(owner0), + "frontend %d owns %s's tunnel after all, so the request below would not be relayed and would prove nothing", + owner0, c.WorkerName(1)) + expectMockedInference(fleet.client, c.FrontendURL(owner0), bravoModel, + fmt.Sprintf("frontend %d must relay to frontend %d, which owns %s's tunnel", + owner0, owner1, c.WorkerName(1))) + + // THIS is what makes the request above a relayed one, and it is the + // assertion rather than the pre-read. Ownership can move between the + // pre-read and the reply, and if it had moved TO owner0 the request + // would have been served directly and still come back 200. That leaves + // the owner CHANGED, and this read reddens on it. The only window left + // is a move away and back inside one request, which takes two claims, + // and no replica dies in this scenario to prompt either. + Expect(fleet.owners.ownerIndexOf(c, 2, fleet.nodeIDs[1])).To(Equal(owner1), + "%s's tunnel is no longer held by frontend %d, so the request to frontend %d was not necessarily relayed", + c.WorkerName(1), owner1, owner0) + servedBy(c, fleet.client, owner0, bravoModel, fleet.nodeIDs[1], fleet.nodeIDs[0]) + + // And the mirror image, so neither replica is the one that happens to + // work. owner1 serves its own worker directly and relays for the other. + expectMockedInference(fleet.client, c.FrontendURL(owner1), bravoModel, + fmt.Sprintf("frontend %d owns %s's tunnel and must serve it directly", owner1, c.WorkerName(1))) + expectMockedInference(fleet.client, c.FrontendURL(owner1), alphaModel, + fmt.Sprintf("frontend %d must relay to frontend %d, which owns %s's tunnel", + owner1, owner0, c.WorkerName(0))) + Expect(fleet.owners.ownerIndexOf(c, 2, fleet.nodeIDs[0])).To(Equal(owner0), + "%s's tunnel moved during the relayed request, so it was not necessarily relayed", c.WorkerName(0)) + + // The control plane crosses the same two paths. Each worker's backend + // listing is the WORKER's own answer to a control verb, relayed through + // the owner when the replica asked is not it, so a listing that comes + // back at all came back over a tunnel. The last spec in this file is + // what says so: with the tunnel gone, this same call fails naming the + // missing route. + for worker := 0; worker < 2; worker++ { + for frontend := 0; frontend < 2; frontend++ { + _, err := nodeBackendNames(c, fleet.client, frontend, fleet.nodeIDs[worker]) + Expect(err).ToNot(HaveOccurred(), + "frontend %d could not list %s's backends (that worker's tunnel is on frontend %d)", + frontend, c.WorkerName(worker), fleet.ownerIdx[worker]) + } + } + }) + + // Scenario 2. One replica dies. Its worker re-homes; the other worker never + // notices; and in between, absence is a fact NOBODY may act on. + // + // A WRONG IMPLEMENTATION reaps the fleet in the seconds before the re-home, + // which is what the last five master commits before this branch were + // fixing, or it takes the surviving worker down with the replica that died, + // or it leaves the dead replica's connection row in place so the survivor + // relays into a corpse. + // + // WHY THE LEGS ARE ORDERED THE WAY THEY ARE. An earlier scenario in this + // programme passed vacuously, with the reconnect grace set to a NANOSECOND, + // because its window sat entirely inside cluster.InstanceLiveness (30s): + // the killed replica still read as a LIVE owner throughout, so the absence + // branch was never entered. Leg 1 here asserts NOTHING and only waits for + // the killed instance to leave the live set. Only then does leg 2 begin, + // and every second of leg 2 is spent in the branch it is about. + // + // ATTACK THAT REDDENS IT: Options.ReconnectGrace set to one nanosecond must + // redden LEG 2. A green leg 2 under that attack means the window is back + // inside the liveness period and the scenario asserts nothing. + It("re-homes only the dead replica's worker, keeps the other serving, and reaps nothing inside the grace", func() { + var balancers []*frontendBalancer + c, dsn := twoByTwo(&balancers) + fleet := awaitFleet(c, dsn) + fleet.awaitOwners() + fleet.pinBoth() + + doomed, survivor := fleet.ownerIdx[0], fleet.ownerIdx[1] + Expect(doomed).ToNot(Equal(survivor)) + + // Both workers serve before anything is killed, or the recovery below + // proves nothing. + expectMockedInference(fleet.client, c.FrontendURL(doomed), alphaModel, + "alpha must serve before its owner is killed") + servedBy(c, fleet.client, doomed, alphaModel, fleet.nodeIDs[0], fleet.nodeIDs[1]) + expectMockedInference(fleet.client, c.FrontendURL(survivor), bravoModel, + "bravo must serve before the other replica is killed") + servedBy(c, fleet.client, survivor, bravoModel, fleet.nodeIDs[1], fleet.nodeIDs[0]) + + // Take worker 0's tunnel away and keep it away, then kill the replica + // holding the live session. Without the block the worker re-homes onto + // the survivor within its capped 30s backoff, which is FASTER than leg + // 1 takes: there would then be a live owner throughout leg 2, the + // absence branch would never be entered, and leg 2 would be the vacuous + // window this scenario exists not to repeat. Registration and + // heartbeats keep flowing through the same balancer, so the worker + // stays registered, approved and heartbeating with no route at all, + // which is exactly the condition the invariant is about. + // + // Only worker 0's balancer is touched. Worker 1's tunnel is on the + // survivor and nothing in this paragraph reaches it. + balancers[0].blockTunnel.Store(true) + Expect(c.KillFrontend(doomed)).To(Succeed()) + Eventually(func() bool { return c.FrontendAlive(doomed) }, frontendExitTimeout, frontendExitPoll). + Should(BeFalse(), "frontend %d did not die, so nothing below is a failover assertion", doomed) + + atSurvivor := newRosterProbe(c, fleet.client, survivor) + roster := newInstanceRoster(openClusterDB(dsn)) + + // LEG 1. No assertion about the fleet, only a wait for the killed + // replica to stop reading as live. Timed, and the number is reported, + // because "it passed" is exactly what the vacuous version also said. + leg1Started := time.Now() + Eventually(roster.addresses, deadInstanceTimeout, deadInstancePoll). + ShouldNot(ContainElement(hostPortOf(c.FrontendURL(doomed))), roster.describe) + leg1 := time.Since(leg1Started) + + // The ownership query joins against live instances, so with the killed + // replica out of the live set there is now no owner at all. This is the + // state leg 2 asserts over, and it is READ rather than inferred. + Expect(fleet.owners.ownerOf(fleet.nodeIDs[0])).To(BeEmpty(), + "a live replica still holds %s's tunnel, so leg 2 would be watching a worker that has a route", c.WorkerName(0)) + + // LEG 2. The worker is heartbeating, its tunnel is genuinely gone, and + // the departure is inside config.DefaultWorkerReconnectGrace. Nothing + // may act on that: not the health monitor, not the reaper, not the + // model evictor. The node_models row is the sharper of the two, because + // evicting it is the step that makes a later request fail for a reason + // nobody can trace. + Expect(config.DefaultWorkerReconnectGrace).To(BeNumerically(">", 60*time.Second), + "the reconnect grace is shorter than this scenario assumes, so leg 2's window may no longer sit inside it") + fleet.client.Timeout = absenceProbeTimeout + leg2Started := time.Now() + Consistently(func() []string { + return absenceViolations(c, fleet.client, atSurvivor, survivor, fleet.nodeIDs[0], 0) + }, absenceHoldWindow, absenceHoldPoll).Should(BeEmpty(), + "something acted on %s's absence while its tunnel was gone INSIDE the grace, which is a routing fact being read as a departure", + c.WorkerName(0)) + leg2 := time.Since(leg2Started) + fleet.client.Timeout = tunnelInferenceTimeout + + // The two legs together have to have fitted inside the grace, or leg 2 + // spent part of its window in the branch where acting on the departure + // is CORRECT. Without this, a slow runner turns a legitimate expiry + // into a report that the fleet was reaped early. + Expect(leg1+leg2).To(BeNumerically("<", config.DefaultWorkerReconnectGrace), + "leg 1 took %s and leg 2 took %s, which is past the %s grace, so leg 2 was not entirely inside the window it claims to assert over", + leg1, leg2, config.DefaultWorkerReconnectGrace) + + // The OTHER worker never noticed, which is the half a one-worker + // cluster cannot state at all. Its status held through the same window + // and it still serves, through the survivor, on its own tunnel. + Expect(atSurvivor.statusOf(c.WorkerName(1))).To(Equal("healthy"), + "%s was demoted because a replica it does not depend on died: %s", c.WorkerName(1), atSurvivor.describe()) + Expect(fleet.owners.ownerIndexOf(c, 2, fleet.nodeIDs[1])).To(Equal(survivor), + "%s's tunnel moved when the other replica died, so it did depend on it after all", c.WorkerName(1)) + expectMockedInference(fleet.client, c.FrontendURL(survivor), bravoModel, + fmt.Sprintf("%s must keep serving while %s has no route at all", c.WorkerName(1), c.WorkerName(0))) + servedBy(c, fleet.client, survivor, bravoModel, fleet.nodeIDs[1], fleet.nodeIDs[0]) + + // LEG 3. Give worker 0 its route back and require it to land on the + // survivor and serve again. The re-home is the assertion, not the + // inference: a frontend that answered without the tunnel moving would + // satisfy an inference-only leg while the worker stayed stranded. + balancers[0].blockTunnel.Store(false) + leg3Started := time.Now() + Eventually(func() int { return fleet.owners.ownerIndexOf(c, 2, fleet.nodeIDs[0]) }, + buslessOwnershipTimeout, buslessOwnershipPoll). + Should(Equal(survivor), fleet.owners.describe) + leg3 := time.Since(leg3Started) + + Eventually(atSurvivor.healthyNames, nodeRosterTimeout, nodeRosterPoll). + Should(ContainElement(c.WorkerName(0)), atSurvivor.describe) + Expect(atSurvivor.idOf(c.WorkerName(0))).To(Equal(fleet.nodeIDs[0]), + "%s re-registered rather than re-homing, so this says nothing about a tunnel moving", c.WorkerName(0)) + + eventuallyMockedInference(fleet.client, c.FrontendURL(survivor), alphaModel, + "the survivor must serve the re-homed worker") + servedBy(c, fleet.client, survivor, alphaModel, fleet.nodeIDs[0], fleet.nodeIDs[1]) + + AddReportEntry("2x2 replica churn, observed windows", fmt.Sprintf( + "killed frontend %d (owner of %s), survivor frontend %d (owner of %s)\n"+ + " leg 1, killed instance leaves the live set: %s (bound %s, cluster.InstanceLiveness is %s)\n"+ + " leg 2, fleet survives with no owner inside the grace: %s (window %s, grace %s)\n"+ + " leg 3, worker re-homes onto the survivor: %s", + doomed, c.WorkerName(0), survivor, c.WorkerName(1), + leg1.Round(time.Millisecond), deadInstanceTimeout, clustersvc.InstanceLiveness, + leg2.Round(time.Millisecond), absenceHoldWindow, config.DefaultWorkerReconnectGrace, + leg3.Round(time.Millisecond))) + }) + + // Scenario 3. THE NEGATIVE CONTROL FOR THIS WHOLE FILE, and the only spec + // here that can tell the two above from a pair of specs that pass with the + // tunnels doing nothing. + // + // Frontends and workers share a host, so every backend port a frontend + // names in a stream target is a port it could have dialled itself. This + // spec refuses BOTH tunnel dials at the balancers while proxying + // registration and heartbeats, so both workers are registered, approved, + // healthy and holding no tunnel, and then requires both to be unreachable. + // + // It cannot use LOCALAI_WORKER_TUNNEL=false: that is a fatal startup error + // now, and a worker that never started says nothing about a worker + // reachable by some other path. + // + // THE FAILURE MUST BE THE ROUTING FACT AND NOT A CLAIM OF ABSENCE. Four + // conditions have to stay apart and no code may report any as another: a + // routing fact, which the scheduler may act on; an absent connection inside + // the grace, which nobody may act on; an unreachable peer, which nobody may + // act on; and the worker's own answer. So the refusal is required to name + // the missing ROUTE, and the roster is required to go on reporting both + // workers healthy with fresh heartbeats while it does. + // + // ATTACK: lift the block for ONE worker. Exactly one set of legs must go + // green and the other must stay red, which is what proves these are + // separate assertions about separate tunnels rather than one assertion + // about the cluster. + It("cannot reach either worker with both tunnels refused, recovers exactly the one whose tunnel returns", func() { + var balancers []*frontendBalancer + c, dsn := twoByTwo(&balancers, func(_ int, b *frontendBalancer) { b.blockTunnel.Store(true) }) + fleet := awaitFleet(c, dsn) + fleet.pinBoth() + + models := [2]string{alphaModel, bravoModel} + + // In every respect the workers the specs above used, except the + // tunnels, and both halves of that are asserted rather than assumed: + // each dialled, and no replica holds either. + for i := 0; i < 2; i++ { + idx := i + Eventually(balancers[idx].tunnelDials.Load, "60s", "500ms").Should(BeNumerically(">", 0), + "%s never dialled its tunnel, so blocking the dial is not what makes it unreachable below", c.WorkerName(idx)) + Consistently(func() string { return fleet.owners.ownerOf(fleet.nodeIDs[idx]) }, "5s", "500ms"). + Should(BeEmpty(), "a replica holds %s's tunnel, so the blocker is not blocking", c.WorkerName(idx)) + } + + fleet.client.Timeout = buslessRefusalTimeout + refuseAt := func(frontend, worker int) { + GinkgoHelper() + refused, err := chat(fleet.client, c.FrontendURL(frontend), models[worker], "ping") + Expect(err).ToNot(HaveOccurred(), + "the request never came back; a worker with no route must be REFUSED, not left parked") + Expect(refused.status).ToNot(Equal(http.StatusOK), + "frontend %d served %s for a worker that holds no tunnel, so something other than the tunnel reaches %s: %s", + frontend, models[worker], c.WorkerName(worker), refused.body) + // One substring, not a disjunction. "no route" is what + // cluster.ErrNoRoute reads as and nothing else on this path + // produces it. It is in particular NOT what an absent worker + // produces, and that distinction is the whole invariant: a worker + // this replica cannot reach must never be reported as one that has + // gone. + Expect(refused.body).To(ContainSubstring("no route"), + "the refusal does not name the missing route, so this cannot tell a worker with no tunnel from a request that failed for one of the ordinary reasons: %s", + refused.body) + } + + // Both workers, asked at BOTH replicas: with no tunnel anywhere, + // neither the owner path nor the relay path exists, and a frontend that + // had some other way in would show up as exactly one of these four + // succeeding. + for worker := 0; worker < 2; worker++ { + for frontend := 0; frontend < 2; frontend++ { + refuseAt(frontend, worker) + } + _, listErr := nodeBackendNames(c, fleet.client, 0, fleet.nodeIDs[worker]) + Expect(listErr).To(HaveOccurred(), + "a frontend answered a backend listing for %s, which holds no tunnel, so something other than the tunnel reaches it", + c.WorkerName(worker)) + } + + // AND NOTHING WAS REAPED FOR ANY OF IT. Both workers kept + // heartbeating, so a request that could not be routed must not have + // cost either its row, its health or its heartbeat. This is the leg + // that separates "unreachable" from "absent"; without it the refusals + // above would be satisfied by a deployment that had simply declared + // both workers gone. + Consistently(fleet.probe.healthyNames, "15s", "1s"). + Should(And(ContainElement(c.WorkerName(0)), ContainElement(c.WorkerName(1))), + fleet.probe.explain("a worker was demoted or removed because requests to it could not be routed")) + for i := 0; i < 2; i++ { + Expect(fleet.probe.heartbeatOf(c.WorkerName(i))).To(BeTemporally(">", time.Now().Add(-1*time.Minute)), + "%s's heartbeat is stale, so the refusals above are about a worker that went away rather than one with no route", c.WorkerName(i)) + } + + // THE DIFFERENTIAL. Put back ONE tunnel and change nothing else. + balancers[0].blockTunnel.Store(false) + Eventually(func() int { return fleet.owners.ownerIndexOf(c, 2, fleet.nodeIDs[0]) }, + buslessOwnershipTimeout, buslessOwnershipPoll). + Should(BeNumerically(">=", 0), fleet.owners.describe) + + fleet.client.Timeout = tunnelInferenceTimeout + eventuallyMockedInference(fleet.client, c.FrontendURL(0), alphaModel, + "the only thing that changed is one tunnel, so the refusal above was that missing tunnel and nothing else") + servedBy(c, fleet.client, 0, alphaModel, fleet.nodeIDs[0], fleet.nodeIDs[1]) + + // And the other worker is STILL unreachable, for the same routing + // reason, at both replicas. This is what makes the four legs above four + // assertions about two tunnels rather than one assertion about the + // cluster: a recovery that had come from anything other than worker 0's + // own tunnel would have carried worker 1 with it. + fleet.client.Timeout = buslessRefusalTimeout + Expect(fleet.owners.ownerOf(fleet.nodeIDs[1])).To(BeEmpty(), + "%s acquired a tunnel, so the refusals below would not be about a worker that has none", c.WorkerName(1)) + refuseAt(0, 1) + refuseAt(1, 1) + + // Then the second one, so the spec ends with the whole fleet proven + // reachable and the refusals proven reversible in both directions. + balancers[1].blockTunnel.Store(false) + Eventually(func() int { return fleet.owners.ownerIndexOf(c, 2, fleet.nodeIDs[1]) }, + buslessOwnershipTimeout, buslessOwnershipPoll). + Should(BeNumerically(">=", 0), fleet.owners.describe) + + fleet.client.Timeout = tunnelInferenceTimeout + eventuallyMockedInference(fleet.client, c.FrontendURL(1), bravoModel, + "the second tunnel came back and its worker must serve again") + servedBy(c, fleet.client, 1, bravoModel, fleet.nodeIDs[1], fleet.nodeIDs[0]) + }) +}) From 276e38f5d7c5778c157d90c99b7e59ec71dda2f0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 6 Sep 2026 02:41:49 +0000 Subject: [PATCH 102/153] test(distributed): prove the fan-out carrier between two real replicas Removing the broker left one thing carrying every broadcast family in the product: PostgreSQL LISTEN/NOTIFY, in core/services/pgbus. It is covered thoroughly in process by test-e2e-distributed, and it was covered nowhere at all by real binaries: grepping the six Cluster spec files for pgbus, bus_messages, LISTEN and NOTIFY returned zero hits. Registration, model staging over the tunnel and inference through both the owner and the relay paths were already proven by real processes; the carrier that now carries everything else was not, so a deployment whose replicas each published to themselves and heard nobody would have left every suite green. Two specs, both on two frontends and no workers against one PostgreSQL, publishing at frontend 0 and reading at frontend 1. 1. A gallery operation admitted at one replica, read out of the other, with the queued state observed before the terminal one. 2. A broadcast of about 9.3 kilobytes, which PostgreSQL refuses as a notification payload, making the round trip byte for byte through the bus_messages spill table. The family is a gallery operation for one property nothing else on this carrier has: the answer a peer gives is held in memory ALONE. GET /models/jobs/ reads galleryop's statuses map, which on a peer is filled by the gallery.*.progress subscriber and by nothing else, because the only other filler, Hydrate, runs once at startup and every operation here is created long afterwards. Every other family has a durable table behind it that a peer would converge through anyway, and a spec on one of those cannot separate "the broadcast arrived" from "the row was read". That is then made checkable rather than argued. The gallery_operations row is written when the gallery worker DEQUEUES an operation, so an operation still waiting in the queue has NO row, and both specs assert zero rows while the peer is already answering with the operation's own bytes. Both also read the instances table and require the reading replica to be a different live instance from the publishing one, so "the other replica" cannot decay into a spelling of "this replica". Holding the queue is what cluster.Options.Galleries is for. The gallery worker runs one operation at a time on an unbuffered channel, so an install parked inside a gated index fetch parks everything behind it; without that the admission broadcast and the terminal one are separated by two database round trips and no HTTP poller could see between them. The option also turns the startup estimate warmer off, because a second fetcher filling the process-wide index cache would leave the operation never blocking and the spec passing on an ordering nothing enforced. The spill spec is written against a failure this branch has shipped three times: a size-limit spec that cannot fail. The oversized body is an ordinary element name that the real consumer decodes and surfaces, so it is not a body the decoder would have refused at any size. The size is ABSOLUTE at 9000 bytes rather than derived from the cap, and a one-byte control operation in the same run is required to leave no spill row, so moving the 8000-byte cap in either direction reddens the spec. pgbus.FitsInline, which shares its encoder and its comparison with Publish, is asked about both payloads and must answer differently. The spilled row is then decoded and its element name compared byte for byte against what frontend 1 answers. The terminal assertion in spec 1 does not re-check the element name: a terminal status does not carry one, because updateError in galleryop.Start builds a fresh OpStatus holding only the error. It asserts the two fields that status does carry, in the relation that one place writes them. Attacks run, each alone, each reverted, each behaving as predicted. Neutering the pg_notify in pgbus.Publish so every replica knows only what it did itself reddens both specs at frontend 1, which answers 500 for an operation it was never told about; the bus_messages row assertion still passes under it, which is right, since the row is written before the notification. Releasing the queue gate reddens spec 1 at the gallery_operations count, because the operation is dequeued and the row appears. Shrinking the oversized name to 100 bytes reddens spec 2 at FitsInline; inverting that guard so the run reaches the row check reddens it there instead, with no bus_messages row written, which is what makes the row a statement about size. test-e2e-cluster is 26 specs in 933.8 seconds of Ginkgo time, 15m37s wall. The two additions cost 7.0 seconds together, 5.0s and 2.0s: they start no workers, so they pay for no registration, and what they wait on is a broadcast rather than a threshold. test-e2e-distributed is unchanged at 223 plus 8 specs, 130.6 seconds. The budget comment and .agents/building-and-testing.md move from 24 specs at 897 to 907 seconds to 26 at 933.8. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .agents/building-and-testing.md | 3 +- Makefile | 9 +- tests/e2e/distributed/cluster/cluster.go | 25 + tests/e2e/distributed/cluster_fanout_test.go | 530 +++++++++++++++++++ 4 files changed, 563 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/distributed/cluster_fanout_test.go diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 228c02bc2098..f1f43c4efa27 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -51,7 +51,7 @@ Rules (both gates): Two suites cover distributed mode (frontend replicas, worker nodes, PostgreSQL), split by a Ginkgo label. Neither starts a message broker: a distributed deployment needs PostgreSQL and the frontends' own HTTP listener, and the `nats-io` modules are not in the build. - `make test-e2e-distributed` runs `Distributed && !VLLMMultinode && !Cluster` over `./tests/e2e/distributed` recursively. Services are wired directly into the test binary. 231 specs in about 2m20s (measured 133.3s for the main suite plus the 8 argument-validation specs in the subpackage). -- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. It **builds `./local-ai` itself** (the `e2e-binary` target), so you do not run `make build` first and cannot forget to. 24 specs, 897 to 907s of specs in a 15m target run (three measurements; the suite grew from 6 specs and roughly 509s, then 21 specs and 802.5s, so budget on the newest number). +- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. It **builds `./local-ai` itself** (the `e2e-binary` target), so you do not run `make build` first and cannot forget to. 26 specs, 933.8s of specs in a 15m37s target run (four measurements; the suite grew from 6 specs and roughly 509s, then 21 specs and 802.5s, then 24 specs and 897 to 907s, so budget on the newest number). Both jobs live in `.github/workflows/tests-e2e-distributed.yml`, with `timeout-minutes: 45` each. They trigger on pull requests *and* on every push to `master`; the `paths-ignore` filter (see [.agents/ci-caching.md](ci-caching.md)) sits on the pull-request trigger only, so a master push always runs both. They are advisory only because `master` carries no branch protection, which is a repository setting and not a YAML key: `continue-on-error: true` would flip the run's *conclusion* to success and hide the failure, so it is not used. @@ -66,4 +66,5 @@ Both jobs live in `.github/workflows/tests-e2e-distributed.yml`, with `timeout-m - **Coverage:** `tests/e2e/distributed` is excluded from the coverage roots (`COVERAGE_E2E_ROOTS = ./tests/e2e`, run non-recursively), and so is the `cluster` helper package beneath it. Neither suite moves the baseline, so production code that these suites are the only cover for reads as **uncovered**. Unit tests for such code belong under `./core/...` with `testutil.SetupTestDB()`. - **The cluster job builds against a stubbed React UI.** `core/http/react-ui/dist` is gitignored and built by Node, so the workflow writes a one-line `index.html` there to satisfy the `//go:embed react-ui/dist/*` in `core/http/app.go` and skips a full Node and Vite install. That holds only while the suite drives the HTTP API and never the UI, which has its own e2e suite. A spec that ever asserts on a UI asset would pass locally, where a real `dist/` exists, and be served the stub in CI: if you write one, the stub step has to go and the real build come back. - **The topology that matters is two frontends and two workers.** `cluster_busless_test.go` runs the whole programme on it, and it is the only shape in which the OWNER path and the RELAY path are live at the same time: with the two workers' tunnels landing on different replicas, every request either goes to the replica holding that worker's tunnel or is relayed through the one that does. A one-worker cluster cannot tell those apart, and cannot state that killing a replica re-homes only ITS worker. Which worker served a request is read back from `node_models` rather than assumed, and the two models are pinned to one worker each through the admin API (`PUT /api/nodes/:id/labels` plus `POST /api/nodes/scheduling`), so a selector that failed open reddens the spec instead of hiding in it. +- **The fan-out carrier is proven under real binaries by `cluster_fanout_test.go`.** Both specs run two frontends and no workers against one PostgreSQL, publish at frontend 0 and read at frontend 1, and cost 7.0s together. They drive a gallery operation on purpose: `GET /models/jobs/` answers from `galleryop`'s in-memory statuses map, which on a peer replica is filled by the `gallery.*.progress` subscriber and by nothing else (`Hydrate` runs once at startup, long before these operations exist), so a peer that answers was told over the carrier. Every other broadcast family has a durable table behind it that a peer would converge through anyway, and a spec on one of those cannot separate "the broadcast arrived" from "the row was read". The specs make that checkable rather than argued: the `gallery_operations` row is written when the gallery worker DEQUEUES an operation, so an operation still queued has none, and both specs assert zero rows while the peer is already answering with the operation's own bytes. Holding the queue is what `cluster.Options.Galleries` is for: it points every frontend at one gated index and turns the startup estimate warmer off, so the operation the spec submits is parked inside a fetch only the spec can complete. The second spec adds the oversized path: an element name of 9000 bytes puts the encoded notification past PostgreSQL's 8000-byte cap (exclusive), so the broadcast becomes a `bus_messages` row plus a notification carrying its id, and the spec asserts the row exists, decodes it, and compares its element name byte for byte with what frontend 1 answers. A one-byte control operation in the same run is asserted to leave NO row, which is what keeps "a row exists" a statement about size rather than one that is true of every operation. - **Do not shorten the cluster suite's waits.** Three of its six specs sit at ~167s each because they wait out a 60s staleness threshold plus a 15s health-check tick. That wait is what stops the assertions from passing before the system could have reacted, which was a real false green earlier on. If the job has to get faster, the levers are CI concurrency or making the thresholds configurable, not shorter waits. diff --git a/Makefile b/Makefile index d5be94cfa003..98fb01f259be 100644 --- a/Makefile +++ b/Makefile @@ -424,14 +424,17 @@ e2e-binary: protogen-go # should stay there: this suite exists to catch nondeterministic cluster # behaviour, and a retry turns exactly that signal into a green run. # -# Budget: 24 specs, measured at 897 to 907 seconds of Ginkgo time (15 minutes -# wall including the compile) on a fast developer box. It was 591 to 612 seconds +# Budget: 26 specs, measured at 933.8 seconds of Ginkgo time (15m37s wall +# including the compile) on a fast developer box. It was 591 to 612 seconds # before the phase 3 control-plane specs and 800 to 830 after them; the three # two-frontend two-worker specs in cluster_busless_test.go added 118 to 127 # seconds (3s, 70s and 46 to 53s), nearly all of it in the churn spec, which # cannot be shortened: it waits for a killed replica to leave the live set # (cluster.InstanceLiveness is 30s, measured at 27s) before it may assert -# anything, and then holds a window inside the reconnect grace. +# anything, and then holds a window inside the reconnect grace. The two +# cross-replica fan-out specs added 7.0 seconds of spec time (5.0s and 2.0s): +# they run two frontends and no workers, so they pay for no registration, and +# what they wait on is a broadcast rather than a threshold. # # --timeout is 30m rather than 20m because of that. The margin is not slack: a # Ginkgo timeout kills the suite mid-spec and reports a spec name rather than a diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index d0045b458474..0cd2bf097088 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -133,6 +133,24 @@ type Options struct { // that never started proves nothing about a worker reachable some other // way. WorkerFrontendURL func(worker int, registrar string, frontends []string) string + + // Galleries is the model-gallery list every frontend runs with, in the JSON + // form --galleries takes. Empty leaves the binary's own default list, which + // is what every spec that does not name this option keeps. + // + // It exists for the fan-out specs, which need the thing a gallery operation + // blocks on to be something the spec can hold open. A model install's first + // act is to fetch its gallery's index, and the gallery worker runs one + // operation at a time, so an index served by a gate the spec controls is + // what turns "the queued broadcast went out before the terminal one" from a + // timing coincidence into an arrangement. + // + // Setting it also turns the startup estimate warmer OFF, and that is not + // tidiness. The warmer fetches the same index in the background at boot, so + // with it on the process-wide index cache can be filled by something other + // than the operation the spec is holding, and the operation then never + // blocks at all. The spec would still pass, on an ordering nothing enforced. + Galleries string } // Process is one running local-ai. @@ -329,6 +347,13 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { if c.opts.ReconnectGrace > 0 { cmd.Env = append(cmd.Env, "LOCALAI_WORKER_RECONNECT_GRACE="+c.opts.ReconnectGrace.String()) } + if c.opts.Galleries != "" { + // Both, together, or neither: see Options.Galleries for why the warmer + // has to be off wherever the gallery index is a spec's gate. + cmd.Env = append(cmd.Env, + "LOCALAI_GALLERIES="+c.opts.Galleries, + "LOCALAI_AUTOLOAD_GALLERIES=false") + } p, err := c.spawn(name, cmd, port) if err != nil { diff --git a/tests/e2e/distributed/cluster_fanout_test.go b/tests/e2e/distributed/cluster_fanout_test.go new file mode 100644 index 000000000000..539c28bcd7a1 --- /dev/null +++ b/tests/e2e/distributed/cluster_fanout_test.go @@ -0,0 +1,530 @@ +package distributed_test + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/mudler/LocalAI/core/services/galleryop" + "github.com/mudler/LocalAI/core/services/messaging" + "github.com/mudler/LocalAI/core/services/pgbus" + "github.com/mudler/LocalAI/tests/e2e/distributed/cluster" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +// The fan-out carrier, under the binaries an operator runs. +// +// Removing the broker left ONE thing carrying every broadcast family in the +// product: PostgreSQL LISTEN/NOTIFY, in core/services/pgbus. That package and +// its callers are covered thoroughly in process, against a real database, by +// test-e2e-distributed. What was not covered anywhere was the same carrier +// running inside two separate `local-ai` processes: nothing in this Cluster +// partition mentioned pgbus, bus_messages, LISTEN or NOTIFY, so a deployment +// whose replicas each published to themselves and heard nobody would have left +// every suite green. +// +// The family both specs drive is a gallery operation, and it is chosen for one +// property nothing else on this carrier has: the answer a peer replica gives is +// held in memory ALONE. GET /models/jobs/ reads galleryop's statuses map; +// that map is filled on a peer by the SubjectGalleryProgressWildcard subscriber +// and by nothing else, because the only other filler, Hydrate, runs once at +// startup and every operation here is created long after both replicas booted. +// Every other family this carrier serves has a durable table behind it that a +// peer would converge through eventually, and a spec on one of those cannot say +// whether the broadcast arrived or the row was read. +// +// The specs then take that further and make it checkable rather than asserted: +// the gallery_operations row for an operation is written when the gallery +// WORKER dequeues it, so an operation still waiting in the queue has NO row at +// all. Both specs hold the queue and assert exactly that. A peer replica +// answering with the operation's own bytes while PostgreSQL holds no record of +// the operation is a peer that was told, and it can have been told only one way. +// +// What holds the queue is the gallery index fetch. The gallery worker runs one +// operation at a time on an unbuffered channel (see galleryop.EnqueueModelOp, +// which exists because of that), so an install whose index fetch is parked +// inside a server the spec controls parks every operation submitted behind it. +// That is the difference between proving an ordering and observing one: without +// the gate the admission broadcast and the terminal broadcast are separated by +// two database round trips, which no HTTP poller can reliably see between. + +const ( + // fanoutTimeout bounds the wait for a broadcast published at one replica to + // become readable at another. Generous against what it covers, which is one + // NOTIFY, one primary-key SELECT for a spilled body, and one HTTP poll. + fanoutTimeout = "60s" + fanoutPoll = "250ms" + + // fanoutHoldWindow is how long a spec requires a held operation to stay + // unfinished at the replica that did not start it. + // + // It is not a tolerance. The gallery worker is parked inside an HTTP read + // that only this spec can complete, so a terminal status appearing in this + // window is a status published ahead of the work it reports on, which is + // the ordering the assertion is about. + fanoutHoldWindow = "3s" + fanoutHoldPoll = "250ms" + + // fanoutGalleryName names the one gallery the fan-out frontends run with. + fanoutGalleryName = "e2e-fanout" + + // fanoutBlockerElement is the operation whose index fetch parks the gallery + // worker. Nothing is ever installed under this name: the gate stops the + // fetch long before any name is looked up in the index. + fanoutBlockerElement = "fanout-queue-blocker" + + // oversizedElementBytes is how long an element name has to be for the + // broadcast carrying it to exceed PostgreSQL's notification cap. + // + // An ABSOLUTE size, chosen against the cap rather than derived from it, and + // that is deliberate. pgbus.maxNotifyPayloadBytes is 8000 and exclusive, + // and the encoded notification carries the subject, the envelope keys and + // the rest of the status alongside the name, so 9000 bytes of name clears + // it by roughly 1300. A name sized as "the cap plus a bit" would move with + // the cap, and a spec that moves with the thing it is measuring cannot + // fail: shrink the constant and both sides shrink together. This one, with + // the one-byte control below, brackets the cap from both directions, so + // moving the cap in EITHER direction reddens the spill spec. + oversizedElementBytes = 9000 + + // tinyElement is the control: an element name whose whole broadcast is a + // few hundred bytes, which no plausible cap would spill. It is what makes + // "a bus_messages row exists" a statement about size rather than a + // statement that is true of every operation. + tinyElement = "f" + + // fanoutNameExcerpt is how much of an element name a failure message + // prints. A nine-kilobyte name rendered whole makes a Ginkgo report + // unreadable, which costs more than it tells. + fanoutNameExcerpt = 48 +) + +// fanoutGalleryIndex is what the gate serves once a spec lets it go. Its +// contents do not matter and deliberately name nothing either spec installs: +// what the gate is for is the WAIT, and every operation here is expected to end +// in a failure once released. +const fanoutGalleryIndex = "- name: fanout-decoy\n description: e2e fan-out probe gallery\n" + +// modelGalleriesJSON is the --galleries value that points every frontend at one +// gated index. +func modelGalleriesJSON(g *gatedGallery) string { + return fmt.Sprintf(`[{"name":%q,"url":%q}]`, fanoutGalleryName, g.URL()) +} + +// withModelGalleries pins the model gallery list every frontend runs with. +func withModelGalleries(galleries string) func(*cluster.Options) { + return func(o *cluster.Options) { o.Galleries = galleries } +} + +// modelOpStatus is the subset of galleryop.OpStatus these specs assert on, as +// GET /models/jobs/ serializes it. +type modelOpStatus struct { + Processed bool `json:"processed"` + Phase string `json:"phase"` + Message string `json:"message"` + GalleryElementName string `json:"gallery_element_name"` + Error string `json:"error"` +} + +// summarise renders a status for a failure message with long fields abbreviated. +func (s modelOpStatus) summarise() string { + return fmt.Sprintf("{processed:%t phase:%q message:%s element:%s error:%q}", + s.Processed, s.Phase, abbreviateName(s.Message), abbreviateName(s.GalleryElementName), s.Error) +} + +// abbreviateName quotes a field, keeping a report readable when it is long. +func abbreviateName(s string) string { + if len(s) <= fanoutNameExcerpt { + return fmt.Sprintf("%q", s) + } + return fmt.Sprintf("%q...(%d bytes total)", s[:fanoutNameExcerpt], len(s)) +} + +// modelOpProbe polls one gallery operation at one frontend and keeps what it +// last saw, so a failing Eventually can name it. +// +// Every accessor returns a zero value rather than raising on a failed read: a +// peer replica answers 500 for an operation it has not been told about yet, and +// that is the state these specs are waiting out, not a defect. +type modelOpProbe struct { + cluster *cluster.Cluster + client *http.Client + frontend int + opID string + + lastErr error + last modelOpStatus +} + +func newModelOpProbe(c *cluster.Cluster, client *http.Client, frontend int, opID string) *modelOpProbe { + return &modelOpProbe{cluster: c, client: client, frontend: frontend, opID: opID} +} + +func (p *modelOpProbe) read() modelOpStatus { + var status modelOpStatus + if err := p.cluster.GetJSON(p.client, p.frontend, "/models/jobs/"+p.opID, &status); err != nil { + p.lastErr = err + return modelOpStatus{} + } + p.lastErr = nil + p.last = status + return status +} + +// elementDigest is the SHA-256 of the element name this frontend reports, and +// it is what the waiting assertions compare. +// +// A digest rather than the name itself, for the failure message alone: an +// Eventually that times out on a nine-kilobyte string prints it twice, and the +// exact-bytes assertion each spec makes afterwards is the one that states the +// claim. Comparing digests is comparing bytes; comparing them HERE keeps the +// report legible when the wait is what failed. +func (p *modelOpProbe) elementDigest() string { return digestOf(p.read().GalleryElementName) } + +func (p *modelOpProbe) processed() bool { return p.read().Processed } + +func (p *modelOpProbe) describe() string { + if p.lastErr != nil { + return fmt.Sprintf("frontend %d: the last read of gallery operation %s failed: %v", + p.frontend, p.opID, p.lastErr) + } + return fmt.Sprintf("frontend %d: gallery operation %s last read as %s", + p.frontend, p.opID, p.last.summarise()) +} + +// explain builds a lazy failure description, for the reason jobProbe.explain +// gives: Gomega renders a (string, args...) description when the assertion is +// CONSTRUCTED, which for a wait is before anything has gone wrong. +func (p *modelOpProbe) explain(format string, args ...any) func() string { + return func() string { + return fmt.Sprintf(format, args...) + ": " + p.describe() + } +} + +func digestOf(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +// applyModel posts a model install at one frontend and returns the operation id +// it was handed. +// +// The endpoint answers 200 with an id the instant the operation is ADMITTED, +// before the gallery worker has seen it, which is exactly the window both specs +// live in. +func applyModel(c *cluster.Cluster, client *http.Client, frontend int, elementName string) string { + GinkgoHelper() + var accepted struct { + UUID string `json:"uuid"` + } + status, err := c.PostJSON(client, frontend, "/models/apply", + map[string]any{"id": elementName}, &accepted) + Expect(err).ToNot(HaveOccurred()) + Expect(status).To(Equal(http.StatusOK), + "frontend %d refused the install outright, so no operation was ever admitted", frontend) + Expect(accepted.UUID).ToNot(BeEmpty()) + return accepted.UUID +} + +// queuedBroadcast rebuilds the admission broadcast galleryop.markQueued +// publishes for an operation. +// +// It exists so the spill spec can ask the carrier's OWN size predicate about +// the exact payload the product built, rather than about a payload of the +// spec's invention. pgbus.FitsInline shares its encoder and its comparison with +// Publish, so an answer from it is the answer Publish gave. +func queuedBroadcast(opID, elementName string) galleryop.GalleryProgressEvent { + return galleryop.GalleryProgressEvent{ + JobID: opID, + Status: &galleryop.OpStatus{ + Message: galleryop.PhaseQueued, + Phase: galleryop.PhaseQueued, + GalleryElementName: elementName, + Cancellable: true, + }, + } +} + +// spilledBroadcasts returns the rows the carrier wrote for one subject, oldest +// first. A broadcast that fit in its notification leaves none. +func spilledBroadcasts(db *gorm.DB, subject string) ([]pgbus.BusMessage, error) { + var rows []pgbus.BusMessage + if err := db.Where("subject = ?", subject).Order("created_at").Find(&rows).Error; err != nil { + return nil, err + } + return rows, nil +} + +// spillCount is spilledBroadcasts reduced to a number, for a wait. +func spillCount(db *gorm.DB, subject string) int { + rows, err := spilledBroadcasts(db, subject) + if err != nil { + return -1 + } + return len(rows) +} + +// galleryOperationRows counts what PostgreSQL records about one operation. +// +// Zero is the interesting answer: the row is written when the gallery worker +// DEQUEUES an operation, so an operation still waiting in the queue has none, +// and a replica answering about it is answering from something other than the +// database. +func galleryOperationRows(db *gorm.DB, opID string) int64 { + GinkgoHelper() + var count int64 + Expect(db.Table("gallery_operations").Where("id = ?", opID).Count(&count).Error).To(Succeed()) + return count +} + +// fanoutFiller builds an element name of exactly n bytes. +// +// Ordinary printable characters, cycling, so the value is a name the product +// would accept anywhere it accepts a short one. That matters more than it +// looks: the point of the spill spec is that an OVERSIZED message survives, and +// a body the decoder would have rejected at any size proves nothing about size. +func fanoutFiller(n int) string { + const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789" + var b strings.Builder + b.Grow(n) + for i := 0; i < n; i++ { + b.WriteByte(alphabet[i%len(alphabet)]) + } + return b.String() +} + +// startFanoutCluster brings up two frontends sharing one database, with their +// model gallery served by a gate this spec holds, and parks the gallery worker +// of the STARTER replica inside that gate. +// +// It returns everything the caller asserts against, positioned on the state +// both specs need: one operation running and stopped inside its index fetch, so +// anything admitted afterwards stays queued for as long as the spec wants. +func startFanoutCluster(starter int) (*cluster.Cluster, *gorm.DB, *http.Client, *gatedGallery) { + GinkgoHelper() + gate := newGatedGallery(fanoutGalleryIndex, true) + c, dsn := startClusterOnFreshDB(2, 0, withModelGalleries(modelGalleriesJSON(gate))) + db := openClusterDB(dsn) + client := controlSession(c) + + blocker := applyModel(c, client, starter, fanoutBlockerElement) + Eventually(gate.fetches.Load, fanoutTimeout, fanoutPoll).Should(BeNumerically(">", 0), + "frontend %d never fetched the gated gallery index for operation %s, so nothing is holding its "+ + "gallery worker and every ordering assertion below would be timing rather than arrangement", + starter, blocker) + return c, db, client, gate +} + +// requireDistinctReplicas fails unless the two frontends are separate live +// replicas of one deployment. +// +// This is the assertion that keeps "the other replica" from becoming a spelling +// of "this replica". Reading a broadcast back at the process that published it +// is the wrong implementation: a carrier that only ever hears itself passes +// every such reading and delivers nothing to anybody, and that is precisely the +// deployment nothing in this suite could previously tell apart from a working +// one. Two different ports would be a weaker version of this; the instances +// table is the deployment's own answer to "how many replicas are there", and it +// gives each a distinct id. +func requireDistinctReplicas(c *cluster.Cluster, db *gorm.DB, starter, reader int) { + GinkgoHelper() + Expect(c.FrontendURL(starter)).ToNot(Equal(c.FrontendURL(reader))) + + roster := newInstanceRoster(db) + awaitReplicas(roster, hostPortOf(c.FrontendURL(0)), hostPortOf(c.FrontendURL(1))) + + starterID := roster.idAt(hostPortOf(c.FrontendURL(starter))) + readerID := roster.idAt(hostPortOf(c.FrontendURL(reader))) + Expect(starterID).ToNot(BeEmpty(), roster.describe) + Expect(readerID).ToNot(BeEmpty(), roster.describe) + Expect(readerID).ToNot(Equal(starterID), + "frontends %d and %d are the same replica, so reading one and publishing at the other proves nothing: %s", + starter, reader, roster.describe()) +} + +var _ = Describe("Cross-replica fan-out", Label("Distributed"), Label("Cluster"), func() { + // starter admits every operation; reader is never sent one and only ever + // reads. Written down here rather than at each call so the one thing that + // makes these specs mean anything is impossible to lose in an edit. + const ( + starter = 0 + reader = 1 + ) + + // Scenario 1. A broadcast published inside one process, read out of + // another, with an ordering the spec enforces rather than hopes for. + // + // A wrong implementation is not exotic. A carrier wired so each replica + // hears only its own publishes satisfies every in-process spec in this + // repository, because those run both ends in one process on purpose. What + // it cannot do is answer here: the reader was never sent the request, and + // while the operation is queued PostgreSQL holds nothing about it either. + It("carries a gallery operation to the replica that did not start it, queued state before terminal", func() { + c, db, client, gate := startFanoutCluster(starter) + requireDistinctReplicas(c, db, starter, reader) + + // A name nothing else in this deployment uses, so an element name read + // back at the reader names this operation and no other. + const probeElement = "fanout-ordered-probe" + probeID := applyModel(c, client, starter, probeElement) + + // Nothing was installed and nothing was written: the gallery worker is + // parked, so this operation is one goroutine blocked on an unbuffered + // send, with an admission broadcast already published and no row of its + // own anywhere. + Expect(galleryOperationRows(db, probeID)).To(BeZero(), + "PostgreSQL already records operation %s, so a peer could read it from the database and this spec "+ + "would no longer be about the broadcast", probeID) + + reading := newModelOpProbe(c, client, reader, probeID) + Eventually(reading.elementDigest, fanoutTimeout, fanoutPoll). + Should(Equal(digestOf(probeElement)), reading.describe) + + // What arrived is the ADMISSION status and not some later one. Phase is + // the product's own marker for "the worker has not started this yet" + // (galleryop.PhaseQueued), so a reader holding it is a reader that was + // told before the work rather than after it. + Expect(reading.last.GalleryElementName).To(Equal(probeElement)) + Expect(reading.last.Phase).To(Equal(galleryop.PhaseQueued), reading.describe) + Expect(reading.last.Processed).To(BeFalse(), reading.describe) + + // Still nothing in PostgreSQL, read AFTER the peer answered rather than + // before. Read only beforehand it would say nothing: the row could have + // appeared in between and been what the peer read. + Expect(galleryOperationRows(db, probeID)).To(BeZero(), + "PostgreSQL now records operation %s, so the reading above could have come from the database", probeID) + + // Held, not sampled. The worker is stopped inside an HTTP read only + // this spec can complete, so a terminal status inside this window is + // one published ahead of the work it reports on. + Consistently(reading.processed, fanoutHoldWindow, fanoutHoldPoll).Should(BeFalse(), + reading.explain("the reader saw the operation finish while its gallery worker was still blocked fetching the index")) + + gate.release() + + // And the LATER broadcast crosses too. Both ends of the ordering are + // now facts at the reader: queued while the gate was shut, terminal + // only after it opened. + Eventually(reading.processed, fanoutTimeout, fanoutPoll).Should(BeTrue(), reading.describe) + + // What the terminal broadcast CARRIES, and not merely that one arrived. + // The gallery index the gate serves names no such model, so the + // operation ends in the failure galleryop.Start builds, whose two + // fields are written together and in one place: the message is the + // error with a fixed prefix. A reader holding both in that relation is + // a reader that decoded a payload rather than a flag. + // + // The element name is deliberately NOT re-asserted here. A terminal + // status does not carry one (see updateError in galleryop.Start, which + // builds a fresh OpStatus with only the error on it), so requiring one + // would be asserting against the product rather than about it. + Expect(reading.last.Error).ToNot(BeEmpty(), reading.describe) + Expect(reading.last.Message).To(Equal("error: "+reading.last.Error), + reading.explain("the terminal broadcast arrived with a message that is not the error it reports")) + }) + + // Scenario 2. The spill, end to end. + // + // PostgreSQL refuses a notification payload of 8000 bytes or more, so + // anything larger is written to bus_messages and the notification carries + // an id. That path is the ORDINARY one for several families here (a gallery + // progress event carries an entry per node; a job result carries a whole + // model output), so it is not an edge case, and until now no spec ran it + // between two processes at all. + // + // The trap this spec is written against is specific and has been fallen + // into three times on this branch: a size-limit spec that cannot fail, + // because the oversized body was malformed and the decoder refused it + // either way, or because the body was sized against the cap so shrinking + // the cap moved both sides. Here the body is an ordinary element name that + // the real consumer decodes and surfaces, the size is absolute, and a + // second operation a few hundred bytes long is asserted NOT to spill in the + // same run. Move the cap in either direction and one of the two goes red. + It("carries a broadcast too large for a notification between two replicas, byte for byte, through the spill table", func() { + c, db, client, gate := startFanoutCluster(starter) + requireDistinctReplicas(c, db, starter, reader) + + oversized := fanoutFiller(oversizedElementBytes) + oversizedID := applyModel(c, client, starter, oversized) + tinyID := applyModel(c, client, starter, tinyElement) + + oversizedSubject := messaging.SubjectGalleryProgress(oversizedID) + tinySubject := messaging.SubjectGalleryProgress(tinyID) + + // The size decision itself, asked of the predicate Publish shares its + // encoder and its comparison with, about the payload the product built. + // The pair is the point: one answer either way from one function means + // the cap is being measured rather than assumed. + oversizedFits, err := pgbus.FitsInline(oversizedSubject, queuedBroadcast(oversizedID, oversized)) + Expect(err).ToNot(HaveOccurred()) + Expect(oversizedFits).To(BeFalse(), + "a %d byte element name still fits in a notification, so this spec would never reach the spill path", + len(oversized)) + + tinyFits, err := pgbus.FitsInline(tinySubject, queuedBroadcast(tinyID, tinyElement)) + Expect(err).ToNot(HaveOccurred()) + Expect(tinyFits).To(BeTrue(), + "a %d byte element name no longer fits inline either, so spilling has stopped being a statement about size", + len(tinyElement)) + + // The wire. One row, written by the publishing replica because its + // notification would have been refused. + Eventually(func() int { return spillCount(db, oversizedSubject) }, fanoutTimeout, fanoutPoll). + Should(Equal(1), "no bus_messages row was written for %s, so the oversized broadcast did not take the spill path", oversizedSubject) + + // And the control: the small operation crossed the same carrier in the + // same run and left nothing behind. Without this, "a row exists" would + // be satisfied by an implementation that spilled everything, which is a + // different product and would hide the cap entirely. + Expect(spillCount(db, tinySubject)).To(BeZero(), + "a bus_messages row was written for the %d byte broadcast on %s too, so the row above says nothing about size", + len(tinyElement), tinySubject) + + rows, err := spilledBroadcasts(db, oversizedSubject) + Expect(err).ToNot(HaveOccurred()) + Expect(rows).To(HaveLen(1)) + + var wire galleryop.GalleryProgressEvent + Expect(json.Unmarshal(rows[0].Payload, &wire)).To(Succeed(), + "the spilled row is not a decodable broadcast, so nothing downstream could have read it") + Expect(wire.JobID).To(Equal(oversizedID)) + Expect(wire.Status).ToNot(BeNil()) + Expect(wire.Status.GalleryElementName).To(HaveLen(oversizedElementBytes)) + Expect(wire.Status.GalleryElementName).To(Equal(oversized), + "the bytes in the spill row are not the bytes that were published") + + // The other end. The reader was never sent this request and, while the + // operation is queued, PostgreSQL records nothing about it, so the only + // account of it there has ever been outside the starter's memory is the + // row above plus the notification naming it. + reading := newModelOpProbe(c, client, reader, oversizedID) + Eventually(reading.elementDigest, fanoutTimeout, fanoutPoll). + Should(Equal(digestOf(oversized)), reading.describe) + Expect(reading.last.GalleryElementName).To(HaveLen(oversizedElementBytes)) + Expect(reading.last.GalleryElementName).To(Equal(oversized), + "the reader reconstructed a different %d byte element name from the spilled row", oversizedElementBytes) + Expect(reading.last.Phase).To(Equal(galleryop.PhaseQueued), reading.describe) + + Expect(galleryOperationRows(db, oversizedID)).To(BeZero(), + "PostgreSQL records operation %s, so the reader could have read the name from gallery_operations "+ + "rather than from the broadcast", oversizedID) + + // The small operation reaches the reader too. A carrier that delivered + // only what it spilled would pass everything above. + tinyReading := newModelOpProbe(c, client, reader, tinyID) + Eventually(tinyReading.elementDigest, fanoutTimeout, fanoutPoll). + Should(Equal(digestOf(tinyElement)), tinyReading.describe) + Expect(tinyReading.last.GalleryElementName).To(Equal(tinyElement)) + + // Released here rather than left to cleanup, so the operations retire + // while the cluster is still up and their logs are still being written. + gate.release() + Eventually(reading.processed, fanoutTimeout, fanoutPoll).Should(BeTrue(), reading.describe) + }) +}) From fbfbb85546e73a2dd09aea4e0adad0baccc3dfcf Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 6 Sep 2026 04:05:56 +0000 Subject: [PATCH 103/153] feat(cluster): make a peer prove which replica it is GET /api/cluster/peer authenticated with the deployment's shared registration token and took the dialling replica's id from ?id= on trust. Every worker holds that token, so anything holding it could open a peer link as any replica: relay through it to every worker tunnel that replica owns, displace a real replica's inbound link by declaring its id, and point the roughly 31 GiB per-session receive window at one replica. Validating the id against the instances table does not fix this, because the attack declares a real replica's id. So the route now checks two credentials and needs both. The shared token still says the dialler belongs to this deployment; a new per-replica credential says which replica it is. The credential follows the per-node worker credential rather than inventing a second mechanism: crypto/rand.Text, stored only as a hex SHA-256, compared in constant time, with no fallback to the shared token. It differs in the stronger direction. A worker's credential is minted by the frontend and handed over once; a replica writes its own instances row, so it mints its own secret, publishes only the hash in the same statement that publishes its address, and never sends the plaintext anywhere but the peer dial. A peer that presents no credential is refused, not waved through. An old replica and an attacker holding the shared token send the same request, so accepting the first accepts the second; there is no safe downgrade here, only a quiet one. The refusal is made loud instead, on both sides, naming the upgrade rather than the network. On the documented frontend-first order a new replica still dials an old one; an old replica cannot dial a new one, which costs relayed requests that land on a not-yet-restarted replica and surfaces as no route, never as absence. A rejected peer gets its own sentinel, ErrPeerRejected, whose unwrap chain carries ErrPeerUnreachable as well and no absence sentinel at all. Keeping the older sentinel means no existing consumer changes behaviour; the cause stays out of the chain, so absence cannot escape through it and nothing can read an authorization failure as a worker that went away. One consequence beyond the fix: a replica with no advertised address has no instances row, so it now cannot dial out either. It was already unreachable inward. The startup error and the docs say so. Registry.Register, NewMembership, NewPeerPool, PeerHandler and RegisterClusterRoutes all gained required arguments, so the identity cannot be dropped without a compile failure. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/dispatch_loop_wiring_test.go | 2 +- core/application/distributed.go | 28 +- core/http/app.go | 5 +- core/http/endpoints/cluster/connect_test.go | 2 +- core/http/endpoints/cluster/peer.go | 119 +++++-- core/http/endpoints/cluster/peer_test.go | 197 +++++++++-- core/http/routes/cluster.go | 10 +- core/services/cluster/connected_among_test.go | 2 +- core/services/cluster/dialer_test.go | 20 +- core/services/cluster/instance.go | 26 +- core/services/cluster/instance_test.go | 10 +- core/services/cluster/membership.go | 37 +- core/services/cluster/membership_test.go | 46 +-- core/services/cluster/ownership_test.go | 22 +- core/services/cluster/peeridentity.go | 96 ++++++ core/services/cluster/peeridentity_test.go | 326 ++++++++++++++++++ core/services/cluster/peerlink.go | 105 +++++- core/services/cluster/peerlink_test.go | 34 +- core/services/cluster/presence_test.go | 2 +- core/services/cluster/relay_internal_test.go | 2 +- core/services/cluster/relay_test.go | 4 +- core/services/cluster/tunnel_test.go | 10 +- core/services/cluster/wsconn_test.go | 12 +- core/services/jobs/claim_test.go | 8 +- core/services/jobs/dispatch_loop_test.go | 6 +- core/services/nodes/agent_control_test.go | 4 +- core/services/nodes/health_test.go | 2 +- core/services/nodes/router_liveness_test.go | 4 +- docs/content/features/distributed-mode.md | 40 ++- .../e2e/distributed/agent_distributed_test.go | 2 +- .../e2e/distributed/cluster_peerlink_test.go | 149 +++++++- tests/e2e/distributed/job_dispatch_test.go | 4 +- .../e2e/distributed/job_distribution_test.go | 4 +- 33 files changed, 1162 insertions(+), 178 deletions(-) create mode 100644 core/services/cluster/peeridentity.go create mode 100644 core/services/cluster/peeridentity_test.go diff --git a/core/application/dispatch_loop_wiring_test.go b/core/application/dispatch_loop_wiring_test.go index f152fcd8b130..1e609eaf0bda 100644 --- a/core/application/dispatch_loop_wiring_test.go +++ b/core/application/dispatch_loop_wiring_test.go @@ -100,7 +100,7 @@ var _ = Describe("building the job dispatch loop", func() { Expect(jobs.MigrateClaims(ctx, db)).To(Succeed()) // Registered, because a replica that is not in the instances table // refuses to claim: its claims could not be told from a dead one's. - Expect(cluster.NewRegistry(db).Register(ctx, "replica-7", "127.0.0.1:8080", "v1")).To(Succeed()) + Expect(cluster.NewRegistry(db).Register(ctx, "replica-7", "127.0.0.1:8080", "v1", "")).To(Succeed()) _, err := jobs.EnqueueClaim(ctx, db, jobs.ClaimKindAgentRun, json.RawMessage(`{}`)) Expect(err).ToNot(HaveOccurred()) diff --git a/core/application/distributed.go b/core/application/distributed.go index 7dc3823c9455..85911ece19f4 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -266,6 +266,18 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // Replica membership. NewNodeRegistry has just migrated the tables this // reads, so it has to come after it. clusterRegistry := cluster.NewRegistry(authDB) + // This replica's proof of which replica it is, minted ONCE per process and + // handed to both halves of the peer mesh: the membership loop publishes its + // hash in the instances row, and the peer pool presents its plaintext on + // every outbound dial. Two mints would leave a replica whose published hash + // and presented secret disagree, which every peer refuses and which reads + // from the logs like a peer running an older release. + // + // The plaintext never leaves this process except in a peer dial's header. + // There is nothing to configure and nothing to rotate: a restart mints a + // new one, and the same registration that republishes this replica's + // address republishes the hash beside it. + peerCredential := cluster.NewPeerCredential() var membership *cluster.Membership if advertised, err := advertisedPeerAddr(cfg); err != nil { // Not fatal, and the cost is worth stating exactly rather than as @@ -292,10 +304,16 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // obvious reading is "the worker is broken". So this is an ERROR, not a // warning, and nagUnadvertisedReplica below repeats it for as long as // the state lasts, naming the workers it is currently costing. - xlog.Error("This replica is not registered in the cluster: no advertised address. Peers cannot reach it, and any worker whose tunnel lands here will be unroutable from every other replica", + // + // It now costs the OTHER direction too, which is why the line says so. + // A peer link is authenticated by the dialling replica's own credential, + // published in the instances row this replica never writes, so this + // replica cannot dial a peer either: its own relayed requests are + // refused as an unproven identity rather than merely arriving nowhere. + xlog.Error("This replica is not registered in the cluster: no advertised address. Peers cannot reach it, any worker whose tunnel lands here will be unroutable from every other replica, and this replica cannot relay OUT either, because a peer link is authenticated by the credential an instance row publishes and this replica has no row", "error", err, "knob", "LOCALAI_DISTRIBUTED_ADVERTISE_ADDR") } else { - membership = cluster.NewMembership(clusterRegistry, cfg.Distributed.InstanceID, advertised, internal.PrintableVersion()) + membership = cluster.NewMembership(clusterRegistry, cfg.Distributed.InstanceID, advertised, internal.PrintableVersion(), peerCredential) // Before Start, so the first sweep already purges on the retention this // deployment's grace requires rather than on the floor. membership.SetReconnectGrace(cfg.Distributed.ReconnectGraceOrDefault()) @@ -339,7 +357,11 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // authenticates with the registration token because that is the token the // peer route checks (see RegisterClusterRoutes); two different tokens here // would make every peer dial 401 with nothing naming the mismatch. - peers := cluster.NewPeerPool(cfg.Distributed.InstanceID, cfg.Distributed.RegistrationToken, clusterRegistry) + // + // And with this replica's own credential, which is the half that says WHICH + // replica is dialling. It is the same value membership published the hash + // of, by construction: there is one mint above and both call sites read it. + peers := cluster.NewPeerPool(cfg.Distributed.InstanceID, cfg.Distributed.RegistrationToken, peerCredential, clusterRegistry) // The one door to every worker. Nothing in the frontend may dial a worker's // advertised address any more: a worker holds ONE tunnel, it lands on ONE // replica, and this resolves which replica that is and relays through it diff --git a/core/http/app.go b/core/http/app.go index 15db18427fb8..c7bc083c3742 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -579,7 +579,10 @@ func API(application *application.Application) (*echo.Echo, error) { xlog.Warn("Replica peer link will refuse every dial: no registration token is configured", "route", clustersvc.PeerPath, "knob", "LOCALAI_REGISTRATION_TOKEN") } - routes.RegisterClusterRoutes(e, distCfg.RegistrationToken, d.PeerSessions.Accept) + // d.Cluster is what the handler resolves a dialling replica's id + // against, so the route can check WHICH replica is on the far end and + // not merely that it holds the deployment's shared token. + routes.RegisterClusterRoutes(e, distCfg.RegistrationToken, d.Cluster, d.PeerSessions.Accept) } // The worker tunnel, registered unconditionally. Both arguments are nil diff --git a/core/http/endpoints/cluster/connect_test.go b/core/http/endpoints/cluster/connect_test.go index 15206d50444c..ebe5013acac7 100644 --- a/core/http/endpoints/cluster/connect_test.go +++ b/core/http/endpoints/cluster/connect_test.go @@ -84,7 +84,7 @@ var _ = Describe("Worker tunnel handler", func() { Expect(nodeID).ToNot(BeEmpty()) reg = clustersvc.NewRegistry(db) - Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1", "")).To(Succeed()) tun = clustersvc.NewTunnelRegistry(reg, "me") e := echo.New() diff --git a/core/http/endpoints/cluster/peer.go b/core/http/endpoints/cluster/peer.go index 9dcaea8b38de..ee92da17e6af 100644 --- a/core/http/endpoints/cluster/peer.go +++ b/core/http/endpoints/cluster/peer.go @@ -8,6 +8,7 @@ package cluster import ( "crypto/subtle" + "errors" "net/http" "strings" @@ -21,10 +22,17 @@ import ( // PeerHandler upgrades an authenticated peer dial to a WebSocket, wraps it as // a yamux server session and hands it to onSession. // +// TWO credentials are checked, and both must pass. token is the deployment's +// shared cluster token and says the dialler belongs here at all; instances +// resolves the replica id in ?id= to its row and checks the dialler's OWN peer +// credential against the hash that row publishes. Neither replaces the other: +// the shared check is unchanged, and identity is added in front of the +// multiplexer it guards. +// // onSession runs on the request goroutine, so it must return promptly; the // session outlives the handler because the upgrade hijacks the connection, and // closing it is the caller's job. -func PeerHandler(token string, onSession func(peerID string, sess *yamux.Session)) echo.HandlerFunc { +func PeerHandler(token string, instances *clustersvc.Registry, onSession func(peerID string, sess *yamux.Session)) echo.HandlerFunc { // gorilla's default CheckOrigin already restricts a browser to same-origin // and lets a header-less client (which every peer is) through, so the // zero value is what this link wants. @@ -38,39 +46,94 @@ func PeerHandler(token string, onSession func(peerID string, sess *yamux.Session return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") } - // SELF-DECLARED, and knowingly so. Unlike the worker route next door, - // which resolves ?id= to a node row and checks that node's OWN minted - // credential, this route has only the shared cluster token to check, - // so the id is a label and not a claim anything verifies. - // - // What that costs, exactly, for anything already holding the shared - // token (every worker holds it, and it is the same token that - // authenticates registration): it can relay to every worker tunnel this - // replica owns, reaching every backend gRPC process and every worker's - // file-transfer server; by declaring a legitimate replica's id it can - // make SessionStore.Accept evict that replica's inbound link, at will; - // and it can aim the per-session receive window, which PeerLinkConfig - // sizes at roughly 31 GiB of unread data per session, at one replica's - // memory. The first two are not new capabilities in KIND - before - // workers stopped listening, a holder of that token could already dial - // any worker's advertised ports directly - but the token is now the - // only thing between an attacker and the whole fleet's tunnels, and the - // third is a figure written down as sizing guidance that is also a - // budget on a route this open. + // Not 401. A frontend with no cluster registry cannot resolve any + // replica's identity, and answering "unauthorized" would send an operator + // hunting a token problem that does not exist. Checked after the shared + // token so an anonymous dial still gets the 401 first, which is the same + // ordering ConnectHandler takes next door and for the same reason. // - // It is deferred rather than patched, because the cheap patch does not - // work: checking ?id= against the instances table stops an invented id - // and stops nothing else, since the attack declares a REAL replica's - // id, and it would buy a false sense of a closed hole. Closing it takes - // a credential per replica, minted where a replica joins the instances - // table and presented here, which is a design with its own migration - // and its own specs. Tracked as the phase-3 item named at - // nodes.BackendNode.TunnelTokenHash. + // This route is registered only in distributed mode, where the registry is + // never nil, so the branch is defence against a future wiring that + // registers it more widely. Failing closed is the point: a handler that + // treated a nil registry as "nothing to check" would publish exactly the + // unauthenticated multiplexer this argument was added to prevent. + if instances == nil { + return echo.NewHTTPError(http.StatusServiceUnavailable, "distributed mode not enabled") + } + + // Read before the credential so a dial that names nobody is still a 400. + // The id is no longer taken on trust: everything below turns it from a + // label into a claim, by resolving it to a row and requiring the dialler to + // present that row's secret. peerID := c.QueryParam("id") if peerID == "" { return echo.NewHTTPError(http.StatusBadRequest, "missing peer id") } + // The identity half, read before any database work so a dial carrying no + // credential costs no query. + // + // A missing header is refused, not waved through, and the choice is the + // whole migration story of this route. Accepting a credential-less dial + // "for compatibility" would leave the hole exactly as open as it was, + // because an attacker simply omits the header too; there is no version of + // a downgrade here that is safe, only versions that are quiet. So the + // failure is loud instead: this is the one line that tells an operator + // mid-rollout why an old replica cannot reach a new one, and it names the + // upgrade rather than the network. + presented := c.Request().Header.Get(clustersvc.PeerIdentityHeader) + if presented == "" { + xlog.Warn("Refusing a peer link: the dialling replica presented no peer credential. It is running a release from before per-replica peer identity, or it never registered a credential of its own. Upgrade it; this replica will not accept an unproven peer id", + "peer", peerID, "header", clustersvc.PeerIdentityHeader) + return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") + } + + inst, err := instances.Get(c.Request().Context(), peerID) + switch { + case errors.Is(err, clustersvc.ErrInstanceNotFound): + // A replica id this deployment has no row for. Reported as 401 rather + // than 404 so a caller cannot enumerate replica ids by status code, + // which is the same choice the worker tunnel makes for node ids. + xlog.Warn("Refusing a peer link: the dial named a replica this deployment has no row for", + "peer", peerID) + return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") + case err != nil: + // A query that FAILED is neither a rejection nor an absence, and this + // is that rule in its HTTP form. Answering 401 on an unreadable + // database would tell a healthy replica its credentials are wrong, and + // nothing above the transport may conclude anything about a WORKER + // from it either: a peer that cannot be linked to is not a worker that + // has gone away. + xlog.Error("Looking up a replica for its peer dial failed", "peer", peerID, "error", err) + return echo.NewHTTPError(http.StatusInternalServerError, "peer lookup failed") + } + + // Split from the mismatch below because they are different operator + // problems with different fixes, exactly as the worker tunnel splits them. + // An empty stored hash means that replica last registered against a + // LocalAI that predates peer credentials, so there is no secret to check + // and it must register again, which a restart does. A mismatch means the + // dialler is presenting the wrong one. + // + // Empty is refused. It is not "no restriction": an empty credential that + // matched would mean every replica registered by an older frontend could + // be impersonated by anyone holding the shared token, which is the whole + // exposure being closed. + if inst.PeerTokenHash == "" { + xlog.Warn("Refusing a peer link: the replica it claims to be has no peer credential published, so nothing here can verify the claim. That replica has not registered since per-replica peer identity was introduced", + "peer", peerID) + return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") + } + if !clustersvc.PeerTokenMatches(presented, inst.PeerTokenHash) { + // This is the impostor case, and it is a WARN rather than a debug + // line: holding the shared token and declaring somebody else's id is + // precisely the attack this check exists for, and it should not be + // invisible at default log level. + xlog.Warn("Refusing a peer link: the dial presented the wrong credential for the replica it claims to be", + "peer", peerID) + return echo.NewHTTPError(http.StatusUnauthorized, "unauthorized") + } + ws, err := upgrader.Upgrade(c.Response(), c.Request(), nil) if err != nil { // Upgrade has already written its own failure to the client. diff --git a/core/http/endpoints/cluster/peer_test.go b/core/http/endpoints/cluster/peer_test.go index dc8cba3da9e6..c7ddface1b93 100644 --- a/core/http/endpoints/cluster/peer_test.go +++ b/core/http/endpoints/cluster/peer_test.go @@ -1,6 +1,7 @@ package cluster_test import ( + "context" "net/http" "net/http/httptest" "strings" @@ -9,6 +10,7 @@ import ( "github.com/mudler/LocalAI/core/http/auth" "github.com/mudler/LocalAI/core/http/routes" clustersvc "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" "github.com/gorilla/websocket" "github.com/labstack/echo/v4" @@ -22,16 +24,40 @@ func wsPeerURL(s *httptest.Server) string { return "ws" + strings.TrimPrefix(s.URL, "http") + clustersvc.PeerPath + "?id=peer-1" } +// peerHeaders builds what a legitimate replica dials with: the deployment's +// shared cluster token AND its own peer credential. Both are always sent, +// because the handler checks both and a spec that sent one would be pinning +// half the door. +func peerHeaders(token string, cred clustersvc.PeerCredential) http.Header { + h := http.Header{} + h.Set("Authorization", "Bearer "+token) + h.Set(clustersvc.PeerIdentityHeader, cred.Token()) + return h +} + var _ = Describe("Peer link handler", func() { var ( srv *httptest.Server sessions chan *yamux.Session + reg *clustersvc.Registry + cred clustersvc.PeerCredential + ctx context.Context ) BeforeEach(func() { + ctx = context.Background() sessions = make(chan *yamux.Session, 1) + + db := testutil.SetupTestDB() + Expect(clustersvc.Migrate(ctx, db)).To(Succeed()) + reg = clustersvc.NewRegistry(db) + // peer-1 is the replica every dial below claims to be, and this is the + // credential that claim is checked against. + cred = clustersvc.NewPeerCredential() + Expect(reg.Register(ctx, "peer-1", "10.0.0.1:8080", "v1", cred.Hash())).To(Succeed()) + e := echo.New() - routes.RegisterClusterRoutes(e, "peer-token", func(_ string, s *yamux.Session) { + routes.RegisterClusterRoutes(e, "peer-token", reg, func(_ string, s *yamux.Session) { sessions <- s }) srv = httptest.NewServer(e) @@ -54,9 +80,7 @@ var _ = Describe("Peer link handler", func() { }) It("accepts an authenticated peer and yields a usable yamux session", func() { - h := http.Header{} - h.Set("Authorization", "Bearer peer-token") - conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(srv), h) + conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(srv), peerHeaders("peer-token", cred)) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = conn.Close() }) @@ -95,16 +119,14 @@ var _ = Describe("Peer link handler", func() { Eventually(accepted, "10s").Should(Receive(Equal([]byte("hello")))) }) - It("reports the peer id it was given", func() { - h := http.Header{} - h.Set("Authorization", "Bearer peer-token") + It("reports the peer id it proved", func() { ids := make(chan string, 1) e := echo.New() - routes.RegisterClusterRoutes(e, "peer-token", func(id string, _ *yamux.Session) { ids <- id }) + routes.RegisterClusterRoutes(e, "peer-token", reg, func(id string, _ *yamux.Session) { ids <- id }) s2 := httptest.NewServer(e) DeferCleanup(s2.Close) - conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(s2), h) + conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(s2), peerHeaders("peer-token", cred)) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = conn.Close() }) @@ -117,11 +139,11 @@ var _ = Describe("Peer link handler", func() { // unauthenticated yamux multiplexer to anyone who can reach the port. e := echo.New() accepted := make(chan *yamux.Session, 1) - routes.RegisterClusterRoutes(e, "", func(_ string, sess *yamux.Session) { accepted <- sess }) + routes.RegisterClusterRoutes(e, "", reg, func(_ string, sess *yamux.Session) { accepted <- sess }) s2 := httptest.NewServer(e) DeferCleanup(s2.Close) - for _, header := range []http.Header{nil, {"Authorization": []string{"Bearer "}}, {"Authorization": []string{"Bearer anything"}}} { + for _, header := range []http.Header{nil, {"Authorization": []string{"Bearer "}}, {"Authorization": []string{"Bearer anything"}}, peerHeaders("", cred), peerHeaders("anything", cred)} { _, resp, err := websocket.DefaultDialer.Dial(wsPeerURL(s2), header) Expect(err).To(HaveOccurred()) Expect(resp).ToNot(BeNil()) @@ -132,7 +154,7 @@ var _ = Describe("Peer link handler", func() { It("accepts the bearer scheme in any case", func() { // RFC 7235 makes the scheme case-insensitive. The token after it is not. - h := http.Header{} + h := peerHeaders("peer-token", cred) h.Set("Authorization", "bearer peer-token") conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(srv), h) Expect(err).ToNot(HaveOccurred()) @@ -145,7 +167,7 @@ var _ = Describe("Peer link handler", func() { // without the handler's own recover the peer would keep a link nobody // ever accepts streams on. e := echo.New() - routes.RegisterClusterRoutes(e, "peer-token", func(_ string, _ *yamux.Session) { + routes.RegisterClusterRoutes(e, "peer-token", reg, func(_ string, _ *yamux.Session) { panic("callback exploded") }) s2 := httptest.NewServer(e) @@ -157,9 +179,7 @@ var _ = Describe("Peer link handler", func() { s2.Close() }) - h := http.Header{} - h.Set("Authorization", "Bearer peer-token") - conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(s2), h) + conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(s2), peerHeaders("peer-token", cred)) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = conn.Close() }) @@ -177,10 +197,8 @@ var _ = Describe("Peer link handler", func() { It("rejects an authenticated dial that names no peer", func() { // The session is keyed by peer id, so a nameless link could never be // looked up again; refusing it is cheaper than leaking it. - h := http.Header{} - h.Set("Authorization", "Bearer peer-token") _, resp, err := websocket.DefaultDialer.Dial( - "ws"+strings.TrimPrefix(srv.URL, "http")+"/api/cluster/peer", h) + "ws"+strings.TrimPrefix(srv.URL, "http")+"/api/cluster/peer", peerHeaders("peer-token", cred)) Expect(err).To(HaveOccurred()) Expect(resp).ToNot(BeNil()) Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) @@ -202,17 +220,25 @@ var _ = Describe("Peer link auth coverage", func() { var ( srv *httptest.Server sessions chan *yamux.Session + cred clustersvc.PeerCredential ) BeforeEach(func() { + ctx := context.Background() sessions = make(chan *yamux.Session, 1) + db := testutil.SetupTestDB() + Expect(clustersvc.Migrate(ctx, db)).To(Succeed()) + reg := clustersvc.NewRegistry(db) + cred = clustersvc.NewPeerCredential() + Expect(reg.Register(ctx, "peer-1", "10.0.0.1:8080", "v1", cred.Hash())).To(Succeed()) + e := echo.New() // A nil DB with one legacy API key is the cheapest configuration that // turns the middleware ON without a database. With neither, Middleware // short-circuits to next() and every assertion below would pass against // a server that has no auth at all. e.Use(auth.Middleware(nil, &config.ApplicationConfig{ApiKeys: []string{"an-api-key"}})) - routes.RegisterClusterRoutes(e, "peer-token", func(_ string, s *yamux.Session) { sessions <- s }) + routes.RegisterClusterRoutes(e, "peer-token", reg, func(_ string, s *yamux.Session) { sessions <- s }) // A route outside the cluster prefix, registered on the same server, is // the control: it proves the middleware in front of both is live. e.GET("/api/nodes", func(c echo.Context) error { return c.NoContent(http.StatusOK) }) @@ -246,11 +272,136 @@ var _ = Describe("Peer link auth coverage", func() { It("completes a full peer handshake through the guarded server", func() { // The status-code assertion above cannot see the upgrade, and the // upgrade is what a peer actually does. - h := http.Header{} - h.Set("Authorization", "Bearer peer-token") - conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(srv), h) + conn, _, err := websocket.DefaultDialer.Dial(wsPeerURL(srv), peerHeaders("peer-token", cred)) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = conn.Close() }) Eventually(sessions, "5s").Should(Receive()) }) }) + +var _ = Describe("Peer link identity", func() { + // The status codes a peer's own retry loop reads, and the order they are + // decided in. Every refusal here happens BEFORE the WebSocket upgrade: a + // handler that upgraded first would answer with a WebSocket error instead + // of a status, and neither an operator nor a dialler could tell an + // authorization failure from a transport one. + var ( + srv *httptest.Server + sessions chan *yamux.Session + reg *clustersvc.Registry + cred clustersvc.PeerCredential + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + sessions = make(chan *yamux.Session, 1) + db := testutil.SetupTestDB() + Expect(clustersvc.Migrate(ctx, db)).To(Succeed()) + reg = clustersvc.NewRegistry(db) + cred = clustersvc.NewPeerCredential() + Expect(reg.Register(ctx, "peer-1", "10.0.0.1:8080", "v1", cred.Hash())).To(Succeed()) + + e := echo.New() + routes.RegisterClusterRoutes(e, "peer-token", reg, func(_ string, s *yamux.Session) { sessions <- s }) + srv = httptest.NewServer(e) + DeferCleanup(srv.Close) + }) + + // dialPeer returns the handshake status a dial was answered with. + dialPeer := func(target *httptest.Server, h http.Header) int { + GinkgoHelper() + conn, resp, err := websocket.DefaultDialer.Dial(wsPeerURL(target), h) + if err == nil { + DeferCleanup(func() { _ = conn.Close() }) + Expect(resp).ToNot(BeNil()) + return resp.StatusCode + } + Expect(resp).ToNot(BeNil()) + Expect(resp.Header.Get("Upgrade")).To(BeEmpty(), + "the handler upgraded a dial it then refused; a peer reads the status, not a WebSocket error") + return resp.StatusCode + } + + It("refuses a dial that carries the shared token and no peer credential", func() { + // This is the mixed-version case, and the answer is deliberately a + // refusal rather than a fallback. A replica running a release from + // before peer credentials sends no such header, and so does an attacker + // holding the shared token: there is no way to accept the first without + // accepting the second, so both are refused and the handler logs which + // replica it refused and why. + h := http.Header{} + h.Set("Authorization", "Bearer peer-token") + Expect(dialPeer(srv, h)).To(Equal(http.StatusUnauthorized)) + Expect(sessions).ToNot(Receive()) + }) + + It("refuses a dial that presents an empty peer credential", func() { + h := peerHeaders("peer-token", clustersvc.PeerCredential{}) + Expect(dialPeer(srv, h)).To(Equal(http.StatusUnauthorized)) + Expect(sessions).ToNot(Receive()) + }) + + It("refuses a dial that declares a real replica with the wrong credential", func() { + Expect(dialPeer(srv, peerHeaders("peer-token", clustersvc.NewPeerCredential()))). + To(Equal(http.StatusUnauthorized)) + Expect(sessions).ToNot(Receive()) + }) + + It("refuses a dial whose declared replica publishes no credential", func() { + Expect(reg.Register(ctx, "peer-1", "10.0.0.1:8080", "v1", "")).To(Succeed()) + Expect(dialPeer(srv, peerHeaders("peer-token", cred))).To(Equal(http.StatusUnauthorized)) + Expect(sessions).ToNot(Receive()) + }) + + It("accepts a dial that proves the replica it declares", func() { + Expect(dialPeer(srv, peerHeaders("peer-token", cred))).To(Equal(http.StatusSwitchingProtocols)) + Eventually(sessions, "5s").Should(Receive()) + }) + + It("answers 503, not 401, when there is no cluster registry to check against", func() { + // A frontend with nothing to resolve replica ids against cannot + // authenticate anybody. Answering "unauthorized" would send an operator + // hunting a token problem that does not exist, and accepting would + // publish an unauthenticated multiplexer, so it does neither. + e := echo.New() + accepted := make(chan *yamux.Session, 1) + routes.RegisterClusterRoutes(e, "peer-token", nil, func(_ string, s *yamux.Session) { accepted <- s }) + s2 := httptest.NewServer(e) + DeferCleanup(s2.Close) + + Expect(dialPeer(s2, peerHeaders("peer-token", cred))).To(Equal(http.StatusServiceUnavailable)) + Expect(accepted).ToNot(Receive()) + }) + + It("answers 500, not 401, when the replica lookup itself fails", func() { + // A query that FAILED is neither a rejection nor an absence. Telling a + // healthy replica its credentials are wrong because the database was + // unreadable sends it re-registering; nothing above may read it as a + // worker having gone away either. + db := testutil.SetupTestDB() + Expect(clustersvc.Migrate(ctx, db)).To(Succeed()) + broken := clustersvc.NewRegistry(db) + Expect(broken.Register(ctx, "peer-1", "10.0.0.1:8080", "v1", cred.Hash())).To(Succeed()) + sqlDB, err := db.DB() + Expect(err).ToNot(HaveOccurred()) + Expect(sqlDB.Close()).To(Succeed()) + + e := echo.New() + accepted := make(chan *yamux.Session, 1) + routes.RegisterClusterRoutes(e, "peer-token", broken, func(_ string, s *yamux.Session) { accepted <- s }) + s2 := httptest.NewServer(e) + DeferCleanup(s2.Close) + + Expect(dialPeer(s2, peerHeaders("peer-token", cred))).To(Equal(http.StatusInternalServerError)) + Expect(accepted).ToNot(Receive()) + }) + + It("refuses an anonymous dial before it looks anything up", func() { + // Ordering: the shared token is still the first gate, unchanged, so a + // dial with no Authorization at all is a 401 and not a 400 about an id + // or a 401 about a credential. The route-coverage test issues exactly + // this shape. + Expect(dialPeer(srv, nil)).To(Equal(http.StatusUnauthorized)) + }) +}) diff --git a/core/http/routes/cluster.go b/core/http/routes/cluster.go index 458122d036e1..d569bbfc9196 100644 --- a/core/http/routes/cluster.go +++ b/core/http/routes/cluster.go @@ -13,6 +13,12 @@ import ( // receives every authenticated session; see clusterep.PeerHandler for what it // is expected to do with it. // +// instances is what turns the ?id= on that route from a self-declared label +// into a claim: the handler resolves the id to a replica row and checks the +// dialler's own peer credential against the hash it publishes. It is a required +// argument rather than an option, so a deployment cannot register this route +// with nothing to verify against; a nil one makes the handler answer 503. +// // The path is core/services/cluster's own constant, so the handler and the // dialler cannot be registered and dialled at different paths. That the path // also falls under auth.ClusterPathPrefix, and so bypasses the global session @@ -21,8 +27,8 @@ import ( // // The route carries no auth middleware: it authenticates itself against the // cluster token, because a peer replica has no session and no user. -func RegisterClusterRoutes(e *echo.Echo, token string, onPeer func(string, *yamux.Session)) { - e.GET(clustersvc.PeerPath, clusterep.PeerHandler(token, onPeer)) +func RegisterClusterRoutes(e *echo.Echo, token string, instances *clustersvc.Registry, onPeer func(string, *yamux.Session)) { + e.GET(clustersvc.PeerPath, clusterep.PeerHandler(token, instances, onPeer)) } // RegisterWorkerTunnelRoute registers the endpoint a worker dials to open its diff --git a/core/services/cluster/connected_among_test.go b/core/services/cluster/connected_among_test.go index 77916d16a70d..dfed0a63e51b 100644 --- a/core/services/cluster/connected_among_test.go +++ b/core/services/cluster/connected_among_test.go @@ -35,7 +35,7 @@ var _ = Describe("ConnectedAmong", func() { // live registers a replica that is heartbeating now. live := func(id string) { GinkgoHelper() - Expect(reg.Register(ctx, id, "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, id, "10.0.0.1:8080", "v1", "")).To(Succeed()) } // kill stops a replica heartbeating, WITHOUT stamping a departure on the diff --git a/core/services/cluster/dialer_test.go b/core/services/cluster/dialer_test.go index 8edd28e5215e..e2c02a55159d 100644 --- a/core/services/cluster/dialer_test.go +++ b/core/services/cluster/dialer_test.go @@ -175,7 +175,7 @@ var _ = Describe("The worker dialer", func() { db = testutil.SetupTestDB() Expect(cluster.Migrate(ctx, db)).To(Succeed()) reg = cluster.NewRegistry(db) - Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1", "")).To(Succeed()) mine = cluster.NewTunnelRegistry(reg, "me") }) @@ -184,7 +184,7 @@ var _ = Describe("The worker dialer", func() { // the worker's own half of the tunnel. ownerRelay := func(nodeID string) (*stubPeers, *yamux.Session) { GinkgoHelper() - Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1", "")).To(Succeed()) ownerTunnels := cluster.NewTunnelRegistry(reg, "owner") frontend, worker := workerTunnel() _, err := ownerTunnels.Attach(ctx, nodeID, frontend) @@ -432,7 +432,7 @@ var _ = Describe("The worker dialer", func() { // client will wait, and this replica is the only one that holds it. // This spec plays the owner by hand so it can read the frame rather // than infer it from a timing. - Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "owner") Expect(err).ToNot(HaveOccurred()) dialling, ownerSide := yamuxPair() @@ -469,7 +469,7 @@ var _ = Describe("The worker dialer", func() { // Zero on the wire would be read by the owner as a caller with // nothing left, and it would refuse traffic that is perfectly // healthy. "Not stated" has to stay distinguishable from "expired". - Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "owner") Expect(err).ToNot(HaveOccurred()) dialling, ownerSide := yamuxPair() @@ -500,11 +500,11 @@ var _ = Describe("The worker dialer", func() { // The catastrophe this phase exists to prevent. A scheduler ACTS on // absence: told a connected worker is gone, it reclaims every model // the worker is running. - Expect(reg.Register(ctx, "owner", "127.0.0.1:1", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "owner", "127.0.0.1:1", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "owner") Expect(err).ToNot(HaveOccurred()) - pool := cluster.NewPeerPool("me", "tok", reg) + pool := cluster.NewPeerPool("me", "tok", cluster.NewPeerCredential(), reg) DeferCleanup(pool.Close) d := cluster.NewWorkerDialer(mine, pool) @@ -518,7 +518,7 @@ var _ = Describe("The worker dialer", func() { // The owner's table row survives a tunnel that has gone. The relay // answers ErrNotOwner, and only that answer tells this replica to // resolve the owner again rather than give up on the worker. - Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "owner") Expect(err).ToNot(HaveOccurred()) ownerTunnels := cluster.NewTunnelRegistry(reg, "owner") @@ -549,7 +549,7 @@ var _ = Describe("The worker dialer", func() { }) It("reports having no way to relay as its own condition", func() { - Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "owner") Expect(err).ToNot(HaveOccurred()) @@ -588,7 +588,7 @@ var _ = Describe("The worker dialer", func() { // is not. The join is what stops this replica dialling a process // that is gone, which is why the spec asserts no peer was dialled // as well as what came back. - Expect(reg.Register(ctx, "ghost", "10.0.0.9:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "ghost", "10.0.0.9:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "ghost") Expect(err).ToNot(HaveOccurred()) Expect(db.Exec( @@ -613,7 +613,7 @@ var _ = Describe("The worker dialer", func() { // dial comes back as ErrInstanceNotFound. That is absence of a // REPLICA, and a consumer matching absence would read it as absence // of the WORKER. - Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "owner", "10.0.0.2:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "owner") Expect(err).ToNot(HaveOccurred()) diff --git a/core/services/cluster/instance.go b/core/services/cluster/instance.go index 2b1e8e4b4163..22ececdaf23f 100644 --- a/core/services/cluster/instance.go +++ b/core/services/cluster/instance.go @@ -31,6 +31,21 @@ type Instance struct { AdvertisedAddr string `gorm:"size:255" json:"advertised_addr"` // host:port other replicas dial Version string `gorm:"size:64" json:"version"` LastSeen time.Time `gorm:"index" json:"last_seen"` + // PeerTokenHash is the SHA-256 of the credential this replica presents when + // it dials GET /api/cluster/peer, and is what turns the ?id= on that route + // from a self-declared label into a claim something checks. The replica + // mints the secret itself, publishes only this hash, and never sends the + // plaintext anywhere but the peer dial itself; see PeerCredential. + // + // Empty means the row was written by a release that predates per-replica + // peer identity. Such a replica is refused, loudly, rather than waved + // through: the column cannot be back-filled, because only the process that + // minted the secret has it, and treating "no credential" as "any + // credential" is the exposure this column exists to close. + // + // json:"-" because this is a stored secret's hash. Instance is serialised + // nowhere today, and the tag is what keeps that true if it ever is. + PeerTokenHash string `gorm:"size:64" json:"-"` } // Registry reads and writes the instances table. @@ -49,7 +64,14 @@ func NewRegistry(db *gorm.DB) *Registry { // Register records this replica's address, refreshing LastSeen. It upserts on // the primary key rather than deleting and re-inserting, so a concurrent Live // never observes a live replica as missing. -func (r *Registry) Register(ctx context.Context, id, addr, version string) error { +// +// peerTokenHash is the hash half of this replica's PeerCredential, written in +// the SAME statement as the address rather than by a follow-up update. A +// registration is what publishes a replica to its peers, and a row that exists +// with an address but without an identity is a replica every peer refuses; a +// process that died between two statements would leave one behind until its +// next registration, which for a healthy replica is never. +func (r *Registry) Register(ctx context.Context, id, addr, version, peerTokenHash string) error { // last_seen is stamped by the database, never by this process. Liveness is // compared across replicas, so it has to be measured on the one clock they // all share; with per-replica clocks the effective Live window becomes @@ -60,12 +82,14 @@ func (r *Registry) Register(ctx context.Context, id, addr, version string) error DoUpdates: clause.Assignments(map[string]any{ "advertised_addr": addr, "version": version, + "peer_token_hash": peerTokenHash, "last_seen": gorm.Expr("now()"), }), }).Create(map[string]any{ "id": id, "advertised_addr": addr, "version": version, + "peer_token_hash": peerTokenHash, "last_seen": gorm.Expr("now()"), }).Error; err != nil { return fmt.Errorf("registering instance %q: %w", id, err) diff --git a/core/services/cluster/instance_test.go b/core/services/cluster/instance_test.go index aa0349496161..b1f6d8a5fa63 100644 --- a/core/services/cluster/instance_test.go +++ b/core/services/cluster/instance_test.go @@ -28,7 +28,7 @@ var _ = Describe("Instance registry", func() { }) It("registers an instance and reads it back", func() { - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) got, err := reg.Get(ctx, "inst-a") Expect(err).ToNot(HaveOccurred()) @@ -37,8 +37,8 @@ var _ = Describe("Instance registry", func() { }) It("re-registering the same id updates the address instead of duplicating", func() { - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) - Expect(reg.Register(ctx, "inst-a", "10.0.0.9:9090", "v2")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.9:9090", "v2", "")).To(Succeed()) live, err := reg.Live(ctx, time.Hour) Expect(err).ToNot(HaveOccurred()) @@ -52,7 +52,7 @@ var _ = Describe("Instance registry", func() { }) It("excludes instances whose heartbeat has aged out", func() { - Expect(reg.Register(ctx, "stale", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "stale", "10.0.0.1:8080", "v1", "")).To(Succeed()) // Age the row directly; sleeping in a spec is forbidden. Expect(db.Model(&cluster.Instance{}).Where("id = ?", "stale"). Update("last_seen", time.Now().Add(-10*time.Minute)).Error).To(Succeed()) @@ -63,7 +63,7 @@ var _ = Describe("Instance registry", func() { }) It("brings a stale instance back with a heartbeat", func() { - Expect(reg.Register(ctx, "revive", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "revive", "10.0.0.1:8080", "v1", "")).To(Succeed()) Expect(db.Model(&cluster.Instance{}).Where("id = ?", "revive"). Update("last_seen", time.Now().Add(-10*time.Minute)).Error).To(Succeed()) Expect(reg.Heartbeat(ctx, "revive")).To(Succeed()) diff --git a/core/services/cluster/membership.go b/core/services/cluster/membership.go index 2dad1de6ae60..3e57e8735076 100644 --- a/core/services/cluster/membership.go +++ b/core/services/cluster/membership.go @@ -92,6 +92,10 @@ type Membership struct { id string addr string version string + // peerTokenHash is the published half of this replica's peer credential. + // Every registration writes it, so the row that tells peers where to dial + // this replica is the same row that tells them how to recognise it. + peerTokenHash string interval time.Duration liveness time.Duration @@ -115,17 +119,26 @@ type Membership struct { // NewMembership returns the membership loop for one replica. The address is // what peers will dial, so it must be reachable from another host, not the // address this process binds. -func NewMembership(reg *Registry, id, addr, version string) *Membership { +// +// cred is this replica's peer credential, and only its hash is used here: the +// loop publishes it, and the peer pool presents the matching plaintext. It is +// taken as a required argument rather than a setter because a replica that +// registered without one is a replica every peer refuses, and a refusal on this +// route is indistinguishable from an older release. The whole value is passed, +// not the hash, so a caller cannot hand the pool one credential and the loop +// another. +func NewMembership(reg *Registry, id, addr, version string, cred PeerCredential) *Membership { return &Membership{ - reg: reg, - id: id, - addr: addr, - version: version, - interval: InstanceHeartbeat, - liveness: InstanceLiveness, - retention: DepartedRetention, - stop: make(chan struct{}), - done: make(chan struct{}), + reg: reg, + id: id, + addr: addr, + version: version, + peerTokenHash: cred.Hash(), + interval: InstanceHeartbeat, + liveness: InstanceLiveness, + retention: DepartedRetention, + stop: make(chan struct{}), + done: make(chan struct{}), } } @@ -173,7 +186,7 @@ func (m *Membership) SetReconnectGrace(grace time.Duration) { // address never reaches the table is invisible to its peers, and starting // anyway would hide that behind a background log line. func (m *Membership) Start(ctx context.Context) error { - if err := m.reg.Register(ctx, m.id, m.addr, m.version); err != nil { + if err := m.reg.Register(ctx, m.id, m.addr, m.version, m.peerTokenHash); err != nil { return err } xlog.Info("Cluster instance registered", "id", m.id, "addr", m.addr) @@ -258,7 +271,7 @@ func (m *Membership) tick(ctx context.Context) { // again or this replica serves workers that, as far as every other // replica can see, are connected nowhere. xlog.Warn("Cluster instance row was reaped, re-registering", "id", m.id) - if err := m.reg.Register(ctx, m.id, m.addr, m.version); err == nil { + if err := m.reg.Register(ctx, m.id, m.addr, m.version, m.peerTokenHash); err == nil { m.reclaimTunnels(ctx) } else { // Re-claiming is skipped and only re-claiming: a claim written now diff --git a/core/services/cluster/membership_test.go b/core/services/cluster/membership_test.go index 188295bb494f..69db4ae6ede7 100644 --- a/core/services/cluster/membership_test.go +++ b/core/services/cluster/membership_test.go @@ -35,8 +35,8 @@ var _ = Describe("Reaping dead replicas", func() { }) It("deletes a replica that stopped heartbeating, and the connections it owned", func() { - Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1")).To(Succeed()) - Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1", "")).To(Succeed()) + Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "dead") Expect(err).ToNot(HaveOccurred()) age("dead", time.Hour) @@ -55,8 +55,8 @@ var _ = Describe("Reaping dead replicas", func() { // The row is what tells "this worker's owner died a moment ago" from // "this worker has never connected". Deleting it on the sweep erased // exactly the departure a reconnect grace has to be measured from. - Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1")).To(Succeed()) - Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1", "")).To(Succeed()) + Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "dead") Expect(err).ToNot(HaveOccurred()) age("dead", time.Hour) @@ -76,8 +76,8 @@ var _ = Describe("Reaping dead replicas", func() { // would push its departure forward on every pass, so the departure // would never age out of any window and the row would be reported as // swept for as long as it existed. - Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1")).To(Succeed()) - Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1", "")).To(Succeed()) + Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "dead") Expect(err).ToNot(HaveOccurred()) age("dead", time.Hour) @@ -102,7 +102,7 @@ var _ = Describe("Reaping dead replicas", func() { It("records a departure rather than erasing the connection when a replica deregisters", func() { // The announced form of the same thing the sweeper does by inference, // and the two must not disagree about what "gone" leaves behind. - Expect(reg.Register(ctx, "leaving", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "leaving", "10.0.0.2:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "leaving") Expect(err).ToNot(HaveOccurred()) @@ -115,8 +115,8 @@ var _ = Describe("Reaping dead replicas", func() { }) It("leaves the connections of a live replica alone", func() { - Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1")).To(Succeed()) - Expect(reg.Register(ctx, "other", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "live", "10.0.0.1:8080", "v1", "")).To(Succeed()) + Expect(reg.Register(ctx, "other", "10.0.0.2:8080", "v1", "")).To(Succeed()) epoch, err := reg.Claim(ctx, "w1", "other") Expect(err).ToNot(HaveOccurred()) @@ -135,7 +135,7 @@ var _ = Describe("Reaping dead replicas", func() { // serving the workers connected to it. Reaping its own row would delete // their connection rows in the same pass, re-homing workers that never // went anywhere. - Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "me") Expect(err).ToNot(HaveOccurred()) age("me", time.Hour) @@ -154,8 +154,8 @@ var _ = Describe("Reaping dead replicas", func() { // Without this a cleanly stopped replica is indistinguishable from a // crashed one, and every peer keeps dialling it for the whole liveness // window. - Expect(reg.Register(ctx, "leaving", "10.0.0.2:8080", "v1")).To(Succeed()) - Expect(reg.Register(ctx, "staying", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "leaving", "10.0.0.2:8080", "v1", "")).To(Succeed()) + Expect(reg.Register(ctx, "staying", "10.0.0.1:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "leaving") Expect(err).ToNot(HaveOccurred()) _, err = reg.Claim(ctx, "w2", "staying") @@ -183,7 +183,7 @@ var _ = Describe("Reaping dead replicas", func() { // one, so an empty id would match every departure in the table and reset // each one's age. Nothing generates an empty id today, which is exactly // why the filter has to be in the query rather than in that habit. - Expect(reg.Register(ctx, "leaving", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "leaving", "10.0.0.2:8080", "v1", "")).To(Succeed()) epoch, err := reg.Claim(ctx, "w1", "leaving") Expect(err).ToNot(HaveOccurred()) Expect(reg.Release(ctx, "w1", "leaving", epoch)).To(Succeed()) @@ -205,7 +205,7 @@ var _ = Describe("Reaping dead replicas", func() { // row per worker that ever dialled this deployment, which is the state // the sweep's held-ness filter exists to make reachable in the first // place. - Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1", "")).To(Succeed()) epoch, err := reg.Claim(ctx, "w1", "me") Expect(err).ToNot(HaveOccurred()) Expect(reg.Release(ctx, "w1", "me", epoch)).To(Succeed()) @@ -214,7 +214,7 @@ var _ = Describe("Reaping dead replicas", func() { `UPDATE node_connections SET disconnected_at = now() - make_interval(secs => ?) WHERE node_id = ?`, (cluster.DepartedRetention + time.Minute).Seconds(), "w1").Error).To(Succeed()) - membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1", cluster.NewPeerCredential()) Expect(membership.Start(ctx)).To(Succeed()) DeferCleanup(membership.Stop) @@ -239,7 +239,7 @@ var _ = Describe("Reaping dead replicas", func() { // the two rows below straddle the difference and one answer cannot // stand in for the other. const grace = 10 * time.Minute - Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1", "")).To(Succeed()) depart := func(node string, ago time.Duration) { epoch, err := reg.Claim(ctx, node, "me") Expect(err).ToNot(HaveOccurred()) @@ -259,7 +259,7 @@ var _ = Describe("Reaping dead replicas", func() { // ticked, and the spec would pass on nothing happening. depart("w-past-derived-retention", cluster.DepartedRetentionFor(grace)+time.Minute) - membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1", cluster.NewPeerCredential()) membership.SetReconnectGrace(grace) Expect(membership.Start(ctx)).To(Succeed()) DeferCleanup(membership.Stop) @@ -276,7 +276,7 @@ var _ = Describe("Reaping dead replicas", func() { }) It("deregisters when the membership loop stops", func() { - membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1", cluster.NewPeerCredential()) Expect(membership.Start(ctx)).To(Succeed()) live, err := reg.Live(ctx, time.Minute) @@ -297,7 +297,7 @@ var _ = Describe("Reaping dead replicas", func() { // The order is asserted on the SQL because the alternative, racing two // transactions until they actually deadlock, is exactly the flaky spec // this one replaces. - Expect(reg.Register(ctx, "leaving", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "leaving", "10.0.0.2:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "leaving") Expect(err).ToNot(HaveOccurred()) @@ -316,7 +316,7 @@ var _ = Describe("Reaping dead replicas", func() { }) It("tolerates a repeated deregistration, because a sweeper may have got there first", func() { - Expect(reg.Register(ctx, "gone", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "gone", "10.0.0.2:8080", "v1", "")).To(Succeed()) Expect(reg.Deregister(ctx, "gone")).To(Succeed()) Expect(reg.Deregister(ctx, "gone")).To(Succeed()) }) @@ -325,7 +325,7 @@ var _ = Describe("Reaping dead replicas", func() { // Nothing calls this today. It exists because the loop channel is only // ever closed by a started loop, so joining an unstarted one blocks // forever, and phase 2 adds callers to this shutdown path. - membership := cluster.NewMembership(reg, "never-started", "10.0.0.1:8080", "v1") + membership := cluster.NewMembership(reg, "never-started", "10.0.0.1:8080", "v1", cluster.NewPeerCredential()) done := make(chan struct{}) go func() { defer GinkgoRecover() @@ -336,10 +336,10 @@ var _ = Describe("Reaping dead replicas", func() { }) It("keeps this replica's row alive and reaps the dead while it runs", func() { - Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "dead", "10.0.0.2:8080", "v1", "")).To(Succeed()) age("dead", time.Hour) - membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1", cluster.NewPeerCredential()) Expect(membership.Start(ctx)).To(Succeed()) DeferCleanup(membership.Stop) diff --git a/core/services/cluster/ownership_test.go b/core/services/cluster/ownership_test.go index d23f9d98d769..b0dd2dfb4826 100644 --- a/core/services/cluster/ownership_test.go +++ b/core/services/cluster/ownership_test.go @@ -225,7 +225,7 @@ var _ = Describe("Connection ownership", func() { } It("names an owner whose replica is live", func() { - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) claimed, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) @@ -236,7 +236,7 @@ var _ = Describe("Connection ownership", func() { }) It("still names an owner whose heartbeat is old but inside the window", func() { - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) claimed, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) age("inst-a", agedButLive) @@ -261,7 +261,7 @@ var _ = Describe("Connection ownership", func() { It("refuses to name an owner whose heartbeat has aged past the liveness window", func() { // The window this task exists to close: the replica is dead, no peer // has swept it yet, and the row still names it. - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) age("inst-a", agedOut) @@ -274,7 +274,7 @@ var _ = Describe("Connection ownership", func() { // Liveness is a window, not a latch: a replica that stalls and // recovers still owns the sockets it never dropped, so resolution // has to follow last_seen rather than remember a verdict. - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) claimed, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) age("inst-a", agedOut) @@ -287,7 +287,7 @@ var _ = Describe("Connection ownership", func() { }) It("still reports the dead owner through OwnerRow, which is why the two reads are separate", func() { - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) claimed, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) age("inst-a", agedOut) @@ -307,7 +307,7 @@ var _ = Describe("Connection ownership", func() { }) It("resolves in one joined statement measured on the database clock", func() { - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) @@ -433,7 +433,7 @@ var _ = Describe("Connection ownership", func() { }) It("reports a released row as no connection from both reads", func() { - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) epoch, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) Expect(reg.Release(ctx, "w1", "inst-a", epoch)).To(Succeed()) @@ -452,7 +452,7 @@ var _ = Describe("Connection ownership", func() { // instances join to miss it. The join only misses an empty owner // for as long as no instance row carries an empty id, which is an // accident of who registers rather than a property of ownership. - Expect(reg.Register(ctx, "", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "", "10.0.0.1:8080", "v1", "")).To(Succeed()) epoch, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) Expect(reg.Release(ctx, "w1", "inst-a", epoch)).To(Succeed()) @@ -462,7 +462,7 @@ var _ = Describe("Connection ownership", func() { }) It("clears the departure when the worker reconnects", func() { - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) epoch, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) Expect(reg.Release(ctx, "w1", "inst-a", epoch)).To(Succeed()) @@ -481,7 +481,7 @@ var _ = Describe("Connection ownership", func() { }) It("does not let a fenced-out replica stamp a departure onto the live claim", func() { - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) stale, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) live, err := reg.Claim(ctx, "w1", "inst-a") @@ -564,7 +564,7 @@ var _ = Describe("Connection ownership", func() { }) It("never purges a row that is still held, whatever timestamp it carries", func() { - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) claimed, err := reg.Claim(ctx, "w1", "inst-a") Expect(err).ToNot(HaveOccurred()) // No writer produces a held row carrying a departure, so it is diff --git a/core/services/cluster/peeridentity.go b/core/services/cluster/peeridentity.go new file mode 100644 index 000000000000..8fa6c91c7811 --- /dev/null +++ b/core/services/cluster/peeridentity.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: MIT + +package cluster + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" +) + +// PeerIdentityHeader carries a replica's own peer credential on a dial to +// GET /api/cluster/peer, alongside the deployment's shared cluster token in +// Authorization. +// +// A header of its own rather than a second value crammed into Authorization, +// and the two are checked separately: the shared token says the dialler belongs +// to this deployment, this one says WHICH replica it is. Folding them together +// would make one of the two answers unavailable, and the shared token check is +// the one that must not be weakened. +// +// The constant lives beside PeerPath, in the package both ends import, for the +// same reason PeerPath does: the dialler here and the handler in +// core/http/endpoints/cluster cannot end up naming different headers. +const PeerIdentityHeader = "X-LocalAI-Peer-Token" + +// PeerCredential is one frontend replica's proof of which replica it is. +// +// The shape follows the per-node worker credential (nodes.BackendNode +// TunnelTokenHash) rather than inventing a second mechanism: a random secret, +// stored only as its SHA-256, compared in constant time, with no fallback to +// the shared token. It differs in one way, and in the stronger direction. A +// worker's credential is minted BY the frontend and returned to the worker +// once, so it crosses the network at birth; a replica writes its own row in the +// instances table, so it mints its own secret, publishes only the hash, and the +// plaintext never leaves the process that made it. +// +// It is one value carrying both halves because they must not be minted twice. +// Membership stores Hash and PeerPool presents Token, and two independent mints +// would leave a replica whose stored hash and presented secret disagree: every +// dial it makes would be refused, on a route whose refusal looks exactly like a +// peer that is merely older. Passing this struct to both makes that a +// non-outcome rather than a bug to be found. +// +// The zero value is a credential nobody may use: an empty Token is refused by +// every handler, and an empty Hash authorizes nobody. That direction is +// deliberate. An empty credential is NOT a credential that matches everything. +type PeerCredential struct { + token string + hash string +} + +// NewPeerCredential mints a fresh credential for this replica. +// +// crypto/rand.Text gives at least 128 bits of randomness with no error to +// handle and no length constant to get wrong, which is what the worker +// credential is minted with too. +func NewPeerCredential() PeerCredential { + token := rand.Text() + return PeerCredential{token: token, hash: HashPeerToken(token)} +} + +// Token is the secret this replica presents when it dials a peer. It is only +// ever sent, never stored. +func (c PeerCredential) Token() string { return c.token } + +// Hash is what this replica publishes in its instances row, and the only form +// of the credential any other replica ever sees. +func (c PeerCredential) Hash() string { return c.hash } + +// HashPeerToken renders a presented peer token in the form the instances table +// stores. Exported so the handler that verifies a dial and the replica that +// registers its row cannot disagree about the encoding. +func HashPeerToken(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +// PeerTokenMatches reports whether a presented token is the one behind +// storedHash, in constant time. +// +// An empty storedHash matches NOTHING, including an empty token. A replica with +// no published credential is one registered by a release that predates them, and +// it authorizes nobody: reading "no credential" as "any credential" is exactly +// the failure this route is being fixed for. +// +// The empty guard is not what makes that true (ConstantTimeCompare already +// returns 0 on a length mismatch, and a stored hash is 64 hex bytes or nothing); +// it is here so a reader does not have to derive the rule from a length, and so +// the caller can log that case as the distinct operator problem it is. +func PeerTokenMatches(token, storedHash string) bool { + if storedHash == "" || token == "" { + return false + } + return subtle.ConstantTimeCompare([]byte(HashPeerToken(token)), []byte(storedHash)) == 1 +} diff --git a/core/services/cluster/peeridentity_test.go b/core/services/cluster/peeridentity_test.go new file mode 100644 index 000000000000..9556510c834f --- /dev/null +++ b/core/services/cluster/peeridentity_test.go @@ -0,0 +1,326 @@ +package cluster_test + +import ( + "context" + "encoding/json" + "net/http/httptest" + "strings" + + "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" + + "github.com/labstack/echo/v4" + "github.com/libp2p/go-yamux/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" +) + +// These specs drive the PRODUCTION handler over a real WebSocket and a real +// yamux session, because the defect they pin is one a double cannot have: the +// id a peer declares is a query parameter on a transport, and whether it is +// checked is decided in the same place the connection is hijacked. +var _ = Describe("Peer identity", func() { + var ( + db *gorm.DB + reg *cluster.Registry + store *cluster.SessionStore + srv *httptest.Server + ctx context.Context + realCred cluster.PeerCredential + ) + + // The victim. "real" is a legitimate replica with a published credential; + // everything below is an attempt to be it. + const realID = "real-replica" + + // The deployment-wide secret. Every worker holds it, and every spec below + // that is meant to fail holds it too: the whole point is that holding it is + // no longer enough. + const clusterToken = "shared-cluster-token" + + BeforeEach(func() { + ctx = context.Background() + db = testutil.SetupTestDB() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg = cluster.NewRegistry(db) + + // A store with no relay: every accepted stream is closed at once, which + // is enough to observe WHICH links exist without standing up a worker. + store = cluster.NewSessionStore(nil) + DeferCleanup(store.CloseAll) + + e := echo.New() + servePeerRoute(e, clusterToken, reg, store.Accept) + srv = httptest.NewServer(e) + DeferCleanup(srv.Close) + + realCred = cluster.NewPeerCredential() + Expect(reg.Register(ctx, realID, "10.0.0.7:8080", "test", realCred.Hash())).To(Succeed()) + // The replica being dialled needs a row of its own, since the pool + // resolves the address it dials out of the table. + Expect(reg.Register(ctx, "listener", strings.TrimPrefix(srv.URL, "http://"), "test", "")).To(Succeed()) + }) + + // dialAs opens a link to the listening replica, declaring id and presenting + // cred. It returns the pool so a spec can assert on what the link did. + dialAs := func(id string, cred cluster.PeerCredential) (*cluster.PeerPool, error) { + pool := cluster.NewPeerPool(id, clusterToken, cred, reg) + DeferCleanup(pool.Close) + st, err := pool.Open(ctx, "listener") + if err == nil { + DeferCleanup(func() { _ = st.Close() }) + } + return pool, err + } + + It("accepts a replica that proves the id it declares", func() { + _, err := dialAs(realID, realCred) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() bool { + _, held := store.Get(realID) + return held + }, "10s").Should(BeTrue()) + }) + + It("refuses a dialler that declares another replica's id while holding the shared cluster token", func() { + // THE attack this whole change exists for. The impostor has everything + // a worker has: the deployment's registration token, and the id of a + // replica it read out of any log line or API response. Before per + // replica credentials that was the whole of the check, so this dial + // succeeded and the link it opened relayed to every worker tunnel the + // listening replica owns. + // + // It is the case the cheap patch does not cover: validating the id + // against the instances table passes here, because the id IS in the + // table. Only a secret the impostor does not have refuses it. + _, err := dialAs(realID, cluster.NewPeerCredential()) + Expect(err).To(MatchError(cluster.ErrPeerRejected)) + Expect(err).ToNot(MatchError(cluster.ErrInstanceNotFound), + "a refused peer is an authorization failure; read as absence it makes the scheduler reap rows and evict models") + Consistently(func() bool { + _, held := store.Get(realID) + return held + }, "1s", "100ms").Should(BeFalse()) + }) + + It("does not let an impostor evict the link a real replica already holds", func() { + // Displacement, which is a denial of service rather than an + // eavesdrop: SessionStore keeps ONE link per peer id, so a dial that is + // accepted under an id closes whatever was there. Repeated at will, it + // keeps a replica's inbound link permanently broken. + _, err := dialAs(realID, realCred) + Expect(err).ToNot(HaveOccurred()) + + var held *yamux.Session + Eventually(func() bool { + sess, ok := store.Get(realID) + held = sess + return ok + }, "10s").Should(BeTrue()) + + _, err = dialAs(realID, cluster.NewPeerCredential()) + Expect(err).To(MatchError(cluster.ErrPeerRejected)) + + // The same session object, still alive. A refused dial that had reached + // Accept would have replaced this entry and closed what it replaced. + current, ok := store.Get(realID) + Expect(ok).To(BeTrue()) + Expect(current).To(BeIdenticalTo(held)) + Expect(current.IsClosed()).To(BeFalse(), + "an impostor closed the real replica's inbound link") + }) + + It("refuses a dialler whose declared replica has no published credential", func() { + // A replica registered by a release that predates peer identity. Its + // row carries an empty hash, and empty must authorize NOBODY: read as + // "no restriction" it would leave every not-yet-upgraded replica + // impersonable by anything holding the shared token, which is the + // exposure rather than a step towards closing it. + Expect(reg.Register(ctx, "legacy-replica", "10.0.0.8:8080", "test", "")).To(Succeed()) + _, err := dialAs("legacy-replica", cluster.NewPeerCredential()) + Expect(err).To(MatchError(cluster.ErrPeerRejected)) + }) + + It("refuses a dialler holding a credential that matches no replica at all", func() { + // The zero value is a credential nobody may use. A pool built without + // one presents an empty token, and an empty token must not match an + // empty stored hash either. + Expect(reg.Register(ctx, "blank-replica", "10.0.0.9:8080", "test", "")).To(Succeed()) + _, err := dialAs("blank-replica", cluster.PeerCredential{}) + Expect(err).To(MatchError(cluster.ErrPeerRejected)) + }) + + It("refuses a dialler that declares an id no replica holds", func() { + _, err := dialAs("invented-replica", cluster.NewPeerCredential()) + Expect(err).To(MatchError(cluster.ErrPeerRejected)) + Expect(err).ToNot(MatchError(cluster.ErrInstanceNotFound), + "the peer answered and refused; nothing here says a worker went away") + }) + + It("still requires the shared cluster token from a replica that can prove its identity", func() { + // Identity is added to the existing check, not swapped for it. A + // dialler with a perfectly good credential and the wrong deployment + // token belongs to another deployment. + pool := cluster.NewPeerPool(realID, "not-the-cluster-token", realCred, reg) + DeferCleanup(pool.Close) + _, err := pool.Open(ctx, "listener") + Expect(err).To(MatchError(cluster.ErrPeerRejected)) + Expect(err).ToNot(MatchError(cluster.ErrInstanceNotFound)) + }) + + It("reports every refusal as unreachable too, so no existing consumer changes behaviour", func() { + // ErrPeerUnreachable has named "the peer refused the credentials" since + // it was written, and every consumer already treats it as a retryable + // transport condition. ErrPeerRejected is added UNDERNEATH that rather + // than in place of it, so gaining a distinguishable authorization + // failure cannot silently re-route one existing caller. + _, err := dialAs(realID, cluster.NewPeerCredential()) + Expect(err).To(MatchError(cluster.ErrPeerUnreachable)) + Expect(err).To(MatchError(cluster.ErrPeerRejected)) + Expect(err).ToNot(MatchError(cluster.ErrInstanceNotFound)) + Expect(err).ToNot(MatchError(cluster.ErrNoConnection)) + }) +}) + +var _ = Describe("Peer credential", func() { + It("publishes only a hash and never the secret", func() { + cred := cluster.NewPeerCredential() + Expect(cred.Token()).ToNot(BeEmpty()) + Expect(cred.Hash()).To(Equal(cluster.HashPeerToken(cred.Token()))) + Expect(cred.Hash()).ToNot(Equal(cred.Token())) + Expect(cred.Hash()).To(HaveLen(64)) + }) + + It("mints a different secret every time", func() { + Expect(cluster.NewPeerCredential().Token()).ToNot(Equal(cluster.NewPeerCredential().Token())) + }) + + It("treats an empty stored hash as permission for nobody", func() { + // The trap this programme hit twice: an empty allow list read as + // unrestricted. An unpublished credential authorizes nothing, not + // everything, and an empty presented token is not a match for it. + Expect(cluster.PeerTokenMatches("anything", "")).To(BeFalse()) + Expect(cluster.PeerTokenMatches("", "")).To(BeFalse()) + Expect(cluster.PeerTokenMatches("", cluster.HashPeerToken("secret"))).To(BeFalse()) + }) + + It("is not serialised out of an instance row", func() { + // The column holds a secret's hash, and Instance carries json tags for + // every other field. Nothing marshals it today; the tag is what keeps + // that safe the first time something does. + cred := cluster.NewPeerCredential() + out, err := json.Marshal(cluster.Instance{ID: "inst-a", PeerTokenHash: cred.Hash()}) + Expect(err).ToNot(HaveOccurred()) + Expect(string(out)).ToNot(ContainSubstring(cred.Hash())) + Expect(string(out)).ToNot(ContainSubstring("peer_token_hash")) + }) + + It("matches only the secret behind the hash", func() { + Expect(cluster.PeerTokenMatches("secret", cluster.HashPeerToken("secret"))).To(BeTrue()) + Expect(cluster.PeerTokenMatches("secret ", cluster.HashPeerToken("secret"))).To(BeFalse()) + Expect(cluster.PeerTokenMatches(cluster.HashPeerToken("secret"), cluster.HashPeerToken("secret"))).To(BeFalse()) + }) +}) + +var _ = Describe("Instance registration identity", func() { + var ( + db *gorm.DB + reg *cluster.Registry + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + db = testutil.SetupTestDB() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg = cluster.NewRegistry(db) + }) + + It("publishes the credential hash in the same write as the address", func() { + cred := cluster.NewPeerCredential() + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", cred.Hash())).To(Succeed()) + + got, err := reg.Get(ctx, "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(got.PeerTokenHash).To(Equal(cred.Hash())) + Expect(got.PeerTokenHash).ToNot(Equal(cred.Token()), + "the plaintext must never reach the table") + }) + + It("replaces the published hash when a replica re-registers", func() { + first := cluster.NewPeerCredential() + second := cluster.NewPeerCredential() + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", first.Hash())).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", second.Hash())).To(Succeed()) + + got, err := reg.Get(ctx, "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(got.PeerTokenHash).To(Equal(second.Hash()), + "a restarted replica mints a new secret; a row that kept the old hash would refuse its own dials") + }) + + It("keeps a heartbeat from clearing the published hash", func() { + // Heartbeat writes last_seen and nothing else. A heartbeat that reset + // the column would make every replica unpeerable five seconds after it + // started, which no spec that only registers would ever see. + cred := cluster.NewPeerCredential() + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", cred.Hash())).To(Succeed()) + Expect(reg.Heartbeat(ctx, "inst-a")).To(Succeed()) + + got, err := reg.Get(ctx, "inst-a") + Expect(err).ToNot(HaveOccurred()) + Expect(got.PeerTokenHash).To(Equal(cred.Hash())) + }) +}) + +var _ = Describe("Membership identity", func() { + It("republishes the credential after another replica reaped this row", func() { + // The heartbeat path, which is a SECOND writer of the identity and the + // one nothing else covers. A replica stalled long enough is reaped by a + // peer; its next tick finds no row, re-registers, and must publish the + // same credential it is still dialling with. A re-registration that + // wrote an empty hash would leave a replica that is up, heartbeating + // and refused by every peer it dials, with nothing naming the cause. + ctx := context.Background() + db := testutil.SetupTestDB() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg := cluster.NewRegistry(db) + + cred := cluster.NewPeerCredential() + m := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1", cred) + Expect(m.Start(ctx)).To(Succeed()) + DeferCleanup(m.Stop) + + // What a peer's sweep leaves behind. + Expect(db.WithContext(ctx).Where("id = ?", "me").Delete(&cluster.Instance{}).Error).To(Succeed()) + + Eventually(func() (string, error) { + got, err := reg.Get(ctx, "me") + if err != nil { + return "", err + } + return got.PeerTokenHash, nil + }, "20s", "250ms").Should(Equal(cred.Hash()), + "the replica rebuilt its row without the identity peers check it by") + }) + + It("publishes the credential it was built with", func() { + // The membership loop is the ONLY writer of this replica's row, so if + // it does not carry the hash there is nothing for a peer to check and + // every dial this replica makes is refused. + ctx := context.Background() + db := testutil.SetupTestDB() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg := cluster.NewRegistry(db) + + cred := cluster.NewPeerCredential() + m := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1", cred) + Expect(m.Start(ctx)).To(Succeed()) + DeferCleanup(m.Stop) + + got, err := reg.Get(ctx, "me") + Expect(err).ToNot(HaveOccurred()) + Expect(got.PeerTokenHash).To(Equal(cred.Hash())) + }) +}) diff --git a/core/services/cluster/peerlink.go b/core/services/cluster/peerlink.go index fec713891802..e0efa2b3885c 100644 --- a/core/services/cluster/peerlink.go +++ b/core/services/cluster/peerlink.go @@ -65,6 +65,56 @@ func unreachablePeer(peerID string, cause error) error { return &unreachableError{peerID: peerID, cause: cause} } +// ErrPeerRejected reports that a peer answered and REFUSED this replica's +// credentials: either the deployment's shared cluster token or, since +// per-replica peer identity, this replica's own peer credential. +// +// It is its own sentinel because an authorization failure has a different cure +// from every other way a dial can end. A peer that will not answer is a +// transport problem an operator waits out; a peer that answers "no" is a +// configuration problem, and during a rolling upgrade it is the specific, +// expected one of a replica running a release that predates peer credentials +// dialling one that requires them. A single sentinel for both would leave that +// window looking like a network fault. +// +// It is emphatically NOT absence. What it says is that a peer is up, listening +// and talking; concluding from it that a worker has gone away is the mistake +// this phase is built to make impossible, because acting on absence reaps rows +// and evicts models. +var ErrPeerRejected = errors.New("cluster: peer refused this replica's credentials") + +// rejectedError reports a peer that refused the dial. +// +// It unwraps to BOTH ErrPeerRejected and ErrPeerUnreachable, and each half is +// load bearing. ErrPeerUnreachable is the condition every existing caller +// already handles correctly for a refused dial (ErrPeerUnreachable's own doc +// has named "the peer refused the credentials" from the start), so keeping it +// means adding an identity check cannot change how one single consumer treats a +// refusal. ErrPeerRejected is what lets a caller that WANTS to tell the two +// apart do so, without any consumer having to opt in first. +// +// What it never unwraps to is any absence sentinel, and the cause is kept out +// of the chain for exactly the reason unreachableError keeps its own out: the +// dial path resolves a peer through the registry, so ErrInstanceNotFound is a +// cause this error could genuinely be built over, and an unwrapped one would +// satisfy an absence check on an authorization failure. +type rejectedError struct { + peerID string + cause error +} + +func (e *rejectedError) Error() string { + return fmt.Sprintf("cluster: peer %q refused this replica's credentials: %v", e.peerID, e.cause) +} + +// Unwrap reports the two sentinels and nothing else. The cause reaches a human +// through Error() and reaches no error-matching caller at all. +func (e *rejectedError) Unwrap() []error { return []error{ErrPeerRejected, ErrPeerUnreachable} } + +func rejectedPeer(peerID string, cause error) error { + return &rejectedError{peerID: peerID, cause: cause} +} + // ErrPoolClosed reports an Open on a pool that has been shut down. It is a // third condition on purpose: the pool being closed is a fact about this // process and says nothing about whether the peer exists or answers. @@ -140,7 +190,16 @@ func PeerLinkConfig() *yamux.Config { type PeerPool struct { selfID string token string - reg *Registry + // cred is this replica's OWN peer credential, the plaintext half. It is + // what makes selfID a claim rather than a label: the peer resolves selfID + // to an instances row and checks this against the hash that row publishes. + // + // Held by value beside selfID on purpose. The two are one identity, and a + // pool that could be built with one and not the other would dial as a + // replica it cannot prove it is, which every peer refuses and which looks + // from the outside like a peer running an older release. + cred PeerCredential + reg *Registry dialer *websocket.Dialer @@ -158,11 +217,21 @@ type peerLink struct { } // NewPeerPool returns a pool that dials peers as selfID, authenticating with -// the deployment's cluster token. -func NewPeerPool(selfID, token string, reg *Registry) *PeerPool { +// the deployment's cluster token AND with this replica's own peer credential. +// +// Both, never one. The cluster token says the dialler belongs to this +// deployment and the credential says which replica it is, and the peer checks +// them separately: the first is what has always been checked and is not +// weakened by the second existing. +// +// cred must be the SAME value the membership loop published the hash of. Pass +// the one PeerCredential this process minted, which is what makes that +// unstateable rather than merely required. +func NewPeerPool(selfID, token string, cred PeerCredential, reg *Registry) *PeerPool { return &PeerPool{ selfID: selfID, token: token, + cred: cred, reg: reg, dialer: &websocket.Dialer{ HandshakeTimeout: peerLinkHandshakeTimeout, @@ -333,15 +402,35 @@ func (p *PeerPool) dial(ctx context.Context, peerID string) (*yamux.Session, err } header := http.Header{} header.Set("Authorization", "Bearer "+p.token) + // The identity half. Sent unconditionally, including when it is empty: an + // empty value is refused by the peer, which is the right outcome for a + // process that never minted a credential, and omitting the header instead + // would make that indistinguishable from an older release at the only place + // that can tell an operator which one it is looking at. + header.Set(PeerIdentityHeader, p.cred.Token()) ws, resp, err := p.dialer.DialContext(ctx, endpoint.String(), header) - if resp != nil && resp.Body != nil { - // gorilla hands back the failed handshake's response so a caller can - // read the status; nothing here needs the body, but it has to be - // drained or the connection is not returned to the transport. - _ = resp.Body.Close() + status := 0 + if resp != nil { + status = resp.StatusCode + if resp.Body != nil { + // gorilla hands back the failed handshake's response so a caller can + // read the status; nothing here needs the body, but it has to be + // drained or the connection is not returned to the transport. + _ = resp.Body.Close() + } } if err != nil { + // A peer that ANSWERED 401 is up and talking, and what it refused is a + // credential. Reported as its own condition so the log names the cure: + // during a frontend rollout this is what a replica running a release + // without peer credentials gets from one that requires them, and it + // must not be read as, or logged as, a network fault. + if status == http.StatusUnauthorized { + xlog.Warn("A peer refused this replica's credentials. Either the deployment's registration token differs between the two, or this replica has no peer credential published in the instances table; during a rolling upgrade this is what an older replica sees until it is upgraded", + "peer", peerID, "self", p.selfID, "addr", inst.AdvertisedAddr) + return nil, rejectedPeer(peerID, err) + } return nil, unreachablePeer(peerID, err) } diff --git a/core/services/cluster/peerlink_test.go b/core/services/cluster/peerlink_test.go index d4406e5e167d..da57f60f3635 100644 --- a/core/services/cluster/peerlink_test.go +++ b/core/services/cluster/peerlink_test.go @@ -26,8 +26,8 @@ import ( // lives in core/http/routes, which imports half the server, and these specs are // about the handler and the dialler rather than about the route table. The path // comes from the same constant the registrar uses, so the two cannot drift. -func servePeerRoute(e *echo.Echo, token string, onPeer func(string, *yamux.Session)) { - e.GET(cluster.PeerPath, clusterep.PeerHandler(token, onPeer)) +func servePeerRoute(e *echo.Echo, token string, instances *cluster.Registry, onPeer func(string, *yamux.Session)) { + e.GET(cluster.PeerPath, clusterep.PeerHandler(token, instances, onPeer)) } // deadlinePassed is a context whose deadline has elapsed and whose @@ -46,20 +46,25 @@ var _ = Describe("Peer pool", func() { db *gorm.DB reg *cluster.Registry pool *cluster.PeerPool + selfCred cluster.PeerCredential srv *httptest.Server accepted chan *yamux.Session ctx context.Context ) // startPeer stands up a real peer server and registers it under peerID. + // + // Every server here is the PRODUCTION handler over a real WebSocket, so a + // dial that this pool cannot prove its identity for is refused by the same + // code path a deployment runs, at the same point, before the upgrade. startPeer := func(peerID string) *httptest.Server { e := echo.New() - servePeerRoute(e, "peer-token", func(_ string, s *yamux.Session) { + servePeerRoute(e, "peer-token", reg, func(_ string, s *yamux.Session) { accepted <- s }) ts := httptest.NewServer(e) addr := strings.TrimPrefix(ts.URL, "http://") - Expect(reg.Register(ctx, peerID, addr, "test")).To(Succeed()) + Expect(reg.Register(ctx, peerID, addr, "test", "")).To(Succeed()) return ts } @@ -69,7 +74,14 @@ var _ = Describe("Peer pool", func() { Expect(cluster.Migrate(ctx, db)).To(Succeed()) reg = cluster.NewRegistry(db) accepted = make(chan *yamux.Session, 4) - pool = cluster.NewPeerPool("self", "peer-token", reg) + // This replica's own identity, published before it dials anything. A + // pool whose id has no matching hash in the instances table is refused + // by every peer, which is the point of the credential and also the + // reason these specs must register one: a spec that skipped it would + // exercise the refusal path everywhere and prove nothing else. + selfCred = cluster.NewPeerCredential() + Expect(reg.Register(ctx, "self", "10.0.0.1:8080", "test", selfCred.Hash())).To(Succeed()) + pool = cluster.NewPeerPool("self", "peer-token", selfCred, reg) DeferCleanup(pool.Close) srv = startPeer("peer-1") DeferCleanup(srv.Close) @@ -109,10 +121,10 @@ var _ = Describe("Peer pool", func() { // link anonymous and indistinguishable from every other. ids := make(chan string, 1) e := echo.New() - servePeerRoute(e, "peer-token", func(id string, _ *yamux.Session) { ids <- id }) + servePeerRoute(e, "peer-token", reg, func(id string, _ *yamux.Session) { ids <- id }) ts := httptest.NewServer(e) DeferCleanup(ts.Close) - Expect(reg.Register(ctx, "peer-named", strings.TrimPrefix(ts.URL, "http://"), "test")).To(Succeed()) + Expect(reg.Register(ctx, "peer-named", strings.TrimPrefix(ts.URL, "http://"), "test", "")).To(Succeed()) st, err := pool.Open(ctx, "peer-named") Expect(err).ToNot(HaveOccurred()) @@ -152,10 +164,10 @@ var _ = Describe("Peer pool", func() { // row. Reporting absence here would evict every worker behind a peer // that was merely rolled out with a stale secret. e := echo.New() - servePeerRoute(e, "a-different-token", func(_ string, s *yamux.Session) { accepted <- s }) + servePeerRoute(e, "a-different-token", reg, func(_ string, s *yamux.Session) { accepted <- s }) ts := httptest.NewServer(e) DeferCleanup(ts.Close) - Expect(reg.Register(ctx, "peer-strict", strings.TrimPrefix(ts.URL, "http://"), "test")).To(Succeed()) + Expect(reg.Register(ctx, "peer-strict", strings.TrimPrefix(ts.URL, "http://"), "test", "")).To(Succeed()) _, err := pool.Open(ctx, "peer-strict") Expect(err).To(MatchError(cluster.ErrPeerUnreachable)) @@ -261,7 +273,7 @@ var _ = Describe("Peer pool", func() { held = append(held, c) } }() - Expect(reg.Register(ctx, "peer-silent", ln.Addr().String(), "test")).To(Succeed()) + Expect(reg.Register(ctx, "peer-silent", ln.Addr().String(), "test", "")).To(Succeed()) deadlined, cancel := context.WithTimeout(ctx, 300*time.Millisecond) DeferCleanup(cancel) @@ -293,7 +305,7 @@ var _ = Describe("Peer pool", func() { Expect(err).ToNot(HaveOccurred()) addr := refused.Addr().String() Expect(refused.Close()).To(Succeed()) - Expect(reg.Register(ctx, "peer-refusing", addr, "test")).To(Succeed()) + Expect(reg.Register(ctx, "peer-refusing", addr, "test", "")).To(Succeed()) _, err = pool.Open(deadlinePassed{ctx}, "peer-refusing") Expect(err).To(MatchError(context.DeadlineExceeded)) diff --git a/core/services/cluster/presence_test.go b/core/services/cluster/presence_test.go index eeb66ca06935..aee4d5429bd4 100644 --- a/core/services/cluster/presence_test.go +++ b/core/services/cluster/presence_test.go @@ -48,7 +48,7 @@ var _ = Describe("Presence", func() { db = testutil.SetupTestDB() Expect(cluster.Migrate(ctx, db)).To(Succeed()) reg = cluster.NewRegistry(db) - Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) }) It("reports unknown for a node with no connection row", func() { diff --git a/core/services/cluster/relay_internal_test.go b/core/services/cluster/relay_internal_test.go index 2a5e631b0ccd..8a02eed98441 100644 --- a/core/services/cluster/relay_internal_test.go +++ b/core/services/cluster/relay_internal_test.go @@ -142,7 +142,7 @@ var _ = Describe("The relay's own budgets", func() { db := testutil.SetupTestDB() Expect(Migrate(ctx, db)).To(Succeed()) reg = NewRegistry(db) - Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1", "")).To(Succeed()) tun = NewTunnelRegistry(reg, "me") }) diff --git a/core/services/cluster/relay_test.go b/core/services/cluster/relay_test.go index 04fa46f20285..3ccc250ab9ff 100644 --- a/core/services/cluster/relay_test.go +++ b/core/services/cluster/relay_test.go @@ -98,7 +98,7 @@ var _ = Describe("The inter-replica relay", func() { ctx = context.Background() Expect(cluster.Migrate(ctx, db)).To(Succeed()) reg = cluster.NewRegistry(db) - Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1", "")).To(Succeed()) tun = cluster.NewTunnelRegistry(reg, "me") store := cluster.NewSessionStore(cluster.NewRelay(tun).Stream) @@ -207,7 +207,7 @@ var _ = Describe("The inter-replica relay", func() { // A relay that resolved the owner and relayed onward would make a // stale row into a loop between two replicas, each certain the other // holds the worker. One hop, always: the dialling replica re-resolves. - Expect(reg.Register(ctx, "other", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "other", "10.0.0.2:8080", "v1", "")).To(Succeed()) _, err := reg.Claim(ctx, "w1", "other") Expect(err).ToNot(HaveOccurred()) diff --git a/core/services/cluster/tunnel_test.go b/core/services/cluster/tunnel_test.go index 26d70350a1df..17ff381550ea 100644 --- a/core/services/cluster/tunnel_test.go +++ b/core/services/cluster/tunnel_test.go @@ -158,7 +158,7 @@ var _ = Describe("The worker tunnel registry", func() { ctx = context.Background() Expect(cluster.Migrate(ctx, db)).To(Succeed()) reg = cluster.NewRegistry(db) - Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1", "")).To(Succeed()) tun = cluster.NewTunnelRegistry(reg, "me") }) @@ -498,7 +498,7 @@ var _ = Describe("Re-claiming tunnels after this replica's rows were reaped", fu ctx = context.Background() Expect(cluster.Migrate(ctx, db)).To(Succeed()) reg = cluster.NewRegistry(db) - Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg.Register(ctx, "me", "10.0.0.1:8080", "v1", "")).To(Succeed()) tun = cluster.NewTunnelRegistry(reg, "me") }) @@ -522,7 +522,7 @@ var _ = Describe("Re-claiming tunnels after this replica's rows were reaped", fu epoch, err := tun.Attach(ctx, "w1", frontend) Expect(err).ToNot(HaveOccurred()) - membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1", cluster.NewPeerCredential()) membership.SetTunnels(tun) Expect(membership.Start(ctx)).To(Succeed()) DeferCleanup(membership.Stop) @@ -565,7 +565,7 @@ var _ = Describe("Re-claiming tunnels after this replica's rows were reaped", fu Expect(deadWorker.Close()).To(Succeed()) Eventually(dead.IsClosed, "10s").Should(BeTrue()) - membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1", cluster.NewPeerCredential()) membership.SetTunnels(tun) Expect(membership.Start(ctx)).To(Succeed()) DeferCleanup(membership.Stop) @@ -730,7 +730,7 @@ var _ = Describe("Re-claiming tunnels after this replica's rows were reaped", fu epoch, err := tun.Attach(ctx, "w1", frontend) Expect(err).ToNot(HaveOccurred()) - membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1") + membership := cluster.NewMembership(reg, "me", "10.0.0.1:8080", "v1", cluster.NewPeerCredential()) membership.SetTunnels(tun) Expect(membership.Start(ctx)).To(Succeed()) DeferCleanup(membership.Stop) diff --git a/core/services/cluster/wsconn_test.go b/core/services/cluster/wsconn_test.go index d7213fd87779..3953b8fab71f 100644 --- a/core/services/cluster/wsconn_test.go +++ b/core/services/cluster/wsconn_test.go @@ -2,6 +2,7 @@ package cluster_test import ( "bytes" + "context" "crypto/rand" "io" "net" @@ -12,6 +13,7 @@ import ( "time" "github.com/mudler/LocalAI/core/services/cluster" + "github.com/mudler/LocalAI/core/services/testutil" "github.com/gorilla/websocket" "github.com/labstack/echo/v4" @@ -237,13 +239,21 @@ var _ = Describe("WebsocketConn framing", func() { var _ = Describe("Peer link payloads", func() { It("carries a payload far larger than one yamux frame end to end", func() { sessions := make(chan *yamux.Session, 1) + ctx := context.Background() + db := testutil.SetupTestDB() + Expect(cluster.Migrate(ctx, db)).To(Succeed()) + reg := cluster.NewRegistry(db) + cred := cluster.NewPeerCredential() + Expect(reg.Register(ctx, "peer-1", "10.0.0.1:8080", "v1", cred.Hash())).To(Succeed()) + e := echo.New() - servePeerRoute(e, "peer-token", func(_ string, s *yamux.Session) { sessions <- s }) + servePeerRoute(e, "peer-token", reg, func(_ string, s *yamux.Session) { sessions <- s }) srv := httptest.NewServer(e) DeferCleanup(srv.Close) h := http.Header{} h.Set("Authorization", "Bearer peer-token") + h.Set(cluster.PeerIdentityHeader, cred.Token()) conn, _, err := websocket.DefaultDialer.Dial( "ws"+strings.TrimPrefix(srv.URL, "http")+"/api/cluster/peer?id=peer-1", h) Expect(err).ToNot(HaveOccurred()) diff --git a/core/services/jobs/claim_test.go b/core/services/jobs/claim_test.go index 8dc4f558d6f2..bb931b887850 100644 --- a/core/services/jobs/claim_test.go +++ b/core/services/jobs/claim_test.go @@ -300,7 +300,7 @@ var _ = Describe("The claim queue", func() { // it, however long it has held it. It("releases a claim whose owner is no longer live", func() { id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) - Expect(reg(db).Register(ctx, "inst-dead", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg(db).Register(ctx, "inst-dead", "10.0.0.1:8080", "v1", "")).To(Succeed()) _, err := ClaimNext(ctx, db, "inst-dead", []ClaimKind{ClaimKindMCPCI}) Expect(err).ToNot(HaveOccurred()) ageInstance(db, "inst-dead", 10*time.Minute) @@ -316,7 +316,7 @@ var _ = Describe("The claim queue", func() { It("leaves a live replica's claim alone however old the claim is", func() { id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) - Expect(reg(db).Register(ctx, "inst-slow", "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(reg(db).Register(ctx, "inst-slow", "10.0.0.2:8080", "v1", "")).To(Succeed()) _, err := ClaimNext(ctx, db, "inst-slow", []ClaimKind{ClaimKindMCPCI}) Expect(err).ToNot(HaveOccurred()) // The claim is hours old; its owner heartbeated a moment ago. Age is @@ -356,14 +356,14 @@ var _ = Describe("The claim queue", func() { Describe("whether this replica may claim at all", func() { It("says yes for a replica that is registered and heartbeating", func() { - Expect(reg(db).Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg(db).Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) live, err := OwnerIsLive(ctx, db, "inst-a", time.Minute) Expect(err).ToNot(HaveOccurred()) Expect(live).To(BeTrue()) }) It("says no for a replica whose heartbeat has aged out", func() { - Expect(reg(db).Register(ctx, "inst-a", "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(reg(db).Register(ctx, "inst-a", "10.0.0.1:8080", "v1", "")).To(Succeed()) ageInstance(db, "inst-a", 10*time.Minute) live, err := OwnerIsLive(ctx, db, "inst-a", time.Minute) Expect(err).ToNot(HaveOccurred()) diff --git a/core/services/jobs/dispatch_loop_test.go b/core/services/jobs/dispatch_loop_test.go index 007017ebb340..e0b884dc9931 100644 --- a/core/services/jobs/dispatch_loop_test.go +++ b/core/services/jobs/dispatch_loop_test.go @@ -155,7 +155,7 @@ var _ = Describe("The dispatch loop", func() { Expect(MigrateClaims(ctx, db)).To(Succeed()) // This replica is registered and heartbeating, which is what makes it // eligible to claim at all. - Expect(cluster.NewRegistry(db).Register(ctx, owner, "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(cluster.NewRegistry(db).Register(ctx, owner, "10.0.0.1:8080", "v1", "")).To(Succeed()) picker = &fakePicker{nodeID: "agent-1", nodeType: nodes.NodeTypeAgent} caller = &fakeCaller{reply: &ClaimReply{}} @@ -532,7 +532,7 @@ var _ = Describe("The dispatch loop", func() { Describe("reaping while it dispatches", func() { It("returns work a departed replica was holding, and takes it on the same tick", func() { id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-orphan"}) - Expect(cluster.NewRegistry(db).Register(ctx, "inst-dead", "10.0.0.9:8080", "v1")).To(Succeed()) + Expect(cluster.NewRegistry(db).Register(ctx, "inst-dead", "10.0.0.9:8080", "v1", "")).To(Succeed()) _, err := ClaimNext(ctx, db, "inst-dead", []ClaimKind{ClaimKindMCPCI}) Expect(err).ToNot(HaveOccurred()) ageInstance(db, "inst-dead", 10*time.Minute) @@ -546,7 +546,7 @@ var _ = Describe("The dispatch loop", func() { It("does not take work a live replica is still holding", func() { enqueue(ClaimKindMCPCI, JobEvent{JobID: "j-busy"}) - Expect(cluster.NewRegistry(db).Register(ctx, "inst-busy", "10.0.0.8:8080", "v1")).To(Succeed()) + Expect(cluster.NewRegistry(db).Register(ctx, "inst-busy", "10.0.0.8:8080", "v1", "")).To(Succeed()) _, err := ClaimNext(ctx, db, "inst-busy", []ClaimKind{ClaimKindMCPCI}) Expect(err).ToNot(HaveOccurred()) ageClaim(db, "j-busy-irrelevant", time.Hour) // no-op: proves nothing is keyed on age diff --git a/core/services/nodes/agent_control_test.go b/core/services/nodes/agent_control_test.go index 773460786602..1359ee7b5710 100644 --- a/core/services/nodes/agent_control_test.go +++ b/core/services/nodes/agent_control_test.go @@ -156,8 +156,8 @@ var _ = Describe("AgentControlClient", func() { registry, err = nodes.NewNodeRegistry(db) Expect(err).ToNot(HaveOccurred()) clusterReg = cluster.NewRegistry(db) - Expect(clusterReg.Register(ctx, selfInstance, "10.0.0.1:8080", "v1")).To(Succeed()) - Expect(clusterReg.Register(ctx, peerInstance, "10.0.0.2:8080", "v1")).To(Succeed()) + Expect(clusterReg.Register(ctx, selfInstance, "10.0.0.1:8080", "v1", "")).To(Succeed()) + Expect(clusterReg.Register(ctx, peerInstance, "10.0.0.2:8080", "v1", "")).To(Succeed()) mine = cluster.NewTunnelRegistry(clusterReg, selfInstance) theirs = cluster.NewTunnelRegistry(clusterReg, peerInstance) diff --git a/core/services/nodes/health_test.go b/core/services/nodes/health_test.go index 9a6d8172a8f9..26400d9417b6 100644 --- a/core/services/nodes/health_test.go +++ b/core/services/nodes/health_test.go @@ -455,7 +455,7 @@ var _ = Describe("HealthMonitor and a worker whose tunnel is gone", func() { Expect(err).ToNot(HaveOccurred()) Expect(cluster.Migrate(ctx, db)).To(Succeed()) clusterR = cluster.NewRegistry(db) - Expect(clusterR.Register(ctx, instance, "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(clusterR.Register(ctx, instance, "10.0.0.1:8080", "v1", "")).To(Succeed()) departed = nil departures = NewDepartureNotifier() departures.OnDeparture("spec", func(node DepartedNode) { departed = append(departed, node) }) diff --git a/core/services/nodes/router_liveness_test.go b/core/services/nodes/router_liveness_test.go index a7ced6d30e3c..f1a8743fb254 100644 --- a/core/services/nodes/router_liveness_test.go +++ b/core/services/nodes/router_liveness_test.go @@ -242,7 +242,7 @@ var _ = Describe("Scheduling against the cluster registry that answers presence" db = testutil.SetupTestDB() Expect(cluster.Migrate(ctx, db)).To(Succeed()) clusterR = cluster.NewRegistry(db) - Expect(clusterR.Register(ctx, instance, "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(clusterR.Register(ctx, instance, "10.0.0.1:8080", "v1", "")).To(Succeed()) reg = &fakeModelRouter{} router = NewSmartRouter(reg, SmartRouterOptions{Presence: clusterR, ReconnectGrace: grace}) }) @@ -337,7 +337,7 @@ var _ = Describe("Eviction and a worker whose tunnel is gone", func() { Expect(err).ToNot(HaveOccurred()) Expect(cluster.Migrate(ctx, db)).To(Succeed()) clusterR = cluster.NewRegistry(db) - Expect(clusterR.Register(ctx, instance, "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(clusterR.Register(ctx, instance, "10.0.0.1:8080", "v1", "")).To(Succeed()) router = NewSmartRouter(registry, SmartRouterOptions{DB: db, Presence: clusterR, ReconnectGrace: grace}) }) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 7497d0e558df..a3a930d56c61 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -96,7 +96,9 @@ That only holds while the database is on **another host**. If PostgreSQL runs on ``` ERROR This replica is not registered in the cluster: no advertised address. Peers cannot reach it, - and any worker whose tunnel lands here will be unroutable from every other replica + any worker whose tunnel lands here will be unroutable from every other replica, and this + replica cannot relay OUT either, because a peer link is authenticated by the credential an + instance row publishes and this replica has no row ``` The replica keeps serving every request that reaches it directly. What it cannot do is be reached by another replica, and on a multi-replica deployment that is worse than it sounds: a **worker whose tunnel lands on this replica is unroutable from every other replica**, because the ownership lookup only accepts an owner that is registered and live. This replica serves that worker fine; the others answer requests for it with `no route from this replica to that worker`. Behind a round-robin load balancer with N replicas, that is (N-1)/N of the traffic for that worker. @@ -118,10 +120,39 @@ environment: LOCALAI_DISTRIBUTED_ADVERTISE_ADDR: "10.0.1.7:8080" # or the pod IP, service DNS name, etc. ``` -The peer link is served at `/api/cluster/peer` and authenticates with `LOCALAI_REGISTRATION_TOKEN`, the same shared secret workers register with. Replicas that disagree about it cannot link. A replica that stops heartbeating for 30 seconds is dropped from the table by the others, along with the worker-connection rows it owned. +The peer link is served at `/api/cluster/peer`. A replica that stops heartbeating for 30 seconds is dropped from the table by the others, along with the worker-connection rows it owned. + +#### A peer proves which replica it is + +The route checks **two** credentials, and a dial needs both. + +| Credential | Sent as | Says | +|---|---|---| +| `LOCALAI_REGISTRATION_TOKEN` | `Authorization: Bearer ` | the dialler belongs to this deployment | +| The replica's own peer credential | `X-LocalAI-Peer-Token: ` | the dialler **is** the replica named in `?id=` | + +Each replica mints its own peer credential at startup, publishes only its SHA-256 in the `instances` table beside its address, and never sends the plaintext anywhere but the peer dial itself. The receiving replica resolves `?id=` to that row and compares. This is the same shape as the [worker tunnel credential](#worker-tunnels), in the stronger direction: a worker's credential is minted by the frontend and handed to it once, while a replica's never leaves the process that made it. + +Nothing needs configuring and nothing needs rotating. A restart mints a new secret, and the same registration that republishes a replica's address republishes the hash beside it. + +What that closes: holding `LOCALAI_REGISTRATION_TOKEN` - which every worker does - no longer lets its holder open a peer link **as some other replica**. It can therefore no longer relay through that link to every worker tunnel a replica owns, no longer displace a real replica's inbound link by declaring its id, and no longer point the per-session receive window (roughly 31 GiB of unread data per session) at a replica of its choosing. Only replicas registered in the `instances` table, each proving its own row, can open a peer link at all. + +What it does not close: replica-to-replica traffic is not encrypted, so anything that can read the wire between two replicas can read a credential off it, exactly as it could read the registration token. Keep `/api/cluster/peer` on a network only your replicas reach, and keep `LOCALAI_REGISTRATION_TOKEN` per-deployment. {{% notice note %}} -**The peer link has no per-replica credential yet.** It checks the shared registration token and takes the replica id in `?id=` on trust. Anything already holding that token - every worker holds it - can therefore open a peer link, relay through it to every worker tunnel a replica owns, by declaring another replica's id displace that replica's inbound link, and hold sessions open against the per-session receive window, which the peer-link code sizes at roughly 31 GiB of unread data per session and which on this route is also a memory budget an attacker can point at one replica. Treat `LOCALAI_REGISTRATION_TOKEN` as a cluster-wide secret with the blast radius of the whole fleet: give it its own value per deployment, do not reuse it elsewhere, and keep `/api/cluster/peer` on a network only your replicas and workers can reach. Per-replica credentials for this route are planned. +**A replica with no advertised address cannot peer in either direction.** It has no row in the `instances` table, so it has no credential published, so peers refuse its dials as an unproven identity - on top of already being unreachable itself. The startup error above names this. Set `LOCALAI_DISTRIBUTED_ADVERTISE_ADDR`. +{{% /notice %}} + +{{% notice warning %}} +**A replica that presents no peer credential is refused, not waved through.** There is no fallback to the shared token alone: an old replica and an attacker send exactly the same thing, so accepting one accepts the other. During a rolling frontend upgrade this means an **old** replica cannot open a peer link to an **upgraded** one, and each refusal is logged by the upgraded replica: + +``` +WARN Refusing a peer link: the dialling replica presented no peer credential. It is running a + release from before per-replica peer identity, or it never registered a credential of its + own. Upgrade it; this replica will not accept an unproven peer id peer= +``` + +The dialling side logs the matching `A peer refused this replica's credentials`. The window closes as each frontend restarts, and it is bounded by the same [frontend-first rollout](#upgrade-the-frontends-first) the tunnel already requires. A refused peer is an **authorization failure**, never node absence: nothing is rescheduled, nothing is reaped, and requests that cannot be relayed during the window fail with `no route from this replica to that worker`. {{% /notice %}} ### Cross-replica in-memory state @@ -256,7 +287,7 @@ A worker can open one long-lived, multiplexed tunnel to the frontend instead of #### Each worker has its own tunnel credential -The dial is authenticated against **that node's own tunnel credential**, which is not the registration token. Registration mints a fresh random secret per node, returns the plaintext once in the registration response as `tunnel_token`, and stores only its SHA-256. So a leaked registration token no longer opens a tunnel: an attacker who has it, and who knows a node ID, still cannot authenticate as that worker. +The dial is authenticated against **that node's own tunnel credential**, which is not the registration token. Registration mints a fresh random secret per node, returns the plaintext once in the registration response as `tunnel_token`, and stores only its SHA-256. So a leaked registration token no longer opens a tunnel: an attacker who has it, and who knows a node ID, still cannot authenticate as that worker. It cannot reach a worker through the [peer link](#a-peer-proves-which-replica-it-is) either, which is authenticated per replica in the same way. A worker that presents a credential belonging to no node, or names a node ID the frontend has never seen, is refused with `401` before the WebSocket upgrade happens. A node still awaiting admin approval is refused with `403`. A frontend that cannot read its node table answers `500` rather than `401`, so a worker retries instead of re-registering under a new identity. @@ -462,6 +493,7 @@ That distinction is the whole point rather than a nicety. A scheduler told that - **Frontends first (correct).** Old workers keep running, keep heartbeating and keep their `node_models` rows: the new frontend reports them as unroutable rather than as gone, so nothing is rescheduled and nothing is reaped. What fails is requests for models on a worker that has not been restarted yet. That is a real degraded window, but it is bounded by how fast you roll the workers, it heals itself as each one comes back, and no state is lost. - **What you will see while it lasts:** requests for models on a not-yet-restarted worker fail with "no route to the worker", while `GET /api/nodes` still shows that node healthy and heartbeating and its models still listed. Restart the worker and it clears. Nothing needs fixing; you are watching the window close. + - **While the frontends themselves are rolling**, a frontend you have not restarted yet cannot open a [peer link](#a-peer-proves-which-replica-it-is) to one you have: it holds no peer credential and the upgraded replica refuses unproven ids. An upgraded replica dialling an older one still works, so the loss is one-directional. What it costs is relayed requests that land on a not-yet-restarted replica for a worker an upgraded replica owns; they fail with `no route from this replica to that worker`, which is a routing fact and not absence, so nothing is rescheduled or reaped. Both sides log it by name. Restart the remaining frontends and it clears. - **Workers first (this fails, do not do it).** An old frontend has no `/api/cluster/connect` route for the worker to dial *and* rejects the new worker's registration outright, because the worker no longer sends an address and the old frontend requires one. A 4xx is a verdict rather than an outage, so the worker reports the reason on the **first** attempt and exits instead of retrying. Every worker you restart is a worker you take out of the fleet until the frontends are upgraded. - **What you will see if you do it anyway:** each restarted worker exits within a second or two of starting, with diff --git a/tests/e2e/distributed/agent_distributed_test.go b/tests/e2e/distributed/agent_distributed_test.go index d044eb89afeb..dc0ffb0808ac 100644 --- a/tests/e2e/distributed/agent_distributed_test.go +++ b/tests/e2e/distributed/agent_distributed_test.go @@ -165,7 +165,7 @@ var _ = Describe("Phase 3: Agent Conversations & SSE", Label("Distributed"), fun registry, err := nodes.NewNodeRegistry(db) Expect(err).ToNot(HaveOccurred()) clusterReg := cluster.NewRegistry(db) - Expect(clusterReg.Register(infra.Ctx, replica, "10.0.0.1:8080", "v1")).To(Succeed()) + Expect(clusterReg.Register(infra.Ctx, replica, "10.0.0.1:8080", "v1", "")).To(Succeed()) tunnels := cluster.NewTunnelRegistry(clusterReg, replica) // The worker's own bridge, and the run registered on it. This is diff --git a/tests/e2e/distributed/cluster_peerlink_test.go b/tests/e2e/distributed/cluster_peerlink_test.go index 7c492c16b9e7..58d3a47ba4bc 100644 --- a/tests/e2e/distributed/cluster_peerlink_test.go +++ b/tests/e2e/distributed/cluster_peerlink_test.go @@ -5,11 +5,14 @@ import ( "fmt" "io" "net" + "net/http" + "net/url" "strings" "time" clustersvc "github.com/mudler/LocalAI/core/services/cluster" + "github.com/gorilla/websocket" "github.com/libp2p/go-yamux/v5" . "github.com/onsi/ginkgo/v2" @@ -129,6 +132,20 @@ func awaitReplicas(roster *instanceRoster, addrs ...string) { Should(ConsistOf(addrs), roster.describe) } +// joinAsPeer gives this spec process a replica identity of its own: a freshly +// minted credential, with only its hash published in the instances table. +// +// It exists because the peer route no longer takes ?id= on trust. A spec that +// plays a sibling replica has to join the cluster the way a replica does, which +// is the point rather than a chore: the address it publishes is never dialled, +// but the credential it publishes is what every dial below is checked against. +func joinAsPeer(roster *instanceRoster, id string) clustersvc.PeerCredential { + GinkgoHelper() + cred := clustersvc.NewPeerCredential() + Expect(roster.registry.Register(roster.ctx, id, "127.0.0.1:1", "e2e", cred.Hash())).To(Succeed()) + return cred +} + var _ = Describe("Cluster peer link", Label("Distributed"), Label("Cluster"), func() { It("publishes an address for every replica that peers can actually dial", func() { // A wrong implementation registers nothing (the whole of phase 1 had no @@ -175,7 +192,11 @@ var _ = Describe("Cluster peer link", Label("Distributed"), Label("Cluster"), fu ctx, cancel := context.WithTimeout(context.Background(), peerDialTimeout) defer cancel() - pool := clustersvc.NewPeerPool("e2e-peer", c.RegistrationToken(), roster.registry) + // The spec joins as a replica before it dials as one. Without this the + // frontend refuses the link: the id would be a claim it cannot check, + // which is exactly what a peer dial is now required to prove. + selfCred := joinAsPeer(roster, "e2e-peer") + pool := clustersvc.NewPeerPool("e2e-peer", c.RegistrationToken(), selfCred, roster.registry) DeferCleanup(pool.Close) // OpenStream is only acknowledged once the far side accepts, so this @@ -214,9 +235,11 @@ var _ = Describe("Cluster peer link", Label("Distributed"), Label("Cluster"), fu Expect(err).To(SatisfyAny(MatchError(io.EOF), MatchError(yamux.ErrStreamReset)), "the peer refused the stream and then left it open: %v", err) - // The same dial with the wrong credentials must be refused, otherwise - // the success above says nothing about authentication. - impostor := clustersvc.NewPeerPool("e2e-peer", "not-the-cluster-token", roster.registry) + // The same dial with the wrong shared token must be refused, otherwise + // the success above says nothing about authentication. The identity + // credential is correct here, so what this pins is that adding identity + // did not replace the token check with it. + impostor := clustersvc.NewPeerPool("e2e-peer", "not-the-cluster-token", selfCred, roster.registry) DeferCleanup(impostor.Close) _, err = impostor.Open(ctx, peerID) Expect(err).To(MatchError(clustersvc.ErrPeerUnreachable)) @@ -224,6 +247,120 @@ var _ = Describe("Cluster peer link", Label("Distributed"), Label("Cluster"), fu "a peer refusing credentials is a live peer; reading it as absence is how a replica evicts healthy workers") }) + It("refuses a peer that declares a replica id it cannot prove, and leaves that replica's link alone", func() { + // The security gap this route carried for the whole phase, closed here + // against real processes. The attacker's position is the realistic one: + // it holds LOCALAI_REGISTRATION_TOKEN, which every worker in the + // deployment holds, and it knows a replica id, which any log line or + // roster read gives it. Before per-replica credentials that was the + // entire check, so it could open a link and relay through it to every + // worker tunnel the replica it dialled owns, and by declaring somebody + // else's id evict that replica's inbound link at will. + // + // It is also the case the cheap fix does not reach. Validating ?id= + // against the instances table passes every dial below except the last: + // the ids ARE in the table. Only a secret the impostor does not hold + // refuses them. + c, dsn := startClusterOnFreshDB(2, 0) + + roster := newInstanceRoster(openClusterDB(dsn)) + awaitReplicas(roster, hostPortOf(c.FrontendURL(0)), hostPortOf(c.FrontendURL(1))) + + target := roster.idAt(hostPortOf(c.FrontendURL(1))) + victim := roster.idAt(hostPortOf(c.FrontendURL(0))) + Expect(target).ToNot(BeEmpty()) + Expect(victim).ToNot(BeEmpty()) + + ctx, cancel := context.WithTimeout(context.Background(), peerDialTimeout) + defer cancel() + + // A legitimate link first, so there is something for an impostor to + // displace and so the refusals below are not passing because the route + // is broken for everyone. + selfCred := joinAsPeer(roster, "e2e-peer") + legit := clustersvc.NewPeerPool("e2e-peer", c.RegistrationToken(), selfCred, roster.registry) + DeferCleanup(legit.Close) + held, err := legit.Open(ctx, target) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = held.Close() }) + + // Declaring a REAL frontend replica's id. This is the dial that used to + // succeed, and the one a scheduler must never be told anything about a + // worker from. + asReplica := clustersvc.NewPeerPool(victim, c.RegistrationToken(), clustersvc.NewPeerCredential(), roster.registry) + DeferCleanup(asReplica.Close) + _, err = asReplica.Open(ctx, target) + Expect(err).To(MatchError(clustersvc.ErrPeerRejected), + "a frontend accepted a peer link from something that could not prove it was the replica it named") + Expect(err).ToNot(MatchError(clustersvc.ErrInstanceNotFound), + "a rejected peer is an authorization failure; read as absence it makes the scheduler reap rows and evict models") + Expect(err).ToNot(MatchError(clustersvc.ErrNoConnection), + "a rejected peer must never be readable as a worker with no connection") + + // And declaring the id whose link is currently held, which is the + // eviction half: SessionStore keeps one link per id, so an accepted + // dial closes whatever was there. + displacer := clustersvc.NewPeerPool("e2e-peer", c.RegistrationToken(), clustersvc.NewPeerCredential(), roster.registry) + DeferCleanup(displacer.Close) + _, err = displacer.Open(ctx, target) + Expect(err).To(MatchError(clustersvc.ErrPeerRejected)) + + // The link opened before the attempts still carries a request. It is + // the stream taken above, not a fresh one: a fresh Open would simply + // re-dial and succeed, which would say nothing about whether the old + // session survived. + Expect(held.SetWriteDeadline(time.Now().Add(peerRefusalTimeout))).To(Succeed()) + Expect(clustersvc.WriteRelayRequest(held, unheldNodeID, peerRefusalTimeout)).To(Succeed(), + "the impostor closed the real peer's link") + Expect(held.SetReadDeadline(time.Now().Add(peerRefusalTimeout))).To(Succeed()) + Expect(clustersvc.ReadRelayReply(held)).To(MatchError(clustersvc.ErrNotOwner), + "the impostor closed the real peer's link") + }) + + It("refuses a peer that presents no credential at all, the way a replica running an older release dials", func() { + // The mixed-version window, made explicit. An upgraded frontend refuses + // a peer that sends no credential, rather than falling back to the + // shared token, and the refusal is a plain 401 before the WebSocket + // upgrade so the dialler reads a status and not a framing error. + // + // This is dialled raw rather than through PeerPool, because PeerPool on + // this release always sends the header: an old replica is the only + // thing that does not, and it is what this reproduces. + c, dsn := startClusterOnFreshDB(1, 0) + + roster := newInstanceRoster(openClusterDB(dsn)) + awaitReplicas(roster, hostPortOf(c.FrontendURL(0))) + target := roster.idAt(hostPortOf(c.FrontendURL(0))) + Expect(target).ToNot(BeEmpty()) + + // A credential that would be perfectly valid if it were sent. + joinAsPeer(roster, "e2e-old-peer") + + endpoint := url.URL{ + Scheme: "ws", + Host: hostPortOf(c.FrontendURL(0)), + Path: clustersvc.PeerPath, + RawQuery: url.Values{"id": []string{"e2e-old-peer"}}.Encode(), + } + header := http.Header{} + header.Set("Authorization", "Bearer "+c.RegistrationToken()) + + dialer := &websocket.Dialer{HandshakeTimeout: peerDialTimeout} + conn, resp, err := dialer.Dial(endpoint.String(), header) + if conn != nil { + DeferCleanup(func() { _ = conn.Close() }) + } + Expect(resp).ToNot(BeNil()) + if resp.Body != nil { + DeferCleanup(func() { _ = resp.Body.Close() }) + } + Expect(err).To(HaveOccurred(), + "an upgraded frontend accepted a peer link carrying only the shared token") + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + Expect(resp.Header.Get("Upgrade")).To(BeEmpty(), + "the frontend upgraded a dial it then refused; a peer reads the status, not a WebSocket error") + }) + It("stops being dialled as soon as a replica shuts down cleanly", func() { // The crash case below is handled by the sweeper, at the cost of a // whole liveness window of peers dialling a corpse. A rolling update is @@ -250,7 +387,7 @@ var _ = Describe("Cluster peer link", Label("Distributed"), Label("Cluster"), fu // replica said it was going. A caller may act on this. ctx, cancel := context.WithTimeout(context.Background(), peerDialTimeout) defer cancel() - pool := clustersvc.NewPeerPool("e2e-peer", c.RegistrationToken(), roster.registry) + pool := clustersvc.NewPeerPool("e2e-peer", c.RegistrationToken(), joinAsPeer(roster, "e2e-peer"), roster.registry) DeferCleanup(pool.Close) _, err := pool.Open(ctx, departingID) Expect(err).To(MatchError(clustersvc.ErrInstanceNotFound)) @@ -305,7 +442,7 @@ var _ = Describe("Cluster peer link", Label("Distributed"), Label("Cluster"), fu // case that matters: the peer is KNOWN and will not answer. dialCtx, cancel := context.WithTimeout(ctx, peerDialTimeout) defer cancel() - pool := clustersvc.NewPeerPool("e2e-peer", c.RegistrationToken(), roster.registry) + pool := clustersvc.NewPeerPool("e2e-peer", c.RegistrationToken(), joinAsPeer(roster, "e2e-peer"), roster.registry) DeferCleanup(pool.Close) _, err = pool.Open(dialCtx, doomedID) Expect(err).To(MatchError(clustersvc.ErrPeerUnreachable)) diff --git a/tests/e2e/distributed/job_dispatch_test.go b/tests/e2e/distributed/job_dispatch_test.go index 070a186d43cb..49fc5ed1ab8f 100644 --- a/tests/e2e/distributed/job_dispatch_test.go +++ b/tests/e2e/distributed/job_dispatch_test.go @@ -51,7 +51,7 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { const owner = "dispatch-instance" // A replica that is not registered may not claim: its claims could // not be told from ones a dead replica left. - Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8080", "v1")).To(Succeed()) + Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8080", "v1", "")).To(Succeed()) dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, owner) @@ -98,7 +98,7 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { It("leaves a plain task job failed with a reason, since no worker in this deployment serves that kind", func() { Expect(cluster.Migrate(infra.Ctx, db)).To(Succeed()) const owner = "plain-instance" - Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8081", "v1")).To(Succeed()) + Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8081", "v1", "")).To(Succeed()) dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, owner) task := &jobs.TaskRecord{UserID: "u1", Name: "plain-task", Model: "m1", Prompt: "p1"} diff --git a/tests/e2e/distributed/job_distribution_test.go b/tests/e2e/distributed/job_distribution_test.go index 61853c130499..f23d56c65321 100644 --- a/tests/e2e/distributed/job_distribution_test.go +++ b/tests/e2e/distributed/job_distribution_test.go @@ -174,7 +174,7 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { It("enqueues a claim and drives it on a worker, persisting what the worker answered", func() { Expect(cluster.Migrate(infra.Ctx, db)).To(Succeed()) const owner = "test-instance" - Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8090", "v1")).To(Succeed()) + Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8090", "v1", "")).To(Succeed()) dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, owner) @@ -202,7 +202,7 @@ var _ = Describe("Phase 2: Jobs & Tasks", Label("Distributed"), func() { It("returns a claim to the pool when the dispatch obtained no answer, rather than losing the work", func() { Expect(cluster.Migrate(infra.Ctx, db)).To(Succeed()) const owner = "lossy-instance" - Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8091", "v1")).To(Succeed()) + Expect(cluster.NewRegistry(db).Register(infra.Ctx, owner, "127.0.0.1:8091", "v1", "")).To(Succeed()) dispatcher := jobs.NewDispatcher(store, infra.Bus(), db, owner) task := &jobs.TaskRecord{UserID: "u1", Name: "lossy-test", Model: "m1", Prompt: "p1"} From 9f1939c8d0ea63ce96b047da012364306fd5ec53 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 6 Sep 2026 05:05:46 +0000 Subject: [PATCH 104/153] fix(distributed): say that skills and collections are replica-local Skills and RAG collections had no cross-replica invalidation, and the two builders that would have published one were deleted earlier in this branch because nothing called them. Wiring one now would be wrong, not merely late. Both features are derived entirely from the frontend's own state directory. A skills.Service indexes /skills, a collections backend enumerates /collections and holds one handle per collection it found there, and no replica reads or writes another replica's copy of either. In distributed mode PostgreSQL carries a skill's NAME and description in skills_metadata, and nothing else: Get, Search, Export and the resource verbs all read local files. So a peer told to drop a cache entry would rebuild it from a directory that does not hold the change. For a postgres-engine collection it would be worse than a no-op, since re-deriving one on a replica with no local index file yields a collection that answers with an empty file list against a populated vector store. What is missing is shared storage, not a broadcast. Recorded rather than left silent: the two cache fields say why nothing invalidates them, a distributed frontend logs the limitation once at startup, and the docs name the two deployments that avoid it. The new spec pins the premise, so a change that moved either directory onto storage every replica mounts reddens and the decision gets taken again. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/agentpool/agent_pool.go | 16 ++++++ .../agentpool/replica_local_state_test.go | 54 +++++++++++++++++++ core/services/agentpool/user_services.go | 22 +++++--- docs/content/features/distributed-mode.md | 40 ++++++++++++++ 4 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 core/services/agentpool/replica_local_state_test.go diff --git a/core/services/agentpool/agent_pool.go b/core/services/agentpool/agent_pool.go index 0c515cef601e..0f755e9773cc 100644 --- a/core/services/agentpool/agent_pool.go +++ b/core/services/agentpool/agent_pool.go @@ -241,6 +241,22 @@ func (s *AgentPoolService) startDistributed(ctx context.Context, apiURL, apiKey collectionsBackend, _ := collections.NewInProcessBackend(s.buildCollectionsConfig(apiURL, apiKey, collectionDBPath, fileAssets)) s.collectionsBackend = collectionsBackend + // Said once, at startup, because it is the one thing about skills and + // collections that a multi-replica deployment cannot discover from its own + // behaviour: both are served out of THIS replica's state directory and + // nothing replicates either of them. The skills list is the exception and + // it is the misleading one, because it comes from skills_metadata and is + // therefore the same on every replica, while reading, searching, exporting + // or fetching the resources of that same skill works only here. + // + // There is no broadcast that would fix it and none is wired: a peer told to + // invalidate would re-read a directory that does not hold the change. See + // "Skills and collections are NOT replicated" in + // docs/content/features/distributed-mode.md for the two deployments that + // avoid it. + xlog.Warn("Skills and collections are served from this replica's own state directory and are not replicated to peer frontends. A skill written here is listed everywhere but readable only here, and a collection created here is invisible elsewhere. Mount one shared state directory on every frontend replica, or route the skills and collections endpoints to a single replica", + "stateDir", stateDir, "collectionDBPath", collectionDBPath) + // User-scoped storage dataDir := cmp.Or(s.appConfig.DataPath, s.appConfig.DynamicConfigsDir) s.users.userStorage = NewUserScopedStorage(stateDir, dataDir) diff --git a/core/services/agentpool/replica_local_state_test.go b/core/services/agentpool/replica_local_state_test.go new file mode 100644 index 000000000000..b647f7c33738 --- /dev/null +++ b/core/services/agentpool/replica_local_state_test.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT + +package agentpool_test + +import ( + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/agentpool" +) + +// The premise behind a deliberate non-decision: skills and collections get NO +// cross-replica invalidation, because both are derived from a directory that +// belongs to one frontend replica and no other replica reads or writes it. An +// invalidation broadcast cannot make a change visible in state that was never +// shared, so wiring one would report coherence the deployment does not have. +// +// This spec pins the premise rather than the absence. A change that moved +// either directory onto storage every replica mounts would redden it, and that +// is exactly the change after which the decision recorded in +// docs/content/features/distributed-mode.md ("Skills and collections are NOT +// replicated") has to be taken again. +var _ = Describe("skills and collections storage locality", func() { + const ( + stateDir = "/var/lib/localai/agents" + dataDir = "/var/lib/localai/data" + userID = "3f2504e0-4f89-11d3-9a0c-0305e82c3301" + ) + + var storage *agentpool.UserScopedStorage + + BeforeEach(func() { + storage = agentpool.NewUserScopedStorage(stateDir, dataDir) + }) + + It("keeps the cluster-wide skills and collections directories inside this replica's state directory", func() { + Expect(storage.SkillsDir("")).To(Equal(filepath.Join(stateDir, "skills"))) + Expect(storage.CollectionsDir("")).To(Equal(filepath.Join(stateDir, "collections"))) + }) + + It("keeps a tenant's skills and collections directories inside this replica's state directory", func() { + Expect(storage.SkillsDir(userID)).To(Equal(filepath.Join(stateDir, "users", userID, "skills"))) + Expect(storage.CollectionsDir(userID)).To(Equal(filepath.Join(stateDir, "users", userID, "collections"))) + }) + + // The assets a skill resource or an uploaded collection file lands in are + // the same story: a peer that learned a collection had changed still could + // not read the file the change is about. + It("keeps a tenant's assets inside this replica's state directory", func() { + Expect(storage.AssetsDir(userID)).To(Equal(filepath.Join(stateDir, "users", userID, "assets"))) + }) +}) diff --git a/core/services/agentpool/user_services.go b/core/services/agentpool/user_services.go index c0cbcfef7b40..4743f4aa54fd 100644 --- a/core/services/agentpool/user_services.go +++ b/core/services/agentpool/user_services.go @@ -16,12 +16,22 @@ import ( // UserServicesManager lazily creates per-user service instances for // collections, skills, and jobs. type UserServicesManager struct { - mu sync.RWMutex - storage *UserScopedStorage - appConfig *config.ApplicationConfig - modelLoader *model.ModelLoader - configLoader *config.ModelConfigLoader - evaluator *templates.Evaluator + mu sync.RWMutex + storage *UserScopedStorage + appConfig *config.ApplicationConfig + modelLoader *model.ModelLoader + configLoader *config.ModelConfigLoader + evaluator *templates.Evaluator + // collectionsCache and skillsCache hold one SERVICE HANDLE per user, and + // neither is a cache of replicated data. Nothing invalidates them across + // replicas, and nothing should: both handles derive everything they answer + // with from this replica's own state directory (see + // UserScopedStorage.SkillsDir and CollectionsDir), which no other replica + // reads or writes. A peer told to drop an entry would rebuild it from a + // directory that does not contain the change, so the invalidation could + // not make that change visible. What is missing is shared storage, not a + // broadcast; see "Skills and collections are NOT replicated" in + // docs/content/features/distributed-mode.md. collectionsCache map[string]collections.Backend skillsCache map[string]*skills.Service jobsCache map[string]*AgentJobService diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index a3a930d56c61..6aaad75cffa5 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -209,6 +209,46 @@ path for every request whose prefix changed, so the frontend refuses to start and says so rather than running slowly and quietly. Nothing here drops an observation to stay under the cap. +### Skills and collections are NOT replicated + +Agent **skills** and RAG **collections** are the two features whose state is not +on this list, and they are absent from it deliberately rather than by omission. +Neither has a cross-replica invalidation, and neither should be given one, +because there is nothing coherent for an invalidation to say. + +Both are backed by the frontend's own state directory: + +| State | Where it lives | What is shared | +|-------|----------------|----------------| +| Skill content, skill resources, git-repo clones, the skill search index | `/skills`, or `/users//skills` | nothing | +| Collection contents and its file list | `/collections/collection_.json` plus the assets directory | nothing | +| Skill name, description and source | `skills_metadata` in PostgreSQL | the row | +| Collection vectors, with the `postgres` vector engine | the vector table in PostgreSQL | the vectors | + +A frontend replica writes a skill or a collection to its OWN disk. No replica +copies it, and no broadcast could: a peer told to drop a cache entry would +re-read a directory that does not contain the change, so the invalidation would +be a guaranteed no-op for skills and, for a `postgres` collection, worse than +one, since re-deriving the collection on a replica that has no local index file +would produce a collection that answers with an empty file list against a +populated vector store. The cache is not what is missing here; the shared +storage is. + +What that means when you run more than one frontend replica: + +- A skill created on one replica is **listed** on every replica, because the + list comes from `skills_metadata`. Reading it, searching it, exporting it or + fetching its resources works only on the replica that wrote it. +- A collection created on one replica is not listed, searched or uploaded to on + any other replica. + +Two deployments avoid it. Mount ONE `ReadWriteMany` volume as the state +directory (`LOCALAI_AGENT_POOL_STATE_DIR`, else `LOCALAI_DATA_PATH`) on every +frontend replica, so all replicas read and write the same files; or route +`/api/agents/skills*` and `/api/agents/collections*` to a single replica. A +frontend running distributed logs this limitation once at startup, so it is +visible in a deployment that did neither. + ### Job and agent streams across replicas The same carrier moves the traffic whose subscriber is an open HTTP response rather than a cache: a job's progress stream, its result, its cancel, an agent's SSE events, an agent cancel and an Open Responses cancel. From c56fcec2f69efa77c5b29b87a81dfe5e2bbe9783 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 6 Sep 2026 05:05:58 +0000 Subject: [PATCH 105/153] fix(worker): check a gRPC port is free before handing it out The backend port allocator allocated from its own bookkeeping alone. That bookkeeping records what this worker did with a port, and the collision it cannot see is with something this worker never did: the default base port is 50051, inside Linux's default ephemeral range of 32768 to 60999, so the kernel hands ports in this range to outbound connections and to anything that binds port 0. A backend handed one of those dies on bind, and the frontend sees a backend that will not start. Every candidate is now probed by binding the exact address the backend will listen on, in all four allocation branches: the key's own port, the free pool, a grown port and a stolen one. Probing the free pool matters as much as probing a grown port, because a port this worker released is exactly as available to the kernel as one it never used. A candidate that fails the probe is quarantined rather than blacklisted, since whatever holds it is usually an ephemeral connection that gives it back, and its affinity claim is dropped so an unbindable port does not stay reserved for the key that last held it. Exhaustion now says how many candidates were skipped, which is what tells an operator "something else is in my range" from "my range is too narrow". This does not remove the race and cannot: between the probe and the child's bind the kernel can still give the port away. It removes the far larger window in which the allocator hands out a port the kernel gave away minutes ago, which was the whole of the observed one-in-three harness flake. The e2e harness comment that recorded the missing check is corrected, and the docs say how to move the range out of the ephemeral one entirely. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/worker/port_bindable_test.go | 149 +++++++++++++++++++++ core/services/worker/supervisor.go | 128 ++++++++++++++++-- docs/content/features/distributed-mode.md | 28 +++- tests/e2e/distributed/cluster/cluster.go | 30 +++-- 4 files changed, 312 insertions(+), 23 deletions(-) create mode 100644 core/services/worker/port_bindable_test.go diff --git a/core/services/worker/port_bindable_test.go b/core/services/worker/port_bindable_test.go new file mode 100644 index 000000000000..3ee5eac3fca7 --- /dev/null +++ b/core/services/worker/port_bindable_test.go @@ -0,0 +1,149 @@ +package worker + +import ( + "net" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The allocator's bookkeeping records what THIS worker did with a port. The +// collision it could not see is with something this worker never did: the +// default base port, 50051, is inside Linux's default ephemeral range +// (32768-60999), so the kernel hands ports in this range to outbound +// connections and to anything that binds port 0. A port the allocator believes +// free is therefore not necessarily bindable, and the backend that is handed +// one dies on bind. +var _ = Describe("Backend gRPC port allocator, against real sockets", func() { + // listenOnAFreePort takes a real port and holds it for the spec, the way + // anything else on the host would. No injected seam: the point of this + // spec is that the allocator asks the KERNEL and not its own map. + listenOnAFreePort := func() int { + ln, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = ln.Close() }) + return ln.Addr().(*net.TCPAddr).Port + } + + It("does not hand out a port another process already holds", func() { + held := listenOnAFreePort() + s := &backendSupervisor{ + processes: map[string]*backendProcess{}, + minPort: held, + nextPort: held, + maxPort: held + 8, + portQuarantine: time.Hour, + } + + port := mustAllocate(s, "model#0") + Expect(port).NotTo(Equal(held), + "a port the kernel has already given to something else cannot be bound by a backend") + Expect(port).To(BeNumerically(">", held)) + }) + + It("refuses the whole range rather than handing out one held port", func() { + held := listenOnAFreePort() + s := &backendSupervisor{ + processes: map[string]*backendProcess{}, + minPort: held, + nextPort: held, + maxPort: held, + portQuarantine: time.Hour, + } + + _, err := s.allocatePort("model#0") + Expect(err).To(MatchError(ErrNoFreePort)) + // The operator has to be able to tell "my range is too narrow" from + // "something else on this host is sitting in my range". + Expect(err.Error()).To(ContainSubstring("already bound by something outside this worker")) + }) + + Describe("with the probe scripted", func() { + // takenPorts scripts which candidates the kernel refuses, so a spec can + // state the rule without depending on what else this host is running. + takenPorts := func(taken ...int) func(int) bool { + held := map[int]struct{}{} + for _, p := range taken { + held[p] = struct{}{} + } + return func(port int) bool { + _, ok := held[port] + return !ok + } + } + + It("passes over a released port the kernel took while it sat in the free pool", func() { + // The free pool is where this bites hardest. A port this worker + // released is exactly as available to the kernel as one it never + // used, so probing only the freshly grown ports would still hand + // out a stolen one. + bp := &backendProcess{port: 50051} + s := &backendSupervisor{ + processes: map[string]*backendProcess{"gone#0": bp}, + minPort: 50051, + nextPort: 50052, + maxPort: 50053, + // Both windows lapse at once, so the released port is an + // ORDINARY free port by the time the next allocation looks: out + // of quarantine and no longer claimed by the key that held it. + // Without the affinity window lapsing, step 2 skips it as owned + // and the probe this spec is about is never reached. + portQuarantine: time.Nanosecond, + portAffinityWindow: time.Nanosecond, + portIsFree: takenPorts(50051), + } + Expect(s.finishBackendStop("gone#0", bp, nil)).To(Succeed()) + Expect(s.freePortsAfterSweep()).To(ContainElement(50051), + "the port has to reach the free pool for this spec to be about the free pool") + + Expect(mustAllocate(s, "other#0")).NotTo(Equal(50051)) + }) + + It("offers a held port again once whatever held it gives it back", func() { + // Quarantined and not blacklisted. An ephemeral outbound connection + // releases its port; a port dropped for ever would be a permanent + // loss of range for a collision that lasted seconds. + free := takenPorts(50051) + s := &backendSupervisor{ + processes: map[string]*backendProcess{}, + minPort: 50051, + nextPort: 50051, + maxPort: 50052, + portQuarantine: time.Nanosecond, + portIsFree: func(port int) bool { return free(port) }, + } + + Expect(mustAllocate(s, "first#0")).To(Equal(50052)) + + // Whatever held 50051 has let go. + free = takenPorts() + Expect(mustAllocate(s, "second#0")).To(Equal(50051)) + }) + + It("does not go on reserving a port it could not bind", func() { + // Affinity is a preference for a port this worker can still bind. + // The claim is dropped when the probe refuses it, and the case that + // shows it is the one where no OTHER port is available either: a + // successful allocation overwrites the claim anyway, so a spec that + // let the range grow would pass with the drop removed. + bp := &backendProcess{port: 50051} + free := takenPorts() + s := &backendSupervisor{ + processes: map[string]*backendProcess{"model#0": bp}, + minPort: 50051, + nextPort: 50052, + maxPort: 50051, // the range is exactly this one port + portQuarantine: time.Nanosecond, + portIsFree: func(port int) bool { return free(port) }, + } + Expect(s.finishBackendStop("model#0", bp, nil)).To(Succeed()) + + free = takenPorts(50051) + _, err := s.allocatePort("model#0") + Expect(err).To(MatchError(ErrNoFreePort)) + Expect(s.portAffinity).NotTo(HaveKey("model#0"), + "a port this worker cannot bind must not stay reserved for the key that last held it") + }) + }) +}) diff --git a/core/services/worker/supervisor.go b/core/services/worker/supervisor.go index de273a586a08..f1ee66ffdafb 100644 --- a/core/services/worker/supervisor.go +++ b/core/services/worker/supervisor.go @@ -154,6 +154,11 @@ type backendSupervisor struct { portAffinity map[string]portOwnership portAffinityWindow time.Duration + // portIsFree reports whether a candidate port can actually be bound right + // now. Overridden only by tests; nil means bindableNow, which binds it. + // See allocatePort for why bookkeeping alone is not an answer. + portIsFree func(port int) bool + // quarantinedPorts holds ports whose process has terminated but which are // not yet safe to re-bind, and portQuarantine is how long they wait. // Overridden only by tests; zero means defaultPortQuarantine. @@ -295,16 +300,45 @@ func (s *backendSupervisor) portBounds() (int, int) { // branch means the worker is genuinely out of concurrent capacity, so the // warning's advice to widen the range is the right advice. // +// EVERY candidate is probed before it is handed out, because this allocator's +// bookkeeping is not a claim on the kernel. The default base port is 50051, +// which sits inside Linux's default ephemeral range (32768-60999), so the +// kernel can and does hand a port in this range to an outbound connection or +// to anything that binds port 0 while the allocator still believes it free. +// The bookkeeping cannot see that and never could: it records what this worker +// did, and the collision is with something this worker never did. A candidate +// that fails the probe is quarantined rather than blacklisted, since whatever +// holds it is usually an ephemeral connection that gives it back. +// // Callers must hold s.mu. func (s *backendSupervisor) allocatePort(key string) (int, error) { + port, skipped, err := s.allocateBindablePort(key) + if skipped > 0 { + // ONE line per allocation and not one per candidate: a range whose + // lower half the kernel is using would otherwise print thousands. + xlog.Warn("Skipped gRPC ports in this worker's range that something outside the worker already holds. The default base port sits inside Linux's ephemeral range (32768-60999), so the kernel can hand these ports to outbound connections; set LOCALAI_ADDR to a base port outside net.ipv4.ip_local_port_range to stop it", + "backend", key, "skipped", skipped, "allocated", port, "error", err) + } + return port, err +} + +// allocateBindablePort is allocatePort's body, and also reports how many +// candidates it had to skip because they were already bound. Callers must hold +// s.mu. +func (s *backendSupervisor) allocateBindablePort(key string) (int, int, error) { s.sweepQuarantine() s.sweepAffinity() minPort, maxPort := s.portBounds() + taken := 0 // 1. This key's own port, if it is back out of quarantine. if own, ok := s.portAffinity[key]; ok { if s.takeFreePort(own.port) { - return s.claimPort(key, own.port), nil + if s.portFree(own.port) { + return s.claimPort(key, own.port), taken, nil + } + taken++ + s.quarantineTakenPort(key, own.port) } } @@ -317,27 +351,105 @@ func (s *backendSupervisor) allocatePort(key string) (int, error) { continue } s.freePorts = slices.Delete(s.freePorts, i, i+1) - return s.claimPort(key, port), nil + if s.portFree(port) { + return s.claimPort(key, port), taken, nil + } + taken++ + s.quarantineTakenPort(key, port) } // 3. Grow into ports never handed out, staying inside the range. - if s.nextPort >= minPort && s.nextPort <= maxPort { + for s.nextPort >= minPort && s.nextPort <= maxPort { port := s.nextPort s.nextPort++ - return s.claimPort(key, port), nil + if s.portFree(port) { + return s.claimPort(key, port), taken, nil + } + taken++ + s.quarantineTakenPort(key, port) } // 4. Steal another key's port rather than refuse to start a backend. - if len(s.freePorts) > 0 { + for len(s.freePorts) > 0 { port := s.freePorts[len(s.freePorts)-1] s.freePorts = s.freePorts[:len(s.freePorts)-1] + if !s.portFree(port) { + taken++ + s.quarantineTakenPort(key, port) + continue + } xlog.Warn("gRPC port range is exhausted; reusing a port that belonged to another backend. A stale controller row for the previous owner could briefly misroute to this backend — raise LOCALAI_GRPC_MAX_PORT to restore headroom", "backend", key, "port", port, "previousOwner", owners[port], "min", minPort, "max", maxPort) - return s.claimPort(key, port), nil + return s.claimPort(key, port), taken, nil + } + + return 0, taken, fmt.Errorf("%w: %d-%d is fully consumed by %d running backend(s), %d port(s) still in quarantine and %d port(s) already bound by something outside this worker; raise LOCALAI_GRPC_MAX_PORT to widen the range", + ErrNoFreePort, minPort, maxPort, len(s.processes), len(s.quarantinedPorts), taken) +} + +// freePortsAfterSweep returns the free pool once expired quarantines and +// affinity claims have been swept. It exists so a spec can state that its +// scripted port really did reach the pool the branch under test reads, rather +// than being skipped as still-owned before that branch is entered. +func (s *backendSupervisor) freePortsAfterSweep() []int { + s.mu.Lock() + defer s.mu.Unlock() + s.sweepQuarantine() + s.sweepAffinity() + return append([]int(nil), s.freePorts...) +} + +// portFree reports whether a candidate can be handed out, deferring to the +// injected probe when a spec supplied one. +// +// The probe is asked for EVERY candidate and not only for grown ones. A port +// this worker released is exactly as available to the kernel as one it never +// used, so a port sitting in freePorts can have been given to an outbound +// connection while it sat there. +func (s *backendSupervisor) portFree(port int) bool { + if s.portIsFree != nil { + return s.portIsFree(port) + } + return bindableNow(port) +} + +// bindableNow reports whether the address a backend would listen on can be +// bound at this instant. +// +// It binds and closes rather than reading a table of open sockets, because the +// question is exactly "would the child's bind succeed", and a table read +// answers a different one on every platform. TOCTOU is real and is not what +// this closes: between this probe and the child's own bind the kernel can still +// hand the port to an outbound connection. What it removes is the far larger +// window in which this worker hands out a port the kernel gave away MINUTES +// ago, which is the whole of the observed failure. +func bindableNow(port int) bool { + ln, err := net.Listen("tcp", backendListenAddr(port)) + if err != nil { + return false } + if err := ln.Close(); err != nil { + xlog.Debug("Could not close the port probe listener", "port", port, "error", err) + } + return true +} - return 0, fmt.Errorf("%w: %d-%d is fully consumed by %d running backend(s) and %d port(s) still in quarantine; raise LOCALAI_GRPC_MAX_PORT to widen the range", - ErrNoFreePort, minPort, maxPort, len(s.processes), len(s.quarantinedPorts)) +// quarantineTakenPort parks a port that something outside this worker holds, so +// the next allocation does not offer it again before the quarantine lapses. +// +// Quarantine and not simply "drop it": the port is not this worker's to +// blacklist for ever, and whatever holds it is usually an ephemeral outbound +// connection that will release it. Parking it costs one retry per quarantine +// window instead of one probe per allocation. The affinity claim is dropped +// first, because a port this worker cannot bind must not go on being reserved +// for the key that last held it. +func (s *backendSupervisor) quarantineTakenPort(key string, port int) { + xlog.Debug("A gRPC port in this worker's range is already bound outside the worker; skipping it", + "backend", key, "port", port) + if own, ok := s.portAffinity[key]; ok && own.port == port { + delete(s.portAffinity, key) + } + s.releasePort(port) } // sweepAffinity drops claims whose window has lapsed, so their ports become diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 6aaad75cffa5..82ee26b97ab3 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -900,8 +900,8 @@ range does fill, backend starts fail with: ``` no free gRPC port in range: 50051-50150 is fully consumed by 100 running -backend(s) and 12 port(s) still in quarantine; raise LOCALAI_GRPC_MAX_PORT to -widen the range +backend(s), 12 port(s) still in quarantine and 0 port(s) already bound by +something outside this worker; raise LOCALAI_GRPC_MAX_PORT to widen the range ``` Raise `LOCALAI_GRPC_MAX_PORT` (or reduce how many models you schedule onto that @@ -909,6 +909,30 @@ worker). A value above 65535 is clamped, and a value below the base port is ignored in favour of the full range, so a typo degrades the setting rather than wedging every backend start on the node. +**The worker checks that a port is actually free before it hands it out.** Its +own bookkeeping records only what this worker did, and the collision it cannot +see is with something this worker never did: the default base port sits inside +Linux's default ephemeral range (`32768-60999`, see +`net.ipv4.ip_local_port_range`), so the kernel can give a port in the range to +an outbound connection, or to any process that binds port `0`, while the +allocator still believes it free. A backend handed one of those dies on bind. +Each candidate is therefore probed, and a port something else holds is skipped +and retried later rather than dropped, since whatever holds it is usually an +ephemeral connection that gives it back. The last count in the message above is +how many candidates were skipped that way, and the worker logs one line per +allocation when it skips any: + +``` +Skipped gRPC ports in this worker's range that something outside the worker +already holds ... skipped=3 allocated=50054 +``` + +Seeing that regularly means the worker's range overlaps what the kernel is +handing out. Move the range with `LOCALAI_ADDR` to a base port outside +`net.ipv4.ip_local_port_range` (for example `61000`) and the overlap goes away +entirely. Nothing dials these ports from outside the worker, so the base port +is free to be anything bindable. + ### NVIDIA GPU support When running workers in a container, two runtime settings affect how VRAM diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 0cd2bf097088..4efd358884b2 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -599,20 +599,24 @@ const ( // // A contiguous block rather than two independent freeport allocations, and the // difference is a defect this suite actually hit. The allocator hands backend -// processes basePort, basePort+1, basePort+2 and so on with no check that -// anything else holds them (core/services/worker/supervisor.go, allocatePort), -// so the moment freeport returned two ADJACENT ports the second backend started -// on a worker was handed the worker's own HTTP server's port and died at -// startup with "address already in use". freeport returns adjacent ports often, -// and no spec started two backends on one worker until the tunnel load -// measurement did, so it presented as a spec that failed about one run in -// three. +// processes basePort, basePort+1, basePort+2 and so on, and it used to do that +// from its own bookkeeping alone, so the moment freeport returned two ADJACENT +// ports the second backend started on a worker was handed the worker's own HTTP +// server's port and died at startup with "address already in use". freeport +// returns adjacent ports often, and no spec started two backends on one worker +// until the tunnel load measurement did, so it presented as a spec that failed +// about one run in three. // -// This is not race free and cannot be: the probe closes each listener before -// the worker binds it. It removes the deterministic self-collision, keeps the -// block out of the range the kernel allocates from, and bounds the allocator to -// the block, which together is the difference between "sometimes" and "not -// observed". +// core/services/worker/supervisor.go, allocatePort, now probes every candidate +// before handing it out, which removes that collision at the other end too. +// This block is kept for the reasons that survive the fix: it keeps the whole +// range out of the ports the kernel allocates from, so the probe has nothing to +// skip, and it bounds the allocator to a known block so an exhausted range in +// this suite is a bug in this suite. +// +// Neither is race free and neither can be: both probes close each listener +// before anything binds it. Together they are the difference between +// "sometimes" and "not observed". func reserveWorkerPorts() (int, error) { for attempt := 0; attempt < workerPortAttempts; attempt++ { base := workerPortFloor + rand.IntN(workerPortCeiling-workerPortFloor) From a5f54b765b85424b732fc7a23f7e5bc862ff985a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 6 Sep 2026 05:06:09 +0000 Subject: [PATCH 106/153] fix(distributed): bound the health monitor's miss streaks to live rows HealthMonitor.misses holds one consecutive-failed-probe count per (node, model, replica) and nothing ever removed an entry whose row had gone. It is the only per-node state in a frontend that grows on model churn rather than on fleet size, so a deployment that loads and unloads models for months accumulates an integer per tuple it ever probed and gives none back. There are four ways a row stops being visible to the pass, not one. A node departs and the pass skips its probes; a node goes offline or unhealthy on a stale heartbeat and the pass skips it entirely; an operator sets a node draining; or the row is removed by an unload, a scale-down or an eviction, and nothing tells this monitor. So the bound is the pass itself, and not a subscription on the departure notifier. The notifier evicts the caches a DEPARTURE invalidates and it keeps that one meaning; this reads a different fact, that there is no longer a row to count misses against, and covers all four cases with one rule. A row the pass could not probe is marked seen before the probe, so an unreachable worker still leaves its streak exactly as it was rather than having it forgiven; a pass that could not list the fleet prunes nothing, since it observed nothing. Forgetting only ever delays a reap by up to the miss threshold and can never cause one. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/nodes/health.go | 76 ++++++++- .../services/nodes/health_miss_bounds_test.go | 152 ++++++++++++++++++ core/services/nodes/health_mock_test.go | 18 +++ 3 files changed, 242 insertions(+), 4 deletions(-) create mode 100644 core/services/nodes/health_miss_bounds_test.go diff --git a/core/services/nodes/health.go b/core/services/nodes/health.go index 4ab3efe78d15..cf6286ea22a8 100644 --- a/core/services/nodes/health.go +++ b/core/services/nodes/health.go @@ -51,9 +51,13 @@ type HealthMonitor struct { // See DepartureNotifier: this monitor is its only caller. departures *DepartureNotifier missesMu sync.Mutex - misses map[modelKey]int // consecutive failed-probe counts; reset on success or model removal - cancel context.CancelFunc - cancelMu sync.Mutex + // misses holds one consecutive-failed-probe count per (node, model, + // replica). It is bounded to the rows the LAST completed pass actually + // walked; see forgetUnseenMisses for why that, and not a departure + // subscription, is what bounds it. + misses map[modelKey]int + cancel context.CancelFunc + cancelMu sync.Mutex } // NewHealthMonitor creates a new HealthMonitor. @@ -228,9 +232,15 @@ func (hm *HealthMonitor) doCheckAll(ctx context.Context) { nodes, err := hm.registry.List(ctx) if err != nil { xlog.Error("Health monitor: failed to list nodes", "error", err) + // No prune. A pass that could not read the fleet observed nothing, and + // pruning against nothing would wipe every streak in progress. return } + // Every model row this pass walked, whether or not it managed to probe it. + // See forgetUnseenMisses. + seen := make(map[modelKey]struct{}) + for _, node := range nodes { if node.Status == StatusDraining { continue @@ -330,6 +340,14 @@ func (hm *HealthMonitor) doCheckAll(ctx context.Context) { if m.WorkerLocalAddress == "" { continue } + // Marked seen BEFORE the probe, so that a row this pass could + // not reach keeps the streak it already had. The unreachable + // branch below leaves that streak exactly as it was, and a + // prune that only counted PROBED rows would clear it instead, + // forgiving a backend that really has died every time a peer + // link blipped. + key := modelKey{NodeID: node.ID, ModelName: m.ModelName, ReplicaIndex: m.ReplicaIndex} + seen[key] = struct{}{} // Through the node's tunnel, never a direct dial to m.WorkerLocalAddress: // that address is a port inside the worker. A worker this // replica cannot reach is not evidence that its backend died, @@ -363,7 +381,6 @@ func (hm *HealthMonitor) doCheckAll(ctx context.Context) { continue } - key := modelKey{NodeID: node.ID, ModelName: m.ModelName, ReplicaIndex: m.ReplicaIndex} hm.missesMu.Lock() if ok { // Probe succeeded — wipe any previous miss streak. @@ -397,4 +414,55 @@ func (hm *HealthMonitor) doCheckAll(ctx context.Context) { } } } + + hm.forgetUnseenMisses(seen) +} + +// forgetUnseenMisses drops every miss streak whose model row this pass did not +// walk, so the map is bounded by the fleet's live rows rather than by every +// (node, model, replica) the process has ever seen. +// +// It is level triggered and it is deliberately NOT a subscriber on the +// departure notifier, though the notifier is where the rest of the per-node +// caches are evicted from and it would have been the shorter change. +// +// A departure subscription would fix ONE of the four ways a streak is +// stranded. The others are a node marked offline or unhealthy on a stale +// heartbeat, a node the operator set draining, and a model row removed by +// anything other than this loop (an unload, a scale-down, an eviction). Each +// leaves a counter with no row behind it, and each would need its own hook. +// What all four have in common is exactly the thing this reads: the row was +// not there when the pass looked. So the notifier keeps its one meaning, "no +// live replica holds this node's tunnel", and this keeps its own, "there is no +// longer a row to count misses against". +// +// Forgetting is safe in one direction only, and it is the safe one: a cleared +// streak DELAYS a reap by up to perModelMissThreshold passes and can never +// cause one. That is what makes it sound to prune on a pass whose +// GetNodeModels call failed, which is otherwise indistinguishable here from a +// node with no models. +func (hm *HealthMonitor) forgetUnseenMisses(seen map[modelKey]struct{}) { + hm.missesMu.Lock() + defer hm.missesMu.Unlock() + for key := range hm.misses { + if _, ok := seen[key]; !ok { + delete(hm.misses, key) + } + } +} + +// missCount reports the streak recorded for one model row. It exists for the +// specs that assert this map is bounded, which is otherwise a property with no +// observable effect until the process runs out of memory. +func (hm *HealthMonitor) missCount(nodeID, modelName string, replicaIndex int) int { + hm.missesMu.Lock() + defer hm.missesMu.Unlock() + return hm.misses[modelKey{NodeID: nodeID, ModelName: modelName, ReplicaIndex: replicaIndex}] +} + +// trackedMisses reports how many model rows currently hold a miss streak. +func (hm *HealthMonitor) trackedMisses() int { + hm.missesMu.Lock() + defer hm.missesMu.Unlock() + return len(hm.misses) } diff --git a/core/services/nodes/health_miss_bounds_test.go b/core/services/nodes/health_miss_bounds_test.go new file mode 100644 index 000000000000..89eba8eb91ad --- /dev/null +++ b/core/services/nodes/health_miss_bounds_test.go @@ -0,0 +1,152 @@ +package nodes + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/cluster" +) + +// The miss map counts consecutive failed probes per (node, model, replica). It +// is the only per-node state in this process that grows on model churn rather +// than on fleet size, so a deployment that loads and unloads models for months +// accumulates one integer per tuple it ever probed and never gives one back. +// +// The bound is the pass itself: a streak survives only while the pass can still +// see the row it counts against. Every spec below is one way a row stops being +// visible, and the last two are the two ways a row must NOT be forgotten. +var _ = Describe("HealthMonitor miss-streak bounds", func() { + const staleThreshold = 30 * time.Second + + var ( + ctx context.Context + store *fakeNodeHealthStore + factory *fakeBackendClientFactory + hm *HealthMonitor + ) + + // deadModel is one model row whose backend answers every probe with a + // failure, so a single pass leaves a streak of exactly one. + deadModel := func(nodeID, modelName, addr string) NodeModel { + factory.setClient(addr, &fakeBackendClient{healthy: false, err: fmt.Errorf("connection refused")}) + return NodeModel{NodeID: nodeID, ModelName: modelName, WorkerLocalAddress: addr} + } + + BeforeEach(func() { + ctx = context.Background() + store = newFakeNodeHealthStore() + factory = newFakeBackendClientFactory() + hm = newTestHealthMonitor(store, factory, true, staleThreshold) + hm.perModelHealthCheck = true + }) + + It("forgets the streak of a model row that is no longer registered", func() { + // An unload, a scale-down or an LRU eviction removes the row without + // telling this monitor. Nothing else in the loop ever revisits the key, + // so the counter is stranded for the life of the process. + store.addNode(makeTestNode("node-a", "worker-a", "10.0.0.1:50051", StatusHealthy, freshTime())) + store.setNodeModels("node-a", deadModel("node-a", "m", "10.0.0.1:50053")) + + hm.doCheckAll(ctx) + Expect(hm.missCount("node-a", "m", 0)).To(Equal(1)) + + store.setNodeModels("node-a") + hm.doCheckAll(ctx) + + Expect(hm.trackedMisses()).To(BeZero()) + }) + + It("forgets the streaks of a node whose heartbeat went stale", func() { + // The pass skips the node entirely from this branch, so every streak it + // held is unreachable from here on. autoOffline deletes its rows too, + // which is what makes the counters count against nothing. + store.addNode(makeTestNode("node-b", "worker-b", "10.0.0.2:50051", StatusHealthy, freshTime())) + store.setNodeModels("node-b", deadModel("node-b", "m", "10.0.0.2:50053")) + + hm.doCheckAll(ctx) + Expect(hm.missCount("node-b", "m", 0)).To(Equal(1)) + + store.setHeartbeat("node-b", staleTime(staleThreshold)) + hm.doCheckAll(ctx) + + Expect(hm.trackedMisses()).To(BeZero()) + }) + + It("forgets the streaks of a node the operator set draining", func() { + store.addNode(makeTestNode("node-c", "worker-c", "10.0.0.3:50051", StatusHealthy, freshTime())) + store.setNodeModels("node-c", deadModel("node-c", "m", "10.0.0.3:50053")) + + hm.doCheckAll(ctx) + Expect(hm.missCount("node-c", "m", 0)).To(Equal(1)) + + store.getNode("node-c").Status = StatusDraining + hm.doCheckAll(ctx) + + Expect(hm.trackedMisses()).To(BeZero()) + }) + + It("forgets the streaks of a node whose tunnel departed", func() { + // The departure branch skips the probes deliberately: there is no route + // to dial. It also announces the departure, and every other per-node + // cache is dropped from that announcement. This one is not, because a + // departure is only one of the four ways a row stops being visible. + store.addNode(makeTestNode("node-d", "worker-d", "10.0.0.4:50051", StatusHealthy, freshTime())) + store.setNodeModels("node-d", deadModel("node-d", "m", "10.0.0.4:50053")) + + hm.doCheckAll(ctx) + Expect(hm.missCount("node-d", "m", 0)).To(Equal(1)) + + hm.presence = &stubPresence{answer: cluster.PresenceGone} + hm.reconnectGrace = time.Minute + hm.departures = NewDepartureNotifier() + hm.doCheckAll(ctx) + + Expect(hm.trackedMisses()).To(BeZero()) + }) + + It("keeps the streak of a row the pass could not probe", func() { + // An unreachable worker is not evidence about its backends. The streak + // is neither advanced nor cleared, so a row that was two misses from + // removal before the peer link blipped is still two misses from removal + // after it. A prune that counted only PROBED rows would forgive it. + store.addNode(makeTestNode("node-e", "worker-e", "10.0.0.5:50051", StatusHealthy, freshTime())) + store.setNodeModels("node-e", deadModel("node-e", "m", "10.0.0.5:50053")) + + hm.doCheckAll(ctx) + Expect(hm.missCount("node-e", "m", 0)).To(Equal(1)) + + factory.refuseForNode = fmt.Errorf("no tunnel for you") + hm.doCheckAll(ctx) + + Expect(hm.missCount("node-e", "m", 0)).To(Equal(1)) + }) + + It("keeps every streak when the pass could not read the fleet", func() { + // A pass that failed to list nodes observed nothing at all, and pruning + // against nothing would clear every streak in the deployment on one + // database hiccup. + store.addNode(makeTestNode("node-f", "worker-f", "10.0.0.6:50051", StatusHealthy, freshTime())) + store.setNodeModels("node-f", deadModel("node-f", "m", "10.0.0.6:50053")) + + hm.doCheckAll(ctx) + Expect(hm.missCount("node-f", "m", 0)).To(Equal(1)) + + hm.registry = &listFailsStore{NodeHealthStore: store, err: fmt.Errorf("connection reset")} + hm.doCheckAll(ctx) + + Expect(hm.missCount("node-f", "m", 0)).To(Equal(1)) + }) +}) + +// listFailsStore is the fake store with its fleet listing broken, which a real +// registry cannot be made to do on demand. +type listFailsStore struct { + NodeHealthStore + err error +} + +func (s *listFailsStore) List(context.Context) ([]BackendNode, error) { return nil, s.err } diff --git a/core/services/nodes/health_mock_test.go b/core/services/nodes/health_mock_test.go index 2685694b9959..a71e5a0e83d8 100644 --- a/core/services/nodes/health_mock_test.go +++ b/core/services/nodes/health_mock_test.go @@ -53,6 +53,24 @@ func (f *fakeNodeHealthStore) addNodeModel(nodeID string, nm NodeModel) { f.models[nodeID] = append(f.models[nodeID], nm) } +// setNodeModels replaces a node's model rows, so a spec can make a row +// disappear the way an unload, a scale-down or an eviction does. +func (f *fakeNodeHealthStore) setNodeModels(nodeID string, models ...NodeModel) { + f.mu.Lock() + defer f.mu.Unlock() + f.models[nodeID] = models +} + +// setHeartbeat backdates a node's heartbeat so a spec can drive the stale +// branch without waiting. +func (f *fakeNodeHealthStore) setHeartbeat(nodeID string, at time.Time) { + f.mu.Lock() + defer f.mu.Unlock() + if n, ok := f.nodes[nodeID]; ok { + n.LastHeartbeat = at + } +} + func (f *fakeNodeHealthStore) getNode(id string) *BackendNode { f.mu.Lock() defer f.mu.Unlock() From d42d70e73bd72de22c21d724a24161d27417861d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 6 Sep 2026 05:06:21 +0000 Subject: [PATCH 107/153] fix(distributed): drop a departed node's cached HTTP clients A frontend keeps two caches of one http.Client per worker: the control client's, built on the first verb issued to a node, and the HTTP file stager's, built on the first file staged to it. Both are keyed by node ID and neither was ever pruned. Their own comments said so and named what a fix would need, a signal that a node has left, which did not exist when they were written and does now. The map slot is the smaller half. Each entry holds an http.Transport whose idle connections are streams on that worker's tunnel, kept until IdleConnTimeout even after the tunnel is gone, so ForgetNode closes them rather than leaving them to the collector. Both are registered on the deployment's one departure notifier, as two subscribers and not one: a node can be in either cache without being in the other, and a single hook would say only that some client was kept. ForgetNode is on the FileStager interface rather than on the one implementation that has state to drop, so a stager that grows a per-node map later cannot be added without answering the question, and so registerDepartureEvictions can take a FileStager and still fail to compile if the registration is deleted. The S3 stager's is a documented no-op that deliberately does not forward to the control client, which registers itself: forwarding would evict a cache it does not own, twice per departure, and the second drop would not appear in the subscriber names the wiring spec reads. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/application/distributed.go | 28 +++- core/application/distributed_test.go | 36 ++++- core/services/nodes/control_client.go | 51 +++++-- core/services/nodes/file_stager.go | 11 ++ core/services/nodes/file_stager_http.go | 38 +++-- core/services/nodes/file_stager_s3.go | 11 ++ core/services/nodes/file_staging_3d_test.go | 3 + core/services/nodes/per_node_clients_test.go | 135 ++++++++++++++++++ .../nodes/router_staging_context_test.go | 3 + .../nodes/router_staging_deadline_test.go | 3 + core/services/nodes/router_test.go | 3 + 11 files changed, 295 insertions(+), 27 deletions(-) create mode 100644 core/services/nodes/per_node_clients_test.go diff --git a/core/application/distributed.go b/core/application/distributed.go index 85911ece19f4..53694f2926bd 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -704,7 +704,7 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade // Every per-node cache a departure leaves stale, onto the one notification // point, after the router that owns two of them exists. - if err := registerDepartureEvictions(departures, prefixDrop, router, galleryProgress); err != nil { + if err := registerDepartureEvictions(departures, prefixDrop, router, galleryProgress, controlClient, fileStager); err != nil { return nil, err } @@ -787,6 +787,8 @@ const ( departureProbeCache = "probe-cache" departureStagingTracker = "staging-tracker" departureGalleryNodes = "gallery-node-progress" + departureControlClients = "control-http-clients" + departureStagerClients = "file-stager-http-clients" ) // registerDepartureEvictions registers every per-node cache that a node's @@ -810,7 +812,7 @@ const ( // *prefixcache.Sync for that decision to be safe, since a nil provider inside // an interface would compare non-nil here and dereference on the first // departure. -func registerDepartureEvictions(departures *nodes.DepartureNotifier, prefix *prefixcache.Sync, router *nodes.SmartRouter, gallery nodeProgressDropper) error { +func registerDepartureEvictions(departures *nodes.DepartureNotifier, prefix *prefixcache.Sync, router *nodes.SmartRouter, gallery nodeProgressDropper, control *nodes.ControlClient, stager nodes.FileStager) error { if departures == nil { return fmt.Errorf("wiring departure evictions: no departure notifier, so a departed node would keep every per-node cache entry it has for the life of the process") } @@ -820,6 +822,12 @@ func registerDepartureEvictions(departures *nodes.DepartureNotifier, prefix *pre if gallery == nil { return fmt.Errorf("wiring departure evictions: no gallery service, so a departed node would stay in every open operation's per-node breakdown") } + if control == nil { + return fmt.Errorf("wiring departure evictions: no control client, so a departed node would keep its cached HTTP client and that client's idle streams on a tunnel that is gone") + } + if stager == nil { + return fmt.Errorf("wiring departure evictions: no file stager, so a departed node would keep the cached HTTP client its transfers ran on") + } // S1. Inside a nil check and not inside the prefix-cache-enabled block, so // that "the disabled deployment registers nothing" is a fact a spec can // hold rather than a property of where a line was written. @@ -841,6 +849,22 @@ func registerDepartureEvictions(departures *nodes.DepartureNotifier, prefix *pre departures.OnDeparture(departureGalleryNodes, func(node nodes.DepartedNode) { gallery.DropNodeProgress(node.ID) }) + // S5 and S6, the two per-node http.Client caches. Two registrations again, + // because they are two caches with two owners: the control client's entry + // is built on the first verb issued to a node and the stager's on the first + // file staged to it, so a node can be in either without being in the other, + // and one hook doing both would say only that some client was kept. + // + // Both are keyed by node ID and both are DROPPED rather than emptied. A + // worker that comes back builds a fresh client on its next verb, over + // whatever tunnel it has by then; keeping the old one would keep a + // transport whose idle streams belong to a session that has ended. + departures.OnDeparture(departureControlClients, func(node nodes.DepartedNode) { + control.ForgetNode(node.ID) + }) + departures.OnDeparture(departureStagerClients, func(node nodes.DepartedNode) { + stager.ForgetNode(node.ID) + }) return nil } diff --git a/core/application/distributed_test.go b/core/application/distributed_test.go index bedd6dfb81b8..3eb05c1dad7c 100644 --- a/core/application/distributed_test.go +++ b/core/application/distributed_test.go @@ -222,6 +222,8 @@ var _ = Describe("wiring the per-node caches a departure evicts", func() { departureProbeCache, departureStagingTracker, departureGalleryNodes, + departureControlClients, + departureStagerClients, )) }) @@ -239,26 +241,54 @@ var _ = Describe("wiring the per-node caches a departure evicts", func() { departureProbeCache, departureStagingTracker, departureGalleryNodes, + departureControlClients, + departureStagerClients, )) }) It("refuses a deployment with no router, naming what its departed nodes would keep", func() { - err := registerDepartureEvictions(nodes.NewDepartureNotifier(), nil, nil, galleryop.NewGalleryService(&config.ApplicationConfig{}, nil)) + err := registerDepartureEvictions(nodes.NewDepartureNotifier(), nil, nil, galleryop.NewGalleryService(&config.ApplicationConfig{}, nil), specControlClient(), specFileStager()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("probe-freshness")) }) It("refuses a deployment with no gallery service", func() { - err := registerDepartureEvictions(nodes.NewDepartureNotifier(), nil, nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}), nil) + err := registerDepartureEvictions(nodes.NewDepartureNotifier(), nil, nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}), nil, specControlClient(), specFileStager()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("per-node breakdown")) }) + It("refuses a deployment with no control client, naming the streams a departed node would keep", func() { + err := registerDepartureEvictions(nodes.NewDepartureNotifier(), nil, nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}), galleryop.NewGalleryService(&config.ApplicationConfig{}, nil), nil, specFileStager()) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("idle streams")) + }) + + It("refuses a deployment with no file stager", func() { + err := registerDepartureEvictions(nodes.NewDepartureNotifier(), nil, nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}), galleryop.NewGalleryService(&config.ApplicationConfig{}, nil), specControlClient(), nil) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("cached HTTP client")) + }) + It("refuses a deployment with no notifier at all", func() { - err := registerDepartureEvictions(nil, nil, nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}), galleryop.NewGalleryService(&config.ApplicationConfig{}, nil)) + err := registerDepartureEvictions(nil, nil, nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}), galleryop.NewGalleryService(&config.ApplicationConfig{}, nil), specControlClient(), specFileStager()) Expect(err).To(HaveOccurred()) }) }) + +// specControlClient and specFileStager build the two per-node client caches a +// registration needs. Neither is dialled: what the refusal specs assert is that +// the wiring refuses a MISSING one, and what the registration specs assert is +// that a subscriber was registered for it. +func specControlClient() *nodes.ControlClient { + return nodes.NewControlClient(nil, "") +} + +func specFileStager() nodes.FileStager { + return nodes.NewHTTPFileStager(func(string) (string, error) { return "", nil }, "", nil) +} diff --git a/core/services/nodes/control_client.go b/core/services/nodes/control_client.go index fe0d7e5f0e8f..dd5d022de79f 100644 --- a/core/services/nodes/control_client.go +++ b/core/services/nodes/control_client.go @@ -56,20 +56,31 @@ type ControlClient struct { // row reuses the tunnel stream its transport already holds instead of // opening a new one. // - // Entries are never pruned, and that is judged rather than overlooked: the - // map is bounded by the number of distinct workers this frontend has ever - // commanded, which is bounded by the fleet, and a departed worker's entry - // holds a map slot plus a transport whose idle connections IdleConnTimeout - // reclaims. It is the same shape, and would need the same missing - // node-departure signal to fix, as HTTPFileStager.clients. + // Entries are dropped by ForgetNode, on the deployment's one departure + // notification, the same way HTTPFileStager.clients is. Without that the + // map grew once per distinct worker for the life of the process, which is + // bounded by the fleet only in a deployment whose nodes never change + // identity. clientsMu sync.Mutex - clients map[string]*http.Client + clients map[string]*nodeHTTPClient +} + +// nodeHTTPClient is one cached per-node client together with the transport it +// was built on. +// +// The transport is kept beside the client rather than read back off +// http.Client.Transport, because what is stored there is whatever the shared +// httpclient constructor decided to wrap it in, and a per-node cache that +// cannot close its own idle connections has only half a ForgetNode. +type nodeHTTPClient struct { + client *http.Client + transport *http.Transport } // NewControlClient returns the control client for the workers dialFor can // reach, authenticating with the deployment's registration token. func NewControlClient(dialFor WorkerNetDialerFor, token string) *ControlClient { - return &ControlClient{dialFor: dialFor, token: token, clients: map[string]*http.Client{}} + return &ControlClient{dialFor: dialFor, token: token, clients: map[string]*nodeHTTPClient{}} } // clientFor returns the HTTP client that reaches one worker, building it on @@ -96,7 +107,7 @@ func (c *ControlClient) clientFor(nodeID string) (*http.Client, error) { c.clientsMu.Lock() defer c.clientsMu.Unlock() if cl, ok := c.clients[nodeID]; ok { - return cl, nil + return cl.client, nil } dial := c.dialFor(nodeID) if dial == nil { @@ -111,10 +122,30 @@ func (c *ControlClient) clientFor(nodeID string) (*http.Client, error) { ExpectContinueTimeout: 1 * time.Second, } cl := httpclient.New(httpclient.WithTransport(transport)) - c.clients[nodeID] = cl + c.clients[nodeID] = &nodeHTTPClient{client: cl, transport: transport} return cl, nil } +// ForgetNode drops the cached client for a departed node and closes the idle +// tunnel streams its transport is holding. See HTTPFileStager.ForgetNode: the +// two are one rule applied to the two per-node client caches this frontend +// keeps, and both are registered on the departure notifier. +// +// Safe on a nil receiver, because the notifier's subscribers are registered at +// wiring time and a deployment can reach it with no control client. +func (c *ControlClient) ForgetNode(nodeID string) { + if c == nil { + return + } + c.clientsMu.Lock() + entry, ok := c.clients[nodeID] + delete(c.clients, nodeID) + c.clientsMu.Unlock() + if ok { + entry.transport.CloseIdleConnections() + } +} + // Call issues one control RPC and decodes its reply. reply may be nil for the // verbs that answer 204. // diff --git a/core/services/nodes/file_stager.go b/core/services/nodes/file_stager.go index ee28a62b8b94..a8523097c8ee 100644 --- a/core/services/nodes/file_stager.go +++ b/core/services/nodes/file_stager.go @@ -43,6 +43,17 @@ type FileStager interface { // ListRemoteDir returns relative file paths within a directory on the remote node. // keyPrefix is a storage-style key prefix (e.g. "models/mymodel"). ListRemoteDir(ctx context.Context, nodeID, keyPrefix string) ([]string, error) + + // ForgetNode drops whatever this stager holds for one node, and is called + // when the deployment decides that node has departed. + // + // On the INTERFACE rather than on the one implementation that has state to + // drop, so that a stager which grows a per-node map later cannot be added + // without answering this question, and so that the wiring in + // core/application can name a FileStager and still fail to compile if the + // registration is deleted. An implementation with nothing per-node is a + // documented no-op. + ForgetNode(nodeID string) } // RequestFileReleaser removes all ephemeral keys staged for one inference in diff --git a/core/services/nodes/file_stager_http.go b/core/services/nodes/file_stager_http.go index 0ac64fd069f6..8165e0d3302e 100644 --- a/core/services/nodes/file_stager_http.go +++ b/core/services/nodes/file_stager_http.go @@ -44,16 +44,13 @@ type HTTPFileStager struct { // connection pool: a client built per request would open a fresh tunnel // stream for every chunk of a multi-gigabyte upload. // - // Entries are never pruned, and that is judged acceptable rather than - // overlooked. The map is bounded by the number of distinct workers this - // frontend has ever staged to, which is bounded by the fleet; each entry is - // a transport whose idle connections the 90s IdleConnTimeout above reclaims, - // so a departed worker's entry holds a map slot and nothing else. It is the - // same shape as PeerPool.links and would need the same thing to fix - // properly: a signal that a node has left, which the deregistration path - // does not publish today. + // Entries are dropped by ForgetNode, on the deployment's one departure + // notification. Without that the map grew once per distinct worker for the + // life of the process, which is bounded by the fleet only in a deployment + // whose nodes never change identity; a Kubernetes worker pool that + // re-registers under a new id on every rollout is not one. clientsMu sync.Mutex - clients map[string]*http.Client + clients map[string]*nodeHTTPClient responseTimeout time.Duration // timeout waiting for server response after upload maxRetries int // number of retry attempts for transient failures } @@ -81,7 +78,7 @@ func NewHTTPFileStager(httpAddrFor func(nodeID string) (string, error), token st httpAddrFor: httpAddrFor, token: token, dialFor: dialFor, - clients: map[string]*http.Client{}, + clients: map[string]*nodeHTTPClient{}, responseTimeout: responseTimeout, maxRetries: maxRetries, } @@ -112,7 +109,7 @@ func (h *HTTPFileStager) clientFor(nodeID string) (*http.Client, error) { h.clientsMu.Lock() defer h.clientsMu.Unlock() if c, ok := h.clients[nodeID]; ok { - return c, nil + return c.client, nil } dial := h.dialFor(nodeID) if dial == nil { @@ -129,7 +126,7 @@ func (h *HTTPFileStager) clientFor(nodeID string) (*http.Client, error) { ReadBufferSize: 256 << 10, // 256 KB } c := httpclient.New(httpclient.WithTransport(transport)) - h.clients[nodeID] = c + h.clients[nodeID] = &nodeHTTPClient{client: c, transport: transport} return c, nil } @@ -221,6 +218,23 @@ func (h *HTTPFileStager) releaseRemoteKeys(ctx context.Context, nodeID string, k return errors.Join(releaseErrors...) } +// ForgetNode drops the cached client for a departed node and closes the idle +// tunnel streams its transport is holding. +// +// Closing and not merely deleting: the map slot is the smaller half. A +// transport left only to the garbage collector keeps its idle connections +// until IdleConnTimeout, and each of those is a stream on a tunnel that is +// already gone. +func (h *HTTPFileStager) ForgetNode(nodeID string) { + h.clientsMu.Lock() + entry, ok := h.clients[nodeID] + delete(h.clients, nodeID) + h.clientsMu.Unlock() + if ok { + entry.transport.CloseIdleConnections() + } +} + func (h *HTTPFileStager) EnsureRemote(ctx context.Context, nodeID, localPath, key string) (string, error) { xlog.Debug("Staging file to remote node via HTTP", "node", nodeID, "localPath", localPath, "key", key) diff --git a/core/services/nodes/file_stager_s3.go b/core/services/nodes/file_stager_s3.go index 18030368e0c9..4f97b074435c 100644 --- a/core/services/nodes/file_stager_s3.go +++ b/core/services/nodes/file_stager_s3.go @@ -30,6 +30,17 @@ func NewS3FileStager(fm *storage.FileManager, control *ControlClient) *S3FileSta return &S3FileStager{fm: fm, control: control} } +// ForgetNode has nothing of its own to drop: this stager keeps no per-node +// state at all. Bytes travel through the object store, and the only per-node +// client involved is the ControlClient's, which the deployment shares with +// every other control verb and registers on the departure notifier itself. +// +// Deliberately NOT forwarded to that client. Dropping it from here as well +// would close a cache this stager does not own, on a departure it would then +// be reacting to twice, and the second drop would be invisible in the +// subscriber names the wiring spec reads. +func (s *S3FileStager) ForgetNode(string) {} + // The two budgets a file-staging RPC gets. They are the ones the NATS // request-reply timeouts carried, kept verbatim: a transfer verb waits out a // multi-gigabyte copy, and a metadata verb does not. diff --git a/core/services/nodes/file_staging_3d_test.go b/core/services/nodes/file_staging_3d_test.go index f67de66821dd..d650bc82b0f2 100644 --- a/core/services/nodes/file_staging_3d_test.go +++ b/core/services/nodes/file_staging_3d_test.go @@ -44,3 +44,6 @@ var _ = Describe("FileStagingClient 3D output", func() { Expect(backend.request.Dst).To(Equal("/remote/tmp")) }) }) + +// ForgetNode drops per-node state, which this double keeps none of. +func (*failingFetchStager) ForgetNode(string) {} diff --git a/core/services/nodes/per_node_clients_test.go b/core/services/nodes/per_node_clients_test.go new file mode 100644 index 000000000000..5fdd5c7b3772 --- /dev/null +++ b/core/services/nodes/per_node_clients_test.go @@ -0,0 +1,135 @@ +package nodes + +import ( + "context" + "net" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// A frontend keeps two caches of one *http.Client per worker: the control +// client's, built on the first verb issued to a node, and the HTTP file +// stager's, built on the first file staged to it. Both are keyed by node ID and +// neither used to be pruned, so each grew once per distinct worker for the life +// of the process. A worker pool that re-registers under a new id on every +// rollout is not bounded by the fleet. +// +// Each cached entry is more than a map slot: it holds an http.Transport whose +// idle connections are streams on that worker's tunnel, kept until +// IdleConnTimeout even after the tunnel has gone. +var _ = Describe("per-node HTTP client caches", func() { + // aDialer stands in for a worker's tunnel dialler. It is never called: what + // these specs assert is which client the cache hands back, not what that + // client can reach. + aDialer := func(string) func(ctx context.Context, network, addr string) (net.Conn, error) { + return func(context.Context, string, string) (net.Conn, error) { return nil, nil } + } + + Describe("ControlClient", func() { + var c *ControlClient + + BeforeEach(func() { c = NewControlClient(aDialer, "token") }) + + It("reuses one client per node until that node departs", func() { + first, err := c.clientFor("node-1") + Expect(err).NotTo(HaveOccurred()) + again, err := c.clientFor("node-1") + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(BeIdenticalTo(first), + "a verb issued twice must reuse the tunnel stream the transport already holds") + }) + + It("drops a departed node's client rather than holding it for the life of the process", func() { + first, err := c.clientFor("node-1") + Expect(err).NotTo(HaveOccurred()) + + c.ForgetNode("node-1") + Expect(c.clients).NotTo(HaveKey("node-1")) + + rebuilt, err := c.clientFor("node-1") + Expect(err).NotTo(HaveOccurred()) + Expect(rebuilt).NotTo(BeIdenticalTo(first), + "a worker that comes back gets a client over whatever tunnel it has now") + }) + + It("leaves the other nodes' clients alone", func() { + kept, err := c.clientFor("node-2") + Expect(err).NotTo(HaveOccurred()) + _, err = c.clientFor("node-1") + Expect(err).NotTo(HaveOccurred()) + + c.ForgetNode("node-1") + + again, err := c.clientFor("node-2") + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(BeIdenticalTo(kept)) + }) + + It("is a no-op for a node it never built a client for", func() { + Expect(func() { c.ForgetNode("never-seen") }).NotTo(Panic()) + Expect(c.clients).To(BeEmpty()) + }) + }) + + Describe("HTTPFileStager", func() { + var h *HTTPFileStager + + BeforeEach(func() { + h = NewHTTPFileStager(func(string) (string, error) { return "worker:1", nil }, "token", aDialer) + }) + + It("reuses one client per node until that node departs", func() { + first, err := h.clientFor("node-1") + Expect(err).NotTo(HaveOccurred()) + again, err := h.clientFor("node-1") + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(BeIdenticalTo(first), + "a client built per request would open a fresh tunnel stream for every chunk of a multi-gigabyte upload") + }) + + It("drops a departed node's client rather than holding it for the life of the process", func() { + first, err := h.clientFor("node-1") + Expect(err).NotTo(HaveOccurred()) + + h.ForgetNode("node-1") + Expect(h.clients).NotTo(HaveKey("node-1")) + + rebuilt, err := h.clientFor("node-1") + Expect(err).NotTo(HaveOccurred()) + Expect(rebuilt).NotTo(BeIdenticalTo(first)) + }) + + It("leaves the other nodes' clients alone", func() { + kept, err := h.clientFor("node-2") + Expect(err).NotTo(HaveOccurred()) + _, err = h.clientFor("node-1") + Expect(err).NotTo(HaveOccurred()) + + h.ForgetNode("node-1") + + again, err := h.clientFor("node-2") + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(BeIdenticalTo(kept)) + }) + }) + + Describe("S3FileStager", func() { + It("does not drop the control client's entry on behalf of a stager that has no state", func() { + // The object-store stager keeps nothing per node; its only per-node + // client belongs to the ControlClient, which the deployment shares + // with every other control verb and registers separately. Dropping + // it from here as well would evict a cache this stager does not own + // and would do it twice per departure, invisibly. + control := NewControlClient(aDialer, "token") + kept, err := control.clientFor("node-1") + Expect(err).NotTo(HaveOccurred()) + + NewS3FileStager(nil, control).ForgetNode("node-1") + + again, err := control.clientFor("node-1") + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(BeIdenticalTo(kept)) + }) + }) +}) diff --git a/core/services/nodes/router_staging_context_test.go b/core/services/nodes/router_staging_context_test.go index 1a577df373c5..37ec8eba1a0b 100644 --- a/core/services/nodes/router_staging_context_test.go +++ b/core/services/nodes/router_staging_context_test.go @@ -78,3 +78,6 @@ var _ = Describe("Route cold-load staging context", func() { "staging context must survive cancellation of the triggering request") }) }) + +// ForgetNode drops per-node state, which this double keeps none of. +func (*cancelOnStageStager) ForgetNode(string) {} diff --git a/core/services/nodes/router_staging_deadline_test.go b/core/services/nodes/router_staging_deadline_test.go index 66026987d9c2..0d1348a8f340 100644 --- a/core/services/nodes/router_staging_deadline_test.go +++ b/core/services/nodes/router_staging_deadline_test.go @@ -219,3 +219,6 @@ var _ = Describe("cold-load staging deadline", func() { Expect(time.Until(deadline)).To(BeNumerically("~", 3*time.Hour, time.Minute)) }) }) + +// ForgetNode drops per-node state, which this double keeps none of. +func (*progressingStager) ForgetNode(string) {} diff --git a/core/services/nodes/router_test.go b/core/services/nodes/router_test.go index e66428cc6684..243de0f39240 100644 --- a/core/services/nodes/router_test.go +++ b/core/services/nodes/router_test.go @@ -1726,3 +1726,6 @@ var _ = Describe("SmartRouter prefix-cache routing", func() { }) }) }) + +// ForgetNode drops per-node state, which this double keeps none of. +func (*fakeFileStager) ForgetNode(string) {} From 8f5f303a75e1e1a7c03b4912afe0a13d71325151 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 6 Sep 2026 05:06:33 +0000 Subject: [PATCH 108/153] fix(distributed): back a failed claim off instead of respinning it The claim queue's attempts counter grew without bound and nothing read it. At the default two-second poll a permanently undispatchable row cost about 43000 UPDATEs a day, and it cost more than writes: rows are claimed oldest first, so the oldest stuck row was re-claimed ahead of every newer one on every tick and held a dispatch slot while it failed. One poison row starved the queue behind it. No dead letter, and that is the decision rather than the omission. Read settleClaim: the only outcome that releases a claim is one where NOTHING was learned about the work. No agent worker was connected, the tunnel broke, a peer could not be reached, the stream was refused before the request body left this replica. Not one of those is a worker saying it ran the job and it failed, and an attempt ceiling would turn "the fleet was away long enough" into a job failure nobody reported, which is the collapse this whole design exists to prevent pointed at work instead of at nodes. The one verdict available here, that no build of any worker serves this kind, is already settled as an answer. So the retry stays unbounded and the RATE does not. Each release stamps the row with the earliest it may be claimed again, doubling from two seconds to a cap of sixty, computed in the release statement from the row's own attempts count and stamped on the DATABASE clock, because that is the clock competing replicas order the queue on. Queued work becomes claimable again within one cap of the fleet returning, and a stuck row no longer holds the head of the queue. A claim released by the reap carries no delay at all: that work was never handed to anyone, so there is nothing to back off from. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/jobs/claim.go | 83 +++++++++++-- core/services/jobs/claim_test.go | 145 ++++++++++++++++++++++ core/services/jobs/dispatch_loop.go | 15 ++- docs/content/features/distributed-mode.md | 6 + 4 files changed, 237 insertions(+), 12 deletions(-) diff --git a/core/services/jobs/claim.go b/core/services/jobs/claim.go index baa7f85f8d59..7cbf6bc2bed2 100644 --- a/core/services/jobs/claim.go +++ b/core/services/jobs/claim.go @@ -46,7 +46,12 @@ type WorkClaim struct { ClaimedBy string `gorm:"size:64;index"` ClaimedAt *time.Time `gorm:"index"` Attempts int - CreatedAt time.Time `gorm:"index"` + // NotBefore is the earliest this row may be claimed again, stamped by + // ReleaseClaim from the DATABASE clock. NULL means "now", which is what + // every freshly enqueued row carries and what a reaped row goes back to. + // See claimBackoff. + NotBefore *time.Time `gorm:"index"` + CreatedAt time.Time `gorm:"index"` } // TableName pins the table this maps onto, because ReapAbandoned and ClaimNext @@ -116,6 +121,7 @@ func EnqueueClaim(ctx context.Context, db *gorm.DB, kind ClaimKind, payload any) "claimed_by": "", "claimed_at": nil, "attempts": 0, + "not_before": nil, "created_at": gorm.Expr("now()"), }).Error; err != nil { return "", fmt.Errorf("enqueueing a %s claim: %w", kind, err) @@ -143,11 +149,12 @@ SET claimed_by = ?, claimed_at = now() WHERE id = ( SELECT id FROM ` + claimsTable + ` WHERE claimed_at IS NULL AND kind IN ? + AND (not_before IS NULL OR not_before <= now()) ORDER BY created_at, id FOR UPDATE SKIP LOCKED LIMIT 1 ) -RETURNING id, kind, payload, claimed_by, claimed_at, attempts, created_at` +RETURNING id, kind, payload, claimed_by, claimed_at, attempts, not_before, created_at` // ClaimNext takes at most one unclaimed row of any kind in kinds, marking it // claimed by owner. It returns ErrNoWork when there is none. @@ -180,19 +187,69 @@ func ClaimNext(ctx context.Context, db *gorm.DB, owner string, kinds []ClaimKind return &claim, nil } -// ReleaseClaim returns a row to the pool, incrementing Attempts. +// The retry schedule a released claim comes back on. +// +// There is no dead letter here, and that is a decision rather than an omission. +// Read settleClaim: the only outcome that reaches ReleaseClaim is one where +// NOTHING was learned about the work. No agent worker was connected, the tunnel +// broke, a peer could not be reached, the stream was refused before the request +// body left this replica. Not one of those is the worker saying it ran the job +// and it failed, and failing a job on any of them would report an absent +// connection as a worker's verdict, which is the one collapse this whole design +// exists to prevent. A deployment whose fleet is down for a day must run its +// queued work when the fleet comes back, not find it failed. +// +// What IS a defect is retrying at the poll interval for ever. At the default +// two seconds a permanently undispatchable row costs ~43000 UPDATEs a day, and +// it costs more than writes: rows are claimed oldest-first, so the oldest +// stuck row is re-claimed ahead of every newer one on every single tick and +// holds a dispatch slot while it fails. One poison row starves the queue behind +// it. +// +// So the retry is unbounded and the RATE is not. Each release stamps the row +// with the earliest it may be claimed again, doubling per attempt from +// claimBackoffBase up to claimBackoffCap. A first failure costs the base delay, +// a fleet that has been down for a minute retries at the cap, and work becomes +// claimable again within one cap of the fleet returning. The exponent is +// clamped so the arithmetic cannot overflow however long a row has been stuck. +const ( + // claimBackoffBase is the delay after the first failed dispatch. One poll + // interval: a row that failed because nothing was connected should not be + // re-tried before the loop would have looked again anyway. + claimBackoffBase = 2 * time.Second + // claimBackoffCap bounds the delay, and with it how long queued work waits + // after a fleet comes back. Kept short for that reason: the point of the + // backoff is to stop a stuck row from spinning, not to give up on it. + claimBackoffCap = 60 * time.Second + // claimBackoffMaxShift clamps the exponent. 2^16 base seconds is already + // far past the cap, so this only keeps power() away from infinity for a row + // that has been retried for months. + claimBackoffMaxShift = 16 +) + +// releaseClaimSQL returns a row to the pool and stamps its next eligibility. +// +// One statement, and the delay computed IN the statement, because it reads the +// row's own attempts count and stamps now() from the DATABASE clock. Competing +// replicas order the queue on that clock; a Go-side deadline would make how +// long a row waits depend on the clock skew of whichever replica happened to +// fail it. +const releaseClaimSQL = `UPDATE ` + claimsTable + ` +SET claimed_by = '', claimed_at = NULL, attempts = attempts + 1, + not_before = now() + make_interval(secs => LEAST(?, ? * power(2, LEAST(attempts, ?)))) +WHERE id = ?` + +// ReleaseClaim returns a row to the pool, incrementing Attempts and stamping +// the backoff described above. // // It is what a TRANSPORT failure does: nothing was learned about the work, so // it must be retried, possibly by another replica against another worker. func ReleaseClaim(ctx context.Context, db *gorm.DB, id string) error { - if db == nil { - return errors.New("releasing a claim: no database handle") + if err := requirePostgres(db, "releasing a claim"); err != nil { + return err } - res := db.WithContext(ctx).Model(&WorkClaim{}).Where("id = ?", id).Updates(map[string]any{ - "claimed_by": "", - "claimed_at": nil, - "attempts": gorm.Expr("attempts + 1"), - }) + res := db.WithContext(ctx).Exec(releaseClaimSQL, + claimBackoffCap.Seconds(), claimBackoffBase.Seconds(), claimBackoffMaxShift, id) if res.Error != nil { return fmt.Errorf("releasing claim %q: %w", id, res.Error) } @@ -227,8 +284,12 @@ func CompleteClaim(ctx context.Context, db *gorm.DB, id string) error { // so this reap and every other reader of replica absence in the deployment move // together, and it runs as ONE statement so a replica cannot die, or come back, // between deciding who is live and acting on it. +// It stamps NO backoff, unlike ReleaseClaim. A claim abandoned because its +// replica died was never dispatched anywhere: there is nothing to back off +// from, and delaying it would punish the work for the death of the process that +// held it. It becomes claimable at once, by whichever replica polls next. const reapAbandonedSQL = `UPDATE ` + claimsTable + ` -SET claimed_by = '', claimed_at = NULL, attempts = attempts + 1 +SET claimed_by = '', claimed_at = NULL, attempts = attempts + 1, not_before = NULL WHERE claimed_at IS NOT NULL AND claimed_by NOT IN (` + cluster.LiveInstanceIDsSQL + `)` diff --git a/core/services/jobs/claim_test.go b/core/services/jobs/claim_test.go index bb931b887850..99f8e10aff5c 100644 --- a/core/services/jobs/claim_test.go +++ b/core/services/jobs/claim_test.go @@ -103,6 +103,15 @@ var _ = Describe("The claim queue", func() { return got } + // expireBackoff brings a released row's next-eligible stamp into the past, + // so a spec can assert what happens AFTER the backoff without waiting out a + // real one. Sleeping for the delay would make every one of these specs a + // spec that fails one run in ten. + expireBackoff := func(id string) { + GinkgoHelper() + Expect(db.Exec(`UPDATE `+claimsTable+` SET not_before = now() - interval '1 hour' WHERE id = ?`, id).Error).To(Succeed()) + } + Describe("taking work", func() { It("reports an empty queue distinguishably rather than as a failure", func() { _, err := ClaimNext(ctx, db, "inst-a", []ClaimKind{ClaimKindMCPCI}) @@ -273,6 +282,10 @@ var _ = Describe("The claim queue", func() { Expect(row.ClaimedBy).To(BeEmpty()) Expect(row.Attempts).To(Equal(1)) + // Claimable again, once the backoff this release stamped has + // passed. The wait is brought forward rather than slept through: + // see "backing a failed dispatch off". + expireBackoff(id) again, err := ClaimNext(ctx, db, "inst-b", []ClaimKind{ClaimKindMCPCI}) Expect(err).ToNot(HaveOccurred()) Expect(again.ID).To(Equal(id)) @@ -293,6 +306,112 @@ var _ = Describe("The claim queue", func() { }) }) + Describe("backing a failed dispatch off", func() { + // A release means NOTHING was learned about the work: no worker was + // connected, the tunnel broke, the stream was refused before the + // request left. So the work is retried for ever and is never failed. + // What is bounded is the RATE, because at the poll interval a + // permanently undispatchable row costs an UPDATE every two seconds and, + // worse, is re-claimed ahead of every newer row on every tick. + It("does not offer a just-released row again immediately", func() { + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + _, err := ClaimNext(ctx, db, "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + + Expect(ReleaseClaim(ctx, db, id)).To(Succeed()) + + _, err = ClaimNext(ctx, db, "inst-b", []ClaimKind{ClaimKindMCPCI}) + Expect(err).To(MatchError(ErrNoWork), + "a row re-claimed on the very next tick spins at the poll interval for as long as the fleet is away") + }) + + It("stamps a longer delay on each successive failure", func() { + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + + delays := make([]time.Duration, 0, 3) + for i := 0; i < 3; i++ { + expireBackoff(id) + _, err := ClaimNext(ctx, db, "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + before := time.Now() + Expect(ReleaseClaim(ctx, db, id)).To(Succeed()) + row := rowOf(id) + Expect(row.NotBefore).ToNot(BeNil()) + delays = append(delays, row.NotBefore.Sub(before)) + } + + Expect(delays[1]).To(BeNumerically(">", delays[0])) + Expect(delays[2]).To(BeNumerically(">", delays[1])) + }) + + It("never delays a retry past the cap, however long the row has been stuck", func() { + // The retry is unbounded and the wait is not: work has to become + // claimable again within one cap of the fleet coming back. + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + Expect(db.Model(&WorkClaim{}).Where("id = ?", id). + Update("attempts", 100000).Error).To(Succeed()) + + _, err := ClaimNext(ctx, db, "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + before := time.Now() + Expect(ReleaseClaim(ctx, db, id)).To(Succeed()) + + row := rowOf(id) + Expect(row.NotBefore).ToNot(BeNil()) + Expect(row.NotBefore.Sub(before)).To(BeNumerically("<=", claimBackoffCap+time.Second)) + }) + + It("keeps a stuck row from starving the newer work behind it", func() { + // Rows are claimed oldest first. Before the backoff, the oldest + // undispatchable row was taken again on every tick and held a + // dispatch slot while it failed, so nothing behind it ever ran. + stuck := enqueue(ClaimKindMCPCI, JobEvent{JobID: "stuck"}) + _, err := ClaimNext(ctx, db, "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + Expect(ReleaseClaim(ctx, db, stuck)).To(Succeed()) + + behind := enqueue(ClaimKindMCPCI, JobEvent{JobID: "behind"}) + + got, err := ClaimNext(ctx, db, "inst-b", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + Expect(got.ID).To(Equal(behind)) + }) + + It("retries a stuck row for ever rather than failing work nobody refused", func() { + // The decision this Describe records. A release carries no verdict, + // so there is no attempt count after which the claim is discarded: + // a deployment whose fleet was down for a day runs its queued work + // when the fleet comes back. + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + for i := 0; i < 25; i++ { + expireBackoff(id) + _, err := ClaimNext(ctx, db, "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred(), "the row was discarded after %d attempts", i) + Expect(ReleaseClaim(ctx, db, id)).To(Succeed()) + } + + Expect(rowOf(id).Attempts).To(Equal(25)) + }) + + It("does not delay a claim its replica died holding", func() { + // A reap is not a failed dispatch: the work was never handed to + // anyone, so there is nothing to back off from and delaying it + // would punish the work for the death of the process holding it. + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + _, err := ClaimNext(ctx, db, "dead-replica", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + + released, err := ReapAbandoned(ctx, db, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(released).To(Equal(int64(1))) + + Expect(rowOf(id).NotBefore).To(BeNil()) + again, err := ClaimNext(ctx, db, "inst-b", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + Expect(again.ID).To(Equal(id)) + }) + }) + Describe("reaping what a departed replica left", func() { // The distinction the whole programme rests on, applied to work: a // claim held by a replica that is GONE becomes claimable again, and a @@ -404,6 +523,24 @@ var _ = Describe("The claim queue", func() { "created_at must not be a literal timestamp from this process's clock") }) + It("releases in ONE statement that stamps the backoff on the database clock", func() { + id := enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) + _, err := ClaimNext(ctx, db, "inst-a", []ClaimKind{ClaimKindMCPCI}) + Expect(err).ToNot(HaveOccurred()) + + rec := newClaimSQLRecorder() + Expect(ReleaseClaim(ctx, db.Session(&gorm.Session{Logger: rec}), id)).To(Succeed()) + + sql := strings.ToLower(rec.only()) + // Read-then-compute-then-update would show up as two statements, + // and would compute the delay from this process's clock. + Expect(sql).To(ContainSubstring("make_interval")) + Expect(sql).To(ContainSubstring("now()")) + Expect(sql).To(ContainSubstring("attempts")) + Expect(sql).ToNot(MatchRegexp(`not_before\s*=\s*'`), + "the next-eligible stamp must not be a literal timestamp from this process's clock") + }) + It("reaps in ONE statement whose predicate is replica liveness on the database clock, not claim age", func() { enqueue(ClaimKindMCPCI, JobEvent{JobID: "j1"}) rec := newClaimSQLRecorder() @@ -452,6 +589,13 @@ var _ = Describe("The claim queue", func() { Expect(err).To(MatchError(ContainSubstring("requires PostgreSQL"))) }) + It("refuses to release", func() { + // make_interval and power() are the backoff's, and they fail on + // SQLite with a parse error that reads like a missing column. + Expect(ReleaseClaim(ctx, lite, "some-claim")).To( + MatchError(ContainSubstring("requires PostgreSQL"))) + }) + It("runs no statement at all when it refuses, so the failure cannot be read as a missing table", func() { rec := newClaimSQLRecorder() guarded := lite.Session(&gorm.Session{Logger: rec}) @@ -459,6 +603,7 @@ var _ = Describe("The claim queue", func() { _, _ = EnqueueClaim(ctx, guarded, ClaimKindMCPCI, JobEvent{JobID: "j1"}) _, _ = ReapAbandoned(ctx, guarded, time.Minute) _, _ = OwnerIsLive(ctx, guarded, "inst-a", time.Minute) + _ = ReleaseClaim(ctx, guarded, "some-claim") Expect(rec.count()).To(BeZero()) }) }) diff --git a/core/services/jobs/dispatch_loop.go b/core/services/jobs/dispatch_loop.go index 616025cc6799..df49f857d0c5 100644 --- a/core/services/jobs/dispatch_loop.go +++ b/core/services/jobs/dispatch_loop.go @@ -372,6 +372,14 @@ func (l *DispatchLoop) rebroadcast(nodeType, subject string, raw json.RawMessage // | | CompleteClaim | | // | anything else | ReleaseClaim | an unreachable peer, a lost tunnel or a refused stream is not a verdict | // +// The second row has no attempt ceiling, and there is deliberately no dead +// letter on it. Nothing that reaches it is the worker refusing the work: the +// row exists precisely for the outcomes where nothing was learned, and a +// ceiling would turn "the fleet was away long enough" into a job failure +// nobody reported. What bounds the retry is its RATE and not its count; see +// claimBackoff in claim.go, which is also what stops one stuck row from being +// re-claimed ahead of every newer one on every tick. +// // The line between the two rows is whether a REPLY LINE was decoded, and it is // deliberately NOT cluster.IsWorkerAnswer, though the plan for this task said // it should be. IsWorkerAnswer accepts the tunnel's stream-refusal vocabulary, @@ -390,7 +398,12 @@ func (l *DispatchLoop) rebroadcast(nodeType, subject string, raw json.RawMessage // which is the dropped-result defect the bus carrier had. func (l *DispatchLoop) settleClaim(ctx context.Context, claim *WorkClaim, reply *ClaimReply, callErr error) error { if callErr != nil { - xlog.Warn("Releasing a claim whose dispatch obtained no answer", "claim", claim.ID, "kind", claim.Kind, "error", callErr) + // attempts is the count BEFORE this release, so the line names the + // attempt that just failed. A row that has been stuck for a while + // prints this at the backoff cap rather than at the poll interval, + // which is what makes it readable as a growing number. + xlog.Warn("Releasing a claim whose dispatch obtained no answer; it will be retried after a backoff", + "claim", claim.ID, "kind", claim.Kind, "attempt", claim.Attempts+1, "error", callErr) if err := ReleaseClaim(ctx, l.db, claim.ID); err != nil { return fmt.Errorf("%w (and the claim could not be released: %w)", callErr, err) } diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 82ee26b97ab3..1379efefa7e0 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -371,6 +371,12 @@ This changes three behaviours an operator can see: - **A plain task job (a task whose model configures no MCP servers) is now failed with a reason.** No agent worker has ever served that kind of job, and it used to be published into silence. It is now marked `failed` with `no worker in this deployment serves plain task jobs`. This surfaces a pre-existing gap rather than introducing one. - **Dispatch is at-least-once instead of at-most-once.** A transport failure (a lost tunnel, an unreachable peer, a replica that died mid-dispatch) returns the work to the pool and increments the row's `attempts`; only the worker's own answer, success or failure, removes it. +**A claim that could not be dispatched is retried for ever, at a backing-off rate.** There is no attempt limit and no dead-letter queue, on purpose. Everything that returns a claim to the pool is a case where nothing was learned about the work: no agent worker was connected, the tunnel broke, a peer could not be reached, the stream was refused before the request body left the frontend. None of those is the worker saying it ran the job and it failed, so failing the job on any of them would report an absent connection as a verdict. A deployment whose agent workers are down for a day runs its queued work when they come back. + +What is bounded is the retry RATE. Each failed dispatch stamps the row with the earliest it may be claimed again, doubling from **2 seconds** to a cap of **60 seconds**, measured on the database clock. Two things follow. Queued work becomes claimable again within a minute of the fleet returning, and one permanently undispatchable row no longer starves the queue: rows are claimed oldest-first, so before the backoff the oldest stuck row was re-claimed ahead of every newer one on every tick and held a dispatch slot while it failed. A claim released by the reap because its replica died carries no delay at all, since that work was never handed to anyone. + +A row that keeps failing logs `Releasing a claim whose dispatch obtained no answer` with a growing `attempt` count, at most once per backoff interval. `SELECT id, kind, attempts, not_before FROM work_claims ORDER BY attempts DESC` is what tells you a job is stuck rather than merely queued. + **A claim held by a replica that has gone becomes claimable again; a claim held by a replica that is merely slow is never taken away from it.** The reap asks whether the claim's owner is still a live replica in the `instances` table, on the database clock, and never how long the claim has been held: a job that legitimately runs for an hour on a heartbeating replica is left alone, and a claim whose owner stopped heartbeating is released on the next tick (within `30s`, the replica-liveness window). **A frontend replica with no advertised peer address claims no work.** Such a replica has no row in the `instances` table, so no peer can tell its claims from ones a dead replica left, and another replica would take the work away from it mid-run. It logs an error naming `LOCALAI_DISTRIBUTED_ADVERTISE_ADDR` and starts claiming as soon as it registers. This is the same configuration that already makes a replica's workers unroutable from its peers. From 55bc99c85abc9dfae6807eea9c3f15a588e70194 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 6 Sep 2026 05:06:47 +0000 Subject: [PATCH 109/153] test: fix the three data races that made -race runs noisy None was introduced by this branch and all three are in test code, which is what made them survive: every suite passed on every run and only the race detector said otherwise. A known-failing -race run is worse than a noisy one, because a real race raised by production code lands in the same report and is read as one of these. galleryop: gatedModelManager guarded the recorded names and not the gate channel itself. A spec frees the parked worker by closing the gate and installing a fresh one, on the spec goroutine, while the worker goroutine reads the field to park on it. The channel is now read and replaced under the same mutex, and cleanup closes idempotently. pkg/model: two specs swapped xlog's package logger to capture output and swapped it back on cleanup. xlog.SetLogger writes an unsynchronised global, so the restore raced with the backend process watcher, which logs while a process is stopping; the captured bytes.Buffer was written by that goroutine and read by an Eventually at the same time. SetLogger is now called once for the whole test binary, from init, before a goroutine exists to race with, and a spec swaps the DESTINATION under a mutex through a routing slog.Handler. Per-spec level filtering is preserved deliberately: one of these specs asserts that a debug emission is filtered OUT and would pass vacuously against a handler that recorded everything. openai: fakeTransport appended to its event and audio logs from the response and turn coordinators' goroutines while a spec ranged over them. Both are behind a mutex and are read through snapshot accessors; the fields are renamed so a raw read from another spec file does not compile. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .../openai/realtime_classifier_test.go | 10 +- .../endpoints/openai/realtime_doubles_test.go | 39 ++++++- .../openai/realtime_semantic_vad_test.go | 6 +- .../openai/realtime_sound_detection_test.go | 4 +- .../endpoints/openai/realtime_stream_test.go | 12 +- .../realtime_voicegate_integration_test.go | 2 +- .../galleryop/cancellable_phase_test.go | 45 +++++++- pkg/model/initializers_load_logging_test.go | 14 +-- pkg/model/log_capture_test.go | 104 ++++++++++++++++++ pkg/model/process_exit_test.go | 10 +- 10 files changed, 199 insertions(+), 47 deletions(-) create mode 100644 pkg/model/log_capture_test.go diff --git a/core/http/endpoints/openai/realtime_classifier_test.go b/core/http/endpoints/openai/realtime_classifier_test.go index b8630ca8120b..757c91dec8ba 100644 --- a/core/http/endpoints/openai/realtime_classifier_test.go +++ b/core/http/endpoints/openai/realtime_classifier_test.go @@ -45,7 +45,7 @@ var classifierTestHistory = schema.Messages{ func classifierResultEvents(t *fakeTransport) []types.ClassifierResultEvent { var out []types.ClassifierResultEvent - for _, e := range t.events { + for _, e := range t.recordedEvents() { if ev, ok := e.(types.ClassifierResultEvent); ok { out = append(out, ev) } @@ -57,7 +57,7 @@ func classifierResultEvents(t *fakeTransport) []types.ClassifierResultEvent { // item — what a classifier response actually "spoke". func replyTexts(t *fakeTransport) []string { var out []string - for _, e := range t.events { + for _, e := range t.recordedEvents() { if ev, ok := e.(types.ResponseOutputTextDoneEvent); ok { out = append(out, ev.Text) } @@ -277,7 +277,7 @@ var _ = Describe("classifierRespond", func() { Expect(t.countEvents(types.ServerEventTypeResponseOutputTextDone)).To(Equal(1)) Expect(t.countEvents(types.ServerEventTypeResponseFunctionCallArgumentsDone)).To(Equal(1)) var fcArgs string - for _, e := range t.events { + for _, e := range t.recordedEvents() { if done, ok := e.(types.ResponseFunctionCallArgumentsDoneEvent); ok { fcArgs = done.Arguments } @@ -656,7 +656,7 @@ var _ = Describe("classifierRespond slot filling", func() { Expect(results[0].Arguments).To(MatchJSON(`{"direction":"up","distance":3,"units":"meters"}`)) var fcArgs string - for _, e := range t.events { + for _, e := range t.recordedEvents() { if done, ok := e.(types.ResponseFunctionCallArgumentsDoneEvent); ok { fcArgs = done.Arguments } @@ -695,7 +695,7 @@ var _ = Describe("classifierRespond slot filling", func() { Expect(handled).To(BeTrue()) var fcArgs string - for _, e := range t.events { + for _, e := range t.recordedEvents() { if done, ok := e.(types.ResponseFunctionCallArgumentsDoneEvent); ok { fcArgs = done.Arguments } diff --git a/core/http/endpoints/openai/realtime_doubles_test.go b/core/http/endpoints/openai/realtime_doubles_test.go index a2c104b3c29b..0d7cc3fa6082 100644 --- a/core/http/endpoints/openai/realtime_doubles_test.go +++ b/core/http/endpoints/openai/realtime_doubles_test.go @@ -16,9 +16,17 @@ import ( // fakeTransport records the server events and audio sent to a realtime client // so streaming behaviour can be asserted without a real WebSocket/WebRTC peer. // It is not a *WebRTCTransport, so handler code takes the WebSocket path. +// +// Every field is behind the mutex, and the recorded slices are read only +// through recordedEvents and recordedAudio. A real transport is written to by +// the response and turn coordinators' goroutines while the spec goroutine +// reads what has arrived so far, so a double that appended without a lock could +// not be driven the way production drives it. Both fields are named with a +// `Log` suffix so a raw read from another spec file does not compile. type fakeTransport struct { - events []types.ServerEvent - audio []fakeAudioChunk + mu sync.Mutex + eventLog []types.ServerEvent + audioLog []fakeAudioChunk } type fakeAudioChunk struct { @@ -27,23 +35,42 @@ type fakeAudioChunk struct { } func (f *fakeTransport) SendEvent(e types.ServerEvent) error { - f.events = append(f.events, e) + f.mu.Lock() + defer f.mu.Unlock() + f.eventLog = append(f.eventLog, e) return nil } func (f *fakeTransport) ReadEvent() ([]byte, error) { return nil, nil } func (f *fakeTransport) SendAudio(_ context.Context, pcm []byte, sampleRate int) error { - f.audio = append(f.audio, fakeAudioChunk{pcm: pcm, sampleRate: sampleRate}) + f.mu.Lock() + defer f.mu.Unlock() + f.audioLog = append(f.audioLog, fakeAudioChunk{pcm: pcm, sampleRate: sampleRate}) return nil } func (f *fakeTransport) Close() error { return nil } +// recordedEvents returns a snapshot of the events sent so far. A COPY, because +// the caller ranges over it while the coordinators may still be sending. +func (f *fakeTransport) recordedEvents() []types.ServerEvent { + f.mu.Lock() + defer f.mu.Unlock() + return append([]types.ServerEvent(nil), f.eventLog...) +} + +// recordedAudio returns a snapshot of the audio chunks sent so far. +func (f *fakeTransport) recordedAudio() []fakeAudioChunk { + f.mu.Lock() + defer f.mu.Unlock() + return append([]fakeAudioChunk(nil), f.audioLog...) +} + // countEvents returns how many recorded events have the given type. func (f *fakeTransport) countEvents(et types.ServerEventType) int { n := 0 - for _, e := range f.events { + for _, e := range f.recordedEvents() { if e.ServerEventType() == et { n++ } @@ -55,7 +82,7 @@ func (f *fakeTransport) countEvents(et types.ServerEventType) int { // delta event — i.e. the text streamed to the client as it is generated. func (f *fakeTransport) transcriptDeltaText() string { var b strings.Builder - for _, e := range f.events { + for _, e := range f.recordedEvents() { if d, ok := e.(types.ResponseOutputAudioTranscriptDeltaEvent); ok { b.WriteString(d.Delta) } diff --git a/core/http/endpoints/openai/realtime_semantic_vad_test.go b/core/http/endpoints/openai/realtime_semantic_vad_test.go index c36e1356356b..7a2fea085dec 100644 --- a/core/http/endpoints/openai/realtime_semantic_vad_test.go +++ b/core/http/endpoints/openai/realtime_semantic_vad_test.go @@ -269,7 +269,7 @@ var _ = Describe("liveTurnState", func() { lts.drainEvents(1.0) var got []types.ConversationItemInputAudioTranscriptionDeltaEvent - for _, e := range ftr.events { + for _, e := range ftr.recordedEvents() { if d, ok := e.(types.ConversationItemInputAudioTranscriptionDeltaEvent); ok { got = append(got, d) } @@ -335,7 +335,7 @@ var _ = Describe("commitUtteranceWithTranscript", func() { Expect(tr.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionCompleted)).To(Equal(1)) var completed types.ConversationItemInputAudioTranscriptionCompletedEvent - for _, e := range tr.events { + for _, e := range tr.recordedEvents() { if c, ok := e.(types.ConversationItemInputAudioTranscriptionCompletedEvent); ok { completed = c } @@ -407,7 +407,7 @@ var _ = Describe("emitPrecomputedTranscription", func() { Expect(tr.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionDelta)).To(Equal(2), "empty deltas skipped") Expect(tr.countEvents(types.ServerEventTypeConversationItemInputAudioTranscriptionCompleted)).To(Equal(1)) - for _, e := range tr.events { + for _, e := range tr.recordedEvents() { switch ev := e.(type) { case types.ConversationItemInputAudioTranscriptionDeltaEvent: Expect(ev.ItemID).To(Equal("item42")) diff --git a/core/http/endpoints/openai/realtime_sound_detection_test.go b/core/http/endpoints/openai/realtime_sound_detection_test.go index e440e80c36d8..02ced312a9e8 100644 --- a/core/http/endpoints/openai/realtime_sound_detection_test.go +++ b/core/http/endpoints/openai/realtime_sound_detection_test.go @@ -38,7 +38,7 @@ var _ = Describe("emitSoundDetection", func() { Expect(err).ToNot(HaveOccurred()) Expect(t.countEvents(types.ServerEventTypeConversationItemSoundDetection)).To(Equal(1)) - ev, ok := t.events[0].(types.ConversationItemSoundDetectionEvent) + ev, ok := t.recordedEvents()[0].(types.ConversationItemSoundDetectionEvent) Expect(ok).To(BeTrue()) Expect(ev.ItemID).To(Equal("item1")) Expect(ev.ContentIndex).To(Equal(0)) @@ -62,7 +62,7 @@ var _ = Describe("emitSoundDetection", func() { Expect(err).ToNot(HaveOccurred()) Expect(t.countEvents(types.ServerEventTypeConversationItemSoundDetection)).To(Equal(1)) - ev, ok := t.events[0].(types.ConversationItemSoundDetectionEvent) + ev, ok := t.recordedEvents()[0].(types.ConversationItemSoundDetectionEvent) Expect(ok).To(BeTrue()) Expect(ev.Detections).To(BeEmpty()) }) diff --git a/core/http/endpoints/openai/realtime_stream_test.go b/core/http/endpoints/openai/realtime_stream_test.go index 2d5d7d7a1162..d6962c087d16 100644 --- a/core/http/endpoints/openai/realtime_stream_test.go +++ b/core/http/endpoints/openai/realtime_stream_test.go @@ -250,8 +250,8 @@ var _ = Describe("triggerResponse", func() { // The single terminal carries the produced output item and the usage — // both empty in the legacy code. var done *types.ResponseDoneEvent - for i := range t.events { - if d, ok := t.events[i].(types.ResponseDoneEvent); ok { + for _, e := range t.recordedEvents() { + if d, ok := e.(types.ResponseDoneEvent); ok { done = &d } } @@ -287,8 +287,8 @@ var _ = Describe("triggerResponse", func() { var created *types.ResponseCreatedEvent var done *types.ResponseDoneEvent - for i := range t.events { - switch e := t.events[i].(type) { + for _, sent := range t.recordedEvents() { + switch e := sent.(type) { case types.ResponseCreatedEvent: created = &e case types.ResponseDoneEvent: @@ -317,8 +317,8 @@ var _ = Describe("triggerResponse", func() { triggerResponse(context.Background(), session, &Conversation{}, t, nil) - for i := range t.events { - if d, ok := t.events[i].(types.ResponseDoneEvent); ok { + for _, e := range t.recordedEvents() { + if d, ok := e.(types.ResponseDoneEvent); ok { Expect(d.Response.Metadata).To(BeEmpty()) } } diff --git a/core/http/endpoints/openai/realtime_voicegate_integration_test.go b/core/http/endpoints/openai/realtime_voicegate_integration_test.go index b0f7f0b49d92..caacab8774b6 100644 --- a/core/http/endpoints/openai/realtime_voicegate_integration_test.go +++ b/core/http/endpoints/openai/realtime_voicegate_integration_test.go @@ -67,7 +67,7 @@ func itSession(gate *voiceGate) (*Session, *fakeModel) { // hasSpeakerNotAuthorized reports whether a speaker_not_authorized error event // was emitted to the client. func hasSpeakerNotAuthorized(tr *fakeTransport) bool { - for _, e := range tr.events { + for _, e := range tr.recordedEvents() { if ev, ok := e.(types.ErrorEvent); ok && ev.Error.Code == "speaker_not_authorized" { return true } diff --git a/core/services/galleryop/cancellable_phase_test.go b/core/services/galleryop/cancellable_phase_test.go index 47c98c660fce..5ef05ecc90c1 100644 --- a/core/services/galleryop/cancellable_phase_test.go +++ b/core/services/galleryop/cancellable_phase_test.go @@ -17,6 +17,13 @@ import ( // Parking is what makes the running phase observable at all: without it the // handler-entry status is overwritten by the terminal write before a spec can // read it. +// +// The gate CHANNEL is under the mutex, not only the recorded names. A spec that +// frees the worker replaces the channel so the next operation parks on a fresh +// one, and the worker goroutine reads the field to park on it; the two happen +// on different goroutines and raced. It was a race in this double and not in +// anything it stands for, which is exactly why it survived: the suite passed +// every run and only -race said so. type gatedModelManager struct { mu sync.Mutex started []string @@ -39,15 +46,44 @@ func (m *gatedModelManager) Started() []string { return append([]string(nil), m.started...) } +// gateCh reads the current gate, for a handler about to park on it. +func (m *gatedModelManager) gateCh() chan struct{} { + m.mu.Lock() + defer m.mu.Unlock() + return m.gate +} + +// releaseGate frees every handler parked on the current gate and arms a fresh +// one, so the next operation parks again. +func (m *gatedModelManager) releaseGate() { + m.mu.Lock() + defer m.mu.Unlock() + close(m.gate) + m.gate = make(chan struct{}) +} + +// closeGate frees every parked handler without arming another. It is what +// cleanup does, and it is idempotent so a spec that already released can be +// cleaned up after. +func (m *gatedModelManager) closeGate() { + m.mu.Lock() + defer m.mu.Unlock() + select { + case <-m.gate: + default: + close(m.gate) + } +} + func (m *gatedModelManager) InstallModel(_ context.Context, op *galleryop.ManagementOp[gallery.GalleryModel, gallery.ModelConfig], _ galleryop.ProgressCallback) error { m.record(op.GalleryElementName) - <-m.gate + <-m.gateCh() return nil } func (m *gatedModelManager) DeleteModel(name string) error { m.record(name) - <-m.gate + <-m.gateCh() return nil } @@ -120,7 +156,7 @@ var _ = Describe("operation cancellability by phase", func() { BeforeEach(func() { manager = newGatedModelManager() - DeferCleanup(func() { close(manager.gate) }) + DeferCleanup(manager.closeGate) svc.SetModelManager(manager) ctx, cancel := context.WithCancel(context.Background()) @@ -188,8 +224,7 @@ var _ = Describe("operation cancellability by phase", func() { // Free the worker: it loops back to an empty channel because the // delivery goroutine gave up on the send. - close(manager.gate) - manager.gate = make(chan struct{}) + manager.releaseGate() Consistently(manager.Started, "500ms", "20ms").ShouldNot( ContainElement("localai@doomed-removal"), diff --git a/pkg/model/initializers_load_logging_test.go b/pkg/model/initializers_load_logging_test.go index e785b03db648..0a863f19213b 100644 --- a/pkg/model/initializers_load_logging_test.go +++ b/pkg/model/initializers_load_logging_test.go @@ -1,12 +1,10 @@ package model import ( - "bytes" "log/slog" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/system" - "github.com/mudler/xlog" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,7 +21,7 @@ import ( var _ = Describe("backendLoader load logging", func() { var ( ml *ModelLoader - captured *bytes.Buffer + captured *syncBuffer ) BeforeEach(func() { @@ -33,16 +31,10 @@ var _ = Describe("backendLoader load logging", func() { // Capture at info level so a debug-level emission is filtered out: the // assertions then fail on severity, not merely on wording. - captured = &bytes.Buffer{} - handler := slog.NewTextHandler(captured, &slog.HandlerOptions{Level: slog.LevelInfo}) - xlog.SetLogger(xlog.NewLoggerWithHandler(handler, xlog.LogLevelInfo)) + captured = captureLogs(slog.LevelInfo) }) - AfterEach(func() { - // xlog exposes no getter for the package logger, so restore the same - // default the suite entrypoint installs rather than the prior value. - xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text")) - }) + AfterEach(stopCapturingLogs) Context("when the model is already resident", func() { BeforeEach(func() { diff --git a/pkg/model/log_capture_test.go b/pkg/model/log_capture_test.go new file mode 100644 index 000000000000..3057eac3f86c --- /dev/null +++ b/pkg/model/log_capture_test.go @@ -0,0 +1,104 @@ +package model + +import ( + "bytes" + "context" + "log/slog" + "sync" + + "github.com/mudler/xlog" +) + +// xlog keeps its logger in a package-level variable that SetLogger writes +// without a lock, so every xlog.Info in the process reads that variable +// concurrently with the write. A spec that swapped the logger to capture output +// and swapped it back on cleanup therefore raced with any goroutine this +// package had left logging: the backend process watcher, which logs while a +// process is stopping, is one of those on every run. +// +// So SetLogger is called exactly ONCE for this whole test binary, from init, +// before a goroutine exists to race with. What a spec swaps afterwards is the +// destination, under a mutex, through the routing handler below. That keeps the +// per-spec level filtering intact, which matters: at least one spec asserts +// that a debug emission is filtered OUT, and would pass vacuously against a +// handler that simply recorded everything. +type routingLogHandler struct { + mu sync.Mutex + to slog.Handler +} + +// sharedLogHandler is the one handler xlog is given for this binary. +var sharedLogHandler = &routingLogHandler{} + +func init() { + xlog.SetLogger(xlog.NewLoggerWithHandler(sharedLogHandler, xlog.LogLevelInfo)) +} + +func (h *routingLogHandler) current() slog.Handler { + h.mu.Lock() + defer h.mu.Unlock() + return h.to +} + +// arm points the shared handler at inner, or discards everything when inner is +// nil. Returns nothing: a spec restores by arming nil, because xlog exposes no +// getter and there is no previous value to hand back. +func (h *routingLogHandler) arm(inner slog.Handler) { + h.mu.Lock() + defer h.mu.Unlock() + h.to = inner +} + +func (h *routingLogHandler) Enabled(ctx context.Context, level slog.Level) bool { + inner := h.current() + return inner != nil && inner.Enabled(ctx, level) +} + +func (h *routingLogHandler) Handle(ctx context.Context, r slog.Record) error { + inner := h.current() + if inner == nil { + return nil + } + return inner.Handle(ctx, r) +} + +// WithAttrs and WithGroup hand back the router itself. xlog never calls either +// (it has no With), and a copy would be a second handler holding the same +// mutex by value. +func (h *routingLogHandler) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *routingLogHandler) WithGroup(string) slog.Handler { return h } + +var _ slog.Handler = (*routingLogHandler)(nil) + +// syncBuffer is a bytes.Buffer that a log handler and a poller may share. +// +// The diagnostic under test is written by a goroutine and read by an Eventually +// on the spec goroutine, which is a plain concurrent use of a bytes.Buffer. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// captureLogs sends everything logged through xlog at or above level into a +// fresh buffer, until stopCapturingLogs is called. +func captureLogs(level slog.Level) *syncBuffer { + captured := &syncBuffer{} + sharedLogHandler.arm(slog.NewTextHandler(captured, &slog.HandlerOptions{Level: level})) + return captured +} + +// stopCapturingLogs sends everything logged afterwards nowhere, which is what +// a test binary wants of a package whose goroutines outlive their spec. +func stopCapturingLogs() { sharedLogHandler.arm(nil) } diff --git a/pkg/model/process_exit_test.go b/pkg/model/process_exit_test.go index 1f0bd9093ff2..4146e9464037 100644 --- a/pkg/model/process_exit_test.go +++ b/pkg/model/process_exit_test.go @@ -1,13 +1,11 @@ package model import ( - "bytes" "log/slog" "os" "path/filepath" "github.com/mudler/LocalAI/pkg/system" - "github.com/mudler/xlog" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -20,12 +18,8 @@ var _ = Describe("backend process exit diagnostics", func() { backendPath := filepath.Join(tmpDir, "failing-backend") Expect(os.WriteFile(backendPath, []byte("#!/bin/sh\nprintf '%s' \"$TMPDIR\" > \"$0.tmpdir\"\necho 'first diagnostic' >&2\necho 'fatal metal pipeline error' >&2\nexit 42\n"), 0o700)).To(Succeed()) - captured := &bytes.Buffer{} - handler := slog.NewTextHandler(captured, &slog.HandlerOptions{Level: slog.LevelWarn}) - xlog.SetLogger(xlog.NewLoggerWithHandler(handler, xlog.LogLevelWarn)) - DeferCleanup(func() { - xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text")) - }) + captured := captureLogs(slog.LevelWarn) + DeferCleanup(stopCapturingLogs) loader := NewModelLoader(&system.SystemState{Model: system.Model{ModelsPath: tmpDir}}) process, err := loader.startProcess(backendPath, "test-model", "127.0.0.1:65535", nil) From 22ef89cba086180c1fc29ff8129ca29951280257 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 6 Sep 2026 07:01:23 +0000 Subject: [PATCH 110/153] test(worker): script the kernel out of the port bookkeeping specs Probing every candidate before handing it out is right, and it made sixteen specs that were never about the kernel depend on it. They build a supervisor directly, name the ports they expect literally, and those literals sit inside Linux's default ephemeral range, so with the real probe each one asks this host whether 50051 is bindable at that instant. The first full -race run over ./core/... and ./pkg/... after the probe landed went red on four of them, and holding 50051, 50052, 50060 and 50061 from another process turns eleven red deterministically. Nothing was wrong with the allocator in either case: something else on the machine held a port, which is the situation the probe exists to survive. So the specs that assert bookkeeping now inject a probe that always says yes, and say why once. The two specs that are about the probe leave the field unset and keep asking real sockets, which is what still fails if the probe is removed. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- core/services/worker/port_allocator_test.go | 26 ++++++++++++++++++++ core/services/worker/port_quarantine_test.go | 10 ++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/core/services/worker/port_allocator_test.go b/core/services/worker/port_allocator_test.go index 8677c840a84a..e5ec3c36fb75 100644 --- a/core/services/worker/port_allocator_test.go +++ b/core/services/worker/port_allocator_test.go @@ -16,6 +16,22 @@ func mustAllocate(s *backendSupervisor, key string) int { return port } +// bookkeepingProbe is the port probe every supervisor in this file and in +// port_quarantine_test.go runs with, and it always says yes. +// +// Those specs are about the allocator's own bookkeeping, and each names the +// ports it expects literally. Those literals sit inside Linux's default +// ephemeral range (32768-60999), so with the real probe every one of them asks +// this host whether 50051 happens to be bindable at that instant. Anything +// holding one turns the spec red: another suite in the same run, or any other +// process on the machine. Eleven of them failed together the first time +// something else on the box held 50051, 50052, 50060 and 50061. +// +// Scripting the kernel out is what keeps these specs about the bookkeeping. The +// probe itself is covered against real sockets in port_bindable_test.go, which +// leaves this field unset on purpose. +func bookkeepingProbe(int) bool { return true } + var _ = Describe("Backend gRPC port allocator", func() { Describe("range exhaustion", func() { It("reports an explicit error instead of handing out an unbindable port", func() { @@ -25,6 +41,7 @@ var _ = Describe("Backend gRPC port allocator", func() { // at the allocator. Exhaustion must be named as exhaustion. s := &backendSupervisor{ processes: map[string]*backendProcess{}, + portIsFree: bookkeepingProbe, minPort: 50051, nextPort: 50051, maxPort: 50052, @@ -47,6 +64,7 @@ var _ = Describe("Backend gRPC port allocator", func() { It("does not hand out a port beyond the end of the range", func() { s := &backendSupervisor{ processes: map[string]*backendProcess{}, + portIsFree: bookkeepingProbe, minPort: 50051, nextPort: 50051, maxPort: 50053, @@ -71,6 +89,7 @@ var _ = Describe("Backend gRPC port allocator", func() { bp := &backendProcess{port: 50051} s := &backendSupervisor{ processes: map[string]*backendProcess{"model#0": bp}, + portIsFree: bookkeepingProbe, minPort: 50051, nextPort: 50052, maxPort: 50060, @@ -91,6 +110,7 @@ var _ = Describe("Backend gRPC port allocator", func() { // the range. Before the fix this consumed a fresh port per restart. s := &backendSupervisor{ processes: map[string]*backendProcess{}, + portIsFree: bookkeepingProbe, minPort: 50051, nextPort: 50051, maxPort: 50053, @@ -140,6 +160,7 @@ var _ = Describe("Backend gRPC port allocator", func() { // by construction rather than by racing a quarantine timer. s := &backendSupervisor{ processes: map[string]*backendProcess{}, + portIsFree: bookkeepingProbe, minPort: 50051, nextPort: 50051, maxPort: 50060, @@ -161,6 +182,7 @@ var _ = Describe("Backend gRPC port allocator", func() { It("reuses an unowned port before growing the range", func() { s := &backendSupervisor{ processes: map[string]*backendProcess{}, + portIsFree: bookkeepingProbe, minPort: 50051, nextPort: 50051, maxPort: 50060, @@ -181,6 +203,7 @@ var _ = Describe("Backend gRPC port allocator", func() { // vanishingly rare misroute for a guaranteed outage. s := &backendSupervisor{ processes: map[string]*backendProcess{}, + portIsFree: bookkeepingProbe, minPort: 50051, nextPort: 50051, maxPort: 50051, @@ -207,6 +230,7 @@ var _ = Describe("Backend gRPC port allocator", func() { // problem. s := &backendSupervisor{ processes: map[string]*backendProcess{}, + portIsFree: bookkeepingProbe, minPort: 50051, nextPort: 50051, maxPort: 50060, @@ -228,6 +252,7 @@ var _ = Describe("Backend gRPC port allocator", func() { // zero would hand a just-released port straight to another model. s := &backendSupervisor{ processes: map[string]*backendProcess{}, + portIsFree: bookkeepingProbe, minPort: 50051, nextPort: 50051, maxPort: 50060, @@ -248,6 +273,7 @@ var _ = Describe("Backend gRPC port allocator", func() { // range has ports, however many distinct keys the worker sees. s := &backendSupervisor{ processes: map[string]*backendProcess{}, + portIsFree: bookkeepingProbe, minPort: 50051, nextPort: 50051, maxPort: 50052, diff --git a/core/services/worker/port_quarantine_test.go b/core/services/worker/port_quarantine_test.go index c5b48b69c3d0..df048f9ea0ad 100644 --- a/core/services/worker/port_quarantine_test.go +++ b/core/services/worker/port_quarantine_test.go @@ -28,6 +28,7 @@ var _ = Describe("Backend port quarantine", func() { bp := &backendProcess{port: 50051} s := &backendSupervisor{ processes: map[string]*backendProcess{"model#0": bp}, + portIsFree: bookkeepingProbe, nextPort: 50060, portQuarantine: time.Hour, } @@ -49,6 +50,7 @@ var _ = Describe("Backend port quarantine", func() { bp := &backendProcess{port: 50051} s := &backendSupervisor{ processes: map[string]*backendProcess{"model#0": bp}, + portIsFree: bookkeepingProbe, nextPort: 50060, portQuarantine: time.Millisecond, } @@ -68,6 +70,7 @@ var _ = Describe("Backend port quarantine", func() { bp := &backendProcess{port: 50051} s := &backendSupervisor{ processes: map[string]*backendProcess{"model#0": bp}, + portIsFree: bookkeepingProbe, nextPort: 50060, portQuarantine: time.Hour, } @@ -83,6 +86,7 @@ var _ = Describe("Backend port quarantine", func() { It("prefers a released port over growing the range", func() { s := &backendSupervisor{ processes: map[string]*backendProcess{}, + portIsFree: bookkeepingProbe, nextPort: 50060, portQuarantine: time.Millisecond, } @@ -98,6 +102,7 @@ var _ = Describe("Backend port quarantine", func() { It("does not sweep a port whose quarantine is still running", func() { s := &backendSupervisor{ processes: map[string]*backendProcess{}, + portIsFree: bookkeepingProbe, nextPort: 50060, portQuarantine: time.Hour, } @@ -113,8 +118,9 @@ var _ = Describe("Backend port quarantine", func() { // The zero value must not degrade to "no quarantine": every // supervisor built outside the tests leaves the field unset. s := &backendSupervisor{ - processes: map[string]*backendProcess{}, - nextPort: 50060, + processes: map[string]*backendProcess{}, + portIsFree: bookkeepingProbe, + nextPort: 50060, } s.releasePort(50051) From 3aab79faef9930ac94168778ec4c6915fb519102 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 6 Sep 2026 07:58:36 +0000 Subject: [PATCH 111/153] test(openai): drop the audio snapshot nothing reads The mutex round the realtime transport double added a snapshot accessor for each recorded slice. Only the event one has a caller, so make lint refuses the build: realtime_doubles_test.go:64:25: func (*fakeTransport).recordedAudio is unused (unused) No spec has ever read the audio log, before the mutex or after it, so the accessor is deleted rather than nolinted and the struct comment says where the next one comes from. audioLog stays written, because a double that silently discarded what a coordinator sent it would be a different double. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .../endpoints/openai/realtime_doubles_test.go | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/core/http/endpoints/openai/realtime_doubles_test.go b/core/http/endpoints/openai/realtime_doubles_test.go index 0d7cc3fa6082..4ad69a024977 100644 --- a/core/http/endpoints/openai/realtime_doubles_test.go +++ b/core/http/endpoints/openai/realtime_doubles_test.go @@ -17,12 +17,14 @@ import ( // so streaming behaviour can be asserted without a real WebSocket/WebRTC peer. // It is not a *WebRTCTransport, so handler code takes the WebSocket path. // -// Every field is behind the mutex, and the recorded slices are read only -// through recordedEvents and recordedAudio. A real transport is written to by -// the response and turn coordinators' goroutines while the spec goroutine -// reads what has arrived so far, so a double that appended without a lock could -// not be driven the way production drives it. Both fields are named with a -// `Log` suffix so a raw read from another spec file does not compile. +// Every field is behind the mutex, and the events are read only through +// recordedEvents. A real transport is written to by the response and turn +// coordinators' goroutines while the spec goroutine reads what has arrived so +// far, so a double that appended without a lock could not be driven the way +// production drives it. Both fields are named with a `Log` suffix so a raw read +// from another spec file does not compile; audioLog has no reader yet, and the +// accessor for it is left to whichever spec first needs one, because an unread +// one does not build. type fakeTransport struct { mu sync.Mutex eventLog []types.ServerEvent @@ -60,13 +62,6 @@ func (f *fakeTransport) recordedEvents() []types.ServerEvent { return append([]types.ServerEvent(nil), f.eventLog...) } -// recordedAudio returns a snapshot of the audio chunks sent so far. -func (f *fakeTransport) recordedAudio() []fakeAudioChunk { - f.mu.Lock() - defer f.mu.Unlock() - return append([]fakeAudioChunk(nil), f.audioLog...) -} - // countEvents returns how many recorded events have the given type. func (f *fakeTransport) countEvents(et types.ServerEventType) int { n := 0 From 3cce3e35036bdcc727ab022cc0d70e519648f5ac Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:08:13 +0000 Subject: [PATCH 112/153] test: guard shared PostgreSQL setup on macOS Apply the existing Docker skip at the container entry point so the maintenance-connection regression cannot bypass it. Assisted-by: Codex:GPT-6 Signed-off-by: Ettore Di Giacinto --- core/services/testutil/testdb.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/services/testutil/testdb.go b/core/services/testutil/testdb.go index bd732ef66862..4201293e4117 100644 --- a/core/services/testutil/testdb.go +++ b/core/services/testutil/testdb.go @@ -87,6 +87,9 @@ var _ = AfterSuite(func() { // true thing. func sharedPostgres() string { GinkgoHelper() + if runtime.GOOS == "darwin" { + Skip("testcontainers requires Docker, not available on macOS CI") + } sharedOnce.Do(func() { ctx := context.Background() @@ -133,9 +136,6 @@ func SetupTestDB() *gorm.DB { // out and lose them on the next one. func SetupTestDBWithDSN() (*gorm.DB, string) { GinkgoHelper() - if runtime.GOOS == "darwin" { - Skip("testcontainers requires Docker, not available on macOS CI") - } dsn := sharedPostgres() name := fmt.Sprintf("testdb_%d", dbCounter.Add(1)) From 595dfb97e9d172515868878f31f73745cf3b5916 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:08:13 +0000 Subject: [PATCH 113/153] test: keep distributed specs in their own targets The general e2e target builds a container, not the host binary that cluster specs require. Run only its own package; dedicated distributed and cluster targets retain the full subpackage coverage. Assisted-by: Codex:GPT-6 Signed-off-by: Ettore Di Giacinto --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 98fb01f259be..574f3cb28e7c 100644 --- a/Makefile +++ b/Makefile @@ -481,11 +481,13 @@ prepare-e2e: run-e2e-image: docker run -p 5390:8080 -e MODELS_PATH=/models -e THREADS=1 -e DEBUG=true -d --rm -v $(TEST_DIR):/models --name e2e-tests-$(RANDOM) localai-tests +# Distributed subpackages have separate targets that provision PostgreSQL and +# build the host binaries; this target exercises the container-backed API only. test-e2e: build-mock-backend build-cloud-proxy-backend prepare-e2e run-e2e-image @echo 'Running e2e tests' BUILD_TYPE=$(BUILD_TYPE) \ LOCALAI_API=http://$(E2E_BRIDGE_IP):5390 \ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v ./tests/e2e $(MAKE) clean-mock-backend $(MAKE) clean-cloud-proxy-backend $(MAKE) teardown-e2e From 553d11afb3ec4e51cfae8a7f036f12b393846a6d Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Tue, 8 Sep 2026 01:05:42 +0000 Subject: [PATCH 114/153] fix(worker): resolve staging directory symlinks Resolve allowed directories before comparing them with resolved files. Otherwise staging rejects valid files under macOS temporary paths. Cover aliased roots, sibling paths, and symlinks escaping the root. Assisted-by: Codex:gpt-6 Signed-off-by: Ettore Di Giacinto --- core/services/worker/file_staging.go | 4 +++ core/services/worker/file_staging_test.go | 32 +++++++++++++++++++++++ docs/content/features/distributed-mode.md | 4 +++ 3 files changed, 40 insertions(+) create mode 100644 core/services/worker/file_staging_test.go diff --git a/core/services/worker/file_staging.go b/core/services/worker/file_staging.go index 9e65f2cedcb5..0c7b81e01ffb 100644 --- a/core/services/worker/file_staging.go +++ b/core/services/worker/file_staging.go @@ -21,6 +21,10 @@ func isPathAllowed(path string, allowedDirs []string) bool { if err != nil { continue } + // Compare both sides after resolving aliases such as macOS /var. + if resolvedDir, err := filepath.EvalSymlinks(absDir); err == nil { + absDir = resolvedDir + } if strings.HasPrefix(resolved, absDir+string(filepath.Separator)) || resolved == absDir { return true } diff --git a/core/services/worker/file_staging_test.go b/core/services/worker/file_staging_test.go new file mode 100644 index 000000000000..8c35be81b209 --- /dev/null +++ b/core/services/worker/file_staging_test.go @@ -0,0 +1,32 @@ +package worker + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("worker staging path containment", func() { + DescribeTable("checks resolved directory boundaries", + func(relative string, allowed bool) { + root := GinkgoT().TempDir() + models := filepath.Join(root, "models") + sibling := filepath.Join(root, "models-other") + Expect(os.Mkdir(models, 0o750)).To(Succeed()) + Expect(os.Mkdir(sibling, 0o750)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(models, "result.bin"), []byte("output"), 0o600)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(sibling, "private.bin"), []byte("private"), 0o600)).To(Succeed()) + Expect(os.Symlink(sibling, filepath.Join(models, "escape"))).To(Succeed()) + alias := filepath.Join(root, "alias") + Expect(os.Symlink(root, alias)).To(Succeed()) + + Expect(isPathAllowed(filepath.Join(alias, relative), []string{filepath.Join(alias, "models")})).To(Equal(allowed)) + }, + Entry("file beneath a symlinked root", "models/result.bin", true), + Entry("the symlinked root itself", "models", true), + Entry("a sibling sharing the directory prefix", "models-other/private.bin", false), + Entry("a symlink escaping the allowed root", "models/escape/private.bin", false), + ) +}) diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 1379efefa7e0..b9b8be17f392 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -431,6 +431,10 @@ operator inspects on the wire has a new shape. The one difference is that a worker now OMITS an empty reply field where the NATS handlers always emitted it, which a client reading a missing field as the zero value cannot tell apart. +`files/stage` accepts files inside the worker's models or staging-cache +directory, including when the directory path contains a symlink. It compares +resolved paths and rejects existing symlinks that point outside those directories. + `POST /v1/control/backend/stop` is served by BOTH kinds of worker, and the frontend sends it the same way to either. A serve-backend worker kills the backend process and recycles its port; an agent worker runs no backend From 06aaed98ad493342a38e31f2b830855be9d0e8a9 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Tue, 8 Sep 2026 03:08:43 +0000 Subject: [PATCH 115/153] test(worker): canonicalize missing upload fixture Resolve the existing model directory before choosing a missing file. On macOS, resolving the missing leaf fails and otherwise compares an aliased path with a canonical allowed directory. Keep this test on the Upload error branch instead of the containment rejection branch. Assisted-by: Codex:GPT-6 Signed-off-by: Ettore Di Giacinto --- core/services/worker/control_files_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/services/worker/control_files_test.go b/core/services/worker/control_files_test.go index 0668e2de6e7e..b45b44199799 100644 --- a/core/services/worker/control_files_test.go +++ b/core/services/worker/control_files_test.go @@ -176,7 +176,11 @@ var _ = Describe("worker file-staging control routes", func() { // allow-list. The two are separate statements of the same rule inside // one handler, and a spec that only ever reaches the allow-list leaves // the upload free to answer 500 for the worker's own verdict. - missing := filepath.Join(modelsDir, "was-never-written.bin") + // A missing file cannot resolve symlinked parents such as macOS /var. + // Use the canonical root so this tests Upload's error, not containment. + resolvedModelsDir, err := filepath.EvalSymlinks(modelsDir) + Expect(err).NotTo(HaveOccurred()) + missing := filepath.Join(resolvedModelsDir, "was-never-written.bin") resp := post(workerctl.PathFilesStage, map[string]string{"local_path": missing, "key": "data/x"}) Expect(resp.StatusCode).To(Equal(http.StatusOK)) var reply struct { From 5612dd195e915141ab50bda117dccce10731b43f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 14 Sep 2026 23:43:46 +0000 Subject: [PATCH 116/153] fix(distributed): preserve master protocol extensions Carry prefix-cache pressure and reported residency over PostgreSQL, keep truthful backend stop replies, and retain capacity-aware S3 staging release over the worker control tunnel after the rebase. Adapt the overlapping master tests to the tunnel-native protocol and cover cross-replica cache events and departed-node cleanup. Assisted-by: Codex:GPT-5 [apply_patch] [exec_command] Signed-off-by: Ettore Di Giacinto --- core/application/cache_fanout_wiring.go | 30 +++ core/application/cache_fanout_wiring_test.go | 32 +++ core/application/distributed.go | 19 +- core/http/endpoints/localai/nodes_test.go | 2 +- core/services/messaging/subjects.go | 44 ++++ .../nodes/file_stager_release_test.go | 85 ++++---- core/services/nodes/file_stager_s3.go | 34 +--- .../file_staging_sound_detection_test.go | 6 +- .../nodes/file_transfer_finalize_test.go | 2 +- .../nodes/heartbeat_checkpoint_test.go | 6 +- core/services/nodes/prefixcache/pressure.go | 6 +- core/services/nodes/prefixcache/reported.go | 15 ++ .../nodes/prefixcache/reported_test.go | 12 ++ .../nodes/router_slot_uncertainty_test.go | 4 +- core/services/worker/control_files.go | 54 ++++- core/services/worker/control_routes_test.go | 13 +- core/services/worker/file_staging.go | 188 ++++++++++++++++++ .../worker/file_staging_release_test.go | 123 +++++++----- core/services/worker/worker.go | 18 +- core/services/workerctl/paths.go | 2 + 20 files changed, 532 insertions(+), 163 deletions(-) diff --git a/core/application/cache_fanout_wiring.go b/core/application/cache_fanout_wiring.go index bbe4d592ae7a..c3d87e9abf01 100644 --- a/core/application/cache_fanout_wiring.go +++ b/core/application/cache_fanout_wiring.go @@ -7,6 +7,7 @@ import ( "fmt" "math" "strings" + "time" "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/core/services/messaging" @@ -114,6 +115,35 @@ func wirePrefixCacheBroadcasts(bus *pgbus.Bus, cfg prefixcache.Config, idx prefi return sync, nil } +// wirePrefixCacheExtensions puts the pressure and reported-residency families +// on the same PostgreSQL carrier as prefix observations and invalidations. +func wirePrefixCacheExtensions(bus *pgbus.Bus, pressure *prefixcache.Pressure, reported *prefixcache.ReportedIndex) ([]messaging.Subscription, error) { + if bus == nil { + return nil, fmt.Errorf("wiring prefix-cache extensions: no broadcast carrier") + } + var subs []messaging.Subscription + if pressure != nil { + sub, err := messaging.SubscribeJSON(bus, messaging.SubjectPrefixCachePressure, func(ev messaging.PrefixCachePressureEvent) { + pressure.ApplyPressure(ev, time.Now()) + }) + if err != nil { + return nil, fmt.Errorf("subscribing to %s: %w", messaging.SubjectPrefixCachePressure, err) + } + subs = append(subs, sub) + } + if reported != nil { + sub, err := messaging.SubscribeJSON(bus, messaging.SubjectPrefixCacheResidency, reported.Apply) + if err != nil { + for _, existing := range subs { + _ = existing.Unsubscribe() + } + return nil, fmt.Errorf("subscribing to %s: %w", messaging.SubjectPrefixCacheResidency, err) + } + subs = append(subs, sub) + } + return subs, nil +} + // prefixCacheIdentifierAllowance is how many bytes of model id plus node id a // prefix-cache observation is budgeted for when its worst case is checked // against the notification cap. diff --git a/core/application/cache_fanout_wiring_test.go b/core/application/cache_fanout_wiring_test.go index 45cf857da318..e9f72459e9bc 100644 --- a/core/application/cache_fanout_wiring_test.go +++ b/core/application/cache_fanout_wiring_test.go @@ -238,6 +238,38 @@ var _ = Describe("wiring the process-lifetime caches onto the broadcast carrier" Eventually(out, 20*time.Second).Should(Receive()) }) + + It("aggregates a peer replica's prefix-cache pressure", func() { + pressure := prefixcache.NewSyncedPressure(time.Minute, busA) + _, err := wirePrefixCacheExtensions(busA, pressure, prefixcache.NewReportedIndex()) + Expect(err).ToNot(HaveOccurred()) + + Expect(busB.Publish(messaging.SubjectPrefixCachePressure, messaging.PrefixCachePressureEvent{ + ID: "frontend-b:1", Model: "m", + })).To(Succeed()) + + Eventually(func() int { return pressure.Count("m", time.Now()) }, 20*time.Second).Should(Equal(1)) + }) + + It("applies a backend's reported prefix-cache residency", func() { + reported := prefixcache.NewReportedIndex() + _, err := wirePrefixCacheExtensions(busA, prefixcache.NewPressure(time.Minute), reported) + Expect(err).ToNot(HaveOccurred()) + + key := prefixcache.ReplicaKey{NodeID: "worker-1", Replica: 2} + chain := []uint64{11, 22, 33} + Expect(busB.Publish(messaging.SubjectPrefixCacheResidency, messaging.PrefixCacheResidencyEvent{ + Operation: messaging.PrefixCacheStore, + Model: "m", + NodeID: key.NodeID, + Replica: key.Replica, + Chain: chain, + })).To(Succeed()) + + Eventually(func() bool { + return reported.Decide("m", chain, []prefixcache.ReplicaKey{key}, time.Now()).HasHot + }, 20*time.Second).Should(BeTrue()) + }) }) // The size bound that lets prefix-cache observations publish like every other diff --git a/core/application/distributed.go b/core/application/distributed.go index 53694f2926bd..7ec4690fc5d5 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -591,21 +591,12 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade return prefixcache.ExtractChain(model, prompt, prefixCfg) } - // The Sync owns observation and invalidation subscriptions. Pressure and - // reported residency are newer prefix-cache families and use the same - // PostgreSQL carrier so no cache signal depends on NATS. - if _, err := messaging.SubscribeJSON(bus, messaging.SubjectPrefixCachePressure, func(ev messaging.PrefixCachePressureEvent) { - pressure.ApplyPressure(ev, time.Now()) - }); err != nil { - return nil, fmt.Errorf("subscribing to %s: %w", messaging.SubjectPrefixCachePressure, err) - } - - // Keep an exact-residency index current so backend producers can report - // their real KV state without coupling to router internals. Routing stays - // on the guessed provider until a backend producer is available. + // Keep pressure and exact residency on the same PostgreSQL carrier as the + // prefix Sync. Routing stays on the guessed provider until a backend + // producer is available. reportedIndex := prefixcache.NewReportedIndex() - if _, err := messaging.SubscribeJSON(bus, messaging.SubjectPrefixCacheResidency, reportedIndex.Apply); err != nil { - return nil, fmt.Errorf("subscribing to %s: %w", messaging.SubjectPrefixCacheResidency, err) + if _, err := wirePrefixCacheExtensions(bus, pressure, reportedIndex); err != nil { + return nil, err } // Background eviction: sweep idle entries on the app context. Stopped // when the app context is cancelled (mirrors the reconciler loop which diff --git a/core/http/endpoints/localai/nodes_test.go b/core/http/endpoints/localai/nodes_test.go index c3da844f3519..4d3366654053 100644 --- a/core/http/endpoints/localai/nodes_test.go +++ b/core/http/endpoints/localai/nodes_test.go @@ -71,7 +71,7 @@ var _ = Describe("Node HTTP handlers", func() { req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) rec := httptest.NewRecorder() - Expect(RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{})(e.NewContext(req, rec))).To(Succeed()) + Expect(RegisterNodeEndpoint(registry, "", true, nil, "")(e.NewContext(req, rec))).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusCreated)) node, err := registry.GetByName(context.Background(), "cpu-worker") diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index b44bdb168856..e7356af03c7e 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -270,6 +270,20 @@ type BackendStopRequest struct { Force bool `json:"force,omitempty"` } +// BackendStopReply is the worker's answer to a backend.stop control request. +type BackendStopReply struct { + Success bool `json:"success"` + Error string `json:"error,omitempty"` + + // StoppedProcessKeys names every modelID#replica process terminated by the + // request so the controller can remove exactly the rows it invalidated. + StoppedProcessKeys []string `json:"stopped_process_keys,omitempty"` + + // ReportsStoppedProcesses distinguishes an authoritative empty result from + // a reply produced by an older worker that predates StoppedProcessKeys. + ReportsStoppedProcesses bool `json:"reports_stopped_processes,omitempty"` +} + // AgentCancelRequest is the body of an agent cancel control request. // // It carries the same three fields the agent..cancel broadcast carried, @@ -471,8 +485,29 @@ func SubjectSyncStateTenantWildcard(name string) string { const ( SubjectPrefixCacheObserve = "prefixcache.observe" SubjectPrefixCacheInvalidate = "prefixcache.invalidate" + SubjectPrefixCachePressure = "prefixcache.pressure" + SubjectPrefixCacheResidency = "prefixcache.residency" ) +// PrefixCacheOperation describes a backend-reported KV-cache residency change. +type PrefixCacheOperation string + +const ( + PrefixCacheStore PrefixCacheOperation = "store" + PrefixCacheRemove PrefixCacheOperation = "remove" + PrefixCacheClear PrefixCacheOperation = "clear" +) + +// PrefixCacheResidencyEvent reports exact backend KV-cache residency. Chain is +// the compatible shallow-to-deep prefix hash chain used by the router. +type PrefixCacheResidencyEvent struct { + Operation PrefixCacheOperation `json:"operation"` + Model string `json:"model"` + NodeID string `json:"node_id"` + Replica int `json:"replica"` + Chain []uint64 `json:"chain,omitempty"` +} + // PrefixCacheObserveEvent announces that the replica (NodeID, Replica) served a // request whose prefix chain ends at the given hashes for model. Chain is the // full shallow-to-deep hash chain so peers can insert the same path. Affinity is @@ -494,3 +529,12 @@ type PrefixCacheInvalidateEvent struct { NodeID string `json:"node_id"` Replica int `json:"replica"` } + +// PrefixCachePressureEvent announces one forced-disturb observed by a frontend. +// ID lets every frontend ignore its own PostgreSQL echo and duplicate delivery +// without inflating the autoscale signal. +type PrefixCachePressureEvent struct { + ID string `json:"id"` + Model string `json:"model"` + Reset bool `json:"reset,omitempty"` +} diff --git a/core/services/nodes/file_stager_release_test.go b/core/services/nodes/file_stager_release_test.go index 519ef6287e41..ef64286d6a61 100644 --- a/core/services/nodes/file_stager_release_test.go +++ b/core/services/nodes/file_stager_release_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net" "net/http" "net/http/httptest" @@ -12,17 +13,13 @@ import ( "strings" "time" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/storage" + "github.com/mudler/LocalAI/core/services/workerctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -type releaseTestSubscription struct{} - -func (releaseTestSubscription) Unsubscribe() error { return nil } - -type releaseTestMessaging struct { +type releaseTestControl struct { subject string payload []byte onRequest func() @@ -32,35 +29,31 @@ type releaseTestMessaging struct { replies [][]byte } -func (m *releaseTestMessaging) Publish(string, any) error { return nil } -func (m *releaseTestMessaging) Subscribe(string, func([]byte)) (messaging.Subscription, error) { - return releaseTestSubscription{}, nil -} -func (m *releaseTestMessaging) QueueSubscribe(string, string, func([]byte)) (messaging.Subscription, error) { - return releaseTestSubscription{}, nil -} -func (m *releaseTestMessaging) QueueSubscribeReply(string, string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return releaseTestSubscription{}, nil -} -func (m *releaseTestMessaging) SubscribeReply(string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return releaseTestSubscription{}, nil -} -func (m *releaseTestMessaging) Request(subject string, data []byte, timeout time.Duration) ([]byte, error) { - m.subject = subject - m.payload = append([]byte(nil), data...) +func (m *releaseTestControl) RoundTrip(req *http.Request) (*http.Response, error) { + if err := req.Context().Err(); err != nil { + return nil, err + } + m.subject = req.URL.Path + m.payload, _ = io.ReadAll(req.Body) m.requestCalled = true m.requestCount++ - m.timeout = timeout + if deadline, ok := req.Context().Deadline(); ok { + m.timeout = time.Until(deadline) + } if m.onRequest != nil { m.onRequest() } + reply := []byte(`{}`) if m.requestCount <= len(m.replies) { - return append([]byte(nil), m.replies[m.requestCount-1]...), nil + reply = m.replies[m.requestCount-1] } - return []byte(`{}`), nil + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(string(reply))), Request: req}, nil +} +func (m *releaseTestControl) control() *ControlClient { + c := NewControlClient(directNetDialerFor, "") + c.clients["node.one"] = &nodeHTTPClient{client: &http.Client{Transport: m}} + return c } -func (m *releaseTestMessaging) IsConnected() bool { return true } -func (m *releaseTestMessaging) Close() {} var _ = Describe("File stager exact-key release", func() { startReleaseServer := func(stagingDir, token string) (*HTTPFileStager, func()) { @@ -77,7 +70,7 @@ var _ = Describe("File stager exact-key release", func() { Expect(err).NotTo(HaveOccurred()) return NewHTTPFileStager(func(string) (string, error) { return listener.Addr().String(), nil - }, token), func() { + }, token, directNetDialerFor), func() { Expect(server.Shutdown(context.Background())).To(Succeed()) } } @@ -161,7 +154,7 @@ var _ = Describe("File stager exact-key release", func() { DeferCleanup(server.Close) stager := NewHTTPFileStager(func(string) (string, error) { return strings.TrimPrefix(server.URL, "http://"), nil - }, "") + }, "", directNetDialerFor) keys := []string{ "ephemeral/audio/request-id/input.wav", "ephemeral/images/request-id/frame.jpg", @@ -178,7 +171,7 @@ var _ = Describe("File stager exact-key release", func() { stager := NewHTTPFileStager(func(string) (string, error) { resolved = true return "127.0.0.1:1", nil - }, "token") + }, "token", directNetDialerFor) for _, key := range []string{ "models/model.gguf", @@ -236,17 +229,17 @@ var _ = Describe("File stager exact-key release", func() { key := "ephemeral/request-id/audio/input.wav" Expect(store.Put(context.Background(), key, strings.NewReader("shared"))).To(Succeed()) - client := &releaseTestMessaging{} + client := &releaseTestControl{} client.onRequest = func() { exists, existsErr := store.Exists(context.Background(), key) Expect(existsErr).NotTo(HaveOccurred()) Expect(exists).To(BeTrue()) } - stager := NewS3NATSFileStager(fm, client) + stager := NewS3FileStager(fm, client.control()) Expect(stager.ReleaseRemote(context.Background(), "node.one", key)).To(Succeed()) Expect(client.requestCalled).To(BeTrue()) - Expect(client.subject).To(Equal(messaging.SubjectNodeFilesRelease("node.one"))) + Expect(client.subject).To(Equal(workerctl.PathFilesRelease)) var payload fileReleaseRequest Expect(json.Unmarshal(client.payload, &payload)).To(Succeed()) Expect(payload.Key).To(Equal(key)) @@ -255,7 +248,7 @@ var _ = Describe("File stager exact-key release", func() { Expect(exists).To(BeFalse()) }) - It("evicts a request's S3 inputs with one NATS round trip", func() { + It("evicts a request's S3 inputs with one control round trip", func() { storeRoot := GinkgoT().TempDir() store, err := storage.NewFilesystemStore(storeRoot) Expect(err).NotTo(HaveOccurred()) @@ -268,8 +261,8 @@ var _ = Describe("File stager exact-key release", func() { for _, key := range keys { Expect(store.Put(context.Background(), key, strings.NewReader("shared"))).To(Succeed()) } - client := &releaseTestMessaging{} - stager := NewS3NATSFileStager(fm, client) + client := &releaseTestControl{} + stager := NewS3FileStager(fm, client.control()) Expect(stager.ReleaseRemoteRequest(context.Background(), "node.one", "request-id", keys)).To(Succeed()) @@ -294,8 +287,8 @@ var _ = Describe("File stager exact-key release", func() { for i := range keys { keys[i] = fmt.Sprintf("ephemeral/inputs/request-id/input-%d.bin", i) } - client := &releaseTestMessaging{} - stager := NewS3NATSFileStager(fm, client) + client := &releaseTestControl{} + stager := NewS3FileStager(fm, client.control()) Expect(stager.ReleaseRemoteRequest(context.Background(), "node.one", "request-id", keys)).To(Succeed()) @@ -306,7 +299,7 @@ var _ = Describe("File stager exact-key release", func() { Expect(payload.RequestID).To(Equal("request-id")) }) - It("falls back to exact NATS releases for an older worker", func() { + It("falls back to exact control releases for an older worker", func() { store, err := storage.NewFilesystemStore(GinkgoT().TempDir()) Expect(err).NotTo(HaveOccurred()) fm, err := storage.NewFileManager(store, GinkgoT().TempDir()) @@ -318,12 +311,12 @@ var _ = Describe("File stager exact-key release", func() { for _, key := range keys { Expect(store.Put(context.Background(), key, strings.NewReader("shared"))).To(Succeed()) } - client := &releaseTestMessaging{replies: [][]byte{ + client := &releaseTestControl{replies: [][]byte{ []byte(`{"error":"batch payload unsupported"}`), []byte(`{}`), []byte(`{}`), }} - stager := NewS3NATSFileStager(fm, client) + stager := NewS3FileStager(fm, client.control()) Expect(stager.ReleaseRemoteRequest(context.Background(), "node.one", "request-id", keys)).To(Succeed()) @@ -348,8 +341,8 @@ var _ = Describe("File stager exact-key release", func() { Expect(err).NotTo(HaveOccurred()) fm, err := storage.NewFileManager(store, GinkgoT().TempDir()) Expect(err).NotTo(HaveOccurred()) - client := &releaseTestMessaging{} - stager := NewS3NATSFileStager(fm, client) + client := &releaseTestControl{} + stager := NewS3FileStager(fm, client.control()) ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -357,13 +350,13 @@ var _ = Describe("File stager exact-key release", func() { Expect(client.requestCalled).To(BeFalse()) }) - It("bounds the NATS release wait by the remaining cleanup deadline", func() { + It("bounds the control release wait by the remaining cleanup deadline", func() { store, err := storage.NewFilesystemStore(GinkgoT().TempDir()) Expect(err).NotTo(HaveOccurred()) fm, err := storage.NewFileManager(store, GinkgoT().TempDir()) Expect(err).NotTo(HaveOccurred()) - client := &releaseTestMessaging{} - stager := NewS3NATSFileStager(fm, client) + client := &releaseTestControl{} + stager := NewS3FileStager(fm, client.control()) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() diff --git a/core/services/nodes/file_stager_s3.go b/core/services/nodes/file_stager_s3.go index 4f97b074435c..9470f135c756 100644 --- a/core/services/nodes/file_stager_s3.go +++ b/core/services/nodes/file_stager_s3.go @@ -264,7 +264,7 @@ func (s *S3FileStager) StageRemoteToStore(ctx context.Context, nodeID, remotePat // ReleaseRemote evicts one exact ephemeral key from the worker before deleting // the shared object. -func (s *S3NATSFileStager) ReleaseRemote(ctx context.Context, nodeID, key string) error { +func (s *S3FileStager) ReleaseRemote(ctx context.Context, nodeID, key string) error { if err := validateEphemeralReleaseKey(key); err != nil { return err } @@ -277,10 +277,10 @@ func (s *S3NATSFileStager) ReleaseRemote(ctx context.Context, nodeID, key string return nil } -// ReleaseRemoteRequest evicts one inference's inputs with one NATS round trip. +// ReleaseRemoteRequest evicts one inference's inputs with one control request. // A worker that only understands the exact-key payload returns an error, so the // frontend retries each key during a rolling upgrade. -func (s *S3NATSFileStager) ReleaseRemoteRequest(ctx context.Context, nodeID, requestID string, keys []string) error { +func (s *S3FileStager) ReleaseRemoteRequest(ctx context.Context, nodeID, requestID string, keys []string) error { if err := validateEphemeralRequestRelease(requestID, keys); err != nil { return err } @@ -305,31 +305,9 @@ func (s *S3NATSFileStager) ReleaseRemoteRequest(ctx context.Context, nodeID, req return errors.Join(deleteErrors...) } -func (s *S3NATSFileStager) releaseWorkerKeys(ctx context.Context, nodeID string, request fileReleaseRequest) error { - if err := ctx.Err(); err != nil { - return err - } - timeout := 30 * time.Second - if deadline, ok := ctx.Deadline(); ok { - remaining := time.Until(deadline) - if remaining <= 0 { - return context.DeadlineExceeded - } - timeout = min(timeout, remaining) - } - reply, err := messaging.RequestJSON[fileReleaseRequest, fileReleaseReply]( - s.nats, - messaging.SubjectNodeFilesRelease(nodeID), - request, - timeout, - ) - if err != nil { - if contextErr := ctx.Err(); contextErr != nil { - return contextErr - } - return err - } - if err := ctx.Err(); err != nil { +func (s *S3FileStager) releaseWorkerKeys(ctx context.Context, nodeID string, request fileReleaseRequest) error { + var reply fileReleaseReply + if err := s.callWorker(ctx, nodeID, workerctl.PathFilesRelease, request, &reply); err != nil { return err } if reply.Error != "" { diff --git a/core/services/nodes/file_staging_sound_detection_test.go b/core/services/nodes/file_staging_sound_detection_test.go index 2fc6fb7e34dc..a194ae5f4e95 100644 --- a/core/services/nodes/file_staging_sound_detection_test.go +++ b/core/services/nodes/file_staging_sound_detection_test.go @@ -34,14 +34,16 @@ func (s *soundStagingFailure) ReleaseRemote(context.Context, string, string) err type soundRouteFactory struct{ client grpc.Backend } -func (f *soundRouteFactory) NewClient(string, bool) grpc.Backend { return f.client } +func (f *soundRouteFactory) NewClientForNode(string, string, bool) (grpc.Backend, error) { + return f.client, nil +} var _ = Describe("FileStagingClient sound detection", func() { It("stages sound audio through the client returned by SmartRouter.Route", func(ctx SpecContext) { node := &BackendNode{ID: "worker-1", Name: "worker", Address: "10.0.0.1:50051"} reg := &fakeModelRouter{ findAndLockNode: node, - findAndLockNM: &NodeModel{NodeID: node.ID, ModelName: "ced", Address: "10.0.0.1:9001"}, + findAndLockNM: &NodeModel{NodeID: node.ID, ModelName: "ced", WorkerLocalAddress: "10.0.0.1:9001"}, } backend := &soundStagingBackend{Backend: &stubBackend{healthResult: true}} stager := &fakeFileStager{} diff --git a/core/services/nodes/file_transfer_finalize_test.go b/core/services/nodes/file_transfer_finalize_test.go index bc0753b0e51f..4069dcc872d7 100644 --- a/core/services/nodes/file_transfer_finalize_test.go +++ b/core/services/nodes/file_transfer_finalize_test.go @@ -41,7 +41,7 @@ var _ = Describe("Recovering unfinished file finalization", func() { DeferCleanup(server.Close) stager := NewHTTPFileStager(func(string) (string, error) { return strings.TrimPrefix(server.URL, "http://"), nil - }, "") + }, "", directNetDialerFor) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() result, err := stager.EnsureRemote(ctx, "worker", local, "model.bin") diff --git a/core/services/nodes/heartbeat_checkpoint_test.go b/core/services/nodes/heartbeat_checkpoint_test.go index 4bea36ec8dd3..744b3e9526de 100644 --- a/core/services/nodes/heartbeat_checkpoint_test.go +++ b/core/services/nodes/heartbeat_checkpoint_test.go @@ -312,7 +312,7 @@ var _ = Describe("heartbeat checkpointing", func() { It("keeps a checkpointing node healthy while it beats normally", func() { registry.SetHeartbeatCheckpoint(200 * time.Millisecond) // perModelHealthCheck off: this spec is about liveness, not backends. - hm := NewHealthMonitor(registry, db, time.Minute, 5*time.Second, "", false) + hm := NewHealthMonitor(registry, db, time.Minute, 5*time.Second, "", false, nil, 0, nil) for range 6 { Expect(registry.Heartbeat(ctx, nodeID, nil)).To(Succeed()) @@ -337,7 +337,7 @@ var _ = Describe("heartbeat checkpointing", func() { Update("last_heartbeat", time.Now().Add(-2*time.Minute)).Error).ToNot(HaveOccurred()) // Zero staleThreshold: the constructor's fallback is what is under test. - hm := NewHealthMonitor(registry, db, time.Minute, 0, "", false) + hm := NewHealthMonitor(registry, db, time.Minute, 0, "", false, nil, 0, nil) hm.doCheckAll(ctx) var n BackendNode @@ -352,7 +352,7 @@ var _ = Describe("heartbeat checkpointing", func() { Expect(db.Model(&BackendNode{}).Where("id = ?", nodeID). Update("last_heartbeat", time.Now().Add(-10*time.Minute)).Error).ToNot(HaveOccurred()) - hm := NewHealthMonitor(registry, db, time.Minute, 5*time.Minute, "", false) + hm := NewHealthMonitor(registry, db, time.Minute, 5*time.Minute, "", false, nil, 0, nil) hm.doCheckAll(ctx) var n BackendNode diff --git a/core/services/nodes/prefixcache/pressure.go b/core/services/nodes/prefixcache/pressure.go index bc78ff20712d..3fa588f41581 100644 --- a/core/services/nodes/prefixcache/pressure.go +++ b/core/services/nodes/prefixcache/pressure.go @@ -27,7 +27,7 @@ type Pressure struct { window time.Duration events map[string][]time.Time seen map[string]time.Time - pub publisher + pub messaging.Publisher origin string seq atomic.Uint64 } @@ -44,7 +44,7 @@ func NewPressure(window time.Duration) *Pressure { // NewSyncedPressure creates a Pressure counter that broadcasts locally // originated events so every frontend sees the same cluster-wide signal. -func NewSyncedPressure(window time.Duration, pub publisher) *Pressure { +func NewSyncedPressure(window time.Duration, pub messaging.Publisher) *Pressure { p := NewPressure(window) p.pub = pub var id [8]byte @@ -103,7 +103,7 @@ func (p *Pressure) record(model, id string, now time.Time) { p.events[model] = kept } -// ApplyPressure records a pressure event received from NATS without +// ApplyPressure records a pressure event received from the broadcast carrier without // re-broadcasting it. Duplicate deliveries, including the publisher's own echo, // are ignored by event ID. func (p *Pressure) ApplyPressure(ev messaging.PrefixCachePressureEvent, now time.Time) { diff --git a/core/services/nodes/prefixcache/reported.go b/core/services/nodes/prefixcache/reported.go index 016041e06435..fd1f482a5654 100644 --- a/core/services/nodes/prefixcache/reported.go +++ b/core/services/nodes/prefixcache/reported.go @@ -108,6 +108,21 @@ func (ix *ReportedIndex) InvalidateNode(model, nodeID string) { } } +func (ix *ReportedIndex) DropNode(nodeID string) { + ix.mu.Lock() + defer ix.mu.Unlock() + for model, byReplica := range ix.residencies { + for key := range byReplica { + if key.NodeID == nodeID { + delete(byReplica, key) + } + } + if len(byReplica) == 0 { + delete(ix.residencies, model) + } + } +} + func (ix *ReportedIndex) Evict(time.Time) {} func equalChain(a, b []uint64) bool { diff --git a/core/services/nodes/prefixcache/reported_test.go b/core/services/nodes/prefixcache/reported_test.go index b3a70abc4120..9f86821b92b6 100644 --- a/core/services/nodes/prefixcache/reported_test.go +++ b/core/services/nodes/prefixcache/reported_test.go @@ -44,6 +44,18 @@ var _ = Describe("ReportedIndex", func() { Expect(idx.Decide("m", []uint64{3, 4}, candidates, t0).Hot).To(Equal(rk("B", 0))) }) + It("drops a departed node from every model", func() { + idx.Apply(messaging.PrefixCacheResidencyEvent{Operation: messaging.PrefixCacheStore, Model: "m1", NodeID: "A", Replica: 0, Chain: []uint64{1}}) + idx.Apply(messaging.PrefixCacheResidencyEvent{Operation: messaging.PrefixCacheStore, Model: "m2", NodeID: "A", Replica: 1, Chain: []uint64{2}}) + idx.Apply(messaging.PrefixCacheResidencyEvent{Operation: messaging.PrefixCacheStore, Model: "m2", NodeID: "B", Replica: 0, Chain: []uint64{3}}) + + idx.DropNode("A") + + Expect(idx.Decide("m1", []uint64{1}, []prefixcache.ReplicaKey{rk("A", 0)}, t0).HasHot).To(BeFalse()) + Expect(idx.Decide("m2", []uint64{2}, []prefixcache.ReplicaKey{rk("A", 1)}, t0).HasHot).To(BeFalse()) + Expect(idx.Decide("m2", []uint64{3}, []prefixcache.ReplicaKey{rk("B", 0)}, t0).HasHot).To(BeTrue()) + }) + It("ignores guessed request observations and keeps cold ordering deterministic", func() { Expect(idx.Observe("m", []uint64{1, 2}, rk("A", 0), t0)).To(BeFalse()) d := idx.Decide("m", []uint64{1, 2}, []prefixcache.ReplicaKey{rk("B", 1), rk("A", 1), rk("A", 0)}, time.Now()) diff --git a/core/services/nodes/router_slot_uncertainty_test.go b/core/services/nodes/router_slot_uncertainty_test.go index b32649197761..4e3380ab0169 100644 --- a/core/services/nodes/router_slot_uncertainty_test.go +++ b/core/services/nodes/router_slot_uncertainty_test.go @@ -37,7 +37,7 @@ var _ = Describe("replica slot lookup under database latency", func() { backend = &stubBackend{loadResult: &pb.Result{Success: true}} factory = &stubClientFactory{client: backend} unloader = &fakeUnloader{ - installReply: &messaging.BackendInstallReply{Success: true, Address: "10.0.0.10:9001"}, + installReply: &messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.10:9001"}, } }) @@ -90,7 +90,7 @@ var _ = Describe("node selection under database latency", func() { reg = &fakeModelRouter{findAndLockErr: errors.New("not found")} factory = &stubClientFactory{client: &stubBackend{loadResult: &pb.Result{Success: true}}} unloader = &fakeUnloader{ - installReply: &messaging.BackendInstallReply{Success: true, Address: "10.0.0.10:9001"}, + installReply: &messaging.BackendInstallReply{Success: true, WorkerLocalAddress: "10.0.0.10:9001"}, } }) diff --git a/core/services/worker/control_files.go b/core/services/worker/control_files.go index 2b05781a915f..e85a55caf19e 100644 --- a/core/services/worker/control_files.go +++ b/core/services/worker/control_files.go @@ -12,11 +12,12 @@ import ( "github.com/mudler/LocalAI/core/services/storage" "github.com/mudler/LocalAI/core/services/workerctl" "github.com/mudler/xlog" + "golang.org/x/sync/singleflight" ) -// The worker's file-staging control plane: four verbs that move model and job +// The worker's file-staging control plane: five verbs that move model and job // artifacts between the object store both sides share and this worker's disk. -// They replace the four nodes..files.* NATS subjects. +// They replace the five nodes..files.* NATS subjects. // // The reply shapes are the ones those subjects already carried, unchanged, so // an operator reading the wire sees the same fields. What DID change is that a @@ -76,7 +77,7 @@ func (cfg *Config) stagingDataDir() string { // from, over the same object store the frontend uses. // // It returns an error rather than degrading, because a worker whose deployment -// asked for object storage and could not reach it would otherwise mount four +// asked for object storage and could not reach it would otherwise mount five // verbs that fail every call, which the frontend cannot tell from a worker out // of disk. func (cfg *Config) NewStagingFileManager(ctx context.Context) (*storage.FileManager, error) { @@ -98,13 +99,19 @@ func (cfg *Config) NewStagingFileManager(ctx context.Context) (*storage.FileMana return fm, nil } -// RegisterFileControlRoutes mounts the four file-staging verbs on mux. +// RegisterFileControlRoutes mounts the five file-staging verbs on mux. // // The caller is responsible for putting mux behind authentication; see // nodes.AuthenticatedRoutes, which is how the worker mounts this so the file // verbs share one bearer check with the lifecycle verbs and the file routes // rather than growing a third one. func (cfg *Config) RegisterFileControlRoutes(mux *http.ServeMux, fm *storage.FileManager) { + cfg.RegisterFileControlRoutesWithCapacity(mux, fm, nil) +} + +// RegisterFileControlRoutesWithCapacity applies worker staging capacity to the tunnel control verbs. +func (cfg *Config) RegisterFileControlRoutesWithCapacity(mux *http.ServeMux, fm *storage.FileManager, capacity *EphemeralCapacityGuard) { + var ensureGroup singleflight.Group cacheDir := cfg.stagingCacheDir() // files.ensure: download an object-store key into this worker's cache and @@ -120,15 +127,23 @@ func (cfg *Config) RegisterFileControlRoutes(mux *http.ServeMux, fm *storage.Fil if err := json.Unmarshal(body, &req); err != nil { return nil, fmt.Errorf("invalid files.ensure request: %w", err) } - localPath, err := fm.Download(ctx, req.Key) + value, err, _ := ensureGroup.Do(req.Key, func() (any, error) { + return ensureWorkerFile(ctx, fm, capacity, req.Key) + }) if err != nil { xlog.Error("File ensure failed", "key", req.Key, "error", err) return fileEnsureReply{Error: err.Error()}, nil } + localPath, ok := value.(string) + if !ok { + return fileEnsureReply{Error: fmt.Sprintf("unexpected file ensure result %T", value)}, nil + } xlog.Debug("File ensured locally", "key", req.Key, "path", localPath) return fileEnsureReply{LocalPath: localPath}, nil }) + registerFileReleaseControlRoute(mux, fm, cacheDir, capacity) + // files.stage: upload one of this worker's files to the object store. // // The path allow-list is what keeps this verb from being an exfiltration @@ -266,3 +281,32 @@ func listStagedFiles(ctx context.Context, dirPath string) ([]string, error) { } return files, nil } + +func registerFileReleaseControlRoute(mux *http.ServeMux, fm *storage.FileManager, cacheDir string, capacity *EphemeralCapacityGuard) { + postControlVerb(mux, workerctl.PathFilesRelease, func(ctx context.Context, body []byte) (any, error) { + var req struct { + Key string `json:"key"` + RequestID string `json:"request_id"` + } + if err := json.Unmarshal(body, &req); err != nil { + return nil, fmt.Errorf("invalid files.release request: %w", err) + } + var err error + if req.RequestID != "" { + err = releaseEphemeralCacheRequest(ctx, cacheDir, req.RequestID, capacity) + } else { + var cachePath string + cachePath, err = fm.CachePath(req.Key) + if err == nil { + err = releaseEphemeralCachePathWithCapacity(cacheDir, req.Key, cachePath, capacity) + } + } + reply := struct { + Error string `json:"error,omitempty"` + }{} + if err != nil { + reply.Error = err.Error() + } + return reply, nil + }) +} diff --git a/core/services/worker/control_routes_test.go b/core/services/worker/control_routes_test.go index 9c28cebf0920..754d12ec50b9 100644 --- a/core/services/worker/control_routes_test.go +++ b/core/services/worker/control_routes_test.go @@ -221,9 +221,14 @@ var _ = Describe("worker control routes", func() { Expect(resp.StatusCode).To(Equal(http.StatusOK)) }) - It("answers backend.stop with 204, the shape a fire-and-forget verb takes", func() { + It("answers backend.stop with the truthful result", func() { resp := post(workerctl.PathBackendStop, messaging.BackendStopRequest{Backend: "no-such-backend"}) - Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + var reply messaging.BackendStopReply + Expect(json.NewDecoder(resp.Body).Decode(&reply)).To(Succeed()) + Expect(reply.Success).To(BeTrue()) + Expect(reply.ReportsStoppedProcesses).To(BeTrue()) + Expect(reply.StoppedProcessKeys).To(BeEmpty()) }) It("answers node.stop with 204 and signals shutdown", func() { @@ -457,7 +462,7 @@ var _ = Describe("the worker's HTTP server", func() { Expect(err).NotTo(HaveOccurred()) srv, err = startWorkerHTTPServer("127.0.0.1:0", filepath.Join(dir, "staging"), dir, - filepath.Join(dir, "data"), token, &nodes.WorkerReadiness{}, sup, nil, sup.cfg, stagingFM, nil) + filepath.Join(dir, "data"), token, &nodes.WorkerReadiness{}, sup, sup.cfg, stagingFM, nil) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { nodes.ShutdownFileTransferServer(srv) }) Expect(srv.Addr).NotTo(BeEmpty(), "the worker HTTP server must report the address it bound") @@ -497,7 +502,7 @@ var _ = Describe("the worker's HTTP server", func() { // that verb" rather than as a file that is not there. dir := GinkgoT().TempDir() bare, err := startWorkerHTTPServer("127.0.0.1:0", filepath.Join(dir, "staging"), dir, - filepath.Join(dir, "data"), token, &nodes.WorkerReadiness{}, sup, nil, sup.cfg, nil, nil) + filepath.Join(dir, "data"), token, &nodes.WorkerReadiness{}, sup, sup.cfg, nil, nil) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { nodes.ShutdownFileTransferServer(bare) }) diff --git a/core/services/worker/file_staging.go b/core/services/worker/file_staging.go index 0c7b81e01ffb..66d9c58aef56 100644 --- a/core/services/worker/file_staging.go +++ b/core/services/worker/file_staging.go @@ -1,8 +1,16 @@ package worker import ( + "context" + "errors" + "fmt" + "os" + "path" "path/filepath" "strings" + + "github.com/mudler/LocalAI/core/services/storage" + "github.com/mudler/LocalAI/pkg/safefile" ) // isPathAllowed checks if path is within one of the allowed directories. @@ -31,3 +39,183 @@ func isPathAllowed(path string, allowedDirs []string) bool { } return false } + +func releaseEphemeralCacheKey(cacheDir, key string) error { + return releaseEphemeralCachePath(cacheDir, key, filepath.Join(cacheDir, filepath.FromSlash(key))) +} + +func releaseEphemeralCachePath(cacheDir, key, filePath string) error { + return releaseEphemeralCachePathWithCapacity(cacheDir, key, filePath, nil) +} + +func releaseEphemeralCachePathWithCapacity(cacheDir, key, filePath string, capacity *EphemeralCapacityGuard) error { + if err := validateEphemeralCacheKey(key); err != nil { + return err + } + relativePath := filepath.FromSlash(key) + expectedPath := filepath.Join(cacheDir, relativePath) + if filepath.Clean(filePath) != expectedPath { + return fmt.Errorf("release path %q does not match key %q", filePath, key) + } + if err := safefile.RemoveExact(cacheDir, relativePath, []string{".sha256", ".sha256.target"}, 2); err != nil { + return err + } + for _, path := range []string{filePath, filePath + ".sha256", filePath + ".sha256.target"} { + if capacity != nil { + if err := capacity.Release(path); err != nil { + return err + } + } + } + return nil +} + +func releaseEphemeralCacheRequest(ctx context.Context, cacheDir, requestID string, capacity *EphemeralCapacityGuard) error { + if err := validateEphemeralCacheRequestID(requestID); err != nil { + return err + } + if capacity != nil { + if err := capacity.BeginRequestRelease(ctx, requestID); err != nil { + return fmt.Errorf("beginning release for request %q: %w", requestID, err) + } + defer capacity.EndRequestRelease(requestID) + } + root := filepath.Join(cacheDir, "ephemeral") + categories, err := os.ReadDir(root) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + var releaseErrors []error + for _, category := range categories { + if !category.IsDir() || category.Type()&os.ModeSymlink != 0 { + continue + } + requestDir := filepath.Join(root, category.Name(), requestID) + info, err := os.Lstat(requestDir) + if err != nil { + if !os.IsNotExist(err) { + releaseErrors = append(releaseErrors, fmt.Errorf("stating request directory %q: %w", requestDir, err)) + } + continue + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + releaseErrors = append(releaseErrors, fmt.Errorf("ephemeral request path %q is not a real directory", requestDir)) + continue + } + entries, err := os.ReadDir(requestDir) + if err != nil { + if !os.IsNotExist(err) { + releaseErrors = append(releaseErrors, fmt.Errorf("reading request directory %q: %w", requestDir, err)) + } + continue + } + for _, entry := range entries { + if entry.IsDir() { + releaseErrors = append(releaseErrors, fmt.Errorf("unexpected directory in ephemeral request %q", filepath.Join(requestDir, entry.Name()))) + continue + } + key := filepath.ToSlash(filepath.Join("ephemeral", category.Name(), requestID, entry.Name())) + filePath := filepath.Join(cacheDir, filepath.FromSlash(key)) + if err := releaseEphemeralCachePathWithCapacity(cacheDir, key, filePath, capacity); err != nil { + releaseErrors = append(releaseErrors, fmt.Errorf("releasing %q: %w", key, err)) + } + } + } + return errors.Join(releaseErrors...) +} + +type ephemeralStagingCapacity interface { + Reserve(path string, size int64) error + Commit(path string) error + Claim(path string) error + Release(path string) error +} + +func ensureWorkerFile(ctx context.Context, fm *storage.FileManager, capacity *EphemeralCapacityGuard, key string) (string, error) { + if capacity == nil { + return fm.Download(ctx, key) + } + if strings.HasPrefix(key, "ephemeral/") { + if err := validateEphemeralCacheKey(key); err != nil { + return "", err + } + requestID := strings.Split(key, "/")[2] + if err := capacity.BeginRequestOperation(requestID); err != nil { + return "", err + } + defer capacity.EndRequestOperation(requestID) + } + return ensureWorkerFileWithCapacity(ctx, fm, capacity, key) +} + +func ensureWorkerFileWithCapacity(ctx context.Context, fm *storage.FileManager, capacity ephemeralStagingCapacity, key string) (string, error) { + if capacity == nil || !strings.HasPrefix(key, "ephemeral/") { + return fm.Download(ctx, key) + } + if err := validateEphemeralCacheKey(key); err != nil { + return "", err + } + cachePath, err := fm.CachePath(key) + if err != nil { + return "", err + } + if info, statErr := os.Lstat(cachePath); statErr == nil { + if !info.Mode().IsRegular() { + return "", fmt.Errorf("ephemeral cache path %q is not a regular file", cachePath) + } + if err := capacity.Claim(cachePath); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return "", err + } + } else { + return cachePath, nil + } + } else if !os.IsNotExist(statErr) { + return "", statErr + } + + meta, err := fm.Head(ctx, key) + if err != nil { + return "", fmt.Errorf("reading size for %s: %w", key, err) + } + if err := capacity.Reserve(cachePath, meta.Size); err != nil { + return "", err + } + localPath, err := fm.Download(ctx, key) + if err != nil { + _ = capacity.Release(cachePath) + return "", err + } + if err := capacity.Commit(cachePath); err != nil { + _ = fm.EvictCache(key) + _ = capacity.Release(cachePath) + return "", err + } + return localPath, nil +} + +func validateEphemeralCacheKey(key string) error { + if strings.Contains(key, "\\") || path.Clean(key) != key { + return fmt.Errorf("invalid ephemeral key %q", key) + } + parts := strings.Split(key, "/") + if len(parts) != 4 || parts[0] != "ephemeral" { + return fmt.Errorf("release key %q must identify one file below ephemeral/", key) + } + for _, part := range parts[1:] { + if part == "" || part == "." || part == ".." { + return fmt.Errorf("invalid ephemeral key %q", key) + } + } + return nil +} + +func validateEphemeralCacheRequestID(requestID string) error { + if requestID == "" || strings.ContainsAny(requestID, "/\\") || path.Clean(requestID) != requestID || requestID == "." || requestID == ".." { + return fmt.Errorf("invalid ephemeral request ID %q", requestID) + } + return nil +} diff --git a/core/services/worker/file_staging_release_test.go b/core/services/worker/file_staging_release_test.go index 1019d475d6d8..f39263cb0a5f 100644 --- a/core/services/worker/file_staging_release_test.go +++ b/core/services/worker/file_staging_release_test.go @@ -8,14 +8,16 @@ import ( "fmt" "io" "net" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" "time" - "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/storage" + "github.com/mudler/LocalAI/core/services/workerctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -55,37 +57,56 @@ func (*stagingObjectStore) List(context.Context, string) ([]string, error) { return nil, nil } -type releaseSubscription struct{} - -func (releaseSubscription) Unsubscribe() error { return nil } - -type releaseMessagingClient struct { - subject string - handler func([]byte, func([]byte)) -} - -func (m *releaseMessagingClient) Publish(string, any) error { return nil } -func (m *releaseMessagingClient) Subscribe(string, func([]byte)) (messaging.Subscription, error) { - return releaseSubscription{}, nil -} -func (m *releaseMessagingClient) QueueSubscribe(string, string, func([]byte)) (messaging.Subscription, error) { - return releaseSubscription{}, nil -} -func (m *releaseMessagingClient) QueueSubscribeReply(string, string, func([]byte, func([]byte))) (messaging.Subscription, error) { - return releaseSubscription{}, nil -} -func (m *releaseMessagingClient) SubscribeReply(subject string, handler func([]byte, func([]byte))) (messaging.Subscription, error) { - m.subject = subject - m.handler = handler - return releaseSubscription{}, nil -} -func (m *releaseMessagingClient) Request(string, []byte, time.Duration) ([]byte, error) { - return nil, nil -} -func (m *releaseMessagingClient) IsConnected() bool { return true } -func (m *releaseMessagingClient) Close() {} - var _ = Describe("Worker exact-key staging release", func() { + It("shares capacity between HTTP uploads and S3 control requests and releases both over the worker transport", func() { + ctx := context.Background() + dir := GinkgoT().TempDir() + cfg := &Config{ModelsPath: filepath.Join(dir, "models")} + stagingDir := filepath.Join(dir, "staging") + cacheDir := cfg.stagingCacheDir() + guard, err := NewEphemeralCapacityGuard([]string{filepath.Join(stagingDir, "ephemeral"), filepath.Join(cacheDir, "ephemeral")}, 4+sha256.Size*4, 0) + Expect(err).NotTo(HaveOccurred()) + store, err := storage.NewFilesystemStore(filepath.Join(dir, "objects")) + Expect(err).NotTo(HaveOccurred()) + workerFM, err := storage.NewFileManager(store, cacheDir) + Expect(err).NotTo(HaveOccurred()) + frontendFM, err := storage.NewFileManager(store, filepath.Join(dir, "frontend-cache")) + Expect(err).NotTo(HaveOccurred()) + sup := &backendSupervisor{cfg: cfg, processes: map[string]*backendProcess{}} + server, err := startWorkerHTTPServer("127.0.0.1:0", stagingDir, cfg.ModelsPath, cfg.stagingDataDir(), "secret", nil, sup, cfg, workerFM, nil, guard) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { nodes.ShutdownFileTransferServer(server) }) + dialFor := func(string) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", server.Addr) + } + } + httpStager := nodes.NewHTTPFileStager(func(string) (string, error) { return server.Addr, nil }, "secret", dialFor) + s3Stager := nodes.NewS3FileStager(frontendFM, nodes.NewControlClient(dialFor, "secret")) + input := filepath.Join(dir, "input.wav") + Expect(os.WriteFile(input, []byte("data"), 0o600)).To(Succeed()) + s3Input := filepath.Join(dir, "s3-input.wav") + Expect(os.WriteFile(s3Input, []byte(strings.Repeat("x", 100)), 0o600)).To(Succeed()) + httpKey := "ephemeral/audio/http-request/input.wav" + s3Key := "ephemeral/audio/s3-request/input.wav" + httpPath, err := httpStager.EnsureRemote(ctx, "worker", input, httpKey) + Expect(err).NotTo(HaveOccurred()) + _, err = s3Stager.EnsureRemote(ctx, "worker", s3Input, s3Key) + Expect(err).To(HaveOccurred()) + Expect(filepath.Join(cacheDir, filepath.FromSlash(s3Key))).NotTo(BeAnExistingFile()) + Expect(httpStager.ReleaseRemoteRequest(ctx, "worker", "http-request", []string{httpKey})).To(Succeed()) + Expect(httpPath).NotTo(BeAnExistingFile()) + s3Path, err := s3Stager.EnsureRemote(ctx, "worker", s3Input, s3Key) + Expect(err).NotTo(HaveOccurred()) + Expect(guard.HasActiveReservation(s3Path)).To(BeTrue()) + Expect(s3Stager.ReleaseRemoteRequest(ctx, "worker", "s3-request", []string{s3Key})).To(Succeed()) + Expect(s3Path).NotTo(BeAnExistingFile()) + Expect(guard.HasActiveReservation(s3Path)).To(BeFalse()) + exists, err := store.Exists(ctx, s3Key) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + It("protects a startup-accounted HTTP cache hit through authenticated repeated probes", func() { stagingDir := canonicalWorkerTempDir() root := filepath.Join(stagingDir, "ephemeral") @@ -113,7 +134,9 @@ var _ = Describe("Worker exact-key staging release", func() { localPath := filepath.Join(canonicalWorkerTempDir(), "input.wav") Expect(os.WriteFile(localPath, content, 0o600)).To(Succeed()) - stager := nodes.NewHTTPFileStager(func(string) (string, error) { return addr, nil }, "secret") + stager := nodes.NewHTTPFileStager(func(string) (string, error) { return addr, nil }, "secret", func(string) func(context.Context, string, string) (net.Conn, error) { + return (&net.Dialer{}).DialContext + }) for range 2 { path, ensureErr := stager.EnsureRemote(context.Background(), "worker", localPath, key) Expect(ensureErr).NotTo(HaveOccurred()) @@ -342,22 +365,22 @@ var _ = Describe("Worker exact-key staging release", func() { Expect(os.WriteFile(path, []byte("data"), 0640)).To(Succeed()) fm, err := storage.NewFileManager(nil, cacheDir) Expect(err).NotTo(HaveOccurred()) - client := &releaseMessagingClient{} + mux := http.NewServeMux() - Expect(subscribeFileRelease(client, "node.one", fm, cacheDir)).To(Succeed()) - Expect(client.subject).To(Equal(messaging.SubjectNodeFilesRelease("node.one"))) + registerFileReleaseControlRoute(mux, fm, cacheDir, nil) request, err := json.Marshal(map[string]string{"key": "ephemeral/request-id/audio/input.wav"}) Expect(err).NotTo(HaveOccurred()) - var response []byte - client.handler(request, func(data []byte) { response = append([]byte(nil), data...) }) + response := httptest.NewRecorder() + mux.ServeHTTP(response, httptest.NewRequest(http.MethodPost, workerctl.PathFilesRelease, strings.NewReader(string(request)))) + Expect(response.Code).To(Equal(http.StatusOK)) var reply map[string]string - Expect(json.Unmarshal(response, &reply)).To(Succeed()) + Expect(json.Unmarshal(response.Body.Bytes(), &reply)).To(Succeed()) Expect(reply["error"]).To(BeEmpty()) Expect(path).NotTo(BeAnExistingFile()) }) - It("releases a request batch through one worker message", func() { + It("releases a request batch through one worker control request", func() { cacheDir := canonicalWorkerTempDir() keys := []string{ "ephemeral/audio/request-id/input.wav", @@ -370,16 +393,17 @@ var _ = Describe("Worker exact-key staging release", func() { } fm, err := storage.NewFileManager(nil, cacheDir) Expect(err).NotTo(HaveOccurred()) - client := &releaseMessagingClient{} - Expect(subscribeFileRelease(client, "node.one", fm, cacheDir)).To(Succeed()) + mux := http.NewServeMux() + registerFileReleaseControlRoute(mux, fm, cacheDir, nil) request, err := json.Marshal(map[string]any{"request_id": "request-id"}) Expect(err).NotTo(HaveOccurred()) - var response []byte + response := httptest.NewRecorder() - client.handler(request, func(data []byte) { response = append([]byte(nil), data...) }) + mux.ServeHTTP(response, httptest.NewRequest(http.MethodPost, workerctl.PathFilesRelease, strings.NewReader(string(request)))) + Expect(response.Code).To(Equal(http.StatusOK)) var reply map[string]string - Expect(json.Unmarshal(response, &reply)).To(Succeed()) + Expect(json.Unmarshal(response.Body.Bytes(), &reply)).To(Succeed()) Expect(reply["error"]).To(BeEmpty()) for _, key := range keys { Expect(filepath.Join(cacheDir, filepath.FromSlash(key))).NotTo(BeAnExistingFile()) @@ -390,16 +414,17 @@ var _ = Describe("Worker exact-key staging release", func() { cacheDir := canonicalWorkerTempDir() fm, err := storage.NewFileManager(nil, cacheDir) Expect(err).NotTo(HaveOccurred()) - client := &releaseMessagingClient{} - Expect(subscribeFileRelease(client, "node-1", fm, cacheDir)).To(Succeed()) + mux := http.NewServeMux() + registerFileReleaseControlRoute(mux, fm, cacheDir, nil) request, err := json.Marshal(map[string]string{"key": "models/model.gguf"}) Expect(err).NotTo(HaveOccurred()) - var response []byte - client.handler(request, func(data []byte) { response = append([]byte(nil), data...) }) + response := httptest.NewRecorder() + mux.ServeHTTP(response, httptest.NewRequest(http.MethodPost, workerctl.PathFilesRelease, strings.NewReader(string(request)))) + Expect(response.Code).To(Equal(http.StatusOK)) var reply map[string]string - Expect(json.Unmarshal(response, &reply)).To(Succeed()) + Expect(json.Unmarshal(response.Body.Bytes(), &reply)).To(Succeed()) Expect(reply["error"]).NotTo(BeEmpty()) }) }) diff --git a/core/services/worker/worker.go b/core/services/worker/worker.go index e75a97693d1e..3fc309acef99 100644 --- a/core/services/worker/worker.go +++ b/core/services/worker/worker.go @@ -218,7 +218,7 @@ func Run(ctx *cliContext.Context, cfg *Config) error { } httpServer, err := startWorkerHTTPServer(httpAddr, stagingDir, cfg.ModelsPath, dataDir, - cfg.RegistrationToken, readiness, supervisor, ephemeralCapacity, cfg, stagingFM, ml.BackendLogs()) + cfg.RegistrationToken, readiness, supervisor, cfg, stagingFM, ml.BackendLogs(), ephemeralCapacity) if err != nil { return fmt.Errorf("starting HTTP file transfer server: %w", err) } @@ -350,10 +350,18 @@ func startTunnelAndArmReadiness(ctx context.Context, readiness *nodes.WorkerRead // the server without the control plane and producing a worker that looks // healthy while answering 404 to every command. func startWorkerHTTPServer(addr, stagingDir, modelsDir, dataDir, token string, - readiness *nodes.WorkerReadiness, sup *backendSupervisor, capacity nodes.EphemeralCapacity, - cfg *Config, stagingFM *storage.FileManager, logStore *model.BackendLogStore) (*http.Server, error) { + readiness *nodes.WorkerReadiness, sup *backendSupervisor, cfg *Config, + stagingFM *storage.FileManager, logStore *model.BackendLogStore, capacities ...*EphemeralCapacityGuard) (*http.Server, error) { + var capacity *EphemeralCapacityGuard + if len(capacities) > 0 { + capacity = capacities[0] + } + var httpCapacity nodes.EphemeralCapacity + if capacity != nil { + httpCapacity = capacity + } return nodes.StartFileTransferServerWithCapacityAndRoutes(addr, stagingDir, modelsDir, dataDir, token, - config.DefaultMaxUploadSize, readiness, capacity, &nodes.AuthenticatedRoutes{ + config.DefaultMaxUploadSize, readiness, httpCapacity, &nodes.AuthenticatedRoutes{ Prefix: workerctl.Prefix, // One registrar for both route sets, because there is ONE control // prefix and AuthenticatedRoutes mounts one mux behind one bearer @@ -361,7 +369,7 @@ func startWorkerHTTPServer(addr, stagingDir, modelsDir, dataDir, token string, Register: func(mux *http.ServeMux) { sup.RegisterControlRoutes(mux) if stagingFM != nil { - cfg.RegisterFileControlRoutes(mux, stagingFM) + cfg.RegisterFileControlRoutesWithCapacity(mux, stagingFM, capacity) } }, }, logStore) diff --git a/core/services/workerctl/paths.go b/core/services/workerctl/paths.go index 8803e2478035..c064fbe71321 100644 --- a/core/services/workerctl/paths.go +++ b/core/services/workerctl/paths.go @@ -36,6 +36,7 @@ const ( // them not at all, and the catch-all under Prefix answers for them. That is // the same 404 an older build gives, which is what the frontend already // reads as "this worker does not serve that verb" rather than as absence. + PathFilesRelease = "/v1/control/files/release" PathFilesEnsure = "/v1/control/files/ensure" PathFilesStage = "/v1/control/files/stage" PathFilesTemp = "/v1/control/files/temp" @@ -73,6 +74,7 @@ func BackendPaths() []string { PathModelDelete, PathModelsRunning, PathNodeStop, + PathFilesRelease, PathFilesEnsure, PathFilesStage, PathFilesTemp, From bc8e84348d708dd47a440157e54cab00db2641fb Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:03:48 +0000 Subject: [PATCH 117/153] test(distributed): use renamed worker address Update the database latency fixture for NodeModel.WorkerLocalAddress. Assisted-by: Codex:gpt-6 Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/db_latency_resilience_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/distributed/db_latency_resilience_test.go b/tests/e2e/distributed/db_latency_resilience_test.go index 471beda7b6af..97121bb8bfcc 100644 --- a/tests/e2e/distributed/db_latency_resilience_test.go +++ b/tests/e2e/distributed/db_latency_resilience_test.go @@ -73,7 +73,7 @@ var _ = Describe("Control plane under a failing database", Label("Distributed"), // would evict when it could not read the slot table. Expect(db.Create(&nodes.NodeModel{ ID: "nm-victim", NodeID: "n-keep", ModelName: "victim", ReplicaIndex: 0, - State: "loaded", InFlight: 0, Address: "10.0.0.1:9001", + State: "loaded", InFlight: 0, WorkerLocalAddress: "10.0.0.1:9001", LastUsed: time.Now().Add(-time.Hour), }).Error).ToNot(HaveOccurred()) From 4f6648b505f2e2b4b968554380174ee4df32b6e1 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Tue, 15 Sep 2026 03:08:50 +0000 Subject: [PATCH 118/153] test(nodes): match the frontend setup command The disabled-state card renders one frontend command. Check its count and required flags without waiting for a second command that does not exist. Worker commands remain covered by the enabled-state test. Assisted-by: Codex:gpt-6 Signed-off-by: Ettore Di Giacinto --- core/http/react-ui/e2e/nodes-roster.spec.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/http/react-ui/e2e/nodes-roster.spec.js b/core/http/react-ui/e2e/nodes-roster.spec.js index b4bcc589be36..d3fca900c356 100644 --- a/core/http/react-ui/e2e/nodes-roster.spec.js +++ b/core/http/react-ui/e2e/nodes-roster.spec.js @@ -98,12 +98,12 @@ test.describe('Nodes join command', () => { // command carrying them fails at kong before LocalAI does anything. await expect(card).not.toContainText('--distributed-nats') await expect(card).not.toContainText('--distributed-db') - // Neither step tells an operator to point anything at a bus. The FRONTEND - // command is asserted first and by itself: it is the one that used to carry - // --nats-url as a required flag, so an operator following this card would - // have stood a broker up before starting LocalAI at all. - await expect(card.locator('.p2p-cmd pre').nth(0)).toContainText('--auth-database-url') - await expect(card.locator('.p2p-cmd pre').nth(0)).not.toContainText('--nats-url') - await expect(card.locator('.p2p-cmd pre').nth(1)).not.toContainText('--nats-url') + // The disabled-state card starts the frontend. Worker commands appear + // after distributed mode is enabled and are covered by the test above. + const frontendCommand = card.locator('.p2p-cmd pre') + await expect(frontendCommand).toHaveCount(1) + await expect(frontendCommand).toContainText('local-ai run --distributed') + await expect(frontendCommand).toContainText('--auth-database-url') + await expect(frontendCommand).not.toContainText('--nats-url') }) }) From b06149b1747d75ff9d431713f9d83a9f7815d735 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 15 Sep 2026 07:49:52 +0000 Subject: [PATCH 119/153] docs(distributed): define binary conformance The existing process tests prove chat routing and model transfer, but file-bearing backend operations rely on in-process coverage. Define the binary deployment shape, feature inventory, fixture contract, master baseline, and CI repair boundary before implementation. Assisted-by: Codex:GPT-5 [apply_patch] [exec_command] Signed-off-by: Ettore Di Giacinto --- ...5-distributed-binary-conformance-design.md | 240 ++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-15-distributed-binary-conformance-design.md diff --git a/docs/superpowers/specs/2026-09-15-distributed-binary-conformance-design.md b/docs/superpowers/specs/2026-09-15-distributed-binary-conformance-design.md new file mode 100644 index 000000000000..2a2f01329735 --- /dev/null +++ b/docs/superpowers/specs/2026-09-15-distributed-binary-conformance-design.md @@ -0,0 +1,240 @@ +# Distributed binary conformance suite + +Status: approved for implementation +Date: 2026-09-15 + +## Problem + +The distributed tests cover routing, model staging, and selected file transfers. +However, most file-transfer tests use in-process servers and direct test dialers. +They do not prove that a normal LocalAI deployment preserves the complete backend +feature set through worker tunnels and frontend relays. + +The existing process tests start real frontend and worker binaries. Those tests +prove chat inference and model artifact transfer. They do not prove deterministic +input and output transfer for each file-bearing backend operation. The mock image +and video methods also return success without writing an output file. + +This gap can hide two regression classes: + +- A new backend RPC can work in local mode but fail through a distributed wrapper. +- A file-bearing RPC can report success while an input or output remains on the + wrong host. + +The current pull request also has failures in both distributed CI jobs. The new +suite cannot provide a useful signal until those regressions are fixed. + +## Goal + +Run a binary-level conformance suite that proves the current distributed backend +feature set behaves like local mode. Use deterministic fixture data to prove each +transfer and response. + +## Deployment fidelity + +The suite starts compiled `local-ai` processes through their public commands. It +does not replace a frontend, worker, tunnel, or backend supervisor with an +in-process substitute. + +The harness uses the deployment shape that users run: + +- two frontend processes share PostgreSQL state; +- two worker processes connect through the public cluster WebSocket endpoint; +- each worker starts the mock backend through the normal backend supervisor; +- frontend and worker processes use separate model and data directories; +- workers do not expose a backend address to the frontend; +- requests enter through the public frontend HTTP API where an API exists; +- the non-owner frontend exercises the inter-replica relay path; +- the owner frontend exercises the direct tunnel path. + +Tests use the same command flags as the documented frontend and worker setup. A +test helper can reduce repeated process code, but it cannot bypass a production +transport boundary. + +## Coverage boundary + +The suite covers the backend feature set that distributed mode routes. It does +not duplicate unrelated admin, gallery, authentication, or user-management tests. + +Coverage has three layers. + +### Public API conformance + +Reuse the existing mock-backend API expectations where possible. Run requests +through a frontend for these operations: + +- non-streaming and streaming text generation; +- embeddings; +- image generation and image inputs; +- video generation and its image and audio inputs; +- 3D generation and its source asset; +- non-streaming and streaming TTS; +- sound generation; +- non-streaming and streaming transcription; +- detection APIs that accept image or audio files; +- reranking, tokenization, detokenization, scoring, and stores operations; +- supported audio analysis and conversion operations; +- model metadata and other path-free backend calls exposed by an API. + +The test must compare structured fixture values. A status-only assertion is not +sufficient. + +### File-transfer conformance + +Exercise each method that `FileStagingClient` overrides: + +- `LoadModel` with the primary artifact and companion files; +- `Predict` and `PredictStream` with image, video, and audio paths; +- `GenerateImage` with source and reference images, plus its output; +- `GenerateVideo` with start image, end image, audio, and output; +- `Generate3D` with a source asset and output; +- `TTS` and `TTSStream` with a model path, voice file, and multiple references; +- `SoundGeneration` with source audio and output; +- `SoundDetection` with source audio; +- `AudioTranscription` and `AudioTranscriptionStream` with source audio; +- `ExportModel` with nested output files; +- quantization progress with its output file. + +For each input, the mock backend reads the remote file and reports a digest or +fixture marker. For each output, the mock backend writes deterministic bytes on +the worker. The test compares the bytes retrieved by the frontend. + +### Protocol conformance + +Exercise representative unary, server-streaming, client-streaming, and +bidirectional-streaming calls through both tunnel paths. Include protocol methods +that do not have a public HTTP route. + +Keep an explicit inventory beside the test cases. A guard compares that inventory +with `grpc.InferenceBackend`, `grpc.ControlBackend`, and the file-staging override +set. A new method must get one of these classifications: + +- covered by a process-level conformance case; +- covered by a representative generic transport case; +- intentionally local-only, with a reason. + +Compile-time interface checks continue to protect wrapper completeness. The +inventory protects behavioral coverage. + +## Fixture backend + +Extend `tests/e2e/mock-backend` instead of creating a second mock protocol. +Existing local E2E tests continue to use this backend. + +The backend emits small deterministic fixtures: + +- a valid PNG for image output; +- a small video fixture or stable opaque bytes when the API does not decode it; +- a valid WAV for TTS and sound output; +- a small GLB fixture for 3D output; +- nested files for export and quantization output; +- fixed protobuf values for path-free calls and streams. + +File-bearing methods fail when a required fixture is absent or has unexpected +content. This behavior proves that the worker received the input. Output tests +compare exact bytes or a digest after the frontend retrieves the file. + +The backend keeps fixtures small so transfer coverage does not dominate the job +duration. Existing large-transfer tests continue to cover multiplexing pressure. + +## Test organization + +Keep process lifecycle helpers in the existing cluster harness. Put the feature +matrix in focused files under `tests/e2e/distributed/`. + +Use table-driven cases for methods with the same request and response shape. Use +separate cases for streaming, file transfer, lifecycle, and error behavior. + +Start the cluster once for the feature matrix when test isolation permits it. +Reset model and request state between cases. A failure must report the endpoint, +frontend replica, worker, and expected fixture marker. + +Run each transport-sensitive group against these paths: + +1. A request enters the frontend that owns the worker tunnel. +2. A request enters the other frontend and crosses the peer relay. + +Do not duplicate path-independent response checks when one response assertion and +one transport matrix can prove the same behavior. + +## Main-branch baseline + +Create a temporary worktree at the current `origin/master`. Apply only the test +and fixture-backend changes needed to run the conformance cases. Do not change +production behavior in the baseline worktree. + +Classify results as follows: + +- A case that passes on `master` must pass on the pull-request branch. +- A case that fails only on the pull-request branch is a regression. +- A case that exposes an existing `master` defect is recorded separately. Fix it + in this pull request only when distributed tunnel compatibility requires it. +- A branch-only feature has a focused expected result and no false baseline claim. + +Delete the temporary worktree after recording the commands and results. + +## Current CI regressions + +Fix each regression with a focused failing test before changing production code. + +### Node health transition + +A tunnel departure can mark a node unhealthy while its last heartbeat remains +fresh. The stale-heartbeat branch later skips unhealthy nodes, so automatic +offline handling never completes. + +The expected state sequence is `healthy -> unhealthy -> offline` when automatic +offline handling is active and the heartbeat becomes stale. + +### Backend stop compatibility + +New workers return a truthful `BackendStopReply`. An older worker can return an +empty successful HTTP response. A rolling upgrade must accept the legacy empty +reply while preserving an explicit failure from a new worker. + +Tests cover legacy success, explicit success, explicit failure, and transport +failure. + +### Restricted database test + +The database latency test constructs a node registry with a role that cannot run +schema migrations. Separate schema setup from the runtime fault-injection role. +The production registry keeps its normal migration contract. + +## Error behavior + +The conformance suite treats these conditions as failures: + +- the mock backend cannot read an expected staged input; +- input bytes or fixture markers differ; +- the frontend cannot retrieve an output; +- output bytes differ; +- a stream loses, reorders, or duplicates fixture frames; +- an explicit backend failure becomes success; +- a direct request works but its relayed equivalent fails; +- a backend method has no coverage classification. + +Assertions use bounded deadlines. Process logs remain available on failure. + +## Verification + +Run verification in increasing cost order: + +1. Focused unit tests for the three CI regressions. +2. Fast in-process distributed tests. +3. The feature matrix through compiled frontend and worker binaries. +4. Existing cluster lifecycle and relay tests. +5. The same compatible conformance cases in the `origin/master` worktree. +6. The repository targets used by `tests-e2e-distributed` and + `tests-e2e-cluster` in CI. + +Do not lower a timeout or coverage gate to make a failure pass. Ask before a full +repository build if targeted binary builds cannot satisfy the suite. + +## Non-goals + +- Testing admin APIs that do not route to a backend. +- Testing model quality or backend-specific numerical accuracy. +- Replacing existing unit and in-process tests. +- Adding a second distributed transport or a test-only frontend endpoint. +- Making the mock backend emulate large model runtimes. From 58d0bfa468ef51700ee770e096ddca3886dd0704 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 15 Sep 2026 09:25:29 +0000 Subject: [PATCH 120/153] docs(distributed): test machine authentication The binary conformance design now covers a WebUI-authenticated deployment. It distinguishes browser sessions, shared registration credentials, and per-node tunnel credentials, including rejection and approval paths. Assisted-by: Codex:GPT-5 [apply_patch] [exec_command] Signed-off-by: Ettore Di Giacinto --- ...5-distributed-binary-conformance-design.md | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/2026-09-15-distributed-binary-conformance-design.md b/docs/superpowers/specs/2026-09-15-distributed-binary-conformance-design.md index 2a2f01329735..28f3a3bcf743 100644 --- a/docs/superpowers/specs/2026-09-15-distributed-binary-conformance-design.md +++ b/docs/superpowers/specs/2026-09-15-distributed-binary-conformance-design.md @@ -51,6 +51,29 @@ Tests use the same command flags as the documented frontend and worker setup. A test helper can reduce repeated process code, but it cannot bypass a production transport boundary. +### Authenticated deployment + +Run one binary scenario with WebUI authentication enabled and distributed +authentication set to fail closed. Set the same non-empty registration token on +the frontend and worker processes. + +The scenario proves these behaviors: + +- a worker with the correct registration token registers and sends heartbeats; +- registration returns a unique per-node tunnel token; +- the worker opens its tunnel with the per-node token, not a WebUI session; +- the authenticated WebUI remains available to a browser user; +- missing and incorrect registration tokens cannot register a worker; +- missing and incorrect tunnel tokens cannot open a worker tunnel; +- a pending node receives `403` until an administrator approves it; +- a backend worker does not receive or require a user API key; +- an approved agent worker receives a scoped API key when it needs the inference + API. + +The global WebUI middleware must delegate `/api/node/*` and `/api/cluster/*` to +their machine-authentication handlers. The test fails if these routes accept an +anonymous machine or require a WebUI cookie. + ## Coverage boundary The suite covers the backend feature set that distributed mode routes. It does @@ -213,6 +236,8 @@ The conformance suite treats these conditions as failures: - an explicit backend failure becomes success; - a direct request works but its relayed equivalent fails; - a backend method has no coverage classification. +- WebUI authentication bypasses machine authentication, or blocks a valid + machine credential. Assertions use bounded deadlines. Process logs remain available on failure. @@ -223,9 +248,10 @@ Run verification in increasing cost order: 1. Focused unit tests for the three CI regressions. 2. Fast in-process distributed tests. 3. The feature matrix through compiled frontend and worker binaries. -4. Existing cluster lifecycle and relay tests. -5. The same compatible conformance cases in the `origin/master` worktree. -6. The repository targets used by `tests-e2e-distributed` and +4. The authenticated binary deployment scenario. +5. Existing cluster lifecycle and relay tests. +6. The same compatible conformance cases in the `origin/master` worktree. +7. The repository targets used by `tests-e2e-distributed` and `tests-e2e-cluster` in CI. Do not lower a timeout or coverage gate to make a failure pass. Ask before a full From a626c9c56d6c4003eebf0fa4cdfd5fa123696928 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 15 Sep 2026 09:35:55 +0000 Subject: [PATCH 121/153] fix(distributed): offline stale unhealthy nodes Allow automatic stale-node handling to advance unhealthy nodes to offline while preserving unhealthy status when automatic handling is disabled. Assisted-by: Codex:GPT-5 [apply_patch] [exec_command] Signed-off-by: Ettore Di Giacinto --- core/services/nodes/health.go | 8 ++++---- core/services/nodes/health_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/core/services/nodes/health.go b/core/services/nodes/health.go index cf6286ea22a8..e4648a6e97c0 100644 --- a/core/services/nodes/health.go +++ b/core/services/nodes/health.go @@ -250,10 +250,10 @@ func (hm *HealthMonitor) doCheckAll(ctx context.Context) { // Workers (both backend and agent) send HTTP heartbeats to the frontend. // If the heartbeat is stale, the worker is presumed down. if time.Since(node.LastHeartbeat) > hm.staleThreshold { - // Skip nodes already marked offline/unhealthy — re-marking them - // every cycle floods the log with the same WARN+INFO pair for - // nodes the operator has intentionally taken down. - if node.Status == StatusOffline || node.Status == StatusUnhealthy { + // Skip nodes already at the monitor's terminal stale state. When + // automatic offline handling is enabled, unhealthy is an intermediate + // state and a stale node still needs to transition to offline. + if node.Status == StatusOffline || node.Status == StatusUnhealthy && !hm.autoOffline { continue } xlog.Warn("Node heartbeat stale", "node", node.Name, "lastHeartbeat", node.LastHeartbeat) diff --git a/core/services/nodes/health_test.go b/core/services/nodes/health_test.go index 26400d9417b6..14ae13d38f6b 100644 --- a/core/services/nodes/health_test.go +++ b/core/services/nodes/health_test.go @@ -172,6 +172,32 @@ var _ = Describe("HealthMonitor (mock-based)", func() { Expect(store.getCalls()).To(ContainElement("MarkUnhealthy:node-2")) }) + It("marks a stale unhealthy node offline when autoOffline=true", func() { + store := newFakeNodeHealthStore() + factory := newFakeBackendClientFactory() + hm := newTestHealthMonitor(store, factory, true, staleThreshold) + + node := makeTestNode("node-stale-unhealthy", "stale-unhealthy-worker", "10.0.0.7:50051", StatusUnhealthy, staleTime(staleThreshold)) + store.addNode(node) + + hm.doCheckAll(context.Background()) + + Expect(store.getNode(node.ID).Status).To(Equal(StatusOffline)) + }) + + It("keeps a stale unhealthy node unchanged when autoOffline=false", func() { + store := newFakeNodeHealthStore() + factory := newFakeBackendClientFactory() + hm := newTestHealthMonitor(store, factory, false, staleThreshold) + + node := makeTestNode("node-stale-unhealthy", "stale-unhealthy-worker", "10.0.0.7:50051", StatusUnhealthy, staleTime(staleThreshold)) + store.addNode(node) + + hm.doCheckAll(context.Background()) + + Expect(store.getNode(node.ID).Status).To(Equal(StatusUnhealthy)) + }) + It("skips draining nodes", func() { store := newFakeNodeHealthStore() factory := newFakeBackendClientFactory() From c06dd46f135578dfb64b041c3a135ee6d47d554f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 15 Sep 2026 09:40:52 +0000 Subject: [PATCH 122/153] fix(distributed): accept legacy empty stop replies Treat an empty backend.stop response as the legacy success acknowledgement while preserving explicit worker failures and transport errors. Assisted-by: Codex:GPT-5 [apply_patch] [exec_command] Signed-off-by: Ettore Di Giacinto --- core/services/nodes/unloader.go | 5 +++- core/services/nodes/unloader_test.go | 40 +++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index bcfca1a9dfb6..632dd005c0ad 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -411,7 +411,10 @@ func (a *RemoteUnloaderAdapter) StopBackend(nodeID, backend string) error { func (a *RemoteUnloaderAdapter) stopBackend(ctx context.Context, nodeID, backend string, force bool) error { // An empty Backend is what the worker reads as "stop everything"; see // decodeBackendStopRequest. - var reply messaging.BackendStopReply + // Workers predating BackendStopReply acknowledge this verb with 204, so + // success is the compatibility default that survives an empty body. A + // decoded reply still replaces it with the worker's explicit verdict. + reply := messaging.BackendStopReply{Success: true} if err := a.control.Call(ctx, nodeID, workerctl.PathBackendStop, messaging.BackendStopRequest{Backend: backend, Force: force}, &reply); err != nil { return err diff --git a/core/services/nodes/unloader_test.go b/core/services/nodes/unloader_test.go index d3efd1d5f77c..916914951aa1 100644 --- a/core/services/nodes/unloader_test.go +++ b/core/services/nodes/unloader_test.go @@ -4,6 +4,9 @@ import ( "context" "encoding/json" "errors" + "net" + "net/http" + "net/http/httptest" "sync" "time" @@ -112,7 +115,6 @@ func (f *fakeMessagingClient) Subscribe(_ string, _ func([]byte)) (messaging.Sub return &fakeSubscription{}, nil } - type fakeSubscription struct{} func (f *fakeSubscription) Unsubscribe() error { return nil } @@ -137,6 +139,20 @@ func stopPayloadOf(s *scriptedControlWorkers, key string) messaging.BackendStopR return messaging.BackendStopRequest{} } +func legacyNoContentControlClient() *ControlClient { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + DeferCleanup(server.Close) + + return NewControlClient(func(string) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, _, _ string) (net.Conn, error) { + var dialer net.Dialer + return dialer.DialContext(ctx, "tcp", server.Listener.Addr().String()) + } + }, "") +} + // --- Tests --- var _ = Describe("RemoteUnloaderAdapter", func() { @@ -309,8 +325,19 @@ var _ = Describe("RemoteUnloaderAdapter", func() { // single case would show that some node gets the path; only two show that // the two kinds no longer differ, which is the whole content of the change. Describe("StopBackend and node type", func() { - It("surfaces a backend worker's stop failure", func() { - locator.nodes = []BackendNode{{ID: "backend-1", Name: "gpu-1", NodeType: NodeTypeBackend}} + It("accepts a legacy empty backend.stop reply", func() { + adapter = NewRemoteUnloaderAdapter(locator, legacyNoContentControlClient(), 3*time.Minute, 15*time.Minute) + + Expect(adapter.StopBackend("legacy-worker", "llama-backend")).To(Succeed()) + }) + + It("accepts an explicit successful backend.stop reply", func() { + scriptStop("backend-1") + + Expect(adapter.StopBackend("backend-1", "llama-backend")).To(Succeed()) + }) + + It("honors an explicit failed backend.stop reply", func() { workers.scriptReply(controlKey("backend-1", workerctl.PathBackendStop), messaging.BackendStopReply{ Success: false, Error: "llama#0: process refused to die", @@ -321,6 +348,13 @@ var _ = Describe("RemoteUnloaderAdapter", func() { Expect(err).To(MatchError(ContainSubstring("process refused to die"))) }) + It("returns a backend.stop transport failure", func() { + workers.scriptUnroutable("missing-worker") + + err := adapter.StopBackend("missing-worker", "llama-backend") + Expect(errors.Is(err, ErrWorkerUnroutable)).To(BeTrue()) + }) + It("sends a backend stop to a BACKEND node over the tunnel, naming the backend", func() { locator.nodes = []BackendNode{{ID: "backend-1", Name: "gpu-1", NodeType: NodeTypeBackend}} scriptStop("backend-1") From 6f1f0aa3821edbc1aa3b431ec54a6909524fa642 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 15 Sep 2026 09:45:15 +0000 Subject: [PATCH 123/153] test(distributed): migrate before restricting DB The failure fixture constructed a second registry with its restricted role. That role cannot create the cluster sequence, so the spec failed during migration instead of the slot lookup. Keep migration on the admin handle and replace only the registry runtime handle after setup. Assisted-by: Codex:GPT-5 [apply_patch] [exec_command] Signed-off-by: Ettore Di Giacinto --- core/services/nodes/registry.go | 6 ++++++ core/services/nodes/registry_test.go | 16 ++++++++++++++ .../distributed/db_latency_resilience_test.go | 21 ++++++++++--------- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/core/services/nodes/registry.go b/core/services/nodes/registry.go index e93e184e7bc5..cce2e28ba941 100644 --- a/core/services/nodes/registry.go +++ b/core/services/nodes/registry.go @@ -588,6 +588,12 @@ func NewNodeRegistry(db *gorm.DB) (*NodeRegistry, error) { }, nil } +// UseDBForTest replaces the registry's database after test setup has finished. +// Callers must use it before starting concurrent registry operations. +func (r *NodeRegistry) UseDBForTest(db *gorm.DB) { + r.db = db +} + // resolveVRAMBudgetBytes turns a budget string into an absolute byte ceiling // against the node's raw total VRAM, clamped to that total. 0 means no cap or // unparseable (fail-open at this layer; the endpoint validates input). diff --git a/core/services/nodes/registry_test.go b/core/services/nodes/registry_test.go index 4287849ec8f2..980e738c08f1 100644 --- a/core/services/nodes/registry_test.go +++ b/core/services/nodes/registry_test.go @@ -40,6 +40,22 @@ var _ = Describe("NodeRegistry", func() { } } + Describe("UseDBForTest", func() { + It("routes registry queries through the replacement handle", func() { + runtimeDB := testutil.SetupTestDB() + _, err := NewNodeRegistry(runtimeDB) + Expect(err).ToNot(HaveOccurred()) + + worker := makeNode("replacement-db-worker", "10.0.0.9:50051", 0) + Expect(runtimeDB.Create(worker).Error).ToNot(HaveOccurred()) + + registry.UseDBForTest(runtimeDB) + fetched, err := registry.GetByName(context.Background(), worker.Name) + Expect(err).ToNot(HaveOccurred()) + Expect(fetched.ID).To(Equal(worker.ID)) + }) + }) + Describe("Register", func() { It("persists CPU telemetry and clamps utilization", func() { node := makeNode("cpu-worker", "10.0.0.3:50051", 0) diff --git a/tests/e2e/distributed/db_latency_resilience_test.go b/tests/e2e/distributed/db_latency_resilience_test.go index 97121bb8bfcc..581ba54d1e15 100644 --- a/tests/e2e/distributed/db_latency_resilience_test.go +++ b/tests/e2e/distributed/db_latency_resilience_test.go @@ -32,10 +32,11 @@ const ( // that failed evicted a healthy model that had done nothing wrong. var _ = Describe("Control plane under a failing database", Label("Distributed"), func() { var ( - infra *TestInfra - db *gorm.DB - routerDB *gorm.DB - ctx context.Context + infra *TestInfra + db *gorm.DB + routerDB *gorm.DB + routerRegistry *nodes.NodeRegistry + ctx context.Context ) openDB := func(dsn string) *gorm.DB { @@ -60,7 +61,8 @@ var _ = Describe("Control plane under a failing database", Label("Distributed"), ctx = context.Background() db = openDB(infra.PGURL) - _, err := nodes.NewNodeRegistry(db) + var err error + routerRegistry, err = nodes.NewNodeRegistry(db) Expect(err).ToNot(HaveOccurred()) Expect(db.Create(&nodes.BackendNode{ @@ -127,11 +129,10 @@ END $$`, routerRole)).Error).ToNot(HaveOccurred()) } It("does not evict a loaded model when the database refuses the slot lookup", func() { - // Built before the revoke: AutoMigrate inspects the schema, and a - // migration that cannot run would degrade this spec into asserting - // almost nothing. - routerRegistry, err := nodes.NewNodeRegistry(routerDB) - Expect(err).ToNot(HaveOccurred()) + // Migration finished on the admin handle in BeforeEach. Only runtime + // queries use the restricted handle, so missing schema privileges cannot + // move the injected failure into registry construction. + routerRegistry.UseDBForTest(routerDB) revokeSlotReads() From 189439f2d030e705e14ace5c229d64d920e7fb4a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 15 Sep 2026 10:25:53 +0000 Subject: [PATCH 124/153] test(distributed): pin cluster health timings Assisted-by: Codex:GPT-5 [apply_patch] [exec_command] Signed-off-by: Ettore Di Giacinto --- tests/e2e/distributed/cluster/cluster.go | 9 +++ .../e2e/distributed/cluster_failover_test.go | 60 ++++++------------- 2 files changed, 28 insertions(+), 41 deletions(-) diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index 4efd358884b2..2fc0a5fa0237 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -191,6 +191,13 @@ type Cluster struct { const ( defaultRegistrationToken = "e2e-token" defaultAdminEmail = "admin@e2e.local" + // The failover specs have bounded observation windows sized around a 60s + // stale threshold. Production deliberately defaults to 5m, so the harness + // pins its shorter test clock explicitly. Checkpoint below the worker's 10s + // heartbeat interval so every beat remains durable and the 60s threshold + // measures worker silence rather than checkpoint lag. + testStaleNodeThreshold = "60s" + testNodeHeartbeatCheckpoint = "5s" // testHMACSecret is shared by every frontend so a session minted at one // replica validates at all of them. See the note in startFrontend. testHMACSecret = "e2e-cluster-hmac-secret" @@ -333,6 +340,8 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { // Pinning makes the cross-replica session a property of the harness. "LOCALAI_AUTH_HMAC_SECRET="+testHMACSecret, "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, + "LOCALAI_STALE_NODE_THRESHOLD="+testStaleNodeThreshold, + "LOCALAI_NODE_HEARTBEAT_CHECKPOINT="+testNodeHeartbeatCheckpoint, // Every replica here shares one host, so the address a peer dials is // this process's own loopback address. It has to be said explicitly: // the automatic discovery asks which local address routes to diff --git a/tests/e2e/distributed/cluster_failover_test.go b/tests/e2e/distributed/cluster_failover_test.go index d007a8ae6ac8..4e50100d862c 100644 --- a/tests/e2e/distributed/cluster_failover_test.go +++ b/tests/e2e/distributed/cluster_failover_test.go @@ -33,9 +33,11 @@ const ( // It is sized from the only eviction path there is. Node liveness is // heartbeat freshness: the health monitor wakes every HealthCheckInterval // (15s) and marks any node whose last heartbeat is older than - // StaleNodeThreshold (60s) offline (core/services/nodes/health.go, defaults - // in core/config/distributed_config.go). Neither is settable from the CLI, - // so 60s + one 15s tick = 75s is the worst case and cannot be shortened. + // StaleNodeThreshold (60s) offline (core/services/nodes/health.go). The + // binary's production default is 5m, so cluster.Start explicitly supplies + // this suite's 60s threshold and a 5s heartbeat checkpoint to every real + // frontend. With the default 15s health interval, 60s + one tick = 75s is + // the worst case. // // Measured rather than assumed: a worker whose registrar was killed goes // offline at both surviving replicas at t=74.2s. A window shorter than that @@ -106,28 +108,6 @@ func (p *rosterProbe) explain(format string, args ...any) func() string { } } -// explainStuckOffline is explain with one extra diagnosis attached. -// -// An assertion waiting for offline has a failure mode that looks like a harness -// bug and is not one, so the message names it rather than leaving the reader to -// find it. See proveHealthCheckingIsAlive and the comment on the dead-worker -// spec for the mechanism. -func (p *rosterProbe) explainStuckOffline(worker, format string, args ...any) func() string { - return func() string { - message := fmt.Sprintf(format, args...) + ": " + p.describe() - for _, n := range p.lastSeen { - if n.Name != worker || n.Status != nodes.StatusUnhealthy { - continue - } - message += "\n\nThe node is stuck at unhealthy, which is a LocalAI defect rather than " + - "a harness one: core/services/nodes/health.go:153-155 skips MarkOffline for a node " + - "already marked unhealthy, so a node whose unhealthy mark lands after its heartbeat " + - "has gone stale never reaches offline at all. Start there, not here." - } - return message - } -} - // proveHealthCheckingIsAlive kills a worker and waits for the roster to settle // it to offline. // @@ -175,7 +155,7 @@ func proveHealthCheckingIsAlive(c *cluster.Cluster, probe *rosterProbe, workerIn Eventually(probe.statusOf, workerDeathTimeout, rosterPollInterval). WithArguments(worker). Should(Equal(nodes.StatusOffline), - probe.explainStuckOffline(worker, + probe.explain( "frontend %d never reacted to a killed worker, so health checking was not running during the window above and the assertion before this one proved nothing", probe.frontend)) } @@ -188,6 +168,13 @@ var _ = Describe("Cluster failover", Label("Distributed"), Label("Cluster"), fun c := startCluster(2, 1) worker := c.WorkerName(0) + frontendEnviron, err := c.FrontendEnviron(0) + Expect(err).ToNot(HaveOccurred()) + Expect(frontendEnviron).To(ContainElements( + "LOCALAI_STALE_NODE_THRESHOLD=60s", + "LOCALAI_NODE_HEARTBEAT_CHECKPOINT=5s", + ), "the failover timing budgets only apply when the real frontend runs with the harness's health timings") + // One session for the whole cluster: register/login/token-login/password // share a five-per-minute-per-IP budget at every frontend // (core/http/routes/auth.go:190) and everything here comes from @@ -309,20 +296,11 @@ var _ = Describe("Cluster failover", Label("Distributed"), Label("Cluster"), fun // the stale-detection path rather than to the transient, which any // not-healthy or not-present matcher would accept at t=8s. // - // KNOWN HAZARD, read this before blaming the harness for a timeout here. - // The staleness branch skips a node that is already unhealthy - // (core/services/nodes/health.go:153-155, `if node.Status == - // StatusOffline || node.Status == StatusUnhealthy { continue }`). The - // skip exists to stop the monitor re-logging nodes an operator took - // down, but it applies to the flap too: if the transient unhealthy mark - // lands AFTER the heartbeat has already gone stale, rather than at the - // ~8s observed here, MarkOffline is never called and this node stays - // unhealthy forever. This spec would then hang to workerDeathTimeout - // and fail with a roster that looks perfectly ordinary. The ordering - // that triggers it did not occur in any run so far, but nothing - // prevents it, so explainStuckOffline says so in the failure message - // when it sees a node stuck at unhealthy. Fixing it is LocalAI work, - // not test work. + // The stale branch must also advance a node that a liveness probe already + // marked unhealthy. Otherwise the transient above can become permanent + // depending on whether it lands before or after the stale cutoff; the + // production health-monitor tests pin that transition, while this spec + // proves it through real processes. // Reading the same verdict at both replicas proves shared-verdict // propagation, NOT two independent detectors. Health checks are // single-flighted by the advisory lock (see proveHealthCheckingIsAlive), @@ -336,7 +314,7 @@ var _ = Describe("Cluster failover", Label("Distributed"), Label("Cluster"), fun Eventually(probe.statusOf, workerDeathTimeout, rosterPollInterval). WithArguments(worker). Should(Equal(nodes.StatusOffline), - probe.explainStuckOffline(worker, "frontend %d never settled the dead worker to offline", probe.frontend)) + probe.explain("frontend %d never settled the dead worker to offline", probe.frontend)) } // And it has to stay offline. Nothing may resurrect a row for a process From 9bd91391e4231b5845030b2cfcd44943d6c3809f Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 15 Sep 2026 17:23:36 +0200 Subject: [PATCH 125/153] fix(ui): complete node operation journeys (#12070) Worker setup now uses a focused drawer without interrupting fleet monitoring. Both generated worker commands retain their NATS settings. Log views preserve their launch context and show one-based replica labels. Bulk controls disclose selections outside the current view. Assisted-by: Codex:gpt-5 Playwright ESLint Co-authored-by: Ettore Di Giacinto Signed-off-by: Ettore Di Giacinto --- core/http/react-ui/e2e/backend-logs.spec.js | 22 +++ core/http/react-ui/e2e/nodes-detail.spec.js | 2 + .../e2e/nodes-fleet-dashboard.spec.js | 46 +++++- .../nodes-per-node-backend-actions.spec.js | 29 ++++ core/http/react-ui/e2e/nodes-roster.spec.js | 24 +-- core/http/react-ui/inline-style-baseline.txt | 2 +- .../react-ui/public/locales/en/admin.json | 47 +++++- core/http/react-ui/src/App.css | 27 ++++ core/http/react-ui/src/pages/BackendLogs.jsx | 2 +- .../react-ui/src/pages/NodeBackendLogs.jsx | 147 +++++++----------- core/http/react-ui/src/pages/NodeDetail.jsx | 2 +- core/http/react-ui/src/pages/Nodes.jsx | 81 +++++++--- docs/content/features/distributed-mode.md | 6 + 13 files changed, 292 insertions(+), 145 deletions(-) diff --git a/core/http/react-ui/e2e/backend-logs.spec.js b/core/http/react-ui/e2e/backend-logs.spec.js index 0390aa6bcebe..9eaf3eced15c 100644 --- a/core/http/react-ui/e2e/backend-logs.spec.js +++ b/core/http/react-ui/e2e/backend-logs.spec.js @@ -61,4 +61,26 @@ test.describe('Backend Logs', () => { // Button label changes to "Show details" await expect(page.locator('button', { hasText: 'Show details' })).toBeVisible() }) + + test('distributed resolver displays zero-based replica indexes as one-based labels', async ({ page }) => { + await page.route('**/api/nodes', route => route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify([{ id: 'worker-1', name: 'Worker 1' }]), + })) + await page.route('**/api/nodes/worker-1/models', route => route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify([ + { model_name: 'llama', replica_index: 0, state: 'loaded' }, + { model_name: 'llama', replica_index: 1, state: 'loaded' }, + ]), + })) + + await page.goto('/app/backend-logs/llama') + + await expect(page.getByText('worker-1 · replica 1 · loaded')).toBeVisible() + await expect(page.getByText('worker-1 · replica 2 · loaded')).toBeVisible() + await expect(page.locator('a[href="/app/node-backend-logs/worker-1/llama"]')).toHaveCount(2) + }) }) diff --git a/core/http/react-ui/e2e/nodes-detail.spec.js b/core/http/react-ui/e2e/nodes-detail.spec.js index 6793b3b9d860..1183f0416e27 100644 --- a/core/http/react-ui/e2e/nodes-detail.spec.js +++ b/core/http/react-ui/e2e/nodes-detail.spec.js @@ -30,6 +30,8 @@ test.describe('Node detail page', () => { await page.getByRole('button', { name: 'Actions for llama-3.3 replica 1' }).click() await page.getByRole('menuitem', { name: 'View logs' }).click() await expect(page).toHaveURL(/\/app\/node-backend-logs\/n1\/llama-3.3%230$/) + await page.getByRole('link', { name: 'Back to alpha' }).click() + await expect(page).toHaveURL(/\/app\/nodes\/n1$/) }) test('is reachable by clicking a roster panel', async ({ page }) => { diff --git a/core/http/react-ui/e2e/nodes-fleet-dashboard.spec.js b/core/http/react-ui/e2e/nodes-fleet-dashboard.spec.js index e7b0ef6e06b9..64eb7ea8ce37 100644 --- a/core/http/react-ui/e2e/nodes-fleet-dashboard.spec.js +++ b/core/http/react-ui/e2e/nodes-fleet-dashboard.spec.js @@ -134,6 +134,7 @@ test.describe('Nodes fleet dashboard', () => { await page.getByRole('checkbox', { name: 'Select page' }).check() await page.getByRole('searchbox', { name: 'Search nodes' }).fill('worker-1') await expect(page.getByText('12 selected')).toBeVisible() + await expect(page.getByText('3 visible, 9 outside this view')).toBeVisible() await page.getByRole('button', { name: 'Drain selected' }).evaluate(button => { button.click() button.click() @@ -150,6 +151,45 @@ test.describe('Nodes fleet dashboard', () => { expect(resumeRequests).not.toContain('n9') }) + test('opens Add worker as an accessible setup drawer and restores focus', async ({ page }) => { + await mockNodes(page, [baseNodes[0]]) + await page.goto('/app/nodes') + + const trigger = page.getByRole('button', { name: 'Add worker' }) + await trigger.click() + const drawer = page.getByRole('dialog', { name: 'Add worker' }) + await expect(drawer).toBeVisible() + await expect(drawer.getByRole('radiogroup', { name: 'Worker type' })).toBeVisible() + await expect(drawer.getByText('CLI', { exact: true })).toBeVisible() + await expect(drawer.getByText('Docker', { exact: true })).toBeVisible() + await expect(drawer.getByRole('button', { name: 'Copy command' })).toHaveCount(2) + await expect(drawer.getByRole('link', { name: 'Distributed mode documentation' })).toBeVisible() + await expect(drawer.getByRole('button', { name: 'Close worker setup' })).toBeFocused() + + await page.keyboard.press('Escape') + await expect(drawer).toBeHidden() + await expect(trigger).toBeFocused() + }) + + test('presents worker setup as a modal drawer on mobile', async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }) + await mockNodes(page, [baseNodes[0]]) + await page.goto('/app/nodes') + await page.getByRole('button', { name: 'Add worker' }).click() + await expect(page.getByRole('dialog', { name: 'Add worker' })).toHaveAttribute('aria-modal', 'true') + }) + + test('opens running-model logs with a fleet return destination', async ({ page }) => { + await mockNodes(page) + await page.route('**/api/nodes/models', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([baseModels[3]]) })) + await page.goto('/app/nodes') + await page.getByRole('tab', { name: /Running models/ }).click() + await page.getByRole('button', { name: 'Actions for Whisper large v3' }).click() + await page.getByRole('menuitem', { name: 'View logs…' }).click() + await page.getByRole('link', { name: 'Back to nodes' }).click() + await expect(page).toHaveURL(/\/app\/nodes$/) + }) + test('disables bulk controls and remove confirmation while removal is running', async ({ page }) => { await mockNodes(page, [baseNodes[0]]) let finishRemove @@ -504,10 +544,12 @@ test.describe('Nodes fleet dashboard', () => { await expect(modelControl).toHaveAttribute('aria-expanded', 'false') await expect(modelControl).not.toHaveAttribute('aria-controls') - await page.getByRole('button', { name: 'Inspect Whisper large v3' }).click() + const whisperControl = page.getByRole('button', { name: 'Inspect Whisper large v3' }) + await whisperControl.click() await expect(page.getByRole('complementary', { name: 'Model inspector' })).toContainText('missing') await expect(page.getByRole('complementary', { name: 'Model inspector' })).toContainText('Unknown') await page.getByRole('button', { name: 'Close model inspector' }).click() + await expect(whisperControl).toBeFocused() await modelsTab.focus() await modelsTab.press('ArrowLeft') @@ -570,7 +612,7 @@ test.describe('Nodes fleet dashboard', () => { const inspector = page.getByRole('complementary', { name: 'Node inspector' }) await expect(inspector).toBeVisible() - await page.getByRole('button', { name: 'Remove selected' }).click() + await page.getByRole('button', { name: 'Remove selected' }).evaluate(button => button.click()) await expect(page.getByRole('alertdialog')).toBeVisible() await page.keyboard.press('Escape') diff --git a/core/http/react-ui/e2e/nodes-per-node-backend-actions.spec.js b/core/http/react-ui/e2e/nodes-per-node-backend-actions.spec.js index 7cbd2ff02c2a..4f8168291feb 100644 --- a/core/http/react-ui/e2e/nodes-per-node-backend-actions.spec.js +++ b/core/http/react-ui/e2e/nodes-per-node-backend-actions.spec.js @@ -108,6 +108,35 @@ async function openBackendActions(page) { } test.describe('Nodes page — per-node backend actions', () => { + test('backend log page uses one-based replica labels, token classes, and a safe direct-link return', async ({ page }) => { + await mockDistributedNodes(page) + await page.route(`**/api/nodes/${NODE_ID}/models`, route => route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify([ + { model_name: 'llama', replica_index: 0 }, + { model_name: 'llama', replica_index: 1 }, + ]), + })) + await page.route(`**/api/nodes/${NODE_ID}/backend-logs/**`, route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([ + { timestamp: '2026-09-15T12:00:00Z', stream: 'stdout', text: 'ready' }, + ]) })) + + await page.goto(`/app/node-backend-logs/${NODE_ID}/llama%230`) + await expect(page.getByText('Replica 1', { exact: true }).first()).toBeVisible({ timeout: 10_000 }) + await expect(page.getByRole('radio', { name: 'Replica 2' })).toBeVisible() + await expect(page.locator('.node-backend-logs__toolbar')).toBeVisible() + await expect(page.locator('.node-backend-logs__output')).toBeVisible() + await expect(page.getByText('ready', { exact: true })).toBeVisible() + await page.getByRole('button', { name: 'Clear' }).click() + await expect(page.getByText('ready', { exact: true })).toBeHidden() + await expect(page.locator('.node-backend-logs [style]')).toHaveCount(0) + await expect(page.getByRole('link', { name: 'Back to nodes' })).toHaveAttribute('href', '/app/nodes') + + await page.getByRole('radio', { name: 'Replica 2' }).click() + await expect(page).toHaveURL(/llama%231$/) + }) + test('upgrade affordance is self-explanatory (not "Reinstall backend" with a sync icon)', async ({ page }) => { await mockDistributedNodes(page) await openNodeDetail(page) diff --git a/core/http/react-ui/e2e/nodes-roster.spec.js b/core/http/react-ui/e2e/nodes-roster.spec.js index d3fca900c356..af9dec52567f 100644 --- a/core/http/react-ui/e2e/nodes-roster.spec.js +++ b/core/http/react-ui/e2e/nodes-roster.spec.js @@ -45,41 +45,27 @@ test.describe('Nodes fleet roster', () => { }) test.describe('Nodes join command', () => { - // The panel emits BOTH the backend and the agent join command from one - // component. Neither worker kind dials a message bus any more: each holds one - // outward tunnel to --register-to and takes every verb on it. A join command - // carrying --nats-url would tell an operator to stand up, secure and pay for a - // broker that nothing in the deployment connects to, which is the one way this - // migration can still cost money after the code stopped using it. - // - // Asserted on the RENDERED command text rather than on the component's - // variables, because the variables are what the fix deletes: a spec reading - // them would stop compiling instead of failing, and a compile error is not - // evidence about what an operator is shown. - test('emits no bus flag for either worker kind', async ({ page }) => { + test('preserves the NATS connection in both worker command forms', async ({ page }) => { await mockCluster(page, []) await page.goto('/app/nodes') await page.getByRole('radio', { name: /^Backend$/ }).click() const backendCli = page.locator('.p2p-cmd pre').first() await expect(backendCli).toContainText('local-ai worker', { timeout: 15_000 }) - await expect(backendCli).not.toContainText('--nats-url') + await expect(backendCli).toContainText('--nats-url "nats://nats:4222"') const backendDocker = page.locator('.p2p-cmd pre').nth(1) await expect(backendDocker).toContainText('LOCALAI_REGISTER_TO') - await expect(backendDocker).not.toContainText('LOCALAI_NATS_URL') + await expect(backendDocker).toContainText('LOCALAI_NATS_URL="nats://nats:4222"') - // The agent tab is the one that regressed: it was the last surface still - // emitting the flag, and it kept emitting it for two tasks after the agent - // worker stopped dialling. await page.getByRole('radio', { name: /^Agent$/ }).click() const agentCli = page.locator('.p2p-cmd pre').first() await expect(agentCli).toContainText('local-ai agent-worker', { timeout: 15_000 }) await expect(agentCli).toContainText('--register-to', ) - await expect(agentCli).not.toContainText('--nats-url') + await expect(agentCli).toContainText('--nats-url "nats://nats:4222"') const agentDocker = page.locator('.p2p-cmd pre').nth(1) await expect(agentDocker).toContainText('LOCALAI_REGISTER_TO') - await expect(agentDocker).not.toContainText('LOCALAI_NATS_URL') + await expect(agentDocker).toContainText('LOCALAI_NATS_URL="nats://nats:4222"') }) test('does not advertise flags the CLI does not have', async ({ page }) => { diff --git a/core/http/react-ui/inline-style-baseline.txt b/core/http/react-ui/inline-style-baseline.txt index 4d0e90cbcbaf..f9aaa4d565f4 100644 --- a/core/http/react-ui/inline-style-baseline.txt +++ b/core/http/react-ui/inline-style-baseline.txt @@ -1 +1 @@ -512 +498 diff --git a/core/http/react-ui/public/locales/en/admin.json b/core/http/react-ui/public/locales/en/admin.json index 7105c97ba616..ca32ff8643cd 100644 --- a/core/http/react-ui/public/locales/en/admin.json +++ b/core/http/react-ui/public/locales/en/admin.json @@ -180,7 +180,32 @@ "backendLogs": { "title": "Backend Logs", "subtitle": "View logs from running backends", - "empty": "No logs available" + "empty": "No logs available", + "backToNodes": "Back to nodes", + "backToNode": "Back to {{name}}", + "noSelectionTitle": "No node or model selected", + "noSelectionPrefix": "View backend logs from the", + "nodesPage": "Nodes page", + "replica": "Replica {{number}}", + "merged": "Merged · {{count}} replicas", + "fromNode": "Backend logs from node", + "replicaScope": "Replica scope", + "allMerged": "All merged", + "allMergedHelp": "Show an interleaved timeline of all replicas — useful for comparing replica behavior side by side", + "filterAll": "All", + "export": "Export", + "clear": "Clear", + "cleared": "Logs cleared", + "textOnly": "Text only", + "showDetails": "Show details", + "hideDetailsHelp": "Hide timestamps and stream labels for easier copying", + "showDetailsHelp": "Show timestamps and stream labels", + "live": "Live", + "reconnecting": "Reconnecting…", + "autoScroll": "Auto-scroll", + "noLines": "No log lines", + "noFilteredLines": "No {{stream}} output. Try switching to All.", + "waitingForLines": "Log output will appear here as the backend process runs." }, "traces": { "title": "Traces", @@ -188,7 +213,25 @@ }, "nodes": { "title": "Distributed Nodes", - "subtitle": "Manage backend and agent worker nodes" + "subtitle": "Manage backend and agent worker nodes", + "selectionVisibility": "{{visible}} visible, {{hidden}} outside this view", + "workerSetup": { + "title": "Add worker", + "eyebrow": "Worker setup", + "description": "Choose a worker role and hardware target, then run one of the generated commands on the machine you want to connect.", + "close": "Close worker setup", + "type": "Worker type", + "backend": "Backend", + "agent": "Agent", + "hardware": "Hardware", + "cli": "CLI", + "docker": "Docker", + "copy": "Copy command", + "copied": "Copied to clipboard", + "docs": "Distributed mode documentation", + "emptyTitle": "No workers registered yet", + "emptyDescription": "Add a worker to bring compute capacity into this fleet." + } }, "scheduling": { "title": "Scheduling", diff --git a/core/http/react-ui/src/App.css b/core/http/react-ui/src/App.css index c6ea6e71d41e..c3d4c30b37c6 100644 --- a/core/http/react-ui/src/App.css +++ b/core/http/react-ui/src/App.css @@ -9863,6 +9863,7 @@ button.collapsible-header:focus-visible { .fleet-select__chevron { color: var(--color-text-muted); font-size: .625rem; pointer-events: none; position: absolute; right: 10px; top: 12px; } .fleet-bulkbar { align-items: center; background: var(--color-primary-light); border-bottom: 1px solid var(--color-border-strong); display: flex; flex-wrap: wrap; gap: var(--spacing-xs); min-height: 42px; padding: 5px 11px; } .fleet-bulkbar strong { font-size: var(--text-xs); margin-right: var(--spacing-xs); } +.fleet-bulkbar__visibility { color: var(--color-text-secondary); font-size: var(--text-xs); } .fleet-bulkbar__clear { background: none; border: 0; color: var(--color-primary); cursor: pointer; font: inherit; font-size: var(--text-xs); } .fleet-bulkbar__count { color: var(--color-text-muted); font-size: var(--text-xs); margin-left: auto; } /* position: relative makes the wrap the containing block for the headers' @@ -10006,6 +10007,30 @@ button.collapsible-header:focus-visible { .local-model-table .fleet-table__row { cursor: default; } .local-model-table .fleet-table__node { cursor: text; overflow-wrap: anywhere; } .local-model-table .fleet-table__row.is-stopping { opacity: .55; } +.worker-setup { width: min(460px, calc(100vw - 2 * var(--spacing-md))); } +.worker-setup__types { width: 100%; } +.worker-setup__types .segmented__item { flex: 1; justify-content: center; } +.worker-setup__commands { display: grid; gap: var(--spacing-md); } +.worker-setup__commands .p2p-cmd pre { max-height: 180px; } + +.node-backend-logs__title-icon { font-size: .8em; margin-right: var(--spacing-sm); } +.node-backend-logs__scope-tag { background: var(--color-bg-tertiary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-sm); color: var(--color-text-secondary); font-size: .6875rem; font-weight: 500; margin-left: var(--spacing-sm); padding: 2px 8px; vertical-align: middle; } +.node-backend-logs__back { color: var(--color-primary); font-size: var(--text-xs); margin-left: var(--spacing-xs); } +.node-backend-logs__replicas { margin-bottom: var(--spacing-sm); } +.node-backend-logs__toolbar { align-items: center; display: flex; flex-wrap: wrap; gap: var(--spacing-sm); margin-bottom: var(--spacing-md); } +.node-backend-logs__filters { display: flex; gap: 2px; } +.node-backend-logs__connection { align-items: center; display: flex; font-size: var(--text-xs); gap: var(--spacing-xs); margin-left: auto; } +.node-backend-logs__connection-dot { background: var(--color-text-muted); border-radius: 50%; display: inline-block; height: 8px; width: 8px; } +.node-backend-logs__connection-dot.is-live { background: var(--color-success); } +.node-backend-logs__autoscroll { align-items: center; cursor: pointer; display: flex; gap: 4px; margin-left: var(--spacing-sm); } +.node-backend-logs__output { background: var(--color-bg-primary); border: 1px solid var(--color-border); border-radius: var(--radius-md); font-family: var(--font-mono); font-size: var(--text-xs); line-height: 1.5; max-height: calc(100vh - 280px); overflow: auto; } +.node-backend-logs__line { align-items: flex-start; border-bottom: 1px solid var(--color-border-subtle); display: flex; padding: 2px var(--spacing-sm); } +.node-backend-logs__line.has-details { gap: var(--spacing-sm); } +.node-backend-logs__timestamp { color: var(--color-text-muted); flex-shrink: 0; min-width: 90px; } +.node-backend-logs__stream { border-radius: var(--radius-sm); flex-shrink: 0; font-size: .625rem; font-weight: 500; line-height: 1.5; padding: 0 4px; } +.node-backend-logs__stream--stdout { background: var(--color-info-light); color: var(--color-log-info); } +.node-backend-logs__stream--stderr { background: var(--color-error-light); color: var(--color-log-stderr); } +.node-backend-logs__text { flex: 1; white-space: pre-wrap; word-break: break-all; } @container fleet-page (max-width: 760px) { .fleet-overview { grid-template-columns: repeat(2, minmax(0, 1fr)); } @@ -10043,6 +10068,8 @@ button.collapsible-header:focus-visible { .node-inspector__header { padding-left: 18px; padding-right: 18px; padding-top: max(16px, env(safe-area-inset-top)); } .node-inspector__body { padding-left: 18px; padding-right: 18px; } .node-inspector__actions { padding-bottom: max(14px, env(safe-area-inset-bottom)); padding-left: 18px; padding-right: 18px; } + .worker-setup { width: 100vw; } + .node-backend-logs__connection { margin-left: 0; width: 100%; } } @keyframes node-inspector-scrim-in { from { opacity: 0; } to { opacity: 1; } } diff --git a/core/http/react-ui/src/pages/BackendLogs.jsx b/core/http/react-ui/src/pages/BackendLogs.jsx index 60e8e94381d3..35e0a39c79ce 100644 --- a/core/http/react-ui/src/pages/BackendLogs.jsx +++ b/core/http/react-ui/src/pages/BackendLogs.jsx @@ -376,7 +376,7 @@ function DistributedBackendLogsResolver({ modelId, fromTimestamp }) {
{node.name || node.id}
- {node.id}{model.replica_index ? ` · replica ${model.replica_index}` : ''} · {model.state} + {node.id}{model.replica_index != null ? ` · replica ${model.replica_index + 1}` : ''} · {model.state}
diff --git a/core/http/react-ui/src/pages/NodeBackendLogs.jsx b/core/http/react-ui/src/pages/NodeBackendLogs.jsx index bda0313cc42e..5028a83217dc 100644 --- a/core/http/react-ui/src/pages/NodeBackendLogs.jsx +++ b/core/http/react-ui/src/pages/NodeBackendLogs.jsx @@ -1,5 +1,6 @@ import { useState, useEffect, useCallback, useRef, useMemo } from 'react' -import { useParams, useOutletContext, Link, useNavigate } from 'react-router-dom' +import { useParams, useOutletContext, Link, useNavigate, useLocation } from 'react-router-dom' +import { useTranslation } from 'react-i18next' import { nodesApi } from '../utils/api' import { formatTimestamp } from '../utils/format' import { apiUrl } from '../utils/basePath' @@ -11,15 +12,16 @@ function wsUrl(path) { return `${proto}//${window.location.host}${apiUrl(path)}` } -const STREAM_BADGE = { - stdout: { bg: 'var(--color-info-light)', color: 'var(--color-log-info)', label: 'stdout' }, - stderr: { bg: 'var(--color-error-light)', color: 'var(--color-log-stderr)', label: 'stderr' }, -} +const STREAM_BADGE = { stdout: 'stdout', stderr: 'stderr' } export default function NodeBackendLogs() { const { nodeId, modelId = '' } = useParams() const { addToast } = useOutletContext() const navigate = useNavigate() + const location = useLocation() + const { t } = useTranslation('admin') + const requestedReturnPath = location.state?.from + const returnPath = typeof requestedReturnPath === 'string' && /^\/app\/nodes(?:\/[^/?#]+)?$/.test(requestedReturnPath) ? requestedReturnPath : '/app/nodes' // The route param can be a bare model name ("qwen3-0.6b") OR a per-replica // process key ("qwen3-0.6b#0"). The worker's BackendLogStore treats them @@ -154,15 +156,21 @@ export default function NodeBackendLogs() { URL.revokeObjectURL(url) } + const handleClear = () => { + setLines([]) + pendingLinesRef.current = [] + addToast(t('backendLogs.cleared'), 'success') + } + if (!nodeId || !modelId) { return (
-

No node/model selected

+

{t('backendLogs.noSelectionTitle')}

- View backend logs from the{' '} - Nodes page. + {t('backendLogs.noSelectionPrefix')}{' '} + {t('backendLogs.nodesPage')}.

@@ -175,58 +183,32 @@ export default function NodeBackendLogs() { const showReplicaToggle = replicas.length > 1 return ( -
+
- +

I6q8DJRWuNr{Q-PTU!i1KubK`KhhiYcO)l7GeB43DA74l3@p+J}0CaLKonR698TYQEVss@rMr| zT1Zm=jUZ1V(slLX5XaWd5ahh>?df*WQ{wu?i1r*WPGFAkZsXZ+f+s5erF&10_QwU* z)_x3jzECz2OzO7-j3mKDEFCb0m{Pz90tYYzgjmVA6LJc$iY5KPal)~&!~+^(rR?)G z;0guePW{btzl#3coC~OY1;EEu{4y{(H1zrWNYw-gA=WhRR8og=VaYNk8ajjCCn04c zd+n;xATU6)7Si1_;T1!VEXDs|RsNMQ`n#P^_1S-JK$hflI)uN)u!W7I842u2_4HXCM*%`%@540S~*R}k2m$)G&R54Pzn&kuFe#3F=7-AlM@f?KFWRk=AUdv0S1)IjW+5_vu7Gaq+nV` z1Be(KQ!$`aB)UnVo`9GjSMWKb5M?a`c1RG>2YAxcz?+R0%^gLDB0vuQjcX|m@^y<} zTzJJKd`qF{<8M?M1vschlHCIN&r1=h3Y zYa!Lp{d7G3{i#y~0#XdPKd;}u^{}s*0#5`qjoSgP+SIBT(7cx93cmDy3wtESEZbbH z39`?u48)IrYU}^v?g80j+d587uWaIptgST#`~2ipacQ5OLo2Z75znr=_jZoCvL zL6qHmUxIf+0+#-oGy?$NMBYC0bm^a3k)j?@th5Xa*BA=&o_<~@c%+Zx$8wy$J-Wts zf%})|_;>3quDDWCiHAAXTxxNpY`!ss{_R8mB@~|YzqR~dcf6zh&w;;aas2rodi(?; z`kN&GPtoJi`|hg#7FWe`4y~qwUQTRX;OdIa3$!oM~5P0zMGfKUYt zN9P?8U23xABKEK$|30wyk5Rt=&5wPEAVgO9=lj{;M0u+4Z({eK1OFk=-`oKNtp39v z{}9#x(BuEh;_(M(uh@PgH7ZSln8u&wGP41MNqM-3o~*#Dajg8}r}mwC=?fsgG<8zYKWq4*%uIM`c(lKn>Ew9ibx$&0f1F$UD( zmkkBrN+y1@y-B{Sv#t}XMY1)mk3omzIAGH!p`==atLSQl7d#0WS5J{vR+9 z{&J$2Fo$H&$kx^PDh-T9VHL~aH(vorX;MXgjkT-)5ArM2PLEDCq`gtIi6;g9 zr^-PMwVd+8I3#)VPajP+FMT=})iDzMaYFf;W~Irf-9Vp~7xV-n@uO#?)eqTN-m7S0 zEU^f)6=83byxjsVzVLQk^{QUM!eXBKash3dg+a5Y9==RA0!|>zKIXp;x4h)F0f^-O$s($mQo_N_+~2b94~r`+_)j$+yeRz*s|9Z6aa1W$;SXMU&Q)%fmscta4*psLIBf zHiz~$Y9UoUkGdJt9sLJU5${I%?LYcglqM;YIflT_kHX55Ni zfx4Kjt0J5Q*8;Gyd!8NMV!h-#OFyWJinV?*TGnrEi>XeX`RS$|x<9Vwv(e5{+7-*6 z8E6uaDfrHn#ia$JK#3UIbX6;RM7c9XV|cyTaVqs{E-t3R|1-z1b?l>k`7l!IrNsek zUUK#-e+lwB{$Qs0Kt(&v8Qv%AQ~1a+y3;{VSIvi<$Gej|gsPbv!Yo-%4v~|WrS6O^ zOh^ZiLw79W0~aAgi5l{EMzQ+L!jJtS-)5(RrVZ<;vl){O>SSFX$%Wx>B82$GM5XQR zva}-y8>!ivb}OMAN-$KC;!1LBRY6VA%*)JREDr5E(=XF3!m~DzYPYkabMdqPsmE5d z!b?FFQnJC_4{|q@CVMuGTIz($=T_Az?9IJRfZcJnR+Goz?Xn01Z(90r>?PwuoS8|p zt@nD8z(_gjAStyn(@$ZTxvLvjGGp9=bdI=2=%?OZ6pf&oK=fJ!V*B;>t}XlMFSKqF zBA@fe7QuX?8b+$X-rcoG_tB*3O2M0vqnCytx-G8iOP$Q9J#WvHs5%x>U|&V@7e=(r z7e8Obk{ouHx;MFe8oO}jy<=Sp582hc(szZRBeG^J1@W>>K1@1UG6S8j)`g;#QUGM( zmEOpS@~hQfYj2ChpS50%8hyggGp%n}@7Nw5wPq$SSSu8Cb}OjTf_f-)0dhj*}Vf z-;i8um%q>{I|a2jwm0AYV7@{_!c+(?O<);qzt8#I*?Ba&lSKH=&}m_0U0!0M$GyPN zd2*nY=SJazpULccRU53Syy;+KIr3&nUh!0x?}S18=1^@8oJ(lkV}g*s$#HBCwvyYc z>~YYwV8Z}1Wznc>8Lx;@5)p^=j~2Y8nFb|JPuqmH8GM=$dl~TIF)B1pq>VS4`0d3G z9%SN2o_sTEp%;`_u=jAimbye`!J^g1HR%U75XN;|(nAsH{6RFLmp8`7d|%=EVqD~a zZ_{Gc`KX&F*vc48Pdb?jWnxY!b!i=w<(40nD`*f5<^C^-CVKR0(d`1CG^)4{*1-B!6f-3 zLV-uV^KxF0zKe=oJQ6Vj1Y{Y{}x0Mui8Fmt;I4DEG0|ADim9Omi6OWh7s( zrI~^L3d)sY>Fg`ZIwSOI-9w9lI6HlMFmZ!55_PwHa`l;YCYz_etKTNVHF?#(l5P51 zPZcu41$B7%o=|78%n;#1o&320gKH;uTO7vM)B10Ox|dm z+}b7YvM#Nkq=&hx^Ci-Y!Z^JR6H}$5-xzbN!Lgm!E}UByl09?I-<}dhLmt^)E1TvE zd0krTx4p7Pbb=I!ZDX+IJLF!EgSD8mtP2!)U{dD=-K4g0o(($En@7#>uo)3Xt#xIi zpAlmGo+BT1DwVBM3=0`*32o6?u5yz;4{F(dczNINMCskY@sCUR&(5tfd8>S_Zg(xl z7*)7CyJJ<4{5~v>NEVaeH{f)30;!|U*Q>cpms}!a2+s#Mq_ws%c^OJZTADFYci}8E zEL@T)I~`rvk_-h(9i#eSJ%77cTfNl!sf!|r1=F~X>>W}b7QQJp^LeZ1aiq!MRRzMFOeAQZ{Qx!$eWTx2Z_{| zCOC1ciHB4Lc&3_DvajYdizV3!sIH)~jF^O}D0_0C)-o|{#6PZnI(^ALBNaPaF{Tdy z$0QgnN-0V8iiHF@U~-0KiB>W2AM`u}87X1hQsBAMj^{23&CXtIYAA4-nP@&BioaWI z-Z3mlOImygMXPpCSu#pPmvK|}nFwj#@D=>jB-u#NqTX#dAw@(wN6pz|hY*thD;x%w zWF)Q_m1?3w-ZrgH1xap-dq}L38`IL-JJL6!`pYsUdnZbb)4h&p0)o%J+?4y#{&_my zHp>b+vCTxP0QE)qKyVeg#Ba7UO^&|fyOJcS+p}+MiO_Qg5@TD%JD`*_c+W}D1Y+87 zYKiYPd(a_RK7DxkvqfU-M(7W6BZZ_ogVxtArxI?NvbxZw>DJ`JvX4?}1z*oRdNK66 zWXU3KI`ia!Ro&Qie2crH6*fs2GGi z+$}t6;y>3s1@4lFGG!*VnhYN6TEC23ZpgwuR;uIRL4ij1z(e8iaOMl9vlaSZ%FXLa ztSP-V;naF@0f92UVl=p0rss{yuOqjTGVW-~2Nbzb9+Z;F)_~adk_>pnGX0S}x9s#y z0|K5j3-#&fnVQwN*7r>xGEQy}G;yu(Xe6e(sWZ!S8yXlBt^l~jO5N_D+7IK#AOpwm zQ;>5e+-iI=HE!q)@Zw40_ey;sF87faL!*TnYTA)^oyHHC6li*zeTKJowPMX(k_e$q zVqktJP(p))qLp`J#<2IX%C%@gqKe*elQ!~wAC~TZc?SEg;cGS+`@l2mBgyYG-~fmM z4!}?qW4URiz+gVf@GVi+nHaOUuI19!P>?jyidx3ubS0yiYl7T z0RZpiVk06|aWyOp&w%CK7*v$~W_op)<5cpy2Yei)gNq6dHWbp`4@Q)T3#m0iuegLN zFU@^P=P`$PqxB7^lw$udHZWjGe)U$*)ih8u0DCjiP47t(SLfrBA4L1EaN0IUIeqJ; z`@BOmBdxB5H0jKZgXI@vHA#U96~&QAg?iTgV(N9eSoYeU=I3hY*cW5dl5pxZk{}g( zhVA#J5|4o3w33+D+Qh)CPQF))KHf(Unp&InNchuOB0KY{?Hp^e7!(|8F5!FkK`|k20@A}AXXaPXY>N|s`tWMAXZ)lABPJ>j=Y$-z2UW?O+B>f8EH2h9~m6Z z^okfgfz_7BG2gqYaN{yvSH2HM_r^`0?r#w@2!3Z~d44z?M!NtzBLR=BWp$CB;n2>a zPp*4njigo3UmFl;71kkBLR$yZxfNZUoX+QaAIIkW?tzyJS9W}-b-Oe`0c&lqt6jxM zMn_aSS}IFr*&b{U^aEZU&JesgshJZrlO}4(IXy%?4Y|4}e;yNi$xik9LX(_qJfo}= zA=b6UQP*r=p7YCqPsTKg&{2nZo{@33j#QR*%i2bnl6|Z*?^OI9Z&ba3o=<(}+Z;VA z_UoebiD2#XHKtZ|RO}DN-ztj6_{E!2&Q@IKa;8BD=W~+pewt~g-K!sfoaVZ&fW8*5 z+X>qq1HJC;c^DhGvK$M5`VOQ;N=3W@);%cd30XpjNNT>0Po5SsniFhM@mUnXE&B!E zLi}=*fGLGSFC}9Ge|%uv@Hi=@wT8uAqC)hW)dTCX^8l}ygio(F8Ext;PfI#qG~&{f zFB0ig_U6UwEHOO|%ktUGhF4b`z;BKSWQjwSNw2Rqh%9OeZ|}WtoZqr3E*8wfStGch z1J8Rs?fX8#-2Ka*3wKita4fGRNn3j;=hd~n1KUbtH#Z&Qt44LJFJof;6G2qmstw%X z`Fr&XtGNMwW&_jT8$|5B45SEoE*kGBO~Dr7aQ-O4yuM6%&(7D=S4BMRwbi#==>%=P z-mcHGH09&)^DVm2`gis7z*3fr-kY7?((Xmkj#Op6#Cqm3{()q7F(WrJTd7km!p!9J zFt-sQCF8WG2-cSc+Fdqh;pyyo>|1$3tC9yRtU6OF$XP5tP1W+fx>}vt%4w{wRyXo- zZCmK?{z`db`8M7UY^Dgjcueed`>LqoC)J%X-DQvgNK-NbSk7R7lQ~}ZiJ81)NiSvw zRcnx7h!o`&7Uimn5aNQ+WM`png)VD6ecNuA+{iXT0)1bMjG5ki(HFkMKfrWKK(w2r zb;Pznk*NfDVNL^8)N6rp$Tnn`xp_i^X(}aI;dOp7uc#llbQ;fv=ZQg@gwhz zK-+KQS=M_SALa{9lfFLDv(`TiKDm_0oP1E=vXLPOq-t@YE6Q_mATurT+Dc^aM2B|DW%&=%h_eIYx~;BdFN4=A60Tf9qMlkkOqJjk zR+AOw@j%;E0}{*lC3!F@EBQGRWA+nbZa?#}j5Ix(iInyxVuV_E+Wo#_3CqQ&~BYCvtCfCcG-fC>TAz zZf~oah$_6BnXxm=c4jtnbwQ*ww~WwdoIUaQn|WSYo}y+}Kx?=T03d7H#;0)vYc+as zQs@MvWZGgf$y_7xE*H*5P%HCFox6GX@p?x~EXA5eZ+d&EmNbvOfj9j+dOo`v!LZeH z>#{R4@cBZ%;5)^9BsKfjte>T?F1;hgNq@@owW(DJzhe|~q|?^!D?Ae34`7rj+S;ik-RE01 zxSa^5iJ(50YaPIZy84uTt9ruF04H?u9W_mSjr<)Xe5f>N+U*4npOQc@~Ku0tBys*=Ri@7Q^bc8mqEZYAp0Kw|NT8 z$nd!zu|_V0A|D40H8(azXMuqE+5Lb+0}Ya5>(_j_8Yz#t>0)LJf^jm+1xpaGMKHiC zCd0xK+^spZo9|5Zs~-_|S0$&?lYR3DHP@#b=RQx+E7-nPS7k|mn+SW$s!b{Q`BLcT z6+=*?Hv~HmYA;$Zo>h1`xw`eKAJnHnX@$=+vSWjhi1v+({Q?Ibp0TYuw|N7$NU0Vr zN#(|yZ8+zEAAS72D!{lEh3bR*`q5$s1!tip-x2CPFj*gTB#M}gTs&Br!M0jZf^8I> zoYVryd9Rj*TuByfdUgX3CsB(Jv9Rn-zYl3U=O-w$qqP7KiouLGpP{Q#@hdhFQq82My`)>IbL(J5{py&0$Cib#{&4yL-CoDi3m zDkQ|yj^op4rGCM4!&dz1!F&-nU$N6s1pmlqm<}7rB-jdTX_S}&d!kQ;rCJA5aj)peh7r{tg6)J%4Z3uuY5^=atolXYY&DaOk2p!vix)VV z3Z48fS~1)BY_nvkoq>EZdZxdeWG3HcV)UJu*#p^e*<0dNjbGwVk+!O-+*vx?`FaIn z`PI+UVAh2BgNa_HeEE|QynV@qTn~o6EK#AUgI8CdGLN*lGG3tC8bK4k;D5x`N@rAz zttPm!hYfYw+kbR%mpvuQxRl7!ct)+VX^bpb9J3*CBeB)1fl)a*YhEJORkR=~Wq*(-=f z^Hcr8dj4hH!80P%NG6~w5+72onUypQz)?jUXbg@yf!UfW%SKl;>CQWH$?wxhy-4t< z>P+UHS=hUQp(zWpw{AmKTgxK-kcK66ZOBxYBnuD0pSIQpTE#h7kz`&j*IZIwX1`dV zPGzx9r+UnWK2&<{Zbls+%$em!8e7Acvo_}=i#`N?`Ih-^ueSVYyM>-kjek(EiAwZ~ zJfU@qWvjx~Czu4T<+o1kVNy~-9&MP|7>RIN@<=n?=7U%B+ZMm0A2JsU`zPFQPWw}g z=^$qjh4S1)_jjWb^DSKr90Br+*0j_)+^kkfAFbs5IZ)M@ciXx&fG#<;4`+~a)OHWPDGtMqFNr6j+V?u2XDdD%Mji=kLZ;A}GU{%+88 z-m)r55ub5Z9Cl!wUG})Ya8!?x+Ar-}uvgc~SPu%wIRGZ=X(kh$y@j~IwgwJw&SSJ- z)F8V?Ih&i(RwHt-!||2^<-qpZhQe#6}*lnvRjAwAMGs&$ZRx?pLK#-K9>@9 zo;`?_cO#aTeoP_xo*kG!?VKckI9Wx#;3M?%DyM*10^e-5qh+DocYB)!`=bxa;695A z8`G{cUdQn)M^zj}*5jYi%N$vMiopta{dC9X=)n5>KWE>9m=*?-d2;r0LJkkclao+i z_aIXnzUVK>u!fH>GbH#rc%*fSZ;8aI4q%GEE;cT;SaUf(4%<>hKIy*?&pUKH+u{Fn zwR=oac!1zp@GqL?-((+q%<>>ns%_QTNzc>+DgqnNHxN_TgofWoaDB)dP1Ci;c3$C4 zjBr{-8th{Umm|OC-8#2+D!>PbVqwBgqx*GJ9Ge1GdbT6Lk#^e*b-3lD-0Sb92Rcem zNC*_odzM)m@BM%(l~t$YyKV1xUau<@LW*tm4YlA%3FfOt&%R#uojEx9YFXCcTy=7v z>Y5NbPfQq(Q`KR~OsN(Wvd^X!+suv&Y62GVxsq%+;caFxPG4q<6kV=ufr}R1RMED) z8_)=hk9Zet-LulUZ>6KUoqB064a{Ry(1j1+H>3L_6o}`h`|mf#cBQ!XzWTQ6#3H)C ziHT=r8tvlrmGKQ@?l) z!0wx!;}Vj^ah9}Mw@r)T6B41GhGn!d82HWG9ZCSoGCi9Ne!a`n&?Vp~UWi73RPrX8 z-)GuDJQ`FK?KWn}I@wfSQBhQ1e}KBm9){YQ-)VV>3dzLYY!xVqTbv16TeX;(&~+Zn zaAXlvS));rZoR`%Gh(5m>qFitJtWGoT$o=J$keHI$z56CvR~NqMH5D3H}6ahFWcTR zpAr244BE*1IcSS%S=38NE7#Oy#fAa+guD^xa$}<#>>NJt2P!a?RBSCl@Q>mTWW7ES zid#o$M1u&yRC>*IHElPR7?e%+&vw)69Jx~LtE78R~bErqNGn z=fRw(O;he5ggr{vokM=_0KBz^(l#?ay(zMAGP{s$!BQ#Sf<uZ8c)AEAE1p+UyTM3BCeP=mN{Iz#s)iTJ_=6dRU>$zhl)x})h zPSe8dv)rzRQb4QeSj#zBWZ9FE%zLXXlVYERPA^V?U+SnJZ#E8wve#ru+ zbJ4?TaUvSsR~x9pa_wd?zB8bZA#XW255I(2I*UK4)s5x?D=G*_*dfA;`c=2SqWg6W z{0hEm^e5GkW=7_1e#&!IzTC9IgtN77PrXRP>j$|9HM9!3(ckdggQh`&KqP1~ct@MX zRV~#)EmgwvnVbHa#Wp^xPQS6%)qA~65$QO8>&N=oY~_5x$Cz|sbxNo|zDFGI+?3^R zcm-UZMlWy>Tp-=P3uQIAkxwlP$*GKDYqHnvx;z?xx;M*&bROyfM7F^>Xsk7iedJEq z$`z;Lq8f>jfPuLcK_h*yyrby5*KU#ZyQgNVWokAlv(;05ann`F6Uf4oGspmS`Q)!p z%DGqWbTZi8eWB{iy%e+?5lrlUQU7sxxsuO(Km{3d0!a%1XUGJ-klkk@747XM+8ZST zMhlShFL_|NDh54b9Z~AaqT^MxthdkCC#o>UirFH8N)~3?9*R4Zy_E4jc_(6mmPKC7 zdaq{sRky{f7P^a`?jKsn0?{b9@8u8>e$-~*CTNE>i&en?Xm!V5&-SIJrsk__eQU+X znQ{{14$(ru0$1cT?x>r%?Se|o-AxkOE;F^P_VM?5ur++dZakKx>q2J7WlitSjgdOvv2*NUo1ZYE-y2E9ey`HR->yqj-@N~^6h=9^A2u1>hGc}?xwD1@1BxoJ%cp1(iApw&tlfZtq))(Z zTI=P@Pa7VGI=s3Y#09R8#{SuqIhNqM_GvsfQdM-j!f#;z8%?OQHhJT!A-E?cb?VTk z*ng>0VNHJ_UaKv;01`CQ7RI?$j|yK--uxPQhkFMfF%vMqFmQzWfmLBO79Io*8v&rO zWcq}M+^#xrlP&p|w5E7|zq)FIHJr+&I5Q%Au55eEJ8)rxjFq<%Bd|>Fvp*U_ooEgA zD68+9>gjsz4=x5ZaR)TEdW|Kl^75CI78R8iNtn$)GNe}pYh1H-+MhDZxN2 z5A6D3$@pM4lFaThL1uKT3e z07u{iqMX#x!4!44mUe;sytu zaH%xJh5g<}Sk-j}FJ6WGap5o2cQ`gz80MzR(-BY5uCE#nGE{0wvcX#NkS!S`xzG8| zVtWTY&NcEnET+~8Q5NADk8!iRR7evHZNxH=0!hOJuTU%U^b)}-jT@&eR=d4D6`0L6MpGJtA(ihDb zc-s#}y0wCms10r#+1KB;ez{dw&S|H;!CZP~KwjQoA$zkW74?8n9xAZD)6fKVTQ?va z!Df~80DMiFy%^=YNX#nZgRe=c5)Y}FV{2xzl{-J)xqe-j`Eoq=Y%`y2eir?~H zH|+%uZE)mkmelb@zipntli2phh3c7}n?G(2hUjyHUmMYjAI`1vg$TI8j(C;U zOY80BiJ}q%fAxC@WY`%6D1K!j?3PY%Y2!UbD+(TulBTRZputPZ(gB?rThHmsF2`3t zSt-#@u=}EsCdkoN6nji`s4sZZOdsctnTSc1zImS%!?C%#rmE(X*;4ClGvFPi)|0%{ z-AZ-?Rtk$$cQ)g5YNh z`cWd%BSbS-8k0(rZYhua*cJ|khgdnmW7)X-W-6v|e6KyIL%&8o9H#zE5%1e!E8T*6 zcK|8WG;tg4%Pae1;iCF%;aW}f)AXp#i&))J8y8$~hBz##CoYZ*Ymhdu3G~RTD+@dh z>h7j~OGp)=^C@@7VI+FxpeyU4^Gh1(wDw~FV@>wahg*g;5?`)v8D+Mpt`7bQk5IWE z{sx#E;qE@aWD~ru3`)RummgKFPPUkcR9jO!A>AgO;pBy&T4XL=oYUCYv|s)_vGMCx zUPb?6b=&f5)k>DN-0-ga6n!Q8e6rxJf@}N6<}`|5Q%5~HW1{aCjb~m{0G1m78Wo>Q zB`3`vMmunmxi&$|gm;N`8kt(y2Wvo}60-9hwh1{~AdY&0sio*kG_P>$?}3+?D9ZP3 zp9^CEoULJeve9ay&+W^ScEg&I3%#|q(GAl59n41bkgdS#ws?=K`31 z`t9{KSVv*BjRy!QkP!)onP+&I_BLH=k`Ti@T41lj;3@<}bF01DJ8xcQEZp*g)dw zcyoeI6<7optEIL2SwLnie_CUEZPpZ#QYj8kWejFPsVpUbE?T-*#J;0xB@wGQk*hv^ zB}Xr~Xt_!J0EjC0=?c9pdwB&%aZ%93$gCFrBWCrMQs*y&^0f`IV-RZIQ@GBFm z{Q||By>Z_Cxm(R#@Vb^J7w7WU=nSvH+^8-g!g+v|@6mxbUUx*P4P4N|^!gz9vEwuO z0f!tq-~WW?uzUI5iWOA1{$yJ37tjGp$C7B6dukETrY`2mPp?-Pm$v5JquBXn4UyI% zg_I=~U&cfrQN`5fRF!ommcQGIbx)_%U_U^L!H6TXek28b?@7NJnjc!Yck^Pw1WdFFl`0aoDf7x%fuxU7THx zVyzExb+Q~)u^VW094E2>Wwg8rqtJ9StC1V1YeuGt&J3;Ukq&*)`YO#%#-#xqt({f zBlqTt&lNOO*Qu3(5Z#)I%M2}^v_R;^NM(2*muI9UK??KAfo zdZLI$map)py|?!V#wKXLSZPd;#qJPJL=Tsd3QY0}n%Oo{+f4f;vrl6;yX@(jnf1A0 zhxLX?Icgs-=ga<+6>PLwD@0tzCZ}Jq*i!Za3fH&9Go7PzJ8iTFVX3N@ZoI#FD+s5MQ7GG}3%np5 zk_$m^m#0wX2PK-;U0sM`FI)a1u0DYVT|2E4A%!BEjo)9rmh6xg2Tz$Ao0*I*!x+ID z#}hzPM}_D1TaY8S2bvGimoEZ!uO8QlTc7I;PdSk0?Zk3#zn zxbw!QKFIk>0>##2k+Y+J!ZR+}Ikt_KeV-alPIB#?X@sJ)=fO0A2Bx7xQgU3kMWDJz zD%s(*PS#I6@`@Z<7w9_^cB@_ekp*K+<1sOBqd$$^3JueOiH2F28G@u;+h5VEqFR>wotH8xrcj^7fB^1qh&^xbho=5#$G{kVQK6&P ze-a@$nl9{bY8-!gM|<@b>OknjH?)oSTKN@EQ%K0)rLuB%-1g@AdFlImfBwQl5PX>z z?%*25I(H0PF=WZfGQim?z5E-);_0({=M?@?^rM&m0EM8S_&?~t!-bXKpiC4`*4$=W6g26`eT8{#+D5;5ekn{v|wF zTWFg|VP)mL#uW;RAM(&07WCNm&L2Mn<0w>5(J3=2JW9H7TF3pf1#-X?iNqKZhc1Fk zd>oPy2GXV4B`lcP-SLUqBJT;5d-m&|M=$7rq?B{dZiP)hl?e$;oel$2$giO{1A@Qc zJqHgmnQ2Kx&Ls+p8r6VBSv~_t61NlsDYjAkbse2UjJUH@|3@mJGxe;gxm{d0S2sPGT8k6&lsC6@M^ z17Gin-wm9k;B3kL+1e8uA5T|`LsFZ5YPhEn&pMuH`MZQ2pL?qE7H$^wW6ajm2Vf7Qi$>3S`3A0i;;3rOd;J9JBoH-289U2%&2pUpdN z1JBRG{_XgX8s$F|9)F&%p*!)P=k`j2o`L(~U ze}D7TU;OfPqM{awq@lLM>{Kkel}fHUMdqmW7ST0qSQsklR5q0S#)$)NeH!s!cO|)p zxX6S&3R8Ua?smK?_Gw&lQUShOHlTUcr!ZZlICLvR66QFQYqxv*_72T;Az!7xs-~cz zIrsO2>|a>l{g^3ISmg&KYNT8w4=CP*MJOk6%xE3?t9(B&^JK)&hE>}eR~a{!8%Rq2 z3tPO;J^6PJ3fu#mPbQ<<$YiHs9Wr~+jTY17?ftBVh}cx+C}R;?Ac!85Rny$&5AXM!`{x zsnEMDZIK-w7uM`a+{YLG+WO9@)BbKK(?aXVv1y}x-;sPY53kJ!0*2QSONDl{B{*jNk!>c>p&pp`hym3!jdxtf+B~yC7^zljL zHqhxy^r-y!Vfv*aTG53{fsrKJd)FyLAb(NP6 z6>WZmuU<#yeP+)<$930uGfsZ{&dVtmHFz^jAFgj29vPSXiO%&cfdDiMR>;{q{LI+Y zDNlI(V7f|r86g*rcppC6B;GrWPwPci)C_SaV06QUdDIi*_i4)j1MzFSr}#G`49(MA zd%sPzb%89FAN5KHG?Vhh=3PxXar;NtNs`uTs7eO>RWJh2GsXxqM#dQ{Up^x?u; zC@P6QJbW%h8z=8IRfe8?G$xnb zjMxc$B8)B3Nkiyn-(`4TwX(b{XZ3pY&3#_h*xF|W#-<)TCJf1UqSYOCXw!xMP*0{* zjvc}>)p;N{GK)z0*pKd+daTu(ZfNI%q$^)oWHWuY9;jTgTjBVYu~#4ThyptrYCJL+ zTV`-4${M0JU~#7jW|vd)Y$gbi{9ez_whLiA?xsHI7=L-I{WJkL0R z!GJU=@Nq)(9`>I3qd!{{JwlGyz3*G8m`VLbWr}~H@@A%J`(#-STS!udluFae;j^1! z0|TV^PG-`aA2eM8wmI|x#n>;N#5PiCV~jn!T}eZj-|W*kaD{VEEl@?a>#Y&>5vo z^PCrSR8*G_sxx+<3*9_zt*m={1KJcUQ191pQ+c@g?S$p~cl7G82S8#S`L2KM?yl_&>$fW~RmQT!W=ZopJvh8@eT@?L&p5#<5 z%Vl@gYm%SS1x{P5h#tLZd=n2VgwD{+fIXJ)?aTcUB>(+=TjQ{qS&g12G9nE#%@LEU zX?aP3)nU8+F2G(iKNhKyfrWDm%db6=k~-D$-E=0_u+lGvMdVFErwQ~AMZ;DwZ;*i{ z-=NUeK^^SI+1lRXtFi%ML3{THuj-{~Sc=|3w*`&texuI%8pNC;leZYcm-_n6I8Su` z|KjScqoVA-_hC#x>23t1yQOvL?iP^l?i>}QLx%3|4(Sk(n4!Btx_jvQ-THi<=eyqb zUi`yi)^MM5_SyT|*RG1)%BwrSj5d3lgP~;D!NG0W{m6Zl<6!9m1iH;<>$H7)l&A&5 z`Bz#pTmRFl@4MKUCT7Z0TOSjJHFR|t6kEp~!gN%$9DsT`a`0#!y5B8o!3WGTI!K2h zV#_II1z5`&y-=S1MVXda!~PrQ$f)|u7)~PF>fJueVYP1%0}R#*vwI(6TxxN;K7}KN7!QB|3(u4ifm*~abvtRSJbwKrk*=L_rtq;`IClD*2 z>iZ^H`e0S;7R&s|&Ajs?JJ9*=>ZRbjK*7~O6=)nvy&?&>A9acez#zuQU*UAt@m0|G zFz^c)MXqP)@0YbN*vJr6_2I6|E=+8*q8WGH-`-5^?;H!opd{eT&g*|{L?O0m81){K z!>(}y62{eW@Zy(;OUjG6rh0C70CwMu8t1=Ts?$5S@IL^6;KIM3p59|d%+TL5A4z+f z$kj1JqR>5UzrHV&*^rXw4RzXl-4lMA%(n>n;LLi2LDYG2qG4vr>nUL^AaMH=VR8C& zeFE|bVR#kxP8g*B*hZ5t=|qI1?63kk+{;xInf5h=nxH&7Cm61Uc6JUNG@tVnS0C{z zCw-%)_$53t5$cL=mGGwa%w zuALyC=El-g=n7O<)~-&L(`qSe(LJ(iCpovPCoJ8HZ4V%Kv2{q#!L;lW+2E3GRr zZ{UMd(WEQ+v>~+e3cSJ1(C}_=F;qW{bxHq)BK0)Gov~-i00KP=xyq2;M*PK1Rt79u6EC3E81)u#8ux8K3Hc^3Q@tk zE-laB}Fs_`r1a>~^o()KcW`EuXJ`ee3a9T&c*z?paUCP*>mxS>O4=;+>Ed>df>Ot3G%EJKF$A5l4znZT;SMp0EpJSWHiCMf6fgS=hTa5B~OT#gU0@cxb#; zQ1fHGIGb5)TttMStc(Wa>VbCm>kWAiVr(53_W|(soxDC{R6i9fgutJk|A(RxMjZch zu09J}v-HlCBjVy7xZ-upMAur^3S|}!q%~WjM^i6AsP813`#rCejAJY(GaLu{e)8&5 z2dtCvX(Y6b6_uCDEW*v8(sl>yr2eBDjt?by zUwUpX?2Wxr7$i5 z^h=8Nhfjv4>hX@e_q{jpxS@Md$*_uUB7v!?#nu&BgIpC6yo|sXFjBk0a+@sWUzO%7o zHPn_EXM8K1SqLn9Cs0Gr<+^EWLOReJK#f|0lCHOd52k=uk(c5mU7|*9KcnR)AYbu8 zT-njKKONTxS>&~N<;qLh~C zhWz{!zP3;khkJ{B{gUFsJ^?q_jPwSyXfiGiZ|WwhWxvHcK8(7o+3Uac=l|brJo-{M zHp5~(nxXNDXa_d4F3L~+NDXMrQ3bc|^%EpZ{?F96tkSv~63r!C=V-PFVHifH*7N$$ zc=MY0O&2vyE%VC}!iIAe^|kf&Fx_A56e((ug*O8<@|d_R^}$B2scgX zRd7#awO!@QJW&VoxpTxa8=Jt##+tmGy0haq>k_j6jF$bgE(^5(vtO2RGRDnw?4P?U z>kAh34SjAHK~kokjXElywpQ#K!ggj#=WqyHp_x~(UEz*B? z_NMmLw<gm(_ksi7L87P>TW{|? z1Zs0~60b36Qy8D9bEHU>gZ@dI=ZdGs4(b#nUtC^l<4yaYl+;>1)gt<_un@;3>peNE z-ZQVywseTPcCf^@G*nTnVtj}N^cZIbl&<7s+jeMsRMU+|(JS$J0ZW6pJc%=Qn=H+< zTV!C?w6(4-(9xHkp_~q)frF=kgClKhuYrS$4eIkptd(S+kuN>f$Sb$#->%Iw1rDWn zGHzo7-hp4TNn<|;ic1?eNwK1u9-PY$+0!Uei5}{J%jpGtN5l~q>8WFSZfQ7r7Gf!6 zE@%0`q+y$-p4>Dyr*f0w754H8tE_~#pIv2Bb@4$Og3EWAvL?zw* z4ap=lU3~*ut`-$MKzkKiBjIAV>2(()6tsIW>$z%;eB83e)R9U1OQhqR#gb?!mx^{= z6Ug&Y-%`hH=Rklgh6X7keKMXsx$hbiR7}oA8pMIO!qDcW=d|J^yQjRhS!eq0zq6y6 zcwVVhhqhUss|_^EVL_b+j?az2{xa~y&7Qzs)6n#uZE6)58NsiiF%Ws{JL(E1cRfGh zu$^lv){&%>m0uGNAha`oHzOd9#?bWNS%(;!x1|CNeVLyN-v$ zKU#vpc4%aPxXP-!iAGmRh48@bWe}i`h*AVXF*@0*{m9iR!F~SgS67L4pVnwH7nfkt zElfPnfd^WLa(*etE^~h+NmGoDvR#W5b<~e7=fOlC;Yh9fwtyX zt_vUdhnI=b!7X@r@tZ7&04a9P#KhDLvp<%+K8lch(lg&`*s`cA8D7t6Ahhu0f6U3q zciIAi`N_S;#=)I0ARaeeZ+sqJ}jQi7ywTFWrpiG$e zi%c0p!y*m<>}I(7zD`qd?f_lnv+35(QrGhIjJ`w^8Gwpvn`>oE85On!T`GN*$h!{s z9!VOnrx?2aXpwGbkG|<&N}g!~K=k)%W@yB6A0(mOG@HW$bONENpcpTkzh9 z1C>SlL4L0x(U+rk!y1xI9y}_yl6Rqm}uvwKdz-_Zgmw z=r<=RS&z4TK3%j=D%vIh-ceu6ATZiq5a>IVoP&>14hrgH9c|6d!?r5V$Q1p zvAsUkE#cHoV{!V$7K`wRg++>|g5QAF7MbeyIN!H~3J4GD&6El{7uOS}*Ni1?)*6i> zJX5fIvnn&d;~?{Eoj+*-J~d|Nj)ZLF?LB(v6hdS)J??LYgrn|oxZ2fXkxW@sR~x=J z&-nYCnoAT3%?WJtwaVDkF{JDr)gL2e2hG-&pYMb*ILu0MQ<0Z!YfnuYDb zVW@czR z>5xy%6LsP1NpV0?9q?EyGYRa#Y3xKQ7!4Ej+*m@)=3_7UOuJnI0986l zG__{uM#!$-=g_wi&4+SMQLn%o6p1S8yM^g0AMrCzZ-Q|3+VZ1ETN=^%KC3%K3O}Ev z`pIgu*lUyJHxoIUBRLXkT8pog?sIFXf%flCM8yW(O#CyV03q|Wvg|8y@9b~R=ET-N zU0fV8xR%(ujpRJjwTKc%9Kx=y2)8Jre?d0Sl*_3C9ldX3^A|K0290Jee}B8IL5I`g z!5M$XnJ%cCnTp)W=C&ks|Do|JsKX}Puni{ZhX-T2sZS|8QQX=0y#rbv6`){g7*jOA z$g=ReU~{`^@YS5{C)1sBVQxENzw<24+HmrX8OI z2dvc{R#b*}Vo?1x0x$q72GhV##rER-7HqIFIoIPY2f?V5Y`y>mp=boj{4tS$*InHT!9~MaM5KS=n0KznAyyi*DI$( zlF!X+hdC8T$&3KhvbAzJr8W#0#vPG6oA^CdpM${pfGRV-^R%P7^GBz$h?gsVW*#6p8iRjl(dG_~t%LU|G9F{}YhPH!YNI8QmJM>L18x+IN z|1k$qI7}aq{*j1JGO#U!gasKpZ;?y4!+6>PBD>*P@5*;P<&%pVn(B*7@H7U2UM76W zcn{emYL5P*A9;E8?Kc-s#D6peRo8~Y$-z4_3?-Iz+8jv$sXa;#78C*&{Sk&B1}p518b5NxWBsscZv8a^Pf{2)apXyJ;EE8Yz_5GUXJ|9!>J zlCRAEjhtC@mMQXFwOzIJ<|3B~v=nEr4#k&sYZ^yzH?&$hSD0@#iSR4|S4><8ptN^H z9w-%M*w?zDcb4Yu!@@R}8wc4;XNO?xDMS|UZjIk_+0TGL%Bf;n>t(x0aY-?Mo!AX7 z6|&^Ex&sBxT7n*+DNSs&p@E^m@VUr`f=TtlYUfba*`bSHL3qY?~zKQ*0sX?W23I6<0z=i?~o@r{VD{~YQ_7TYsm z`oy2-y=rj@CwR%zdpy26d?21ohlDuccuA6GOcQ`(u{>wfkJM%NAgUPd`fM0%p{v)f z?Yi7^9Qfl{EnGLlhHe-;WnBL~?cczEY5Pn2GxtLgKS=A+5fG`K$qaT*mFFtZL4&I0^58 zVH{lI>u=Ifj$d4e96p_k*Uy`t$8CEBs@bQ!@hJSPcl#Bq;B6>;li2iC&`W`l%L1st zqrJ>t0qtM1XdCu_8&iYR^N~5cagJ&Pn7pSJwbVMZq)$Vn36M@p3i#v#+VM*Z0~*k> zb>fOLhe)DTexpQRr8ftuD(bknc}4Cf$d9-N5qX2XiQfeI&NowzyoQrkI2d44vq@UJ z;DwT=VwDo5+Tx>pbxW?D<@>k7UMg&60cYQ(XojQfv^~=s)-|g4C0h2|b9n-j<{~Za zD{fZ-FS7a(;**}}i#w$6+{a_)B0`<98gF^oV!_NK< zq9SCzfF0&0EK1V`XwA=EY_9=RO)EDm%MSLLk_KQ->cAWTW>xy5MOE2Xx}%$N7 z`c9iIWk2Hp5+-@D4*00e0Ryh7mGl?>_Dn(EilmZF%g|hWGvV=2qH@7YC&tL)@?xy} zABym4VMbz&sQ2K=dilQl?>G)rzHR6ORSIHKxtJa>&MBmuw#nEtB4U;~*s= zAgRNhAL{G`>8Ytv&?}j4S@pw`XVvH2b;sN0=h`-Qs!S#y&1K{0Xs^%Y0dlE#>YHa59tNn_wtrSg^F&<$t`%UrzXd`mt^CM+vaK7eKpGh^!;~KSf;rJvams+H-EK+ z^M5%8ncNJMR@!%9(h0!z&Wy)N^bRglR?niBmSd2Xg1tms-)k0q zKZ33D@+s5A0bkU)++vWb&$-0OB=IK8bA-&Z6@Ez%fxBpW-WiHvm`3y`e~Z{`I{JAR zyL>u&(Q;(-Jw>a$X!8w%^lxmf{`a1nZAc@6W8uHJTA^c!omm}du17y8Y1ySkat}y| zc9uTTaLc%C-a%!ZoYDW@=AU=}pWKRVvYAc?_f5jajs>-i>shB4j-$E(N>l*xY*eIM z_KsTdgCP8wmYSiL@Su^ik3m+<&I=@5&w7rt(>#W6r5`^i2S7KSWf5gor5PD$o>(*-mRHwsIYHX zrXM5h?KdM!zG-;^cArlM!ZjO!HBh)u4@ogX|S z=60Llbe2GM;DjWMt>3qu9#mix(aaNIGPd8?1+pMWn(Bq?w_jVN<{CZ=Qf|qVr%HbH zNg8`ex8=bE+fV?+fzFiZ@>A{EY=nA^c?D&_)zD0%ss3`j{@)=(pATREL06yc0#!`R zsioItie!poW$8OdedA+~e$yPE*cU|U{Ledlzx50E3fc1)8)@DEUI=ckqo5460tTqbA9bLGHZ1$i+I(E%NCIi3*p()L*`P z*p~-n4WDMecKhok-#+)JYyyR1`+w7tc~uQA3R!lFAYv{-)y~ z%n|-Q<;7>}snJfmGqY6YECv1H^;lO?B*I>s@WQMmbkD~P)8ljwU000vad|6w*;)QrET2Md ze^K+lq!P9-YnUs8?^rpt$dzdD80p8@B=e#R%r|$)iJ8>4Ny~AT2Q&sx?GjJ3aQHk` zx>R2WJP~YV!=P@vx;^E7%pVU-ve;OdY2=``TS_rtedTU76+tPEL=g%X@e`Es#-m9X z9h24aIrzEB33p_FIEJd;!=EW8@AvPCJUlq$*X@)eQ{)QX&!7*>{z{n|Acol=B5)IK z@!BcoMFp(55&aSNOzgjQBD0@Sfv?JP&Ax!6KgEtgx*a3UkK%^+QChK;%YK0FSEY_L z;}Db0L(Upr=K2^(BbQ{X91?}38z$&o=R;S{=1|Ja;1W{=lj6jBCqOd%c}89_1tA!i zL*7g+mov#hEDE?k-YIB%aJD;|_4TuU-l9Ao6Zn{(u98c*GV^9DXK*+ZO6z!c_boy= zUCcD?;rg6=>r7WL@HBV1-Ans=)4l&?BX1l_HXVp7_ub0&irU;Aw*TZ9ayAaVK>dTP z0Vl0R$K4l&h-*;!G%pJ$l>#u-$J!;=wrQu0D%qak zxfsF1QqX;q-eZD321Y#Ih@HoO zJ7fKx4r<$Qx-ZuLS2S@MO}jZWwK8*}r6kW=MjoDw(_THF(;bZfxpC7R7(tkL+qpDl zERn}Qd?F{<=B9+XJJ{>qAp!o33qC-{{Mn|tuUHvN)fUh?Gay_XS3o|0a&fo8_&7M3 z=#8KU$$u-L`|y1z52u@a{{G^w`;5l`-!;cQ3}Uk3E*0><$ufZ6r3f->{+L=}uEGel3v-mOG8Ql?nLunI^Sn-2+^b=72?FJNrcqJimQxophVr8&0W|5U&G&8%d9(IXT$wB)vj;F z3-Ok%;_VI?C%PK>qp zFxV&A-BJ=si^)G;xM+_Fmz!J1ku2dourGN%Er+ch61s{s?pPtR5e+==lb&1)7 z(|7Z7u1j37C(23GOmF8@KP+CKe7GsZ24Bfq=kE<_iiZr}wOwN!6cO4*chN|^e1Vim z6hg;Bu>s3N@h3LCzHV2{L(Q0;UQZLw8?ka4i*X9yH{(eD5l*oTKB2WXo76Y8gne<< z#U^g&8_J1qh0EEt$VL15<@k9-V)7sT==LYI(s4*Vd57E-2oaDLxZLbRrnvhYOqY>9 zh{j}fl;=mze03vUfj2Ox9)EepD*fb0{Q2CDzUE?7W99C&VB_es;%b&&pHgaKeYh*P zF$7??5#|v8>R7!v8UeHM=`tWSqp!lRb%XZk2V-aRgz}R94P+jEV%i>W>~DVw|s|WL?Ia*uE*2dwOpKH<8R_ zS~_o*$?-M$Ob)&45~fagY~V89MnT6$W(c*QSM6Wm)J`?)A)-6!Ue>g20o{61D6D_2 z(2SZgU{Q*$GQLDj4ku)_*r7dQDh1DQ*vye+O99{2t#EJpoYNTHd}w=q!kAFJt#9T| zGeA1I{yD`w?Yo(RP3y@rYp46_7agj-f#`*(&LL+}3~p)itLD5#9efVWfm!zHErYd>WSn<&^bm3ff;Rgo@i9UQ-j^TnR>yd7V;nQI)D)ug( zXek2}DsV43P|g*K0Xj5KgCe7%m|%gV9H+{+o8Y*dpR~5&QXf7i2-URE(F=3lSZKs2 zV}M>68@Hq5`e1Z-MBxMeDSpr&UhX}X#npNqr3j-Cr(&!@BCD$7 zb79Ob#BZm>!graW*a|*v2i4V+v=*O_sMmRYszF7$5Kz(B#DdPXIhsKt>77YTlmIt% zwp=uw0kwKOU91kZ;jyQU@N_gQM&K1+-D+_?t@t67bp9f@&i56&Olov`x*lCz(J4K1 z#@f#nfmf|}_pP%BobSaARjlIa+Laih8TnW!qLp**zN`L~#Lp~W_egdR-g9htr4;!@ zCsJ}_Yg@h=hnq#LdIY?h;WqH#RDcmJHQ?r(_Jw4G9ex8bU=a^!b;HpvDh#f^W9hX= zE2t87fk$QgCALN0|9+|XV4{69Ze32WU}qgOlp9Rzd6n2OKCTpc*D^;~n<`2h0X{vc ze9}hdLy2!13gKAyIm$oU`+bW`Y(J7$u@5qDuQPg(rhwZ=%>=A@j9gwAs!B`R8U%K3|1GjNO7aO}#G>h9h*O&gY78w=+` z=8T>0oCwTkW1&#AC+=mX%k*O?gXHcJ-W+{P0Ea2U3+aRT+SWBg;tuy2;&W<5D!LERHHKJ)trx|Mdi(;8Wv$RO4G%R|tUm3=TOrc@3$DmFPw4H1l zK8tygN^|>Z<^l!P%M3L2qZPPp$_aR}nwUnWrv!|WtQFEVzQaE8$+AckyI+k$DdlL2 zn#eQoG&RO)K22RI(g;lBd^%z`s_y;$@O5}fKSC25RG6FC^0n{t=J3!TKPl~>oNK+9 ze70-;K@!kFB`r^dCr{_6i|fX-46pl}XvL|cNZzY=MI78Td?K*LGQY6ZoA_DAVQ!zz zJ`%VW{D%EUl!fdnn%#2cnv>g_q@&!jAuF>k7w^KGZ^}`0}FhLa>%?`Y-8((+Koo4MqRzV;SY>$OM+c7 z0IH1L9y1{3ArG;yt;_6^@;mNXDL&g<_P*7^WT)BcgY-sm&0%qYFE^8(rg{rJdaAy+}kDpVb~>$y=Pym_p}rENomuHRdMmqgWnia zCi{N{fPBAjTM{CxDCXUa;+tD@sZ9D@Epz=6-qF@nrYr*@nvOkF%=jp#n6p;hnDX z#SJMx3|MVa+p7%OA81EeY=X`uZOs_7$lEO_b`8Y->n%gFqhf zZo;1o<;8Zg8arzZJAVLvYu}k_-)#(pIrp~kbu`f%NX6P_6$~TcD;A9Yf zl6-t$*xhCSG|mM+VeTEK7SXF8(}s1`)^5~?L0huvYT8|AUX84b36fF|se>ZmQ3S9Z z*jTG3s286y07tc3n>+W$8ofeWdC2R8H9uuKcckG;D-%(6=gNAXpGZ^m51asmduX{06vpJb0c=r>!)hIX92d6@ux8<1UCr#L;~BX{|m zy@clmNfX>aKXu#`XI`*LZ#MFx5=T3z-geoAEidGC9**9X@A$Xv`!#%49butd?wIdC z0^_Z`$#XrIQL5puM|beE!Dkt@+c|m0Pm`}*v_xHS@o4WZdSKu&R1xT)$#3Vjzp_8# z%9eH)VE0!25&3g9n|wC4og63_1YxoAaqhmn_&vSwwtCTD=-udQ;O!abSa9! zY*pN^*W-F^9~g!lIEU>!?`Fdaf`|wGEMM81jreLYZcP;D@ow*Bu2Xi+8(U_@QVPp2 zrF%y_n#=}lF0CiO9h8zAZ>}(VCHX>NIxe$MT5il>Y?OklMbC^3%!Nafi;D#`)|}C? zoNQu38(mVoZPyzzXkXofy$Qfz5GfP(o9Gg*gY)mW2Q6w+;67v#D z8%$=ZJbHp#`=`#acecVHG&Wj|ZeX)B)-ay?-}&EWsZ*vEQEjK3KQL2>#W>oHN8-!{ zV5qu(;2vbnOsn+i8Af`fk2fn5{iY^viDc>1h%;->Tl#RQrr_L68@oLbAGL~&=l7#2 z3BOpLPA1RN`KfFS>DhHMvm{3fVIZTT$i-szT}#DOV)!L@VKfE^#}aHO1X0jr2=l)K z3ZH<-h5BQ~Z-k}x>znaU_#xFB$qqbWN~W9Tr8gTo-p+E2vQv@D{KPZ#&X82W z{(4fp$gSt7B&1{{m}FdB3k4y=O!Q%5SXA?>VNz$x8=?93NrYLPG?H9<7j@FqmVwyr z$akO@u9}Hh1-ZfSim+uRkzShSK(BE9Eb6D&R^ z;Wq0!(~k^@{biulL@lderEm}R7Ny)rrXRdn&3T;w1N2{~5x2NaQUFHT>2Bj3TwPBV zB;~0LpYC;EETI(hJuum9?F)YYl45Smdh0EB7IoWnNd*ew^nASZg!E{E@QtR~F;l^L z7jORCb{bI$FOSyir9_I001OU0m-pM7;5O+(O0hINR>qy9(bOX&Q-w`(TI{38eZ7~W z7tJL-<-^%7I+WsNEh=A24^jnmpuZyL5cDze@DQt%9Y-b!RSK{;t#~9!8~dgD!E`m< ze@9g#&;gm6zvi}?z%#~|(fD!BmEPz6o&WVUcjGOOou%|mad`k~#{ThaRVnnvS}l$^ zk}@^90G6Ui9##yXXf3SGCA|t8o`T&sO`D1m6K{4(!m*Dk7TnaQ_RFoMjop~EnG5u4 zs%u8qdt2I_#$QPHM`mOFy&93{*eM@3A=yZy-&cQ8Nvc>r(?>C@aSHt_p9pC$d#O58 z9Q8%6GI7oO=vO-^8I(gCPFT#rB0)_lw0Mei>(TH)cS9Lvh#2=^gnX>0WuUW@_x0`B zZicu8oy-SKoP=H;M{v*j41355X@f>iCK5Yoc{MTH)Nm{5ria)kcnhTac&upnO zyV&P;W%b9~pXX@Y(QYO)yofOil6a?$2}-;m8#fQqmyMJK%yvA@hSFN9O-LQrfkIJv zkvZ5Pqlx;F;Nr^{5+?&`B&a+DSI*(E=<;25(6lb2M5M)`ir0_Q7d14~CQ&o`LqcUW z>u~`XW{=P;?j#0*+xN6iMd3gX>@fzJ1P>5>R&yVE`A0?ro((P&K3>0pY zk|C~JqwyuqdFLWA!qKR(63w`}t&Ak^Z2=SbX~@XnVqRH>Y6i5j)1Ueew~z1-BZ1vZ zE%u&;z{D=LN+5pViq_;twStvI9={9dVx$9~blCd-W;nh=s6a-ag)W>L=QcojEjx8^QmraHTvNa@6PxOKY6fHq>4#eOS-uUZA!NZ zhWw(;Hz1%6wai#QT_hnAx=?@nSnKD_IHt*ME`7HZWBV|#Dar4;|7ou7c^1uO-{t|c zyul8g@#D>*+*MPrUh3fB@tD;qXR&0T`%ma+FVwC7cJouZ>Ra)bkO-8#;+x7-Oug%r{Bal#>wKLs zwx{p-e2B?3I`*QIeC1wdV00QC zx8C2!Z3JLMFfr<$I;1BQiX~uGTh@7_5FUA!Vtp4z%D2%N6{RC{QtOKn+dCAlyId znOHaPrp^5rAI9uiTLriMP(Fo7}&1XQ+do>Rv1BBr1Eq+y2 zQ5F_oF!i+b%5XY4ef^u;18VfTH5Gauc+G69@l}hzHzO`VPGGP?S4XdyFTI$QH z;uU4Zq4Y}g4eu1Jb55;m`W+Q{g!-b1a1Ie!8yj*}-lOQXLR0dHKl2A4F3&uPQF`{c zP4}k6F>N1m?M1thZ$+Z;2<^7SgicJ9{CgWh;rEED+x(TUlP;GuWkI`}Ka!8HCUC=I z+Vxv66qBhh#3a#ZM97!E#C&qRr?Fe$_>1NYSar_}9;zX3_nNkIHZ~9jasGQXtCru> zQ!VwG!n}Cgu4u}+B>Ig`K@VRcNmY^@=7Gs_iOKO)UZ<5}MZVC4G?M453g z7Ue2sA@ewuDA1d1aAgo1@gwn(dQ0|1k%){Y$kPGDq*QELdb+B@xRjgmT2s!0fyik6 z;YG&Id;CxaeA$$bY?w{0FRh0$LBVy!JYE*vQ-fmNpQiF+1trB>aV%xeT7OmYe!M$0 z+&oTntNXFsr%oM_L8Cm^l$e0qI~2;LV^mS>*ialgrm^F`IJc6Lt!_pQeZ@e0Ioj2x zX|Hebn7I=@6e;mQXc!c=&@|9nl^-)Uo)YC3BZmp9<%4){d}!Ub(Dbf-l_+4__57A* z2`$j~bIN2=HG`hL_QnUHgL56F#_1ThdtDhM3Za&#**;!Bj)j0Y?PrCTf8)6bL? zGg;1f6*23rm}T_=8F!uW@{ZM%px}9Uo&j@-{6!@RRX(eB+|AOh`T9%R?_1{m++BOJ zMxLl%emt(i6x6~sZS1sG55+q#V|1zndxw!9=})R3g1o3HqajH0Xpn5!9<9mhbyNnI zA3Iz362z$&47NBrW40E&It`siHLUseJT;ozwn`>)8XP>nME7uN8;? z79C`U8J+|@JcZ=C%ZlStZ}i_|i$xsq6CQSudGX5o*aMI<4jy=^iKUz@b{MOi3#LhD z1C3nehwPB-n$-`#fr^(nrM7a@{0sE2<47PU=?!ub`G%tKJ~PwqRXrqk=274c{U}zX?#~j| zmt1ke>)~1#1HqAm&tp1F=67ivR5mwDik8kfrHW3&4yVP^5?PvpA!*FLI=r;;{PkM$ zUejoWwOlMgG@mCGx++v%k3J4O>ak2>b)pg0uEb?(C{TA-k_*oEa`G`JU}O&oc$8Zl z+kDtFpEh=QdV5A-DXybuPaAeKY}b-$*tl=xGpG}eLi_)7W4}52oBREUKG{ho)ubL2 z4upk?)CoMy*yU@^4161I6C5h=vNU=5%vGu?Ow>OhqK?Mr|dewgWAE>G0TI56^*K)DRJm~S4RAdWu0zAF&1H#63s zIV(wM!71XE#GzJ@9xkvv5VRV4Gq+CI?>(}xa=R8qD9D&#sux#C)8N_}9?fv?#V_B< zW&u0v(%YWn{1&ZDF|V>p3>IBCH<9M1$j=v*U2HYs2L5fu=-w1-x>9>#fb9DQ;HQ!G zZ|W=1SH!tt8e-sR%bZKT=U2zlX|$J^GGXNWQ+2^3hHd#+N6 zOC6xr@TdJ0Zx;MN4v(-(Lpc zYD_fL-mdkA=K9~NzERm@#)uq}Em#x%kxF^u1?ahXJF7BNsY;9RBNT;9{GbI@ERfUM zIE3bP;KV>gu3RWIw7q~PfNj56$Mf{U=!j%jV)Y29?O2VXq5g&zGJ>o9C*pk2&girC zo|X}WG#NpvPAD&<$hL#s0Q>a@$rt1`to{pg`^82g9?owO9 zzsQQO@uwKrVEx!J6@Oy;p8tSLTZIl)UipB%H7e7m;B$X_L|IaaYz@$0DUjA_x>6V*cYi1!EiXt@j3DdJR#Kq!nWO#x^b z1pGd$doKQTLqnnMF6#xPA%H?D{x~|oxXQhflZX9`AFgd)!sbZ^GBA8xQC97d7>g@_ zO@WORfcbA*oWJ@{h zHnGR6De1+NAB4?sMB2SZpWFL9C;Z14Bi6Akq>evdQ@Ui9EZ>RlXb?;rU^_2aO0;QKsZo zt@UVxL@O-pP)L3lt1A^p(!3w>R`Is@qvgHoD>sYxV)DsBglzn8xM*04AQ+ZYO+R2O zS%Q%oo=4?x<+ERt&7zSVOpdpNkObZbBkLp9{|T5wYNM71PLLfhOz6fTCf~Tn4YbcQ zNH8*G1!hlr#7`m*;w+K>kdscXbC0teA<1K7?u$!Z=349rrAS4T0XXR*&HHFUdf-xA zLoL20z9<4x{_=WT&ZgCf#NW7S@zk^jY{+=y(5)oGBx&!r0O6elQP9(J+kGl8@a3rV zL7v&-Pa3)2|D0TMk>Aj`48^1bw-uHB$$;2~U+}w4iK^P!-Zou{L|siW?yZ(Tm2WC!1Sk z;^f&>O9GT@fLsPvhrL37PzYxARhMgxs9Y;6QJh+F_UzD^1Lwo1;G=`1p_bHguxYJI zNW2RO2Eshpw}WNxPh{i4MT>9-ugrd$pSUu*1+wbb70FP$3bUn57rMFO!cq^WI{d+nf8RWz8uZ%g@ z3bRdyvSiAD?_^)Kl{E?;2V-~-n_7d0LkJGmVq)XfrICslVqKwLD8u=bT5fh9hFpf}2tJ~vVEA*P zj8lW&Czu*m@2q`@)er^i1F2QFmn5+P@K^Ix;oa2B7N4332#e`HjibK7m=Yn>4f zKkp^_CxLGy&EdtvlNRuqsXW)3xA(E-VPN86$05ns#Na#0cRy1A*uKHxHJa*+ED&Q6 znb0f#So!8AC?DpYUTM;!#5t&Wu+D3Na0K)8mo1QMjDAaeaHt{B!9gC{Z~sN*FB>??)|q#Qv1(jf&Re6l3V>|D*0LqvGhAwo!5? z1QLRW5d0K%VD&-yi4SIX_OX zUb9%!O!w|BRkf?Gx=QYp3{r#a@34Ooun^SNrK6jIO()ms#txuw^gpzk(P|zcH2=)2<@L(2H?aGUz z{2$4qkzd9vqKz=#g`$z*%&!0A`(iQR8L31Vrc zsD$mdMD6I|`?4K>3hgl%WsvP!AK|6aF0VZu!hB6f>5b*Rj-@Fe5@F-mc72e`yu*jA z?!UG_FlM;bt?p9xgcRN|c#|xc5M@hFwIOf{y4@;Psb-wd=RE#5>K8x(R_tMIY9B-g z&_9}piw@+HD{!Cn8?s$dvhWHvkZ>73kQ5>rw&MCdf4yapv&CargW@*4NMqZ0>k+RU zDe09Z{O?Cc^?@^@W<&8_D3R6gU8uqfgiA>=u};`diM?|xs#To?6|QCveZsaXM@g;< zC$(p}Mf*uvbRDAxo~#+7btYIEQaQV{{fJZ7@-`dSLkJNhk5Fs34L!P6CurCq{5TAcRF_DRv5{ z-R?(AN}y6J#zy1nilVl;wD=*Tg`6905OTsx#Jo)xel)>3^JBQu%rO#;V^1&d3a%Hqrm)-md-{+2!hDtKBTnU8H z{0rO<;JORj);D1>_E@;O7sZ|Cc(=^{(1gyL*x3yTIKqp zwtG2WR%_yS>7@mni?TFuflVyJ)1HY(!R1B)EK7ClL2o8 zU{pQI0M42e@q%gnw?O1rOpCgkHmYK92l2|hMWcbcUac4hoBpMUPHz!AGq7R zu9K2p@n+>`Y~THoN9;$pwy3O}K!k3o-a-0|$5z7?i_um==$7yY;&)#Ag+#BA`HP&R zzC>%6Hl+Wt%2Qq#U0k7lNCc4+_4#;0JhZHC18HC6sZ{c^#ePKw5a`XD^8RHQwos>k zChn~upYa|fJj*#U8mod!!PmGEiXL~y|E?D;wbpubex*ggb8Rqm&@#(Hoqz-=_NTK5 zqiQ_s8(XJ>%rxxWysm9|bb>Yq4?+WRMT6?UcWgpud|v;KHrSvQ|5jF89&Jk@rPdq9 zkS8(16i-|ylv4g)Gl#6E;@Gb=?&v2xqAfcI%1b-{_jOLFHY4=IVm@2cENQK;0+ zclB5R$MW|GZn_-1yy}G}VqB=)+_TXBRK-467T;7H=hibKe0#*GQUO42# zo#+vK zw?F+w&#HaLA-Oy)>xyeE)47P4t`q&fVPh28A@yN;la$Rhi($YKS(wWhuXb{H{UAJy zH9m7m2iCh$Xy~(-TY@SpHVWPOhv$I*&1`O1uYk#o6jFyRnrf--m1NyTfEXCz(pcJJ zR#nZQxhNfrPbEvRliDjI{%XYB2(59C%z|ZS>j|yo(0itate|l_DtUG0Pl*;Aa(c)t zBmz(TOc-6R&v=lT`~Y|=yl3x(Et3zV4j0aYx~+-9O;vUIZD#?~(<76$ps-- zTU*mV!~K(7*$QMg4f>3^)@=z=wjvTmB1z*+j%rnMBEr{cBAqP4BS!a!BhSN@7;DX6 zkzbq8s-8sn%tu(J9ymU_A|Dzeay*IZds+rQsnL6uS%&M&YNT6-a#J5+?|s!Tgddj8 zPev?VA?N*Oue;yu3<(&3?l!l<@WFCfpTaN_C=;D5gkwwOXt2q(<@TiJg^exP{K3GzVSKGLva+RW) z4uZJnm=~BNFF6gIYmRh^_wHm3AVFP9VU!3LM_aRS!}An`A`Iq8?X$Cv&)3J`K1nO| z>pk{Xfbf-zZeWmZc`Y$p+Sc;n7XBYA+j0uSCFQ0duk1>SBYzJscNGcx1zfLGk4#}b zdorFYXiLy~<41jZi?RHhm0cT_$10cJ zd`m>PE`B>p$}sd$>R5*bHlb1h<3=t8#GT!rV@MW9;iGdIs3+@R319Vr>#j&<8@HX~Bi z@^?;~S$U}Z_+f7#ru}EIbfc+(V;$@qH|J7z@u5kx5=ofl8aCpLMG&1?adrA_XcL)! z<|Rw!U$ztz+38%`rs`M+#q45 z=qKkBu`JEqZjBA6haK|U?w3YH@HxnDm__^UJS!6o>o6P}lwHRoCO1CO%fZAwaD4*E z;VQ@qL#k>8&Uc!Fr6s;4TV_@9OT8_ADQS1FoID%1;vbo)CPUf;QCn?BICUEs8#_ku z8T*`bJgebow$wlf3v>6OKnQ#t;nQ1}D915X;#l2J!yBghX)1oA75uC z;KHT}g)`o>UUh%>ItQ$;@xjwHX5Oq7dsPhv1S&&hir%zB63K1HsU9hL1l%&Qb$$Bf zLbmp`pa0Rc^0R?)rE3*)c6JnjAY;dkcZW-{HO=QS)yp%9y6#kbWDcwH-Vzy-K+yv6 zNi&HYNmMlJEg%!(4gbMz!rP^T8gmX3LVowJVnip__iN=pACwP9hj493BFpvAwAZMK zkxOJYC_uUyYoQ;!@7GUyu#})=q<6w|Nh}bB7u9a5_NqQ`ZU2L|JdSJ=_|ceG@BWCO z9F8ZSe3lc_dtq+;&ebc?%gFVdEf-U66oj=5PLyS-W|AkbCRa~|qaTampRC+ezN+&f z$;5Se&whSL7YyntZ*-E%huB%}WMTym`G|$za=M(NyoW{(ty4@j8~TKxreSITF#`pL6%{PK7>ji-}YaDL< zQJ9R|5jFnd1RE3OZ)#U$Jj_qjQz8?@K-dVEU@eS8AzJgliCOo}{KS4w znEGj|<3)U#uH787ilrXDS3TfvlJ6)1u#IWYpuZ9;^od!_;azBLDX1$aBrl$3WJedA zgF~o^Z^rxo6`A8U#hId=P7|#bjP+!#$dc__rruMMIkI%255YZeS2oLeIxOYcc|rso zwGTHuZ9z@oH3%Rg_xX4MgDr1up_{F+f8e$s3@R=Zsn!jFm6epJzaCB4bb4@RK6K`{ z!@mKFP!g`KMv}cWMpjTX(~r{JDJ!_gK>Igo0%;qvrtQL?>t{@!MXd~0&jLVZ^^9ubCx zi(}qMJdxY>s?$=iX{I3H-6b2E^gv2Ufzar-G00Tld>wlAZ6mwr6A^^)g!?E~&ol%XcC-C=ja z9==@gc0JzzO=6QO?(I5pw;fQc-Y!nFl!zV;k>&6_Ufp2%XE%;PpBZ%|($I#uZoO^E zJKTKs3v?4e(Uq;C8MhkIRUb_?FET@~10jxNwLew#c{Gi-_#IcrDsm;!9nGP002Ob# z%hB@ED5X~tpm*#YfqdD1&Hqe2o_DW0RPmpY)2XA_DGU69d}DG|w_M*QlKb;%{M z68+;`|CjkYC=_wAhVTrZoU5^b&cat(3=717tj;gMITT?=Xl`(<(XgQ5^A9FF1AeY_ zN_}QVw~Rz@f8Su7Svltl-iFdd|-pC&gI!{BZ)c;PHVdc&ea&88^>y0t#L-vn7{QQFW&aA zk~GN>i;PRrd>@PV1wgY@)tAU1j+PA@IuvXimFk4(V|rtNG=#|QUMS`{glv6mqp63? zfNH3{q1iBMv2)i{S4ZdcCv{Y3F{r-Idv~Yqp5JMVEOmstOTkglrU>07vOrF zaJV>$DL$;&zn=p}CjX)J{-591(fo%@EFCvu3Oos9Pz-0s@aM7pm&^I+=YQ!foygs3){1G!{VDE9YCe{)T zkMmP+zd~kcEI(-jv{Echzb483zab&Ti}rqhTX(9Ma#Ln=qvd>ip5Z0#Q28H*_ivWS zcXs6VtmV99QBl#ydC#NnK!OVVWFCjbdE~#M@*6<_t|ftGR&O7J)uOBQ%v^S@t{whg zD|O*MgU$+cT>`v6lDN=kh2hFtLJ-F%8d}xb-LdSxB?g+v#CplU&2i3~mUh>x53Hi#I61Jeezh#N zdF*bM5c_IT&PE5=Vl-62uB#4W@Cfau{w`3>_{u}CkEXS?v?j6V1r>!M)s5N>3q<$a zV*@~m*#pgQCCHR5!11nT``8 z8zP?z6uC6E84h|BF$~RyP&ly0)QVPlx>TuL{NT2R-`IH^^lRcG#VaWaJDfs9VQ`Pj zyyw36@`f?#Kb&@(gz3e&MknlPS^NBsW<&gnYQ<%S;v*=BDa{|vJ8-6!zc~_HIh;MN z2munM(=`>8`M1@)GVO~^?-}npoXJQIxEBZA&+d20_P8szKQg^;9O3gjv9Tgv^|I0D zHx~6|bB{O>?=p#k(=KWF4~;H51T z-Ah6&iV>&lZhK%$xUXDnQoa7eziqpk8g^^XaK+b08k)GJ)P2-YCa{_V2;3g}viMIq z1*FJXZ#KH8kbm~4%an2Z?qOr{mX`ZX?NtQ`)bO)Mx2nvR)@-t^Sp!Hpq_M*t?7~mn zS^w}gqHe3cm_6DDYwjBXgbrgZsXpB?J*EqGy0GL#`>gB`db(Hg!uDUE=q1jR4H8<}@oidb>j3{Bi^Q96esNcNofG+ce5~8>nxXRst8*@)g!B*zShn@gUL1wO*K!R?8nxeOoGrI_ zryy?3u9JOjo>lj2S`{k3oR66ra5Ol;VS~OIYG7 z0Xv<_^l%h^EP(n_#}EwQS5c?trhMr=6zTgK9ag$Ghni<>897T@=LH{z!};_HP~c-(^y7SMl&JI6*ECy)PEmOAjPJ&Ur0wk0%Ui zhSVA!x~hZASJ3tbBX98J6Q*?OB0Ic}=H93l~?RE}P&wYSR%Ak&_m^NliYp9fZB_97iBo zgx>8~d}@Ay$f+w97&fc+U`YQ_QGbKWc6()3eTS*pN-`ewmxdVvX_!X1CvdCeUGVc@ zgP<=0MHaQhslmCBKP_u*ylx+Z>=j>WOy72VmIb(_)i@xH$PpSYyB*M6Tp?s%TVaYa zhnLRI&gW!IG`-c7o(i$Qofg8@{aW5S%zl@NCHvRMSyHRqy(sI$GWL0`23uCJ%KW;2o}!wlvPi{#1qSv`c*?@4 z_$6u((Zo(EKVtk#jeJPH8+u3y;v-^)?oi7L5G6{}#?-p>Wa6UG2k&hy%f*Q?OT0_y zNk|x?kH zLqih~W6h`ZQBY<#)%#3mKliMx#-8H57C*Ku{sngxN!MW5KvhLW#hUg0w4P}jgB9r@C6cxX zCMnPG1{ihZ)=T#@ooTaqmZh1AiR!9X&j7z~b~Z)$)^__AC)CBADiwEW1Xxz>TinMa zSJFIGd}$U&bFjs`^_`wnR;43MueHu%J6(GQqHPyZCW_u#Cv{z0AXxGmz^nPc^U(f| zQ0SXzKqIfyL84Qmt1GP3!f3)9r8AL)+a^aIrPLgGEmd>4S%B1QU2K6^fmF~?65UA~ zizwA>tsz<<(rDUye-Qo{eN=f8qiKcQ!wsz_=A@syr#g~?wVuW<{2=uB7dk&RjXjDh z%5z_hNGB?$3QKVz-|bkHnQD|=e@LFP7CLQ>4eWxbgLTV>jK}JWHl~^G-fH{5KE@Pf zC__`ZB}K@$R3hkT*sjww!q1p)!YTI*I|^QcxL*>T_li;7s#emNjVfz6f%vVc)P3Lc z)zX_PCB(gtM8~UPMkN zrZ;Vh#LMW{+n1mFd>8Pvz}#@@VRAU8wOnpcHkzPk?S4kc=?%~T5Wn1cqmtP!IrEiD zxy=FH!bcvI%Bs~plEQw{Jt$>l7D*gqfMY^oV}K>Fp75;JA+CLvdpE{+4Q5KU9Z9** z%8RSQ>o=CiJ{j@`k6=W-S|bKmK%=wa^6;h1Q*t%c)$la3?xJ`1qWvm@2N0IS{}h}A zRXL4{;3WplH^6|?r#;JWj)ZvMcahi4wZ_HG#nrXW?d(*D=FXZ;uMfIvCd)gqy%Q7# z&zKThG8J}XPal*WP(liZA^>!`k2zu~ZOOe}Dh|wi95HEmPBvBSCgZX(^9X%Z0R4@zXG(DdULGW>ZHtYgYP)&4p%fhHB)C&d$b%_&70~O+~~!(s*>dtfb}2BkxR- zgiOwgFN!qf!sWC0^NTtj8boka-2t>0ryMk+$iHdOgphYz0H2!K61BX@L>A!cVB~VSGk9CgCeO8XB z`uc7xG=6KxG?*WZq}CY-(LTfpiR_cW-N<(BBYCco;{D^FA2*l&PmrIUJu9u=`nwx3 zE*zcbaS&f?(|!4+eW0b?j}@ zSCOcqwfQ#qjZhRb%P8TCl^-8DjD{yM5&Mv0tXmG$eGd|-P25H)EpqwMImV-UvCQVa zP=plw`>X1dbj0NrN+9Jd4WP+K1tE}EVTEY9vO1G&Rg(@>C0As{nKkL2bqrJ)dTTQc zuQ5CtNTh=|1{#xhS6^sO_s)uUk2xq(_t8#L^wb=t^2}|XKSAOel;)Asl#+z^&F70f z4|<0zY&6udtyNe5?1wkZ+xzj2zpvjJI@?3puwg=TDk@sail9tMc02ULF{4?%P5Qsg zk6dr>{2|K`r8{%Q&9|f$BjaAnPL8W3x9ceYXmUW)ikn%CT2U!)4_~s>aCl%^f1p@~ z+GGMx?E`rzuF4BDGu@hhwfhv53X-D63QW{4->w6RTrZZ=!=}s)`V()V9NL^3pcy6{R+pXX?;8) z7^n~uE7jcO@uu|#BLI~MYh-9X`%Yzjs#sqP2?ypkP}RnFFp;!&9oY{5?;bAY#SE-^>$%dFw~Sc^HahBKQ8izYy>tVEUc zT56+kTM}Z1a{b;cn?1mlq+lGqL6cpek*$#5J3*u9axk^AvHM&&K|oQfB0DSVcbeS_ z_cR~zrmWs+G)H{urWX)XY5w`>_6CD++UqDCzS;EWZYqBIC7|Vc=AQ;hZ8WY+_!WMU z=d`qXbI$D4cy|?eJ(qi3yr3$w*{DcXBs0`t#HGlpv)`=uoZ10#xxZbDPm$@;`Ie($t5VLDZHbN{1O~wFsiFsrl3R@BdfqoC5MZU zkcKf_#gTY1s`kTg1>T^Xg%iKV9`70&9UYy=$4t!m`d%i5)9S7OTKbuWriLFniT)u? zy1WFpCzs=#qm=gzNN|!)39+^1$HxiKAotFnJ$NBL<=*?l>}8_;u07`l5x1m3dkiuY zf`i9I3**8A2+a>&$=*Ee@-`A|uN#*&I9ymg`HT@7DWhhf<#_wp7#u9vMOSDv;IyKG zTT4%%sPtae*w+y~O6$QmpG58`;x%e2(~cG`gjKV<%I9I{2ulePA@uQWNP4HcZs=c`P`}pZv zQMxxc>S<{;1+oEN#vz`w_R*p<--`dwmx8gTL9_EWet0 z3e0&@{P{vaboC(R=Cz2+q<$}rka8G^-n^J!{pcXfYvb)H>J3{_`_YnyFx2ydx5jjFdcEhQz% zgP(!oOxh$r^MesJt$Zq{+fM(Cgbx;`Q_FTlTW45MP{6j);mA~Xcf$n?&FZ}(4N!8& zw*z1rp>w;%h6&%*@mu&QlF{FUBS9?@BV0b`AEuZfNi#h?hv|7Hyj1pfK+^#&T3JEN zARFk(St*l*a^Zu%-ssDx+tR&#>TIBT1xl+6@syU3_vnjKfKQD6zLwOs(w8smLx*mF z{F&u_H`jL zU>&&5=cT|Fy}10}lA6teYpxM#yqpiTgR}~hJgoUe-L!>XKXPI>7MtW%?R<+kNd>6% ziIKRHQd19kc^CXK2vn*mKyvYsu2j@l(qDE{#^-jvD7qAK58+BULT0MnhWJoWZe-j1 zLkLJ%^}jyp^D_Xq6!cC2ZcDkirZ>P2Mj3HD1+Um*@PNUe_y2o0Pc;*Hs4gxGSbKjM zy2Jgj5P^Dc;~M^~+9r;@2Cf3fr+b@mn-7D8^LPSrtD>f6SRl*l;?UC?_lj4~zyR=N z4fx}8*og#eSFb0gD+0YLBDsm~Z8(`4Whne?t_n#96aPDHPybkBUU%cs)*;&8J@K~| ztQU!VG>86Utk^|C^L=iJ2J>m(O6}?PPo8_ZE9H*pWbB82+tvrawM>swRBULk^P@M! z?GqT)@nupE%kzR++UqPyd=80c3HuI-mtaPe7X1e$U8#7Hn_?g_Ym>5kw|pD{@l_yME-;8sPQzjcVGzPz9S zWnyCDL2S(47X#hKu(j(TD_L-UOJCL#n+A3<>KyOt*DRS)g|_83C@3ATwIsS^Y$tM{ zyE#t2*HO=D2J-g&y1v=y1lHs17!6kAihOWZg0ESzqlFF#8oVXGm@hK1plefN+})wh zOv88=^LoD=TRMe^a@OwO{GEga+UMmCQJUEJki;JF@8{yF^c|ges4f&&x6%xlX3*?R#qp_@JrOx)cnrSNSWNbyLG(X zE?)5^m%Z-|0fU_ZpEnxQ)S{QuO#AUKGHIri(CRqt6Q7Lxca{Xi$N~HYsEYe-aQo&~M0VHfZKr24PQ1*4XU6dD=CgiTr!|R^xb`06Mog#P4NwFY%4j zJMQLZOCrPUbfCBQZO0fQ-tGM`VOr7LZ%EqP|JAoAV9jv#Dwy~4&`zPGT_8sHIY*i> z{8aG+(MtCf+0A0DJLRUkuH_nH9YB!)DT%$y|7y9xQvO&6#5bl#+Gry}H8iLDCOfLb zMQ`NMxqKu0b%_=Jj+K)tR*k+Of2d`w?X$9%YHHpm<>KJsN(*`4N1xdLlXhz zv#IXX6TEX8V@HO-rd_fY(tg-DBF!Q_L@}NR2Z7w|DwPUZM?~1&Se8a?ChDyKed)N4 zbv26;@!QSZW6P^1hm$PfHx!i^p|O0qm1xIhHWh)!7W%Pjs?+K`DanXv#$@j$=S4?% ztw_D$k&kSJuWW_*m?TE%3YYdw+~bh64kuXVRM*%(wxoKkq1ssPx%){Ys|?#>C)r1@ zv2@BXGif5E?N}b(`K`eOCa1v{L8F~il$y1!x1$?GPLM#gS(#Lo@~slDbJ$sUO0k8$ zJ|aRkqRk)O#?caissPby{jgMYW51B|X5k-Rd$3k`5*xIZUCZlghjWbVaO2J`ZCr+c z2gJV1jyL}EyMpMb)$a2cB{tpk->9U;J)XkeqKq9^3P25#1A5)(r$NaF!1&z+184wJ zPDx5VK$v)=oPpq-w8*}~aU~kzwYF=n<_!Y*rn3A%*rXO3H zdXI$Fd3uz>(LJ@c@2=&5LIfP(ci!9C2G=^GIYmuP3)ve2B^k`>9eJvRdcb6N9&TM)9Pe}ue2M%QffCMyxZ{GR| zi8R!ooXt6n4r$fc9%@LsZZ^g`$^nL*C*t>2!bMFcrY{E%$ zIJO((tgSv)hazi6eW22iZ z;Oua_`#e9jwkjv%U()hASgf+USmKBOmXOcLuz&)g6`-&AT&`QMD+_`BW8w|Xon0|b zbTd*xqVuaYtfBI@4~*^;ZHarb_pFtKakI!GPvD+nhW(v4Eaj~w%|vU=3<(&7~}(duPLUov|QTMA_4OFOcMn} zK|D8h2h7UYB_!|;MqQ*WJ*6%A$e+2+xt%W8)Nycgb8~W?TH9M&+y8bdtxiMqN8JkC zz$y{ewO=fX+8+CdgX(bVcMr00kJ30F=H1=Hmm}`$;@IQnSC7fbjh(SECxlQLNi#W_ z02dFw4XyDqS&ZTW(n!FPB3G-?7211N^JGn4u2NVvp}a^kAtJ$`2EdRcV(Gq_r;F)T zYPt{ymi`2!k;%OKUpo-n-7_AJZy5JXpZ2&Ym}!jj94@2I^!^ryqrk;k)uo^5zyb{h zxk@X7n`~HCb}N&V!At#vgC3~4lforHwWJTIl z4Bir7UttFa)_gD<6J9K^b?{GfcYz&|Z8nKLwEO;(-8v_VO6n!l?WdrPvodtE{ z`{tA4;N$0-$Wuxu0jvX=N`+(0hX(rd^==F|(yk&RoxE+JXbS~?yx8LxO5%a@^U8m9 z+}(ZC+a9+2E#KdEp|kTbc|_Lk0kfGAQ(b)y7}cAKQ^chAJJ1iXyJ+g?BpStQO(V^KrWG45oEj?GWx2L|qM52fa} zPcTS$bvPb(<*f+LI96@a0|@kV=*cO-)jxwR02Dk!uQsFkc!liS(ZTEsl(%M3rdFh> ztkX~g%f%G-P6RP`9Q>BFY4VAG0RliWA3O|3_Nj6c-J*de4By6V1|sGvN>*ZjjQ_4T z8bpywGru1c;Nr6E!_;;exxpZ^Og_2fu$uzoRV6FOE$2LSDlmdYB zo+qf5G?xDvoE)4N1Xjp5QGr$~FC6b71H5d5`2T=8oPF*gLYnHWUOj-IZVcshwe~|N zc$f-x;gL93%1NFaw)?+#44WD~CcWxt82?P&J3apV63~3p>~@~H;`n(<`@a8yRw9LC z-TCR#&A2^Jl2h>Bb|N*(B8fwes-@|K<(A;0dMk6 zRBX59*90c%wuQdUOrt0U``xM?c2`)8X#~OTFKxLt`1JeH82I^lzv&&`?;_OhF^}7` z(kU5N$v~y^wE@jL5?-gDNwje1gH9rwz?>xXz>v#JZ@fdxe;)7?;mP$z^sg;nc2n@R=-Vo%{Kmt6Go-xyXh?&q6f%LeK^-syot!5+6nbvk+y>f zVR`}rS}?acmhU+TCRu$EuZzA~djV8z&FKsD!llwPS?NvEJ5%CQ@I%i|A8_w4W@YxlhxM`Q=&3EE2kLHiLvjy>?;W5 z$<&WGY0u9!{@$9tr29MPl-0+1oRjsBNu-xIPtwue^OZSX-)y}ETp6gE@Y{1Ecatr# z(EyCblYVw~Y5{xP3JaHh?60$Hu1UEafiAt@g z_CV-7iEU)Tja@FoGh=PV-4cp+Yu~<-_jZl1e*<*Ox7Y{4R%*AU`+^-1V&nH}*k19A z;?*X^6K)2{0Cp9)13WzQeCkk3A!pFvX4mCyl%!m07k^v>A7%pyrp&;(`=z215@|Vg6(QOX8i&9RBw$1qZ7kAH8dO+S3@Yr~|Zt^yUo}cr!wh0W} z4@I2)jr8^C1_b}r7b$kPs}3U&sC;<7<4y|$37jNp7wzGQIGdCH+i7P2!}6z#tm1b$ zvy&_!Y~HU*&%YFQK|PDBpan=kQj1uf9Ktc^sP3Rg;ZN?DK_QnLynLk-*jfPffaB`{ zG~73^C|xX_`%bZK<*ZN=AQjE{RAvg}svn|E#_d+ED$URI1~i2)l#?ADFH&*iqHF=| z)bk+5D_#$Q)*P7>E{kX@?~c|yb#<+z<~@3Hp-Y&|h=hk~P_o|5xlme2V}kUtU~4DO z6^uyUY}PDDJ{&URbY!~v0efSejLm#EDmcf-rBuyH3pZ~ytD^a^-kp|LK3XeXo2%~T z=IJG7GQ9sc(^+lub@`P%U$_=aw|In|`BIbM7OvI1rCbj1BfwMX>Uo&;?djf_iLE1p z+fENahl$xJ)%JL5lj`OvhG=S3jKD!9>4tSOTuFSxn!L3omg?b`Z^w=!V7+m@H3 z;-)+W-~tnx=5)Zj-!dhY4RE_rAqVi zJp{%3A%wEB@xKx)>wL3XBLB6;&q-Oj;#!4%?WNa+XvF$-0w%t;8LCYZ2Wp_DLLJ=^ zmhUG^08!yQ{(!ujs6|tL!LEDz8JH!LoF@=!+#OnO^||WUM%eiN&L%xBFt3pMKfetI zDk{|1!{t9ti-<#VHER$XzHQAIE@yLR71y>qVPosW8n?4XtYt)Rq?(9RlGP;PkiLAa zqSbbWxDVmdB_`C;z}N0qwxcM`U_$w{pT-9+L<;|Mo-fQ|Ud)3*=3zA0R7+S?OB_z? zM|y>mo{STH|8@wQ1mQNTTpdCjSl&^ijY=Q7T&&Nm6;6UZBH7M_@u&_wjCL^DV=7Ca z5UoY%#zxfE;`HSTTDTaU_Xew>0_32l4k>hZaw2&(@W=Jbus2Y1*MaIsHfU~iS$bp2 z{Xg@^RE>VEDfk#iNq6e=UP)@VO^3dLUdAM*4{g`#sJcl9^m`n0VK+fOk*_5y0PN9n zfnwsqI|4q0s%mvvpSgQdJ|0*+;L#89ViZ4L~V~l*t-+FW5&T=w9HYJJnar?7iw5oqI=(MM{ZYL#s2SGAh z3cFmPYM!~2Q)pAkTH(j;{YO^g`}@O_3=gVB6~rN0#_y#<1)F)Wa_j-_M1y`gW8WR5 z9ZUBC)~hC%q0m7!KRVZ~t%6C!HGCf#vKKcGODWUD?F{2-KzNDWXFKu~IP;JO}Vke>$BXyZURtLqVD=r=lTXoL+TQ5 zIjdF{oQOnd2^F0XmsDh)F2aVTnhB2UTT3RBxwIVBD{TxzecN@B#0V2*&Fg~H)6Qzz zW{&qc{9e^|84g^|PV zY84bEojd^eU{^_4c~?gQx>(61hS}Q-H_d3aVr$MvsckI-2;pCig?l(FA&fC`l=)Zm zM%NfGhGpoE7vfFoo7iCqnUNQs6%6uv1%w-+Dd?E`)(ft6P$Jk{>y;!Y&o6W=9Fh@a zH)}FyJB3e;<}WsH{13-dMo;;%uh^S3uXt~VN|?7UW&DL47!oAuPn3ba{&uegWw#i# z06hKo#ZGeXr_N9v2<+ zo|P<{^ZZRzl!Z1(LgY=@jSw~)A`Ytsr>-zWncG@n_!Wz((<|a@lcf*7i!ON$wEl=N zd7HO6o~0IBGlKjc6{e=Q0aX}5ae#b#`v~ijt3p)cvVI23(u~LU@JwUc zP;j2aSCc4=G{(#Crk^70&UY6QO9Zv%Jr3J9+Ump7INbN7;pLvx0rguHcQn0*~Sh4epSG&g&1B*62hbJ77`}GqwLUDKIe3(>zb5 z7bE~6T&)bRjq%bTwygKP+6R!waoYX!RHFVhIIX0mr^U)|oa?e4^-&vN=glVAW^t;@ z-4X|1_BMI1raQ5D9?$&f%HfC3h7*Ba~S!&%JMuiWk9<7A$<vHO?sstJ%Tz)YEa~f>FJp-JyVAc$zguug_Wz2doAHk8Eu(NTBQLOGSj5 z_?K8+nuE;POgqvwT6%t`;H$*mhGe>Y7b@0B^&U>vxhs+;c>3X~579>t1(^l3<11W; z&%B`=$6qx7ilTqohl_VKWj&l!s8xXY-aGUW{uZcVl*LBY z^NvPJQz>$f+h{`Fsz2qXWixH?(8DRjGreT_q$jzlunFNXP=qjo3_f0^f!v1ke^*s8 zL}hub^Q{!}9ZNvqRVI@N5vvWTKnvi4pR7_-zo~GYo86clnCoyYtY)p!Du$2ODTFFT znSLb1m68uQXMUMzgn$ERa+!FtHcd>DORu6j!z z(x0uMJX!Rm?wTKQ+Jg`AxV|t#TwSDjvZlTQ9D;Y!XLX?>Z-D~c{R5P@5$@BCGV88Yrbgu^HEJwA8-V|c4I?a7$VpRqNTZk!pY(n8b zp#WP6Zgw-v;|$d|t8&SCpLNY_H>((jnGGJ7^6?SrdO0b=XW7k4cC!IIwMTO z=4y2%l{aUW3!;dsH@Q^$Mn~u;t?NvOnV-@4=vN&+nL0X?)v4kDGT5wY+19l7ZjbwE za8Sm#!SY}u*V~rQG*YvEeWOpCEN2iyk>5q7O!+QX9#voo7%PJuZL+?g_Snq&#}~fn zF&lka-;|I<#6>KCs(ujhR2(%Ua3DBbDtHWy+-LmjS0h{j4)dJZmI?Teu{a7h{)R}^ zX1HJS%CRi}*U0Qiu5%gx$@psL(y`4c!MMoRo&*|0&->QX&XN@ zDes3@4T>hV-F1^XH=y;CPo1pWH7vwZl0P0UL7i9)E)gfGeVxsBkSXl_xja_SfiyuP z|C)^9F#Ng`)6;b#laUZm!r!o8+5)=!PqdyqKg9g3e|CH=TlTz?l2?o297BQLA)dk? z>lfSb>x#Fmzs@s4rC8qc8{0f5_t7^i|00l#k8UflI#nd@Q3;kqB`=CWG3A~V^$!f| zZQJk;_oa7h>o4Jd@AK33ow!eymi0h6zL(Q5wa^wm0c2Q}H2vw=x<@FzT#IkhLC{b)dj88o%%`550fRw9iv^ea z#Z3((`eU_@@|@BNa9y6Vnx&p= z{s3Or4;WVds7mti+XFU8cq4%}}Eip@@-V>7Fy@Cl#J4;uCZ ziurNHarK$d@8dQ{0|@6+FSMsDSOP?G(Xlx)>W!3*hmA{(Al`ElMV*L5Yy#w%FU5qC znt7+PgfUwm9!n$ZO5fBxzToqo6d6m7;YxeMH#>b+7^Rg@+-(!$SBluI-yoiy*vA0+LkXO8hseCIx6miPkp3QCac zTQVvC@S@vrly$IKnB7OOlQB_vwYK&|l>s|!y@k6(jL6vV?F9T*e!?01Rg7u>`=~Fx zTx1z%v#T${xUHXrD$%xo@=yA1fG$cpqQg~s5g(ud^!kNq+L)@_@c$v|EyJSxzVBg7 z5R_1mP(TDkI;3kpf^?VU07^4-!w@PZEgjO$z|cbtozl`dbi*)o$8+QN_rIR|;tlg= z?m6e|z1LZL?Y&1+Kn}B?XN2o_(wRPO$D&JlUguYUU+`phMU1IKT<|rY!v%fHDUvF$ zlFvvBw*fXn-L8E_n#AC>#p@acq?N+ll(x@bPC~c}Jqyw*Y_U1dl*UQ|vnxPha#JY= z)Ek^R$D0rcuhRuRjpMVvigM}?G??@wjGHzM(O@b>03!dpbDg1Lw~=gT-8uG-~oaJ51<5h9;Fm1T@N9s@;R{RUfBBXYzw-a zH4y-5@zy;8_bxgdhCNZqdt3+fB5z~euT0`V5Q-U*0Rg3912lKmwmKBeE<@r;buIq@ z>zmABG!j_zTuBFMYy!#?GWCKO_g;sUI|31S`qITq>N$TkQhaW3tnR|e{XJTLI`Y8x z+WIToZKoZs?9cGb_<4}9(PHd~fQ_r8xOIhfX+hX{jj*WOUO?RSEx$Ph!ekZNhzwEp+PN09C?ed8)CA^0{;FaqW zajUEIQ5FWe<0YPg2L^%=0xGz*s7JcPf-dSH#9=`3W-BH?8S+``DY4_qm|5Cu^W@x~ z8dGtHKjdiFaJ5%a@mUA(9b$c0M>viDnU!V)DqdYDpKtQf1 zw9!k^cW-hg{H}2OROs<BtIO!9Sd)n1SWLfqqJXaHPrF^oFu180X7nSBB)vWG@wQW_=d^8mJ2nT{ zy_l2tMY|7(^u%{2esfIBNz-$$P|5)QYPR-W4~L*~tn--l^mUWR%HZ%Rre0WMlnP^C|T86q~MKc$srPh1xhns-HPa&&zwqy9o59o%qeat%gh|G)%sirSK&fb1 ziUwNEhgw7?eb*gXI?l}2z){&<>j9v&SKPZ5f?zf3H>v)>Ad-gSoZoVL`+R`8dkX{c7X*sjk%%%~kW;?S?dObVHiQ=NvuH zF^wsG1#iKkI_Tjs$m@6$Sq(z-L?-mwYT4_rmiYb6hU*d?TN+gmwqA8a`%t_dW7CbM zqVxYl+jE6rDM@Gc0&xY_&H;4MFluae=5A%S^tG*$=?jm>C;{8Y_fMINPoADQyZofzUIw|} zD1ne`O8P=!^$WQnU$LofH=L{IkRG>~lM?{$19y2KX9wbGZ-14&)-%4i?r%AzF<~6p z2oR{ztV^JRQDG4; zNyjvM3CLvrqfiP21hZ~ ziK3uGvJuuMbCshh=8wMmqvt`<)h>b(ZbO@_0G;#>1oyg8uN_?k|jOj?CPZcJ_gt?(mJA9-F3p zw@SpvSRDla2k`&pf101dZm;ApE!lExQOM%MZ!SZqh+Bazc=b)0mydmp!tc8G&?%GZ z`8rW{S^A=+_hEzp))<+#ir%mJ(reRjnwiyfS)^h9b$r%tP^tL!jvei2nWpf;yt}4X z>3NB51G3tlUo%4i?05{Xo@@A`lHgh?R?aQgiS#*IFEuG?yz*9QiL@;IE5WL7f0^Ps z7X#|Vh46-ql`iG^oF5*L&RulPWiIu6?k0?`+9`)N*&kn=9kO`=Jtn4CdJm0mROa0* z+erg3w#I`1C`Ko(_}HGI za4NI^qPm~G{bm`1@hUwvmynBK;$pN#!3jLuAeW`dnCsOs2)_5hZXk9rbHvzZ-Xx&k z@$j#t=<3CCpQe`68z5ii2`F5#_~T!*vBkblose*dH)bTR&-|Ba<}j0KXc%Y|jcEBL zokb&{vO7GWL30i|t6acN&HA#B<*G7@h$)y+|(wY;~fdgtIt74C2I9>53Jcm1>J?iTTmIp1DwAHq9}{wAP6 zLQK3kb}e`q&~40h_Yqct{aprRJ<3-?svbLj)wK8jd24OvCg;YRdXY8o7C%w+*@-)>C^Gm-HNI zS^Iyb?HoVv&~37cwgzeha!RI`oLH>9=kQN%)a*{P`$NN?v?vOR+b06s!3gVKr4-DH zQ$jBdMC-x3b@!g)>NdHrd(?&iudvQDSN0ey`2#LNya&OWu~|2mRMB1s*2rQL??>pJ zls#MI_AG?*|8KjJuzZ({9w7IZpr7*<5GVq@$BsuF{Eh6?r@7zi7?(%3)k@Y;6ARb}!#pR8M(`24H9`KTZcr zJU&j0+u@3*X5$h7G8;rmWu7YP-%8vQ!*|&+Kr2)~K`yiIOl1Id)m(;?l74ndAr||Y z{&RMg4yEV*`rkplTh?zCL5;+K7w&9n1;S=;99tydKB9VuXuH8}m5!&03JiL*k1qtL z>l9s+*t+LXv*W>Sw7G*j;0_kWKm2~ZyK%~AZz5D}&&nO@sr!a3F9bjr&$ zK=oliLCTPH^33!X3LRPw<{rB>VQpCi+1=W8u9-bl|JXv~VdQ!B<6Pe->gY7ZT+oKgbp{GW zdlg3m4UBP~j`dYIALD0d|Hxc=%QG@z+-qWuCQJVOpO$=iX4zQ{7-wU=e}VaKNjMg|>H)z10IW|r1JWwvc&Z5}MGpcI1pITfvnd^NLG$U3k{R4_3<1nnCh zi;n&P`xui|*qC3_y++mWAikjxNZow;&s_eiUh9v|qWK}zkZ3}a>h?=Nzn{OpEuR3y z^UV14_=~I|#oDnrgKumslY?A;)q5FbY-s*<0SDL_XLTxwfZp@eNXtGQI#jm%ZL9%29NfJ_wa10P=ybaX==|K8(O6?Asw%E$?NP6hFSfX1 zRPEM=-GybUY}T_UsjM z^Q7dkGbR2`L-VoaW6Q-G|BZ@Ko~Ntv#5DFHQ7?|}-H72-k51{SIrO~+9701uF|FVX znQ%w3{LM$aa)-8}XC0Ri4}Ze9Z&*b5Iyn46?_)AlIQLaN!di!>#`Sc~sD1`bO$$Ov zA49Hz%eIXLkKBV-*qtk((m^r|Jt2?dWGkzA(|+RdXr*U?j^u^G7);FD?##_3TCtA3}9e= z}CI%4Z8b0p3ZEz9cO__{bYLW$e81EZRG3 z2)%)*qAn@fC6j&jxPL-nUs%Z=7l@=;M%!6USI1PB#=IJjUL6t9L~?Q_k_XkwamByu zPy3Ka2AOKwbvKl2Yv$|mgQj_GglAVE9S&I5N=J95Yt3d(GQD_EMmE{p#%t?f=<2jw zykAOZMXMgT27L|Ib&caa3Ps4Ia|$`41LQE1zswWNZO6;2G42b#j|Ufj;ZdJ&+D`Px zqlNGDOTBjA21i8Z`bw}t)829{RG$13LN&|ZrK|a|iK+LAntSx+v5W3~j|}QO&(j|U zH-M{JxFeb9-i|KAD&HD^l>V=xK?!HE+;vJ1??LY5;Yx2owq`8I6QqN#HAmAWKljl(Qx8Sj0ZKDc`>F^2;3>_X3DcZJmCvy7%)mu;{o0 zaiQKWf#&+Z`Hv`W*6w4;^A&Vi!8m_gTH=9bj>`$fs`px-Y0i2%>cb>=dj=BVh@rQx z&nkYksqc4wC=ssiqEo0T7}2|!!p=$a+A}_xhGJ?wm$Qwu@Z~y$ejChRP3;Haf8iq` ztyj_88<;7ic3ZgE=K7;@FM;S}V4$iTH{zpDZ1B;)p^AntGk@RzckEX3PYu^Hxq)Lx zsZIfI{^JxeL;ox%M*ZGB{aR{}#@=5`KqI)an|6jZq6O0tsiQLR`FU2r{oPmh*abmg z(hUT8!6B|n<#eX^cEON~u!8yVx#_!K-d<0bwX0sp*z|4h(R7BIb7se|tIoQm{x42? z5liEpr$A?)A$QNy?ZG)`%m_QBD2*$l*&mBy-l-Kfpg+lHpOcEf+fsL~%|arp4y5<( z9<8gW_9WLn%0sT|7Cj(zv$xW|S*Vts)X?O!V0J33$rW_{pgD$Thk1{R`#>8|Lj#6e z^oj2=R<|wwE0N`p}qUoloWDo27<1{dZpyKp9a8x zH?g9Q=LuH2!p6?k1(~cx2fvzWV6FQ~8r&?}Sf&3)j<#YsMy9>jv z!w1HfpRuA*<2Hq;)jXW1`1tsoEb`K!xL?Ayr_!5{o|tu><dkwb0a+p2frvMT}~_ zx-9r1?!3rpVWFZA8XK@tQxjK%n7O(}pjJQU|KOpa$%x#S!E;{UYLWxs^h;X6h7-`Z zA+q(@q-G|6z!NN7OvEdEgr!@gKun1sxx7HNWgT7-ztFc7W^D5$K8zk-hRSZ$0jx;{ zGOGbP+JDGh)-7{k;o&oAV!cULXSH6Xfxa!b41I<|q2at)yzQ5^dH+LyShdB|z>tq2 zv_tJ|qnM3NJohX~e+1|T%W&6`%4faIj3)wQXCV1&HYXWBu;6H!>oO5zwp+^6c(}5i zm(oaKc(Ji+(9tezFBdxA71j`_?{Q(4m)J-#cRTO>(#UNo?h`t3=Mk19gq87-Dij>c z;0oq<9N1k8SO()eZ?sPCf8bz4BC-mKvI@=4VMTjSTA#PgtifNGSvR52B7HkpMe?c~ zaR1pLHy}>wd{hY@YPM$23>$0BUhSV}cod;T5wG88r2eP8x0=;|0m;kO>YfW8ysObfCd~ zpSxhIPiU4w$~FaE!_!8bGkS$_9jVjfUy9j;)xVGvO*+YlfGL4L)~?@N2M#Zs{PqCz zt6UP{2h2}%XFF;;s^ej4=TaWk?oax=xRjM8YZLGFK6E+n^SQfz9T1N)m#)RfeuS!q?3(OHH4vwYpwXn1aF&9C!{ou1INx6+#ouWbP2Wm6?H zL{SpGyel-cN;W+UGOQc70q}?hP3I@ylTk?B$Pi$JPI3dBysqPo`?F^8q8<`3(;wG5 z3Sl{6l;Z<7A@B*7CvYa|5`y8!ua>yJE^+7WXHLuv8^UZwzU=N)X?ygGkr$+u4EeWr zw6~M{zB?px3@liw&6{J}No>z`s7RtHgee@uDOpootUvV)UnR3#*O>KZthfR zdC=Ld@~#_gVAPMJ9)dd@ZDOPAW~OkOvu>s(Z4 zb5ZAJ*BMwGW)`h&=oB!ml+zP?9V5S4=SBJB(<;qajY`?KTA$h_!|FSf(h^d*uROL~ zoT{lytShB6^D>Hf1-ZSP)W42)UgPUZsX6;fJQsPk>RfhF`Tb$Qh-Z->BLiC}$ll@mv)m zqoT3b5FwKbCOp`@AVr$^7g2#4u@R%d6ph)sMB3`YNTs&y!h zl^ju4F)^&8IWfHK;;gQLkYou_#iRM;gV~SDY&p_cxcak)<7J*+K5t`TZM9K~7oSs9 zBNh!h(*tgv2~>A?{|fNtkOQWrsLu%qi3s9*G~a_w0|EH&Ts=+KmT#)UrmL%oX^!{) zPV@1B&y!@!OuFM~`f6UgJ*caal-#B9q0C4-YDQgOf8JRYCqw5~a&+B)LcTumqeKPp z@Je-6rL)}OvkHhcC4eM&|C_wSp7W||6BJ1lBpj9sJydF7go!!48w>3T+$HVUWb0S> zl)#Z&#}q3Z8uf>>+e+SRS+0@l^~<$C>gS8`4jaTbKXnffq(Z$S2FLV)X#DfCkK z{v09q4iS@q$k?ZcqxWsc1?=8-c5|alY)B@J7lmOS3^8aBC=luR{87=Ut*t(Ib90c z0YGlOZkw&aMWK(ZAS`tAu-v%10{?}!?@Y-dz6z3;A4NVkJRh2Q*o<1;irqk}R8@+=zdfApnP@Qg?U zsLsm!mK?OVlIlovH>C~xvt>Z{u0}I1_V$Q;U}SAPyX+@Clh{-vc8J@_SAnnEd{Jqn zPpu#r0?iA&#nREwy0gF=i5h9xwAz)x{f8n0XraNnbk&HQ3}zXr`(}~?$ltFlBdT4T zQ#S0M&;U@yCSLbv^B)ODs#~-7+ySL~MDBm~ek!k~0%Oiw$51A&Y)@6gE9~7q{VYYp z%W%8LO&lc)V#WB8TQFUKHB95>wAGm2i)LlA`O?1Vg`MQ_Nl8u|`qgbzV2TGQ0`61w zwY3o30Xt}NF^*`g*K*BowtBAft_f9h8qjo80%) ze9+0V0-_7qY5t9ec*$OXlC4@<-XOQj1hY%2clvj5(10ged;eyxI4;9D>VE{2fh`A8 zkU8Qt1;2$CxC=?w_G2oxSv7#==TlA%*E9g5cdLF@p0&{qn&+gR48Ih!;%W{&@ zXnH7?_0s(3{POI?JbV9cfF=!wYjSay4K(ow(MH#q#f)ka=up_U(W8nfYkJ&ciB0mXCYx1Iqq5GZ35Hi4%-0WM4Q|fQ8;fmdJ{^_s_!q_}Gc#DG4@5Jd%<8fn zQ`^+**Of<_{c-%o79DE!*s2Lz%6bpp>8UH%P;!Az|{D zbEpN|(5T}{!ifc5`bebf&=j!^VH^~XdPfc5&s1>=<(^(&30cD_MSyCD*U0zwPNPcj zh%g)**d|f&oK1nxlHhzg*bv8*7k ztHSo9hh=6OHQ2Lrg_Ld@fc~*MmEnb?;aBk08uwSXn5`( z_UlQci!zCd=##oh4&HYy$$>-nF2be_lR=%%Apg#M0!y1M!1tKN7xYq9E%b#8p{{-y zslMow;PKTK(6sH^uM1yx7B$UOl8(S#gFVOU6Zlv=0Y=yls?AV{*p_mSS53xINFrzj z_g5VUZ+4v1&>@}`#F(=|C^If~E#^B30qYf*1O}Gym+@&-ESzJmi|K6ulCM|yHmHuhvR*& z0xtr0FS{Q{eAZI@9q}zmjAu6uY-zOCza5IKc1tGASx?ChXRp>ggK^lcqN=@kjEY86 zxQ;K7-a}`B*wr3sd%IpUa@&XO!N0-o&RFZE2^zI3wc3;+MPc+wFCBKf7t9 z2jF*!tZ?=(nBTn@X0`mjz7hJ)jP2X)CI^+b%G+$ z#zr|HRw2sAdv+uou9{YEB@xq$&cnxF5$x{Uv@Q{{amLFC)+=Wcl3iwhjg~s4&aalW z9%B_DlA(E#49y$0b_HlJ)tuCg^~(7#Xv)dY1xGK$j)dxvXn=*m9fHD(cB!8?IhiE1 zu@blFF#v@2h~-|yOJ~FU*;x6W#0i040YaK<(6t22Z0A3(oDRE4KkW>u;4&Lb+}D&h z)lRMGv%P5@4V^pSQY+ELp;44=I2pna!`f8UUWf&jfZscn?Q^15!%z$&F4?$p@&xGj z0^{g?fe#2#uD5@<2xXr~Wobs;r*mye!1r#k?;c8_S3oV-M{k$wQuN@<6x0@c$xHKh zn9)_JYH@$GNh{u2AC)!g=MPERvc59}O9%!pJmj`Xa&u)BJ5SnJ{&>h%UqNP~W=iBJm}yCf<5SY7zLo=HJ7mC#DYd*HW>*>cY$>)LRb*k`#PhJt z`1uAlhuJ3kb@qmH7~ZKH-_&zeZHu7QO#Wr36!!(s4CTiI0rAV^4!J(I4}&L3@K#dW zEe2fY>JP7dII0^8Cs_S(@D0ddC!}I$*qfvQ*t|TGs1qBcuG@;O9=;qD$sZeQ=fW3L z4UDGqrXk8mEs>{sC4}4#ud4GS4F20^E4_)Z747s)??~n0C64Vm_U6+fVZM`hnkX~mk4yRS0$BqF$Je@{t`TT>h$FesR zqLHEju&HgdkGV8-Jz3y6k6~qNCw&gHb>-9`zKTO&D=UnrE zA?y`8n<~@|3FTZPtHqG{gv*1P~;ZmhOx0Z;_IH8Re#`0BnADHx3`_ zWVOdF@1#>84%AB%nc8fdqc;S!kTrxQGH zW&TEATa>5YnsO{;a6V>{UGg{)GFY zyKh#6k!xnd{P-*Aar&|6pbk4nv9e~Pg&qpwUL~?{l1ofv##=Mfp33F5b!@iW>;#wrcq#q23-BLQof6UM8?hPNv?KWN>D99W( zxMp|PY%nx6zGtK5wAne0>mQ@n5Irys?sL{=e9=HFA{os}<3VxClR9af4D@B?@9^6C zplbsFVH}#Pn7M|2R9wAYKxT_)A=ck|yz-5Jrh>7{S^trjy& z{Zrd6V(=z%+-%{aWEG#uDuf?qHuv0nzcN!FMkompig>UNzQhGm#q>ua< zYl?XxD;u>NvJ&H`utkt!M|6KaCgVxr1ik!$6+~p4_$EUJGET2>p3l@4b#*lYbLBF5 zOcsjwtkXYw-uuhImW=H)tc0;_PA>U83My^cRA_;D=<43nMAB7oQamB~4j13JXYFD0 zE`>oEEVXA{pGoIZ`bL?Q-b_y0Kf?7-OeY@ZX3OhU`C~`WaL`5hFl$NEspNMvNm-`3 zOT{UP+pM!?XUI=LPuS-fD}R}HG`JZTM;X4bm_r5PCCrMSGU>IBbzl>}Uut2)Cj zjFGA)aC_tiec+=V<~aIQcpL;dUQQFrs~uXmY2%81H$wF0J)2OA=9jgLEgPvhPY<9U zBH*%G0thA40(h-r;9GGFGs^lDWnvbQ_PW zHS#rNY3LSR1W?yvZm;1@)yTpwnyb;Izl1#B;pAM;l)f!Y&aRSoc5@dV;Ni>Z$B?({MG91W!W zIRyT(1o1Oyw!-`aveAdw9UX)eiCME=41ykyh1#( zEzrRl;K_$QTDxlADT@%t!R9LEePC|KDh4unN{{xGcjTy@gl_LwgaUdN!#HzA9t_OqU%F&Qb~i`YuBk2Zk@4NuSwTXgPE@oJ%xOA*T{XwQ7-N z4yP}Hu)!(9>MFC=vl>)O&iOc&Ow-m2Q8MtQB_QUf4E#@4kW8rhF2c>^wg}EZM|4*L z`^ImNAXr|>;-AdVf_>*%X7`_>3o%l1q#1~e+ zZs-a*A+DBbowGijqW4a^84?PTi;qnEuReri1lSQ@S4M8tn^L$>+8?1lB?xMO+ zmHgOQOUJ`l(>|o3aIJYj&4-4$Rdv{l8#a;)2d8kK8S+mPU_|~Jlb-AX`0>omPts;5 znoBF|B_$h;Gh!xru}-Cl=>pA4k1fk;lcAuOl1A0DOaRDjpWUejcbtpr#)sDT0C7kx zjVm}oz}p)*nHTO>}paIM`q(| zsJL}LKDuBZF`K$S9cV)x*+`KtkFq;d>Inzcj6s|Q%+SRtVXk!amR&_1CN?{z(wvc0 z>_f0pgzNmtSSO(LC}5sg)vFU`)c}d4aaU_o=z4PB;&y}MDK-VjViMCVX1`J-#r9UN z`%+UC*PfPL?*2*Ox22P5U>(bCdHJ=@PmWKok`qKj&nR!QozGMVx70Aj?~~@T_G%jK zDo||NWjIZhHAoR;b=7mVXjzMpGF#3r&lJzSs#)xkAgSF%y2d7@dgqz8N+h9k4>GM+ zC1&(V#U6ID4I*tuB;wb?BHN>qs|NI`my>=4Md|vECJ{DB{!}c%?l4Uoo(=~xlST96 zsx!0b=VJNyePeM!2%Rh*KdDcj$hcozGcHg+PbAhhvnWRjT5i?7eW?J$9yj0m`;$iiXt=cr$IOx zJX1N}MT{w=xCI@x!mD)vjSD52^le4r$o!0o>#P;2*!%mwW4NFkofnxCQhrkYf$~R( zMdJ+Z+qyk0yQ#}#@cj(Dyt!QYf}p?H6URqCQb_gE==wbxo2gF(Nl170#+ZjvDXlGo=ir6X!-oZxwiM z=42uw*18mk1-Pu;w>|;`?EzNn*A!@`O5LD+^hv6~V8-xhL z>+s#UBv>3(H2}i7cs*6a?uu;Rg#Zcx!xJXf!Xv6aoljU0Hnl){XgJxu;YGy+@K*MTCH7U3GVx^kVz`9U~b6>&qy-UJ2WO# z@U6W`X#RnYtkRYhx7=GrdOu}~4_wPJv2!%6a{I;o9Y}ymw)oRGSnH%P09&=Nr{H|S zeL?0Mp|R(otCP}N2!V|nm3wGFAaIMwcld_Sg0(NBp;Fl;^^+9LFi)qIEjG`aZXtuy z^Ww;n>8%EQ!;;#OO~S9ndDsZi?Zz(egv*;5AJ2mHmQz~MfYfDV+I+q3AbpeD$_1iR zrrt3pSvTazU`r(!lSNTZjascc~{Ot=`7y&{$FRhs) zo3-DClEr3=C@LpobeLmnd|_i}$G=FF&v#(w$pSTTWDQ46*6z`7!}Gv)c70QjQ)$bV zONu#@&@A`XXN^yiiao??9}{qW5Fbn66*?#erW0>H&V!dA4Rd3i@l5|^-d5E>h7P}~ z2Ds#Vy@glFw3`**DFS`@seYDlUPON)zsIq=VQlVbGlW0{t40r^*n5LKYE?t^Rw zPi_%+?jJCq?yHdLk3$yEa^*nH+3J?ovReq8TD&pz7c|E8eUOSh3m;MhQ9&{0A^uyt8f7h|XHaJtm#sJ~sOz&3dGPdbc z7S`S&n%$i{y-AH))eN;1q<7x&K<2kVNqf>|%-V<(f3s=BsLgStR`uI2-ri)EB7?jMq$a&cZ zNmj79%woj|3X9%^J|}B6sNfI+aUZyW#Y1X)$7!F%WDc`wB}B!2rSsyc7&v;8{xmFt zKuCXjvAlZpbp(M@lE({{+w{GnF$irEMR;|ki!%Vis)3BhkG9O{VYQcvHTYBmKw|K{ zG)>W*k8WK~aqe(i(yuFfYPH*6{y{{}X3txQEyY#mbiCxiZTl{Au7eB97}9)$goL=~ zg}*QA?%Hm>xGT#|$N}`6?yH;;9~;=XzkaOwSO4B;Qv!R1VfWDity;RZx*5KJxOt? zTChK1u6lK=kpvWy41L*4tf!!1x6{~j=86|;VX}_S-BlA+RyTdfi5L#kY)QrP=O#-z zy=|R7X_8xIN;U>Besv0DB>mCYeya&bs^ct{I;&H%UAwKPMx2i7B)>pcns$}&GKgL- z*Z*lz%L4%FlSM>s#wfD_*LV7J{dQ%N*U~q*%@!X>;&*_;DeT7LtA*tr0@22Upsvs< zK(pvSvcy5j3M;F*5_@qNNB?S6{3Bi-yf!;MzhigH664Zt`NL0Higku?)sgh3F8iuC zTJl*~!@gEb(w<+@-osj@u5+_;C9E8p?WHhLpPp;qr32pfFbWFja@iTSe^+i`*Rb-f zm19z2aIX$M(d*y8_WW0Hc73HuJ`i8L$E9-2}F3{ccRsH*HwvK|=2shBG0trKuPAcsclw`$FUlZ-?U5lYM(N zF4eJje+zp-H>eL7G$h%W?;UhZs>Cy^W~r;`X4*J7`Ik$#Mw46ma$p2$ zf@Fl;>Z9Mic)Srl>+Ohnq(DDMH!XP>iWO>&Jf>fOzLQEG6nk za^+EbVWcC6)eSc-a?~0+@4WalF8=4IqYVf+C({$tE21%@N$+lpNvZoIgb#yo^J?pi zCkAA7PgY%5`l&?{(`!O=?8)V98+El5NRU;#Zky=L$C$tg?L&L*E`&^^<(=@fkHfMm zLHL8Z$Qn#BFX0ofA`q8O-3YKHw;pH@K5E|Fw7{6NvL2FIl_d1MElyB~`JO?;G(=k~ z6uxs_DY?nVy<2%I4dgGS)hgyooo9f#$@B-J$BB{sF5C0dF?ZJ;;9x?giw!$_^I z#`_7(P*--g*8|nJ7$c|P$h`kw9hKcnPv(4jurQ} zi|!Ls2uwJQY;E{{w6o08Cx$MI0-GcI)Y#71?DP9rioN^35_7(XS$S2{uT6;%_9}`= zS^2GRWlmCw*dx5ZPnYibm+j-V1sU{N+?_R9@#$gt%a-BK$lkdROAjc#nXI+kMqAG& z^WRVa{LHu7yXN7BaZJpIWA~W zDzXFeu`b=7SFEa`7{g!v>RKack)mLZa1uI)NsqClH#B}a)X~Uy?_AAirGpJ3nXk~$> z^e$1#;cTbxFwSK#ebSib?n($WEP{kIEGUs^q_i9`Yc;ixk*9{E!&bH1Kxgk)QJ=w4bgm@7 z0P^DSw~zErd5F>KUC;0Zmb-$Dgs}fV+>DYDvHdF_Fas~qH@R;GkB-=6WCv-Jzc{lm zG-F_zSrkE19NL`3T3a%nS!rfPh;*1RY>D)gB0OsyC$=$|sR;^4%re33B7)D_YC3na z5<&j)RJcB?Ww}Z*y8E4K;15wtK#%ggwUA~cUWi%eATGSYFRjA+5-4YMYuQ`!y|_Z& zTuFP+TBH?!NDu3lA3}4vAEj(DdU-+mb53*5cNk6BE_aUMk6xB-KLbpc8(3!(WBnR7 z3^VzmKOt=4P1vcI4uNnrJ?L==kE- z)dGp<&0Qt&ms*hF8es+rTC9bmgy0A_0=7Qg|5xxUA9{Dvcwza%YYxZIr))Z(E}4NE zfdaMu!5$@bk{F=*r3X{m)%8qdgnrXFKNHWYIe+PSt(tqTTFjI9HS$0)fq}U4OjcW; zdUVBf1}k$K^q`?ns$}qZOhAT&$Oz}tu>WFZaZ0RNsI43GoR2%5wY7=9XI306 zr@UM4cSThjr-#G>t5k6+tZyuYa5t3o{-F|n6bp0@_&RR-aCdGw(HSy%&=XBQ@8t$g zFw5>9ud$XJ`l)8k7aO0{!>sdmc-*v46EB1JU9qexv4>=eo<<4i+`1PHsug1Ibv>+j zVbFMV;Mg&Dub}x)Xe30MY59PCz9#2@aBr>%WkK;wx2BuA2ur!{lmT*DC~x>~1y?ZpF!hAeS$8ZVCS%pw?QGib6(e|SHo{L6dlEvGk%EJ);AVbj&$iC zodX6lyU_Fba280KyrP_q>;cu>eoumwAdQy~+9PYp19R3=*5{ay6b&8?QjaOg@qiQ= zTrLGGbKEe!9niqW4_3f1E;-%h@oK4DKn;E36>+=X5uxNJD;7##0GU`dsAglfp%CIN zXdTy&za6G3(Kl$VM3>EvQa5hxZac)`CzJp4jIR1V7)ne+43{Z7DkFVHO9AFrxR^!S zxIY91SXjNfcOMkaCnJg4g5NGxfJ7@y3<@Jq+Y{hM`^=8Ff!pj}*HhRAX&X*#P1|EqhqSA~ zBlq~X-#X5&Cd@}*v9Yl7WYU~ZDFqBfcOx}`DZ4{z zNOS?n3Tl}tvf;4XNV`5N_$?_xt|DxPLLv1x;&bDn{HhQj-Bu@(%wkh!b)J|=pf^B< zf};eOuCh7TOvwz2IfDv*Y@m<7XqA)+h88 zO8qw2P!~PEx&}l&gbn&yI>@eR-*=cyIimn0`JS4qUUrmfEqrLbY7Mwg#CTU3MIkmP zzz!BgX8-k+uB{o&T6ydp>kNL_6mCTC&s#xo%&bOYeIo zP^Jcxd*OJ9GHBosSvgXYAKbT&03C0qP==4pcbLOU<(ix)IupwH>|7$E8=nJs{YJ!A zeUQOx`~d+tUZA^Eb@`|ECABO2G0897o`&UA11lYVNyHj#DHBDF_AuL?LqR(g4K^L9 zyjRvr$Oh|WvDOj(?zPp82YPcNt`cn8$*!ljwKs_jz(B0KU-qs0pZ|6S{I|Bh8J1k( zlW=a}?sL~N&oPc^hxv5wu=!x*4-+}N*G6jKgr9pTEX@6O|L-be9%=0p#hPmXr@A{E z4V&sY%;z%Zw8uoc$KRY3bb-9k>pmC9BA}b`RnkuR)j8ijYS8HkTd&3J5Ff&Rv#sf7 z7oBGhi#1A#WA;9r+)M>P6d~*g0lFRhJ+akQCt!ts0P;8nu^YgI;0;|xH~Sb&EgPe` zhaxo)f!=Z$cK1)%UbL0O&w3dBx8Yga`){KR%e8(GVsUJ%PmI>;T~qmLYm@)0f3*uA zX`j8?AfPqySl&Dhq40R;8nGkieem}f(zRh_tmSZ>mKyB7TU$N-1S%@7dtrgFF@ZL^9+H=gVYvFZH&5<2P@kLKo6w}Rfp z_CqB}XLah}>C4QoTHitc1m%ZmpNgD>ATUigm+L2=6?WA_5|Xj0>dV#lm^#x$J$wSmgm!KHvPvw1IT&++K#74T*6-U!x>maioegO6X7j? zZC&~rn#lS1aFLSThqqon`|SyH_3hH!m3r9S#sbfP**`$7=-$DJX*n7-{?`_k{}k{| zb6)e)Na!{JsWr;ma3Xb>GfL}wS+66nb&a`^cjlMbg>YU*3^ZaD1qPoDTtY!??+YxQ z!y2rm*CgUSk1wte3gHNqGpg3`v*_T)?WsF}uookpw&gGll@;cF9hZKD zLY=dL=k zSsriZXe0XfK*{y(JA?k-`>*xxn&nz@>zs4X*=NURe|8&-<_k)dFE7Y5 zBDmVy`qJzg6iBO9PaPWE(1@Pw@HlOdPv+R*ID|}!l=>cdbV5DD+gt%F^da2mZ0CLC z=NtJ??tz%{C#_86qk?!0^M57L`JFDIM1~R*RF06PmVK$SLR zgL-4lpQK-Jpqzc$4zy~oxmr4%Egi5NzNXaJc=Y8r{QRVELkuiu?D*kY&YBB8M-*`S z+{d}|3!N)6y1clah26&O3`1%K5yPwWAeqtqbyd;_52=73FU^?-WoYDe^Vg$M-1gNm zVn>6yI29wGV|*k5TE2Bae4}dR&%tTOS;M%Twd;#$!Uri#l}zle3c9Nm8)|v{?S(w; zg(7HiAteFqE|r1uefu26p2>vxavo7`HglU&v5U_*bN&JdtZ=Qx01o8P+$+O7Lwf?W zz^SF1wyvmjZP-BSAoCV_Z0*za04Dj&Tf?hMYd0ugD5xg2O%9F^47yJ)nC~=^m@D5@ zLY<5bO$_Rms_`UIQXK$!KeEz>FB42(dOZs+%4{02v*91n5sZr&NiZw+oO_pby|&=G z|K3;}v@adZ?OkDI5BbdON}nti&cn5PO+=s zn;A&8HO8+;rTHC8xwKYuF56d$c*VxVE%JkNs6m|y|I~e`8zsvVXHdI$Ei<2zLQ!QP z@QX$oXF7Kp@em|$-KG=vJpT0xC_7;%F;S_rdG8UGf>RR~7IC4Bi!Czas}!tF0X&z@PlFN)m8UA!SsWL8<6*wtme>Zah`s^Ku5-m4sZxWUM^w{+&~WWlg@=d!<{Vs!&_1&f|C-FRgi0PK=RORhRB zSFS8_0P#|sR*OqiN-*kTi}1K}VVe5qd@D5A^pZL_g{87_39H%OxGQ|*dsG1KC8P%m zeL9s^RruDP)JEG^leCy_u()Fui-YFZe2>3Px~6y)*P2a*WPNqgd1swDa$U{(411O$ z3Yik9EoiU!#1{noq1W#m)#+P3sbxOSz}(h}>y{7=eyF<#GGl4!{|oeUGHbXPYO=&M zFSMYs5wxwt0O^H7X9T4xAeBse`s!VzF|KQX&gj(tT%fDa%dd3V7-pJ>uoE`jymQyCK<>fxXql4fprk}A1 zvyFCQBk9@TC;Wev-x-&u3qZAX{}-`ZO_z09@Jb%=&?Js?3{-;yUU6db>Kg~2Wg8EU z?Su_48E5dtK6}WTuIO5K{*P%RgqycD;p<~4K2wb#`7SQQzK4vo!<*xU{6%N+tpEaG zcX4i<|Lm*lS9Mc5zG+iYTfaWQD>4@sG_T9xy2~~^N&~Gu6~8_N0{tD*i}U_Fct6m8 zS|bO@fQP8RExtHcPKg)U>K=uoUR?Z)ukg$I;uHY^#hr_ju7!PoJm**V|2sKD)2#9I z92AB6`@N`N^H(0|i{K>JpqfkcIUKD?L$oPEVC-X{m|fbC+={YUyy z(^*Mvo%Uu1zR5!PI)u9gxrIBl$||^tf`82*!gX$?-a!0%gwOt`zf;>$7H7gN=yyo=s7+Ue*@1F^V@+vEaz0}i4fNUoG zjmdX$X3*E#sZH|{SK5Ht*`O~ZMR<2QZVZMmrZDiW|IgJl1;yeXyWcxZY!oy8Hp}07 z)U%?+{N>$m0u0V~@sD=22mJzWoAK6}1;llX^Z9teLX78z8^Cwx@9aN>iosVKPZr16 zXu+i^h0S`pV(N3tmlqU%ECNw-_I02$O2Dv%;}+K!J*^Qy%X1d@KSwA&(pNktcHZ%HYm6ksdHtqT?;G8#rm`CIE7UfW(e+a1PRx#me?0Q_yi0Xfx^odngj$+tD9Acy<`zT7QVYnN_z5Lv_%kgZ_f4B zj*||41vFR8Za@h9lP~s>GR!0Vv+)1SP~&$0M>+ZC{Kx>7&i^y*m#^b`@hdMQW8902 zg~fQYF8h(C>F}UJ&R4gWFQR3Vn1Vt8sng1%o->xF@FeJoR1D~_Lr z7NhDNfk@`)_+RM+j`i_L+99Z zE;<*DV`+bsR97b(UodSBEX<3|_~IlmknFg|jA?4p>;bDlHRi8`<3fwrt|hzGPio;W zv`0MeWM?y6Xt{+5y~BMvs3tBhN}+@8sE(6)y2ydr7ZQbvA9dA-0Y>|S zfokuqIfo@vpuCGoCHTLU9 zHTv`$OdwgyJ&BIS3VQ`?oy|cCv$I9qwcz?^l6snFy?M`yVysHloGckj9bBhll@5!@y+yv6$H@bC;~ zEK=j1!M2gOwn5u75`&yL;iPdI9~(gpuZSvzIUNyLi=d|@M0ms{dz-b zFzfKUoMPBhsg|(Ev!Jo)^7`%8sQ?t)%v9Z8REfr zG^?sz^@@DZ0KWLm`~6u{h0pOt{ro|pBdo)VZ=Q+PK-9E+K98*KB7z>yYYYh%_J6#+ zKiM~^4${>%m>L`#MUIUjU^joSwz|IJ{^W6jLe<@kznPpOn(ASZ1Q3AG)$g>QImrMv zwn0RNzL8Y@em>nK1-Vd-7H9xd)}jw`t{)jka6Vu)(seg9Q$&XB?>oW>Zy)y2pWWHy zu+J4dsQaL3#g`4!EoCtfj~$vX8`j(q*K{o`%`7UY?q~cUl|?wx_DsLb7m|;79q8yr zC#vB(^F~ElDd*ojZw*s;$l_7^DYI&e-Rm?$G#_A7`ArD-7^;j6-xWWTWU;qdEjjc^ zQ}aR>_<6#<+5Vh@X!T>V0gQ{m+o>Z#8zp1!#mez_w{`CdPWl;Pet}FKI48jxnj-_F z!n#o;m`j(?KGL?k*BA<$-WClY=GV)rTwHGZWEzxi+9G}u_dlnz9}cFPZ}|!G6CMW0 zh&2q%bHoK1)I+<+eAa)@PRS)Ej@K6nbTWQ;@<^U8|FdGRn5)D_c2R%BN$1D*%Hr`9 zxR*-~Nqz;D{Ykredp8oP>7HAdkyT#QSYD)bFXWL?g8hrve+EfwJxYH_JEsj7e{kfZ zc_ylEXfX9>Y;mKolunE$0P6+vY(NlKLZ4ZEYDKnrFLME@U?~oJ*A_;MYR|@WZVT4Sh9CT0;1EEVC^fku0#V4du38bL zD*G|xKin>GUt2kl0`ODmXavK#JcI(~*TuF+CBdR*G_D8fKxNX`~H_cNlJFl)2%lLWi?5#>f%+Bt| z3M>bO2`&XixL%d<*MI!8dH>3l_YIsYS7kYB(MfOJ;sH!TrRluR>UyE~hF>+gYZ#@= z!9mqm&k1D}nKLCpWbIDWpU$82)GA=_9lAs zQZWEepLVWTke%5X5o-azthDuh^my;dP+i3n|MDN+h1$ttC~k_#uN+XL@?pto?^m1m zvX{Ki{NA1PK>%T3-UJBG?2}HMo+PG)^19WshjZW%%mg%|qKh&Ibwd0fZG4Og>vlQr zy|Y%|xb^2%Q<0+!yR|=RdlbFVm+nT}XdlJJHE%bnUhFc@NnZEf>|^UNl~@1hDy`<93G)}}qsfXuv08CNAF&dewU&Cg4# ztSfU~N3%WA;4SA1nC9EUdV7_2&P5u5uFG}EGBe|T9f1*PK$Hr*z4r>mCi6vm-fwlH zDI4Oj0ZV%+&^M;BJAN36V#E5YZ+u)X8ZNQwFt8B6za@Wjc3on7Gnf`t5P52I2fa}g zH9F)+MDVM1aeK#^rc?;@SGP}4#qr$zYTj`3x>3TH_)UHuplk$EbG`&OmX^JK8uDF5 z&!?MT>H6r^^9??qE8`5}iwV8ZT`wR?j)%k45Z_T$#a%d6jR7i}(;U03pT z{{SR}y(#7)&I)_)WM8w{ZfNw41c-1Rk0x=_p@1x}_`KY~MDop%tKE#m$XDrufvXwx zBQ0ZkH9kUY*&)4fN07aXi>~~)vgJQk6WFT8JhhvT>kc>b^;5pwPk+#NI6g&R1`a9t z(=hpyAqAdQWwV6|Qsust!ao76ygdR7F{(2|hx}Y{^iwh)8LgIpl>po1%G0y&b_zgA z1d`%x>3Zm$<+{sHv|-?=Xv)Ut0~3aNm9~iV5DviI>_cn-<@-JrwPLGP|F+-CWU}ba zv|m9Zs#g*+CR)O#(j|Tb78p+Mk&{)uX__ZuILLooY2T(b-N{vx%kZ~Fvhkkth`_nZlxs{1`}{t;r%7oMZvg z*8@PHH)UGtZfy00PdHaV zmpi6oOAxf@jzX*CiB7%QjMm(BH;cx{j8B#bCvJ>-uNR2a)C(-O53>{*-g<7?zAl4d zwSc>@O61*C&oO$N_s?IDf>)u`br35~9*3mzDym=b%IDahGtb~NVivx-Yl%Pn{z%^| zFCJsKU9tb)E+Wvl8|d--5kRQ zz{QCl7a!LBAc^N;U=bCWk=IHik=i#9@idZJHZK*D`FT1aaGUebvK=Q22~!XCLv+JsqB0NGJk_ys)o|GpuX%$3+ZTk;%Rzl_>_F)UqYL zzhYk~A}}n<`pA5WMlgF7z_-54S(+@%JNSesCy5M(LClLgFtz=hdK9sh1niWpm0MCt%$zWlrVV%S;S1*X5Qu zKGQCzrhi&RJu63CHBFktw?k@>w(_ETy%#Aj;{!+|`5! zr!j?mDv)=O^UKtbN*ZaI{HXG<)5zs#Opn>polQ!`Am2y;oGZ;i>j|5=o$5lay+YUa zpN6YiGG76wxB$qt-n`V!MKntxr9W?%JY@D7#?Z%y<8jcPafiy=oPAlhHA9`#+ebWI zwhXpti&JM@RFofnGvE?q8Jg{d=Y<$0r=}%E#(g7H=oya@3N#A)gS6OMFN9UN9SIk@ z1y8x;^Tv^ce}XQo7t@tSrDwTgMYZTH?Kgxf8(!CsY(99Gb3fq9qohKZaqQd0?EE{> zj~;tZoXI$U$w%W0aRJEqhd5Q7h?Uj0;RYyb9`Db2F7Ph$VZnVxWD!l*+`enf`j}ps z6`)OByES0_*?DwHcFiY9^|DLmgxN{N*I8hH-Wo$MF;}L5lkR60{ojbK9C zpx#TTg~T))8|-xfF>M*InoD%dBVcdkutHDEh$yuAV=qCiT+hZg65^S&)&;BsqwrUMm2 z_XkNleyKBJ(680FSo?_;;^P}P+#8_2Z+yxJmPEWqj>YOsW!t#j(8H>rj1{)b{E|a z$v3B^-&^WcQV73TvwGJk5lCMv;~bQ(g^Gfnvd#I!# ziN`!+YF zr zxy?EyJ8~JZry@zr#?|%<8SffXj5xT|whJL78n!0Q=mo}# z!!~DxQI>ORWo2Egk|9IYIZop1s8Oxg9$WB)q0~)3BPlb_*S~+iLmzOz)ngerz2)Ru z{E3b!zLme>(CnM1#d{y1`2a?@@n$OheaK80TzO7MBB=P zz(zp9`_gqJSYi2EhKOv&rihg#O~^lMv}5JL*^Tjc3I}oq_2u`2sh$hhC*F>!6u3jp z32?r}B$m{0y@KF?E(4l_n;)s}{&A`jBo(+_88t`|-IC+YqZT~1-#Lo&Qk&{QC*4ee zyo>~v_hdxN(S|Bshk_PHWa6H5;v>Obi8tVf2&&3?>K_{CDBHBkaexr=DY{HaAnQ8e zhXgKvey1xI>UOWc zP`EGEh=j0;?zOb4iL2F9{#T%%WCUdKf3x@-)F9{)35ahWewU6TJ+U%geonb*_;u^+ z-%ql%?QOei(0Jp&-huUP$tHj*qR#_wV>!LLR`Mr7Y5*~2NBOJK#ctwmqY237 z=kxdNNTcaN(TXu^H#?2c8qo3p+KbFVH;IB>BUcX;q;MG)Og<7?CCa2PbX2J+h{6d=4AKr4Q%ps9GN~G$;$eQ{K0E|4d2ANiC>%7P1;0(bS!I zSSc@9NfOD!Ak42!q?!*mMM$P?)-S{zd>YUXoEk@JPH0wOe%MO+*B|09*M z;UP{vsZzVm7zk5E5R1uBr-ae8qE{+r10Z#>9K$QCb~aTOGAq}QTx`g)qc(yw9Q3S- zDxh(6%jp+2rhXY(b?V9U~gPMa(vpi zNS9)^aCB{nHju{=$7}LsZ13ng&bfhS^XT?oX2nt0OxK11RxvO4^qBV{E{Y~J!OB+9 z`2T7Vva2L$_u+95YAFEn6;B1q5goI9Gb%peo75_8u8vHec`omXd6$i$(in$RP+A;!|jbA{?FWFP+H{vFdjQhFAv<97B(KI^lBA>7o}*Q^9) zlsi9KJn`qaA5u9uV?7B+e?m*#S)!|V)2p*@aFWu9TcEO);65OHY#0`t=5y}&{LPzh z-Q(7xvF-OS4S2M&l=^?UM)_(Akfkl!2fOoHoP=8A#MR#P^z5vfns;Q23K_B9d(7B< zS22T)?g%Vus*Q@Sfsy-lq0wT{2Zq8L_RQJJvUg0nMQIJkScz9xQBvq4no&?uS<4{8 zN{zc7&vFeoZ^a~Qf#78_lr`T5(B`6X4t|1UL`hQn>hSXS?|tEbHGenJY5o~o zc7w}Qb6+cnqnr4^^)oP@K~!WnR9u7KePXkRKeqVs@oqfiHQ;Nl+*yScQQz*m%GqQ$ zWs?a%@_o?GSPf1gdBpVnV;_7yj?~oLHvTsps7=lZKqsGVF&L38Pj08{} z52K*H7Hju4b7T!0Q|JeEE^&XY1`^Cg&@g}Ju-ZD+e^IEnc!qr~`|2~@N*umzl+9{GEE8S$w_N}u3jHT4o9I*3Hz%9o5+ge(J zZZ1GHLX+^`JslXhQG{F9n)q$41X6!=@g6HAftzCz{1NlXE%)QE#`PZ`3Nz-)T6pl@ zTR}&1EzR~am83YLXV!u*si5-+2L(~hiX(OD#?Eo(59)7*v zqE=M7_DuexyrM#9w`%B+%Bn^LkJ!QJS>|3i_36Urd>hb~x;18;%iPecW%WFCe><{q zbhwkM7|_Videv-1jjja#%nHs6R9|uC^x`VCIZEA)(g!6Kfp4bsw_6tJl|DvL%qich z*l>4Nlg?4P`XQ*T?J7&h#7OhiOhF%XUvj3!jS$-cD$$9$>x7^WvdczJKe8vc%TK;t z5^~m}^KqK-!-8RB88_8An|8(vsfu6O^8ckyZVpXAysB5~8P&R19uGUBVBt7kAUf(y zOvI}JfTE@DTDTNs(6q^2!{dNTOFc#Mbn}xBNWd-Z?jxMLJ503gKA|pD%?WEWNINqE>0E z#~ylmEH^{bzaAm|-txSnHwj5hoUpH3V{V-;?(Cm9A9m>XeaI2d1|7M+wG+*()|Vvc zQ0OIo++4j`H(y`{AAdN4m_m=kr#4odi6UM;>qQ=)fjf;5p-JL?VD7(^A2{`<*z7{R zN!ZfteQEP%spRTof|){ASm7b9vXaUL;K0OEsGiM6bDx4A483~8Vtey zB42zL3Hp5GrpH8h0FC$!2YXdz<-h0%?|UUrfHXdTP=|o+U$EZ)Va9LPU~SiRTlWn$ zSnTVUYqB3o3=jKG%Yv5DwJ0(*?)$(bZxSrWWJ?D(-Z;a#yYZ$%!d+a{#|lX1h&?j# zq{J2GA>fi@an(Edl-2-*+>WbYncqvy3n8TKjEF-1vhzp&&MEb9_{N*sj&E>{yfD6h zfC}FCK+I;QP()uF#hsyk%*@OvL^Fj4t`PLx^f|19jkP6B6a-Beo{&(MiutvjPS)I< z(*Zh3qt&5WH$@v@iemA_U@q<;Xb#1trU`i|7j0-8M0A)qX=J3OrCr3*dF5!Tcgwb5 z4a%E5KvID<=g|OK8XwnqC#X(xi@x(cl>lp88tk)tPF8J@<74RP; z9=DOamg={&l|71sL=cz&k>pxD4rnNl@#ZB0tJZ;CRsgl|4?bf5_CT1O-}N7SKu_id z09hPU-(>TT#lZ!D!(aIO%t^fBv~T|yf;@Lu z@&B;X1+?^9biV9=!{N!qxW&m^TwFYWBSz?vSBm`KT-Fa!5|cfpiTVeV5x@x;lXjY- z?NW@1?5W{x6Q8xUYNo+yLIS2)M}-qM7#g(63cSZLTZA>dwn4%>OX*{5eesw~<}J^C zxg!9d2%t+|qt<=bci>&Xe+Zm$&Zp*3HqN8F&5CMDKqBwM3O+BOl#23S&s^!e3IDre;34WG_vORVd5%KT9e>JGAt~mH!A+Xfe`-)2$@fX;+VxUw1=Nj3(((7yG+j9lr#vjW{we;x zzI(VO_jR%U3K@P@0%)S-=SFY3f|UNbq{_o^6Owl=7a*2Q-&RxjA2b7hKSA~-z-~tS zVt)=fK*m_R)73(vff5~Nsguk809l{M(%Cc|d$Ow4y~yDm6V%RBP}G6LekkN={*%KO zj~N1Lo@S>>JME8e0Fyj9!Vryiil0Hf(F;5>8O;FbUI8qbjTfAKh3agpb2#Sv0Xg^) z^e-5ExeB26w8EUu=~nmMN%33LfyLI#0uXV0Dg*B?FZ7GYFD;pHlf(VcTdxEP{-IlK z?9rC>4_nix?y531WPYQ>%sc~^cf;j@&xlCq27uIMyuAMxm4?EDb3v2#N3&5p5ieZ* zMYy?D|5jAG=*k6EEpJv`b{~Ef;LEQArwAOHmtkXBi!!H20P!5Oaq_% zG;d#L{|6c?%%Hyugd-o&39pLR@CARy?W}+maE-$Yj565rAky~P{}(ZFtiogaJ~lNq zttNU#`~bpjDIA*r%1ZIQc~;;kOider2D0idcCKUyUNFpUM`kjsrS1ciDQ$S12Jrm5 zMT`>q;I+96XE7kh%ENH^WU|yO0P}W_aq>U(nTh8yk;%jI<7qlvvmB9lK;_-wh~t`@ zNsS43&^GRA5IMF6U};MzkC*p<>!{~}xd4g{_ps&@oa8k*<>#+< zaZTB#0QO*Q*4V!rJ3+SGWFi2x)k7FD{uOh-aP@gjPD#&zTAkbgY8dwnuHXgaiNg+g zAuUFLosk>U;{C4%(e?juW>n7$04d#gL5%+Qdk8#!6&Wvanps>I)c(HZgpfWn?)ATI zbr+{iICntdSaUx~Y^iei&f~8Eb60Y%C_?@-cI=|S~bFLqT6T2p2RkZb!jQGp__sDUh%_pkq^3#ip znOO-hCRL;JM$VV&)9^wXf1OF38BPn}V-wO{;GWys>IVuy{qOJ>#ueQ#5g8ffPZcsf zpfHR{&A6@pf3ajk5%^0;$!HE~4dd!{|EH?(#Ku*|>Hr{R{Nu{sUkJsnA1(CA0Ht_f zcyT!q4=Ctc1SAira2s#3M2daWBx8?7hnsg57Us9cZ}5S{G*6*mv%Vt%kno3M^*d#>o|y>mS`}9lcSeKL{;6^c zKXE0ltcM0nC2(3&LIMf7h~&%vEf}HQ2x!od&J|#XpL)jhcR@Z^#jTm4?3v%DTR*ad zE8zHfW)l8Jcs=u=QpiK&Z>KNXRw1nE4vcyHKe2KDJJMJQ0^n|+JbG>$`cENScnK&A zTo6F6#o-*7z~L{Pejxu9T3(Qw+_raK(}%~n3c+?FxB*<({{DDDzODFm;_E(|F~CNO zAB$$bH?Z7&+R&cPe<H6FJlt}=w65-0?g5JtT;POr6LX%I0oXdBCr z{@C0g9WexZy*|;rdQ|xWNW!3Z-WMB&5G}_Dyx>;(GN>)|qI{hLa8` zK;Y&Ru+*~EcnG%5@Sf@z*hK|^y~b@#n_ff;il2^O`Kq4=EyV6)93*^xuZ7Q36PBxV z#QQZKq(}5*wr5~AI3vVk9*S*;W!iM7OrLvY@i_(TkU?A$Y@wL7OhHM^+)OF5^maZ^ zHoL)BD3Bq$#^d@@2-riotZ=v?uFGzD$UBXn|&*p@oB4RAM|=n^T&r~sKv=D+%3BDWUZI7#kGuRk3xEs z!*=EW2qz-F&m%Y=gx4<3xGQ_ndf8YNNpCviw)A9tqAzcymBYKEnZywQAb26H98(8^ zI6o5}0B?mf!l+{_(emiy+OLQ7T1FCc+~LLJK^$AAT85VS#+%Xpo!mO4TJ9BsptfbJ zpxYeSiSi4ez@doRv4ohq;~7o28^XClXG^pNSqN4IDbm7`tF{N}k#Bd#B8P@Wy2rH*^>wUl1gl;2DrZ0I z6xRwZvI{OoB^Knfu1@sCAuSbn-bhuR{ywf5uBe%@)=M53*D!cBVrXS!D{y2(KOb+D z5D!sT@excRJ!=@v3RN*Gq(jDP8t9MAmA;d}%n41c*sNkO?U#}$l30ho>FBF$m~-;| zsXt0dA&sAmE5hX`Gy^6002^P~2h--e^@IP!r8(;J>mr!A*=pHWubwK7at6LD@jS%o zk?kLbt?LFac1BA^EnP*M7$N;-S7|xvXsz#_FAWP1vMMv3xf2(jqICCuaXb5HmAlIP zS@9BK-*6?{$_&o7kPVB>h)g7eVir%JNH#mV^8G)_1@>@MQvCODn%SrjLF{p{j;xBR zbLSBfbCOu{4yEaHdJMvD3%&d9lI1$pg}-UaY#B%j&bW2*>t^i_lF7#L)67jExU{jKodk@-`EKZqu!( z`Y463#UlN&C#vz~V$;6yE=!`9xHb&?n!X-G)Te*HM(s~R4DBkIesA&JhH_whH=(^z zWGD?eEtDMgbOB8R*3@rDb|rWY1d@aOdas?lwK~a6a}L!!>f>}clSa)D8n4Ipi|uW7 zSC~7Tw{tq2rVT1==#tuS1<6v9zm(>t}@mIFVoNixvF<_wSByVbypFSbh|lFw*UEs}$69 zR+u)kPnodQ8Evl@Ud*5Ub1e`m(xnYX*pX(=zY^~a?9GgpdPVMcpufhewngm$1zBXZ z@lu;Z@@x;AhzdjWmkJOumA+VTs^{$1S$n84@fXSvq-WtuQT*QK>88=NAwSCR9wWE9 zyxc6?ksmfi5tcv9x;X4Co+1jy`~rY`xf=G933Kz5{*{Mm{bFaw^;+`>@5?kd&gDk> z{2HP>+l(Z!n{C$7gn-L#&^pKR~Tihay4 ziOB+cF^oU*JzojufK64Xiwg=zwFm$C+H2vde=VApBTy+@b`oBAr*mwEshf~rT3KNP{C?qm_fM?PiEw>=E;~OtK?aSQl8GFcRyqeako`OrEkd`!h26J zuC9mJXep@f-s2DMSeeHkm?Vx`2<;m;%7hP1gxOVG@B41z@|~L3)v+fSBRG1f84(Pg zimJi9=f@sVIgtmJT4%(!u|V}3MsHN`2~IF@^T;i(}0ppqtVbKEY;C-EP=cNBx*+Itum!Qwrpf!~qDJ(Jr_H za&_-1YmrlaPRV>XGTi?^l(3JIfKL;U79rJhD^c@;T^Q+eyLoTtDQDRQ_S6ir-C)QrKlH||U!&hZgzxEE7-4M{zeRZ8>1lDf3%kKo zy59)RDPkf|&}SJjZTL`9hxS>g=7dy-axw>=#oIW!9wxVZ>aBGNWOV;JpzKSh6D1RK zc9=(cdL2Q0-!91aGN&ygcO25W37MUbfZ$yF7`Ic?-qJ}Y=z|E~GhxFA0$$-0?0Bt>O|O~MaqymTzA>&=Vof5N}_#j(s8o53qjCr^>D(oUGOJoAY&3bQ!H%6p-`i89cs zRB{2&o`K!Ah$d`D3g|p0QuDa)4jrN2PAT!N`^vvjUp|nC%T+t zOmoyXKh~uyDM?K7W;bSux!#(*dLB0Ry&64VbcQ`WLUY=|Aj!km#Pf=>QEAw%00*=s z_Q(BbFX$D71xE6^hcSzN0r0bt zq-pH5d04dWC@?_Dgt+*nTMS}aQ0a9cRm`Cb#aOZ zlz>l_B0_+J_eJIMN2{K!>f(H(dxSw}M0#s$vcaA)uUH23)1&4xGJn3SyDKPvAh%{m zo@OI!VJkW~mtuYsoP5tS|5Ie<#?bP#5RdFo_ODo5j(RSuQe$g=Et*N4{n;iFuu)XK zn*_EmHFmnmy~Xv^uJ@V*c?&>6mZg}L>|)E9=QcrzJt^MGjh=1sqHS!?1ji0H%*W=q zorzWOI4vf2YjY^_=2K_a1q5b!x#@Vw{H$|p+IiI$SZxt8L9AUSLPH!YN7-JZmz6xs z6BIP;fH@za3hpbb-A@5ag7{8<+yWm>d0`+=jV+6&Pim~(Rcnu{9Q>TMR%XZ&x33nF zJP|q@cQ`-M2auDx6RY7s>30MT3q+ffn1=pv)MFbgBaz{hxY8zjV)i;=LZmyQVcSv~ zpkV$|wf_D?f`a5wGQS-5{LVaG6Rl`qvC))DKV~bkf{sIpu0aereSBgi=~2h9FH51gf(@T}7s3V5F2ZHuoUkBtSyb_NI!8>gUApk%Gc! zm9ZU(tNP>m?BbN>EoFo{>;j<;mQP(D_#P+0y-h_ou7sL=3=@1KyKcD3oiyH4wIdON zD4TEWx9@6rY@eaFB_gsGJ~C3dp8!eGP8wp0x$XlrotI+1T*HN@dcT@>8dD8#@Z9e) z_^8mU6&rg1u~bs~yukNHivPNix1EiR;r71r?{T+}hHBJ*7L3od z|FrZLWV*%^7IsdacwKILaH+%Lq+_e$_i4q%c3ycp)4{JKhmAoaSuQq=_6a&F?)6K7! zBVnAE2-LsUdpRhbhLGGQS0S7yH2U)r5-7xCyA?TUNJ`+(Ab~Nsi$&ig!_YqmzRjr+ zkqPjHR-5)pb@r*~TclOQi#wzn85qMef=KQj506&%2d;>p3qdiwu(meu2FwvdP;A6@3ALReUIPD zi#HrQ>yDG0+jCt$Tz_x%%yTQ2OOSq-=OBxP`A*8mPmPh9Cb>_mvuuCn)|RIwQonv= zq6ywdh> zbvIW{GhH|2XT`(MX<;OkJq0W{g6_(0Ih`5j*`5quM{ZjJg#3^6E_E1Yg%i8{U1n%Gk_*{0VCGp4|fO-YM zh2(NZS}uN3%u>w^OZprbR+$$ZB=XvV*{L#!y;hZ1FHuKIYHzBM!6?V(fzqQn-Xu(G z>|2e|!|QrcsVp}fjOWSu2gT4@3paKA(neiZel^s5APR}(_9lB~d^hXtYJ4pAdxXyG z9z8FaEeW+}64ClRuV#=xp1FzZf%GR0Uv#VTygOLC0f@8YBZ?>c{_e3P-9Q+!L? zNGytsjiVkUd(SZZ=^uwB22OC*=Hu6{V{OqWtDhFaS*%$>mc15dAq`C7dp3f3uM7@b zb2_kWS14u`gTS3^z`o%_~$o7F}!>9+#n$) z)xFx9ZnmeCPOZ$AlE8q3UxAA+0XvqX6Rk<`JcN@pKKND`tPe;GxtVT!>xwQg&+76N zjZ+e{MWZd-%&uRb1C`tf%#bq7220Cxc?^V-tPbvdF#i+kTEO^OJ7qX!HH!3ecg#F> zXu&23?Bd>@PrP7m!1Q5{(Vhn$=benJn$!Mqq-dTYKDjq%-hnT_2;WkB*gd8LTh-V;1h z*u`}7Ka6nPr#nD00aOfhcApS%`^ppRHvXZ!%ewdnT6Cumt8&CV*=nIP;8>rMJdBlb zP$gVZy<4`a9(^=@gW{<7QFl#=;Cu^m+M9{v)bZqLdgRwU&o|Vji(%8+#UfwQu`e$F zXwD(Ii`%z;o69H07EM3Bd7JN`Mm;A%0u$~iTRg$e)(QHerh7LG>(lrRJyra<`U`-w zupE`*6s;MkTG4Ivb(;Hqn6D9GD zEOw=NdNA=w3QKbrXA)?Sr*XpUsj(!DvmKzy$*`D1$0uH#^WTd zdHpTt-m7XaZ>6rBpSz62FB%Pati7A1T^bS*1c96wwThi%q}9K99nmRiAVq3yH%9kj z*i_1%SUn?JZJ~5%Osg_XJ{*bMx9!C$4r`26g5vLOX*OWB-Q|bls`^6625R4;lP@2-0or-T%a}M0M#fb!O1YkD_ZUD|~Q!xDogq9+UBCYXE#2cXW9Szi2q9 z_~WRP=A<{1VjG0;69zD?ULt@DVg6B*uqRx~w9P6Y#DnV5&H?EOAUPGyo&S=hx5q4( zmW?h82H<9;rBpGUx^Z+u`WJwC`T&rLnK}0SF5K!}Y`&JXQRaJ@jI9dNuxjuHyCn!~ z-IpBClc2J?M*^eqKDDS&+5t!|@*KIXt*Gc^w8rw(v6Xou2LR0DytG+G1m37&u%C6O zf!m2rhE$d!;x?D7kLO;r3_0BfbC_86H0<<5fTr{HKk)&_98-2HsnUFSUpV!ry?5cJ z#m3i%Qg*epazPnk$?Q}Bu<;6VuuKA%(KNZ`Q^+}DTB}rM*_%rdG@bqL8dk5anXypq(kQ&o^l8K z6*~U+ci#e6$hv4m46rsHj_#P^2#)?vFw&f!FCAq3Sd2K=wQkqsRgx?O4QV+d=MY7L zY#1TPC-_Y?LzQd>5I}xARYpElqrhLEZN8vN*tVW|N$UdXE{W^w_q{?oL%G5h-!usZ zz_{GgiH+{06Yg)z5tuynXq>9*CXgYZ~+pq z3>ZoK?3f}?WXsB_(}Q)t3iTBm(K49r%e_gNXwSj}iYw$z(&1?N#N2A}lDSs5F3yB5X8hP>fDXviEg z_ezqjBlcuU1Vxm@5jHlf1SppGMym2?e*-7@9Em8CX~V{N6E`IrDoUT&Y*NqalA#cO zi~SvzJwm314z=>(vE(HxP^oi#jWaKGy{4r(a9>qW3si)zmKX&8rLh+Nu{nXcVM^wT9` z)W<6ngewBPRe8(r^o=GExQA%!N zu7i2Auf0t_zqIha|DuiiC6C4UVxhOzwrG|1PMaung>mI4o(73*B zIPh$5C8RxPZdi!`yLlO)N~NdS$)xd7j^*XDZJW$C;CTHF@`fJD!W5|o(H zn*;4u|Hp5tiQHzxh5=(I7ttRqpaGF#*x`-D_s8F`KP6^SmVLm+z1AzI9e%tvy+&cJ z%nW67tFs=HC!w=!E$znUVxBCfE=zL>MdDug!aQy3`-1(5|He)J*OHU}wd8YBDR5#l z%**ZYHg31Kre-@pSa&*oLbd-wm;L2acfEza4_O-hw*l8oNbARZy?|_$rKKzt^9(*K z+0mk#K`JM?ZdP`;JCe)H3W8-xzimZb{k?=+Iz0A^WtPFcCfa?H$Cs_JY{r{&->h%t zBSLr@wuTgQk%pf@W6~igSW=^OW#UdmPU`OYmVE(A3?3uR>a4<=BIe+`%HraA3)Bk7*R zeItxdo`ZrA4(@AbefKBNv3IV7 zm9@Uc^-U%ys!GPo&Z&w?PR0(;UH*WGK9{|=uy60h;SgwvFUc3~3{G2W@+LjV+J{0` z+}@t^lMO((Wo1O;Gnj-Rubr7_Z$--TOfDO{v1C8Qw+-%+zFIZY^M;NS9wJ#zUzVvT z-DWRv{%4d<#wMw1no0DKXp3R}0i|5Mmrq^W{Nch4lp^(iKde0gFiL{Fam<{9<%eeTX z$o&ln2y9eP&T5PX5fPk&>jg{gt`fwySom%bmsl*n?EOG9`b@UT&E#U-xcQl)?JoSo zNXT+zQkAB!00%H6!TG9^l%nL*5k;G8gBwureX@g|fx3}pWz2XhSyGgTizUZra=Dl7 z7}L?0mcSX4K40JaMRjv)E5E@I4vQ-DqlFq84&%u^;j;}XyH{^mErTDi#+LiSK|Qbl zj`vFoLy$??0C4U4Gdo3ccbmd!`!c};=2_4m$vbWuU?ItqPfVcN1AG-(yaZfGv$j~& zmkdJxZCPu&g?#&?nl29oBu|E7IAToh?^J&fZfeM(B+aDMYn*D@7hM?HG^L}qSt*Tm zQ@UcHl?>9-T`9oF1xOt*iO;>87$3{BP+bi z$J*~p$*JC?#S!D>A^Pr)Vl-y3vq3wf-u?`=qU98%))L6e`Lj zdmn3e1UsJiE6Y51aEfgNOc@ICKH*&8L8MNo-*qc|c1laGji(M9{8HZo-d>?jUIr`u zo>odW&+IdN&YhSFwTh;m|GG-c>t$oI0J^5jg3~Hw>reSyKm;*!? zPf+6Ym|wsD^r#0QEwgT*1msYUrN5I@biElSEWJr@RnM8Xas6zZD z=wIS`JKsdzak}=pHV|iKvf`{d=8t;YXy=g!Wtk&q7*?*P##KQ)eC1ee*~T!J!#7Z% zgr#QF6DYn&pXSl%Xx%oOO{KH>U-`Be)nZl zoRPOU6sEQlB6~vw1PGYEhL%yIxR3%8cKjPFmJ1CIq_Z@%6EL&MVFA@hDGkqg8ole| zfa5ZB6g!=UD0=mR;kF+I8MEg0rR-rwoT)e_(*U`n+z7Muaa8qdb@1e;#)At?MZmI1iL%BbQL8f(AgRfI} z2S-hH*Tn*9kYit*l_ z3#FN@h|=Z*L#7j`ZYiu3f#u0qhj_lM$)x3fJ_ElJvtVJ(P8c}wfN3G??3O*f_7nm; zmEw&FpNc&(3~?&*>D`QkWTMtvhD)`NCS!iXHbGpZ#opwP|m!amk{! zF}gpJw0Llz_u>g$_U!YBN0X#iwDeP>xT@9Ja;cR_Lt0?Cumwq{!+cxzt&+uZIhhq+ zea#ByeZ_-JMXfp_;3sur=7E#7R0ICc4Of_`6=8VTpK4`mG(Md9+Fcgn7B-&BWHxFO zXtelsFXiw(q_}c3_oL}k`jtT``QCr6gFr>tz7v1d@h z;Z5h@FuI1mE(Yd52ni=KN z$S`ZnrDrvNiL&GPe{psUA0K}*qb+}n7!S?nB#QmDkI%LrbHdL|PwR5DgJ0%;1k$!B zjus*k47D2LskD^8OL!!+N_Gyxq7 zbmM@v#Oym45N5_DMiHl9p zJO0t(SZkn`li!??I~2yyR0=NU&c8Gt50%QKo7WGlUR2SJ7*Q>sn|m#ln0JtaVRvd< z6+IAS&~+g3*a_F9`Jke%OHKijo^vhLbkc)eclVgad|(3liA%{9s|EUjYy^zb_Yt80 z@HP@3fou#`sriMQTitcarcvcdB>?a5`;ZVgjW=RvP`r(~&epxDf5Pf6_GAWZpKOAxfSpdVaBSbQ&f3;`^F06}6s< z(QF8g8!+PM${miIk5mrtI1`^F7EKl8a3NynarU2O>rexjwrKn%3%$2tYS<~5gUOA4 z^LTxxjIs9bWnv`%^)#PIYheMz)qBvru7{i^kXtC@oVPC z8>-IhqW5PPvD=Jm2>$8L9VAQmVx?vT|+S`Pde*=NS@Md9VGD8* zj|S)_wOs~8y?wxA%wg#&_^Ol0`#KhEreCOElf(d^S0` zYymJHKt_=C=vJ4287$-)*QO%vjRq(VN69S1<&W5UIrc_Ma$pUO)z7g%Ju?OVDZfR; zUmRto=BgMNu5PFsx`MPZ_^Oy4`cHPE=l=Q*BH5>)sCiDbT7FbmY3)1Q$`t70OPH~u zDsLPP+7luv(vR{to72f#kMzBnW-$JiZ6D^z(fKCHBCgCn03H1?Zz%KxORe_m%EK%( z{y3^ZoSxPEtN00S6}1B~{MI;Nz*c8NDO@mo%nWgC)MT*UPP>JEaMeSKLnU?_Rt z@vt=g8vZ_#*H`?lXnZP8k#Nun#e+?CWNXmkJIYDQYv1Tq+;iz8sd<&p`v(2Ee33d7 z04&X7`FEZ_!>(8ZkyYiQ(xQi3hhL|H79v*jtNsl$A+?q|XW!n1$E|l>PwHO!pR+XY z+-E0;7PJquMj|h0_aM0yP3Lf2|F~e2a^}ZCN3h4-2yKcMVlWJ9a*05G?Oq=bW#6D; z`DcUd>}nSI$|a<)P5Or@*xSP9mpq?3+i$}+`%J7aMc}4l{7igzbwg7#X~&W6e`lE- zr3TZO`%LAxH*4~UuQc&2;1P8vRK;-iN!X4Y*FJ3=9LG|mC}gn(&kBD4tLHd$0~c?B;h|Kae%(&SRod%#z! zp$+VQGr%WmVl)F+gfM5)PkNwBL*rAujx3+&n}u)zsoKF7sY0f74-zTrq$0;$){`Mu zCcs`6hCxIlfr-qCy#;5Hq0w+pxD7pEx{ySuL7u47(|eacSQ}{7(+M0+zj@Jbj$gRPLB^REa1n1RoLC3jNXNBXbJK_n&AXV#O3$EQKc`) zP!+IC>6_b2Yk%YAV44}i4{9Jm7w={uDNo0mo_SKPN{~*( zJR=MY-fpp~EQ3&6d}4Q_#RX>Hmf%^*LMUTuThnvNMJjR;Uw(|@Q*`2KicwGfISw)F z{*NvCKw3#gqcS*b70ZiwR*Nroa$`XXd?y>oXhEnNV|`|JcL@d$hlZ`2?3dAg_KmF` zRbO>9i>IQ;kv2g45EuC*bO*-zd`3c8IFuc5}< z25mI%t&I%tVp@|q!V&4pBY0de6NEXU!$WkbwPjw@Yfj}P@g{5Ia>q0#BN(5YQi>e{ z+afZJ!EVy}^wbu7waC?S&UJ7vM2W#8l8a1|Wd+g+atq;uQPVt+Xr zFcfY##Omg`9zuXxD_;CjFHd(dxZM5oVm|AseDFgP*wN|807aEFhE6BC1I_1(r!=e(tI&P)H2e7@t^>#r}xit7>5fNfDutFM#K($}mYb-j-?z zG~%yYRd>cTzfLu?>Q|LL!yG4Eh-BOfRqH;B^?r*aAY$v?5_kA<>^HNr#%cuWjj!Dt$vhk(Vr`X2Cy9NpG%tRI)jqC;&|Ma5aIZ>;S7#2d5MypEh zUx3;-=JP%kH<1zkL-|{93DtgL;n>?@+WSppXMT1t5Zvtk6sMKyZF8;lbibw~=)ceI ztH1X-94VgMZt2RSe?l$RS~_A*W%usI<&SCFW)|QV`?J~9iQj?L<@!184VZw$1C zo#pwsnfAeS35Kc~L<)uD?7W@9Y(#?*oleu)dr}e!20%dCY$St4wCxM9s}$QYSQLnI zSW@lnWvpEzPq0lyYn@q8A{{d}FJy-EhU#}0M0VV6BGT!5Dq-Srr*nVND1{t$%p%;`JPFKROimc9t|MLU_8~7sg;M29qGw;nv>+0Ct8Ps z=%4oqJTUWt3J^fgQ$X4@p!Csqg!7X9BXl^%ESWJflgW6i9k<6h@pySSVf=I+kiz7=%sd)lX$fMG6;P<4hm`^s z;5|ZC`c+o$Uerh=KTPb=9`eRuK8VoUo@ z^hX67Ml4ZQ6y9f74=w0ms#@#(tH(-L({^uCpJEH@nWJ=BUf5=Bk(BFC@L(^-q=M{= z;0oygRMFqUSjm#-T^L`9h17jX$PO(_%HubhE8j5>yijldz8DUg%=%{BBD5_Heon+= z@t!&LcQ{2?A!Jclf7^EjM`vPbk#TvVZ44^hHGya?R9VKF<4Rm?rcl4NG<1Twu)<1O zr(MaTM+fo_GsrAt>gfxDXy&D9`OQtyfRS3AR5u!}+8TYHl{@EG1iP#Ch zQ4Udt25@GDK8uX z{{I6%H1aA5Qp6(EqY!V`zMxS+v(=2Ramss^>Qs5=uu|cfJFEnGRY7UR;dl8vjCWZS zf9SI)(4|!je-_a9jWRO(SsjO`m(c7~iRQ__gW42f!Ax5a{jf_=a0 z<_cf`iK7xwY1N<)8{nAX40Zd{4^JY0T|kjI``jK`zV3@>Hd(@b6l0g|1M0MwT{G5u zpKJTU4KAm^JFYyZJs+lSpFKs%yjij~i?U=#Z*;bN2TnPooQ+wW4ZA0UD7$NaYt9B1 zJxQI~zY+7FNIz%uvyOs!mav8+)#hNI+pmN$#;nW|iLb2@yVN@zlVkk)u36i^S$>W$ zYqmT#V`%gfZP{OMAPFs*?hCn}#m&Q}qqn+E4gc}FT~Jdhr1f9;zKgvh?RI49rEyg@ zD8R{-V~NG-a~2-A)JRRhFQKeX#`R1tJjLu=Fy!EX{G*V%=pGmR$&eALel* zJMWyZr+r$WpC&W&9CkDA>S`xR5-q6-j;0fzVS4VDLb*Qy3JTmTevRTmc9 zV8KKTWdpg>#mbcY$UH!?qPP^c8pd#$X}#Jpr{~r3*2v`T;~YUn@YVszOJv96qXxC{ zciP-^3CR3W(c+lye|S6QVHU>G$&op$Q^=v~-@nlciC1@vI}2&qxL@Z}We#@p_x+D<3;jxUT z5FA|$+HJM;9E%@65ex=@fgcnSc(c;j*j`93d~R?XYf9eph8t$_ee*D$zA|D$#7JRl z(WMU9%&J{ihJ{C77*!s|!mR5}()E)GkN#X;jP4d5j+BZ|cvET7v1e_~woW64h^)*~ zRLqzd3qY~@hX1cI3{8h|kt9<_K>Z|T-aasR>NU_ieVfNrLHXbok=ii4^>J33e|-cJ zs;6C1&;FAjI(?RabLr?nrg4S8JBkotN$h%%FvrPGHNKbi0gdH`s^W*IK|ZMf;JLct zR{idvI*-&~C2V!@bEWEScon=^co^Rgmw65i=9dlBtw+ z?Kxfb`ZurL4W8q(`nc2Z+`+eUwS1g}m8HfK&O&MzX*Ide(%k7wR{xY-;d9RatyD4d ziZ;9&-cO@)6J&?@3=g*lXLnU=9bS&E_H)ot#pV$T=oU<}v3h@98WGZJJ>#OBhsLez z-sn>_X!mz$-^9pWG|}jI(9e&0J?ApRD^RV=5b9zkHcOKsi=_v6|0W4c73=O*b4W@8 zH5*(A1=ThSIZYw?)KZxYxq^3k@LJC8Yj^tD_a&M4k?6G2(&8j$CS6?`ti0|Jj?)a| zBF9U;{Wr{LZDuDkWlu&Cl^%gwp_k}6luUZx5SVsV~6$%&kYm$6p5{MDWluv<#haRaAZ9eSG3oJWcZ0))c2Q zo8oF=8&amL{4hk54^GwS4R??OA0OwNT;g%H61_f+g((9<$;5mkwYp*qYvIGMv>%+3 zRcJNDId@^dww&-JXRY|NZ*{ln82I{Kgr#(J`FB3{Y0VvemG`+$Y!K+=^QNeJ|34YU zpyX29T~(r*+WtYRw{CHIbLDF^k%Xmgdk7(P`gr*U#38`^pt)&0w-pYgBfRd7@jvHp zaRDD&gof$je!DgC3K8zuIqPz-^y<0!dt9DDgJ)Dm;-JX%WK&N1BvltxVP75nQYdkMSPRlc81heDFGf4hoQ|s@}%=@Nf$q2k`aN> z#ZBxj899Aoi{;rjwZpyh6hMeMP3Ivq{1vFOtE;=)f6h4Q$wq3z?=7aFmXM$J$PP z?1`;2In(Q0f@KB>azgGT;t*YxHe@K^O6F#x{ev?(l92R4B z_d+@*!yliVQ2G%b(fQ<+An2dnp57JJE25Wg&)Kp!GMNKUDpnaD2w6?7NNotE;d1($ z%svz2!5f_ran1=?9w--AmC)1CjxaDW)2YP4%!Lo^IhPZ4bcAFsu~NTh+DNIdZxj3V z@$L5hPo-{a(QO8(ROawEuj}lg7!A~q#0M|Df-C9Nc5!vG3Mlk|>ROCPpV0f3HFs1i zb_~I`vndOo9cPaR-DWZ2t){zcW{>goRGRX+5VDEiVl6Kaw+v>DHUtwCCd#b_rfDWDkv8my#2)YCAEaj{gU525g1^D6OAvqj$CZMe2bNT>?AM8{7rjZgHJMvp z#~e`nDeulVYy^-@Vx*-zE6*RW=G%%|t8wx1UGbS(G7?_i+^WPX3O@oYU0qbNSHxy- zeWB;`b)GKZ#(Y-u4c@rGxal`VbI^x!93wL6S}V;@Yt2tj>5uq=yk9HIVk*i?Bm(~h zA@A@1RB483k&%%@jpctg#gOpKLULx?OYzC*O$>k1;M&o6Mubd=p!y*?%} zRzL&^Mxxsc3aDPr=r2%KUz1u52>Z$yc~}*M#}0cwMrImA23l{})HvVmQ5IwPp2|@Hv<)+n=G+QHi+sAs zP!~vcVicsCV(9;w4Ga~&lfW@RS7E^MP8_bKhIFJDo@9<`I@GOL)M&gjYTT2;B4!mt zM{VG*(zEZNVepC_@M~Es7E3uevo*>toAqD$c1%hffyW00JtE_o!qQ09wCAE4k1s-H z8?2Eo3K?;X@<+!?D2k9tO2`eh^eTE&!V@Tfn@$q@7YvhQC$1be+`uv+Z<0h9!Lcr~TA_#6Ia+mnr_`bvm z2=r5TdHm4rw#wo1Y*)@J+wqK|C3btGvZo$2g7HEkUBTKrvo;iXbvkFCUoVHR1>4?! zoEmWMM}iv2QttCvB%>cGg_pT|_<9e?i@zLkOCoY02Pq>)ck-p$cQHaGx_#Agj5H>P}#64RV)=! zTO=Xg9OY;J%^*1$^XNSTjz7uAh4)&63+1KmK936eP2L$cOD;y`-9_A$ndJHOe&z>4 z)|Z@tXdu>i=5ilBAcy5{!7Z;4_q;japj~Krt0S-(aX!fBEYa0?9ulyrx&@h={(pQG zA!|njQOH}FWoipqyr9orpwV~}IU&x!#X^swzAYX1eqAO-k26KTUl~{t`q8e(nJbt> z!$5*$jf&-@^&V}vaxxZMJprI8EZ|eMr*6Ql+Uh))870DW-w}IRK(a=Ma_eI{q8Z9P zI;1HUb0i}BOH+d=p@uGo`P@Mi1tCT9>GJXGS@a9r2-YB&8TWG0hC~hr-|sH9PyeCI zLq_D~Sy_Sza{pH|<%9B(Q{rg$3e4IXr2bDasWymxai^AwXM;6b-Tyu6VnTX^isMNb>=Yx1?}5;mjiCC)^8Dkmp&* zM&KT-u9W(k%PUm9%o|-qD|9L(9Xpt{M1l4bd5REql36Hm0pf)KyKhVnzfj&%m;_$A zQ3>n!uINT1Q~6*z8okycn=!VYiCxHAdL8}7ASI%EzB+1!%cho#rhm&(V78Be!Fu0> zjs97$ftPATR{y^doKjnAH^&Upsuj_EH}y|%ld$DAH(TP{&2D{Qud ziz4Rp355}L;ny zeAU8oIM!Ww-*!-M@aiJg$qv|R0F!AYt~%dlD8*?*V$z+2&j)RuL1r}k>O;7U%zgo7 zUtnK-ZLO1|)3sB>k?8qkB#-TD=ui?-yys#%^qeiHxBj{WC~KQ19oYC3N)&EjOtY^@recrM6FUS*N4v{cG0le{ROyy;P& zN1y=Q)Mzf?q9w>Z7B)f7SxQf(RiG$IiRF=RO_7{#ZDxLMKXqAs$XkP8=H}K?FPh=6 zQE9hQ4jWa%LT-JL{Spc$fP+PHb#dbcPygpT{ukU}{LL3*KBRd$I{qqGWQpaPm}XYn zZwWPyk1<;t9>$1nKSdbZmr*keTJ@3m<-=ziR zdj?y!*xL2O3Pv??h7 zU$lkX27AYAAu&Mo_v>_u7>(@mv+Chd88&0!QW+vm2L*mNXMN~OKp01nANq#ZV`+ne z&o;regq&OcYizIKDoY06sN-X4YQxhGf}bddqB=t<@s-(^Z)XpO0e-vvPI-(>j1cVp zzrXtM4P^fn4a7~Nvv_Y`4BIMkB8(;EQvb*oBdfiSKqKTct|t?XE`P-$abQF+--ga>O!Zwlk7a;-3{vG7r%3cA=({M8JS|u9vYVOU zIK1msxOnUL@i8VL=P(3>|L-e!!{*`Oiot`~As=sdgo^Oaqo-*jWh=@pjy5-Oz}eX* zuPq2yo#C4!-(Wc(8D8LSntuN!#D`Di)lgAYcY;gZyR0yC9B7&(f(NK-u@hB}=Ivsw#aI6*Qhhasb8xCW-zEtVK5yC4>F67PB` z2$@Rrw*0?uP|+a}VrGxUukylvWC@oHz#fz*=4TD(sv{YvLE-N>-=aPJ%e%!L40;zF zB1!Lt-ns22BQcBUenML)9gTiF{dES>hPA(uKHPkj%7!^v+#+9ohoY|S4Z1E%uMzV_ zT9_lh=r_3E(GAPzdmY$oVah^h8_c3|Hq%Qe1+WH~8T-Gv{GeRmk0K;Ek;0Hj|Nps1 zYJ8b5M7GKq!fs$qna9#FexajTl%SHg!y*zZW4I@$a>UX7D1(JDW@@w8O5a$avIZAm zPvzW^9YhNZ3Xh+?K*HZCyH@GXpfa6{mP&u$ZMYV)JV!hA^LeqL5t2_UN?&yf6;oy5 zVPOd#W2hJys%mK5on-rf*A6FAKNsy@eEg54#_-hAKK;Ry;zIx!vf%TY6>~Xt=?Uu( zU-e=PMSPF~00?rX5@;&}*)M+5j^2Zd(rNk^3{7~RkN%ipc$G7^fBe@$L29Osze0b* zgQbwLX}-#!*^-aV#q7_QfO#xW;^JyQynLDT);WcB=dO2dfy^}2)AKrY-Dd;*Ohpue z6$9yhi6+b?Wm581=#nYqi;%jc_*!j*j-<#F9cX2D_~pT*u+npDVW;t^Ox)UP8)ay2 z{qUX-n!Tpu$;8?^fj`Rpzv!~DOnr?8NAbmiK?57Mt^dDv@4tb(tQ2$k;Ttf` zBrcWYx!obh2TKW6`DLEj`Bdh z3_=W46C)_rH%%D+=h^9V=&u zICv!2n^BP=wp3DVXkM*IMonS$xI$F2shSN=5uYCkSJs7wdo61Wt#$Qb)g`0ZHvOZ& zGwbh)Ib+5~STm?sdMmfY#qLY|0iqJ?v{&94u^WT&%TehPzo*{AEVlphNH@4OgwHqA zcyaj{QVY`6xVOHJ#V=yMm!+pu>RAFTMM5>s2Zwo=VgHMxM1L7*5c=mLHrlj?53nWN zu950xVGyXX9&UQqQ$o2kX@ynQ;UizA^q|Bp8yu} z4`5_SvhkZOcODnZ*0`GJP!2wOf8^MarB=GqOj`w~`B?QXS`=j{Ck2D5VE^%KY9ib2 zzl^X0?d-M9^ZDjALfz$B)~~Sy?te1Z|Do1F=p{4C>aFDCR!9IbV0B6YxEP{+T+@CX zU8T5xIU+Z3Dc$7?&v*_OfE|!}7}$y9r<%J2!wD0L$S1NWr$ue~@qbC+>Y6^Ur z>)kFF>-V$-!sBtlGE<{3z{izMikLRqD-PTZz2Z%nDpyO)y1%&HceQf2cWXh%U9@FE zISDlC?l6yWJfZ;D0 z4jzRD>Yvdw+5dZ+Wo*IfO+bT;Ahk}k*@Rms@WNbm%`Qhqu@l*%kZFIMd7NLfD>R+II$o1nXo|l=$-aCu zjXHmA#fcE~dq(qK8y#ryyo=%WADfSsA3F4B}wsptk*;mI&pC3B`^N_yKGZ z(E&Y#x0jUAH{JN!A3ZGRa-HPno~_9;#?JN;#P|{|2?pn@n$pnK#yd|^0Uc6oCXRp6 z^)i=rv^)*By{Yqni`3Vn$J&XKUG^WONT&9E0~}=|0g~mMtG4E0(@2U__4q) zMFec?5E>d(1k++mm@2RxiP4+wN9oOs1A9a}Zm<%H^d3)D2e~$Guvk-UEy6{$I4$*z z{^+YUh)Qu`_ih6F3tk^gm>f|Wxdym^fQ1FFuG2@EF3=a%=B-aT{{<;vaqL@eE;f+{ z1ipqJICfi}56)_1OJJ*~=&m(k&_WVF54ddOuW^UpcxK+HwY_jCi#+ic{LDMtJ)Vm; zc;w2Xk~d{@KK~V1-k7A35Sst3D410P6prNtYy zwMmnzGbx%3FCpg_t8M`BaiU6lR->Yx$G^jsS7PoSRx9CQfjamQ@ss}B(W%FhH55RS zBFHHJ`A^|4|4~`vhbjG6X7$aZzKH)_I4n&l#2jI%iXT+c4P@^&s< ztQE^F1$Q0p>djr3iB0FqeYpVg1>N;Aht#SU)fja7IaKHx|2KGI^abUOx9B2qLv2EU z=8x3TK1b>yM_MpXT!z6E3{uvJ3Dl2$zOGjWo2v#+ zCokQD3Hqi&taHhcZ4#Y14Q?o64)RRhl+~d8HJ`~*{A^DZ>Wycvk=>V^qy(|}u=vi` zvVE)`uiigqN)!84l1E`1cDZ_jZBDUSIp<$RmAzlP#s_;;2R7ETPg=|MU?IcKfs<#D zU}0{+M}LdT zc$%r8@;Eb^g6A6*`9^#ICXL5d1|jM>7N2ohbs;Oak;(PF5Y3kkOsmfTfYH7cG=Nk_ z`jM-|>yK<3o=m7LwiB-@cw#yK!q0AeITpbpe3ZOGuNoe59{C?PA%y)mQ(=K5J0bU$ zi9{FqN*g0paW+^xNO*MBbD6z4YZGaLl!seoHM2f;>8XNM1F`SIpA*I1newbgqmAs1 z#MEX+rMerXS{ba;6WC3}#bdk^OFsw!SXZBQmBttkeIy~2SbmLtC2sitMmV%3W>5ZcY{8-S z`b57Vp!g-`7O(lX>x@oLg>I$B%S^=!Qg?EW8=xRfh5;PXDP4@a9smHEk3iV26rK8= zEl3}Pf<1G2kn4B6fWlMl*{q};wew(MeW83OdUMZ&JIC#?{WY0H0Kn8MRun4fF`hI0 zCjDfw$eaf@)-o$Ecd>nM&}x$&V{3g&P!#pvUXx^|Jt0TIwR*tEVXL*`eZcY82=nyt9MO+q0`IhN_lSjgbn4a?bnYv zYy%wL~re1SFNluwjo zyWY?PrPNw2{JEZEfXT&3dlaYslAyL(6Ktnkpo<82@gnw%pn>_OGxsU0jc3T%nAN}n z{;OAdbE0}|f_kB2!Kfe9gvs=GJ3p}x5e&{{YLuB&H>jMIR+*Z#L z4}TP-n)gtP^Iti5F^@@WXJ;)P2ttoK4o!hKyAI^%l!Jj-O;q?SH2gBW_b*j&U3ot? zO`zUJlM z6MsradTo-oo_Ma^cP|$|awH-9TNS&d58WRl?ov)LkZDKy(DBw!h&eLSOc|Xw*+W(} zVSm&tAG+)eI$$(B-YElz$GZvB-53Eh*s}G&*L0*iN=1)84zGw6u9JLN7lxEIqb(IN zNy2{vRw}s2R09w1vqX;7AVlb6%<8_6tp1q!%P>A+tNAuho4kU&C*-F~M!D@Lby&bp z{+1VVi}PP2Mjw5#0n>WoF)x0u$@=leQfCuY{j z@;!eA9}pav-i5k&MLjN?pA7pPwtDg+{Xb zej+Cp4o4{#<0q84oIaWzbo<{;Q>k1VSr!)m9j@Z`ZH&aPREF&LUz=`Peb_0s8Cxr_ zlc;VE4t16S$|Y&WWim`m#%{>db!=#RWt1yhO96sZ6BC>E;u!^n7+2LkH#udsl?Pid zEU=NR)#e-;XWY z8jWd{jFb}d|04tai^3hme`%by^8%WI3-{DNrPH}Op^(GUt*+(yaszpf)=pu@X%#so z)M-&%ZQ!!28xaRJx)u6@O&4Dzj@*sXY_*j}mwhMYO`eMMF& zOP;q{6J2JtoOY|d8Lx^z_nk0BMk405sC;uQN}?mI)h*#FMdTW=NBV!5`s%o-!lvy- zBveYJQ$FyTklI~DSNu_J)?rwHLB&55$Yw2BJiT#c~kKgmM{I&aov*$kd z%sq3>HP?8R`84m)Q;r3LEM%N{u_`k3L1Dk|LYRM5RT^QODT)0488lVq^~KUyf`^q3 zcXw0pW&EkEb=FFA-<9gDybxm;a~ZLwo}llZu-nqq5-lbC<9>K!7JyJ3q+G>(RNv(* zm3u%HT-1n_U;p3yZTt8y>ca(JFzIGL(gF^XuWYo1??qfzm@8El%9L3n6(3*WJxp?Q zb=IFLQ~fP1W0a_Rw*ptF^hP!P$JYv zGZIhqZU!BhnM>aNVO0J-CtcoWzI1Wovr-um+7YKu1=?Ib zaj>iX0!Vqsy~t7I7!QwGiYdHkeAcdiy@-llNv0D%9T-)%(c0kqb3V@nG}phtvAezR!Sz{&Ig;zDJK-1~VGo{0D3g@Nbcn(&PL)bLN$9xCd6 z9<6nrX^89To)2B;hm-q%ZwaWX@dJr&L6mCz;i6N{E6<*mH+Q_&Zr{Xg>@3kniqSVp zjU=rj6S}AA6dww?4aa3&ganY?GAm*FHz}y+Cd5AL*S^~dUnd0>bXMkz7Dc&(XFF+y zG8)|XM>F%X^t!+y-wx&qxyjfJpm|HQw4_`P*A}W#W!5r%&+&u?CgZJlKmRG!`{)|H zdD=be*#=@(CAAak9Yg0cu%7zSV+0oypfD<5LnW zDd)pIVQBlokNB%E^YDX`tLl!jl{=yq?)VcQy>Tc&&p%{2&Kf*`lK~}DiO#z zguw@W>jWx(@~$Y+V!(1iF9q_vaJSMegL$LmSyJ980hsR>AK8=uuuYYoM-EC z>2E;MhHiffHOZqK1IPTBb2e|7B)`LKJ;h6vovoX{VMQ#dLG~}cIeY&%do;XR;OM82;hYt<)6!We?XE2ng4l(LDIWaeGi z*jr6il(LyL?`(^ZT9_G2;0z}Tj2Crk4zWsCN8CrGY;GN7oncNRC=;H-e=62o9G6zXG0NL4wJrV65@fudMixonfAZ%$JiIlIXr^%x zs49^=cZ7DZbmx_duicm*Rm}TfUazBJ|AaKU?Fq=qbYScEfv>jS_XdO-h;{>x4hWw$g5uPY=;^u<(^?F%$)9`UGjWja zM&Ntf%pQA%mZ&6PkfW9dWcq0BV4 z5+q@A;R}Xxb-eVQHVk>~54)Oksy!TLg>PG0|4-3@tqnj=e+9bdLtT4KV zMO*nz1g=%jK|MvIPTjI8eGU)3Q}Q009ZyrgvTeEU&FPcfd@=oi1yr(GLctw0lead1 z9kcqG?Wx9H?G)F|lP;JQwv;1VH{Xjb;tT($|2=EKY=5hVTWM9OvIbPy#jU3@^Wn<{ zXo~aI=Wr#qQkiHCaN^*PO3~{xK*&Jv##NE9erpqKyF2324i~2AR!p0)yk6N_QcM$q zAv@{=Fo9?H20x&{B8T0^%3uwd~4gliP(85CJIObcx#66kgIgF|l2Ztyw zuj;oBr)IiN-F=y&ut0%9VKG=aJXo1_XwQ$x@`0;{EBmzG8-6yJ(ml{?rj(-4#|pIU z0|SjWs;bxpOpGg~SSZ3`hrE8or>8;%&+RrTHg-J(Y8_fdTqL_L^$giFDW)5sXVH6b z`OAp7ru%D;L_^2+QjPh|%Wz;R7%WdxMbgqeuTZe6yz;oJM0mbQa`{a7h=_*G6=qS+ zy&YQx`uelFw`<+Ig_qWSpC5xohWHF{PLRiahcqy`%7iwT9Ir9YNEQYh#XaGmlX%;Y z9-Q~^fW9dGt zqt|9jg6nmKZmEV_r7ZBt_VH^DLDl#m0<&$~3RfKz@@PeVqp%-a0pI;gVLjoP@lNk@ zyT}8L)4ZCdfX?Gb=mYQBS1m4$6NqfKQD=iKrij}M@vHw3xKnv_hBJn6l z;IDNT^i{Is7)o&o?u1~){)X*ZfjIO*%R`Q@e2 zEI%(d#WNAuVM_LI%{}mL-PewOi^?ZG%i}eljxC^BlDD8TdX{=ebVk5z=M|2k!J3I8 zX#fM>5^CWhAKdR>;cQ0$2%LXR>51G}#)@(EC24j3BGE^HXN_z#iV~FVMnk=3t1>kp zWChQ(e1A^HUEhzE=}R{7AEcv%UUuhCiX4Jj{MT$2t~p3!c-_Z`2gF1X3b79%ZYQIY zBjsYXr`f{}7%VAWuO7#6pX46TX?Us-PkxvgNZf1-k8=Sm91*Zpsr0qk?OR0ne!%c>4 zgvuS*BbFLPy({n`6?JC3kRT21p#M<20mtbnxBMs&sHeF<%%Asg>9i`1&$@fd&7B`f zL1Ma#;hIz(v)0EBKJQS3QOstlFzKGm3P0|S@^qW9=Hr{pqhQ0jUofC!^Jo_TrYW+YdV6Yj>$?pH%q(|0DX{Z>dJ;wjc6xi(&z5dK_t0}7)nV3% z(Al?V*-aceU$-L2-icI81T&_?$6WXy!#{X^LuHhIo0?DqSles6d0j356D z;k#9;DGzS5^R)f_z_7$GeSmhjU9?*v6(Z!+{uv#a&9V0aDBe)T5sz*&oPY zH;wkQ{Xv?QP0#=P-~V3X={VlsORQzplw1R&msKZ=&iE@6;|+a}IRn}`5aiAFKO2-+ zi~$|7>jS0!dlFY_np56qzq=CY^C$gI(pp;~g(36Y<5eZ63~Vn<<480oFop%wK67s> zkpF65I_w%F?%cmvwJ}{I)5tuX>PJ-ZAQFkhdjd)O)4{IJh-^}H$ z+%6{#^-Jz?19e=8@BfNW|M{YiJMIJGnrB)T{=t|bk;-}H{R92=ey2g%^h@I=ErvFA z6ZzI?`>iO$dv^(F_T#@Nir8-c6hmIm4G$abK3%GB*7idmU9AMRtQK_@nNe!H8f@xE z6ovq{Tww3y49u&AccDvlJ2s}BE9U$-8S)48KCO-m{%5QFZ^gJcOTGg|tf+?I;9y@6 zC}U!AY^m31xZZw&Lu=i9*BF`jJ>yNt?abxP5Db3p5+#|PJD;zSi(1OFC!`asy^T2+ znb(&GuiK3gF9@VR=R~{`(o8^3@e4m91{pm~y&Yu!tfxv?fK#xM;Cr!|Zi5fFH2oE8@VM?H=T`Gf9_ylAf20&N3gv~9a-9%J=T`d_Vk(P z8S z%QKdj#AA1=9Dt=!x|Te%2~=YD?orqP_s8)wTnoD2>Em~^&Oe+VurO+8-k)6Wenr?k zVFl>v&VyYxW=`U*G@T8W!?!lswfUTKS{n;-Cew5b^h!7oPdt;0_Uqnaf?jVC5(Dz$ zujzj?MFYLram8fi`luqd!=GujbUxK4OSum3OFBzL4-nQsLnmyk}gGDJA z&O`AiA!g>uZn~1$%Qa25!7l`^VXf5s@Et5KH2C_JTEd?b{9n2Xs2mXdX#+g7u(J5` zvu*yw6AN^MLd;!XN+~}CWSU2DIy)8+<%ad;zUD8&EGwG6>y*yi{?A*tEM$xn>?Na9 z4oFEjMSA*%!iH)a(naY&>ZWP^<0jjdwb3Q3lz_a0K%>Jp2xR4n_vA zi1nwdZ-!}-R^pyot3)0ODXUrU#sC{iLi`Ih9DS9LVCWgy&Y|^2JG11*xg+7!(7m*v z;Me%tAI|>&?tzx?kGk|QeJ-cl1PC8w^vZK$ySf(5&m{D7e9fl-ZA`*E(NiGS%G7kj<$OuLJnP2>KUuS*^BT{~MK#mX0 zGX&12wfjYEii>{F6emu27iO7xJ|I(fRDb$-l;-yGbdp7NlUav*IskqX>x)rK-Fi@~ z-|egWDp&wgJLy0r6WH7MbiHJ-F;|pZkqwl<3}cZ5I+7n>Wvu4|F7-f z510G-f&6`qEFM0=hKLFYRaG>cHPgAKpzw4oCtN_UO+>zLoO-u_DV!zuH>_)GnR;me zIhxz)YV`7o$!S_rrP1v^24;zz4_GLt_sVOmtWVuGg-81es#8e_vXLrr-?6;PTH?Q< zHN*n7ee>Lb>?kCc?4cnIF9*xiMi$IqykbA0mxQ3&XY4O7uj67xq;pFoF<;kLd~k4A zFvyhX#umi4Aln&@HN3d-3pwEdy>|0LE(95CoYaJEDK7OeC-hkJ7#Vkv;ii7cmXvho z?Tk7IJujV08byW{j}rP`Z@vK$$?2SITKJ|M?DZZ#PltW?Zhq@qpQnK7`~axhWRGQJ zv@Jj3x0^FAsM1kn^s<2+A%HyEnB3Sa*H~Y$WO04xcCrDC>5!->ERI3A|SkRGUHF-^irxp+2?t=GBHX1Iz+&|F{jod zrl^RexQK=M1@m5G&=9sZ72Wfqf#%zxv^DAIIz&28QF8C~8SuMqVvd>aVeFU)?lfd! z(l(t8lJ%pra>AwzdM+j#ogNqasB(3K)6SMl%dJ`gebJYDJ*lJT!W-R^L+??ql9t%R zZI`jzd%EOPEe2Z%YG~amCh?j*eA!rMI#8PzPATP7ueIp-;fsGJreG<<&t-S~<0=K2 z@BF^n^vVSoK%hyhh@WvTT%1iA`8_EvCMnKjS6OUQ=q8RS?l-^P?LjkKSn=2X4!m;H zApfLiR&h#SnPS@NVa>;hc#1*D+c4st^wN3n8m0WW1K4VS{YznITG?mJQ6+~)z(7Ja z#a6_cO6IMuZcK&KL2@y;`l7;VJidI3&~(RMNw#`SF}=m{>>AO})Dbe*EJZ+t9M2if zWQzxSyZ-ka3CY>s02*gLDz=uHnTp^m#NJf-G4}|zg?1#-o3*%|sdCF7k{vvh@FF)c zPK1$HI1_qy<+0jMF%?ez{emG5uOD zN4*~Kr9riZr{a?&|F7Cqr!MQe|uCJdwT(+kJP*w4Mvgg zog#}a=D6RZnu_da80aiiUP$q?R$zBRaLj%S5-8h!Y1@h z?93g{-SYEuxtXXcqsi?RK@2tL556taIsPp`U*z|H4^5@|zcMDY4y+#8k@QkgnNaUH z{f%~Zdf)p7+cxq@n}XT~hWeUzm|L?*QvEr(B`=%q+2C-1UaR^7WTyBF5{Q`D%+JCB zS^Qu;Dj_c4rAe?GWxchG%4#V6e9J4+XX`40gNK3K|CaNK;N0 zn}wDV)=+Kd0vkL*&XNl|sNMV~tyP#RTkWs%67O_>Ny zJ(e&^byasA|Dl0MiwbY<2lG^9U{imqaGib}CdhtV7R^8N4I?A|g+2P(C?4DY0J-0M z+A&+o-E+RFlsE^N_ld|FK%={XdO-a5nJ`Y`?P#22kuXMd!E{I*2plMldv0l5L+AT zvl+$_qZ!o@8 z50JprT$p(MvDy}@=3cM{E1=n;Prt1wn0U`i)Q{*>Pw5MT9&j4Ho^9_Kp?HONEwh-_%{`1W@c8-o8eqtT%r;^ z6QgpP9mMDaz!t~Pq3wz4^~^>S2`EEP5tRTV22`QD&-dCqov z70+{l>!&}wpPw`|7YHh{H0V{ z^goJU^ZUQsIueB6e$61Pu)3;M#$YLYJ5psRf0A|{BRsA7w0NWq1cB*lAw@KrKaUKs zZnLs4o{8-y_nu5T8rIqI!HYdOOK_TW(%F7cc?qh=B}EuBHAm|fjNUq6abo0xs}HTkxmNY9T54As&o=|+;NE}5>Et9hlt*~Pp8X(rLEqUd^b zD)qH%0jB9_sF5@O=iEV9+V{dqYcm3)A8sT<@QuR!>f#W`X~J8Zi6l+7U&@k<3zBQ72MTNRS2UQEO~;p2CDD#Wr41sJwIWB)@oRk!tZ9^B45k zkj0zYh~(}}Wr~e(q{+bN=Cg^5Ps>zRQKHkxq zpKO+%*nFK? zTB6hJn>{l=;I$n5IbxRx6~{fZVsuG_-N~%)u1X*i$@n4Cuq4&0nlQ-Ez>f}yf)VA< z9~jXA$uh7J*hR!bB(sxH>$D!DW5NVc6c^&eozMK8?`(IG zIPe-4=1htayCCyE4y%?5)W>JUVfs=|UN0Y|uTas7b1&eMrmKl$blmaAX+`XEKG0|v zi->|C^Rbxr3;mYOtb?K;b<@I)T9eDD#U-!*6O=st+XaIQ)=9qq;f;iZ1kOgq!9w%- zrZAoDGvA&`9cccw-L)_e`<1H2a$en^gc}aeWh*M|;>I&R<^vvM9QzoS*ed zB3F6hl@8QME0G(})OA0r8{tqGKI1oU1b>Z^pdc;b+j}%QQNRmF-;T_yPq-AXl{KUw z4hO+xv)BzbEnWS|A|qs)v~K}^@$Fxj&uD&nNr z@P>33#^qlOD-@O`Dmr~!FiCfYhquaIqIKJjG>?qm!+WxJ2H!1MA@n5qS?8BX?iD)U zPM>ztT6Lpvb*P0s`1#3M&j-hucmmf;+vqGgS=rCo_hc4%d0v~j5=;a~-r023D~phi zx)GQtnwSXc1djbW>QjuD?jT^31N{h=kV1i4lSFMURc;hgUuV0HrR#yQwxwQn$gaguYEBR(a zNoUG|g@NA}i{beb7PUII2?RTe`Q|{E-q-dg&ggkP2)QKsSVJ|Ng}Iyqz`%AnBO|@@ zHr7;Dx4kQLym`%Wf!>`eQ-)n_|kY> zmQ(9aq+l%1?fe+Aspcd>rCoxz3UlbG?p8*r`y`>+J4vC0?yP_o2PRZmPKhMkYzFD0 zv~kmFIknN1LB3$w)f+^QwSa2Q<@_|+V3#48#!!fP#V2N{9#>zzLAZXcW0Elzl^!fLYs4gFhedu%H zkEOZX+AMS?nO?7q@cA(02mD4a)QT z3-1_kLAeKobT?Nrv8P{+B0s}cam&!JbsJQ%UvFJt3U#^fZ|<#|t?YKHYNuJd-&a+&eXw71e1IhHX`)m$3^o|(@ABITQ_LxYv5hULxE^h-nTVv1=!Zuc zIlJ8ZnYMRv-eT8yp>LATuoSD*#j@@SbNbS47)e2KdZr|j`k(Ew=1u|j#K z4}Rm}tDGcIn@uD0z}CA-Po^kO&e3IWXk#@pUM<-7CQvbwKt~|O*lrPio_pLD9PG6q zObz&iU3^N)14JoBG?Z6I#PIKHsat9&&E6jB`8N%vb+|!cyatX@|1PwJV(Daw&duc% z)e<2-#mC(r$Z#tDvB|PI4Hccmc2QE&%|3&)^Zv0I+9@U)XRjp&Y@Lm7DWT$VkXt(` z1EsbniqJU8G#7>q4H@k|^JV95e~$K>@n*aF!Hkz^)TR666M}^=TkAajc8s`RKeg7u z7*o|eE~mg8g3tP=oJ8ijH|c&-c552{#NAl zVCu&M^UGu-K_Vc6SpKy$&0c>HtMOZSI>g%KW^ymAD>9S=VMOPHunO*|pkS6-_jP_# z)XcioPlj{H&s%vKscAa(SgQ6?^AQ&!1rnq@C?CYtnD*wDZ)(l?ixMzY+g?3#V@|X! zhN0n20AyL=;Y3|?q13HAyl07e$-lt(vvTI`n>3s1O+E(4q8b#x-$Eo%$n5_hKl7HjksBT^;#o6!oie36IYI~X3YTpp}Hm_nee z_C3$$5y;EI@K4RKxD)K73*`5S8XI-}ksvV4@V`K&Xh1t1m+i+@w*f@vZYbN~oRc=} zfso&YyP@5A-=`6HKv%eQkMO)+u#CDR12l)~#^PhAX+#pogS5x_$5jPP+_IF-wXRMd zcOoj;*q`R=e=Zej{M9Bq#!Ae@i|3w7+VQqv&VpyP{l+Kk`~gVGz~Fd=!B8(kcK>3! zFs8fprmNV-b%!sb#)Xgms3RaeVjJdKE@V_RnDMO{9wwlifvg9( zMHAlMX)Bjcc zhENmcdvn!`cW-b@WcJ9L&ek0oRfppMZN1zz{#&-ubNBM-PdPnHIV>qH8D~vZGSq}Q zPT+WMf4+N&z)9bg7Q0$11rhVI6;lQ@~4hsBZm@Qhe_+ zTU|$bir^^Rx7QE@#BoZv8i_K6&eP#!S(A!* z7WZX2aj`L9n|h;pXYNfoo~1OlcU+WGy!=8V(x23jmRY^tvT7zC&EotB{U-RU;2UO5 za2yQv8i0|&6Qlimn%w1vH#i{j>x>^OV+@30lsyLl9v*aB}NfTGIw%|c6u znLiz{q?Hek2kb^ZWoHjxxR#$IO~}l=Rg@>4a z)ay6am9NSyPSCAXL~fg-msHx(e01kyr5Bh1!I3m1%X}F)!R-}j?C0SIZBhAz2(i_^s(`AWPX?LyI|T8hVu z+Sjm|MR0G-0&?Oq;7%(AMWGOudYqtCxn^Y2{?Tg*1r1CFUufF#49ah{H$qfW-*s*L zGgUqT(ctY^ddl(YdvlhrK}4S3TEfEei!F=e+1r6E3<<16!~J-q%ESuPZuL1VduM_V zsI0f1D&nS7cdxpXCi51m(RyuVhn|HKATuBSdd%KWHjr$$yZ8TF7483u&4CT(^-|q# zk2bz^GXH6`=Yxp&_@djtQ;abrd4}u9L?O_t#HWdOTa+^E3o%QX893XF7Pmhta02y^Mmo{;Rg# z!|Dttnl#Fe+ixU>BE}c$EnRG$wk(~bx8v&8`L4I43W~61K(ewhWl*-OA3MDQbXa=?NOyr=D~iq?~p(E-ZBX;g8R(av&%pbZ7NEXyY{W zg*zj(Wrn7Ie5m^K#?+023p$15AGY|A2d|&8z63b!C9^5NBNtCpA zBD+rc!?F>8xUI$R zUb)Sc=;7!D&Y9y$mWl)*|E8sE4--8OQ{?pdCx=lW^{1|0BSc5ck%;MyMeWAcy%^5I zRNnKW$1!L3z$dZoBjKkH)$V%n;1pYlsc4y?x0yxJ8ee61V+l`U|1}`5I?N!QpTrdv z38|~Q)C$04%HZE_2NI!Wt_n8sk5it+g+8+k0^U|xDwX|JOL9ET$MS%7?h8m=y|zdu zGy1)$+jEzN#!a(LaZO={mwn%C-wc0NV+vgleI~8lbY`HaH+3r_1?@?L|1#y=$zWlJ zCTc{_l~y0%-@&;o_qIKbFL&Fni9xK-6^qSWZAHZ%mn^OQA||FvE~&HS>bfW2ECToHr_1_ z%w~vTUu4?j(N%ZdJ!i)j^11cblm%-4Z>53d*$r@s7A8g%s1Q)QhT-}VvG<;zvNk-* zUpf$d_Tl10v%hpk|L_sue*adpvVwv3o|>FG#@JtgutB20y7cFuMbVGOuBh~_Xp+Tkt;$kq%0KK7!FHrKAaMZTzgy6Dmxx6H=Db zEmJTik5{(Ez%Kc9_SR@vZGYCq+wk<8uCON?H5FIz>Lbes)nl&d3Q#k>@o()l6pv{I zMiiIZxxSwi#^}pGLMMGsupWQ$p@EQifNGWYnUS*Z=-b}DvfNxc84mY$Mg{7kr~|eyO}H@jyG?9a&sgqj(m$VUsl@U#&Kabv*aqz*;I5r z^4NZGt$hp$Ie~;kreK8j#->n^l|R>ONz+j>wQy6{>+kIUF?6@0>4Xw+0S75MxuG6# zpP-yCj@B+AaixbLL%%Bd&uI(F?IKuor>EaS5k-B&PtQk~|CJ%%VgP{Q4|NwE(DG%V zugJns5^Cx0%044XOf%s~l?>XdY%1_@e96*ix!B}gmV)`IcIwen`kyy}A(4i&ahWaX zTA@Ud`cBoK-GKCp0}fn9h{(jrb zGx~xwq~%#jI5>pb)bNA(<0v{Og^P-ZEA2m9ImvGOw+kflvbR3$%4sqGOs3-vr{jJ8 zobz_KzVgMRY=ss<3Nt>V-F|8I=3PVG(kh1DZ%LEEE_5QN`+ZnJokH+AziXHI#D@xZ zq*Muqw;x|R<4YGQLK<;F9U(!`8x$8Ip(dc$`X~GIf}&m5LhBb5<(jZSkGmp(S=-Fs zUc$hq82-WI%m*JgAE8iA_rmJ)n4J04fI3j8d11s4Z~%|+ijLqrVUNA2#uzk*@7HyB zV8hRcqG^&iu%?ib!GQo^EwXa)ML{XygX8(POv zcG5{~yc+L{HNfn~4GtQagGV(%Mtkr}e}k!1uu*S3KEo1v-Of@l;t94w8d1w9TH|jp ztbY|`gb1EdX14*P=>0vsw4vh1*FxX!j?5@O?&ey#Amcw!F|8oaI#RaIy3}`%P3}zR zbGmbQbj<$iJg&V8iF@-gFI|f4mbY3wi@|0&?Pgmm^P%!{%If2@hP1G5N*P05?)nry zpSKgx>Z|(&ieG;_0peM!(w|3$I#W)~2U0)!U5qEJHmiY=Zj=J+5Jqv6A;{o;3O?5W zMD9?eDOI)e6AaMs(t>@Dh-MLnbj>40IcnM|N)a9QBXf2c*D0{BSoa~(5BP%Ck1c6wd;Ydcf zRPs8TQey|R={`n{_beISTx1!zuv_|iu>Prbo)t9cs(8ZZ==xObnhB^L)t$XK#n8y3 z{ef|VqL}>v-+gr4RZMsgSE>&&M~c%N6*eT6lF-Oyt;G7hW5tZ*2{Y+QZTkz0WR^%FfMYA_!vCTb3Zc$ICvCr8+^9e`^ zAm1s=xH{%^EE_A+62tF53U{Z*1cE0}P|j+PZt;WFvKSfD%|fS;c&IirJW0(q$|(#l zJ7v$z+4Al~4wKz+TSE@GP4k2`*(dPA|Bm_X&%mfR&w0iiE-Xy}c?;uxj7^~L`#=GX zDT!Bi+Oj=ZB;hii`5#Nf$m^{QwWe64cx{%!?m zx%Nev9t3hW$8R(5dTtl-uO}am>y*0gL=asjx;iWltkzAtT~o}b%8I&t#UFq$?|cbk z88uokvucfgkHqv+U5$o|f-p<&4>ap-KIvOlVlgl>8Jm!<5BohHkR@4>NPl}65zl@# zw&OeYa^P~UJG0N=Cl zhv8j;_k{>sM=X$cgaatpBkHQ94zBRN7?oU|>E9rM*-}gBvy^{J>;JsHjV>K8{Wo_! z0T;?6+O;TV(EQBecTrZDX0N9RZ0H2{D9_z4XMejp1#R!w)GnkIn(9|~5H$wFF!_Xe zaUm@xzODQTF-OPD#pOcTXP0N8=CZJR_}M9nn(v_cd?7oS&~tOg-1o+K`b3k(22gVT z(`NKHS}y2z_F3Qxq-896#)umrc=xr3=e1_HeOTOpZokb-1zJSWcdoULd&o782fZms z-nF}4j<1l>TfU8lqZGeeC*uQSeK526#5T%=TR#ro+kDuaqYrsva)G+{YV{4}%t(j1 zU(15ok7%lT^!lYYmzZ8oN?jFEQ&m#^UMvkPQc!q01{ll1AnbE6TBfV+$m*Ze*u(zu zSc|=v1O%tS8Z>)x5+4M%=>KknF_DwZ$8fuEz9sW~wu_w`i6KEwd3>uy)(X*nMi=QK z`HlKdzvzlwQI_dP)5!$1K|~y%p_2nPNP=8w^`^3^+-`06MHQO>eJc+32>I)yeQoT% zdlofOa#$=Na^huv^T`^p7vWCL;f&X(+m<8(@3cakzq1$55>=_@#RbF~oqKoG(K$@b z!!)4R+Fd)kZj7_=j(YE#pGpc+A(Lc!?zTI0g`NRS@89=$chNVb@Ol;|HC~7ay0=9z zqa@b=%`h6@-{X_C>O7)VC@b$?4Kyd+aHATl@p?z6>QNe}diC%Elc?SAe+ z!7-_6Xm7>6HVWMieo-?w`3C&1#!o&E-#g-v@M^-NS>k)Zvr(l=0d{see$@Fq)E zp4`_v1-(#|ww$D$PYNqSz5Bx;mv#)P%7m>WDQ;?i?-K2!HhE`LE!@=M{fM@5(gw@$Y3Hk7HVr(lrS~* z--u|@P90Ek#C`N6c4z*G6}J77YrmD4Ck z{pE}2&tHIE`!SHcsfd#O%Dhy_cB3Ww?j3V*V-Ws?m@rj57N(FE^80A({9HY(*@p_0 zsCQ*VnDB`X`n^=hRaWxhde%cexlml<_K$PJnFZM((h4n7kWyOwg}EzaP>S~C`i#3% zJTyF6a1J6Lu*nkJ3SRoQTB80a0db-V~sGP(6|_f5nP z0yD4>A5a3lxWD>HP2KUvRoIX&{o5MtAUSb#XakMdmO)DvjPW1@62n){y+iUZI|!8f z!zXy-p%CAY>>1M{qRB!wtGj2*VJ^t2izzv9Kte)HNGP6X(2|V7`vI=RbkSdk8V~hNmj3|h_=O^ zC{A-&;wcqlfchaKg2bN-g|t<^g`%D!ztq?h=8%xYZmCS%HTfh+t2Q;fXxbA@CQ%S# zKR8MUim`3a`CL_PUDgwcMgr=3&Gzz3C;k1Fuv=((PECoeiG)PQBwWnzva>snmgdim z4V#1qDirBfrsl&0Hdc!;3$3zI#R;aV(88?gwzID%rc@&s9WE8aSdI_Q_+s@H->c(> zAV0-11`n-w>h{nr&5zvRk^23rPA{k4lyN?H2l5DBa#G?T>;bL*5xQG|_J0GcYh+7h zQ(jA|qw0=)WnNjDU8INRzmq454B(^Z#Kkh>mECCO_Y}Tt*YS%^GfH-wE~e=CHn8b1 zLwg50oVI2>RL7t1QX>YLNEOp_}sSFAPu&&|po?9kwwa7v+H~Wc-V_ z9Gdt&<;wjJXG=1>weciQ{czW;bTBo7B!7|gl8^@h`Wu$e>O!`PO0XlfF7;0OVC$4) zx-+GLRBCQ-ezmnN4^GnAcnX@MjkeaXf;_1Js^L-@Ev1!QXt68{rWwoC$nB9PEH#pO ze^qB0d0U;(@9WSYF*LaWJPK6@f(Ml(p6Yq#wYJn($CRW)2P=3P9Bd;>sG^5?|Ayh= zf`fsO=(|Sd6MVnqO!Z0SJbMc{N_n&@(aRx^O%7Nlo9kc+!I}nYTYnw~#vU`1Yp|#Z zX`4cR78x!rD0@|uz4&(0y0DBk!he%HYb-I=Jb$F9`Jz#p{);4s=X%L@&EQobOC&3Q zS#VqS2Ox38^l7zU`aN#Bbb>e+_Ph8elIJP+BZt#V8(YrT>^4pwwzRWSJ*jf02FYY3W+3Gz)24V+E8I{#8px0DpN_`k#xz5PN`pI3 zaH+XH&Yac)3A8Tm0}U2Q6~6G_=VEd53ij8I<5%{lI5;9Tnp5LbE)~2NmyLO?0{?vF zcWL0X%cT9l_k>I6dcw#zgg3F~exuC5hncf=m`E^(MNjdF0WFQj`ZJPHCZWE?`Dq20 zf;gs*bX)QZjsgU=~>SjuD;ZTV+q-_3^Y3>hU?mGt8N4;TS zhalX#p(D;iJi1eY@pv<`ypLpNq>Hv*vf|U9^x=(V-q3(*Rx&3|+)SxhL9e!PTdGs` zZgsMRBb>`)?;yg{37P7|HLkg&OWX;S~+giB2 zP!puW4rBeG+Et{c&MNzu3r;2m-j&u}0QXjpAz$Pe4#1f0=eY5E((a!P_tP^(`gADP zuajzPf7&j(kdri3ZnjbNtL;*gerBsqV+sn}_&7&|=%3rdMTn>r_KSGrtt#Yc zlmfWmIW_Ok;k5RlA%Omq#4t1|V~UwChpT;5(#YG)%>DycqS{DR@?Q zrVqu$$On8R6WtL*a)naQ3dP*wjzKPL}i1LW9cWxjj)xrb)j3k$CFFz%R$>j&H3 zy*=m=huOWUS+?ebu>WH%7p{qHeXu_N*#WAL4pL(?qUYAgP_bJ$gj2aK;r(VcdLX>@ zu(_Br?pSntybiQNLIarqSBz&v$O_kQQ4r0KW^3rV@+c@C1rfq$83NPNfO=bd4ld^! zjRJ>uBu|v)nZ_8yNvqWsLnM$L#&s7I67;O6ZiHb?c#oqi6-}OR0?r^`4}g)zi-~sH zaKE)uH_eE*z@G2y`2r~C@l@ev20Ev8L$?ogo~4E4Xa(n38cA;Af-ZBRO`q|78fj33 z*=}#4G#_fIBa?NEuBanIqmlN9C#jG)&-SsN3!Oi@a}gd((Q9k>!Xl_lss8%?5nq!$ zr0f{2jgPV>3kAjCbw7~wX^dfVO-{4sdBYho^cAOWh5q#z!8mNE`CQ()kkRAb?b55&)sV8SB5#tUdlQyOSy&+~K)+S7WU-y#@|Coy9~7u-zzfQoYrY|Jkbe zlx}ulM=bQX`P@8YskC=EB|B;OWC~~;hQV6R^buau7fNg0<9`kzRlPc=SpyH27c@+J zXGV}h@dVauQ@+cKIwQ9~<@ZGMrDHmtIDfrI%gKz$a*gD)ydKIv!^ZcmtmvxuI>Yla z`9v3mA^PZrU*o5KE`;Zsu|R^)^=1yjPFx^M*;jh9Ml~N1> zmO_TKVwgnFMy!(ySfjcpsRJOV^xt{>6bTKFU8+=KEFco zFL-7Tl(B2T$0vA84S%5*Ruz8UG+Omg;kEHAsb%*{SJ^BoXSI#II;Q%_xxx3z&IsO7$i$%0vT@U}oa+h?()XgsixV%?Iqe_UG~O-%Rg# zzb998D%EF52=CfZY{9XQMi)*t!12C{;=+6j(OI~0E@as`${HdWpr8}5PKfKw9r}9w z1wLt zR9J3UZdh6#3?74aHVG}%l3WWhpJv$@pRQzndioFX zX>)dbFq;M1r((bdE}iW~+>yg{^t(_{xGUWG`8yx&EYl)ieRfPCUmJlHaYsPrBqbdF zbf1*;O&@*h*N0)BTCJQOx3^!dcq;hpfu@9!8E2p`!jc>^_>Uyn}}lnwaYe(LWt9j|;l?0zj$4GtO8 z-nMnkH6DZ~^CA5A21ui0V+YPMFcZ{Pba|;;!|Szmii_)kUb+*$HE5TnY5wk%a$^Rn z@y;dwm95i$)FSP@Rc4mrUE@pd)pT8!m%%X#6Y{mMaX#c($eDBmA*Jk@m77GChJ@Pv0p!VXX`tbHTOow zh~9oPv-?tfuGulR9f!*&oz~`UyupQ-49ZhdhG&4lzAM^JIF>HgY)yN$r}1TEuZ&)oiTK;)zWoaL*k#Oos;0nfM%7s z#ka{_S`&gdv&93>uE_h@P&mhMclOA_D=d&=0)<6Y&4{<{{aoUK>hFYp#%n)xZihO_PwW~v+68)qN^v`m z&l|`iz1D1IS!K&{XVqf^M6rk>I@Y5uI58p-J4y(#K-s!E+UaB+wsWE_d{zb!X-uz% zZQ7rW+qs?v_3n?J{LcB36UDie_JDzvDOJ>|qgIXKlV4D*<;2cti~M!(7)~Cto5G3$aof9>rI*>^F4`!SbENOll*F`F(cC zc)y3L{rn|*ZXC{G! zLIx{38fN_xc;T?#*U=l&;HdBk3ofMCwubk|aZlRFkuJezejwL+-4&Zg#((dV4 zDrR{yHfZT1lnY9;ZT^i1I*UNri3^{My)0|}o9=hkT2NjN@+RpUcbr`*_7ch{_V&3D zZi*P%d`exiszv0Vq$yXK71p@(71U&gxP+npzB4~IQP)FeVOfk+38A#aNF`gBx--X^||SLT_!JaxuZiT=_qKO*w_;5R3_AUhYMc;Jxk+apv3m;Kj^KO zKT3p1fV^cKxm723bps=tclua7h_P#I=NqXZYNVJz(WMP+Qnk~-->$P=YzZqPKWurH zEo6Iuhf3DhVH|XC8?W&#KsCLpo^()ti-7AUwLot2X7D0luHvsj=I^f#uhPF+|#=lX!gvItOFQK ztc(?sX$Tl8!I-!DeGq!5x+Jz-2A$)A!1ZVonuarhHrm20f8xFj(`xm`=vxSidN+l=naq!prfV(Y9; zTHIo%-85c67Q)}XZzVpKTu_))Q0P2)cvLQQ_7{Ei*OlgB_0Jh+yVZrrUVGkw38b(! zX*sLDb#JuHk}ctbMr+Xx^W(*xj zLvkx&>B>MFAeOg3hYnw*c=AE=bV@d)UpUiRpmz;t0XxO&WCY;T9pxvIzU1ITtlNkc z;waym%C|~+<=vK$r@RtlkIq@AOb$X?@VJjDXCSEn8!}rfuhbYPyGz(e9ej4wHEty) zjw*Fu&zQ_c7UfXJh|g)54AZ!tkENfIwWgSJdnY!>RxC~&!GR^NKgrdZnn%ZY2&$Y( z=|-x!g$10%54M>)o29bg0RxE?fXh0(F~6}w`4-!m?L@QWsax5Pa_^2c$ACcgngzh+@Dix83AFxQNW3L;l-8< zdLdYC8=(fPGWp$K;o!AW!7yt#*jK}X0Sj_x_}X~=C+7X&&Aeo(fIyv?Wy%YqP>4SYc}!e+MbGW`Z$7!TYH#wM=daz(7Kv zV<-8EK&YUM*V-ad4vpxy*g33lYsL>j#0{dxHPo*%hxlPWre*P!`7fDeT?GKaT|k$$ z*S_s?e(w@HB{y|A2s)bF*E1>@{6!T9f$M}EBpVoS@Qdox2XC&OvtrG#V(+Y=iD*kNOBEr% z+C(#%rZ3YgGe-xC=W1HT>k?9S!Zr}rlT&FtE4g4-KZI=xtF^>>)FzD&M&&+3o@*y} z{2P2SR~<5YIQ<<+`5o%nyN-8et|v_3CW_cj(Eo|kdlou74NLH62)| zQZTADgPn9!z0Hta$xbNUMe{vA43l@At#{};rt#aDZiV3JFrR%yK?I!lkG@sd*bY9P z&HP~gc?DVkmeSl1K;CMbif5Z?t+PH;fML_MsC{Nu{NSDg(s*iuDav}JKrzFjMYecH zGvo-((&a*DMErow5rFRLxt_O`GK;&Ntg`__sfxtiiXIPXs4RkSfGVPmS&%1Zsk4WU zU7ZB{jQ?`jr#N|jz4c>M;V8|VnLhEbq3e54!SsaAz{NaN4Q^+3smp!1@*=kYJKeV% zQ~R{9&1ipUGO{SvxFEjm5sWV-370+T-1a1i2Zy>Ab(YYb8zTyYsIz8sd-s zEbn2Q<>Gar$+)JF6$!GRl=^k5$XgHO-!+@0-!5EV_v&%fn!E>wA#myVo2<}cOLvHv z-#HC@R=xp})1R5>v)EYnMAMbEJK?d&t4_Gadt>#&65ddh@{OgBy-$XPtvi#7{%n3sRF-8HNU0k@ z9Us?axM?>@{kIHWXa4Z0(#Nl}FLcJz(K2#9*USUwp`DMKC+!a~A&wIDd}o_e{U}&T z3t{8*mY1HF?ePs-;q_a-G6&m-bC$CW_T=!57rJZ@4f^E5bTrCi!NFZ(>jY^U`Xe)P zJB*KAs#=-2<&~d{Mlsz{Z`@5IX4%sseY4ocyR7yeL4B751q)AiC z^B*9yMQv?cpw3UCuE+kVy=CVR0_zHAsi$Q2e6Grr3WY}42if5~oR-31&|$t;q<{@{ zKKPR0?8H^R-?P$Yko&YS9fiiXuQt%Qf`Qo|Jf%Ccayx8CZCN0njmNIPXxN^!4EX!1 zWSMSG=VMH@cW*VtN0z%1n11*F`PAPMSy)CIT!wS<`|e1_`CbG!nUqj<;}t}qrp_NLd!p#ETE^DTT%!v zm-#(Z^MU_*s8h6Zo@3W|4YosVz$1Tcryk-R;zRpmzLE=WHf=D#_NtcVe17u$ODaIV za_KMe|1{A96V95jk6oRe`@02t5^|!=eaOXMhV?k|8BQBkI;KbcpSEnhIo>1;O_&(z zniqMaIK5(nhhSZLf$5E5k2sAfYWn7{_ zM)tUFIXCBfL>vZ0;q<$2t+o+%R>SXmq?gEY`rP7?$GYMgCoMp*sTzo_S822G$nb#-rjvIH zU77aB+kq_+;Srqv>l!o~Eo+a3*YUAf^jCfBfV00@@Mbu653i%xGQ*uyt5;Xaz2lGh z3C8Aeh=fA(-hC`Q+c*NTMPxC(b5l%+R0H+Ltq!IB%c!&+c;!FN_nd&6w1+Q^u$)pkUv{i zL{iG8bDaxv+s*7hBlMzqjQd7$Bzihy-nIcgu_MMku8mztqQ zLEH7DUW_SCVgCNx?3PEyM+=Oo#t%mUOP9RS9{*2n@>VeU&zCI+C^zs^yzJtrH#2~=8N8JYE-qSr<^~w z^p!25zqp;4cOJ*yx2_22Gi#RNLZ>DBpN_Q(>!`x%Egy*9wolf0kAmyw;a&)QXnrO&w1?~nCbTLlV01h{@D!A zO>M{Vt?84@XaJmOn*Y6y4?ErQG#K zIqZ3hMjaG^64&6Jgj)~8#evQv81o}N3^UN_gvC_$o(1aN9=<&a#KJb8bW;svDyD3!!-yt%YYFeJcTrq%zqAm3-pupFC8 zxlT85*DlcNW#IDT91DZ%d^~*!P6@B^t;cL`%dTeyXEq5IW-;R2Klx-v`DE1B5-;7R zgPnze%!8J9KBD!*q#nue{E~R#v39$HKsqD1%jYf?;b73Xd-*zW*dmcI#TPk`(24%4 zq@-@O9u@K+F65+8WoBZCUe@DQoW|zXjm&2MDCc9m3ZvZ`sKP>o$@Xx9s<22bboLrU zBB|M;?8v`|OhU|>>Kjiy=`I|Qt&=>MSk@XBI%b>Ta0H7Sa%CYzG}q& zC~9~?YG~Y9h!$_z4Ny3rqAvF@g)dVx2&w^PXFKlzZl%nrT98mRl_jk5C?lMBUfdZ# zQ4fGu;A#+kFpz0|6>JK%i016e%qEXv*gfqsvvg|a5lvNaToakBasI1Z^?_fy8pi2*|2u zrZ!lmW6&>1a4)=$b)&D~kl%l&^PsgFkJkmbE54rvex$*WzX@5e`D_6B?LdQA2~+NrVTc} zRzBN`LJwvZNLv?H@RP(I!;58XyS}6Gp7T*`euWiEjw9ni!DX|XUlI%KdzYtKtLeuJ z$|sY?T143;e^m6gKjD5XSF`|M&ICJrZc*_r#T{(X4V?S*7ld~8QnAk`oJ9AB*R$21 z&8Eb=MvntX=hZ^fnP8sP02a&v@v|$FsS(qg2E^DhiTz^L*Ly5ska3Ikj>a4G*h@;j9~eE!EtW9R-}M>|?aT#F;9`o0?|VaQIk zkaL221!7PAwQ%Gcmvv~pka?x|9vD+$Bz&HVkHaem(SiD|ubTC`ujUuZP0ujaJf|lH zk-8%T}ch`KOezUCuhVQf04zXd!H|4^dM6l6CoX@G+!3Q&e8du zTG;`+p|YO?^{fN1VJMlMKjApNA<2&ZXgSOe)rKFAeyt5w9r0}be-q=){g)^A(rQ}- z#}U078!>5)E(aX%dIHZsGr*5}p+c?O^K@(w@u&xTVF)VT>X993zx)BSo!=G*CAm#^ z^*Ix}Iiu<8b|Jlsr(N)%`32d@rM=H$1%A1^LnqSRhL zW%i7}*=%FGQR_-A=&WuFg|`SiuJ=1xteA2d@DU8kP4ztozZsbUWe3=u{@TDE{~qlZ zk~>?iZqy6$#oXI#+y|D-v**HBA5&Pq1iW$`l;_9s1`9K%;kPyViLMA$5KuG&@H4-u z{65KTb&r;dcEuxyb>$l)7Xa6#T^vMqTMYI=rq0a0@efV?4hAax=My42ZY2@#f4ie$ z>8_1Y52iQGHfveR|G~{x%+H&WMkD8?G89d7Qm?AiK5F?q^7eufl<8pZky`>MEiI1` zdDF!S#u-Nb+T&GP)5{v*K>QAj@jC-;Vs{=x!tp{zU7_FUXT09H+aIuSK-j@B+ic_0 z)BL!1P3nwr8Fw7lJl(f3(At2V^|akgos8zx)@+PsZD0kwe6JaDa z#i=o-&8L?V<`!}uJ8N+s)_-5&mTR+2665vcrHmSz;E9yj4tCwzqt9e<)I9w|)*^$9 z7#|MhEWJBXft_ED6j+9y97Hfi%E6O-(H3cy)-GWYZ#JoVl}EAPUY6UeDJ%7@xydx_ zf-5%N-2grS9;_9Tlk*@5&B{36O<>*nRXyFdA`oeMj_m}+y9A8|rVUh?g4*}!f;{0q zm@GA4*(qTT$k8~4IB}dV&fTK|HrAgWZP7iPUlJL}+S|zv!IvzSyA}9#(%uqp-JFUQ z@)nFn?qIgCck|(b0PuNi^$$PA&-Y5opSjw*jKu2d>qA;xQvXj!-c3{9h2$YxTKa|! z$K8%qmD2G{$0jv|$<%eN`@A^jmyfZl%p?=yNUnl~nYRA)z2LUofil(G)Wmdzx|`#} zo*D9AIeeYz%7KhK*g?lN`6=$WFH~*eBTl-Sms7F4d(`Oz8fmDya`I1+Ei$tf?WYWE7=dBj!&S6=MSVz8T(8(d%W{QKT z=Nhs)lOGe3|6+EMM&>Lx4VRi&{9Q(Zk1O|^%CX~^Q+#>EYSXbaZ;c$8aDty~ZJY_wX>1Nd z%+HQazEukxM}5p*<0DM7$&wd}S+BfcOmDzv2idn~J^!6@?rb1hZ41N*3`JAir?C8n zFsU1@^T10;CnNsj9`AMBewVGC@(a9`%=7;6-s;u;^T2*-nrZ)lS3i&uqi97m5w)P z?Gj{ud^Z$gCnkb$jqH7Il|~ixT!V<+&@a*qP9Gui$LR3$0OI&jrL2|F{C$}NEZ5AD zJ7MA0D`(2W=c9O-o%|8n!lK#hsBeOzHsVk{#&kayGv0DA4%Q~9Is=P4y19INzL>Hdk>#_G-c=!A zFU%yr5ykhqyh1SzMM%wKRWQwGJ9w$HbS0(fFK+%e{n99NrXXU#xy%<3s+PDQHgLMy zXY#k-fO>cOfd)G}cC|G4Xw?Aq zaPU5-aGH|LZ-$f2t*i3DzuF(gk4uoOE$fI6XfP+vx;K*eKCOG<7#PIycdg}QoZK=6Tv*UY?;ssJ5$zOg& zkIsL@qHfxIrWP3d?K@8Z?sSd!=?iY*$KxfHK+3@hHev}ZGfxu2N=cgwX$g z?xfD^@!x(r_)V92(^ClE|KDq}*3LK-ggLF~=Hr2YA&SBO^`d%1I(16i6<24v*7S`1 zD}JXH*XTBdhE4gh2|cgj1OG=KFMgMT!puDez&Z~y`6?PtV(S6+H68NEphCjFmZ%B} zkM77Bko{tg<)7b>@Ue>-FbYr{(#?WBtDYI&80_vaK{<@1n(yne zw|o00_04T>-wju>F&61eJ*T_0I^?c?&E26)w{9T4q_soxYg*c5$nU;(O9=F|q)+B+ zd38jD{l@~$x4Zh05jFMMQo*7sey^3<#Vn6^xhpESWiBK-HNqMQo0$*E!1kp8zT*C~ zA(|w?|19`>HHQCQI2U`6mA*}bfi)CLCdyJK2~sg6bWGUjHif;z%$R@zu*13EYwJuT zMqN$R1!BkOB^|orGG$!!n;;hx(V4ACI^P+DvWa8HnYAQX&%JSV=TM!ssEeBv4c@c> z?YWKhLm1giHCSWJ^YOnsti{D@O*U$z13sx|r$6)Y{5{&Oa|yw)?DE=)P$^5UBTU!3 zKzfFH#_Rr0d6;^^J|hm!)SEoP_&a`>T1@AcjlS*WcV%=J&j(g18`&Q`kn$S`JvccM z$jAS;is$ZjsEbg%S1@gMpOGe=j+A&D)RU*M*oQ1E;WcV8&k5xM`zm3&jd2IwE#Fke zF^0bVw5#(=2-enH2%I(;H!(AGivi@|P#5vW6xN97mKZEY@lU=W9Xcdlf%b`T69i)S zY&kkQYFQL5YRBLuj=S-eALTDP)RwCw6wUf@qA`Vf_rebaAi|JWn@xCGf6Buq9KfPAZ|3f|8A`W1It^! zXAu!aSLg(z559EHx2Z^t)6*u5r`O1e;U4GhA2*+NIjOVH&#{Wh@=W7?73ha|_6+BE zf~~8EhUGK~>&t&x9)nLoxQ&hZ!*;q zRtc1J0>o1s##Qcbu6Ih|)#g;MAPu^?9o-gO^38yT#LGJivR8{{Y=COB=})Lzj1sXUBHr z7xI=mUcPdYgdpCvcA%rxY4rDRXliTW(=H zOpJJ=W~}4#XYpmdLRcjW_%yMAysj32m9m_LP$Ck$2g0u-Nxw<7OcL zl9^FKc1O}+mNDV@wbiOMCsA5X=k1g8O=qegbD`bNezf`+1$(5Cv1rW_A6a**xPzh{Ov3U_F9$qg<~#$$>FGQuK=~u2qP2k zYrX%hC{S;B%juibo#F!f^nvAp@VMiJ1gz3zTE^fn&YIESOF3@{!o;d}IFm$x)cp=r;u7()7-!def$^ zFkNZsDH(cdM~T|dn!4zS4tqy8I**5g@51u+V+;Ib_b+QuJNhvs*BB|~8H22U`0suF z_#?+D=Vfha@1vDnyk5rlf|9M6h=`b->$_6Y7EXqACC4UFK&rtd0{-t{krP9LNNn<6{|5y2LqTh2aVA(s26pA+=@pq##WxmI1-GvJ z<_`n9`*UKMh4H)hji21MKun0&x0(*$4;T80@gXZStN%WwF|54Ib12aNnsUcyvhlfp zY5#<_Pf1=*fzE7^Tjh%)v+3z+nIKLjrEAa^JUAaqcMR%5ubv-EY1M@32))~xb5A^||oyiH9Uz*~3hcc$|y zej9JwZyEWYEze43T3FwZi6z{R&c~VmDaTo@v(oo}kN~;@&}B7~jPF%7d99Pbp1g4U z2nk`lqZZxT%2C*=DG@*%y|MD(-qdhTh1(DYzpJE^@|HdTpY=e;Is}_Cc z4K$a@;;3$~0!??wA7?Y0<<9LddQ!QaUa#y?kBfepom)8!VNz_z*3<3!-)`f3fFi(G zfFGus{RON>S3sgkA%~F$);%BEP{$qdNrD93S+Z8EFF{oeRbD3RW&Ni+J4`IKKVtIsc~;_0(KxlRtq z4(%E1R)`_656|dAo_%hZ%FNs!ACa=gtjD3-#QdIIOaevjzm=`re5HBHqmSii=>i3Y z!Z^Dqy&)=n{d=}t(T=9=`pKylB#B9_k(C}G824CDS-DnVSU17zdAm^ez(sIQ<Q#e|dd5q zL;CyO0689!glT>sU zL8F#l>`FfuRpS0~)y9SujuC+0Dq69781tWgeC0mLQX@YxhzeKH*aOCb%KEU#&}!tk z@$G(gc9%V_=n@(0&@5>G+ev+6eeGE>OkTCG4A8d+l}uGLK`y2WD!Y>I#3%Auw9oR~ zo-m2jHy`U{n*%$x%jLoLw%?+MW!tnFqfv*z#tVy90e*DK?JPJa?BONy=-e2yE zUE=s$SmNC@qH&+rLtC=O6sv6Z@ve1HIXefFmYcjhrkd(QFp&VINve6HW}REvueC67 z&=dS2*p&dK_A8qjUS3(`wki|iFLGm`P4@e3JMyHdiz)ri-NnRKlKeAxaGomne@ZuI=Ja$ulzvA0UnssPd~V^Bq+Gr_?SAu#Hte5{9-Npi8Z zEe%NJe{1)``};sDSyc?WUjs(!m0=GG9yU83(W`>|)^wrAfkobXz(%p*B17QQ*Bb65 z{|+~|5<#>8Dbr)%R~x4AN%6wmY8fk6KVT7bEljittPHEX<%Ld~$@JO7d%_fkpolIj zaRA4O1*Ng7saboiG>21D$F1KwGR!YG1`Q>v>;8{0O&IqM|CBD^v{WZRYyhqiidO@o-K?cSKIXOs5tNA8jQMI9OBaB;Pt5Ii4ROoVqE6iMV%X-#Uytf zu<+m;=!$fU`R_+}3=7WRm$Eke0$(;WrCB~^pk!d+qD`#H=qIq7odafNxECW(?oqV+L0E3Zceu7Y*i$_ z!Jjg`D;(OG2_lxZG`2QXilpal-FP_L1>;JFeyev1Fb0UIF{s#SYfm1vKrKO8A6ZH3 z|d?U}NS)l;r5jM~_3mu8onh(!XRKgMV|Y+jL4tweyG7xSb0?Z`*LnGYj0cJ!-Q zrH3s~?k;mar@#HoSD&@_aQWQaSr%rZM16kR1_aLKsAzv1R=&ib<)tyi14Bkg+4tBc zhT>Q{j1mJjwJT@z!P-Oj?kssA;{T+iyw*cT0LDjjrHS6s!9f(T`k{#hK52SM?5~%F!E{{0t#cQ-8hxw(Ba`!dg*iOhdfRWQU==e zYb=3?>+_a~5aIfo2g*thMVTo7C-{;7ZF>maVld+rI-3NzX7;Hn#wXJ2f&63f?~>A` zRgGXJfO|CN9{_&s*@=nk1H}4EycL6+k+GBcv5+#~y#3gigR=fVq#hn0HxkReIg?ph z)w_w6lQy2>we{WZ?(VmusU7}sJJsyA#O3F=KE+4y_y(4d2CKCR0tU%p=-|dc%7ERY zf$`4Tf3Z>y}B#)+2kAE`6VkWwHwF_2b=txao)TnueR!+xKu>dq`gB^mt+@o zv{dOxOS_f&$2ju$7-bBV&(+TsvS<1kl{*WTUa+=hmF(s<3la~Q!F{NP(dQaKOAj?R zJ^b#`4*6?cAHTn{ny-wpqAjH`MCyPnavh|Qjo2R#7ZT&K{aJN;0;+aGSg@eBE$%iQ*?m6yT?wYyOAw>L-m%ZL`1v(&v`Gjg(5MCDLs6xmL|z z;kO@%cuItfWVNmj8tM1|k*w+$8_ucM%*)@P#N)h@?cxT>>+-;KyL zdugMT(`4K$t5?z(pE(Kb2aH(g*`=>jO2A;9D+Uz6=Y&13k##MTda0QBs2i()eEx@U z{xjW2P0ncVQi_y_hcZD)6yu*4pUDMc>4#Q~x;jSWu_GZr$w1Jm=9}FtOWX!awYx_Q z0jo*;@V5F&Vi^;mY+9?B+Lew|XwB#6{s_mU14|-T)3JtX%%&G;?@TOv&D-bi)<#nt z&rL06b(idt?@LjcGEbHT;~vaSj+RUX=AHP5Wbu}=vH~5q zXU0nUXv9pm`LD{v6u~Ge#*{bcNoq%iv&|IJH8D0^tDB#laU^)*jcT zer_yyVQb;*tE}jwp{VH8#o*xxNiMG4`5E_p=e|%h!P;cmT=v=hLef-KeO`m^EN`62 z?9YZZz5D%f8ycZq!l%NuobV!Wy2LbC=5(WDLN@{F z=H~m_K`|7u(D%YowiYO0K4LfUl^Or5!R)7{q~pekGTYG#S3>kMG;HSsOX_765F3q; zM`E|WPgPOp6YPM&?WL^fk z)uJ20?d!|R8I?MaXLZq?{UZxzbiXbsF3c@2eCI8#tx1B*(9g5}%@>x)nE$FfwqL@$!rS423)WN% zBkmKYORdHWW(bgDf*w)eFQUubr33RX?+c?{LFjTp1 zuIOA>%e1?64Hoe3UC%EbzuDSn=F&;k?_6IrzUxm(>M_?7u4nS*xCVZEaX-SqF9>3; z9j^y*RiFK_6`I>}Ajl5HWvoQ3&02^_&p27uJDD(--qO=+ebX)MO)6|`s`=4W^9;GW zZc!e$L0es|&oa3^bNu@n9_*c8F|AmEI*Mvca9yF6G-jf9GeXL?AaA)GW2@Rsdxzt+ znik?Xx)s?fdX1X#y?*ukt>}D>ijg&+rY~i5JR>PMD2!jv>vHJbovhyYkS25;tm{@q zFewWIgq{J>cdZ^<3QY_20!@kGzS%70Dof0CY?jPq_sy(k_x(?CWp7P)ifdq9r5eM- z6IOmn#B9E)rJ^l&zl0Hi;hd4$ngGc?DYJ~$&52)$fgYRLHI{}&(h?Hxk6$!h>*z`% z4V2W5+T}YBth@T%h>G8?3viP!!dr|ghzP-#F$dBj|Dp7^R(G^1ejcUEI;FcTd}vci zLfpzt2n{XS>5t5>+w`|n@C|-PI;Xd>xwkGw8RfpSqs<3?2AqdW&tLx|j^8y9av4*p zFP}6Yw{kO998YL#saD7$tj)Tc4#8ZkL24I7J=b2(+OSQPgu2iKiOo3HUDSC`hQ z{Or~ZaK@#~BU)(IokeY(Cnxz}OB?G;GYhAY2=~*bQPzRj{EfLfF(63C@f;v}BkVbW zF}#B&CWZy|J4us};^XaES>POm=)+TA@>}Z)k?|+6e5%`T)9)wV`@1&X688N$vh(Q` zoebO)AYfcHd*TPdX5qc>yYNQ#g=#_z!VBUDT4ALrjVDh3E4PTdlFD7N`Qxjtn#&aa zre+H38-pgPId*#|gn*O*=E5y5AyGa)BEc={J6v|8W)}{3vQ$>Vyf3-|=tBe_Z;3GV zqWyTL%w?G|9CUSW+qF#IAfJJDsG-6B4SephgNXjU6bTdHT;lw4!B`;q+UE|9>=yf9 zfUDuU?1kB+xP9ZoO@~fh@sIzy8qs|6J3F>e3%|3i)v2Vy3ubvuPYFR;lID-o4QXk; ztbGh-Evg(Y=R65Di~MqhqEiaiojRq~q{|a`c0(@(9T$1iQw)=|^iZ;H$=;wVXFGJ3 zn9+J1^#}^z!YSIJqazPIM^m$9yz_oEds!4@f*F@?+-fsZ+O-YeNtXug#~AtS%o~4q z)no_oywG^ik&8-GQ60Y`-sqk>f=17>|@B#^GMQw}PIZsT`gPt$3RXAsE{Cx~OHrEt+#bA9f;1)=e=El~Z zyN6@XZ0#OfmLm6>uJ1~J>{q6_rvmwnjI~+N@y@=MrDGnJ1vRm3b?}akszPbR8^yof z>nYFTe`Uws2P-&WdeDzCvjKZKaS4^X9r&|4p8SPx$y_6jcAva!>PK0njO^A-LdLWz zvM^!>R;OD673b1`O-m_pJ4o)o8C#S6q)Hy4Gcph12(vfst#gyq9Zc&QUa=9YQM%R? z5h1$7)+b0Zkz89I^V1?uw={AtwAB0EuZfD_f%uthEpiN9Nq5y{mXLNZ`7}PQo#OFS zsIX3?aqQphbl-qgcGTl(rz(B60X|7-y7_I>6yLrXdw%K7ekk*;Jr>MMJKF{C_+Ocf z6rPg*r_>7zcKuOI6U<4~nKj>aS~$P9| zRV##051+Kl-DFapq{#1UlIVLFV1w)Do&WYTLXAZVE*x}=+Kx<@_;06lN?3U|+bm-q z@WT+f`CS1nIAQnneQ;(->BLzhe^@Ws=1iB zw&gs^Fn+0j2{Y?M$ceKYU;L!J<9WWfKVB2ZGM=)8=YJ>2>*eHSeb9@j4o87jo_cGV zzf%JVdiOX!i;n2w>uDTd@3%{Atp%Jh_gh+PNvWIWG7K}pIKL-i?Cc3!YSwYEe`nHc zJmZZ2KepZiD(da~1I5HZ5s?r?uAm|v64FZ{kQlmKQKV72ySuwox<N5$&f#!j6bCnz5s=6NPFva3F>bK0{~>=$V;*J-Kq6tDzBJ|0PAC*6pO;sp<2! z-- zU)Am_R(=_%*o2by0M)&{M7|x1PhDNl-5n!CW)%L36^M6t4qQPJQ8e*~P^vD)oA-kX2+`e#*jQxd)r7D|jLc z@T%=pWR=iPkg}VeZ5l`mB1=kd@6goD7^_b1mpplzy>b`7(&Vj{!u%&sT`MZ>OEh#f zBcxoF8vy}kAL;OKy9ybLhi)(4XBJnG-rtHEGE(p8c4bz^fBYu+H(34%@`KS*R!A$*O zdDBaV%a_}qft{s6ZD46#NtOA#XAb*|ZxK4W^6y`T(V~v1)?)mN>q=C-YLlH1TZyz` zVXn6Wmq@r7#o`?+Xvv4w6o!~yK9ud(LBkc&h>5;+8(N{*Svh$ZH_MYj+i(qEbo{6cKq=v7Lg1#_Rn*D?CNEm;AXko*Qt~g`j zk?uXmgx^)`;kTHF1g~-isIZ3Uywn&OFb?$dqc=o}%Wd6A4+|;I#OUEt27K0r=t3!Z;JvZW1t|%$vp_fjn4cWuVIJ~& zbm5L|HToJN?O-r3qmK61(61CLBE)PL8JTeYv!HbDPG!DJYLMVl?sC6yZtK~1u!@;V zZQVZ3=nQIlNnsiq4cOP5)??8d8TnzZR+aSlUQO z$|%{SICBK#++LT8q5M{wTC!0;HaZ#P!TmG;?e`e}kVjqJkOryIY1hh-gR<()=Z z6FQ08>`H3bI)pW9zo_%=(;ywq&qYKcK{$qt0zpPmynoX2q+KmG*0F4Me(pZ6xmZPL z)&^mV#WU!O1eF1Ufe}uM*E8DqPN=}A&h2G$-y^;sfZd;2o&X|Pyq_(04dPlB_KvbT z(DtG11F+|p%~=5{$R&Q?Yfxn$Ec-Y_wkmnfEen#-=ZG)MAf6wrlm};?WJ-LFHv18! z<5s;_C08A_>Q-D(%q%^n9aY8ZuzbL%Du*!ly`$WcfM|TNGM`8yop0kk^q{Ju3M`nq zFxR2@)SHc7M=4|8jC}qK{=wiOHRxpo(|W?c*48HJJBv+R)Kv8E1)bPwXk;Ij7d599 z=nqv#(ikvI94$#BEk>^=l-cUc{SzjZ_kln(wtvGYM;BLw0 z=DQFSyr*k9ZGFJNl*|8Qvs35N$|8wdk2WN^aLT2o>?m>?cususVGagvt^53zNo0jo z`$-q<{WNzHYe-=7bvNb=%$~11fLg1$x%srU9oMy-ff`<3zluPqCAQT^9^Udlrqj$h z{rz<~QZam}+s}1~YgFHcX87GYFfI=-ivcJHp`oga)558#}DPsw16^fpAe;7bu%FvC1``=o-(L3pbxltp*v7Oeb##lU(==~>Du?jVJ24eWS^dJMm9AkD zOSNX!<80|PZ7@Oha@`jkQq^G(CzS~$*xUP6m%rhJK>#ekANj(qP3|_YlDaWZS!QsI zXHVY54jGN%;v$6rF9+v_n*YppoJs2e4=X$%CDEsC+IX#AG}HFMtJ6Fm>9;=$n}gKV z<)dr_SZS*ZQp(24Ggt-6mT$uFt^5ya@Urpo*hqQJDw0^z%+1DD9*leB(c03`7+9Ke zR0jG@T75k7)VepABwk-VJO@ZDLGI+ugM-z0V`5JM_!o zwhb?(pXrb)rxR*wpslS`RjIv=(X~ua?y!1@ri;2>DOrEsjI!5M5A=IcE2~{~7SLI4(%Z(!<|3=No8Uqdbs82H zl-^NA&9w1gIj?H_?OMa?_`~d@X_KGTLOl63j1nXbt8T)&4+jqj$n!PdIGXW{Z6ZM- zL4z?7kqIp06CP`H{}hKTTt~i?NdBcI*)joTpcBO1&Kw&N=Hh(BS%BQ&jx?7Lc{iA1 zeDr*Hv|36c^i^M7dwY==e{H?vjH!ZycAWvd(j&eps}|9iFYxil@|Dkhnk)vE42()qTJ)%if+$zGX<0V%gH5U+3cOtz~$oIOuY z?j)^To2q(uy*^tvqcowHOFbEakkIvaoFs87zRh2Ji84TugK~bwerdM%)_weXu*=6j z;jT3iNYi=YWR+4}#3eN*`rh(IY?zUqtp-DNA%?p!RYpV!2wwDjQ3 zjIGjGZBnrhnQ{#bN-8*BU7Ow zCw;+3>ouxiVXbg0#M~66CEiJOnGRNBN~|b#1B9LJCpNe*&o(m9w~C4c8HpcSNFZ(; zIs3Nan~I2xOup5b7zmwYt^Rym9CT~dPpXoWG?|L<&B9U<#pc9YUI)T>{qAv($Aa%I z%c41>79p^>umO*xHYMFutJuwDny9F_<)W0ZT!He87f(FYWgU7Nnn-$0I@iJ^;_)4(SO=~vw;+l6yl(Fk#XF5~6hC-) z`#(Gw@mfk^xt~m6jk|bg;y+;R6^~71vNJ#0ciCz)23b_dx6zrcE^aY)BdO6^=Pfn$ zHgjEq4?Qn_p~i^KUf2&U5fK4{tlpafwT)#mB21Ybvu-Cb<=NSRqAo-INJQ~bq1 zqu^n`AuaCYdQzuBz~>Q4{*{5vSmUelSCiLEf&=;H?i{7S`JY=dblhNuHNyR8np9!c zt}0(g)%k|BPYV)@O|~ku)tBJ+xjbeM0%AY&InwDtMJ0pLLNq-}D0^sG+`Yc( zEi27nLY0m7WONZaDI~a3F!{-jR8Xif`X$h3(1Iiqt4D&E`7#;rfBl_jSF!I@qqT^x zQh&43OpN!l-@vl*7lunhHmC&hM$mTO*PM*dnlOE*844jyAAEw>l9^f9XP>IeDz>mO z@E%c|tL%m5zn`vjVW{}H$#SFFF}iY6_xpg?&qK#SHLxwk{6)Q1 zy#2whX5Dh9-HkZcj=s|)A->W3SMMsmh9C$Fli2gEVrPiNDTgPvWn%e_U~!FGsdGv**npxi@SPsd|_BS4pzAJOPhIm&Mx zs#eLsVS=i9=<~_nuiNd^3I5CLB(zX_4A7?39c4|%TybK(N!g@E!B(c$%N)?~fM;x! z3@)47*MAfk>aSkf+I4Ud{Vn*@5ha6`^T7tp81vX(nN%}1?^;a@@)jS>n9ibgg-RH`yM zTUFxSI|bX`VF^OyODd;AuihXx#NG@_wEM7NQU#dKQ#lYS6g2w1_-|;E_;z69LDN_D zHSsD$A7hRMJg%@`euI*n^gg?j&Z`F4Nw1pIqnwLbtX^z1^jprO0|ni?6d)WyOPFz6 zHm1}z^MFnEVd+h3*B9v3CZDO$h=^pW9;q#1b~esOrURRuNhg&r6K+MfL<-kE*_W97 zy&ra_;F0f_Q;~XIQo1-MY%|;J`rW-cD?-&ck)?HE_7<4>NLoI9(fzlMFa>9(Oiq1= zMYhDkDBt-(v7UcNIq$*yEn-}2bzuXns+g3L$RVriQ|K)+q!a}cP1@lNf}l7H4+_!g zc~wuucYh?eMHkDjN&t_|(+$F+Sd#A|v^2Mv3`!=;pPud_1iv}k!m5TN8uH40VtLI$ zwq~oGoJP!@Rd$7|`j^`M@`!o)lab|;R(HOIMGSZE7d&b9vc__fjr~c}&lz?RS6GE6Mhgoi<@Un`v;??NI}Z{ybXI-JMU&) z%v8uMMuh2g{_QEE@M~-{YCPK4$S+uU!(X zBB-AaDPiodjSbh_-~T)?M5fashRdW!1z1mTbljmKbi<_oczMmh10lYCk zOXNt_KbErFu^2Po^CM)SHZI>huJ4>GF7o=U$I(SX>E@9`+SVSkcJL>=^bHRWqe?cnOVhUS=Zn?ZH}NTW5j-p* zNhm6w{+DaE0Q1+`EIbhFVLZ+}^ITB)*Q-UW;{{5Z6T2PUpY^ zuD{p@xl|&Qoo*>)*gpC#I9d^r)Qg}P>EeIJ*4pDLVN1sfEBR-vNIZmvWnKS1qN^ak zdo_;b?Gnjd(naG&O|sCrYsK(u8B+QB3G{OcZ(Rh^O-}P(Pj;A}mqZN&U`sxJnobTc z%IJDRMq(T{Ta|lGsQ5tyNLxmH%)R1i&v3cq#!8!oxnWRvdAno=?MLNCgjcarYb$>3 zVGX_EfNBI-pWvsi{qiPpm0O@gIais12!u2;?V%f15pRO`ZP#q(qI|m(3i66BnJ*B# zYx`Q0R7}9|y5#WICTi;gaYLk_oAH9IAHMvcf0KoAwj7p+8W%Ue4ubcfv0Z-i@){nwti1ehP*15644mG=!ot^+ z)F8kSlf9cnzKWPW=zPAvJ{8S;V}3=!FhqK;j6aGs_}3wFI1fdfHN6`ik4S=ZxznbY z&PQuIb!_56ni)ZWtf&dK{6$6F^_fQ!o)+hjWn))WX=z&7yjtCPk>9DcG#MUSukoAs zGIKd8XcXWXdLYvi<2LaoQyMHVkbmIRkJdd}U6NA$a1+|aXuA|jqpEXusoq(p9^Gy_ z#yI1JM;m0NW#73^ZzLX~4q|`OK3R_$&ueuEuvnZeo}0Pr4=zc@;S%s&o$LpP@sLYu zo;#&dW1sg9cJr;8(Cx5@qKo7eVvXoCr`@$?up3N;=von;&*<=An*DVE+}&}Zpz>VB zFbm3|!I&Rd=>h(R$M2;S2J<#JMy)n2V_m0$0fPbdrbh40qOy!L^YJ1E%q$>WRSeo# z>%$_73p6mfC^Sedy)Ugeun=%P`-Gbrz`b->kQ1Q-bh6I`?OR#7`P_dCK!07AQoW3Q zP)m*owz{Y{AtB-ElU6Ee=;p+G3$A6ddt&0E=haCs7}0SgJ7WcJ>uw~lMy980Y-QM! zw3OJ8^H|O?=0BjSL9<7_c)|Bf8|!=D!0B0ZLWCYk%iV7a_tmj)hHbtxp>VB(bbeKO z?JFrkjM4q|wrc|5L%=&^h7fS!=^e|+%9cQSUwkXH*T!A(=MTa2zju5`=N*~!61?qy zjcpVcsc-%Y>S$Tz^4q5yK8}v6;5zyI9K9D<@oL@a;x50F#A~-KVjS{UEPu{mOTdz8 zu=sWLzWR$hvA@?`&WQl!U2NP>=$=e+XC-%@S9|?R&g)GuOfJfo68@chhi9e3$<6l* zgEpXp0F9Oks~K0=9m*QpX_?$=N8>snY39E{FnkkKET`D76_57lWtJojV z0WRck|8g1GE;_cns9CwXijY5NIARfd3?%<-K%R)gBv^T&G^hahtN(r5V=cEx zh#`EG>U{q669Zjqo%-U%i#JK{1g}Mvj;Lw5OZ_da_ol+yvie$euBvreO%08|rFGue z*Tb8;7#BfqJ{99hBmGxK-X1%EgTm~(gp^Y43r1{I6*O8hT=#I&v-el+h^Q2u>j4rQ zHax<^z`rce`SzZy%}9~`nbG~S6zcGTUiMP}p2X;v5ChJ*Uedt9&zI6?Yx*&iFYnH8 z1$jrt#-{&O*A}73z&&MFp~P|*O*5{`V5HJ6z6+IVxw3 zWO>lCeKV0fZKQ7PKt_;K?%h&Fbv6GBwi|y^lW7NzuRkjH+P{PwDZpe8&-Ga0wS4BH z|L#iqDloHf?%lWZ55avU8v#x|nCi>#7#pqNBFebmv44JgW2N;YEzYTc^6`09Ut-FCoeublaFOmg2y{vrQfdz{TowWj7uwxBpg(vPenn$vj zDRIR0PB~ElLj{aVPE)G<8YulBU%*ME=GdA4LP2jqsldh2DKDf5_7G^~SA&;GkgzjN zXw*+HjB4u-m!c*isc7y~a)LfW4BY6i+zEs)s*w|U9_&%i+N~6>G*c^@r~Bn33P!+x ziN5mcsM1FiuQ<2KYLB;G@26N%F*JvYVatm^(_7?sSoKp5tgYm7>Z!`DhGqO6azXbl zm$1<|yScfzxg}yegP1Kr5PnCg?lswU6xCF_niny#VaNj+#6;Q{Pa*?j@}0W{f3HlD zD>{TvCQesI1KDHS%mb{svX+$aD}R1dUxnM}?q%g<-`K~e3} z@v2e=TE%m#M6V=&|FMkTN?bsmW4;%$xY$ zM3xDF{Spr^WvF%H`Shi;TgoGXl-eF1kwxYi$lu!4_v7NOR(92UVRC9~)z^Mp08tot zGcdG7!Vb$J0>62XEiSz&vRMJzQEAg}`_;^$>Jgaj^6 z_z7N;#cTfpm^5kbR?U@B3cK@LD|`e=y4bt3DX?7x1AxXqG_VTN+uxTgY*qCah;eSW1J$A^ENxEA^Kn!JNlm7n>`UeI|JFi~O zZ%+$T>|^fx+n3mP06lsKH_Zvy*sxax*SdWEYgGUD;Es)>$)jG?{~4aXe%RqV?;lu7 z+|u38{PuqjN@Ut!0Hqod?phTPLreG*o{K84+rG{NgdlZznq*U3j z(QfZuME&(}f3FuO7grCWz?k90*2vU2yL))FhmIu)6(S~L660aBvqko%WFrS-EV?_1 zynjYZRAB3PT6(h~IT$l6mFSC(T-D@qq>+qRTtw^FRJQ&LhIX)Bu_Tf=<+MZ8^T6|r zq{z5P&(J&gku3VwES1ok`%V_%29zjp1sMfyLe7))-%(*UF3+qvj`-UZ-XBiqQ(+Dn z#UiZ+oDTv2siI=}Ulm9p$0C$K29~GYRqXsca({U^Ex0dewX95XD?L>&-KsEi}&hs|-Go&h7+eD_gmzT+&;g$#0iGC9kBBs)? z(oMCqCeOCNZm8apArdKO$WGzLJ-WEeH8NaxasqN?aNU0cH-K|~{G{XK&FwXo2IKMp zcwaodck=m(5x|^!6G<%&ZMFRXf9}wCAtPfg7P!F{{r&eQ-kfxf))5Y+sa!7bOq1!s0y;Regq& z7&8m^U==%0v&4TAdtU*sD!(P0@ci7ayv5?EmNrr{V#riXv@WPE#oa(J#Td1Uzd(1{ zNjONO?io7nyiyL&2se*dcln2e05Hk;>>RnVy_IEQ;qlyd+(|;+YI~x;x8nTy3;kc@ z7JyT?ae)rE%U6$;RYvwLm|m!xal~@5NReSzM)2^iGFf~#0)CDnm|C#yUi&>Ton0C4 zf91T~8~bA(6xrj(IPM^xo;6ku_wG5rL}$XrXIKUyN_hepX>A+h} z&Fueu>P75RISe2d)o8-PHB-(7`Aq7NT3{C{SH|sOWXi>${Rdu?MXsMYfmHH4K=&qq z?veB-^JxCXVl8v-JHHSNA(>8R$hmrIE!1wgrw#i#XcGN_g$G8h?P8PQx|Q>}g*;*K zQtr$eBM~Sp*z`O(IIQA6uVbgd3jeK`d%_G+i7dlkutuH`VAxThL2)Bcj!H`TuT^3J z1i0YdHFY({QUP|m6<8wsmGdI5knNr(_1pFPb*57avdCJH4BX0>TvD4iQJh$lt zuf{+2SWpHYQN1I5g%w}RlaIg-{>yH|?r8j|x6A0yuXtj_fL39L_+ueOu&c*`Khxi5 z%^AFPI1@GulwFg$`mSdE0W*q<<&P1W8i!kny!VC}s;DdnAbRENDgxhG2x9|yArRad zWXZY8#aO@ZS-XMd{p+k9EA-Bzi$mGng%tUAWeo;TJ3F15ZPj{ST~d6=M6Ml7eebL=Ef5s8~L)U_*_i0oZ4&(wmtSf`~6(3VCFB zkAyR?kQ{LqH136)*xL7DI4LB>YdEcqb_Lbm_VUjy|35FyBH|^!J7EY2dAtoCJa~|n zPE$Z3mprPfI>zvqT$((AR~RJ^pfW>nds~nBXrogR8X14?n~J6Z01o0!UR{To^H`vbSM;j+hKz~6*@Q%Q-(>$IrYs3BD|L^VUB>8VGgv}(f)JDU zonqc+=VF4rF!Ae=`bi)7`c-uMRpUcf7;|WKmkyFQ(7ut z73^~5^UW0~=*bTZg33v8u5Z>D@buo)x_{RKE)i(`V>|oAk;+_Bsu%G7O?DR$#6nc> zQXj10lqyU7_*SG0z}7!hIoR`oLw*6L@d~fpgTQ3}5RA+JTFdIc*irr%y7LeX3NeC3 zMKhm+EDkmU10y32f|_7CFg#lK1wFe8_PmxSc^MU(MQ+qM5d5nz zmh0fNiDx!$F$=p)h(&%WB{4CbR!3ANX1#r^xadX1;OMB1rlu1bc>(X4s8bydID_Z4 zJo$tj7y34(-vF+ty#i`}bkMuq;o;#6SWqMOfvWyc_+VNW#5lm&HRKJA?N(=m2+s?6 zH*#_c8h4O{xB!JAcKy=(he3L}1*P2{802Fv^8A3oUJHAj*abQ|I-D$9uE^Uk(QYkN zD${XX3Y*XJ!i0EBp2hH##Cx73Mrf48lj{woRvXE3fUOds3S* z<^w;QmCh~IHT+CqN&lTU*zA236W;R=cwf+hgqBC6+5@EW+vf$5e~pp%&Cgg7yRfqZ z5xboHA0CJcFAwDY1l&L8&4=!6+t79eA=6&Kc(h*9CL0W!S*R&(tz4Ed`V8ORyNd^_wRbA^7lz> z?}<&I)=K$>6d+Rl-^*g{q@IXh@QrFS5KejHdEqaq5F3Lii;mvN-I$0RdwJ~BK+uH& zCj*hkd<+%{G%W}My3i2=ic&y}V_oX8n+&Z&Ob?9>Ua>Pl9gD+M>%_oC2|xF#r=zMJ z+>Tza>@0*^ofTn0mUsE>heD@5)abn8%nujPtb&+NgIs_L(G2_gLD$#^e*e`@U>E;& zaWN7VHPV%)yNu6(<9g_qpZiyI1WIeNW}iJ2Vzgh$ zX>Z_%CFWp@`*YJ$;0aH-{Pl#%3qNMd9{S%TAqo!6{_C<@U*LU)MI$sc`l*j(Wk*Zr zdWm_@5f4@XU~^F{3=q4FkB>j~vILYd^>j00zy81GErf;bLHF1PK;QH7{!a{wycTzH zKuwG_z)}L({ycAg!XA1VexxPYRE zG0#4m@|>uM;1Z}$F?9JnsEoUStrrJCThwhz3H?vGe| z;5ol!94GpJY)S`!IBL&j-*%%4Q3Wx5#&nM5e8J9IfEVem6{~k-qT+v0W_7af=~O!a zu;&r5-`HWz_VE~r452OtJGwA2SXx=Bo~yUW1%Lml;7t76=Q!)94_3F|Nd|>hnz6?- z*SZjV#EGuuJqd^1W2zTgYi5%bz+=-kUlM zsIWixqseif#$?TRC=y=AG zm_a(yYQwpbwXk7rH@P|u&8tMNewE+}-YZ*MXU}jp!Fxbz3(kEIH8MlkA$P5Id4x>K ztn$By;QH0O^7weQ!tlPJrbcjKTU$1`&3}q85TU7xFPE&46jfy76nN5qSNs-sS7R_C zkmF=J{}_<#f#m4N&-p{ojZhq5vIYPCQ=XWQcktNeTRTLju<@L8A^%auQJiikv^w)Q zf9~tb!4i?}B#S1Vf7$Vb^4N0-}0RJ_yhhQXl6Loz5{)qc&HOGbUss;{kxA9+~hdb8NiNhOsn|`68z?bjF z&eEW@kn0aqmI=eO9B8B|?~yyS(O`~WWHRF5zMdcML()Fg!~jOnL_o7@vQWD4XLSW! zO5hWA>5F6IcHY3R;T<-oJsuvfeI64Kt-!9?g4n@5+#-oT2)?JK|0tw5IL+D(U^oY~ zD;wi<8}&n~*c2|_!!C{ULE=Y#0es#o87aS>9@Yx8-tO*Ow{N`ttJT&zc^n*t8n)FI z@q6KJz%CeS%*^UeqcsA)=Z!eP7chuoRV1S}Hsawu5R45+&?>!pRR96oemyu>w-xiV z);sy6hUrbaYQF!yDpqG{hciTgW@`O+CgXk?AS4k*9-ia;*WZC(lXnwy#h1mlvH zRD?k;nEE|B>XkUxqF-)gbs-w8+@;vDrs}Xj??uVDorfyH!6eFJjrtPE*Z2q*GnIdw}L^QtT9^F{qb_o}VEW(PN+yQM6Q6aM|sEm zw9veI*5o{g8|7(R`>}b15Fz;;Lx)p;j(qMT$#4Vc)m8g7{O-bC)w_ZY#wi+WgFw%` ziN0V}elafM4hWK;hkXHS<0+w~!$aXF6Y1#yMi4w59&-NDab;3d?^#cwRay1F(T2p| z*D{-)HMYWimjfCo!Xf2OoxT4qhfz`mMYl!$}WF?Jyl*+Ev+1)@d$^6EWjHF|GR=5$o_<}=+cLxIZnZQZ-e4mNU6!H2N%;U*O=Sy@?NEY&Am1Sx?D znIBZtC#E}Jylat2KMeN271c`U4e=ejQJb!jVTG*IyaW3Y(VraNplUA1x%iCs>+J8a4Pmd zAFwJgx)sgUj`Qz~h_2kw8=*pN$tA&DLrM}Q&?ZjW2kKso!_=;u?UdBS-&PTx+tE{W z=A!P)&2=_tJlXdLO_kIFM~A65rbIt{FP)WLnZ+szTN!SqdGCMSM&$w%QCDZ@BkvKO z@TMBJMg@}`^StM)>u=^~cQh5Kp(!nX@UCp;Y-& zMysa7EcK>_*O%}JXd!xUYO2Sx(h1r`mt5DVLd_V2HD>wg?=>uDA=Y7@b|%n1MmxI= znefU+8llsGG}|fbFg|DB#?q|)Hhh>1$|sa?|9<1T9)qMN#(naXs~mnhqAgt8>y=U* z`gHAC#^I{?k>I|1k&H4$<ULr&78#J6lIOvtLbCO^Od( z_{WyFw2<tS0uTdC#Og;vmaOXffPv8_rLp6iYwgElsCXSO@jIv0VjdhbIv{AS{CP2t-`A}8Xes-Urae1p65u3W@wuBL|ThNsyc;8_r zm;5g@c9xY5&d>*-o-I>);0UQa=F;=Q8k$?3PAH8rLVYEUwZ z@TQD{{=;7%e%s8d6mMkh&QJ>;4@cKJ{(R#Y`QUW!H*s#Y@OHQ0(AC3>rgw=3Ry@x} z$s9w|MsJ^9Npr-Q6kC;3lkKV3>`szz9I>>|H$pMLi{p3uR%R>xB<$d-RXb|vTTMMKyQP%5m-^h*;Wk& zg*1p^JBUebP^g>9S-2c5d3hwl+~LZMc20$TGnI$BrK&UTh%IR{;-3^Wb}pU=Eml;7 zA0<^))AAYVyYCR4oV_NpAxv(v{$5zkN^K|?RoYarnm`OPG}`YNelLPy)`jUTnj;dN z+bF%7PQtF~#WFOV9D&W1Zmu_-*dNI}x00|`t^4uqrPnGTPG@Ob#=k-gKi)GGmb1P~ zq&U(P?OEQ}9~-2*1K&#!!^%C_M5`kC0dut_k5aa3T=B+9oq3Ls)&O#em-A`=R5Q!4 znVmM=pJIWg_#rQ~ooG^`M6lZ%3+Liwy4V}dE{LXV$H`a+_=C71;ZrYRZh`IgVq^Sd zM(uYT{^Xg5t7P#UTg%A_oIX_lNO{fsR5w@Xmb=YsJN52v%@`MqBgbDCZ#`K2qBS%C z^+YznzCc+6Ux?%PiG}-dW_q27=Aph|*#qLYuk92%yUfM0yLoH_2=xIW_glphyPZqxX{>U@tzKvuZ zJvb51KMdWkw|KdGaP#6N*H#`GlGsobS}&vx{xl@%!)({kLXj>hk?`z*4(qz&;o_!j zv9PkusZwGWlNq#SXJ;ohkW@sMbcurl^9e*_=l(dHDE| zQ?E>EFGjdSwet&on>nH&t|*`HS;c(hw+m1`RqG$zM|su)s~$W<7^cQ+2oAP;bCST( zFjh-U+iS-`RoR!S(+BX$6|}_Z4M3&S$D<3jtXuG*2J6#QznF~_54y^NG1F&5!Vlk2 ztxolDYa8`d>%v`*kEVDc)EX;!A|Fb)to{y$dW7vk#ANhAf6y2VDvxf$VowLlHy)m9 zdak9wLBt<;;y4T)2Np;MGmeLc4RM=2B}|7)hJ(FWWNot2tY%L}hWLG58Ox2wZZ>0r z*Sc1kC{wBCAF?R>mhE>3KDc1fuZ^6aX>)2u4_9^>{+;qdu|CCYHN0h3mY-lSUCp;f)Ew5tpX-jG5hVA5Zr;P%NSH)B}m zYERFYTdS}?E#MFu;0i~3o|W|d^N&M4CR#;cW0y_#DW2c^pHD?|jxsgz-Qxpy&ED8L zqMbZ!(VZ#r(FUmVh7B}^jeAoAHOhMZ`yV#RVbSHvG_-8S4 zuGBu?D=xQ*Lk*gxDbe8QO{roT8W{Q(DMms>}7=>?8wdQCC8zE54g7E6+^V<9mt0UnP z2jj3Fdk@QBDm6|}>fYTSp=p=IG{QiN0Rbo;2GeJ>1u;m6+9y`OToTmgrsHAMxt4#s zEvxT z;&v))b(XtbO&Q`snAR&3S?$`*-TW2N!{pXMYE8QhE61@OmuoS|^<+d-uUkocY~axr z->-KjAC7umhAy!VchyS}@Iq%X-O-s;gj!jGXN&ng?pt2jIkz?T8`Uv*D&}w$bbT|z4%tL z`hI(EQHF+9LdKSRp&0|8{3ue;SJxsaTTFzQoOpv-R8Y5qA@KHC{IWyi6x(RD>UdNB z_HDV_f^Kf#)yj?*thR1#q}|p%7`A$-Go1J%)?-GdTG)L)5T6?P&xm7kq?<7Muzs(J za!~GF$75qfuc7IDF8C#PQ?^J1dbKqAXad#v6`4@4ct0`}=4E7?5-w>q z_}*W7A&pnN>ahH7qMzRziQB=}l{#?lt=(!b5*uy3SV$UVmJE)hNRv`W|m>S9u&T;?-th|{SQ6_4=ADeniH z)P<2gt~K%9F;985aT-vJusx?6ekoaPu3r}b4{r4*N1(B#ur$`nV^{i>D z>a20pqg=jsaOLuB@h`_U^(XH~u6n%IHc~(RAgG%?fjmgP=YdX6ua?!kBXE4Nd(Zay zZg=-aJm)N0U%&ME(*q2#m?A{SQp`qO)F|8;iU0XCov7S4)eozO!7`PRBP0hSq9h{@ z2gdUCC(dqW9s-8x3m?B2zJKN+s&DD3E-&gJ>Jjdy*BbP>JlxGA9+D?mw-k!l=y-f3j0bv5dJO~N)sb4B<^Le=Qe*_=vZcy>5BCItVq zF3mRHxrWa|VRJ&FGsxMt(A*ea$g6;?zD<{N%gbH z{vNecx}>iBDA&g*-CbqXqB__0-qkzQ10+jinJa7VKl)Lo{m*20lTin$(S#Gyy0uSX z$AM&?eEj2pV<%m+di*}vr(~ko)23P`ONxK3`wN2Cd1WKE5vq04G$VRy8h%=vsn5x! zycp?}bRQ98q^j*QlQus7UN|`Q>57P@Y$l&eiJ>V)zG7&UdSz|Le{f}6@Hp4x0ZWG8QaU&)X{klS?1h@Q@D^2#~Fgm8B9gEajNA+vW_064U z+>IRdgBgcAvPnrbu159^GDXAob+$*N36Ql&I!28+rjR6|g@y3+VhvJ3+MJGB%Odxq z*gi@|QGv8%Pma>xff1oc!%CXNH@!|9#aC(wGLmXERfWD;jH5(PRo4!(DcXcNtO3ve z6zSzUE>$?129;0C#q032`rqco#TWIW6bw>TkiLmTZLhMjHr&H#7hX4&Gug=J^bPd+ z?hBflS4U;2#WoJ zGN|JEn69$aF0p1rRj6^WxZHc^QJ{*i7PsK`go#23p!}ovkW(K?sQD1u4+5JYI}-#_ zzp+f{;B7dIMGZR8_o*e?wDgm|TF-^Kp>h$>k`s=-QGUxNM9YetaPzWXA6cz$ui$-> zn;(7pt5ylmmfF8K?KiCCBzeFlSE3zf(jXEu+vuB^am{I@N&=rNy0N)#Wz>}WfHDFtXNEPCSP8&