From be838ad0f2709f420b5557c6d9484ebd79ac677d Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Tue, 6 May 2025 08:48:45 +1000 Subject: [PATCH] chore: convert a bunch of manual schema construction to builders --- backend/admin/admin_test.go | 96 ++++++----- backend/admin/local_client.go | 16 +- backend/admin/service.go | 3 +- backend/console/console.go | 16 +- backend/console/console_test.go | 155 +++++++++--------- backend/ingress/view_test.go | 14 +- common/schema/builder/builder_test.go | 9 +- common/schema/map.go | 9 + common/schema/realm.go | 5 + common/schema/schema.go | 5 + common/schema/validate.go | 20 +-- common/schema/verb.go | 13 ++ internal/buildengine/deploy.go | 56 ++++--- internal/buildengine/engine.go | 48 +++--- internal/buildengine/engine_test.go | 10 +- .../buildengine/languageplugin/plugin_test.go | 5 +- .../buildengine/sql_migration_extract_test.go | 7 +- internal/routing/routing.go | 3 +- internal/routing/routing_test.go | 49 +++--- .../schemaeventsource/schemaeventsource.go | 15 +- .../schemaeventsource_test.go | 64 +++++--- internal/sql/sql.go | 5 +- 22 files changed, 357 insertions(+), 266 deletions(-) diff --git a/backend/admin/admin_test.go b/backend/admin/admin_test.go index fc10c54930..cc7c9ecab5 100644 --- a/backend/admin/admin_test.go +++ b/backend/admin/admin_test.go @@ -14,6 +14,7 @@ import ( adminpb "github.com/block/ftl/backend/protos/xyz/block/ftl/admin/v1" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" "github.com/block/ftl/internal/configuration" "github.com/block/ftl/internal/configuration/manager" "github.com/block/ftl/internal/configuration/providers" @@ -141,58 +142,55 @@ func testAdminSecrets( } } -var testSchema = schema.MustValidate(&schema.Schema{ - Realms: []*schema.Realm{{ - Modules: []*schema.Module{ - { - Name: "batmobile", - Comments: []string{"A batmobile comment"}, - Decls: []schema.Decl{ - &schema.Secret{ - Comments: []string{"top secret"}, - Name: "owner", - Type: &schema.String{}, - }, - &schema.Secret{ - Comments: []string{"ultra secret"}, - Name: "horsepower", - Type: &schema.Int{}, - }, - &schema.Config{ - Comments: []string{"car color"}, - Name: "color", - Type: &schema.Ref{Module: "batmobile", Name: "Color"}, - }, - &schema.Config{ - Comments: []string{"car capacity"}, - Name: "capacity", - Type: &schema.Ref{Module: "batmobile", Name: "Capacity"}, - }, - &schema.Enum{ - Comments: []string{"Car colors"}, - Name: "Color", - Type: &schema.String{}, - Variants: []*schema.EnumVariant{ - {Name: "Black", Value: &schema.StringValue{Value: "Black"}}, - {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, - {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, - }, +var testSchema = builder.Schema( + builder.Realm("test").Module( + builder.Module("batmobile"). + Comment("A batmobile comment"). + Decl( + &schema.Secret{ + Comments: []string{"top secret"}, + Name: "owner", + Type: &schema.String{}, + }, + &schema.Secret{ + Comments: []string{"ultra secret"}, + Name: "horsepower", + Type: &schema.Int{}, + }, + &schema.Config{ + Comments: []string{"car color"}, + Name: "color", + Type: &schema.Ref{Module: "batmobile", Name: "Color"}, + }, + &schema.Config{ + Comments: []string{"car capacity"}, + Name: "capacity", + Type: &schema.Ref{Module: "batmobile", Name: "Capacity"}, + }, + &schema.Enum{ + Comments: []string{"Car colors"}, + Name: "Color", + Type: &schema.String{}, + Variants: []*schema.EnumVariant{ + {Name: "Black", Value: &schema.StringValue{Value: "Black"}}, + {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, + {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, }, - &schema.Enum{ - Comments: []string{"Car capacities"}, - Name: "Capacity", - Type: &schema.Int{}, - Variants: []*schema.EnumVariant{ - {Name: "One", Value: &schema.IntValue{Value: int(1)}}, - {Name: "Two", Value: &schema.IntValue{Value: int(2)}}, - {Name: "Four", Value: &schema.IntValue{Value: int(4)}}, - }, + }, + &schema.Enum{ + Comments: []string{"Car capacities"}, + Name: "Capacity", + Type: &schema.Int{}, + Variants: []*schema.EnumVariant{ + {Name: "One", Value: &schema.IntValue{Value: int(1)}}, + {Name: "Two", Value: &schema.IntValue{Value: int(2)}}, + {Name: "Four", Value: &schema.IntValue{Value: int(4)}}, }, }, - }, - }}, - }, -}) + ). + MustBuild(), + ).MustBuild(), +).MustBuild() type mockSchemaRetriever struct { } diff --git a/backend/admin/local_client.go b/backend/admin/local_client.go index b707c6c0f1..51155a2c4f 100644 --- a/backend/admin/local_client.go +++ b/backend/admin/local_client.go @@ -8,6 +8,7 @@ import ( "github.com/alecthomas/types/either" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" cf "github.com/block/ftl/internal/configuration" "github.com/block/ftl/internal/configuration/manager" "github.com/block/ftl/internal/projectconfig" @@ -46,17 +47,14 @@ func (s *diskSchemaRetriever) GetSchema(ctx context.Context) (*schema.Schema, er moduleSchemas <- either.LeftOf[error](module) }() } - realm := &schema.Realm{ - Name: s.projConfig.Name, - Modules: []*schema.Module{}, - } - sch := &schema.Schema{Realms: []*schema.Realm{realm}} + sch := builder.Schema() + realmBuilder := builder.Realm(s.projConfig.Name) errs := []error{} for range len(modules) { result := <-moduleSchemas switch result := result.(type) { case either.Left[*schema.Module, error]: - realm.Upsert(result.Get()) + realmBuilder = realmBuilder.Module(result.Get()) case either.Right[*schema.Module, error]: errs = append(errs, result.Get()) default: @@ -66,5 +64,9 @@ func (s *diskSchemaRetriever) GetSchema(ctx context.Context) (*schema.Schema, er if len(errs) > 0 { return nil, errors.WithStack(errors.Join(errs...)) } - return sch, nil + realm, err := realmBuilder.Build() + if err != nil { + return nil, errors.WithStack(err) + } + return errors.WithStack2(sch.Realm(realm).Build()) } diff --git a/backend/admin/service.go b/backend/admin/service.go index c82817b36e..d7c4157f48 100644 --- a/backend/admin/service.go +++ b/backend/admin/service.go @@ -30,6 +30,7 @@ import ( "github.com/block/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" schemapb "github.com/block/ftl/common/protos/xyz/block/ftl/schema/v1" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" "github.com/block/ftl/common/sha256" islices "github.com/block/ftl/common/slices" "github.com/block/ftl/internal/channels" @@ -75,7 +76,7 @@ type streamSchemaRetriever struct { func (c *streamSchemaRetriever) GetSchema(ctx context.Context) (*schema.Schema, error) { view := c.source.CanonicalView() - return &schema.Schema{Realms: view.Realms}, nil + return errors.WithStack2(builder.Schema(view.Realms...).Build()) } // NewAdminService creates a new Service. diff --git a/backend/console/console.go b/backend/console/console.go index 3f6ba4d17d..d6205d7817 100644 --- a/backend/console/console.go +++ b/backend/console/console.go @@ -22,6 +22,7 @@ import ( ftlv1 "github.com/block/ftl/backend/protos/xyz/block/ftl/v1" schemapb "github.com/block/ftl/common/protos/xyz/block/ftl/schema/v1" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" frontend "github.com/block/ftl/frontend/console" "github.com/block/ftl/internal/buildengine" "github.com/block/ftl/internal/channels" @@ -428,14 +429,17 @@ func (s *Service) sendStreamModulesResp(stream *connect.ServerStream[consolepb.S realms := []*schema.Realm{} for _, realm := range unfilteredSchema.Realms { - realms = append(realms, &schema.Realm{ - External: realm.External, - Name: realm.Name, - Modules: s.filterDeployments(realm), - }) + filteredRealm, err := builder.Realm(realm.Name). + External(realm.External). + Module(s.filterDeployments(realm)...). + Build() + if err != nil { + return errors.Wrap(err, "failed to build filtered realm") + } + realms = append(realms, filteredRealm) } - sch := &schema.Schema{Realms: realms} + sch := &schema.Schema{Pos: schema.Position{}, Realms: realms} builtin := schema.Builtins() for _, realm := range sch.InternalRealms() { realm.Modules = append(realm.Modules, builtin) diff --git a/backend/console/console_test.go b/backend/console/console_test.go index 750104d2a9..6451fac997 100644 --- a/backend/console/console_test.go +++ b/backend/console/console_test.go @@ -6,6 +6,7 @@ import ( "github.com/alecthomas/assert/v2" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" ) func TestVerbSchemaString(t *testing.T) { @@ -16,74 +17,80 @@ func TestVerbSchemaString(t *testing.T) { } ingressVerb := &schema.Verb{ Name: "Ingress", - Request: &schema.Ref{Module: "builtin", Name: "HttpRequest", TypeParameters: []schema.Type{&schema.String{}, &schema.Unit{}, &schema.Unit{}}}, + Request: &schema.Ref{Module: "builtin", Name: "HttpRequest", TypeParameters: []schema.Type{&schema.Unit{}, &schema.Unit{}, &schema.Unit{}}}, Response: &schema.Ref{Module: "builtin", Name: "HttpResponse", TypeParameters: []schema.Type{&schema.String{}, &schema.String{}}}, Metadata: []schema.Metadata{ &schema.MetadataIngress{Type: "http", Method: "GET", Path: []schema.IngressPathComponent{&schema.IngressPathLiteral{Text: "test"}}}, }, } - sch := &schema.Schema{ - Realms: []*schema.Realm{{ - Modules: []*schema.Module{ + sch := builder.Schema( + // TODO: Need a realm + builder.Realm(""). + Module( schema.Builtins(), - {Name: "foo", Decls: []schema.Decl{ - verb, - ingressVerb, - &schema.Data{ - Name: "EchoRequest", - Fields: []*schema.Field{ - {Name: "Name", Type: &schema.String{}}, - {Name: "Nested", Type: &schema.Ref{Module: "foo", Name: "Nested"}}, - {Name: "External", Type: &schema.Ref{Module: "bar", Name: "BarData"}}, - {Name: "Enum", Type: &schema.Ref{Module: "foo", Name: "Color"}}, + builder.Module("foo"). + Decl( + verb, + ingressVerb, + &schema.Data{ + Name: "EchoRequest", + Visibility: schema.VisibilityScopeModule, + Fields: []*schema.Field{ + {Name: "Name", Type: &schema.String{}}, + {Name: "Nested", Type: &schema.Ref{Module: "foo", Name: "Nested"}}, + {Name: "External", Type: &schema.Ref{Module: "bar", Name: "BarData"}}, + {Name: "Enum", Type: &schema.Ref{Module: "foo", Name: "Color"}}, + }, }, - }, - &schema.Data{ - Name: "EchoResponse", - Fields: []*schema.Field{ - {Name: "Message", Type: &schema.String{}}, + &schema.Data{ + Name: "EchoResponse", + Visibility: schema.VisibilityScopeModule, + Fields: []*schema.Field{ + {Name: "Message", Type: &schema.String{}}, + }, }, - }, - &schema.Data{ - Name: "Nested", - Fields: []*schema.Field{ - {Name: "Field", Type: &schema.String{}}, + &schema.Data{ + Name: "Nested", + Visibility: schema.VisibilityScopeModule, + Fields: []*schema.Field{ + {Name: "Field", Type: &schema.String{}}, + }, }, - }, - &schema.Enum{ - Name: "Color", - Visibility: schema.VisibilityScopeModule, - Type: &schema.String{}, - Variants: []*schema.EnumVariant{ - {Name: "Red", Value: &schema.StringValue{Value: "Red"}}, - {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, - {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, + &schema.Enum{ + Name: "Color", + Visibility: schema.VisibilityScopeModule, + Type: &schema.String{}, + Variants: []*schema.EnumVariant{ + {Name: "Red", Value: &schema.StringValue{Value: "Red"}}, + {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, + {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, + }, }, - }, - }}, - {Name: "bar", Decls: []schema.Decl{ - verb, - ingressVerb, - &schema.Data{ - Name: "BarData", - Visibility: schema.VisibilityScopeModule, - Fields: []*schema.Field{ - {Name: "Name", Type: &schema.String{}}, + ). + MustBuild(), + builder.Module("bar"). + Decl( + &schema.Data{ + Name: "BarData", + Visibility: schema.VisibilityScopeModule, + Fields: []*schema.Field{ + {Name: "Name", Type: &schema.String{}}, + }, }, - }}, - }, - }}, - }, - } + ). + MustBuild(), + ). + MustBuild()). + MustBuild() - expected := `data EchoRequest { + expected := `export data EchoRequest { Name String Nested foo.Nested External bar.BarData Enum foo.Color } -data Nested { +export data Nested { Field String } @@ -97,7 +104,7 @@ export enum Color: String { Green = "Green" } -data EchoResponse { +export data EchoResponse { Message String } @@ -114,31 +121,31 @@ func TestVerbSchemaStringIngress(t *testing.T) { Request: &schema.Ref{Module: "builtin", Name: "HttpRequest", TypeParameters: []schema.Type{&schema.Ref{Module: "foo", Name: "FooRequest"}, &schema.Unit{}, &schema.Unit{}}}, Response: &schema.Ref{Module: "builtin", Name: "HttpResponse", TypeParameters: []schema.Type{&schema.Ref{Module: "foo", Name: "FooResponse"}, &schema.String{}}}, Metadata: []schema.Metadata{ - &schema.MetadataIngress{Type: "http", Method: "GET", Path: []schema.IngressPathComponent{&schema.IngressPathLiteral{Text: "foo"}}}, + &schema.MetadataIngress{Type: "http", Method: "POST", Path: []schema.IngressPathComponent{&schema.IngressPathLiteral{Text: "foo"}}}, }, } - sch := &schema.Schema{ - Realms: []*schema.Realm{{ - Modules: []*schema.Module{ - schema.Builtins(), - {Name: "foo", Decls: []schema.Decl{ - verb, - &schema.Data{ - Name: "FooRequest", - Fields: []*schema.Field{ - {Name: "Name", Type: &schema.String{}}, + sch := builder.Schema( + builder.Realm(""). + Module( + builder.Module("foo"). + Decl( + verb, + &schema.Data{ + Name: "FooRequest", + Fields: []*schema.Field{ + {Name: "Name", Type: &schema.String{}}, + }, }, - }, - &schema.Data{ - Name: "FooResponse", - Fields: []*schema.Field{ - {Name: "Message", Type: &schema.String{}}, + &schema.Data{ + Name: "FooResponse", + Fields: []*schema.Field{ + {Name: "Message", Type: &schema.String{}}, + }, }, - }, - }}, - }, - }}, - } + ). + MustBuild()). + MustBuild()). + MustBuild() expected := `// HTTP request structure used for HTTP ingress verbs. export data HttpRequest { @@ -167,8 +174,8 @@ data FooResponse { Message String } -verb Ingress(builtin.HttpRequest) builtin.HttpResponse - +ingress http GET /foo` +verb Ingress(builtin.HttpRequest) builtin.HttpResponse` + " " + ` + +ingress http POST /foo` schemaString, err := verbSchemaString(sch, verb) assert.NoError(t, err) diff --git a/backend/ingress/view_test.go b/backend/ingress/view_test.go index 997dcc4b98..84df3799e3 100644 --- a/backend/ingress/view_test.go +++ b/backend/ingress/view_test.go @@ -8,6 +8,7 @@ import ( "github.com/alecthomas/assert/v2" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" "github.com/block/ftl/internal/log" "github.com/block/ftl/internal/schema/schemaeventsource" ) @@ -18,9 +19,8 @@ func TestSyncView(t *testing.T) { source := schemaeventsource.NewUnattached() view := syncView(ctx, source) - assert.NoError(t, source.PublishModuleForTest(&schema.Module{ - Name: "time", - Decls: []schema.Decl{ + timeModule := builder.Module("time"). + Decl( &schema.Verb{ Name: "time", Metadata: []schema.Metadata{ @@ -33,9 +33,13 @@ func TestSyncView(t *testing.T) { }, }, }, + Request: &schema.Ref{Module: "builtin", Name: "HttpRequest", TypeParameters: []schema.Type{&schema.Unit{}, &schema.Map{Key: &schema.String{}, Value: &schema.String{}}, &schema.Unit{}}}, + Response: &schema.Ref{Module: "builtin", Name: "HttpResponse", TypeParameters: []schema.Type{&schema.Unit{}, &schema.Unit{}}}, }, - }, - })) + ). + MustBuild() + + assert.NoError(t, source.PublishModuleForTest(timeModule)) time.Sleep(time.Millisecond * 100) diff --git a/common/schema/builder/builder_test.go b/common/schema/builder/builder_test.go index 1639256c5d..24a354f6a6 100644 --- a/common/schema/builder/builder_test.go +++ b/common/schema/builder/builder_test.go @@ -9,10 +9,7 @@ import ( ) func TestBuildSchemaError(t *testing.T) { - builder := Schema(). - Realm( - Realm("myrealm"). - Module(Module("service").Decl(&schema.Config{Name: "user"}).MustBuild()).MustBuild()) + builder := Module("service").Decl(&schema.Config{Name: "user"}) _, err := builder.Build() assert.EqualError(t, err, "user: missing config type") } @@ -48,7 +45,3 @@ func TestBuildSchema(t *testing.T) { } assert.Equal(t, expected, actual) } - -func TestBuildModule(t *testing.T) { - -} diff --git a/common/schema/map.go b/common/schema/map.go index 56d6cb6abf..acd80b1051 100644 --- a/common/schema/map.go +++ b/common/schema/map.go @@ -15,6 +15,15 @@ type Map struct { var _ Type = (*Map)(nil) var _ Symbol = (*Map)(nil) +func (m *Map) Validate() error { + if m.Key == nil { + return errorf(m, "map key type missing") + } + if m.Value == nil { + return errorf(m, "map value type missing") + } + return nil +} func (m *Map) Equal(other Type) bool { o, ok := other.(*Map) if !ok { diff --git a/common/schema/realm.go b/common/schema/realm.go index ce9b4e42dd..e75362582c 100644 --- a/common/schema/realm.go +++ b/common/schema/realm.go @@ -25,6 +25,11 @@ type Realm struct { var _ Node = (*Realm)(nil) +// Validate Realm clones, normalises and semantically validates a realm. +func (r *Realm) Validate() (*Realm, error) { + return errors.WithStack2(ValidateModuleInRealm(r, optional.None[*Module]())) +} + func (r *Realm) Position() Position { return r.Pos } func (r *Realm) String() string { out := &strings.Builder{} diff --git a/common/schema/schema.go b/common/schema/schema.go index a29cbe6854..2a292bd20b 100644 --- a/common/schema/schema.go +++ b/common/schema/schema.go @@ -28,6 +28,11 @@ type Schema struct { var _ Node = (*Schema)(nil) +// Validate Schema clones, normalises and semantically validates a schema. +func (s *Schema) Validate() (*Schema, error) { + return errors.WithStack2(ValidateModuleInSchema(s, optional.None[*Module]())) +} + func (s *Schema) Position() Position { return s.Pos } func (s *Schema) String() string { out := &strings.Builder{} diff --git a/common/schema/validate.go b/common/schema/validate.go index aef9ef7081..90b8f9fbb7 100644 --- a/common/schema/validate.go +++ b/common/schema/validate.go @@ -56,16 +56,6 @@ func MustValidate(schema *Schema) *Schema { return clone } -// Validate Schema clones, normalises and semantically validates a schema. -func (s *Schema) Validate() (*Schema, error) { - return errors.WithStack2(ValidateModuleInSchema(s, optional.None[*Module]())) -} - -// Validate Realm clones, normalises and semantically validates a realm. -func (r *Realm) Validate() (*Realm, error) { - return errors.WithStack2(ValidateModuleInRealm(r, optional.None[*Module]())) -} - // ValidateModuleInSchema clones and normalises a schema and semantically validates a single module in it's internal realm. // m can be a new or updated module that will be added to the schema before validation (in the internal realm). func ValidateModuleInSchema(original *Schema, m optional.Option[*Module]) (*Schema, error) { @@ -404,6 +394,9 @@ func (m *Module) Validate() error { duplicateDecls := map[string]Decl{} _ = Visit(m, func(n Node, next func() error) error { //nolint:errcheck + if m == n { + return next() + } if scoped, ok := n.(Scoped); ok { pop := scopes scopes = scopes.PushScope(scoped.Scope()) @@ -415,6 +408,13 @@ func (m *Module) Validate() error { return errors.WithStack(err) } + if n, ok := n.(ValidatedNode); ok && n != m { + if err := n.Validate(); err != nil { + merr = append(merr, err) + return nil + } + } + if n, ok := n.(Decl); ok { tname := typeName(n) duplKey := tname + ":" + n.GetName() diff --git a/common/schema/verb.go b/common/schema/verb.go index c150670696..be840f455c 100644 --- a/common/schema/verb.go +++ b/common/schema/verb.go @@ -61,6 +61,19 @@ func (v *Verb) Kind() VerbKind { } } +func (v *Verb) Validate() error { + if !ValidateName(v.Name) { + return errorf(v, "invalid name %q", v.Name) + } + if v.Request == nil { + return errorf(v, "%s: missing request", v.Name) + } + if v.Response == nil { + return errorf(v, "%s: missing response", v.Name) + } + return nil +} + func (v *Verb) Position() Position { return v.Pos } func (v *Verb) schemaDecl() {} diff --git a/internal/buildengine/deploy.go b/internal/buildengine/deploy.go index c22efb0951..e91f419dd6 100644 --- a/internal/buildengine/deploy.go +++ b/internal/buildengine/deploy.go @@ -26,6 +26,7 @@ import ( schemapb "github.com/block/ftl/common/protos/xyz/block/ftl/schema/v1" "github.com/block/ftl/common/reflect" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" "github.com/block/ftl/common/sha256" "github.com/block/ftl/common/slices" "github.com/block/ftl/internal/key" @@ -187,12 +188,11 @@ func (c *DeployCoordinator) processEvents(ctx context.Context) { if !c.schemaSource.Live() { logger.Debugf("Schema source is not live, skipping initial sync.") c.SchemaUpdates <- SchemaUpdatedEvent{ - schema: &schema.Schema{ - Realms: []*schema.Realm{{ - Name: c.projectConfig.Name, - Modules: []*schema.Module{schema.Builtins()}, - }}, - }, + schema: builder.Schema( + builder.Realm(c.projectConfig.Name). + Module(schema.Builtins()). + MustBuild()). + MustBuild(), } } else { c.schemaSource.WaitForInitialSync(ctx) @@ -200,10 +200,11 @@ func (c *DeployCoordinator) processEvents(ctx context.Context) { // If there are no realms yet, initialise the internal. sch := c.schemaSource.CanonicalView() if len(sch.Realms) == 0 { - sch.Realms = []*schema.Realm{{ - Name: c.projectConfig.Name, - Modules: []*schema.Module{schema.Builtins()}, - }} + sch.Realms = []*schema.Realm{ + builder.Realm(c.projectConfig.Name). + Module(schema.Builtins()). + MustBuild(), + } } c.SchemaUpdates <- SchemaUpdatedEvent{schema: sch} @@ -503,20 +504,18 @@ func (c *DeployCoordinator) mergePendingDeployment(d *pendingDeploy, old *pendin func (c *DeployCoordinator) invalidModulesForDeployment(originalSch *schema.Schema, deployment *pendingDeploy, modulesToCheck []string) map[string]bool { out := map[string]bool{} - sch := &schema.Schema{} + schemaBuilder := builder.Schema() for _, realm := range originalSch.Realms { - newRealm := &schema.Realm{ - Name: realm.Name, - External: realm.External, - } - sch.Realms = append(sch.Realms, newRealm) + newRealm := builder.Realm(realm.Name).External(realm.External) for _, module := range realm.Modules { if _, ok := deployment.modules[module.Name]; ok { continue } - newRealm.Modules = append(newRealm.Modules, reflect.DeepCopy(module)) + newRealm.Module(reflect.DeepCopy(module)) } + schemaBuilder.Realm(newRealm.MustBuild()) } + sch := schemaBuilder.MustBuild() for _, m := range deployment.modules { for _, realm := range sch.Realms { if realm.External { @@ -544,10 +543,7 @@ func (c *DeployCoordinator) publishUpdatedSchema(ctx context.Context, updatedMod logger := log.FromContext(ctx) overridden := map[string]bool{} toRemove := map[string]bool{} - realm := &schema.Realm{Name: c.projectConfig.Name} - sch := &schema.Schema{ - Realms: []*schema.Realm{realm}, - } + realmBuilder := builder.Realm(c.projectConfig.Name) for _, d := range append(toDeploy, deploying...) { if !d.publishInSchema { continue @@ -557,7 +553,7 @@ func (c *DeployCoordinator) publishUpdatedSchema(ctx context.Context, updatedMod continue } overridden[mod.moduleName()] = true - realm.Modules = append(realm.Modules, mod.schema) + realmBuilder.Module(mod.schema) } for mod := range d.waitingForModules { toRemove[mod] = true @@ -567,8 +563,20 @@ func (c *DeployCoordinator) publishUpdatedSchema(ctx context.Context, updatedMod if _, ok := overridden[mod.Name]; ok { continue } - realm.Modules = append(realm.Modules, reflect.DeepCopy(mod)) + realmBuilder.Module(reflect.DeepCopy(mod)) + } + + realm, err := realmBuilder.Build() + if err != nil { + logger.Errorf(err, "failed to build realm") + return + } + sch, err := builder.Schema(realm).Build() + if err != nil { + logger.Errorf(err, "failed to build schema") + return } + // remove modules that we need to rebuild so that the schema is valid for { foundMoreToRemove := false @@ -598,7 +606,7 @@ func (c *DeployCoordinator) publishUpdatedSchema(ctx context.Context, updatedMod break } - sch, err := sch.Validate() + sch, err = sch.Validate() if err != nil { logger.Errorf(err, "Deploy coordinator could not publish invalid schema") return diff --git a/internal/buildengine/engine.go b/internal/buildengine/engine.go index cea583c171..96e9b871b5 100644 --- a/internal/buildengine/engine.go +++ b/internal/buildengine/engine.go @@ -6,7 +6,6 @@ import ( "crypto/sha256" "fmt" "runtime" - "sort" "strings" "sync" "time" @@ -26,6 +25,7 @@ import ( langpb "github.com/block/ftl/backend/protos/xyz/block/ftl/language/v1" "github.com/block/ftl/common/reflect" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" "github.com/block/ftl/common/slices" "github.com/block/ftl/internal/buildengine/languageplugin" "github.com/block/ftl/internal/dev" @@ -1115,11 +1115,11 @@ func (e *Engine) handleDependencyCycleError(ctx context.Context, depErr Dependen fakeDeps[dep] = sch continue } + // not build yet, probably due to dependency cycle - fakeDeps[dep] = &schema.Module{ - Name: dep, - Comments: []string{"Dependency not built yet due to dependency cycle"}, - } + fakeDeps[dep] = builder.Module(dep). + Comment("Dependency not built yet due to dependency cycle"). + MustBuild() } _, _, _ = e.build(ctx, module, fakeDeps, ignoredSchemas) //nolint:errcheck close(ignoredSchemas) @@ -1194,7 +1194,14 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[ return "", nil, errors.Errorf("module %q not found", moduleName) } - sch := &schema.Schema{Realms: []*schema.Realm{{Modules: maps.Values(builtModules)}}} //nolint:exptostd + realm, err := builder.Realm("").Module(maps.Values(builtModules)...).Build() + if err != nil { + return "", nil, errors.Wrap(err, "failed to build realm") + } + sch, err := builder.Schema(realm).Build() + if err != nil { + return "", nil, errors.Wrap(err, "failed to build schema") + } configProto, err := langpb.ModuleConfigToProto(meta.module.Config.Abs()) if err != nil { @@ -1285,25 +1292,28 @@ func (e *Engine) gatherSchemas( } func (e *Engine) syncNewStubReferences(ctx context.Context, newModules map[string]*schema.Module, metasMap map[string]moduleMeta) error { - fullSchema := &schema.Schema{} //nolint:exptostd + schemaBuilder := builder.Schema() for _, r := range e.targetSchema.Load().Realms { - realm := &schema.Realm{ - Name: r.Name, - External: r.External, - } - if !realm.External { - realm.Modules = maps.Values(newModules) + realmBuilder := builder.Realm(r.Name).External(r.External) + if !r.External { + realmBuilder = realmBuilder.Module(maps.Values(newModules)...) } for _, module := range r.Modules { - if _, ok := newModules[module.Name]; !ok || realm.External { - realm.Modules = append(realm.Modules, module) + if _, ok := newModules[module.Name]; !ok || r.External { + realmBuilder = realmBuilder.Module(module) } } - sort.SliceStable(realm.Modules, func(i, j int) bool { - return realm.Modules[i].Name < realm.Modules[j].Name - }) - fullSchema.Realms = append(fullSchema.Realms, realm) + realm, err := realmBuilder.Build() + if err != nil { + return errors.Wrapf(err, "could not build realm %s", r.Name) + } + schemaBuilder.Realm(realm) + } + + fullSchema, err := schemaBuilder.Build() + if err != nil { + return errors.Wrap(err, "could not build full schema") } return errors.WithStack(SyncStubReferences(ctx, diff --git a/internal/buildengine/engine_test.go b/internal/buildengine/engine_test.go index 53db0e8dc0..559ae1f0e5 100644 --- a/internal/buildengine/engine_test.go +++ b/internal/buildengine/engine_test.go @@ -9,6 +9,7 @@ import ( errors "github.com/alecthomas/errors" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" "github.com/block/ftl/internal/buildengine" "github.com/block/ftl/internal/log" "github.com/block/ftl/internal/projectconfig" @@ -34,9 +35,8 @@ func TestGraph(t *testing.T) { defer engine.Close() // Import the schema from the third module, simulating a remote schema. - otherSchema := &schema.Module{ - Name: "other", - Decls: []schema.Decl{ + otherSchema := builder.Module("other"). + Decl( &schema.Data{ Name: "EchoRequest", Fields: []*schema.Field{ @@ -54,8 +54,8 @@ func TestGraph(t *testing.T) { Request: &schema.Ref{Module: "other", Name: "EchoRequest"}, Response: &schema.Ref{Module: "other", Name: "EchoResponse"}, }, - }, - } + ). + MustBuild() engine.Import(ctx, "test", otherSchema) expected := map[string][]string{ diff --git a/internal/buildengine/languageplugin/plugin_test.go b/internal/buildengine/languageplugin/plugin_test.go index 732743fc3e..74d7a060a2 100644 --- a/internal/buildengine/languageplugin/plugin_test.go +++ b/internal/buildengine/languageplugin/plugin_test.go @@ -17,6 +17,7 @@ import ( langpb "github.com/block/ftl/backend/protos/xyz/block/ftl/language/v1" "github.com/block/ftl/common/builderrors" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" "github.com/block/ftl/internal/log" "github.com/block/ftl/internal/moduleconfig" "github.com/block/ftl/internal/projectconfig" @@ -132,7 +133,7 @@ func setUp() (context.Context, *LanguagePlugin, *mockPluginClient, BuildContext) Dir: "test/dir", Language: "test-lang", }, - Schema: &schema.Schema{Realms: []*schema.Realm{{Name: "test"}}}, + Schema: builder.Schema(builder.Realm("test").MustBuild()).MustBuild(), Dependencies: []string{}, } return ctx, plugin, mockImpl, bctx @@ -269,7 +270,7 @@ func TestRebuilds(t *testing.T) { checkResult(t, <-result, "first build") // send rebuild request with updated schema - bctx.Schema.Realms[0].Modules = append(bctx.Schema.Realms[0].Modules, &schema.Module{Name: "another"}) + bctx.Schema.Realms[0].Modules = append(bctx.Schema.Realms[0].Modules, builder.Module("another").MustBuild()) sch, err := bctx.Schema.Validate() assert.NoError(t, err, "schema should be valid") result = beginBuild(ctx, plugin, bctx, true) diff --git a/internal/buildengine/sql_migration_extract_test.go b/internal/buildengine/sql_migration_extract_test.go index 6a225fd307..881efa9a2f 100644 --- a/internal/buildengine/sql_migration_extract_test.go +++ b/internal/buildengine/sql_migration_extract_test.go @@ -11,6 +11,7 @@ import ( "github.com/block/scaffolder" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" "github.com/block/ftl/common/sha256" "github.com/block/ftl/internal/moduleconfig" ) @@ -26,7 +27,7 @@ func TestExtractMigrations(t *testing.T) { // Define schema with a database declaration db := &schema.Database{Name: "testdb"} - sch := &schema.Module{Decls: []schema.Decl{db}} + sch := builder.Module("test").Decl(db).MustBuild() // Test files, err := extractSQLMigrations(log.ContextWithNewDefaultLogger(t.Context()), getAbsModuleConfig(t, tmpDir, "db"), sch, targetDir) @@ -47,7 +48,7 @@ func TestExtractMigrations(t *testing.T) { t.Run("Empty migrations directory", func(t *testing.T) { tmpDir := t.TempDir() - sch := &schema.Module{Decls: []schema.Decl{}} + sch := builder.Module("test").MustBuild() files, err := extractSQLMigrations(log.ContextWithNewDefaultLogger(t.Context()), getAbsModuleConfig(t, tmpDir, "db"), sch, t.TempDir()) assert.NoError(t, err) @@ -56,7 +57,7 @@ func TestExtractMigrations(t *testing.T) { t.Run("Missing migrations directory", func(t *testing.T) { tmpDir := t.TempDir() - sch := &schema.Module{Decls: []schema.Decl{}} + sch := builder.Module("test").MustBuild() files, err := extractSQLMigrations(log.ContextWithNewDefaultLogger(t.Context()), getAbsModuleConfig(t, tmpDir, "/non/existent/dir"), sch, t.TempDir()) assert.NoError(t, err) diff --git a/internal/routing/routing.go b/internal/routing/routing.go index fac8afcfeb..a09dfbef7c 100644 --- a/internal/routing/routing.go +++ b/internal/routing/routing.go @@ -9,6 +9,7 @@ import ( "github.com/alecthomas/types/pubsub" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" "github.com/block/ftl/internal/channels" "github.com/block/ftl/internal/key" "github.com/block/ftl/internal/log" @@ -101,7 +102,7 @@ func extractRoutes(ctx context.Context, sch *schema.Schema) RouteView { logger := log.FromContext(ctx) if sch == nil { - return RouteView{moduleToDeployment: map[string]key.Deployment{}, byDeployment: map[string]*url.URL{}, schema: &schema.Schema{}} + return RouteView{moduleToDeployment: map[string]key.Deployment{}, byDeployment: map[string]*url.URL{}, schema: builder.Schema().MustBuild()} } modules := sch.InternalModules() diff --git a/internal/routing/routing_test.go b/internal/routing/routing_test.go index d3bef1287a..9ef7f9033b 100644 --- a/internal/routing/routing_test.go +++ b/internal/routing/routing_test.go @@ -11,6 +11,7 @@ import ( "github.com/alecthomas/types/optional" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" "github.com/block/ftl/internal/key" "github.com/block/ftl/internal/log" "github.com/block/ftl/internal/schema/schemaeventsource" @@ -18,34 +19,38 @@ import ( func TestRouting(t *testing.T) { events := schemaeventsource.NewUnattached() - assert.NoError(t, events.PublishModuleForTest(&schema.Module{ - Name: "time", - Runtime: &schema.ModuleRuntime{ - Deployment: &schema.ModuleRuntimeDeployment{ - DeploymentKey: deploymentKey(t, "dpl-default-time-sjkfislfjslfas"), - }, - Runner: &schema.ModuleRuntimeRunner{ - Endpoint: "http://time.ftl", - }, - }, - })) + assert.NoError(t, events.PublishModuleForTest( + builder.Module("time"). + Runtime( + &schema.ModuleRuntime{ + Deployment: &schema.ModuleRuntimeDeployment{ + DeploymentKey: deploymentKey(t, "dpl-default-time-sjkfislfjslfas"), + }, + Runner: &schema.ModuleRuntimeRunner{ + Endpoint: "http://time.ftl", + }, + }, + ). + MustBuild(), + )) rt := New(log.ContextWithNewDefaultLogger(context.TODO()), events) current := rt.Current() assert.Equal(t, optional.Ptr(must.Get(url.Parse("http://time.ftl"))), current.GetForModule("time")) assert.Equal(t, optional.None[url.URL](), current.GetForModule("echo")) - assert.NoError(t, events.PublishModuleForTest(&schema.Module{ - Name: "echo", - Runtime: &schema.ModuleRuntime{ - Deployment: &schema.ModuleRuntimeDeployment{ - DeploymentKey: deploymentKey(t, "dpl-default-echo-sjkfiaslfjslfs"), - }, - Runner: &schema.ModuleRuntimeRunner{ - Endpoint: "http://echo.ftl", - }, - }, - })) + assert.NoError(t, events.PublishModuleForTest( + builder.Module("echo"). + Runtime(&schema.ModuleRuntime{ + Deployment: &schema.ModuleRuntimeDeployment{ + DeploymentKey: deploymentKey(t, "dpl-default-echo-sjkfiaslfjslfs"), + }, + Runner: &schema.ModuleRuntimeRunner{ + Endpoint: "http://echo.ftl", + }, + }). + MustBuild(), + )) time.Sleep(time.Millisecond * 250) current = rt.Current() diff --git a/internal/schema/schemaeventsource/schemaeventsource.go b/internal/schema/schemaeventsource/schemaeventsource.go index 110e133410..664fa37e2a 100644 --- a/internal/schema/schemaeventsource/schemaeventsource.go +++ b/internal/schema/schemaeventsource/schemaeventsource.go @@ -16,6 +16,7 @@ import ( ftlv1 "github.com/block/ftl/backend/protos/xyz/block/ftl/v1" "github.com/block/ftl/common/reflect" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" islices "github.com/block/ftl/common/slices" "github.com/block/ftl/internal/key" "github.com/block/ftl/internal/log" @@ -44,7 +45,7 @@ func (v *View) GetCanonical() *schema.Schema { return v.eventSource.view.Load(). func NewUnattached() *EventSource { return &EventSource{ events: pubsub.New[schema.Notification](), - view: atomic.New(¤tState{schema: &schema.Schema{}, activeChangesets: map[key.Changeset]*schema.Changeset{}}), + view: atomic.New(¤tState{schema: builder.Schema().MustBuild(), activeChangesets: map[key.Changeset]*schema.Changeset{}}), live: atomic.New[bool](false), initialSyncComplete: make(chan struct{}), subscribeLock: &sync.Mutex{}, @@ -116,7 +117,15 @@ func (e *EventSource) ActiveChangesets() map[key.Changeset]*schema.Changeset { } func (e *EventSource) PublishModuleForTest(module *schema.Module) error { - return errors.WithStack(e.Publish(&schema.FullSchemaNotification{Schema: &schema.Schema{Realms: []*schema.Realm{{Modules: []*schema.Module{module}}}}})) + realm, err := builder.Realm("", module).Build() // TODO: Realm name should not be empty + if err != nil { + return errors.WithStack(err) + } + sch, err := builder.Schema(realm).Build() + if err != nil { + return errors.WithStack(err) + } + return errors.WithStack(e.Publish(&schema.FullSchemaNotification{Schema: sch})) } // Publish an event to the EventSource. @@ -196,7 +205,7 @@ func (e *EventSource) Publish(event schema.Notification) error { modules = er.Modules existingRealm = er } else { - existingRealm = &schema.Realm{Name: realm.Name, External: realm.External} + existingRealm = builder.Realm(realm.Name).External(true).MustBuild() clone.schema.Realms = append(clone.schema.Realms, existingRealm) realms[realm.Name] = existingRealm } diff --git a/internal/schema/schemaeventsource/schemaeventsource_test.go b/internal/schema/schemaeventsource/schemaeventsource_test.go index b839db2ee4..76a4475696 100644 --- a/internal/schema/schemaeventsource/schemaeventsource_test.go +++ b/internal/schema/schemaeventsource/schemaeventsource_test.go @@ -16,6 +16,7 @@ import ( "github.com/block/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" schemapb "github.com/block/ftl/common/protos/xyz/block/ftl/schema/v1" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" "github.com/block/ftl/internal/channels" "github.com/block/ftl/internal/key" "github.com/block/ftl/internal/log" @@ -59,29 +60,24 @@ func TestSchemaEventSource(t *testing.T) { panic("unreachable") } - time1 := &schema.Module{ - Name: "time", - Decls: []schema.Decl{ - &schema.Verb{ - Name: "time", - Request: &schema.Unit{}, - Response: &schema.Time{}, - }, - }, - } - echo1 := &schema.Module{ - Name: "echo", - Decls: []schema.Decl{ + time1 := builder.Module("time"). + Decl(&schema.Verb{ + Name: "time", + Request: &schema.Unit{}, + Response: &schema.Time{}, + }). + MustBuild() + echo1 := builder.Module("echo"). + Decl( &schema.Verb{ Name: "echo", Request: &schema.String{}, Response: &schema.String{}, }, - }, - } - time2 := &schema.Module{ - Name: "time", - Decls: []schema.Decl{ + ). + MustBuild() + time2 := builder.Module("time"). + Decl( &schema.Verb{ Name: "time", Request: &schema.Unit{}, @@ -92,8 +88,8 @@ func TestSchemaEventSource(t *testing.T) { Request: &schema.Unit{}, Response: &schema.String{}, }, - }, - } + ). + MustBuild() time1.ModRuntime().ModDeployment().DeploymentKey = key.NewDeploymentKey("test", "time") echo1.ModRuntime().ModDeployment().DeploymentKey = key.NewDeploymentKey("test", "echo") time2.ModRuntime().ModDeployment().DeploymentKey = key.NewDeploymentKey("test", "time") @@ -133,7 +129,7 @@ func TestSchemaEventSource(t *testing.T) { assert.True(t, changes.WaitForInitialSync(waitCtx)) var expected schema.Notification = &schema.FullSchemaNotification{ - Schema: &schema.Schema{Realms: []*schema.Realm{{Modules: []*schema.Module{time1}}}}, + Schema: builder.Schema(builder.Realm("", time1).MustBuild()).MustBuild(), } assertEqual(t, expected, recv(t)) @@ -147,7 +143,14 @@ func TestSchemaEventSource(t *testing.T) { } actual := recv(t) assertEqual(t, expected, actual) - assertEqual(t, &schema.Schema{Realms: []*schema.Realm{{Modules: []*schema.Module{schema.Builtins(), time1, echo1}}}}, changes.CanonicalView()) + expectedCanonical := builder.Schema( + builder.Realm("", // TODO: This should be something + schema.Builtins(), + time1, + echo1, + ).MustBuild(), + ).MustBuild() + assertEqual(t, expectedCanonical, changes.CanonicalView()) }) t.Run("Mutation", func(t *testing.T) { @@ -174,7 +177,14 @@ func TestSchemaEventSource(t *testing.T) { } actual := recv(t) assertEqual(t, expected, actual) - assertEqual(t, &schema.Schema{Realms: []*schema.Realm{{Modules: []*schema.Module{schema.Builtins(), time2, echo1}}}}, changes.CanonicalView()) + expectedCanonical := builder.Schema( + builder.Realm("", // TODO: This should be something + schema.Builtins(), + time2, + echo1, + ).MustBuild(), + ).MustBuild() + assertEqual(t, expectedCanonical, changes.CanonicalView()) }) t.Run("Delete", func(t *testing.T) { @@ -203,7 +213,13 @@ func TestSchemaEventSource(t *testing.T) { } actual := recv(t) assertEqual(t, expected, actual) - assertEqual(t, &schema.Schema{Realms: []*schema.Realm{{Modules: []*schema.Module{schema.Builtins(), time2}}}}, changes.CanonicalView()) + expectedCanonical := builder.Schema( + builder.Realm("", // TODO: This should be something + schema.Builtins(), + time2, + ).MustBuild(), + ).MustBuild() + assertEqual(t, expectedCanonical, changes.CanonicalView()) }) } diff --git a/internal/sql/sql.go b/internal/sql/sql.go index 9eca38d622..99a6db5eca 100644 --- a/internal/sql/sql.go +++ b/internal/sql/sql.go @@ -18,6 +18,7 @@ import ( "golang.org/x/text/language" "github.com/block/ftl/common/schema" + "github.com/block/ftl/common/schema/builder" "github.com/block/ftl/common/slices" "github.com/block/ftl/common/strcase" "github.com/block/ftl/internal" @@ -90,9 +91,7 @@ func AddDatabaseDeclsToSchema(ctx context.Context, projectRoot string, mc module } // Generate queries for each database (one config per database) - sch := &schema.Module{ - Name: mc.Module, - } + sch := builder.Module(mc.Module).MustBuild() for i, m := range out.InternalModules() { if m.Name == mc.Module { out.InternalModules()[i] = sch