diff --git a/docs/docs/reference/project-files/index.md b/docs/docs/reference/project-files/index.md index 4cac2803119..78827be17b1 100644 --- a/docs/docs/reference/project-files/index.md +++ b/docs/docs/reference/project-files/index.md @@ -14,6 +14,12 @@ It is possible to define resources (such as [models](models.md), [metrics-views] ::: +:::info Custom fields + +Rill validates project files strictly and rejects unknown properties, with one exception: properties prefixed with `custom_` (for example, `custom_owner: data-team`) are ignored by validation. You can add such properties at any nesting level, in any project file (including `rill.yaml`), to attach your own metadata for use by external tooling. + +::: + Projects can simply be rehydrated from Rill project files into an explorable data application as long as there is sufficient access and credentials to the source data - figuring out the dependencies, pulling down data, & validating your model queries and metrics view configurations. The result is a set of functioning exploratory dashboards. You can see a few different example projects by visiting our [example GitHub repository](https://github.com/rilldata/rill-examples). diff --git a/runtime/parser/parse_node.go b/runtime/parser/parse_node.go index c72f3addc1a..a08406089df 100644 --- a/runtime/parser/parse_node.go +++ b/runtime/parser/parse_node.go @@ -350,7 +350,7 @@ func (p *Parser) decodeNodeYAML(node *Node, knownFields bool, dst any) error { // Using node.YAMLRaw instead of node.YAML because we need to set KnownFields for metrics views dec := yaml.NewDecoder(strings.NewReader(node.YAMLRaw)) dec.KnownFields(true) - err = dec.Decode(dst) + err = filterCustomFieldErrors(dec.Decode(dst)) } else { err = node.YAML.Decode(dst) } diff --git a/runtime/parser/parse_rillyaml.go b/runtime/parser/parse_rillyaml.go index 1814d5bc9d7..cba0995204d 100644 --- a/runtime/parser/parse_rillyaml.go +++ b/runtime/parser/parse_rillyaml.go @@ -134,7 +134,7 @@ func (p *Parser) parseRillYAML(ctx context.Context, path string) error { dec := yaml.NewDecoder(strings.NewReader(data)) dec.KnownFields(true) - err = dec.Decode(tmp) + err = filterCustomFieldErrors(dec.Decode(tmp)) if err != nil && !errors.Is(err, io.EOF) { return newYAMLError(err) } diff --git a/runtime/parser/parser.go b/runtime/parser/parser.go index 225b66e0941..fb79fd03244 100644 --- a/runtime/parser/parser.go +++ b/runtime/parser/parser.go @@ -15,6 +15,7 @@ import ( runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime/drivers" "github.com/rilldata/rill/runtime/pkg/fileutil" + "gopkg.in/yaml.v3" ) // Built-in parser limits @@ -1216,6 +1217,34 @@ func newYAMLError(err error) error { } } +// yamlCustomFieldErrRegexp matches strict YAML decoding errors for unknown fields prefixed with "custom_" +var yamlCustomFieldErrRegexp = regexp.MustCompile(`^line \d+: field custom_\S* not found in type `) + +// filterCustomFieldErrors removes strict YAML decoding errors for unknown fields prefixed with "custom_" at any nesting depth. +// Such fields are reserved for user-defined metadata and should not fail parsing. +// It returns nil if all the errors were for "custom_" fields. +func filterCustomFieldErrors(err error) error { + var typeErr *yaml.TypeError + if !errors.As(err, &typeErr) { + return err + } + + var remaining []string + for _, msg := range typeErr.Errors { + if !yamlCustomFieldErrRegexp.MatchString(msg) { + remaining = append(remaining, msg) + } + } + + if len(remaining) == 0 { + return nil + } + if len(remaining) == len(typeErr.Errors) { + return err + } + return &yaml.TypeError{Errors: remaining} +} + // duckDBErrLineRegexp matches the line number in a DuckDB parser error var duckDBErrLineRegexp = regexp.MustCompile(`\nLINE (\d+):`) diff --git a/runtime/parser/parser_test.go b/runtime/parser/parser_test.go index d8b28163a89..0eacd180520 100644 --- a/runtime/parser/parser_test.go +++ b/runtime/parser/parser_test.go @@ -2857,6 +2857,85 @@ tests: requireResourcesAndErrors(t, p, resources, nil) } +func TestCustomFields(t *testing.T) { + // Unknown fields prefixed with "custom_" are reserved for user-defined metadata, + // and should be exempt from strict validation at any nesting depth. + // Other unknown fields should still produce a parse error. + files := map[string]string{ + `rill.yaml`: ` +custom_project_tag: hello +`, + `models/m1.sql`: `SELECT 1 AS id`, + `metrics_views/mv1.yaml`: ` +type: metrics_view +version: 1 +model: m1 +custom_owner: team-a +dimensions: +- name: foo + expression: id + custom_tag: bar +measures: +- name: count + expression: COUNT(*) + custom_format: usd +`, + `metrics_views/mv2.yaml`: ` +type: metrics_view +version: 1 +model: m1 +custom_owner: team-a +not_a_real_field: oops +measures: +- name: count + expression: COUNT(*) +`, + } + + resources := []*Resource{ + { + Name: ResourceName{Kind: ResourceKindModel, Name: "m1"}, + Paths: []string{"/models/m1.sql"}, + ModelSpec: &runtimev1.ModelSpec{ + RefreshSchedule: &runtimev1.Schedule{RefUpdate: true}, + InputConnector: "duckdb", + InputProperties: must(structpb.NewStruct(map[string]any{"sql": strings.TrimSpace(files["models/m1.sql"])})), + OutputConnector: "duckdb", + ChangeMode: runtimev1.ModelChangeMode_MODEL_CHANGE_MODE_RESET, + }, + }, + { + Name: ResourceName{Kind: ResourceKindMetricsView, Name: "mv1"}, + Refs: []ResourceName{{Kind: ResourceKindModel, Name: "m1"}}, + Paths: []string{"/metrics_views/mv1.yaml"}, + MetricsViewSpec: &runtimev1.MetricsViewSpec{ + Connector: "duckdb", + Model: "m1", + DisplayName: "Mv1", + Dimensions: []*runtimev1.MetricsViewSpec_Dimension{ + {Name: "foo", DisplayName: "Foo", Expression: "id"}, + }, + Measures: []*runtimev1.MetricsViewSpec_Measure{ + {Name: "count", DisplayName: "Count", Expression: "COUNT(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}, + }, + }, + }, + } + + parseErrors := []*runtimev1.ParseError{ + { + FilePath: "/metrics_views/mv2.yaml", + Message: "field not_a_real_field not found in type", + }, + } + + ctx := context.Background() + repo := makeRepo(t, files) + p, err := Parse(ctx, repo, "", "", "duckdb", true) + require.NoError(t, err) + requireResourcesAndErrors(t, p, resources, parseErrors) +} + func requireResourcesAndErrors(t testing.TB, p *Parser, wantResources []*Resource, wantErrors []*runtimev1.ParseError) { // Check errors // NOTE: Assumes there's at most one parse error per file path diff --git a/runtime/parser/schema/project.schema.yaml b/runtime/parser/schema/project.schema.yaml index fa8f5c1aba0..614aa48856e 100644 --- a/runtime/parser/schema/project.schema.yaml +++ b/runtime/parser/schema/project.schema.yaml @@ -11,6 +11,12 @@ description: | ::: + :::info Custom fields + + Rill validates project files strictly and rejects unknown properties, with one exception: properties prefixed with `custom_` (for example, `custom_owner: data-team`) are ignored by validation. You can add such properties at any nesting level, in any project file (including `rill.yaml`), to attach your own metadata for use by external tooling. + + ::: + Projects can simply be rehydrated from Rill project files into an explorable data application as long as there is sufficient access and credentials to the source data - figuring out the dependencies, pulling down data, & validating your model queries and metrics view configurations. The result is a set of functioning exploratory dashboards. You can see a few different example projects by visiting our [example GitHub repository](https://github.com/rilldata/rill-examples).