From d898b564986a8ea075605f5a3208def0b3dd9fe5 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Sat, 10 Oct 2026 01:22:31 +1100 Subject: [PATCH 1/8] expanding-omnisdk-capabilities --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 53410cd3..4507cd9f 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.10.1 - github.com/stackql-labs/omnisdk v0.1.3-beta03 + github.com/stackql-labs/omnisdk v0.1.4-alpha01 github.com/stackql/any-sdk v0.6.0-alpha01 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.3-beta01 diff --git a/go.sum b/go.sum index be73baf8..f28abb34 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stackql-labs/omnisdk v0.1.3-beta03 h1:VAzd8cix45oiRLbexCEBfOGHrYsnmsvqjRcBqroW96o= -github.com/stackql-labs/omnisdk v0.1.3-beta03/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY= +github.com/stackql-labs/omnisdk v0.1.4-alpha01 h1:eC8iPGbCyMpNHjW8UJV//Hg20fzAzNv0lEyQFrIxJoE= +github.com/stackql-labs/omnisdk v0.1.4-alpha01/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY= github.com/stackql/any-sdk v0.6.0-alpha01 h1:mqy0bmZ1wghr7mUgLyaexoRU6rsUZCWXU9Mf60fC8dg= github.com/stackql/any-sdk v0.6.0-alpha01/go.mod h1:ahkRgwHHEn7RTmSfKrshPVk6i4X1BdHknSeBflSyRac= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= From 6c937d0c232cc121e1b59ed04f8ef6f5dc12041f Mon Sep 17 00:00:00 2001 From: General Kroll Date: Sat, 10 Oct 2026 09:29:05 +1100 Subject: [PATCH 2/8] test-updates --- internal/stackql/intrinsic/translate.go | 94 +++++++++++-- internal/stackql/intrinsic/translate_test.go | 100 +++++++++++++- .../stackql_mocked_from_cmd_line.robot | 129 +++++++++++++++++- 3 files changed, 308 insertions(+), 15 deletions(-) diff --git a/internal/stackql/intrinsic/translate.go b/internal/stackql/intrinsic/translate.go index a082ce9e..3a90d901 100644 --- a/internal/stackql/intrinsic/translate.go +++ b/internal/stackql/intrinsic/translate.go @@ -123,13 +123,16 @@ func translateSelect(node *sqlparser.Select, currentProvider string) (docQuery, // translateSource translates a SELECT's FROM and WHERE: the joins omnisdk runs // and the conjuncts it applies to them. func translateSource(node *sqlparser.Select, currentProvider string) (*translator, []query.Predicate, error) { - if len(node.From) != 1 { - return nil, nil, fmt.Errorf("a comma-separated FROM cannot be applied to %s relations; use JOIN ... ON", - UnstablePrefix+"*") - } t := &translator{currentProvider: currentProvider} - if err := t.from(node.From[0], query.Base, nil); err != nil { - return nil, nil, err + // A comma-separated FROM is a cross join of its items; WHERE then says how they relate. + for i, item := range node.From { + form := query.Cross + if i == 0 { + form = query.Base + } + if err := t.from(item, form, nil); err != nil { + return nil, nil, err + } } if node.Where == nil { return t, nil, nil @@ -210,17 +213,23 @@ func (t *translator) from(expr sqlparser.TableExpr, form query.JoinForm, on []qu if err != nil { return err } - if len(node.Condition.Using) > 0 { - return fmt.Errorf("JOIN ... USING cannot be applied to %s relations; use ON", UnstablePrefix+"*") - } if err = t.from(node.LeftExpr, form, on); err != nil { return err } var rightOn []query.Predicate - if node.Condition.On != nil { + switch { + case node.Condition.On != nil: if rightOn, err = conjuncts(node.Condition.On); err != nil { return err } + case len(node.Condition.Using) > 0: + // USING (c) is ON left.c = right.c, left being the table joined just before. + if rightOn, err = t.using(node.RightExpr, node.Condition.Using); err != nil { + return err + } + case rightForm == query.Inner: + // A JOIN with no condition, CROSS JOIN among them, pairs every row with every row. + rightForm = query.Cross } return t.from(node.RightExpr, rightForm, rightOn) default: @@ -242,6 +251,29 @@ func (t *translator) resource(tableName sqlparser.TableName, as sqlparser.TableI return query.NewResource(alias, address) } +// using is the ON a JOIN ... USING spells: each column equal on the table joined just before and on +// the right side. +func (t *translator) using(right sqlparser.TableExpr, cols sqlparser.Columns) ([]query.Predicate, error) { + aliased, isAliased := right.(*sqlparser.AliasedTableExpr) + if !isAliased || len(t.joins) == 0 { + return nil, fmt.Errorf("JOIN ... USING needs a table on each side") + } + rightName, isName := aliased.Expr.(sqlparser.TableName) + if !isName { + return nil, fmt.Errorf("JOIN ... USING needs a table on each side") + } + rightAlias := aliased.As.GetRawVal() + if rightAlias == "" { + rightAlias = rightName.Name.GetRawVal() + } + leftAlias := t.joins[len(t.joins)-1].Resource().Alias() + out := make([]query.Predicate, 0, len(cols)) + for _, c := range cols { + out = append(out, query.NewEq(query.NewColumn(leftAlias, c.GetRawVal()), query.NewColumn(rightAlias, c.GetRawVal()))) + } + return out, nil +} + func qualifiedName(tableName sqlparser.TableName) string { var parts []string for _, part := range []string{ @@ -326,6 +358,34 @@ func predicate(expr sqlparser.Expr) (query.Predicate, error) { return query.NewNot(inner), nil case *sqlparser.ComparisonExpr: return comparison(node) + case *sqlparser.IsExpr: + // IS NULL is never unknown: is_null is true or false, so NOT of it is exact. + e, err := expression(node.Expr) + if err != nil { + return nil, err + } + switch node.Operator { + case sqlparser.IsNullStr: + return query.NewTest(query.NewCall("is_null", e)), nil + case sqlparser.IsNotNullStr: + return query.NewNot(query.NewTest(query.NewCall("is_null", e))), nil + } + return nil, fmt.Errorf("condition '%s' cannot be applied to %s relations", + sqlparser.String(expr), UnstablePrefix+"*") + case *sqlparser.RangeCond: + args := make([]query.Expr, 0, 3) + for _, part := range []sqlparser.Expr{node.Left, node.From, node.To} { + e, err := expression(part) + if err != nil { + return nil, err + } + args = append(args, e) + } + between := query.NewTest(query.NewCall("between", args...)) + if node.Operator == sqlparser.NotBetweenStr { + return query.NewNot(between), nil + } + return between, nil case *sqlparser.FuncExpr: call, err := expression(node) if err != nil { @@ -352,6 +412,20 @@ func comparison(node *sqlparser.ComparisonExpr) (query.Predicate, error) { return query.NewIn(left, right), nil case sqlparser.NotInStr: return query.NewNot(query.NewIn(left, right)), nil + case sqlparser.LikeStr, sqlparser.NotLikeStr: + args := []query.Expr{left, right} + if node.Escape != nil { + escape, escErr := expression(node.Escape) + if escErr != nil { + return nil, escErr + } + args = append(args, escape) + } + like := query.NewTest(query.NewCall("like", args...)) + if node.Operator == sqlparser.NotLikeStr { + return query.NewNot(like), nil + } + return like, nil } op, known := compareOps[node.Operator] if !known { diff --git a/internal/stackql/intrinsic/translate_test.go b/internal/stackql/intrinsic/translate_test.go index 4556ea8a..68e84c0e 100644 --- a/internal/stackql/intrinsic/translate_test.go +++ b/internal/stackql/intrinsic/translate_test.go @@ -4,6 +4,7 @@ import ( "fmt" "path/filepath" "reflect" + "strings" "testing" "github.com/stackql-labs/omnisdk/pkg/omnisdk" @@ -108,10 +109,6 @@ func TestTranslateSelectRefusals(t *testing.T) { "stackql_unstable_* relations", "select login from stackql_unstable_github.orgs.members limit 1, 2": "OFFSET cannot be applied to " + "stackql_unstable_* relations", - "select login from stackql_unstable_github.orgs.members where login like 'a%'": "condition " + - "'`login` like 'a%'' cannot be applied to stackql_unstable_* relations", - "select a.login from stackql_unstable_github.orgs.members a, stackql_unstable_github.orgs.members b": "a " + - "comma-separated FROM cannot be applied to stackql_unstable_* relations; use JOIN ... ON", "select a.login from stackql_unstable_github.orgs.members a right join " + "stackql_unstable_github.orgs.members b on a.login = b.login": "RIGHT JOIN cannot be applied to " + "stackql_unstable_* relations", @@ -322,3 +319,98 @@ func TestDocProviderUnderOmniAll(t *testing.T) { } } } + +// Predicates SQL spells as operators become calls omnisdk evaluates: IS [NOT] NULL, [NOT] LIKE with an +// optional ESCAPE, and [NOT] BETWEEN. +func TestTranslateOperatorPredicates(t *testing.T) { + withUnstable(t, true) + for cond, want := range map[string]string{ + "login is null": "is_null(login)", + "login is not null": "not is_null(login)", + "login like 'a%'": "like(login, a%)", + "login not like 'a!%' escape '!'": "not like(login, a!%, !)", + "id between 1 and 10": "between(id, 1, 10)", + "id not between 1 and 10": "not between(id, 1, 10)", + } { + sel := parseSelect(t, "select login from stackql_unstable_github.orgs.members where "+cond) + dq, err := translateSelect(sel, "") + if err != nil { + t.Fatalf("%s: %v", cond, err) + } + where := dq.getQuery().Where() + if len(where) != 1 { + t.Fatalf("%s: %d conjuncts", cond, len(where)) + } + if got := describePredicate(where[0]); got != want { + t.Errorf("%s: got %s, want %s", cond, got, want) + } + } +} + +// A comma-separated FROM, CROSS JOIN and a JOIN with no condition are cross joins; JOIN ... USING is +// an inner join on each named column. +func TestTranslateCrossAndUsing(t *testing.T) { + withUnstable(t, true) + for sql, want := range map[string][]query.JoinForm{ + "select a.login from stackql_unstable_github.orgs.members a, stackql_unstable_github.orgs.members b": {query.Base, query.Cross}, + "select a.login from stackql_unstable_github.orgs.members a cross join stackql_unstable_github.orgs.members b": {query.Base, query.Cross}, + "select a.login from stackql_unstable_github.orgs.members a join stackql_unstable_github.orgs.members b": {query.Base, query.Cross}, + } { + dq, err := translateSelect(parseSelect(t, sql), "") + if err != nil { + t.Fatalf("%s: %v", sql, err) + } + var got []query.JoinForm + for _, j := range dq.getQuery().From() { + got = append(got, j.Form()) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: forms %v, want %v", sql, got, want) + } + } + dq, err := translateSelect(parseSelect(t, "select a.login from stackql_unstable_github.orgs.members a "+ + "join stackql_unstable_github.orgs.members b using (login, id)"), "") + if err != nil { + t.Fatal(err) + } + b := dq.getQuery().From()[1] + var on []string + for _, p := range b.On() { + on = append(on, describePredicate(p)) + } + if b.Form() != query.Inner || strings.Join(on, "; ") != "a.login = b.login; a.id = b.id" { + t.Errorf("using: form %v, on %v", b.Form(), on) + } +} + +// describePredicate renders the predicate shapes these tests build. +func describePredicate(p query.Predicate) string { + switch p := p.(type) { + case query.Not: + return "not " + describePredicate(p.Negated()) + case query.Test: + return describeExpr(p.Cond()) + case query.Compare: + return describeExpr(p.Left()) + " " + string(p.Op()) + " " + describeExpr(p.Right()) + } + return fmt.Sprintf("%T", p) +} + +func describeExpr(e query.Expr) string { + switch e := e.(type) { + case query.Column: + if e.Qualifier() == "" { + return e.Name() + } + return e.Qualifier() + "." + e.Name() + case query.Literal: + return fmt.Sprint(e.Value()) + case query.Call: + var args []string + for _, a := range e.Args() { + args = append(args, describeExpr(a)) + } + return e.Func() + "(" + strings.Join(args, ", ") + ")" + } + return fmt.Sprintf("%T", e) +} diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index ec64d074..854e1f69 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -11117,6 +11117,36 @@ Unstable Github Org Members Jsonl Row Set Matches Expectation ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GITHUB}", ... "insecureSkipTLSVerify":true,"unstable":true} ${expected} = Catenate SEPARATOR=\n + ... {"id":"1","login":"some-jimbo-40","type":"User"} + ... {"id":"1","login":"some-jimbo-39","type":"User"} + ... {"id":"1","login":"some-jimbo-38","type":"User"} + ... {"id":"1","login":"some-jimbo-37","type":"User"} + ... {"id":"1","login":"some-jimbo-36","type":"User"} + ... {"id":"1","login":"some-jimbo-35","type":"User"} + ... {"id":"1","login":"some-jimbo-34","type":"User"} + ... {"id":"1","login":"some-jimbo-33","type":"User"} + ... {"id":"1","login":"some-jimbo-32","type":"User"} + ... {"id":"1","login":"some-jimbo-31","type":"User"} + ... {"id":"1","login":"some-jimbo-30","type":"User"} + ... {"id":"1","login":"some-jimbo-29","type":"User"} + ... {"id":"1","login":"some-jimbo-28","type":"User"} + ... {"id":"1","login":"some-jimbo-27","type":"User"} + ... {"id":"1","login":"some-jimbo-26","type":"User"} + ... {"id":"1","login":"some-jimbo-25","type":"User"} + ... {"id":"1","login":"some-jimbo-24","type":"User"} + ... {"id":"1","login":"some-jimbo-23","type":"User"} + ... {"id":"1","login":"some-jimbo-22","type":"User"} + ... {"id":"1","login":"some-jimbo-21","type":"User"} + ... {"id":"1","login":"some-jimbo-20","type":"User"} + ... {"id":"1","login":"some-jimbo-19","type":"User"} + ... {"id":"1","login":"some-jimbo-18","type":"User"} + ... {"id":"1","login":"some-jimbo-17","type":"User"} + ... {"id":"1","login":"some-jimbo-16","type":"User"} + ... {"id":"1","login":"some-jimbo-15","type":"User"} + ... {"id":"1","login":"some-jimbo-14","type":"User"} + ... {"id":"1","login":"some-jimbo-13","type":"User"} + ... {"id":"1","login":"some-jimbo-12","type":"User"} + ... {"id":"1","login":"some-jimbo-11","type":"User"} ... {"id":"1","login":"some-jimbo-10","type":"User"} ... {"id":"1","login":"some-jimbo-9","type":"User"} ... {"id":"1","login":"some-jimbo-8","type":"User"} @@ -11218,6 +11248,103 @@ Unstable Google Kms Key Rings Joined To Crypto Keys Jsonl Row Set Matches Expect ... stdout=${CURDIR}${/}tmp${/}Unstable-Google-Kms-Key-Rings-Joined-To-Crypto-Keys.tmp ... stderr=${CURDIR}${/}tmp${/}Unstable-Google-Kms-Key-Rings-Joined-To-Crypto-Keys-stderr.tmp +Unstable Github Org Members Filtered By Like Jsonl Row Set Matches Expectation + [Documentation] LIKE is evaluated by omnisdk on the streamed rows, every page included. + [Teardown] Remove Preview Mock Environment + ${preview} = Catenate SEPARATOR= + ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GITHUB}", + ... "insecureSkipTLSVerify":true,"unstable":true} + ${expected} = Catenate SEPARATOR=\n + ... {"login":"some-jimbo-1","type":"User"} + ... {"login":"some-jimbo-10","type":"User"} + ... {"login":"some-jimbo-11","type":"User"} + ... {"login":"some-jimbo-12","type":"User"} + ... {"login":"some-jimbo-13","type":"User"} + ... {"login":"some-jimbo-14","type":"User"} + ... {"login":"some-jimbo-15","type":"User"} + ... {"login":"some-jimbo-16","type":"User"} + ... {"login":"some-jimbo-17","type":"User"} + ... {"login":"some-jimbo-18","type":"User"} + ... {"login":"some-jimbo-19","type":"User"} + ${query} = Catenate SEPARATOR=${SPACE} + ... select login, type from stackql_unstable_github.orgs.members + ... where org = 'dummyorg' and login like 'some-jimbo-1%'; + Should StackQL Exec Inline Jsonl Set Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Filtered-By-Like.tmp + ... stderr=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Filtered-By-Like-stderr.tmp + +Unstable Github Org Members Filtered By Between And Is Not Null Jsonl Row Set Matches Expectation + [Documentation] BETWEEN orders text as text, and IS NOT NULL holds for every member with a type. + [Teardown] Remove Preview Mock Environment + ${preview} = Catenate SEPARATOR= + ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GITHUB}", + ... "insecureSkipTLSVerify":true,"unstable":true} + ${expected} = Catenate SEPARATOR=\n + ... {"login":"some-jimbo-2","type":"User"} + ... {"login":"some-jimbo-20","type":"User"} + ... {"login":"some-jimbo-21","type":"User"} + ... {"login":"some-jimbo-22","type":"User"} + ... {"login":"some-jimbo-23","type":"User"} + ... {"login":"some-jimbo-24","type":"User"} + ... {"login":"some-jimbo-25","type":"User"} + ... {"login":"some-jimbo-26","type":"User"} + ... {"login":"some-jimbo-27","type":"User"} + ... {"login":"some-jimbo-28","type":"User"} + ... {"login":"some-jimbo-29","type":"User"} + ... {"login":"some-jimbo-3","type":"User"} + ${query} = Catenate SEPARATOR=${SPACE} + ... select login, type from stackql_unstable_github.orgs.members + ... where org = 'dummyorg' and login between 'some-jimbo-2' and 'some-jimbo-3' + ... and type is not null; + Should StackQL Exec Inline Jsonl Set Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Filtered-By-Between.tmp + ... stderr=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Filtered-By-Between-stderr.tmp + +Unstable Github Org Members Comma Joined To Themselves Jsonl Row Set Matches Expectation + [Documentation] A comma-separated FROM is a cross join, narrowed by WHERE. + [Teardown] Remove Preview Mock Environment + ${preview} = Catenate SEPARATOR= + ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GITHUB}", + ... "insecureSkipTLSVerify":true,"unstable":true} + ${expected} = Catenate SEPARATOR=\n + ... {"login":"some-jimbo-3","type":"User"} + ${query} = Catenate SEPARATOR=${SPACE} + ... select a.login, a.type from stackql_unstable_github.orgs.members a, stackql_unstable_github.orgs.members b + ... where a.org = 'dummyorg' and b.org = 'dummyorg' and a.login = b.login and a.login = 'some-jimbo-3'; + Should StackQL Exec Inline Jsonl Set Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Comma-Joined.tmp + ... stderr=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Comma-Joined-stderr.tmp + Unstable Github Org Members Filtered By In List Jsonl Row Set Matches Expectation [Documentation] A condition no request parameter can carry is applied by ... omnisdk as a filter on the streamed rows. @@ -11290,7 +11417,7 @@ Unstable Github Org Members Staged Group By Exact Match ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} ... ${query} - ... type,member_count\nUser,10 + ... type,member_count\nUser,40 ... \-o\=csv ... --preview\=${preview} ... stdout=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-Staged-Group-By.tmp From 59ff30bc0aefc727a42c645d95cec39ef617a50a Mon Sep 17 00:00:00 2001 From: General Kroll Date: Sat, 10 Oct 2026 09:40:40 +1100 Subject: [PATCH 3/8] test-updates-again --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4507cd9f..53f87936 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.10.1 - github.com/stackql-labs/omnisdk v0.1.4-alpha01 + github.com/stackql-labs/omnisdk v0.1.4-alpha02 github.com/stackql/any-sdk v0.6.0-alpha01 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.3-beta01 diff --git a/go.sum b/go.sum index f28abb34..3e803382 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stackql-labs/omnisdk v0.1.4-alpha01 h1:eC8iPGbCyMpNHjW8UJV//Hg20fzAzNv0lEyQFrIxJoE= -github.com/stackql-labs/omnisdk v0.1.4-alpha01/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY= +github.com/stackql-labs/omnisdk v0.1.4-alpha02 h1:FnbBQphdHwY7r/nHsDibDPkO0rz+HJaExSiQFr2OqBU= +github.com/stackql-labs/omnisdk v0.1.4-alpha02/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY= github.com/stackql/any-sdk v0.6.0-alpha01 h1:mqy0bmZ1wghr7mUgLyaexoRU6rsUZCWXU9Mf60fC8dg= github.com/stackql/any-sdk v0.6.0-alpha01/go.mod h1:ahkRgwHHEn7RTmSfKrshPVk6i4X1BdHknSeBflSyRac= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= From a75856d37df4be976acebc048fe1b844f0580f9f Mon Sep 17 00:00:00 2001 From: General Kroll Date: Sat, 10 Oct 2026 10:34:02 +1100 Subject: [PATCH 4/8] linter-fixes --- internal/stackql/intrinsic/translate.go | 61 ++++++++++++++----------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/internal/stackql/intrinsic/translate.go b/internal/stackql/intrinsic/translate.go index 3a90d901..cc25ce96 100644 --- a/internal/stackql/intrinsic/translate.go +++ b/internal/stackql/intrinsic/translate.go @@ -359,33 +359,9 @@ func predicate(expr sqlparser.Expr) (query.Predicate, error) { case *sqlparser.ComparisonExpr: return comparison(node) case *sqlparser.IsExpr: - // IS NULL is never unknown: is_null is true or false, so NOT of it is exact. - e, err := expression(node.Expr) - if err != nil { - return nil, err - } - switch node.Operator { - case sqlparser.IsNullStr: - return query.NewTest(query.NewCall("is_null", e)), nil - case sqlparser.IsNotNullStr: - return query.NewNot(query.NewTest(query.NewCall("is_null", e))), nil - } - return nil, fmt.Errorf("condition '%s' cannot be applied to %s relations", - sqlparser.String(expr), UnstablePrefix+"*") + return isNull(node) case *sqlparser.RangeCond: - args := make([]query.Expr, 0, 3) - for _, part := range []sqlparser.Expr{node.Left, node.From, node.To} { - e, err := expression(part) - if err != nil { - return nil, err - } - args = append(args, e) - } - between := query.NewTest(query.NewCall("between", args...)) - if node.Operator == sqlparser.NotBetweenStr { - return query.NewNot(between), nil - } - return between, nil + return between(node) case *sqlparser.FuncExpr: call, err := expression(node) if err != nil { @@ -398,6 +374,39 @@ func predicate(expr sqlparser.Expr) (query.Predicate, error) { } } +// isNull is never unknown: is_null is true or false, so NOT of it is exact. +func isNull(node *sqlparser.IsExpr) (query.Predicate, error) { + e, err := expression(node.Expr) + if err != nil { + return nil, err + } + switch node.Operator { + case sqlparser.IsNullStr: + return query.NewTest(query.NewCall("is_null", e)), nil + case sqlparser.IsNotNullStr: + return query.NewNot(query.NewTest(query.NewCall("is_null", e))), nil + } + return nil, fmt.Errorf("condition '%s' cannot be applied to %s relations", + sqlparser.String(node), UnstablePrefix+"*") +} + +func between(node *sqlparser.RangeCond) (query.Predicate, error) { + parts := []sqlparser.Expr{node.Left, node.From, node.To} + args := make([]query.Expr, 0, len(parts)) + for _, part := range parts { + e, err := expression(part) + if err != nil { + return nil, err + } + args = append(args, e) + } + test := query.NewTest(query.NewCall("between", args...)) + if node.Operator == sqlparser.NotBetweenStr { + return query.NewNot(test), nil + } + return test, nil +} + func comparison(node *sqlparser.ComparisonExpr) (query.Predicate, error) { left, err := expression(node.Left) if err != nil { From fe3703db28e20319d0d82ac667f426e7ee1cd3b8 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Sat, 10 Oct 2026 10:50:15 +1100 Subject: [PATCH 5/8] more-sql-functions --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 53f87936..470fa576 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.10.1 - github.com/stackql-labs/omnisdk v0.1.4-alpha02 + github.com/stackql-labs/omnisdk v0.1.4-alpha03 github.com/stackql/any-sdk v0.6.0-alpha01 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.3-beta01 diff --git a/go.sum b/go.sum index 3e803382..ab95885c 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stackql-labs/omnisdk v0.1.4-alpha02 h1:FnbBQphdHwY7r/nHsDibDPkO0rz+HJaExSiQFr2OqBU= -github.com/stackql-labs/omnisdk v0.1.4-alpha02/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY= +github.com/stackql-labs/omnisdk v0.1.4-alpha03 h1:pXJv/9XdPyAX/JR2CscqpRCSUDtiptBYert0mQWYeeQ= +github.com/stackql-labs/omnisdk v0.1.4-alpha03/go.mod h1:G8QSu3NlKSsiWuvGSNPQH+Bqqc7MT2GV2qSK4drtJEo= github.com/stackql/any-sdk v0.6.0-alpha01 h1:mqy0bmZ1wghr7mUgLyaexoRU6rsUZCWXU9Mf60fC8dg= github.com/stackql/any-sdk v0.6.0-alpha01/go.mod h1:ahkRgwHHEn7RTmSfKrshPVk6i4X1BdHknSeBflSyRac= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= From feee233190772fcb4c5378af99159605de871a5e Mon Sep 17 00:00:00 2001 From: General Kroll Date: Sat, 10 Oct 2026 13:48:22 +1100 Subject: [PATCH 6/8] more-sql-functions --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 470fa576..4647477b 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.10.1 - github.com/stackql-labs/omnisdk v0.1.4-alpha03 + github.com/stackql-labs/omnisdk v0.1.4-alpha04 github.com/stackql/any-sdk v0.6.0-alpha01 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.3-beta01 diff --git a/go.sum b/go.sum index ab95885c..290d2092 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stackql-labs/omnisdk v0.1.4-alpha03 h1:pXJv/9XdPyAX/JR2CscqpRCSUDtiptBYert0mQWYeeQ= -github.com/stackql-labs/omnisdk v0.1.4-alpha03/go.mod h1:G8QSu3NlKSsiWuvGSNPQH+Bqqc7MT2GV2qSK4drtJEo= +github.com/stackql-labs/omnisdk v0.1.4-alpha04 h1:9kUYDp9i2INLfZvaH0Cd64g66ehLpcdtqGvrCPb9tn0= +github.com/stackql-labs/omnisdk v0.1.4-alpha04/go.mod h1:2P2vFi+c/wEeQ3kFgIt0QuIlefpuzIoTm6bPJAiLAvQ= github.com/stackql/any-sdk v0.6.0-alpha01 h1:mqy0bmZ1wghr7mUgLyaexoRU6rsUZCWXU9Mf60fC8dg= github.com/stackql/any-sdk v0.6.0-alpha01/go.mod h1:ahkRgwHHEn7RTmSfKrshPVk6i4X1BdHknSeBflSyRac= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= From de284886f4f34fbc4674f740d884d623f591f3f3 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Sat, 10 Oct 2026 14:14:39 +1100 Subject: [PATCH 7/8] further-expansion --- internal/stackql/intrinsic/dialect.go | 27 ++++++++++++++ internal/stackql/intrinsic/dialect_test.go | 37 +++++++++++++++++++ internal/stackql/intrinsic/doc.go | 11 +++++- internal/stackql/intrinsic/dynamic.go | 7 +++- internal/stackql/intrinsic/iac.go | 8 ++-- .../stackql/intrinsic/preview_services.go | 13 +++++-- internal/stackql/intrinsic/translate.go | 2 +- internal/stackql/intrinsic/translate_test.go | 4 +- 8 files changed, 95 insertions(+), 14 deletions(-) create mode 100644 internal/stackql/intrinsic/dialect.go create mode 100644 internal/stackql/intrinsic/dialect_test.go diff --git a/internal/stackql/intrinsic/dialect.go b/internal/stackql/intrinsic/dialect.go new file mode 100644 index 00000000..38cb34e6 --- /dev/null +++ b/internal/stackql/intrinsic/dialect.go @@ -0,0 +1,27 @@ +package intrinsic + +import ( + "fmt" + + "github.com/stackql-labs/omnisdk/pkg/sqlfn" + "github.com/stackql/any-sdk/pkg/constants" + "github.com/stackql/any-sdk/pkg/dto" +) + +// backendDialect is the SQL dialect of the session's backend, which decides what each function a +// query calls means: the embedded SQLite's, or Postgres's. It is read from the backend's +// configuration; a backend omnisdk has no catalogue for is refused. +func backendDialect(ctx queryContext) (sqlfn.Dialect, error) { + cfg, err := dto.GetSQLBackendCfg(ctx.GetRuntimeContext().SQLBackendCfgRaw) + if err != nil { + return "", err + } + switch cfg.GetSQLDialect() { + case constants.SQLDialectSQLite3: + return sqlfn.SQLite, nil + case constants.SQLDialectPostgres: + return sqlfn.Postgres, nil + } + return "", fmt.Errorf("SQL backend %q has no omnisdk function catalogue (want %q or %q)", + cfg.GetSQLDialect(), constants.SQLDialectSQLite3, constants.SQLDialectPostgres) +} diff --git a/internal/stackql/intrinsic/dialect_test.go b/internal/stackql/intrinsic/dialect_test.go new file mode 100644 index 00000000..b5ebc164 --- /dev/null +++ b/internal/stackql/intrinsic/dialect_test.go @@ -0,0 +1,37 @@ +package intrinsic //nolint:testpackage // tests the unexported dialect selection + +import ( + "testing" + + "github.com/stackql-labs/omnisdk/pkg/sqlfn" + "github.com/stackql/any-sdk/pkg/dto" +) + +// backendCtx is a queryContext whose only state is the SQL backend's configuration. +type backendCtx struct { + queryContext + raw string +} + +func (c backendCtx) GetRuntimeContext() dto.RuntimeCtx { + return dto.RuntimeCtx{SQLBackendCfgRaw: c.raw} +} + +// The dialect is the backend's: the default and sqlite3 are SQLite, postgres is Postgres, and a +// backend with no catalogue is refused rather than given another's functions. +func TestBackendDialect(t *testing.T) { + cases := map[string]sqlfn.Dialect{ + ``: sqlfn.SQLite, + `{"sqlDialect": "sqlite3"}`: sqlfn.SQLite, + `{"dbEngine": "postgres_tcp", "sqlDialect": "postgres", "dsn": "postgres://u:p@h:5432/db"}`: sqlfn.Postgres, + } + for raw, want := range cases { + got, err := backendDialect(backendCtx{raw: raw}) + if err != nil || got != want { + t.Errorf("%s: got %q, %v; want %q", raw, got, err, want) + } + } + if _, err := backendDialect(backendCtx{raw: `{"sqlDialect": "snowflake"}`}); err == nil { + t.Error("snowflake: want a refusal, got none") + } +} diff --git a/internal/stackql/intrinsic/doc.go b/internal/stackql/intrinsic/doc.go index e06141c9..f34e8a11 100644 --- a/internal/stackql/intrinsic/doc.go +++ b/internal/stackql/intrinsic/doc.go @@ -224,14 +224,21 @@ func openDocQuery(ctx queryContext, translated docQuery) (omnisdk.Rows, string, } else { relation = q.From()[0].Resource().Alias() } - res, resolveErr := omnisdk.Resolve(q, tables) + dialect, dialectErr := backendDialect(ctx) + if dialectErr != nil { + return nil, "", dialectErr + } + res, resolveErr := omnisdk.ResolveIn(q, tables, dialect) if resolveErr != nil { return nil, "", resolveErr } // omnisdk takes one credential per run: the first relation's cloud - a // mutation's target - leaving the rest to the canonical environment // variables. - args := previewArgs(ctx, translated.getBundles()[0], res.Params()) + args, argsErr := previewArgs(ctx, translated.getBundles()[0], res.Params()) + if argsErr != nil { + return nil, "", argsErr + } args.Tuning.Limit = translated.getLimit() plan, planErr := omnisdk.NewGraphSelectQuery(registry, res.Graph(), args) if planErr != nil { diff --git a/internal/stackql/intrinsic/dynamic.go b/internal/stackql/intrinsic/dynamic.go index 47d6a847..616d6965 100644 --- a/internal/stackql/intrinsic/dynamic.go +++ b/internal/stackql/intrinsic/dynamic.go @@ -83,8 +83,11 @@ func dynamicSelectFunc( if err != nil { return internaldto.NewErroneousExecutorOutput(err) } - plan, planErr := omnisdk.NewGraphQuery( - registryRoot(ctx), graph, previewArgs(ctx, graphCloud(graph.Addresses()), params)) + args, argsErr := previewArgs(ctx, graphCloud(graph.Addresses()), params) + if argsErr != nil { + return internaldto.NewErroneousExecutorOutput(argsErr) + } + plan, planErr := omnisdk.NewGraphQuery(registryRoot(ctx), graph, args) if planErr != nil { return internaldto.NewErroneousExecutorOutput(planErr) } diff --git a/internal/stackql/intrinsic/iac.go b/internal/stackql/intrinsic/iac.go index 6bbc75eb..f422703a 100644 --- a/internal/stackql/intrinsic/iac.go +++ b/internal/stackql/intrinsic/iac.go @@ -86,9 +86,11 @@ func iacSelectFunc( if err != nil { return internaldto.NewErroneousExecutorOutput(err) } - plan, planErr := omnisdk.Converge( - registryRoot(ctx), collection, state, runID, resources, - previewArgs(ctx, convergeCloud(resources), params)) + args, argsErr := previewArgs(ctx, convergeCloud(resources), params) + if argsErr != nil { + return internaldto.NewErroneousExecutorOutput(argsErr) + } + plan, planErr := omnisdk.Converge(registryRoot(ctx), collection, state, runID, resources, args) if planErr != nil { return internaldto.NewErroneousExecutorOutput(planErr) } diff --git a/internal/stackql/intrinsic/preview_services.go b/internal/stackql/intrinsic/preview_services.go index 422b4a87..d79dd9ef 100644 --- a/internal/stackql/intrinsic/preview_services.go +++ b/internal/stackql/intrinsic/preview_services.go @@ -141,16 +141,21 @@ func streamPlan( } // previewArgs assembles the SDK arguments both services need: the scope left -// over after the control predicates, the credential for the cloud in play, and -// the backend tuning. -func previewArgs(ctx queryContext, cloud string, params map[string]string) omnisdk.Args { +// over after the control predicates, the credential for the cloud in play, the +// backend tuning, and the backend's SQL dialect. +func previewArgs(ctx queryContext, cloud string, params map[string]string) (omnisdk.Args, error) { + dialect, err := backendDialect(ctx) + if err != nil { + return omnisdk.Args{}, err + } input := previewCfg return omnisdk.Args{ Params: params, AuthByProvider: providerAuthByProvider(ctx, cloud), Endpoint: input.getEndpoint(), InsecureSkipTLSVerify: input.getInsecureSkipTLSVerify(), - } + Dialect: dialect, + }, nil } // previewSelectFunc routes a SELECT over an extended preview relation. diff --git a/internal/stackql/intrinsic/translate.go b/internal/stackql/intrinsic/translate.go index cc25ce96..e1cfa85a 100644 --- a/internal/stackql/intrinsic/translate.go +++ b/internal/stackql/intrinsic/translate.go @@ -422,7 +422,7 @@ func comparison(node *sqlparser.ComparisonExpr) (query.Predicate, error) { case sqlparser.NotInStr: return query.NewNot(query.NewIn(left, right)), nil case sqlparser.LikeStr, sqlparser.NotLikeStr: - args := []query.Expr{left, right} + args := []query.Expr{right, left} // like(pattern, value), as SQLite's like() takes them if node.Escape != nil { escape, escErr := expression(node.Escape) if escErr != nil { diff --git a/internal/stackql/intrinsic/translate_test.go b/internal/stackql/intrinsic/translate_test.go index 68e84c0e..6095c682 100644 --- a/internal/stackql/intrinsic/translate_test.go +++ b/internal/stackql/intrinsic/translate_test.go @@ -327,8 +327,8 @@ func TestTranslateOperatorPredicates(t *testing.T) { for cond, want := range map[string]string{ "login is null": "is_null(login)", "login is not null": "not is_null(login)", - "login like 'a%'": "like(login, a%)", - "login not like 'a!%' escape '!'": "not like(login, a!%, !)", + "login like 'a%'": "like(a%, login)", + "login not like 'a!%' escape '!'": "not like(a!%, login, !)", "id between 1 and 10": "between(id, 1, 10)", "id not between 1 and 10": "not between(id, 1, 10)", } { From 386e614084479306f5653a4bbd6d997df70b6f90 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Sat, 10 Oct 2026 15:10:46 +1100 Subject: [PATCH 8/8] re-run-ci --- go.mod | 2 +- go.sum | 4 +- internal/stackql/intrinsic/doc.go | 12 ++- internal/stackql/intrinsic/staged.go | 10 ++- internal/stackql/intrinsic/staged_test.go | 5 +- internal/stackql/intrinsic/translate.go | 78 ++++++++++++-------- internal/stackql/intrinsic/translate_test.go | 36 +++++++-- 7 files changed, 101 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index 4647477b..c26c5a30 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.10.1 - github.com/stackql-labs/omnisdk v0.1.4-alpha04 + github.com/stackql-labs/omnisdk v0.1.4-alpha05 github.com/stackql/any-sdk v0.6.0-alpha01 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.3-beta01 diff --git a/go.sum b/go.sum index 290d2092..661bcd3d 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stackql-labs/omnisdk v0.1.4-alpha04 h1:9kUYDp9i2INLfZvaH0Cd64g66ehLpcdtqGvrCPb9tn0= -github.com/stackql-labs/omnisdk v0.1.4-alpha04/go.mod h1:2P2vFi+c/wEeQ3kFgIt0QuIlefpuzIoTm6bPJAiLAvQ= +github.com/stackql-labs/omnisdk v0.1.4-alpha05 h1:Z22XGi99mjKPJEAAYv3qkW1J0iqqFRd9OANK+9cw4Xs= +github.com/stackql-labs/omnisdk v0.1.4-alpha05/go.mod h1:2P2vFi+c/wEeQ3kFgIt0QuIlefpuzIoTm6bPJAiLAvQ= github.com/stackql/any-sdk v0.6.0-alpha01 h1:mqy0bmZ1wghr7mUgLyaexoRU6rsUZCWXU9Mf60fC8dg= github.com/stackql/any-sdk v0.6.0-alpha01/go.mod h1:ahkRgwHHEn7RTmSfKrshPVk6i4X1BdHknSeBflSyRac= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= diff --git a/internal/stackql/intrinsic/doc.go b/internal/stackql/intrinsic/doc.go index f34e8a11..b75df734 100644 --- a/internal/stackql/intrinsic/doc.go +++ b/internal/stackql/intrinsic/doc.go @@ -166,7 +166,11 @@ func docSelectFunc( if previewCfg.getStagingEnabled() && needsStaging(node) { return stagedSelectFunc(ctx, node, currentProvider), true } - translated, err := translateSelect(node, currentProvider) + dialect, err := backendDialect(ctx) + if err != nil { + return refuse(err), true + } + translated, err := translateSelect(node, currentProvider, dialect) if err != nil { return refuse(err), true } @@ -190,7 +194,11 @@ func docMutationFunc( } else if err != nil { return refuse(err), true } - translated, err := translateMutation(stmt, currentProvider) + dialect, err := backendDialect(ctx) + if err != nil { + return refuse(err), true + } + translated, err := translateMutation(stmt, currentProvider, dialect) if err != nil { return refuse(err), true } diff --git a/internal/stackql/intrinsic/staged.go b/internal/stackql/intrinsic/staged.go index 4e9e75bc..8a8d0823 100644 --- a/internal/stackql/intrinsic/staged.go +++ b/internal/stackql/intrinsic/staged.go @@ -18,6 +18,7 @@ import ( "github.com/stackql-labs/omnisdk/pkg/omnisdk" "github.com/stackql-labs/omnisdk/pkg/query" + "github.com/stackql-labs/omnisdk/pkg/sqlfn" "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/omnistaging" @@ -139,7 +140,11 @@ func stagedSelectFunc( node *sqlparser.Select, currentProvider string, ) func() internaldto.ExecutorOutput { - staged, err := planStagedSelect(node, currentProvider, ctx.GetASTFormatter()) + dialect, err := backendDialect(ctx) + if err != nil { + return refuse(err) + } + staged, err := planStagedSelect(node, currentProvider, dialect, ctx.GetASTFormatter()) if err != nil { return refuse(err) } @@ -248,9 +253,10 @@ func (r *stagedRefs) collectSelectExprs( func planStagedSelect( node *sqlparser.Select, currentProvider string, + dialect sqlfn.Dialect, formatter sqlparser.NodeFormatter, ) (stagedSelect, error) { - t, where, err := translateSource(node, currentProvider) + t, where, err := translateSource(node, currentProvider, dialect) if err != nil { return nil, err } diff --git a/internal/stackql/intrinsic/staged_test.go b/internal/stackql/intrinsic/staged_test.go index fe81d28e..f33a5c0e 100644 --- a/internal/stackql/intrinsic/staged_test.go +++ b/internal/stackql/intrinsic/staged_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "testing" + "github.com/stackql-labs/omnisdk/pkg/sqlfn" "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/any-sdk/public/sqlengine" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" @@ -84,7 +85,7 @@ func TestPlanStagedSelectOuterStatement(t *testing.T) { }, } for _, tc := range cases { - staged, err := planStagedSelect(sel, "", tc.formatter) + staged, err := planStagedSelect(sel, "", sqlfn.SQLite, tc.formatter) if err != nil { t.Fatal(err) } @@ -102,7 +103,7 @@ func TestPlanStagedSelectOuterStatement(t *testing.T) { } } if _, err := planStagedSelect(parseSelect(t, - "select * from stackql_unstable_github.orgs.members order by login"), "", + "select * from stackql_unstable_github.orgs.members order by login"), "", sqlfn.SQLite, astformat.SQLiteSelectExprsFormatter); err == nil || err.Error() != "'*' cannot be staged for stackql_unstable_* relations; name the columns" { t.Fatalf("star refusal: got %v", err) diff --git a/internal/stackql/intrinsic/translate.go b/internal/stackql/intrinsic/translate.go index e1cfa85a..ad86b3f0 100644 --- a/internal/stackql/intrinsic/translate.go +++ b/internal/stackql/intrinsic/translate.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/stackql-labs/omnisdk/pkg/query" + "github.com/stackql-labs/omnisdk/pkg/sqlfn" "github.com/stackql/stackql-parser/go/vt/sqlparser" ) @@ -96,7 +97,7 @@ func fromDocProviders(from sqlparser.TableExprs, currentProvider string) (bool, // relations. What omnisdk leaves to the caller - ordering, grouping, // aggregation, de-duplication - is refused until stackql applies it over the // streamed rows. -func translateSelect(node *sqlparser.Select, currentProvider string) (docQuery, error) { +func translateSelect(node *sqlparser.Select, currentProvider string, dialect sqlfn.Dialect) (docQuery, error) { if unsupported := unsupportedDocClauses(node); len(unsupported) > 0 { return nil, fmt.Errorf("%s cannot be applied to %s relations; remove %s from the query", strings.Join(unsupported, ", "), UnstablePrefix+"*", pluralClause(len(unsupported))) @@ -105,7 +106,7 @@ func translateSelect(node *sqlparser.Select, currentProvider string) (docQuery, if err != nil { return nil, err } - t, where, err := translateSource(node, currentProvider) + t, where, err := translateSource(node, currentProvider, dialect) if err != nil { return nil, err } @@ -122,8 +123,10 @@ func translateSelect(node *sqlparser.Select, currentProvider string) (docQuery, // translateSource translates a SELECT's FROM and WHERE: the joins omnisdk runs // and the conjuncts it applies to them. -func translateSource(node *sqlparser.Select, currentProvider string) (*translator, []query.Predicate, error) { - t := &translator{currentProvider: currentProvider} +func translateSource( + node *sqlparser.Select, currentProvider string, dialect sqlfn.Dialect, +) (*translator, []query.Predicate, error) { + t := &translator{currentProvider: currentProvider, dialect: dialect} // A comma-separated FROM is a cross join of its items; WHERE then says how they relate. for i, item := range node.From { form := query.Cross @@ -137,7 +140,7 @@ func translateSource(node *sqlparser.Select, currentProvider string) (*translato if node.Where == nil { return t, nil, nil } - where, err := conjuncts(node.Where.Expr) + where, err := t.conjuncts(node.Where.Expr) if err != nil { return nil, nil, err } @@ -184,6 +187,7 @@ func pushedLimit(limit *sqlparser.Limit) (int, error) { } type translator struct { + dialect sqlfn.Dialect currentProvider string joins []query.Join bundles []string @@ -219,7 +223,7 @@ func (t *translator) from(expr sqlparser.TableExpr, form query.JoinForm, on []qu var rightOn []query.Predicate switch { case node.Condition.On != nil: - if rightOn, err = conjuncts(node.Condition.On); err != nil { + if rightOn, err = t.conjuncts(node.Condition.On); err != nil { return err } case len(node.Condition.Using) > 0: @@ -299,19 +303,19 @@ func joinForm(join string) (query.JoinForm, error) { } // conjuncts splits a condition at its top-level ANDs. -func conjuncts(expr sqlparser.Expr) ([]query.Predicate, error) { +func (t *translator) conjuncts(expr sqlparser.Expr) ([]query.Predicate, error) { if and, isAnd := expr.(*sqlparser.AndExpr); isAnd { - left, err := conjuncts(and.Left) + left, err := t.conjuncts(and.Left) if err != nil { return nil, err } - right, err := conjuncts(and.Right) + right, err := t.conjuncts(and.Right) if err != nil { return nil, err } return append(left, right...), nil } - p, err := predicate(expr) + p, err := t.predicate(expr) if err != nil { return nil, err } @@ -327,10 +331,10 @@ var compareOps = map[string]query.CompareOp{ //nolint:gochecknoglobals // fixed sqlparser.GreaterEqualStr: query.Ge, } -func predicate(expr sqlparser.Expr) (query.Predicate, error) { +func (t *translator) predicate(expr sqlparser.Expr) (query.Predicate, error) { switch node := expr.(type) { case *sqlparser.AndExpr: - parts, err := conjuncts(node) + parts, err := t.conjuncts(node) if err != nil { return nil, err } @@ -341,23 +345,23 @@ func predicate(expr sqlparser.Expr) (query.Predicate, error) { } return query.NewNot(query.NewOr(negated...)), nil case *sqlparser.OrExpr: - left, err := predicate(node.Left) + left, err := t.predicate(node.Left) if err != nil { return nil, err } - right, err := predicate(node.Right) + right, err := t.predicate(node.Right) if err != nil { return nil, err } return query.NewOr(left, right), nil case *sqlparser.NotExpr: - inner, err := predicate(node.Expr) + inner, err := t.predicate(node.Expr) if err != nil { return nil, err } return query.NewNot(inner), nil case *sqlparser.ComparisonExpr: - return comparison(node) + return t.comparison(node) case *sqlparser.IsExpr: return isNull(node) case *sqlparser.RangeCond: @@ -407,7 +411,24 @@ func between(node *sqlparser.RangeCond) (query.Predicate, error) { return test, nil } -func comparison(node *sqlparser.ComparisonExpr) (query.Predicate, error) { +// like is value LIKE pattern [ESCAPE escape] as the backend's dialect calls it: SQLite's function +// like(pattern, value[, escape]), or Postgres's like(value, pattern), an ESCAPE clause rewriting the +// pattern through like_escape(pattern, escape) as Postgres's parser does. +func (t *translator) like(value, pattern, escape query.Expr) query.Expr { + if t.dialect == sqlfn.Postgres { + if escape != nil { + pattern = query.NewCall("like_escape", pattern, escape) + } + return query.NewCall("like", value, pattern) + } + args := []query.Expr{pattern, value} + if escape != nil { + args = append(args, escape) + } + return query.NewCall("like", args...) +} + +func (t *translator) comparison(node *sqlparser.ComparisonExpr) (query.Predicate, error) { left, err := expression(node.Left) if err != nil { return nil, err @@ -422,15 +443,14 @@ func comparison(node *sqlparser.ComparisonExpr) (query.Predicate, error) { case sqlparser.NotInStr: return query.NewNot(query.NewIn(left, right)), nil case sqlparser.LikeStr, sqlparser.NotLikeStr: - args := []query.Expr{right, left} // like(pattern, value), as SQLite's like() takes them + var escape query.Expr if node.Escape != nil { - escape, escErr := expression(node.Escape) - if escErr != nil { + var escErr error + if escape, escErr = expression(node.Escape); escErr != nil { return nil, escErr } - args = append(args, escape) } - like := query.NewTest(query.NewCall("like", args...)) + like := query.NewTest(t.like(left, right, escape)) if node.Operator == sqlparser.NotLikeStr { return query.NewNot(like), nil } @@ -566,8 +586,8 @@ func mutationTables(stmt sqlparser.Statement) (sqlparser.TableExprs, bool) { // translateMutation builds the omnisdk mutation for an INSERT, UPDATE or // DELETE whose target is a document-driven relation. A RETURNING list becomes // the mutation's outputs. -func translateMutation(stmt sqlparser.Statement, currentProvider string) (docQuery, error) { - t := &translator{currentProvider: currentProvider} +func translateMutation(stmt sqlparser.Statement, currentProvider string, dialect sqlfn.Dialect) (docQuery, error) { + t := &translator{currentProvider: currentProvider, dialect: dialect} var ( target query.Target where []query.Predicate @@ -665,7 +685,7 @@ func (t *translator) insertSelect(rows *sqlparser.Select) ([]sqlparser.Expr, []q if err := t.from(rows.From[0], query.Base, nil); err != nil { return nil, nil, err } - where, err := whereConjuncts(rows.Where) + where, err := t.whereConjuncts(rows.Where) if err != nil { return nil, nil, err } @@ -705,7 +725,7 @@ func (t *translator) update(node *sqlparser.Update) (query.Target, []query.Predi } assignments = append(assignments, query.NewAssignment(set.Name.Name.GetRawVal(), translated)) } - where, err := whereConjuncts(node.Where) + where, err := t.whereConjuncts(node.Where) if err != nil { return nil, nil, err } @@ -724,7 +744,7 @@ func (t *translator) delete(node *sqlparser.Delete) (query.Target, []query.Predi if err != nil { return nil, nil, err } - where, err := whereConjuncts(node.Where) + where, err := t.whereConjuncts(node.Where) if err != nil { return nil, nil, err } @@ -770,9 +790,9 @@ func (t *translator) sources(from sqlparser.TableExprs) error { } } -func whereConjuncts(where *sqlparser.Where) ([]query.Predicate, error) { +func (t *translator) whereConjuncts(where *sqlparser.Where) ([]query.Predicate, error) { if where == nil { return nil, nil } - return conjuncts(where.Expr) + return t.conjuncts(where.Expr) } diff --git a/internal/stackql/intrinsic/translate_test.go b/internal/stackql/intrinsic/translate_test.go index 6095c682..86660100 100644 --- a/internal/stackql/intrinsic/translate_test.go +++ b/internal/stackql/intrinsic/translate_test.go @@ -9,6 +9,7 @@ import ( "github.com/stackql-labs/omnisdk/pkg/omnisdk" "github.com/stackql-labs/omnisdk/pkg/query" + "github.com/stackql-labs/omnisdk/pkg/sqlfn" "github.com/stackql/stackql-parser/go/vt/sqlparser" ) @@ -32,7 +33,7 @@ func TestTranslateSelectJoins(t *testing.T) { "inner join stackql_unstable_google.cloudkms.crypto_keys c on c.keyRingsId = split_part(k.name, '/', 6) "+ "left join stackql_unstable_google.cloudkms.crypto_keys c2 on c2.name = c.name "+ "where k.projectsId = 'p' and k.locationsId = 'global' limit 5") - dq, err := translateSelect(sel, "") + dq, err := translateSelect(sel, "", sqlfn.SQLite) if err != nil { t.Fatalf("translate: %v", err) } @@ -68,7 +69,7 @@ func TestTranslateSelectResolvesAgainstRegistry(t *testing.T) { withUnstable(t, true) sel := parseSelect(t, "select login from stackql_unstable_fixture.orgs.members "+ "where org = 'dummyorg' and (type = 'User' or not id = 2) and login in ('a', 'b')") - dq, err := translateSelect(sel, "") + dq, err := translateSelect(sel, "", sqlfn.SQLite) if err != nil { t.Fatalf("translate: %v", err) } @@ -113,7 +114,7 @@ func TestTranslateSelectRefusals(t *testing.T) { "stackql_unstable_github.orgs.members b on a.login = b.login": "RIGHT JOIN cannot be applied to " + "stackql_unstable_* relations", } { - _, err := translateSelect(parseSelect(t, sql), "") + _, err := translateSelect(parseSelect(t, sql), "", sqlfn.SQLite) if err == nil || err.Error() != want { t.Errorf("%s:\n got %v\nwant %s", sql, err, want) } @@ -235,7 +236,7 @@ func TestTranslateMutations(t *testing.T) { where: 2, }, } { - dq, err := translateMutation(parseStatement(t, tc.sql), "") + dq, err := translateMutation(parseStatement(t, tc.sql), "", sqlfn.SQLite) if err != nil { t.Errorf("%s: %v", tc.sql, err) continue @@ -275,7 +276,7 @@ func TestTranslateMutationRefusals(t *testing.T) { "delete from stackql_unstable_google.compute.firewalls where project = 'p' limit 1": "ORDER BY and " + "LIMIT cannot be applied to a DELETE of stackql_unstable_* relations", } { - _, err := translateMutation(parseStatement(t, sql), "") + _, err := translateMutation(parseStatement(t, sql), "", sqlfn.SQLite) if err == nil || err.Error() != want { t.Errorf("%s:\n got %v\nwant %s", sql, err, want) } @@ -333,7 +334,7 @@ func TestTranslateOperatorPredicates(t *testing.T) { "id not between 1 and 10": "not between(id, 1, 10)", } { sel := parseSelect(t, "select login from stackql_unstable_github.orgs.members where "+cond) - dq, err := translateSelect(sel, "") + dq, err := translateSelect(sel, "", sqlfn.SQLite) if err != nil { t.Fatalf("%s: %v", cond, err) } @@ -347,6 +348,25 @@ func TestTranslateOperatorPredicates(t *testing.T) { } } +// On a Postgres backend LIKE is Postgres's like(value, pattern), an ESCAPE clause rewriting the +// pattern through like_escape as Postgres's parser does. +func TestTranslateLikePostgres(t *testing.T) { + withUnstable(t, true) + for cond, want := range map[string]string{ + "login like 'a%'": "like(login, a%)", + "login not like 'a!%' escape '!'": "not like(login, like_escape(a!%, !))", + } { + sel := parseSelect(t, "select login from stackql_unstable_github.orgs.members where "+cond) + dq, err := translateSelect(sel, "", sqlfn.Postgres) + if err != nil { + t.Fatalf("%s: %v", cond, err) + } + if got := describePredicate(dq.getQuery().Where()[0]); got != want { + t.Errorf("%s: got %s, want %s", cond, got, want) + } + } +} + // A comma-separated FROM, CROSS JOIN and a JOIN with no condition are cross joins; JOIN ... USING is // an inner join on each named column. func TestTranslateCrossAndUsing(t *testing.T) { @@ -356,7 +376,7 @@ func TestTranslateCrossAndUsing(t *testing.T) { "select a.login from stackql_unstable_github.orgs.members a cross join stackql_unstable_github.orgs.members b": {query.Base, query.Cross}, "select a.login from stackql_unstable_github.orgs.members a join stackql_unstable_github.orgs.members b": {query.Base, query.Cross}, } { - dq, err := translateSelect(parseSelect(t, sql), "") + dq, err := translateSelect(parseSelect(t, sql), "", sqlfn.SQLite) if err != nil { t.Fatalf("%s: %v", sql, err) } @@ -369,7 +389,7 @@ func TestTranslateCrossAndUsing(t *testing.T) { } } dq, err := translateSelect(parseSelect(t, "select a.login from stackql_unstable_github.orgs.members a "+ - "join stackql_unstable_github.orgs.members b using (login, id)"), "") + "join stackql_unstable_github.orgs.members b using (login, id)"), "", sqlfn.SQLite) if err != nil { t.Fatal(err) }