Feature/omni rdbms fidelity

.github/workflows/build.yml

@@ -666,11 +666,18 @@ jobs:
       id-token: write
       contents: read
     runs-on: ubuntu-latest
+    # sdk omni runs the mocked suite with omnisdk's document-driven paths opted in, best effort.
+    continue-on-error: ${{ matrix.sdk == 'omni' }}
     strategy:
       matrix:
         registry:
           - test/registry-mocked
           - test/registry
+        sdk:
+          - any
+        include:
+          - registry: test/registry-mocked
+            sdk: omni
     timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 120 || vars.DEFAULT_JOB_TIMEOUT_MIN }}
     steps:
 
@@ -986,7 +993,7 @@ jobs:
     - name: Run robot mocked functional tests
       env:
         PYTHONPATH: '${{ env.PYTHONPATH }}:${{ github.workspace }}/test/python'
-      if: success()
+      if: success() && matrix.sdk != 'omni'
       run: |
         if [ "${{ matrix.registry }}" = "test/registry" ]; then
           robot \
@@ -1000,25 +1007,41 @@ jobs:
         fi
 
     - name: Output from mocked functional tests
-      if: always()
+      if: always() && matrix.sdk != 'omni'
       run: |
         cat ./test/robot/reports/output.xml
 
+    # Omni cutover readiness, best effort: the same suite with every stackql run opted in to the
+    # document-driven (omnisdk) paths. Reports a traffic light; never fails the job.
+    - name: Omni readiness traffic light
+      if: matrix.sdk == 'omni'
+      continue-on-error: true
+      env:
+        PYTHONPATH: '${{ env.PYTHONPATH }}:${{ github.workspace }}/test/python'
+        STACKQL_ROBOT_DEFAULT_PREVIEW: '{"unstable":true,"staging":true}'
+      run: |
+        pgrep -f flask | xargs kill -9 || true
+        if python cicd/python/build.py --robot-test --config='{ "variables": { "SHOULD_RUN_DOCKER_EXTERNAL_TESTS": "true" } }'; then
+          echo "🟢 Omni readiness: all robot tests pass with the omni preview" >> "${GITHUB_STEP_SUMMARY}"
+        else
+          echo "🟡 Omni readiness: some robot tests fail with the omni preview" >> "${GITHUB_STEP_SUMMARY}"
+        fi
+
     - name: Run robot mocked functional tests with aggressive concurrency
-      if: success() && matrix.registry != 'test/registry'
+      if: success() && matrix.registry != 'test/registry' && matrix.sdk != 'omni'
       run: |
         echo "## Stray flask apps to be killed before robot tests ##"
         pgrep -f flask | xargs kill -9 || true
         echo "## End ##"
         python cicd/python/build.py --robot-test  --config='{ "variables": { "SHOULD_RUN_DOCKER_EXTERNAL_TESTS": "true", "CONCURRENCY_LIMIT": -1  } }'
 
     - name: Output from mocked functional tests with aggressive concurrency
-      if: always() && matrix.registry != 'test/registry'
+      if: always() && matrix.registry != 'test/registry' && matrix.sdk != 'omni'
       run: |
         cat ./test/robot/reports/output.xml
 
     - name: Run robot integration tests
-      if: env.AZURE_CLIENT_SECRET != '' && startsWith(env.STATE_SOURCE_TAG, 'build-release') && matrix.registry != 'test/registry'
+      if: env.AZURE_CLIENT_SECRET != '' && startsWith(env.STATE_SOURCE_TAG, 'build-release') && matrix.registry != 'test/registry' && matrix.sdk != 'omni'
       env:
         AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
         AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
@@ -1031,17 +1054,17 @@ jobs:
         python cicd/python/build.py --robot-test-integration
 
     - name: Prepare Test DB
-      if: success() && matrix.registry != 'test/registry'
+      if: success() && matrix.registry != 'test/registry' && matrix.sdk != 'omni'
       run: cp test/db/db.sqlite test/db/tmp/python-tests-tmp-db.sqlite
 
     - name: Test Script
-      if: success() && matrix.registry != 'test/registry'
+      if: success() && matrix.registry != 'test/registry' && matrix.sdk != 'omni'
       run: python3 "${TESTSCRIPT}"
       env:
         TESTSCRIPT: ${{env.TESTSCRIPT}}
 
     - name: install and test deb package
-      if: matrix.registry != 'test/registry'
+      if: matrix.registry != 'test/registry' && matrix.sdk != 'omni'
       env:
         PYTHONPATH: '${{ env.PYTHONPATH }}:${{ github.workspace }}/test/python'
         IS_SKIP_MCP_TEST: 'true'
@@ -1060,7 +1083,7 @@ jobs:
         echo "stackql version: ${stackqlVersion}"
 
     - name: Output from mocked deb package functional tests
-      if: always() && matrix.registry != 'test/registry'
+      if: always() && matrix.registry != 'test/registry' && matrix.sdk != 'omni'
       env:
         PYTHONPATH: '${{ env.PYTHONPATH }}:${{ github.workspace }}/test/python'
       run: |

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/pflag v1.0.10
 	github.com/spf13/viper v1.10.1
-	github.com/stackql-labs/omnisdk v0.1.3-beta03
+	github.com/stackql-labs/omnisdk v0.1.4-alpha05
 	github.com/stackql/any-sdk v0.6.0-alpha01
 	github.com/stackql/go-suffix-map v0.0.1-alpha01
 	github.com/stackql/psql-wire v0.1.3-beta01

go.sum

@@ -368,8 +368,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
 github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
 github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4=
 github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U=
-github.com/stackql-labs/omnisdk v0.1.3-beta03 h1:VAzd8cix45oiRLbexCEBfOGHrYsnmsvqjRcBqroW96o=
-github.com/stackql-labs/omnisdk v0.1.3-beta03/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY=
+github.com/stackql-labs/omnisdk v0.1.4-alpha05 h1:Z22XGi99mjKPJEAAYv3qkW1J0iqqFRd9OANK+9cw4Xs=
+github.com/stackql-labs/omnisdk v0.1.4-alpha05/go.mod h1:2P2vFi+c/wEeQ3kFgIt0QuIlefpuzIoTm6bPJAiLAvQ=
 github.com/stackql/any-sdk v0.6.0-alpha01 h1:mqy0bmZ1wghr7mUgLyaexoRU6rsUZCWXU9Mf60fC8dg=
 github.com/stackql/any-sdk v0.6.0-alpha01/go.mod h1:ahkRgwHHEn7RTmSfKrshPVk6i4X1BdHknSeBflSyRac=
 github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8=

internal/stackql/intrinsic/dialect.go

@@ -0,0 +1,63 @@
+package intrinsic
+
+import (
+	"fmt"
+
+	"github.com/stackql-labs/omnisdk/pkg/query"
+	"github.com/stackql-labs/omnisdk/pkg/sqlfn"
+	"github.com/stackql/any-sdk/pkg/constants"
+	"github.com/stackql/any-sdk/pkg/dto"
+)
+
+// sqlDialect is the SQL of the session's backend as the translator writes it: the omnisdk function
+// catalogue a query's calls resolve in, and how SQL that is an operator in the parse is spelled as a
+// call in that catalogue.
+type sqlDialect interface {
+	// catalogue is the omnisdk function catalogue of the backend.
+	catalogue() sqlfn.Dialect
+	// like is value LIKE pattern [ESCAPE escape]; escape is nil without the clause.
+	like(value, pattern, escape query.Expr) query.Expr
+}
+
+// sqliteDialect is the embedded SQLite's: its function like(pattern, value[, escape]).
+type sqliteDialect struct{}
+
+func (sqliteDialect) catalogue() sqlfn.Dialect { return sqlfn.SQLite }
+
+func (sqliteDialect) like(value, pattern, escape query.Expr) query.Expr {
+	args := []query.Expr{pattern, value}
+	if escape != nil {
+		args = append(args, escape)
+	}
+	return query.NewCall("like", args...)
+}
+
+// postgresDialect is Postgres's: like(value, pattern), an ESCAPE clause rewriting the pattern
+// through like_escape(pattern, escape) as Postgres's parser does.
+type postgresDialect struct{}
+
+func (postgresDialect) catalogue() sqlfn.Dialect { return sqlfn.Postgres }
+
+func (postgresDialect) like(value, pattern, escape query.Expr) query.Expr {
+	if escape != nil {
+		pattern = query.NewCall("like_escape", pattern, escape)
+	}
+	return query.NewCall("like", value, pattern)
+}
+
+// backendDialect is the dialect of the session's backend, read from its configuration; a backend
+// omnisdk has no catalogue for is refused.
+func backendDialect(ctx queryContext) (sqlDialect, error) {
+	cfg, err := dto.GetSQLBackendCfg(ctx.GetRuntimeContext().SQLBackendCfgRaw)
+	if err != nil {
+		return nil, err
+	}
+	switch cfg.GetSQLDialect() {
+	case constants.SQLDialectSQLite3:
+		return sqliteDialect{}, nil
+	case constants.SQLDialectPostgres:
+		return postgresDialect{}, nil
+	}
+	return nil, fmt.Errorf("SQL backend %q has no omnisdk function catalogue (want %q or %q)",
+		cfg.GetSQLDialect(), constants.SQLDialectSQLite3, constants.SQLDialectPostgres)
+}

internal/stackql/intrinsic/dialect_test.go

@@ -0,0 +1,37 @@
+package intrinsic //nolint:testpackage // tests the unexported dialect selection
+
+import (
+	"testing"
+
+	"github.com/stackql-labs/omnisdk/pkg/sqlfn"
+	"github.com/stackql/any-sdk/pkg/dto"
+)
+
+// backendCtx is a queryContext whose only state is the SQL backend's configuration.
+type backendCtx struct {
+	queryContext
+	raw string
+}
+
+func (c backendCtx) GetRuntimeContext() dto.RuntimeCtx {
+	return dto.RuntimeCtx{SQLBackendCfgRaw: c.raw}
+}
+
+// The dialect is the backend's: the default and sqlite3 are SQLite, postgres is Postgres, and a
+// backend with no catalogue is refused rather than given another's functions.
+func TestBackendDialect(t *testing.T) {
+	cases := map[string]sqlfn.Dialect{
+		``:                          sqlfn.SQLite,
+		`{"sqlDialect": "sqlite3"}`: sqlfn.SQLite,
+		`{"dbEngine": "postgres_tcp", "sqlDialect": "postgres", "dsn": "postgres://u:p@h:5432/db"}`: sqlfn.Postgres,
+	}
+	for raw, want := range cases {
+		got, err := backendDialect(backendCtx{raw: raw})
+		if err != nil || got.catalogue() != want {
+			t.Errorf("%s: got %v, %v; want %q", raw, got, err, want)
+		}
+	}
+	if _, err := backendDialect(backendCtx{raw: `{"sqlDialect": "snowflake"}`}); err == nil {
+		t.Error("snowflake: want a refusal, got none")
+	}
+}

internal/stackql/intrinsic/doc.go

@@ -166,7 +166,11 @@ func docSelectFunc(
 	if previewCfg.getStagingEnabled() && needsStaging(node) {
 		return stagedSelectFunc(ctx, node, currentProvider), true
 	}
-	translated, err := translateSelect(node, currentProvider)
+	dialect, err := backendDialect(ctx)
+	if err != nil {
+		return refuse(err), true
+	}
+	translated, err := newDocTranslator(currentProvider, dialect).selectQuery(node)
 	if err != nil {
 		return refuse(err), true
 	}
@@ -190,7 +194,11 @@ func docMutationFunc(
 	} else if err != nil {
 		return refuse(err), true
 	}
-	translated, err := translateMutation(stmt, currentProvider)
+	dialect, err := backendDialect(ctx)
+	if err != nil {
+		return refuse(err), true
+	}
+	translated, err := newDocTranslator(currentProvider, dialect).mutation(stmt)
 	if err != nil {
 		return refuse(err), true
 	}
@@ -224,14 +232,21 @@ func openDocQuery(ctx queryContext, translated docQuery) (omnisdk.Rows, string,
 	} else {
 		relation = q.From()[0].Resource().Alias()
 	}
-	res, resolveErr := omnisdk.Resolve(q, tables)
+	dialect, dialectErr := backendDialect(ctx)
+	if dialectErr != nil {
+		return nil, "", dialectErr
+	}
+	res, resolveErr := omnisdk.ResolveIn(q, tables, dialect.catalogue())
 	if resolveErr != nil {
 		return nil, "", resolveErr
 	}
 	// omnisdk takes one credential per run: the first relation's cloud - a
 	// mutation's target - leaving the rest to the canonical environment
 	// variables.
-	args := previewArgs(ctx, translated.getBundles()[0], res.Params())
+	args, argsErr := previewArgs(ctx, translated.getBundles()[0], res.Params())
+	if argsErr != nil {
+		return nil, "", argsErr
+	}
 	args.Tuning.Limit = translated.getLimit()
 	plan, planErr := omnisdk.NewGraphSelectQuery(registry, res.Graph(), args)
 	if planErr != nil {

internal/stackql/intrinsic/dynamic.go

@@ -83,8 +83,11 @@ func dynamicSelectFunc(
 		if err != nil {
 			return internaldto.NewErroneousExecutorOutput(err)
 		}
-		plan, planErr := omnisdk.NewGraphQuery(
-			registryRoot(ctx), graph, previewArgs(ctx, graphCloud(graph.Addresses()), params))
+		args, argsErr := previewArgs(ctx, graphCloud(graph.Addresses()), params)
+		if argsErr != nil {
+			return internaldto.NewErroneousExecutorOutput(argsErr)
+		}
+		plan, planErr := omnisdk.NewGraphQuery(registryRoot(ctx), graph, args)
 		if planErr != nil {
 			return internaldto.NewErroneousExecutorOutput(planErr)
 		}

internal/stackql/intrinsic/iac.go

@@ -86,9 +86,11 @@ func iacSelectFunc(
 		if err != nil {
 			return internaldto.NewErroneousExecutorOutput(err)
 		}
-		plan, planErr := omnisdk.Converge(
-			registryRoot(ctx), collection, state, runID, resources,
-			previewArgs(ctx, convergeCloud(resources), params))
+		args, argsErr := previewArgs(ctx, convergeCloud(resources), params)
+		if argsErr != nil {
+			return internaldto.NewErroneousExecutorOutput(argsErr)
+		}
+		plan, planErr := omnisdk.Converge(registryRoot(ctx), collection, state, runID, resources, args)
 		if planErr != nil {
 			return internaldto.NewErroneousExecutorOutput(planErr)
 		}

internal/stackql/intrinsic/preview_services.go

@@ -141,16 +141,21 @@ func streamPlan(
 }
 
 // previewArgs assembles the SDK arguments both services need: the scope left
-// over after the control predicates, the credential for the cloud in play, and
-// the backend tuning.
-func previewArgs(ctx queryContext, cloud string, params map[string]string) omnisdk.Args {
+// over after the control predicates, the credential for the cloud in play, the
+// backend tuning, and the backend's SQL dialect.
+func previewArgs(ctx queryContext, cloud string, params map[string]string) (omnisdk.Args, error) {
+	dialect, err := backendDialect(ctx)
+	if err != nil {
+		return omnisdk.Args{}, err
+	}
 	input := previewCfg
 	return omnisdk.Args{
 		Params:                params,
 		AuthByProvider:        providerAuthByProvider(ctx, cloud),
 		Endpoint:              input.getEndpoint(),
 		InsecureSkipTLSVerify: input.getInsecureSkipTLSVerify(),
-	}
+		Dialect:               dialect.catalogue(),
+	}, nil
 }
 
 // previewSelectFunc routes a SELECT over an extended preview relation.

internal/stackql/intrinsic/staged.go

@@ -139,7 +139,11 @@ func stagedSelectFunc(
 	node *sqlparser.Select,
 	currentProvider string,
 ) func() internaldto.ExecutorOutput {
-	staged, err := planStagedSelect(node, currentProvider, ctx.GetASTFormatter())
+	dialect, err := backendDialect(ctx)
+	if err != nil {
+		return refuse(err)
+	}
+	staged, err := planStagedSelect(node, newDocTranslator(currentProvider, dialect), ctx.GetASTFormatter())
 	if err != nil {
 		return refuse(err)
 	}
@@ -247,10 +251,10 @@ func (r *stagedRefs) collectSelectExprs(
 // formatted over the staged table.
 func planStagedSelect(
 	node *sqlparser.Select,
-	currentProvider string,
+	translator docTranslator,
 	formatter sqlparser.NodeFormatter,
 ) (stagedSelect, error) {
-	t, where, err := translateSource(node, currentProvider)
+	src, err := translator.source(node)
 	if err != nil {
 		return nil, err
 	}
@@ -282,7 +286,7 @@ func planStagedSelect(
 		refs.outputs = append(refs.outputs, query.NewOutput("", query.NewStar("")))
 		refs.names = append(refs.names, stagedColumnPrefix+"0")
 	}
-	q, err := query.New(t.joins, where, refs.outputs)
+	q, err := query.New(src.joins(), src.where(), refs.outputs)
 	if err != nil {
 		return nil, err
 	}
@@ -303,7 +307,7 @@ func planStagedSelect(
 	buf := sqlparser.NewTrackedBuffer(refs.formatter(formatter))
 	outer.Format(buf)
 	return newStagedSelect(
-		newDocQuery(q, refs.names, 0, t.bundles), refs.names, outputNames, buf.String()+limit), nil
+		newDocQuery(q, refs.names, 0, src.bundles()), refs.names, outputNames, buf.String()+limit), nil
 }
 
 // stagedLimit renders LIMIT and OFFSET in the form both backends accept.

internal/stackql/intrinsic/staged_test.go

@@ -84,7 +84,7 @@ func TestPlanStagedSelectOuterStatement(t *testing.T) {
 		},
 	}
 	for _, tc := range cases {
-		staged, err := planStagedSelect(sel, "", tc.formatter)
+		staged, err := planStagedSelect(sel, newDocTranslator("", sqliteDialect{}), tc.formatter)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -102,7 +102,7 @@ func TestPlanStagedSelectOuterStatement(t *testing.T) {
 		}
 	}
 	if _, err := planStagedSelect(parseSelect(t,
-		"select * from stackql_unstable_github.orgs.members order by login"), "",
+		"select * from stackql_unstable_github.orgs.members order by login"), newDocTranslator("", sqliteDialect{}),
 		astformat.SQLiteSelectExprsFormatter); err == nil ||
 		err.Error() != "'*' cannot be staged for stackql_unstable_* relations; name the columns" {
 		t.Fatalf("star refusal: got %v", err)