feat: complete the BigQuery driver for 0.2.0

.github/workflows/ci.yml

@@ -36,6 +36,48 @@ jobs:
       - run: cargo clippy --all-targets -- -D warnings
       - run: cargo fmt --all -- --check
 
+  emulator:
+    name: BigQuery emulator integration
+    runs-on: ubuntu-latest
+    services:
+      bigquery:
+        image: ghcr.io/goccy/bigquery-emulator:latest
+        env:
+          BIGQUERY_EMULATOR_PROJECT: test
+          BIGQUERY_EMULATOR_DATASET: ds
+        ports:
+          - 9050:9050
+    steps:
+      - uses: actions/checkout@v7
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Wait for the emulator
+        run: |
+          for attempt in $(seq 1 30); do
+            curl -sf http://localhost:9050/bigquery/v2/projects/test/datasets && exit 0
+            sleep 2
+          done
+          exit 1
+      - run: cargo test --test emulator --test emulator_metadata
+        env:
+          BIGQUERY_EMULATOR_HOST: localhost:9050
+
+  explain:
+    name: EXPLAIN parser package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 9
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: pnpm
+          cache-dependency-path: explain/pnpm-lock.yaml
+      - run: pnpm --dir explain install --frozen-lockfile
+      - run: pnpm --dir explain typecheck
+      - run: pnpm --dir explain test
+
   validate-manifest:
     name: Validate .tabularium manifest
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -18,14 +18,23 @@ jobs:
         with:
           node-version: 20
           cache: pnpm
-          cache-dependency-path: ui/pnpm-lock.yaml
+          cache-dependency-path: |
+            ui/pnpm-lock.yaml
+            explain/pnpm-lock.yaml
       - run: pnpm --dir ui install --frozen-lockfile
       - run: pnpm --dir ui typecheck
       - run: pnpm --dir ui build
+      - run: pnpm --dir explain install --frozen-lockfile
+      - run: pnpm --dir explain typecheck
+      - run: pnpm --dir explain test
       - uses: actions/upload-artifact@v7
         with:
           name: ui-dist
           path: ui/dist/
+      - uses: actions/upload-artifact@v7
+        with:
+          name: explain-dist
+          path: explain/dist/index.iife.js
 
   build:
     name: ${{ matrix.platform-label }}
@@ -74,22 +83,29 @@ jobs:
         with:
           name: ui-dist
           path: ui/dist/
+      - uses: actions/download-artifact@v8
+        with:
+          name: explain-dist
+          path: explain/dist/
       - name: Package on Unix
         if: runner.os != 'Windows'
         run: |
-          mkdir -p staging/ui/dist
+          mkdir -p staging/ui/dist staging/explain/dist
           cp target/${{ matrix.target }}/release/bigquery-plugin${{ matrix.binary-suffix }} staging/
           cp .tabularium staging/
           cp ui/dist/*.js staging/ui/dist/
+          cp explain/dist/index.iife.js staging/explain/dist/
           (cd staging && zip -r ../bigquery-plugin-${{ matrix.platform-label }}.zip .)
       - name: Package on Windows
         if: runner.os == 'Windows'
         shell: pwsh
         run: |
           New-Item -ItemType Directory -Force -Path "staging\ui\dist" | Out-Null
+          New-Item -ItemType Directory -Force -Path "staging\explain\dist" | Out-Null
           Copy-Item "target\${{ matrix.target }}\release\bigquery-plugin${{ matrix.binary-suffix }}" staging
           Copy-Item ".tabularium" staging
           Copy-Item "ui\dist\*.js" "staging\ui\dist"
+          Copy-Item "explain\dist\index.iife.js" "staging\explain\dist"
           Compress-Archive -Path "staging\*" -DestinationPath "bigquery-plugin-${{ matrix.platform-label }}.zip"
       - name: Smoke test on Unix
         if: runner.os != 'Windows' && !matrix.cross

.gitignore

@@ -5,3 +5,5 @@ ui/dist/
 .DS_Store
 .idea/
 .vscode/
+explain/node_modules/
+explain/dist/

.tabularium

@@ -1,29 +1,43 @@
 {
   "$schema": "https://registry.tabularis.dev/manifest.schema.json?kind=driver",
-  "name": "bigquery",
-  "version": "0.1.0",
-  "description": "Google BigQuery driver for Tabularis with GoogleSQL, datasets, views, routines, editable results and cost controls.",
+  "id": "bigquery",
+  "name": "BigQuery",
+  "version": "0.2.0",
+  "description": "Google BigQuery driver for Tabularis with multi-project browsing, cancellable paged queries, sessions and transactions, query plans, GoogleSQL, materialized views, routine management, typed row editing and cost controls.",
+  "category": "database",
+  "tags": ["driver", "bigquery", "google-cloud", "gcp", "sql", "googlesql", "analytics", "data-warehouse", "database-driver"],
   "kind": "driver",
   "engine": "bigquery",
+  "explain_parsers": [
+    {
+      "engine": "bigquery",
+      "format": "bigquery-job-json",
+      "label": "BigQuery job plan (JSON)",
+      "module": "explain/dist/index.iife.js"
+    }
+  ],
   "paradigms": ["relational", "columnar"],
   "license": "Apache-2.0",
+  "readme": "README.md",
+  "min_runtime_version": "0.29.0",
   "homepage": "https://github.com/TabularisDB/tabularis-bigquery-plugin",
+  "documentation_url": "https://github.com/TabularisDB/tabularis-bigquery-plugin#readme",
   "support": {
     "issues_url": "https://github.com/TabularisDB/tabularis-bigquery-plugin/issues"
   },
   "default_port": 443,
   "default_username": "",
   "executable": "bigquery-plugin",
   "color": "#4285F4",
-  "icon": "cloud",
+  "icon": "https://raw.githubusercontent.com/TabularisDB/tabularis-bigquery-plugin/main/bigquery-icon.svg",
   "connection_fields": {
     "host": {"hidden": true},
     "port": {"hidden": true},
     "username": {"hidden": true},
     "password": {"hidden": true},
     "database": {
       "label": "GCP Project ID",
-      "placeholder": "billing-project"
+      "placeholder": "my-data-project"
     }
   },
   "ui_extensions": [
@@ -39,14 +53,35 @@
     "views": true,
     "materialized_views": true,
     "routines": true,
-    "routine_management": false,
+    "routine_management": true,
     "triggers": false,
     "file_based": false,
     "folder_based": false,
     "no_connection_required": false,
     "connection_string": false,
     "identifier_quote": "`",
-    "sql_dialect": "generic",
+    "sql_dialect": "mysql",
+    "sql_syntax": {
+      "extends": "generic",
+      "quotes": [
+        {"open": "'", "kind": "string", "escape": "backslash"},
+        {"open": "\"", "kind": "string", "escape": "backslash"},
+        {"open": "`", "kind": "identifier", "escape": "backslash"}
+      ],
+      "triple_quoted_strings": true,
+      "string_prefixes": ["r", "b", "rb", "br"],
+      "hash_comments": true,
+      "nested_block_comments": false,
+      "line_comment_requires_space": false,
+      "script_blocks": true,
+      "dollar_quoting": false,
+      "string_escape": "backslash",
+      "like_escape": false,
+      "boolean_literals": "keywords",
+      "formatter": "bigquery",
+      "keywords": ["QUALIFY", "PIVOT", "UNPIVOT", "TABLESAMPLE SYSTEM", "UNNEST", "WITH OFFSET", "EXCEPT", "REPLACE", "WINDOW", "OVER", "PARTITION BY", "CLUSTER BY", "OPTIONS", "FOR SYSTEM_TIME AS OF", "IGNORE NULLS", "RESPECT NULLS", "UNION ALL", "UNION DISTINCT", "INTERSECT DISTINCT", "EXCEPT DISTINCT", "MERGE", "WHEN MATCHED", "WHEN NOT MATCHED", "WHEN NOT MATCHED BY SOURCE", "CREATE OR REPLACE TABLE", "CREATE TEMP TABLE", "CREATE TEMP FUNCTION", "CREATE OR REPLACE PROCEDURE", "CREATE SCHEMA", "RETURNS", "LANGUAGE js", "EXPORT DATA", "LOAD DATA", "ASSERT", "DECLARE", "BEGIN", "BEGIN TRANSACTION", "COMMIT TRANSACTION", "ROLLBACK TRANSACTION", "EXCEPTION WHEN ERROR THEN", "ELSEIF", "END IF", "LOOP", "END LOOP", "WHILE", "END WHILE", "REPEAT", "UNTIL", "END REPEAT", "FOR", "DO", "END FOR", "END CASE", "LEAVE", "BREAK", "CONTINUE", "ITERATE", "RETURN", "CALL", "EXECUTE IMMEDIATE", "RAISE USING MESSAGE", "INT64", "FLOAT64", "NUMERIC", "BIGNUMERIC", "BOOL", "STRING", "BYTES", "DATE", "DATETIME", "TIME", "TIMESTAMP", "INTERVAL", "GEOGRAPHY", "JSON", "STRUCT", "ARRAY", "RANGE"],
+      "functions": ["SAFE_CAST", "SAFE.", "SAFE_DIVIDE", "IF", "IFNULL", "COALESCE", "NULLIF", "COUNTIF", "ARRAY_AGG", "STRING_AGG", "LOGICAL_AND", "LOGICAL_OR", "ANY_VALUE", "APPROX_COUNT_DISTINCT", "APPROX_QUANTILES", "APPROX_TOP_COUNT", "APPROX_TOP_SUM", "ROW_NUMBER", "RANK", "DENSE_RANK", "LAG", "LEAD", "FIRST_VALUE", "LAST_VALUE", "NTILE", "PERCENTILE_CONT", "ARRAY_LENGTH", "ARRAY_CONCAT", "ARRAY_TO_STRING", "GENERATE_ARRAY", "GENERATE_DATE_ARRAY", "SPLIT", "FORMAT", "STARTS_WITH", "ENDS_WITH", "CONTAINS_SUBSTR", "REGEXP_CONTAINS", "REGEXP_EXTRACT", "REGEXP_EXTRACT_ALL", "REGEXP_REPLACE", "REGEXP_INSTR", "FARM_FINGERPRINT", "GENERATE_UUID", "JSON_VALUE", "JSON_QUERY", "JSON_VALUE_ARRAY", "JSON_QUERY_ARRAY", "JSON_EXTRACT", "JSON_EXTRACT_SCALAR", "PARSE_JSON", "TO_JSON", "TO_JSON_STRING", "CURRENT_DATE", "CURRENT_DATETIME", "CURRENT_TIMESTAMP", "EXTRACT", "DATE_TRUNC", "DATE_ADD", "DATE_SUB", "DATE_DIFF", "DATETIME_TRUNC", "DATETIME_DIFF", "TIMESTAMP_TRUNC", "TIMESTAMP_ADD", "TIMESTAMP_SUB", "TIMESTAMP_DIFF", "FORMAT_DATE", "FORMAT_TIMESTAMP", "PARSE_DATE", "PARSE_TIMESTAMP", "UNIX_SECONDS", "UNIX_MILLIS", "TIMESTAMP_SECONDS", "TIMESTAMP_MILLIS", "LAST_DAY", "ST_GEOGPOINT", "ST_GEOGFROMTEXT", "ST_ASTEXT", "ST_DISTANCE", "ST_DWITHIN", "ST_CONTAINS", "ST_INTERSECTS", "ST_AREA", "ST_LENGTH", "ST_BUFFER"]
+    },
     "alter_primary_key": true,
     "alter_column": true,
     "create_foreign_keys": true,
@@ -56,20 +91,33 @@
     "readonly": false,
     "serial_type": "",
     "auto_increment_keyword": "",
-    "inline_pk": false
+    "inline_pk": false,
+    "column_comments": true,
+    "drop_table_sql": true
   },
   "type_mappings": {
     "INTEGER": "INT64",
+    "BIGINT": "INT64",
+    "SERIAL": "INT64",
     "REAL": "FLOAT64",
+    "DOUBLE": "FLOAT64",
+    "DECIMAL": "NUMERIC",
     "BOOLEAN": "BOOL",
+    "TEXT": "STRING",
+    "VARCHAR": "STRING",
+    "UUID": "STRING",
+    "BLOB": "BYTES",
+    "JSON": "JSON",
+    "DATE": "DATE",
+    "TIME": "TIME",
     "DATETIME": "DATETIME",
-    "TEXT": "STRING"
+    "TIMESTAMP": "TIMESTAMP"
   },
   "data_types": [
     {"name": "INT64", "category": "numeric", "requires_length": false, "requires_precision": false},
     {"name": "FLOAT64", "category": "numeric", "requires_length": false, "requires_precision": false},
-    {"name": "NUMERIC", "category": "numeric", "requires_length": false, "requires_precision": true},
-    {"name": "BIGNUMERIC", "category": "numeric", "requires_length": false, "requires_precision": true},
+    {"name": "NUMERIC", "category": "numeric", "requires_length": false, "requires_precision": false},
+    {"name": "BIGNUMERIC", "category": "numeric", "requires_length": false, "requires_precision": false},
     {"name": "STRING", "category": "string", "requires_length": false, "requires_precision": false},
     {"name": "BYTES", "category": "binary", "requires_length": false, "requires_precision": false},
     {"name": "BOOL", "category": "other", "requires_length": false, "requires_precision": false},

CHANGELOG.md

@@ -2,11 +2,64 @@
 
 All notable changes to this project are documented in this file.
 
-## Unreleased
+## 0.2.0
+
+Requires Tabularis 0.29.0 or later (`min_runtime_version`): it relies on plugin connection fields and secrets, the DDL connection context, the declarative `sql_syntax` capability and the bundled Visual EXPLAIN parser.
+
+### Query runtime
+
+- Handle JSON-RPC requests concurrently with a single ordered writer, cancel the running BigQuery job when Tabularis sends a `cancel` notification, and answer with an internal error instead of never replying when a request handler fails.
+- Start jobs with a `requestId`, the `client: tabularis` label and a server-side `jobTimeoutMs`; cancel jobs that exceed the query timeout. The default query timeout is now 110 seconds, below the host's 120 second call timeout.
+- Retry HTTP 429, 5xx and 403 `rateLimitExceeded`/`backendError` responses with jittered backoff and refresh the token once after HTTP 401. Judge job failure by `status.errorResult`.
+- Fill result pages across `pageToken` responses, keep the rows returned by `jobs.query`, and read later pages from the finished job instead of running the SQL again.
+- Return TIMESTAMP values with exact microseconds.
+- Serve plain `SELECT * FROM table` table browsing through the free `tabledata.list` API.
+- Support Tabularis editor sessions with BigQuery sessions, transaction tracking (including COMMIT and ROLLBACK inside scripting blocks) and `release_session`; run stateful batches in a short-lived session.
+- Return the last result set and summed DML row count of multi-statement scripts.
+- Return Visual EXPLAIN as a raw `bigquery-job-json` payload with a bundled parser: dry-run bytes and on-demand cost estimates, and executed stage plans for EXPLAIN ANALYZE.
+- Map common BigQuery error reasons to messages with hints.
+- Run metadata queries, DDL, row edits and routine statements through the same job path as editor queries, with typed named query parameters, so they get request IDs, retries, job timeouts, labels, error hints and cancellation.
+- Clear the metadata cache after any editor statement other than a plain query, so DDL and DML run from a query tab show up at once.
+
+### Projects and metadata
+
+- Browse several projects from one connection: `get_databases` lists visible projects plus the `additional_projects` setting, and metadata follows the selected project.
+- Run and bill every job in the optional `billing_project` (default: the GCP Project ID), while unqualified names and table browsing resolve in the selected project.
+- Read routines through the free routines REST API, return host-compatible routine parameters with ordinal positions and a named return row, and add routine management: run, create from templates, edit with rebuilt `CREATE OR REPLACE` scripts (SQL, JavaScript, Python, remote, table and aggregate functions, procedures) and drop.
+- Fetch each table resource once with bounded concurrency for schema snapshots, ER diagram batches and AI context, cache metadata briefly, and return the requested tables from `get_all_columns_batch` and `get_all_foreign_keys_batch`, skipping tables that cannot be read instead of failing the batch.
+- Fill table descriptions, parameterized column types, numeric precision and scale, string lengths and composite foreign-key order.
+- Add `get_table_ddl` and use `INFORMATION_SCHEMA.TABLES.ddl` for table, view and materialized view definitions, with a metadata-based fallback that keeps partitioning, clustering and options.
+- Keep view options when saving a view and refuse to replace legacy SQL views.
+- Refresh materialized views on demand with `BQ.REFRESH_MATERIALIZED_VIEW`.
+
+### DDL and row editing
+
+- Quote every part of generated identifiers and fully qualify objects as `` `project`.`dataset`.`object` ``. Table, column, foreign-key and routine template builders use the browsed project from the connection parameters the host sends with every DDL request, and fall back to dataset-qualified names on older hosts.
+- Declare `drop_table_sql` and implement `get_drop_table_sql`, so the explorer drops tables, external tables, snapshots, views and materialized views with the matching statement.
+- Declare `column_comments`: column descriptions from the column editor are written with `OPTIONS(description = ...)` on create and add, and changed or cleared with `ALTER COLUMN ... SET OPTIONS(description = ...)`.
+- Drop views and materialized views with the matching statement, and reject row edits and column changes on snapshots and external tables.
+- Generate only DDL that BigQuery accepts: no `NOT NULL` on added columns, no `SET NOT NULL`, type changes limited to supported widenings, column descriptions through `SET OPTIONS`, primary keys added and dropped as `NOT ENFORCED`, and optional partitioning, clustering and description on new tables.
+- Edit rows with typed query parameters; support tables without a primary key by matching comparable columns inside a transaction that rolls back when more than one row matches; report real affected-row counts and explain streaming-buffer errors.
+- Declare the GoogleSQL syntax (backtick identifiers, triple-quoted and raw strings, `#` comments, procedural `BEGIN … END` blocks, backslash literals, `TRUE`/`FALSE` boolean literals, keywords and functions) through `sql_syntax`, so Tabularis splits scripts, formats SQL and builds filters the GoogleSQL way. Older hosts fall back to the `mysql` dialect.
+- Write valid GoogleSQL literals for strings with newlines and control characters, NUMERIC, BIGNUMERIC, GEOGRAPHY, INTERVAL, RANGE, JSON, BYTES, ARRAY and STRUCT values.
+
+### Connection and authentication
 
 - Hide unused common connection fields on compatible Tabularis hosts.
 - Store plugin-owned credentials in the OS keychain while preserving legacy Password-based connections.
-- Author the connection UI in React and TypeScript and generate its release bundle with Vite.
+- Author the connection UI in React and TypeScript and generate its release bundle with Vite, with advanced fields for the billing project, additional projects, API endpoint and impersonation.
+- Support `impersonated_service_account` ADC files and an optional service account to impersonate, reject service account keys whose `token_uri` is not a googleapis.com endpoint, and tolerate a byte order mark in credentials.
+- Add an API endpoint override for Private Service Connect and emulators, and a `none` authentication method for localhost endpoints.
+- Fix the client cache key to cover the plugin-owned credential and every connection option, including the billing and data project.
+- Use a single `datasets.list` call for `ping`.
+- Fix BigQuery REST URLs that contained a double slash after the API version.
+
+### Packaging and tests
+
+- Fix the GCP Project ID placeholder, which suggested the billing project.
+- Let `pnpm install --frozen-lockfile` run the esbuild install script with pnpm 10 and later.
+- Complete the manifest type mappings, stop requiring precision for NUMERIC and BIGNUMERIC, add registry metadata and declare the Visual EXPLAIN parser.
+- Add mocked HTTP, host-shape conformance and BigQuery emulator integration tests for the query runtime and for metadata, DDL and row editing.
 
 ## 0.1.0