From f8a467a530a21826d506daa50b7a425f8e400d8b Mon Sep 17 00:00:00 2001 From: Evan Rusackas Date: Tue, 22 Sep 2026 16:07:25 -0700 Subject: [PATCH 1/4] test(bigquery): verify string-literal escaping against a real GoogleSQL engine Adds testcontainers coverage for superset/db_engine_specs/bigquery.py's _monkeypatch_bigquery_string_literal, using the new BigQueryContainer (testcontainers/testcontainers-python#1121) to run apostrophe, percent-sign, and combined-value queries against a real GoogleSQL emulator rather than reasoning about the sqlalchemy-bigquery dialect and BigQuery DBAPI paramstyle handling from source alone. Also pins down, with a direct reproduction, why the doubled-single-quote escape #38835 replaced doesn't work on BigQuery. Co-Authored-By: Evan Rusackas Co-Authored-By: Claude Sonnet 5 --- .../db_engine_specs/test_bigquery.py | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 tests/testcontainers/db_engine_specs/test_bigquery.py diff --git a/tests/testcontainers/db_engine_specs/test_bigquery.py b/tests/testcontainers/db_engine_specs/test_bigquery.py new file mode 100644 index 000000000000..55a362bcca97 --- /dev/null +++ b/tests/testcontainers/db_engine_specs/test_bigquery.py @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Tests Superset's BigQuery string-literal escaping (superset/db_engine_specs/ +bigquery.py's ``_monkeypatch_bigquery_string_literal``) against a real +GoogleSQL query engine, spun up on demand via testcontainers. Run via +.github/workflows/testcontainers.yml. + +sc-120493-adjacent investigation: an apostrophe in a filter value used to +break BigQuery queries (apache/superset#35857 / #38835, doubled single +quotes -- BigQuery rejects ``'Armando''s'`` as two adjacent string literals +needing whitespace between them). The current fix backslash-escapes instead. +Reasoning about correctness from the ``sqlalchemy-bigquery`` dialect source +and the BigQuery DBAPI's ``pyformat`` paramstyle handling is necessary but +not sufficient; this locks in the actual compiled-and-executed behavior +against a real engine instead. +""" + +from collections.abc import Iterator + +import pytest +import sqlalchemy as sa +from sqlalchemy.engine import Engine + +pytestmark = pytest.mark.testcontainers + +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.community.google") + +from google.cloud import bigquery # noqa: E402 +from sqlalchemy_bigquery import BigQueryDialect # noqa: E402 +from testcontainers.community.google import BigQueryContainer # noqa: E402 + +# Importing this triggers _monkeypatch_bigquery_string_literal(), exactly as +# it runs in a real Superset process. +import superset.db_engine_specs.bigquery # noqa: E402, F401 + +DATASET = "ds" +TABLE = "t" + + +@pytest.fixture(scope="module") +def bq_client() -> Iterator[bigquery.Client]: + with BigQueryContainer() as container: + client = container.get_client() + client.create_dataset(f"{client.project}.{DATASET}") + client.query(f"CREATE TABLE {DATASET}.{TABLE} (name STRING)").result() + yield client + + +@pytest.fixture(scope="module") +def engine(bq_client) -> Engine: + # user_supplied_client=true is a URL query param, not just a connect_args + # key: parse_url() only sets BigQueryDialect.create_connect_args() to + # accept the connect_args={"client": ...} override when it's present, + # otherwise it tries to build a client from real GCP credentials. + return sa.create_engine( + "bigquery://?user_supplied_client=true", connect_args={"client": bq_client} + ) + + +def _compiled_literal(expr: sa.ColumnElement) -> str: + """Render ``expr`` exactly as Superset's actual code path does: compiled + with ``literal_binds=True``, then executed as a plain string with no + separate bind parameters (superset.db_engine_specs.base.BaseEngineSpec + .execute() calls ``cursor.execute(query)``, nothing else).""" + return str( + expr.compile(dialect=BigQueryDialect(), compile_kwargs={"literal_binds": True}) + ) + + +def _insert_and_find(engine: Engine, value: str) -> list[str]: + t = sa.table(TABLE, sa.column("name")) + with engine.connect() as conn: + conn.execute(sa.text(f"DELETE FROM {DATASET}.{TABLE} WHERE TRUE")) # noqa: S608 + insert_literal = _compiled_literal(sa.literal(value)) + conn.execute( + sa.text( + f"INSERT INTO {DATASET}.{TABLE} (name) VALUES ({insert_literal})" # noqa: S608 + ) + ) + where = _compiled_literal(t.c.name == value) + rows = conn.execute( + sa.text(f"SELECT name FROM {DATASET}.{TABLE} WHERE {where}") # noqa: S608 + ).fetchall() + return [row[0] for row in rows] + + +def test_apostrophe_value_round_trips(engine: Engine) -> None: + """Regression test for apache/superset#35857: an apostrophe in a filter + value must not corrupt the compiled query or fail to match.""" + assert _insert_and_find(engine, "O'Brien") == ["O'Brien"] + + +def test_percent_sign_value_round_trips(engine: Engine) -> None: + """ + A literal percent sign must survive Superset's actual execution path + unchanged. Superset's literal_processor does not double it (unlike the + upstream sqlalchemy-bigquery function it replaces), which is correct + specifically because Superset always executes via cursor.execute(query) + with no separate `parameters` -- the BigQuery DBAPI's own pyformat + handling only applies `%%` -> `%` de-escaping in that case + (google.cloud.bigquery.dbapi.cursor._format_operation), so a lone `%` + passes through untouched either way. A doubled `%%` would also survive + (de-escaped back to one `%`), so this test would not by itself catch a + regression toward doubling -- it exists to pin the actually-shipped + behavior, not to distinguish the two. + """ + assert _insert_and_find(engine, "100% sure") == ["100% sure"] + + +def test_combined_percent_and_apostrophe_round_trips(engine: Engine) -> None: + assert _insert_and_find(engine, "50% off for O'Brien") == ["50% off for O'Brien"] + + +def test_doubled_single_quotes_are_rejected_by_bigquery(bq_client) -> None: + """ + Documents *why* the fix in #38835 was needed: BigQuery does not accept + the standard-SQL doubled-single-quote escape convention Superset used to + emit. If this test ever starts failing because the query succeeds, that + is a BigQuery/GoogleSQL behavior change worth knowing about, not a + Superset regression. + + Goes through the raw client with retries disabled, not engine.connect(): + the emulator reports this syntax error as a generic retryable INTERNAL + rather than a 400, so the client library's default retry policy spends + close to a minute retrying a failure that will never succeed. + """ + with pytest.raises(Exception, match="concatenated string literals"): + bq_client.query( + f"SELECT * FROM {DATASET}.{TABLE} WHERE name IN ('Armando''s')", # noqa: S608 + job_retry=None, + ).result(retry=None) From d7bf85297ecd1d8598ec9caace52f8944f015a19 Mon Sep 17 00:00:00 2001 From: Evan Rusackas Date: Wed, 23 Sep 2026 13:00:44 -0700 Subject: [PATCH 2/4] test(bigquery): add docstring and fixture type hint per review Adds a one-line docstring to the combined percent/apostrophe round-trip test and a `bigquery.Client` annotation on the `bq_client` fixture parameter so the new tests match their siblings. Co-Authored-By: Evan Rusackas Co-Authored-By: Claude Fable 5.1 --- tests/testcontainers/db_engine_specs/test_bigquery.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/testcontainers/db_engine_specs/test_bigquery.py b/tests/testcontainers/db_engine_specs/test_bigquery.py index 55a362bcca97..bbba559a25e3 100644 --- a/tests/testcontainers/db_engine_specs/test_bigquery.py +++ b/tests/testcontainers/db_engine_specs/test_bigquery.py @@ -125,10 +125,13 @@ def test_percent_sign_value_round_trips(engine: Engine) -> None: def test_combined_percent_and_apostrophe_round_trips(engine: Engine) -> None: + """Round-trips a value containing both a percent sign and an apostrophe.""" assert _insert_and_find(engine, "50% off for O'Brien") == ["50% off for O'Brien"] -def test_doubled_single_quotes_are_rejected_by_bigquery(bq_client) -> None: +def test_doubled_single_quotes_are_rejected_by_bigquery( + bq_client: bigquery.Client, +) -> None: """ Documents *why* the fix in #38835 was needed: BigQuery does not accept the standard-SQL doubled-single-quote escape convention Superset used to From a4d1960f2483754bad54e1561466f0df278b033f Mon Sep 17 00:00:00 2001 From: Evan Rusackas Date: Tue, 29 Sep 2026 20:07:00 -0700 Subject: [PATCH 3/4] test(bigquery): vendor a local BigQueryContainer and add it to the CI matrix testcontainers-python has no released BigQueryContainer -- it's still an open, unmerged upstream PR (testcontainers/testcontainers-python#1121), missing from every published release including the current latest (4.15.0). Importing it from testcontainers.community.google as this test did previously would ImportError on any real installation. Vendors a local BigQueryContainer (tests/testcontainers/db_engine_specs/ _bigquery_container.py) wrapping the same goccy/bigquery-emulator image, following the same pattern this suite already uses for StarRocks and ClickHouse when testcontainers-python has no dedicated module. Adds the missing `bigquery` entry to testcontainers.yml's matrix so this coverage actually runs in CI, and documents why no extra is needed for it in requirements/development.in. Verified locally against real Docker with the same env vars the CI job uses -- all 4 tests pass. Co-Authored-By: Evan Rusackas Co-Authored-By: Claude Sonnet 5 --- .github/workflows/testcontainers.yml | 1 + requirements/development.in | 7 +- .../db_engine_specs/_bigquery_container.py | 71 +++++++++++++++++++ .../db_engine_specs/test_bigquery.py | 7 +- 4 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 tests/testcontainers/db_engine_specs/_bigquery_container.py diff --git a/.github/workflows/testcontainers.yml b/.github/workflows/testcontainers.yml index 51a54d0c6852..75f783ffa4ff 100644 --- a/.github/workflows/testcontainers.yml +++ b/.github/workflows/testcontainers.yml @@ -86,6 +86,7 @@ jobs: {"dialect": "risingwave", "timeout": 10}, {"dialect": "firebird", "timeout": 10}, {"dialect": "ydb", "timeout": 10}, + {"dialect": "bigquery", "timeout": 10}, {"dialect": "oceanbase", "timeout": 20, "nightly_only": true} ] run: | diff --git a/requirements/development.in b/requirements/development.in index d75496be099a..510e7f3c882f 100644 --- a/requirements/development.in +++ b/requirements/development.in @@ -43,5 +43,10 @@ # already provides. databend/risingwave/firebird/ydb are the same story: # none has a dedicated testcontainers module, so each test uses a generic # DockerContainer plus whatever driver its own extra above already -# provides. +# provides. bigquery is the same story too, and for the same reason as +# db2/oceanbase above -- testcontainers-python has no BigQueryContainer in +# any release yet (still an open upstream PR), so its test vendors one +# locally against a generic DockerContainer (see +# tests/testcontainers/db_engine_specs/_bigquery_container.py) using the +# google-cloud-bigquery client the bigquery extra above already provides. testcontainers[cockroachdb,cratedb,mongodb,mssql,mysql,oracle,postgres,trino]>=4.15.0,<5 diff --git a/tests/testcontainers/db_engine_specs/_bigquery_container.py b/tests/testcontainers/db_engine_specs/_bigquery_container.py new file mode 100644 index 000000000000..125f76e6a354 --- /dev/null +++ b/tests/testcontainers/db_engine_specs/_bigquery_container.py @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Locally vendored ``BigQueryContainer``. + +testcontainers-python has no released ``BigQueryContainer``: adding one is +still an open, unmerged upstream PR (testcontainers/testcontainers-python +#1121, tracking the older #393/#925) as of this writing, and the class does +not exist in any published release (including the latest, 4.15.0) or on the +project's default branch. Rather than depend on an unreleased upstream +class, this follows the same pattern this test suite already uses for +StarRocks and ClickHouse (see test_starrocks.py): wrap the container image +directly with ``testcontainers.core.container.DockerContainer``. Delete this +file and import from ``testcontainers.community.google`` instead once that +PR ships in a release. + +Wraps `goccy/bigquery-emulator `_, +a GoogleSQL implementation over an embedded SQLite database. It is not +BigQuery itself, so treat query results as a strong signal rather than a +guarantee for anything outside standard GoogleSQL (BigQuery-specific +services like BigQuery ML, row access policies, or external tables are out +of scope). +""" + +from google.auth.credentials import AnonymousCredentials +from google.cloud import bigquery +from testcontainers.core.container import DockerContainer +from testcontainers.core.waiting_utils import wait_for_logs + + +class BigQueryContainer(DockerContainer): + def __init__( + self, + image: str = "ghcr.io/goccy/bigquery-emulator:latest", + project: str = "test-project", + port: int = 9050, + grpc_port: int = 9060, + **kwargs: object, + ) -> None: + super().__init__(image=image, **kwargs) + self.project = project + self.port = port + self.grpc_port = grpc_port + self.with_exposed_ports(self.port, self.grpc_port) + self.with_command(f"--project={project} --port={port} --grpc-port={grpc_port}") + + def get_rest_endpoint(self) -> str: + return ( + f"http://{self.get_container_host_ip()}:{self.get_exposed_port(self.port)}" + ) + + def get_client(self, **kwargs: object) -> bigquery.Client: + wait_for_logs(self, "REST server listening at", timeout=30.0) + kwargs.setdefault("project", self.project) + kwargs.setdefault("credentials", AnonymousCredentials()) + kwargs.setdefault("client_options", {"api_endpoint": self.get_rest_endpoint()}) + return bigquery.Client(**kwargs) diff --git a/tests/testcontainers/db_engine_specs/test_bigquery.py b/tests/testcontainers/db_engine_specs/test_bigquery.py index bbba559a25e3..fd5ef429e304 100644 --- a/tests/testcontainers/db_engine_specs/test_bigquery.py +++ b/tests/testcontainers/db_engine_specs/test_bigquery.py @@ -38,18 +38,15 @@ pytestmark = pytest.mark.testcontainers -from ._driver import require_driver # noqa: E402 - -require_driver("testcontainers.community.google") - from google.cloud import bigquery # noqa: E402 from sqlalchemy_bigquery import BigQueryDialect # noqa: E402 -from testcontainers.community.google import BigQueryContainer # noqa: E402 # Importing this triggers _monkeypatch_bigquery_string_literal(), exactly as # it runs in a real Superset process. import superset.db_engine_specs.bigquery # noqa: E402, F401 +from ._bigquery_container import BigQueryContainer # noqa: E402 + DATASET = "ds" TABLE = "t" From 615e64e4829c2ae5c91e14740ebf58d112d465d8 Mon Sep 17 00:00:00 2001 From: Evan Rusackas Date: Wed, 30 Sep 2026 03:46:15 -0700 Subject: [PATCH 4/4] test(bigquery): guard optional import, add container docstrings Adds the shared require_driver() guard around the testcontainers import so environments without the testcontainers/BigQuery extras skip this module at collection time instead of failing, matching the pattern the other per-dialect testcontainers tests already use. Also adds one-line docstrings to the vendored BigQueryContainer class and its methods. Co-Authored-By: Evan Rusackas Co-Authored-By: Claude Sonnet 5 --- tests/testcontainers/db_engine_specs/_bigquery_container.py | 5 +++++ tests/testcontainers/db_engine_specs/test_bigquery.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/tests/testcontainers/db_engine_specs/_bigquery_container.py b/tests/testcontainers/db_engine_specs/_bigquery_container.py index 125f76e6a354..fff88e95c9a7 100644 --- a/tests/testcontainers/db_engine_specs/_bigquery_container.py +++ b/tests/testcontainers/db_engine_specs/_bigquery_container.py @@ -43,6 +43,8 @@ class BigQueryContainer(DockerContainer): + """Wraps the `goccy/bigquery-emulator` image in a plain ``DockerContainer``.""" + def __init__( self, image: str = "ghcr.io/goccy/bigquery-emulator:latest", @@ -51,6 +53,7 @@ def __init__( grpc_port: int = 9060, **kwargs: object, ) -> None: + """Configure the emulator's REST/gRPC ports and startup command.""" super().__init__(image=image, **kwargs) self.project = project self.port = port @@ -59,11 +62,13 @@ def __init__( self.with_command(f"--project={project} --port={port} --grpc-port={grpc_port}") def get_rest_endpoint(self) -> str: + """Return the emulator's host-mapped REST API base URL.""" return ( f"http://{self.get_container_host_ip()}:{self.get_exposed_port(self.port)}" ) def get_client(self, **kwargs: object) -> bigquery.Client: + """Wait for the emulator to be ready and return a client pointed at it.""" wait_for_logs(self, "REST server listening at", timeout=30.0) kwargs.setdefault("project", self.project) kwargs.setdefault("credentials", AnonymousCredentials()) diff --git a/tests/testcontainers/db_engine_specs/test_bigquery.py b/tests/testcontainers/db_engine_specs/test_bigquery.py index fd5ef429e304..36dec4a7c930 100644 --- a/tests/testcontainers/db_engine_specs/test_bigquery.py +++ b/tests/testcontainers/db_engine_specs/test_bigquery.py @@ -38,6 +38,10 @@ pytestmark = pytest.mark.testcontainers +from ._driver import require_driver # noqa: E402 + +require_driver("testcontainers.core.container") + from google.cloud import bigquery # noqa: E402 from sqlalchemy_bigquery import BigQueryDialect # noqa: E402