diff --git a/.github/workflows/integration-tests-sqlserver.yml b/.github/workflows/integration-tests-sqlserver.yml index c31c55f6..0aa6f2f3 100644 --- a/.github/workflows/integration-tests-sqlserver.yml +++ b/.github/workflows/integration-tests-sqlserver.yml @@ -43,31 +43,31 @@ jobs: matrix: python_version: ["3.10", "3.11", "3.12", "3.13"] backend: [pyodbc, mssql-python] - # Baseline on 2022 - sqlserver_version: ["2022"] + # Baseline on 2025 (newest) + sqlserver_version: ["2025"] msodbc_version: ["18"] collation: [SQL_Latin1_General_CP1_CI_AS] exclude: - backend: mssql-python python_version: "3.10" - sqlserver_version: "2022" + sqlserver_version: "2025" - backend: mssql-python python_version: "3.11" - sqlserver_version: "2022" + sqlserver_version: "2025" - backend: mssql-python python_version: "3.12" - sqlserver_version: "2022" + sqlserver_version: "2025" include: # Keep pyodbc on every supported Python version, but retain # SQL Server ODBC 17 coverage for the oldest and newest Python. - backend: pyodbc python_version: "3.10" - sqlserver_version: "2022" + sqlserver_version: "2025" msodbc_version: "17" collation: SQL_Latin1_General_CP1_CI_AS - backend: pyodbc python_version: "3.13" - sqlserver_version: "2022" + sqlserver_version: "2025" msodbc_version: "17" collation: SQL_Latin1_General_CP1_CI_AS # Older SQL Server versions stay on pyodbc only, with a single @@ -82,17 +82,22 @@ jobs: sqlserver_version: "2019" msodbc_version: "17" collation: SQL_Latin1_General_CP1_CI_AS - # Add the case-sensitive collation only on the latest SQL Server - # and latest Python/backend rows. - backend: pyodbc python_version: "3.13" sqlserver_version: "2022" msodbc_version: "17" + collation: SQL_Latin1_General_CP1_CI_AS + # Add the case-sensitive collation on the SQL Server 2025 baseline + # and latest Python/backend rows. + - backend: pyodbc + python_version: "3.13" + sqlserver_version: "2025" + msodbc_version: "17" collation: SQL_Latin1_General_CP1_CS_AS # mssql-python stays on the latest Python only. - backend: mssql-python python_version: "3.13" - sqlserver_version: "2022" + sqlserver_version: "2025" msodbc_version: "18" collation: SQL_Latin1_General_CP1_CS_AS runs-on: ubuntu-latest diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 0a7bdb8f..ce9a1e9b 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -44,7 +44,7 @@ jobs: publish-docker-server: strategy: matrix: - mssql_version: ["2017", "2019", "2022"] + mssql_version: ["2017", "2019", "2022", "2025"] runs-on: ubuntu-latest permissions: contents: read diff --git a/CHANGELOG.md b/CHANGELOG.md index 83acd090..7e547975 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Add `drop_unmanaged_indexes` config (`false` (default) / `warn` / `true`) for indexes dbt didn't create. - Validate cross-index config conflicts (multiple clustered indexes, clustered vs `as_columnstore`). - Document the minimum supported SQL Server version (2017). Partitioning, `XML_COMPRESSION` and ordered columnstore are not yet expressible in the `indexes` config. +- Add SQL Server 2025 to the integration-test matrix (pyodbc and `mssql-python` backends, ODBC Driver 18) and document it as a supported version. - Add `dbt_sqlserver_enable_safe_type_expansion` behaviour flag to allow safe column type widening during schema expansion: `varchar` → `nvarchar`, integer family promotions (`bit` → `tinyint` → `smallint` → `int` → `bigint`), and `numeric`/`decimal` precision/scale upgrades. Gated by the per-model `column_type_expansion_max_rows` config (default 1,000,000 rows). [#699](https://github.com/dbt-msft/dbt-sqlserver/issues/699). - Add `prefer_single_alter_column` model config to use a single `ALTER COLUMN` statement instead of the add+update+drop+rename pattern when altering column types on tables. - Add `string_type_instance()` to preserve the NVARCHAR/NCHAR type family during column expansion, fixing incorrect promotion of NVARCHAR/NCHAR to VARCHAR. @@ -27,6 +28,7 @@ - Fix seed table ingestion of empty numeric cells by inlining `null` literals instead of binding parameters. [#425](https://github.com/dbt-msft/dbt-sqlserver/issues/425) - Guard `run_hooks` commit with `@@trancount` check for autocommit safety when running post-hooks. [#444](https://github.com/dbt-msft/dbt-sqlserver/issues/444) - Fix `columnstore IF EXISTS` guard to check `object_id('schema.table')` correctly. +- Gate the `optimize_for_sequential_key` and `resumable` index options on the detected engine version: they require SQL Server 2019+, so on 2017/2016 the index is now built without them (with a warning) instead of failing with "is not a recognized CREATE INDEX option". - Escape single quotes in `query_tag` before building `OPTION (LABEL)` clause. - Map Python `float` to SQL Server `float`, not `bigint`. - Set default port to `1433` (instead of Postgres `5432`) in `dbt init` profile template. diff --git a/README.md b/README.md index 26f7d506..c17e8751 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,20 @@ The adapter supports dbt-core 1.10 or newer and follows the same versioning scheme. E.g. version 1.10.x of the adapter is compatible with dbt-core 1.10.x. -The minimum supported SQL Server version is SQL Server 2017. +## Supported SQL Server versions + +The adapter is tested against the following SQL Server versions: + +| SQL Server version | Supported | +|---|---| +| SQL Server 2017 | ✅ (minimum supported version) | +| SQL Server 2019 | ✅ | +| SQL Server 2022 | ✅ | +| SQL Server 2025 | ✅ | + +The minimum supported SQL Server version is SQL Server 2017; older versions are not supported. + +SQL Server 2017, 2019, 2022, and 2025 are covered by the integration test suite. Azure SQL Database and Azure SQL Managed Instance are not covered by the integration test suite, but are expected to be compatible. ## Documentation @@ -180,6 +193,29 @@ flags: **Compatibility notes:** Enabling `dbt_sqlserver_use_dbt_transactions: true` may expose transaction-state assumptions hidden by autocommit-only mode. Explicit transaction macros may interact with dbt-managed transactions, and cleanup after failed DDL/DML may differ. Review pre/post hooks for in-transaction vs out-of-transaction semantics. +### `as_columnstore` + +*(default: `true`)* When building a table, the adapter creates a [clustered columnstore index](https://learn.microsoft.com/en-us/sql/relational-databases/indexes/columnstore-indexes-overview) (CCI) on it. Set `as_columnstore: false` to build a plain rowstore table instead. + +This matters for any table containing a `(n)varchar(max)` or other LOB column, because SQL Server does not allow those data types to participate in a columnstore index. The table build fails with: + +> Column '...' has a data type that cannot participate in a columnstore index. + +A common case is dbt's [test failure storage](https://docs.getdbt.com/reference/resource-configs/store_failures): the audit tables can contain `VARCHAR(MAX)` columns (dbt's `STRING` type maps to `VARCHAR(MAX)`), so disable the CCI on those resources: + +```yaml +# dbt_project.yml +data_tests: + +store_failures: true + +as_columnstore: false # avoids CCI on (n)varchar(max) audit columns +``` + +You can also set it per model: + +```sql +{{ config(materialized="table", as_columnstore=false) }} +``` + ## Contributing [![Unit tests](https://github.com/dbt-msft/dbt-sqlserver/actions/workflows/unit-tests.yml/badge.svg)](https://github.com/dbt-msft/dbt-sqlserver/actions/workflows/unit-tests.yml) diff --git a/dbt/include/sqlserver/macros/adapters/indexes.sql b/dbt/include/sqlserver/macros/adapters/indexes.sql index 72802eab..464e0408 100644 --- a/dbt/include/sqlserver/macros/adapters/indexes.sql +++ b/dbt/include/sqlserver/macros/adapters/indexes.sql @@ -238,6 +238,20 @@ {% endmacro %} +{% macro sqlserver__server_major_version() -%} + {#- Detected engine major version: 13 = 2016, 14 = 2017, 15 = 2019, + 16 = 2022, 17 = 2025. parsename() on the always-4-part productversion + works on every supported release, unlike + SERVERPROPERTY('ProductMajorVersion') which is 2014+ only. Returns none + outside an executing context (e.g. parse time). -#} + {%- if not execute -%}{{ return(none) }}{%- endif -%} + {%- set result = run_query( + "select cast(parsename(cast(serverproperty('productversion') as varchar(128)), 4) as int) as major_version" + ) -%} + {{ return(result.columns[0].values()[0]) }} +{%- endmacro %} + + {% macro sqlserver__get_create_index_sql(relation, index_dict) -%} {%- set index_config = adapter.parse_index(index_dict) -%} {%- set index_name = index_config.render(relation) -%} @@ -274,6 +288,30 @@ {% if index_config.where -%} where {{ index_config.where }} {% endif %} + {#- optimize_for_sequential_key, RESUMABLE and RESUMABLE's MAX_DURATION are + CREATE INDEX options that SQL Server only recognizes on 2019 (major 15) + and newer. This adapter still supports 2017/2016, so on older engines + they are dropped (with a warning) and the index is built without them, + rather than failing with "is not a recognized CREATE INDEX option". + ONLINE is intentionally kept: it is recognized on 2017 (edition-gated, + not version-gated). The server version is only queried when one of these + options is actually requested, so the common path adds no round-trip. -#} + {%- set v2019_only_build_options = ['resumable', 'max_duration'] -%} + {%- set _build_options = index_config.build_options or {} -%} + {%- set _wants_v2019_option = index_config.optimize_for_sequential_key + or _build_options.get('resumable') or _build_options.get('max_duration') -%} + {%- set drop_v2019_options = false -%} + {%- if _wants_v2019_option -%} + {%- set _major = sqlserver__server_major_version() -%} + {%- if _major is not none and _major < 15 -%} + {%- set drop_v2019_options = true -%} + {%- do log( + "Index [" ~ index_name ~ "] on " ~ relation ~ ": optimize_for_sequential_key" + ~ " / resumable require SQL Server 2019 (15.x) or newer; building the index" + ~ " without them on detected major version " ~ _major ~ ".", info=true + ) -%} + {%- endif -%} + {%- endif -%} {%- set with_options = [] -%} {%- if index_config.data_compression -%} {%- do with_options.append('data_compression = ' ~ index_config.data_compression | upper) -%} @@ -287,14 +325,16 @@ {%- if index_config.ignore_dup_key -%} {%- do with_options.append('ignore_dup_key = on') -%} {%- endif -%} - {%- if index_config.optimize_for_sequential_key -%} + {%- if index_config.optimize_for_sequential_key and not drop_v2019_options -%} {%- do with_options.append('optimize_for_sequential_key = on') -%} {%- endif -%} {%- if index_config.sort_in_tempdb -%} {%- do with_options.append('sort_in_tempdb = on') -%} {%- endif -%} - {%- for option_key, option_value in (index_config.build_options or {}).items() -%} - {%- if option_value is sameas true -%} + {%- for option_key, option_value in _build_options.items() -%} + {%- if drop_v2019_options and option_key in v2019_only_build_options -%} + {#- skipped: not recognized before SQL Server 2019 -#} + {%- elif option_value is sameas true -%} {%- do with_options.append(option_key ~ ' = on') -%} {%- elif option_value is sameas false -%} {%- do with_options.append(option_key ~ ' = off') -%} diff --git a/devops/server.Dockerfile b/devops/server.Dockerfile index d5743136..a7a3fe08 100644 --- a/devops/server.Dockerfile +++ b/devops/server.Dockerfile @@ -1,5 +1,5 @@ -ARG SQLServer_VERSION="2022" -FROM mcr.microsoft.com/mssql/server:${SQLServer_VERSION}-latest +ARG MSSQL_VERSION="2022" +FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION}-latest ENV COLLATION="SQL_Latin1_General_CP1_CI_AS" diff --git a/tests/functional/adapter/mssql/test_store_failures_passing.py b/tests/functional/adapter/mssql/test_store_failures_passing.py new file mode 100644 index 00000000..7aa44f18 --- /dev/null +++ b/tests/functional/adapter/mssql/test_store_failures_passing.py @@ -0,0 +1,102 @@ +# flake8: noqa: E501 +"""Regression test for dbt-msft/dbt-sqlserver#601. + +With ``--store-failures``, a test that *passes* must leave behind an empty +audit relation, not drop it. dbt's contract: "A test's results will always +replace previous failures for the same test, even if that test results in no +failures." The SQL Server adapter was reported to ``DROP`` the audit table on a +passing test instead of replacing it with an empty table (Postgres creates the +empty table). + +This exercises the exact reported scenario: a passing test configured with +``store_failures`` materialized as a ``table``. It asserts the audit relation +exists, is a base table (not a view), is empty, and survives idempotent re-runs. +""" + +import pytest + +from dbt.tests.util import run_dbt + +# the default audit schema (_dbt_test__audit) plus the test schema can exceed +# identifier limits; use a short suffix as the rest of the suite does. +TEST_AUDIT_SCHEMA_SUFFIX = "dbt_test__aud" + +model__chipmunks = """ +select 1 as id, 'alvin' as name +union all +select 2 as id, 'simon' as name +""" + +# returns zero rows -> the test passes +test__passing_601 = """ +{{ config(store_failures=true, store_failures_as='table') }} +select * from {{ ref('chipmunks') }} +where 1 = 2 +""" + + +class TestStoreFailuresPassingKeepsEmptyTable: + @pytest.fixture(scope="class") + def models(self): + return {"chipmunks.sql": model__chipmunks} + + @pytest.fixture(scope="class") + def tests(self): + return {"passing_601.sql": test__passing_601} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "vars": {"dbt_sqlserver_use_default_schema_concat": True}, + "data_tests": {"+schema": TEST_AUDIT_SCHEMA_SUFFIX}, + } + + @pytest.fixture(scope="function", autouse=True) + def setup(self, project): + self.audit_schema = f"{project.test_schema}_{TEST_AUDIT_SCHEMA_SUFFIX}" + run_dbt(["run"]) + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=self.audit_schema + ) + project.adapter.drop_schema(relation) + + def _assert_empty_audit_table(self, project): + # type_desc proves the relation exists AND is a user table (not a view). + # On the #601 bug the relation is dropped, so this returns no rows. + # Queried via sys catalog (lowercase column names) so it is safe under a + # case-sensitive database collation. + rows = project.run_sql( + f""" + select o.type_desc + from sys.objects o + join sys.schemas s on o.schema_id = s.schema_id + where s.name = '{self.audit_schema}' + and o.name = 'passing_601' + """, + fetch="all", + ) + assert len(rows) == 1 and rows[0][0] == "USER_TABLE", ( + f"audit relation [{self.audit_schema}].[passing_601] should be a user " + f"table that persists after a passing store-failures run, got: " + f"{[tuple(r) for r in rows]}" + ) + # and it must be empty (the failures were replaced with nothing) + count = project.run_sql( + f"select count(*) from [{self.audit_schema}].[passing_601]", + fetch="one", + ) + assert count[0] == 0, f"audit table should be empty, has {count[0]} rows" + + def test_passing_test_keeps_empty_audit_table(self, project): + results = run_dbt(["test", "--store-failures"], expect_pass=True) + assert len(results) == 1 + assert results[0].status == "pass" + assert results[0].failures == 0 + self._assert_empty_audit_table(project) + + # idempotency: a second run must still leave the empty table in place + results = run_dbt(["test", "--store-failures"], expect_pass=True) + assert results[0].status == "pass" + self._assert_empty_audit_table(project)