Release 0.7.4.1: SQLAlchemy 2 result correctness fixes and wheel publisher

Jenkinsfile

@@ -0,0 +1,112 @@
+// Fork publisher. Pull-request wheels are versioned <version>+pr.<number>.<sha>
+// (PEP 440-normalized by ci/release_version.py); stable wheels are published only
+// from reviewed master. A published artifact is never overwritten.
+podTemplate(
+    imagePullSecrets: ['preset-pull'],
+    containers: [
+        containerTemplate(name: 'ci', image: 'preset/ci:latest',
+            ttyEnabled: true, command: 'cat'),
+        containerTemplate(name: 'py-ci', image: 'preset/python:3.9.18-2024-02-21-ci',
+            ttyEnabled: true, command: 'cat'),
+        // Disposable server for the test suite; reachable on localhost inside the pod.
+        containerTemplate(name: 'mongo', image: 'mongo:8.0',
+            envVars: [
+                envVar(key: 'MONGO_INITDB_ROOT_USERNAME', value: 'admin'),
+                envVar(key: 'MONGO_INITDB_ROOT_PASSWORD', value: 'secret'),
+            ])
+    ]
+) {
+    node(POD_LABEL) {
+        checkout scm
+        def revision = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
+        boolean isMaster = env.BRANCH_NAME == 'master'
+        boolean isPR = env.CHANGE_ID != null
+        if (!isMaster && !isPR) {
+            error('Only master and pull-request builds publish; use a PR.')
+        }
+
+        container('py-ci') {
+            stage('Test and build') {
+                def args = isMaster ? '' : "${env.CHANGE_ID} ${revision.take(12)}"
+                sh '''
+                    set -eu
+                    python -m venv .venv
+                    .venv/bin/pip install 'sqlalchemy==2.0.52' 'pymongo==4.17.0' \
+                        'antlr4-python3-runtime==4.13.2' 'jmespath==1.1.0' 'pandas>=2.2,<3' \
+                        'tenacity==9.1.2' 'pytest==8.3.5' 'boto3>=1.36,<2' 'packaging==25.0' \
+                        'build==1.4.4' 'setuptools==80.9.0' 'setuptools_scm==8.3.1' 'wheel==0.45.1'
+                '''
+                def version = sh(script: ".venv/bin/python ci/release_version.py ${args}",
+                    returnStdout: true).trim()
+                env.PUBLISH_VERSION = version
+                env.WHEEL = "pymongosql-${version}-py3-none-any.whl"
+                env.KEY = "pymongosql/${env.WHEEL}"
+                sh '''
+                    set -eu
+                    # The checkout is owned by another uid; setuptools_scm runs git during builds.
+                    # The pod is ephemeral, so its global git config is disposable.
+                    git --version
+                    git config --global --add safe.directory "$PWD"
+                    .venv/bin/pip install --no-deps -e .
+                    for attempt in $(seq 1 30); do
+                        .venv/bin/python -c "import pymongo; pymongo.MongoClient('mongodb://admin:secret@localhost:27017', serverSelectionTimeoutMS=2000).admin.command('ping')" && break
+                        sleep 2
+                    done
+                    .venv/bin/python tests/run_test_server.py setup
+                    .venv/bin/python -m pytest -q tests
+                    .venv/bin/pip uninstall -y pymongosql
+                    python - <<'PY'
+import os
+import re
+from pathlib import Path
+path = Path('pymongosql/__init__.py')
+source = path.read_text()
+pattern = re.compile(r'^__version__: str = "[^"]+"$', re.MULTILINE)
+assert len(pattern.findall(source)) == 1
+path.write_text(pattern.sub('__version__: str = "' + os.environ['PUBLISH_VERSION'] + '"', source))
+PY
+                    SOURCE_DATE_EPOCH=$(git -c safe.directory="$PWD" log -1 --format=%ct)
+                    case "$SOURCE_DATE_EPOCH" in
+                        ''|*[!0-9]*) echo "Invalid commit timestamp for reproducible build" >&2; exit 1 ;;
+                    esac
+                    export SOURCE_DATE_EPOCH
+                    # Pin the build backend and remove stale output for reproducible retries.
+                    rm -rf build dist pymongosql.egg-info
+                    .venv/bin/python -m build --wheel --no-isolation
+                    test -f "dist/$WHEEL" || { echo "missing dist/$WHEEL"; ls -1 dist; exit 1; }
+                    .venv/bin/python -c "import os, sys, zipfile; names = zipfile.ZipFile('dist/' + os.environ['WHEEL']).namelist(); sys.exit('wheel ships tests or ci' if any(n.startswith(('tests/', 'ci/')) for n in names) else 0)"
+                    .venv/bin/pip install --force-reinstall --no-deps "dist/$WHEEL"
+                    .venv/bin/python - <<'PY'
+import importlib.metadata as im
+import os
+import sqlalchemy as sa
+assert im.version('pymongosql') == os.environ['PUBLISH_VERSION']
+engine = sa.create_engine('mongodb://user@localhost/db')
+assert engine.dialect.name == 'mongodb'
+engine.dispose()
+PY
+                    sha256sum "dist/$WHEEL"
+                '''
+            }
+        }
+        container('ci') {
+            stage('Publish immutable wheel') {
+                withCredentials([[
+                    $class: 'AmazonWebServicesCredentialsBinding',
+                    credentialsId: 'ci-user',
+                    accessKeyVariable: 'AWS_ACCESS_KEY_ID',
+                    secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
+                ]]) {
+                    withEnv(["ALLOW_IDENTICAL_PR_ARTIFACT=${isPR && !isMaster}"]) {
+                        sh '''
+                            set -eu
+                            python -m pip install --quiet 'boto3>=1.36,<2'
+                            python ci/publish_wheel.py
+                        '''
+                    }
+                }
+            }
+        }
+        archiveArtifacts artifacts: 'dist/*.whl,published.sha256', fingerprint: true
+    }
+}

ci/publish_wheel.py

@@ -0,0 +1,71 @@
+"""Publish an immutable wheel; retries may reuse identical archive content."""
+
+import hashlib
+import io
+import os
+from pathlib import Path
+from zipfile import BadZipFile, ZipFile
+
+import boto3
+from botocore.exceptions import ClientError
+
+
+def same_wheel_content(stored, fresh):
+    """Compare sorted member names and bytes, ignoring ZIP metadata such as timestamps."""
+    if stored == fresh:
+        return True
+    try:
+        with ZipFile(io.BytesIO(stored)) as old, ZipFile(io.BytesIO(fresh)) as new:
+            old_members = sorted(old.infolist(), key=lambda member: member.filename)
+            new_members = sorted(new.infolist(), key=lambda member: member.filename)
+            return [member.filename for member in old_members] == [member.filename for member in new_members] and all(
+                old.read(a) == new.read(b) for a, b in zip(old_members, new_members)
+            )
+    except BadZipFile:
+        return False
+
+
+def publish_wheel(s3, bucket, key, body, is_pr=False):
+    try:
+        s3.put_object(Bucket=bucket, Key=key, Body=body, IfNoneMatch="*")
+    except ClientError as error:
+        if error.response["Error"]["Code"] != "PreconditionFailed":
+            raise
+        print("Artifact already exists; verifying archive content without overwriting.")
+
+    response = s3.get_object(Bucket=bucket, Key=key)
+    try:
+        stored = response["Body"].read()
+    finally:
+        response["Body"].close()
+    if not same_wheel_content(stored, body):
+        raise RuntimeError(
+            "Stored wheel differs from this commit's freshly built artifact; "
+            "refusing to overwrite or accept it (local sha256={}, stored sha256={}).".format(
+                hashlib.sha256(body).hexdigest(),
+                hashlib.sha256(stored).hexdigest(),
+            )
+            + ("" if is_pr else " Bump __version__ before publishing different content.")
+        )
+    print("Published wheel verified by archive content: " + key)
+    return hashlib.sha256(stored).hexdigest()
+
+
+def main():
+    wheel = os.environ["WHEEL"]
+    receipt = Path("published.sha256")
+    # Do not leave a previous build's success receipt after a failed retry.
+    if receipt.exists():
+        receipt.unlink()
+    digest = publish_wheel(
+        boto3.client("s3"),
+        "preset-pypi",
+        os.environ["KEY"],
+        Path("dist", wheel).read_bytes(),
+        is_pr=os.environ.get("ALLOW_IDENTICAL_PR_ARTIFACT") == "true",
+    )
+    receipt.write_text(digest + "  " + wheel + "\n")
+
+
+if __name__ == "__main__":
+    main()

ci/release_version.py

@@ -0,0 +1,39 @@
+"""Compute the published version of this fork, normalized as the wheel will be.
+
+Stable builds publish the declared ``__version__`` (a four-part Preset release
+such as 0.7.4.1). Pull-request builds publish ``<version>+pr.<change>.<sha>``.
+PEP 440 normalizes local segments (lower case; a numeric segment loses leading
+zeros), so the filename must be derived from the normalized form or it will not
+match the file the build backend writes.
+"""
+
+import re
+import sys
+from pathlib import Path
+
+from packaging.version import Version
+
+INIT = Path(__file__).resolve().parents[1] / "pymongosql" / "__init__.py"
+
+
+def declared_version(source=None):
+    source = INIT.read_text() if source is None else source
+    matches = re.findall(r'^__version__: str = "([^"]+)"$', source, re.MULTILINE)
+    if len(matches) != 1:
+        raise SystemExit("expected exactly one __version__ declaration")
+    version = Version(matches[0])
+    if str(version) != matches[0] or version.local or len(version.release) != 4:
+        raise SystemExit(f"__version__ must be a normalized four-part release, got {matches[0]!r}")
+    return matches[0]
+
+
+def release_version(base, change_id=None, revision=None):
+    if change_id is None:
+        return str(Version(base))
+    if not re.fullmatch(r"[0-9]+", change_id) or not re.fullmatch(r"[0-9a-fA-F]{7,40}", revision or ""):
+        raise SystemExit("pull-request builds need a numeric change id and a git revision")
+    return str(Version(f"{base}+pr.{change_id}.{revision}"))
+
+
+if __name__ == "__main__":
+    print(release_version(declared_version(), *sys.argv[1:]))

pymongosql/__init__.py

@@ -6,7 +6,7 @@
 if TYPE_CHECKING:
     from .connection import Connection
 
-__version__: str = "0.7.4"
+__version__: str = "0.7.4.1"
 
 # Globals https://www.python.org/dev/peps/pep-0249/#globals
 apilevel: str = "2.0"

pymongosql/executor.py

@@ -232,6 +232,10 @@ def _execute_aggregate_plan(
             _logger.debug(f"Pipeline: {pipeline}")
             _logger.debug(f"Options: {options}")
 
+            # A pipeline generated from SQL carries the WHERE clause's ? placeholders
+            if parameters and execution_plan.aggregate_parameterized:
+                pipeline = self._replace_placeholders(pipeline, parameters)
+
             # Get collection and call aggregate()
             collection = db[execution_plan.collection]
 

pymongosql/helper.py

@@ -6,9 +6,12 @@
 """
 
 import logging
+from decimal import Decimal
 from typing import Any, Optional, Sequence, Tuple
 from urllib.parse import parse_qs, urlparse
 
+from bson import Decimal128
+
 from .error import ProgrammingError
 
 _logger = logging.getLogger(__name__)
@@ -102,6 +105,20 @@ def parse_connection_string(connection_string: Optional[str]) -> Tuple[Optional[
 class SQLHelper:
     """SQL-related helper utilities."""
 
+    @staticmethod
+    def to_bson_value(value: Any) -> Any:
+        """Convert a bound parameter to a type BSON can encode.
+
+        ``decimal.Decimal`` has no BSON encoding; ``Decimal128`` stores it exactly.
+        """
+        if isinstance(value, Decimal):
+            return Decimal128(value)
+        if isinstance(value, dict):
+            return {k: SQLHelper.to_bson_value(v) for k, v in value.items()}
+        if isinstance(value, (list, tuple)):
+            return [SQLHelper.to_bson_value(v) for v in value]
+        return value
+
     @staticmethod
     def replace_placeholders_generic(value: Any, parameters: Any, style: Optional[str]) -> Any:
         """Recursively replace placeholders in nested structures for qmark or named styles."""
@@ -120,7 +137,7 @@ def replace(val: Any) -> Any:
                         raise ProgrammingError("Not enough parameters provided")
                     out = parameters[idx[0]]
                     idx[0] += 1
-                    return out
+                    return SQLHelper.to_bson_value(out)
                 if isinstance(val, dict):
                     return {k: replace(v) for k, v in val.items()}
                 if isinstance(val, list):
@@ -138,7 +155,7 @@ def replace(val: Any) -> Any:
                     key = val[1:]
                     if key not in parameters:
                         raise ProgrammingError(f"Missing named parameter: {key}")
-                    return parameters[key]
+                    return SQLHelper.to_bson_value(parameters[key])
                 if isinstance(val, dict):
                     return {k: replace(v) for k, v in val.items()}
                 if isinstance(val, list):

pymongosql/sql/ast.py

@@ -4,7 +4,7 @@
 
 from ..error import SqlSyntaxError
 from .delete_handler import DeleteParseResult
-from .handler import BaseHandler, HandlerFactory
+from .handler import BaseHandler, ContextUtilsMixin, HandlerFactory
 from .insert_handler import InsertParseResult
 from .partiql.PartiQLLexer import PartiQLLexer
 from .partiql.PartiQLParser import PartiQLParser
@@ -275,6 +275,7 @@ def visitOrderByClause(self, ctx: PartiQLParser.OrderByClauseContext) -> Any:
             if hasattr(ctx, "orderSortSpec") and ctx.orderSortSpec():
                 for sort_spec in ctx.orderSortSpec():
                     field_name = sort_spec.expr().getText() if sort_spec.expr() else "_id"
+                    field_name = ContextUtilsMixin.normalize_field_path(field_name)
                     # Check for ASC/DESC (default is ASC = 1)
                     direction = 1  # ASC
                     if hasattr(sort_spec, "DESC") and sort_spec.DESC():
@@ -289,6 +290,23 @@ def visitOrderByClause(self, ctx: PartiQLParser.OrderByClauseContext) -> Any:
             _logger.warning(f"Error processing ORDER BY clause: {e}")
             return self.visitChildren(ctx)
 
+    def visitGroupClause(self, ctx: PartiQLParser.GroupClauseContext) -> Any:
+        """Handle GROUP BY keys; they become the _id of a $group stage."""
+        keys = []
+        for key in ctx.groupKey() or []:
+            if key.symbolPrimitive() is not None:
+                self._query_parse_result.unsupported_clauses.append("GROUP BY key alias")
+            keys.append(ContextUtilsMixin.normalize_field_path(key.exprSelect().getText()))
+        if ctx.PARTIAL() is not None:
+            self._query_parse_result.unsupported_clauses.append("GROUP PARTIAL BY")
+        self._query_parse_result.group_by = keys
+        return None
+
+    def visitHavingClause(self, ctx: PartiQLParser.HavingClauseContext) -> Any:
+        """HAVING is not translated; record it so the query fails instead of ignoring it."""
+        self._query_parse_result.unsupported_clauses.append("HAVING")
+        return None
+
     def visitLimitClause(self, ctx: PartiQLParser.LimitClauseContext) -> Any:
         """Handle LIMIT clause for result limiting"""
         _logger.debug("Processing LIMIT clause")