python-checks.yml

name: Python checks

on:
  pull_request:
  push:
    branches: [main]

permissions:
  contents: read

jobs:
  cpu-and-package-layout:
    runs-on: ubuntu-latest
    timeout-minutes: 20

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: pip

      - name: Install CPU test dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install "torch>=2.8,<3" --index-url https://download.pytorch.org/whl/cpu
          python -m pip install pytest wheel numpy "clickhouse-driver>=0.2.0" "pyyaml>=6.0.0"
          # DMI-configurator's [ui] extra. Without fastapi and httpx the API
          # contract suite skips itself rather than failing, so the whole
          # backend would go untested on every PR.
          python -m pip install "fastapi>=0.115" "uvicorn>=0.30" "httpx>=0.28"
          # The native capture suites import botocore at collection (their
          # missing-driver skip is a mark, evaluated after the import), and
          # `--no-deps -e .` would otherwise leave them erroring here.
          python -m pip install "botocore>=1.34"
          # boto3 too, and it is a DIFFERENT dependency: the uploader's
          # head-to-head test drives the Python reference SpoolUploader,
          # which imports boto3. That test skipped here until the compile
          # step below started building the drivers its skipif looks for,
          # so the gap only became visible once the test could run at all.
          # Pinned to the range pyproject's `s3` extra declares; a pip line
          # rather than the extra because `--no-deps -e .` installs none.
          python -m pip install "boto3>=1.40,<2"
          python -m pip install --no-deps -e .

      - name: Fail on an undefined name in src/dmi
        # An undefined name is a NameError that waits for its line to run,
        # and nothing here would see it first: `make check`'s compile stage
        # is `python -m compileall`, which accepts any name, and a branch
        # the tests do not reach never raises. ruff's F821 finds it
        # statically. That one rule only, not a style gate.
        #
        # Enabling it found 14, none of them a NameError today, and each is
        # handled in the source where a reviewer sees it rather than
        # excluded here: config.py's string annotations now import their
        # types under TYPE_CHECKING, and hooks/specs.py's HOOK_TYPE_* names,
        # bound by a globals() loop no linter can follow, carry a per-line
        # noqa saying so. Pinned, so a ruff release cannot move the gate.
        run: |
          python -m pip install "ruff==0.16.5"
          ruff check --select F821 --no-cache --output-format=github src/dmi

      - name: Compile every CPU-only native target
        # The C++ was compiled NOWHERE in CI until this step. `make check`'s
        # compile stage is `python -m compileall` -- Python only -- and the
        # one native build test runs `make -C native -B -n host`, which is a
        # DRY RUN (`-n`) of the CUDA target: it checks the command lines and
        # compiles nothing. So a source file that did not build on this
        # image could sit in the tree indefinitely, and one did: pack_sink.cpp
        # threw std::runtime_error without including <stdexcept>, invisible
        # on a dev box whose libstdc++ pulls it in transitively, and it only
        # surfaced when a later job happened to ask for the sink driver.
        #
        # `cpu-goals` rather than a list of binaries: the set is what a
        # developer can build without CUDA and therefore what CI can check,
        # and a list written here goes stale the moment CPU_ONLY_GOALS
        # gains a target -- which it did, when the catalog driver landed on
        # the Phase B branch and the hardcoded list did not know. Adding a
        # target to CPU_ONLY_GOALS now puts it under the gate with no
        # change here. libcurl dev headers for the store client; the runner
        # ships the runtime .so but not the headers.
        run: |
          sudo apt-get update -q
          sudo apt-get install -y -q libcurl4-openssl-dev
          make -C native cpu-goals \
            CURL_INCDIR=/usr/include/x86_64-linux-gnu \
            CURL_LIBDIR=/usr/lib/x86_64-linux-gnu

      - name: Run compile, CPU, and package-layout checks
        run: make check PYTEST_ARGS="--junit-xml=cpu-results.xml"

      - name: Fail if a cpu test skipped for anything but absent hardware
        if: always()
        # The same reasoning as the live job's gate, arriving late because
        # nothing here built the native binaries until the step above: a
        # test whose skipif looks for a driver that was never compiled
        # SKIPS, and a skipped suite reports green. Measured on this
        # branch: the cpu job was 1125 passed / 58 skipped where a machine
        # with the binaries present runs 1197 -- seventy-two tests that
        # nobody was running and nothing was saying so.
        #
        # Not "zero skips", unlike the live gate: a CPU runner genuinely
        # has no GPU, so the one CUDA-shaped skip is legitimate and stays
        # allowed BY REASON. Every other reason fails and names itself,
        # which is what keeps this from being widened by habit -- a new
        # skip has to be argued for here, not just added.
        run: |
          python - <<'EOF'
          import sys
          import xml.etree.ElementTree as ET

          # A skip is allowed only if its reason says the runner lacks the
          # HARDWARE. Anything else -- an unbuilt driver, a missing
          # dependency, a lost fixture -- is coverage that silently is not
          # there, which is the failure this gate exists for.
          ALLOWED = ("no CUDA device", "requires CUDA", "no GPU")

          try:
              suites = ET.parse("cpu-results.xml").getroot().findall("testsuite")
          except (OSError, ET.ParseError) as exc:
              sys.exit(f"cannot read cpu-results.xml: {exc}. The gate is not "
                       "running, which is indistinguishable from it passing.")
          if not suites:
              sys.exit("cpu-results.xml has no <testsuite>: the gate cannot read it")
          ran = sum(int(s.get("tests", 0)) for s in suites)
          if ran == 0:
              sys.exit("no cpu tests ran: the marker selection matched nothing")

          unexplained = []
          allowed = 0
          for suite in suites:
              for case in suite.iter("testcase"):
                  for mark in case.iter("skipped"):
                      reason = mark.get("message") or ""
                      if any(ok in reason for ok in ALLOWED):
                          allowed += 1
                      else:
                          unexplained.append(f"{case.get('name')}: {reason}")
          print(f"{ran} collected, {allowed} skipped for absent hardware, "
                f"{len(unexplained)} unexplained")
          if unexplained:
              for line in unexplained:
                  print(f"  skipped: {line}")
              sys.exit(f"{len(unexplained)} cpu test(s) skipped for a reason "
                       "that is not absent hardware; a skip is not a pass")
          EOF

      - name: Fail if a live-marked test lives outside tests/*_live.py
        # The live job runs `pytest tests/*_live.py -m "clickhouse and manual
        # and not garage"`, which makes the FILENAME load-bearing: a new test
        # under those markers written as `test_..._e2e.py` is never collected
        # there, and that job's skip gate cannot catch it, because uncollected
        # is not skipped. This step is that missing half -- it fails when a
        # test the live job is meant to run is sitting where the glob cannot
        # see it.
        #
        # It runs HERE, in the cpu job, and not in the live job, because it
        # has to collect the WHOLE tree and only this job's environment can
        # import the whole tree: `make -C native cpu-goals` above builds
        # CPU_ONLY_GOALS, while the live job builds only what its suites
        # drive; when this step was written that was the four conformance
        # drivers, and the tree walk there hit a module-level skip in the two
        # suites that import build/_dmi_native_sink. Doing the tree walk in
        # the live job is precisely the regression this replaces.
        # This job already collects the same tree (`make check` -> `pytest -m
        # cpu`, no path), so the walk costs seconds and adds no new imports.
        #
        # A module that cannot be imported contributes NO node ids and is
        # therefore invisible here -- ignored by construction rather than by a
        # special case on "collection skipped". That is deliberate: an
        # unimportable module in THIS job is already a failure, caught by the
        # skip gate above, so the two checks compose instead of overlapping.
        run: |
          python - <<'EOF'
          import glob
          import os
          import subprocess
          import sys

          MARKERS = "clickhouse and manual and not garage"
          LIVE_GLOB = "tests/*_live.py"

          proc = subprocess.run(
              [sys.executable, "-m", "pytest", "tests", "-m", MARKERS,
               "--collect-only", "-q", "-p", "no:cacheprovider"],
              capture_output=True, text=True,
          )
          # NOT `|| true`. `--collect-only` still exits non-zero on a
          # collection ERROR (exit 2) and on an empty selection (exit 5), and
          # swallowing either turns this gate into a no-op that reports green
          # -- the same silent-green it exists to end. Both are failures, and
          # both name themselves.
          if proc.returncode != 0:
              print(proc.stdout)
              print(proc.stderr, file=sys.stderr)
              sys.exit(f"collecting '{MARKERS}' over tests/ exited "
                       f"{proc.returncode}; this gate cannot run, which is "
                       "indistinguishable from it passing")

          collected = set()
          for line in proc.stdout.splitlines():
              node = line.strip()
              if "::" not in node:
                  continue
              path = node.split("::", 1)[0]
              if path.endswith(".py") and os.path.exists(path):
                  collected.add(os.path.normpath(path))

          # Self-check: an expression that matches nothing would make every
          # comparison below vacuously true.
          if not collected:
              sys.exit(f"'{MARKERS}' selected no tests at all; the live job "
                       "would run nothing and this gate would pass vacuously")

          covered = {os.path.normpath(p) for p in glob.glob(LIVE_GLOB)}
          outside = sorted(collected - covered)
          print(f"{len(collected)} file(s) hold '{MARKERS}' tests; "
                f"{len(collected & covered)} matched by {LIVE_GLOB}")
          if outside:
              for path in outside:
                  print(f"  outside the glob: {path}")
              sys.exit(
                  f"{len(outside)} file(s) hold '{MARKERS}' tests but do not "
                  f"match {LIVE_GLOB}, so the clickhouse-live job never "
                  "collects them and its skip gate cannot see the gap. Rename "
                  "them to *_live.py."
              )
          EOF

  clickhouse-live:
    # The `manual`-marked suites, which `make check` cannot run: `addopts` in
    # pyproject.toml carries `-m 'not manual'`, so every test that needs a real
    # server was deselected everywhere, CI included. That left the catalog's
    # load-bearing regression -- tests/test_clickhouse_schema_migration_live.py,
    # which holds `main`'s DDL verbatim and the measured `skipped=2 indexed=0`
    # data-loss path -- passing on nobody's machine on any schedule. A CLI `-m`
    # overrides the one in `addopts`, which is what selects them back in here.
    runs-on: ubuntu-latest
    timeout-minutes: 30

    services:
      clickhouse:
        # Pinned, and pinned to the version the suites were written against:
        # tests/_catalog_fakes.py records UUID collation as "verified on 25.12",
        # and that ordering is what the lease fence resolves a contested head
        # with. A floating tag would move that out from under the fakes.
        image: clickhouse/clickhouse-server:25.12
        env:
          # Without this the image's entrypoint writes a users.d override
          # restricting `default` to ::1 and 127.0.0.1 -- it says so in the
          # container log: "neither CLICKHOUSE_USER nor CLICKHOUSE_PASSWORD is
          # set, disabling network access for user 'default'". The health check
          # below still passes, because clickhouse-client runs INSIDE the
          # container and so connects from localhost, while the suites connect
          # from the runner host and got `Code: 516, Authentication failed` on
          # all 61 tests. Skipping the entrypoint's user setup leaves `default`
          # as the stock config has it: no password, no network restriction.
          # The suites take only host/port/database from the environment and
          # have no way to present credentials, so opening `default` up is the
          # fix rather than creating a user.
          CLICKHOUSE_SKIP_USER_SETUP: "1"
        ports:
          - 9000:9000
          # The native catalog driver (B1 lease/allocator port) speaks
          # ClickHouse's HTTP interface, not the native TCP protocol
          # clickhouse-driver uses.
          - 8123:8123
        options: >-
          --health-cmd "clickhouse-client --query 'SELECT 1'"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 12

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
          # Not the default depth 1. The migration suite extracts its
          # predecessor's catalog writer with `git show bea3ed8:...` rather than
          # transcribing it -- a hand-copied fixture would drift towards this
          # branch and prove nothing -- and on a shallow clone that commit is
          # absent, so the test calls pytest.skip. Verified: in a depth-1 clone
          # the `git show` fails. Depth 1 would therefore reproduce the exact
          # bug this job exists to catch, reporting green while the regression
          # never ran.
          fetch-depth: 0

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: pip

      - name: Install CPU test dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install "torch>=2.8,<3" --index-url https://download.pytorch.org/whl/cpu
          python -m pip install pytest wheel numpy "clickhouse-driver>=0.2.0" "pyyaml>=6.0.0"
          # The configurator pipeline imports yaml/fastapi at module level,
          # and its suites must not silently skip here either.
          python -m pip install "fastapi>=0.115" "uvicorn>=0.30" "httpx>=0.28"
          # Same reason as the cpu job above, and it reaches this job through
          # the ported catalog suite: it imports the fake-S3 fixture at MODULE
          # level (a fixture imported inside a function does not register), and
          # that module imports botocore at import time. Missing, it is not the
          # S3-backed tests that skip -- collection of the whole module errors
          # and the live job dies at exit code 2 with every other ported test
          # uncollected.
          python -m pip install "botocore>=1.34"
          # boto3 as well, and note this is NOT the same dependency: the
          # reader-parity suite compares the native reader against the
          # PYTHON one, and the Python side hydrates through S3PackStore,
          # which imports boto3 and raises "S3 support requires the optional
          # 's3' dependencies" without it. The pip line rather than the `s3`
          # extra because `--no-deps -e .` below installs no extras. Pinned
          # to the range pyproject's `s3` extra declares.
          python -m pip install "boto3>=1.40,<2"
          python -m pip install --no-deps -e .

      - name: Build the native drivers the live suites drive
        # The ported lease/allocator suites drive the native coordinator
        # over ClickHouse's HTTP interface. The runner ships the runtime
        # libcurl.so.4 but not the dev headers; install those and build
        # against the system paths.
        #
        # ALL FOUR CPU-only conformance drivers, not just the catalog one:
        # the ported catalog suite reaches the others through its indexer
        # tests, which stage packs through the sink, recover the staged
        # entry through the spool driver, upload through the store driver
        # to the in-process fake S3, and then index what came back. A
        # missing binary there is not a skip -- it is FileNotFoundError at
        # Popen -- and the spool one is reached by a path built from
        # STORE_DRIVER.parent rather than a module constant, so it does not
        # show up by grepping for driver names. Build the set; the marginal
        # cost is seconds and it ends a class of failure that was found one
        # binary per CI round.
        #
        # And _dmi_native_store, the storage service and reader that
        # test_native_capture_storage_live.py drives in-process. It takes
        # only pybind11's headers from the torch this job installs and links
        # nothing from it.
        #
        # And _dmi_native_sink, the torch-facing NativePackSink the ring
        # hands envelopes to. Every other suite here stages packs through
        # the conformance_sink DRIVER, JSON and base64 in, so the real
        # adapter never reached a catalog in CI;
        # test_native_capture_chain_live.py drives it with torch CPU tensors
        # through the service into ClickHouse and reads the bytes back. It
        # links torch_cpu from the wheel this job installs (C++20, which the
        # Makefile already passes for this one target).
        run: |
          sudo apt-get update -q
          sudo apt-get install -y -q libcurl4-openssl-dev
          make -C native \
            build/conformance_catalog \
            build/conformance_store \
            build/conformance_sink \
            build/conformance_spool \
            build/_dmi_native_store \
            build/_dmi_native_sink \
            CURL_INCDIR=/usr/include/x86_64-linux-gnu \
            CURL_LIBDIR=/usr/lib/x86_64-linux-gnu

      - name: Run the ClickHouse live suites
        env:
          DMI_CLICKHOUSE_HOST: 127.0.0.1
          DMI_CLICKHOUSE_PORT: "9000"
        # `python -m pytest`, matching the Makefile. `not garage` drops the one
        # suite that also needs a Garage S3 endpoint; everything else under
        # these markers needs nothing but the server above.
        #
        # The GLOB, not the whole `tests` tree. Walking the tree here makes
        # pytest IMPORT every test module, including ones this job never
        # builds for: it builds the four conformance drivers above, NOT
        # build/_dmi_native_sink, and tests/test_native_adapter_torch.py and
        # tests/test_native_rollback.py take a module-level `pytest.skip`
        # when that .so is absent (they import it at module scope, so they
        # cannot convert to a per-test skipif -- tests/test_ci_guard_skips.py
        # records exactly those two as the permanent exceptions). A
        # collection skip lands in the JUnit as `<skipped/>`, and the gate
        # below then fails the job -- correctly, by its own doctrine -- over
        # two files this suite does not even want. Measured in CI: `201
        # collected, 2 skipped`. (This job has since started building the
        # sink, for the capture-chain suite, so those two import here now.
        # The glob stays: the argument is about any module that imports an
        # artifact this job does not build, not about those two.)
        #
        # So the RUN stays scoped to the files it needs to import. The hole
        # a glob leaves -- a `manual`/`clickhouse` test named outside the
        # convention is silently UNCOLLECTED, and this gate cannot catch it
        # because uncollected is not skipped -- is closed instead by the
        # "Fail if a live-marked test lives outside tests/*_live.py" step in
        # the cpu job, which collects the whole tree in an environment that
        # can actually import it.
        run: |
          python -m pytest tests/*_live.py -m "clickhouse and manual and not garage" \
            -q --junit-xml=live-results.xml

      - name: Fail if any live test was skipped
        if: always()
        # A skip here is indistinguishable from coverage that does not exist,
        # which is the whole finding this job answers: an unreachable server
        # raises, but a missing `clickhouse_driver` or a lost fixture commit
        # skips, and a skipped suite reports green. So a skip is an error, and
        # names itself.
        run: |
          python - <<'EOF'
          import sys
          import xml.etree.ElementTree as ET

          # EVERY testsuite, not the first. pytest writes one today, but a
          # plugin (xdist among them) can split the run across several, and
          # reading only the first would report a subset as the whole -- which
          # is the failure mode this gate exists to prevent. An absent node
          # used to crash here with an AttributeError traceback; it is now a
          # named failure, because a JUnit file this script cannot read is a
          # gate that is not running.
          suites = ET.parse("live-results.xml").getroot().findall("testsuite")
          if not suites:
              sys.exit("live-results.xml has no <testsuite>: the gate cannot read it")
          ran = sum(int(s.get("tests", 0)) for s in suites)
          skipped = sum(int(s.get("skipped", 0)) for s in suites)
          print(f"{ran} collected, {skipped} skipped, across {len(suites)} suite(s)")
          if ran == 0:
              sys.exit("no live tests ran: the marker selection matched nothing")
          if skipped:
              for suite in suites:
                  for case in suite.iter("testcase"):
                      for mark in case.iter("skipped"):
                          print(f"  skipped: {case.get('name')}: {mark.get('message')}")
              sys.exit(f"{skipped} live test(s) skipped; a skip is not a pass")
          EOF

  native-backend-compile:
    # The full ring backend, _native_backend, which nothing in CI compiled:
    # the cpu job builds CPU_ONLY_GOALS, and the one build test there
    # (tests/test_cpu_native_build.py) dry-runs `host`. So the .cu sources,
    # bindings.cpp and the torch/CUDA link could break on any PR and stay
    # broken until someone built on a GPU host -- which is how the C++20
    # floor torch 2.14 imposes went unnoticed (#138). This job compiles and
    # links `make -C native all`, imports the result, checks that the capture
    # sink built beside it derives from its RecordSink, and builds the ring
    # test binaries, running the two that need no device.
    #
    # No GPU is needed to COMPILE, only the toolkit, and this runner has
    # none: everything that needs a device stays with the GPU job (D0).
    #
    # On the runner, not in an nvidia/cuda:*-devel container. A container
    # job pulls its image before the first step runs, so nothing can free
    # disk first, and the image is 3.68 GiB COMPRESSED
    # (nvidia/cuda:13.0.1-devel-ubuntu24.04, amd64, per its registry
    # manifest) -- several times that unpacked -- before the ~4.4 GB that
    # torch 2.14+cu130 and its nvidia-* wheels take installed (measured in a
    # cu130 venv). The free disk on a hosted runner is the image's to
    # decide, not this repository's, so the job keeps its footprint small
    # rather than depend on it. Installing only the packages the build
    # reads is ~2.7 GB:
    # the header set comes from the -MMD dependency files of a real build,
    # mapped to packages with dpkg -S on a host whose toolkit came from the
    # same apt repository, and the sizes are those packages' Installed-Size.
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    env:
      # Explicit, so the Makefile's resolver takes this toolkit rather than
      # searching; it still checks the major against torch.version.cuda.
      CUDA_HOME: /usr/local/cuda-13.0
      # Explicit because `native`, the default, asks the GPU this runner does
      # not have. Ada, the architecture the GPU hosts build for.
      SM_ARCH: sm_89

    steps:
      - name: Free runner disk
        # A precaution, not a measured need. On run 35933626430 of this job
        # the runner had 87 GB free before this step, 103 GB after it, and
        # still 94 GB once the CUDA toolkit and torch were installed, so the
        # job fits without it today. It stays because that figure belongs to
        # the runner image and can shrink between runs, and it removes only
        # the two largest preinstalled trees this job never uses (the Android
        # SDK and .NET). It cost about two minutes on that run. `df` either
        # side, so a disk failure can be read from the log.
        run: |
          df -h /
          sudo rm -rf /usr/local/lib/android /usr/share/dotnet
          df -h /

      - uses: actions/checkout@v4

      - name: Fetch the clickhouse-cpp submodule
        # Only this one. `make all` links clickhouse-cpp and needs none of the
        # framework submodules, and `recursive` would also clone
        # DMI-Megatron-Integration's nested Megatron-LM onto a disk this job
        # is rationing. clickhouse-cpp has no submodules of its own.
        run: git submodule update --init third_party/clickhouse-cpp

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          # No pip cache: the torch CUDA wheel set is gigabytes, and caching
          # it would spend the repository's cache quota to save a download.

      - name: Install the CUDA 13.0 compiler and the headers torch includes
        # NVIDIA's apt repository, and only what the build reads: nvcc
        # (pulling cccl, crt and nvvm), cudart-dev (pulling driver-dev, whose
        # stubs/libcuda.so satisfies the backend's -lcuda with no driver
        # installed), and cublas, cusparse and cusolver for their headers,
        # which ATen's CUDA context includes. 13.0 because torch 2.14's CUDA
        # build here is cu130 and the resolver requires the same major.
        run: |
          curl -fsSLO https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
          sudo dpkg -i cuda-keyring_1.1-1_all.deb
          sudo apt-get update -q
          sudo apt-get install -y -q --no-install-recommends \
            cuda-nvcc-13-0 cuda-cudart-dev-13-0 \
            libcublas-dev-13-0 libcusparse-dev-13-0 libcusolver-dev-13-0
          "$CUDA_HOME/bin/nvcc" --version
          df -h /

      - name: Install the CUDA build of torch
        # 2.14, the release the GPU hosts run (the plan's pin), from the cu130
        # index. A CUDA build is required, not the CPU wheel the other jobs
        # use: the Makefile reads torch.version.cuda to pick the toolkit and
        # links libtorch_cuda and libc10_cuda.
        run: |
          python -m pip install --upgrade pip
          python -m pip install --no-cache-dir "torch==2.14.*" \
            --index-url https://download.pytorch.org/whl/cu130
          python -c "import torch; print(torch.__version__, torch.version.cuda)"
          df -h /

      - name: Build the ClickHouse C++ client
        # docs/install.md step 5, verbatim.
        run: |
          cmake -S third_party/clickhouse-cpp -B third_party/clickhouse-cpp/build \
                -DCMAKE_BUILD_TYPE=Release \
                -DCMAKE_POSITION_INDEPENDENT_CODE=ON
          cmake --build third_party/clickhouse-cpp/build -j"$(nproc)"

      - name: Compile the full native build
        # `all` is the documented build: the backend plus the capture
        # extensions (#144), and the store needs libcurl's headers. With the
        # dev package installed, the Makefile finds it through pkg-config --
        # its default lookup, so no CURL_* override here: this job builds
        # exactly what docs/install.md tells a user to run.
        #
        # LIBRARY_PATH puts the toolkit's libcuda stub where the linker finds
        # -lcuda; the result does not end up NEEDing libcuda (checked with
        # readelf on a local build: the link is --as-needed and nothing calls
        # the driver API), so check-link's ldd has no driver to find either.
        env:
          LIBRARY_PATH: /usr/local/cuda-13.0/targets/x86_64-linux/lib/stubs
        run: |
          sudo apt-get install -y -q libcurl4-openssl-dev pkg-config
          make -C native -j"$(nproc)" all PYTHON=python
          ls -l src/dmi/_native_backend*.so src/dmi/_dmi_native_sink*.so src/dmi/_dmi_native_store*.so

      - name: Import the backend
        # A shared library links with undefined symbols; only loading it
        # proves the link. No device is needed to load it.
        run: |
          PYTHONPATH=src python -c "
          from dmi.transport import native
          module = native._load_extension()
          print(module.__file__)
          assert hasattr(module, 'RecordSink'), 'no RecordSink in the backend'
          "

      - name: Build the capture sink beside the backend
        # _dmi_native_sink decides its RecordSink base once, when it loads:
        # it derives NativePackSink from _native_backend's registration when
        # that backend is loadable, and registers module-local stand-ins of
        # its own when it is not. Only the first is attachable --
        # create_record_runtime checks isinstance against the BACKEND's
        # RecordSink -- yet the cpu and live jobs build the sink with no
        # backend, so the stand-in branch was the only one CI ever took.
        # This is the one job that has a backend to bind to. `all` above has
        # already built it; the step stays so the dependency is explicit.
        run: make -C native build/_dmi_native_sink PYTHON=python

      - name: Check the sink binds the backend's RecordSink
        # Two import orders, one process each, because the binding is fixed
        # for the process by whichever order ran. Through dmi's loaders, the
        # order the engine uses: backend first, then the sink. And the bare
        # `import _dmi_native_sink` the adapter suite does, where the module
        # has to find the backend itself through dmi.transport.native. Both
        # assert the stand-ins were NOT taken, which is what the pytest run
        # after them cannot: its test passes on either branch by design, so
        # on its own it would stay green with the binding broken. It still
        # runs, because here it takes the test's backend branch, which no
        # other job reaches; the grep refuses a skip as a pass.
        run: |
          PYTHONPATH=src python -c "
          from dmi.storage.capture.native_sink import _load_native_sink_extension
          from dmi.transport import native
          backend = native._load_extension()
          sink = _load_native_sink_extension()
          assert not sink.RING_TYPES_ARE_STANDINS, 'the sink bound stand-ins'
          assert issubclass(sink.NativePackSink, backend.RecordSink), sink.NativePackSink.__mro__
          assert not hasattr(sink, 'RecordSink'), 'the sink registered a second RecordSink'
          print('loaders:', sink.NativePackSink.__mro__)
          "
          PYTHONPATH=src:native/build python -c "
          import torch  # first, as the suite does: the .so has no rpath to libtorch_cpu
          import _dmi_native_sink as sink
          from dmi.transport import native
          assert not sink.RING_TYPES_ARE_STANDINS, 'the sink bound stand-ins'
          assert issubclass(sink.NativePackSink, native._load_extension().RecordSink), sink.NativePackSink.__mro__
          print('bare import:', sink.NativePackSink.__mro__)
          "
          python -m pip install pytest
          python -m pytest -q -p no:cacheprovider \
            "tests/test_native_adapter_torch.py::test_the_sink_derives_from_the_engines_record_sink" \
            > sink-test.out || { cat sink-test.out; exit 1; }
          cat sink-test.out
          # "1 passed in ..." or "1 passed, N warnings in ..." -- a warning
          # (torch's NumPy one appears here) is not a failure; a skip is.
          grep -Eq '^1 passed(,| in )' sink-test.out
          ! grep -Eq '[0-9]+ skipped' sink-test.out

      - name: Build the ring tests and run the host-only ones
        # All six binaries are compiled, so a .cu test that no longer builds
        # fails here. Only test_record_consumer and
        # test_clickhouse_record_sink RUN: they use ATen CPU tensors and no
        # CUDA call, and pass with no device visible (14 and 21 checks,
        # measured with CUDA_VISIBLE_DEVICES empty). The other four launch
        # kernels and belong to the GPU job. CXX_STD because this Makefile's
        # default is c++17, which torch 2.14's headers refuse.
        run: |
          make -C tests/native/ring -j"$(nproc)" all CXX_STD=c++20 PYTHON=python
          tests/native/ring/build/test_record_consumer
          tests/native/ring/build/test_clickhouse_record_sink