Add validation + compatibility tests for DeepLabCut-live

.github/workflows/testing-ci.yml

@@ -54,8 +54,9 @@ jobs:
             libxcb-cursor0
 
       - name: Run tests (exclude hardware) with coverage via tox
+        shell: bash -eo pipefail {0}
         run: |
-          tox -q | tee tox-output.log
+          tox -q 2>&1 | tee tox-output.log
 
 
       - name: Append Coverage Summary to Job
@@ -78,3 +79,43 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./.coverage.py312.xml
           fail_ci_if_error: false
+
+  dlclive-compat:
+    name: DLCLive Compatibility • ${{ matrix.label }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: pypi-1.1
+            tox_env: dlclive-pypi
+          - label: github-main
+            tox_env: dlclive-github
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+
+      - name: Install Qt/OpenGL runtime deps (Ubuntu)
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            libegl1 \
+            libgl1 \
+            libopengl0 \
+            libxkbcommon-x11-0 \
+            libxcb-cursor0
+
+      - name: Install tox
+        run: |
+          python -m pip install -U pip wheel
+          python -m pip install -U tox tox-gh-actions
+
+      - name: Run DLCLive compatibility tests via tox
+        shell: bash -eo pipefail {0}
+        run: |
+          tox -e ${{ matrix.tox_env }} -q

dlclivegui/gui/main_window.py

@@ -196,14 +196,17 @@ def __init__(self, config: ApplicationSettings | None = None):
 
         # Validate cameras from loaded config (deferred to allow window to show first)
         # NOTE IMPORTANT (tests/CI): This is scheduled via a QTimer and may fire during pytest-qt teardown.
-        QTimer.singleShot(100, self._validate_configured_cameras)
+        # NOTE @C-Achard 2026-03-02: Handling this in closeEvent should help
+        self._camera_validation_timer = QTimer(self)
+        self._camera_validation_timer.setSingleShot(True)
+        self._camera_validation_timer.timeout.connect(self._validate_configured_cameras)
+        self._camera_validation_timer.start(100)
         # If validation triggers a modal QMessageBox (warning/error) while the parent window is closing,
         # it can cause errors with unpredictable timing (heap corruption / access violations).
         #
         # Mitigations for tests/CI:
         #   - Disable this timer by monkeypatching _validate_configured_cameras in GUI tests
         #   - OR monkeypatch/override _show_warning/_show_error to no-op in GUI tests (easiest)
-        #   - OR use a cancellable QTimer attribute and stop() it in closeEvent
 
     def resizeEvent(self, event):
         super().resizeEvent(event)
@@ -2023,6 +2026,8 @@ def closeEvent(self, event: QCloseEvent) -> None:  # pragma: no cover - GUI beha
         if self.multi_camera_controller.is_running():
             self.multi_camera_controller.stop(wait=True)
 
+        if hasattr(self, "_camera_validation_timer") and self._camera_validation_timer.isActive():
+            self._camera_validation_timer.stop()
         # Stop all multi-camera recorders
         self._rec_manager.stop_all()
 

dlclivegui/services/dlc_processor.py

@@ -10,12 +10,13 @@
 from collections import deque
 from contextlib import contextmanager
 from dataclasses import dataclass
+from enum import Enum, auto
 from typing import Any
 
 import numpy as np
 from PySide6.QtCore import QObject, Signal
 
-from dlclivegui.config import DLCProcessorSettings
+from dlclivegui.config import DLCProcessorSettings, ModelType
 from dlclivegui.processors.processor_utils import instantiate_from_scan
 from dlclivegui.temp import Engine  # type: ignore # TODO use main package enum when released
 
@@ -33,10 +34,74 @@
     DLCLive = None  # type: ignore[assignment]
 
 
+class PoseBackends(Enum):
+    DLC_LIVE = auto()
+
+
 @dataclass
 class PoseResult:
     pose: np.ndarray | None
     timestamp: float
+    packet: PosePacket | None = None
+
+
+@dataclass(slots=True, frozen=True)
+class PoseSource:
+    backend: PoseBackends  # e.g. "DLCLive"
+    model_type: ModelType | None = None
+
+
+@dataclass(slots=True, frozen=True)
+class PosePacket:
+    schema_version: int = 0
+    keypoints: np.ndarray | None = None
+    keypoint_names: list[str] | None = None
+    individual_ids: list[str] | None = None
+    source: PoseSource = PoseSource(backend=PoseBackends.DLC_LIVE)
+    raw: Any | None = None
+
+
+def validate_pose_array(pose: Any, *, source_backend: PoseBackends = PoseBackends.DLC_LIVE) -> np.ndarray:
+    """
+    Validate pose output shape and dtype.
+
+    Accepted runner output shapes:
+    - (K, 3): single-animal
+    - (N, K, 3): multi-animal
+    """
+    try:
+        arr = np.asarray(pose)
+    except Exception as exc:
+        raise ValueError(
+            f"{source_backend} returned an invalid pose output format: could not convert to array ({exc})"
+        ) from exc
+
+    if arr.ndim not in (2, 3):
+        raise ValueError(
+            f"{source_backend} returned an invalid pose output format:"
+            f" expected a 2D or 3D array, got ndim={arr.ndim}, shape={arr.shape!r}"
+        )
+
+    if arr.shape[-1] != 3:
+        raise ValueError(
+            f"{source_backend} returned an invalid pose output format:"
+            f" expected last dimension size 3 (x, y, likelihood), got shape={arr.shape!r}"
+        )
+
+    if arr.ndim == 2 and arr.shape[0] <= 0:
+        raise ValueError(f"{source_backend} returned an invalid pose output format: expected at least one keypoint")
+    if arr.ndim == 3 and (arr.shape[0] <= 0 or arr.shape[1] <= 0):
+        raise ValueError(
+            f"{source_backend} returned an invalid pose output format:"
+            f" expected at least one individual and one keypoint, got shape={arr.shape!r}"
+        )
+
+    if not np.issubdtype(arr.dtype, np.number):
+        raise ValueError(
+            f"{source_backend} returned an invalid pose output format: expected numeric values, got dtype={arr.dtype}"
+        )
+
+    return arr
 
 
 @dataclass
@@ -60,9 +125,6 @@ class ProcessorStats:
     avg_processor_overhead: float = 0.0  # Socket processor overhead
 
 
-# _SENTINEL = object()
-
-
 class DLCLiveProcessor(QObject):
     """Background pose estimation using DLCLive with queue-based threading."""
 
@@ -269,8 +331,17 @@ def _process_frame(
         # Time GPU inference (and processor overhead when present)
         with self._timed_processor() as proc_holder:
             inference_start = time.perf_counter()
-            pose = self._dlc.get_pose(frame, frame_time=timestamp)
+            raw_pose: Any = self._dlc.get_pose(frame, frame_time=timestamp)
             inference_time = time.perf_counter() - inference_start
+        pose_arr: np.ndarray = validate_pose_array(raw_pose, source_backend=PoseBackends.DLC_LIVE)
+        pose_packet = PosePacket(
+            schema_version=0,
+            keypoints=pose_arr,
+            keypoint_names=None,
+            individual_ids=None,
+            source=PoseSource(backend=PoseBackends.DLC_LIVE, model_type=self._settings.model_type),
+            raw=raw_pose,
+        )
 
         processor_overhead = 0.0
         gpu_inference_time = inference_time
@@ -280,7 +351,7 @@ def _process_frame(
 
         # Emit pose (measure signal overhead)
         signal_start = time.perf_counter()
-        self.pose_ready.emit(PoseResult(pose=pose, timestamp=timestamp))
+        self.pose_ready.emit(PoseResult(pose=pose_packet.keypoints, timestamp=timestamp, packet=pose_packet))
         signal_time = time.perf_counter() - signal_start
 
         end_ts = time.perf_counter()

pyproject.toml

@@ -115,6 +115,7 @@ markers = [
   "unit: Unit tests for individual components",
   "integration: Integration tests for component interaction",
   "functional: Functional tests for end-to-end workflows",
+  "dlclive_compat: Package/API compatibility tests against supported dlclive versions",
   "hardware: Tests that require specific hardware, notable camera backends",
   # "slow: Tests that take a long time to run",
   "gui: Tests that require GUI interaction",

tests/compat/conftest.py

@@ -0,0 +1,10 @@
+# tests/compat/conftest.py
+import sys
+import types
+
+# Stub out torch imports to avoid ImportError when torch is not installed in DLCLive package.
+# This allows testing of DLCLive API compatibility without requiring torch.
+# Ideally imports should be guarded in the package itself, but this is a pragmatic solution for now.
+# IMPORTANT NOTE: This should ideally be removed and replaced whenever possible.
+if "torch" not in sys.modules:
+    sys.modules["torch"] = types.ModuleType("torch")

tests/compat/test_dlclive_package_compat.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import importlib.metadata
+import inspect
+import os
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+
+def _get_signature_params(callable_obj) -> tuple[set[str], bool]:
+    """
+    Return allowed keyword names for callable, allowing for **kwargs.
+
+    Example:
+    >>> params, accepts_var_kw = _get_signature_params(lambda x, y, **kwargs: None, {"x", "y"})
+    >>> params == {"x", "y"}
+    True
+    >>> accepts_var_kw
+    True
+    """
+    sig = inspect.signature(callable_obj)
+    params = sig.parameters
+    accepts_var_kw = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
+    return params, accepts_var_kw
+
+
+@pytest.mark.dlclive_compat
+def test_dlclive_package_is_importable():
+    from dlclive import DLCLive  # noqa: PLC0415
+
+    assert DLCLive is not None
+    # Helpful for CI logs to confirm matrix install result.
+    _ = importlib.metadata.version("deeplabcut-live")
+
+
+@pytest.mark.dlclive_compat
+def test_dlclive_constructor_accepts_gui_expected_kwargs():
+    """
+    GUI passes these kwargs when constructing DLCLive.
+    This test catches upstream API changes that would break initialization.
+    """
+    from dlclive import DLCLive  # noqa: PLC0415
+
+    expected = {
+        "model_path",
+        "model_type",
+        "processor",
+        "dynamic",
+        "resize",
+        "precision",
+        "single_animal",
+        "device",
+    }
+    params, _ = _get_signature_params(DLCLive.__init__)
+    params = {
+        name
+        for name, p in params.items()
+        if p.kind in (inspect.Parameter.KEYWORD_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+    }
+    missing = {name for name in expected if name not in params}
+    assert not missing, f"DLCLive.__init__ is missing expected kwargs called by GUI: {sorted(missing)}"
+
+
+@pytest.mark.dlclive_compat
+def test_dlclive_methods_match_gui_usage():
+    """
+    GUI expects:
+    - init_inference(frame)
+    - get_pose(frame, frame_time=<float>)
+    """
+    from dlclive import DLCLive  # noqa: PLC0415
+
+    assert hasattr(DLCLive, "init_inference"), "DLCLive must provide init_inference(frame)"
+    assert hasattr(DLCLive, "get_pose"), "DLCLive must provide get_pose(frame, frame_time=...)"
+    # NOTE: frame_time is passed as a kwarg, so we only check for "frame" as a required param.
+    #  This is used by DLCLive Processor classes, rather than the DLCLive class itself.
+
+    init_params, _ = _get_signature_params(DLCLive.init_inference)
+    init_missing = {name for name in {"frame"} if name not in init_params}
+    assert not init_missing, f"DLCLive.init_inference signature mismatch, missing: {sorted(init_missing)}"
+
+    get_pose_params, _ = _get_signature_params(DLCLive.get_pose)
+    get_pose_missing = {name for name in {"frame"} if name not in get_pose_params}
+    assert not get_pose_missing, f"DLCLive.get_pose signature mismatch, missing: {sorted(get_pose_missing)}"
+
+
+@pytest.mark.dlclive_compat
+def test_dlclive_minimal_inference_smoke():
+    """
+    Real runtime smoke test (init + pose call) using a tiny exported model.
+
+    Opt-in via env vars:
+    - DLCLIVE_TEST_MODEL_PATH: absolute/relative path to exported model folder/file
+    - DLCLIVE_TEST_MODEL_TYPE: optional model type (default: pytorch)
+    """
+    model_path_env = os.getenv("DLCLIVE_TEST_MODEL_PATH", "").strip()
+    if not model_path_env:
+        pytest.skip("Set DLCLIVE_TEST_MODEL_PATH to run real DLCLive inference smoke test.")
+
+    model_path = Path(model_path_env).expanduser()
+    if not model_path.exists():
+        pytest.skip(f"DLCLIVE_TEST_MODEL_PATH does not exist: {model_path}")
+
+    model_type = os.getenv("DLCLIVE_TEST_MODEL_TYPE", "pytorch").strip() or "pytorch"
+
+    from dlclive import DLCLive  # noqa: PLC0415
+
+    from dlclivegui.services.dlc_processor import validate_pose_array  # noqa: PLC0415
+
+    dlc = DLCLive(
+        model_path=str(model_path),
+        model_type=model_type,
+        dynamic=[False, 0.5, 10],
+        resize=1.0,
+        precision="FP32",
+        single_animal=True,
+    )
+
+    frame = np.zeros((64, 64, 3), dtype=np.uint8)
+    dlc.init_inference(frame)
+    pose = dlc.get_pose(frame, frame_time=0.0)
+    pose_arr = validate_pose_array(pose, source_backend="DLCLive.get_pose")
+
+    assert pose_arr.ndim in (2, 3)
+    assert pose_arr.shape[-1] == 3
+    assert np.isfinite(pose_arr).all()

tests/conftest.py

@@ -168,7 +168,7 @@ def init_inference(self, frame):
 
     def get_pose(self, frame, frame_time=None):
         self.pose_calls += 1
-        return np.ones((2, 2), dtype=float)
+        return np.ones((2, 3), dtype=float)
 
 
 @pytest.fixture