From 581f1f347e76849e9c15b6b1a319ab14b6bf4ae3 Mon Sep 17 00:00:00 2001 From: David Pastl Date: Fri, 13 Mar 2026 13:30:55 -0600 Subject: [PATCH 01/15] Fix for ldap errors --- .../tests/test_authenticators.py | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/bluesky_httpserver/tests/test_authenticators.py b/bluesky_httpserver/tests/test_authenticators.py index 53c6bbe..1d92a09 100644 --- a/bluesky_httpserver/tests/test_authenticators.py +++ b/bluesky_httpserver/tests/test_authenticators.py @@ -1,4 +1,5 @@ import asyncio +import os import time from typing import Any, Tuple @@ -10,20 +11,29 @@ from respx import MockRouter from starlette.datastructures import URL, QueryParams +LDAP_TEST_HOST = os.environ.get("QSERVER_TEST_LDAP_HOST", "localhost") +LDAP_TEST_PORT = int(os.environ.get("QSERVER_TEST_LDAP_PORT", "1389")) +LDAP_TEST_ALT_HOST = os.environ.get("QSERVER_TEST_LDAP_ALT_HOST") +if not LDAP_TEST_ALT_HOST: + LDAP_TEST_ALT_HOST = ( + "127.0.0.1" if LDAP_TEST_HOST == "localhost" else LDAP_TEST_HOST + ) + + # fmt: off from ..authenticators import LDAPAuthenticator, OIDCAuthenticator, ProxiedOIDCAuthenticator, UserSessionState @pytest.mark.parametrize("ldap_server_address, ldap_server_port", [ - ("localhost", 1389), - ("localhost:1389", 904), # Random port, ignored - ("localhost:1389", None), - ("127.0.0.1", 1389), - ("127.0.0.1:1389", 904), - (["localhost"], 1389), - (["localhost", "127.0.0.1"], 1389), - (["localhost", "127.0.0.1:1389"], 1389), - (["localhost:1389", "127.0.0.1:1389"], None), + (LDAP_TEST_HOST, LDAP_TEST_PORT), + (f"{LDAP_TEST_HOST}:{LDAP_TEST_PORT}", 904), # Random port, ignored + (f"{LDAP_TEST_HOST}:{LDAP_TEST_PORT}", None), + (LDAP_TEST_ALT_HOST, LDAP_TEST_PORT), + (f"{LDAP_TEST_ALT_HOST}:{LDAP_TEST_PORT}", 904), + ([LDAP_TEST_HOST], LDAP_TEST_PORT), + ([LDAP_TEST_HOST, LDAP_TEST_ALT_HOST], LDAP_TEST_PORT), + ([LDAP_TEST_HOST, f"{LDAP_TEST_ALT_HOST}:{LDAP_TEST_PORT}"], LDAP_TEST_PORT), + ([f"{LDAP_TEST_HOST}:{LDAP_TEST_PORT}", f"{LDAP_TEST_ALT_HOST}:{LDAP_TEST_PORT}"], None), ]) # fmt: on @pytest.mark.parametrize("use_tls,use_ssl", [(False, False)]) From 2412733af1e88b1ec9d8bc419fb27da160e5b4b8 Mon Sep 17 00:00:00 2001 From: David Pastl Date: Fri, 13 Mar 2026 14:10:39 -0600 Subject: [PATCH 02/15] Corecting formatting errors from pre-commit I didn't run it beforehand. Oops. --- bluesky_httpserver/tests/test_authenticators.py | 7 +++---- docs/source/usage.rst | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/bluesky_httpserver/tests/test_authenticators.py b/bluesky_httpserver/tests/test_authenticators.py index 1d92a09..7b7dd4b 100644 --- a/bluesky_httpserver/tests/test_authenticators.py +++ b/bluesky_httpserver/tests/test_authenticators.py @@ -11,17 +11,16 @@ from respx import MockRouter from starlette.datastructures import URL, QueryParams +from ..authenticators import LDAPAuthenticator, OIDCAuthenticator, ProxiedOIDCAuthenticator, UserSessionState + LDAP_TEST_HOST = os.environ.get("QSERVER_TEST_LDAP_HOST", "localhost") LDAP_TEST_PORT = int(os.environ.get("QSERVER_TEST_LDAP_PORT", "1389")) LDAP_TEST_ALT_HOST = os.environ.get("QSERVER_TEST_LDAP_ALT_HOST") if not LDAP_TEST_ALT_HOST: - LDAP_TEST_ALT_HOST = ( - "127.0.0.1" if LDAP_TEST_HOST == "localhost" else LDAP_TEST_HOST - ) + LDAP_TEST_ALT_HOST = "127.0.0.1" if LDAP_TEST_HOST == "localhost" else LDAP_TEST_HOST # fmt: off -from ..authenticators import LDAPAuthenticator, OIDCAuthenticator, ProxiedOIDCAuthenticator, UserSessionState @pytest.mark.parametrize("ldap_server_address, ldap_server_port", [ diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 299bdcb..bcae133 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -169,7 +169,7 @@ If you are already in a browser context, open: This redirects to the OIDC provider login page and then back to the server callback. -This can similarly be acheived using ``httpie`` by opening the URL in a browser after getting +This can similarly be acheived using ``httpie`` by opening the URL in a browser after getting the authorization URI from the server:: http POST http://localhost:60610/api/auth/provider/entra/authorize @@ -183,7 +183,7 @@ spawn a browser for the user to log in to the provider. CLI/device flow *************** -For terminal clients (i.e. no browser possible), start with +For terminal clients (i.e. no browser possible), start with ``POST /api/auth/provider//authorize``. The response includes: From 86838851e5245355a73282edcf521c836367619c Mon Sep 17 00:00:00 2001 From: David Pastl Date: Fri, 13 Mar 2026 15:16:15 -0600 Subject: [PATCH 03/15] Updates to LDAP server for test support These seem to resolve the LDAP server issues locally, pushing the changes to see if that fixes them remotely as well. --- .github/workflows/testing.yml | 6 - .../docker-configs/ldap-docker-compose.yml | 11 +- .../dockerfiles}/test.Dockerfile | 0 continuous_integration/scripts/start_LDAP.sh | 132 +++++- docs/source/configuration.rst | 2 +- docs/source/contributing.rst | 2 +- scripts/run-ci-docker-worker-matrix.py | 438 ++++++++++++++++++ scripts/run-github-actions-local.sh | 379 +++++++++++++++ scripts/run_ci_docker_parallel.sh | 56 +-- 9 files changed, 966 insertions(+), 60 deletions(-) rename {docker => continuous_integration/dockerfiles}/test.Dockerfile (100%) create mode 100755 scripts/run-ci-docker-worker-matrix.py create mode 100755 scripts/run-github-actions-local.sh diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 5355c05..5f88dc1 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -36,12 +36,6 @@ jobs: run: | # sudo apt install redis - pushd .. - git clone https://github.com/bitnami/containers.git - cd containers/bitnami/openldap/2.6/debian-12 - docker build -t bitnami/openldap:latest . - popd - # Start LDAP source continuous_integration/scripts/start_LDAP.sh diff --git a/continuous_integration/docker-configs/ldap-docker-compose.yml b/continuous_integration/docker-configs/ldap-docker-compose.yml index 2b2c45a..5fbfc53 100644 --- a/continuous_integration/docker-configs/ldap-docker-compose.yml +++ b/continuous_integration/docker-configs/ldap-docker-compose.yml @@ -1,14 +1,13 @@ services: openldap: - image: osixia/openldap:latest + image: osixia/openldap:1.5.0 ports: - - '1389:1389' - - '1636:1636' + - '1389:389' + - '1636:636' environment: - - LDAP_ADMIN_USERNAME=admin + - LDAP_ORGANISATION=Example Inc. + - LDAP_DOMAIN=example.org - LDAP_ADMIN_PASSWORD=adminpassword - - LDAP_USERS=user01,user02 - - LDAP_PASSWORDS=password1,password2 volumes: - 'openldap_data:/var/lib/ldap' diff --git a/docker/test.Dockerfile b/continuous_integration/dockerfiles/test.Dockerfile similarity index 100% rename from docker/test.Dockerfile rename to continuous_integration/dockerfiles/test.Dockerfile diff --git a/continuous_integration/scripts/start_LDAP.sh b/continuous_integration/scripts/start_LDAP.sh index ecfa1cf..897e9c6 100755 --- a/continuous_integration/scripts/start_LDAP.sh +++ b/continuous_integration/scripts/start_LDAP.sh @@ -1,7 +1,131 @@ -#!/bin/bash -set -e +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +COMPOSE_FILE="${LDAP_COMPOSE_FILE:-$ROOT_DIR/continuous_integration/docker-configs/ldap-docker-compose.yml}" +COMPOSE_PROJECT="${LDAP_COMPOSE_PROJECT:-}" +LDAP_HOST="${LDAP_HOST:-127.0.0.1}" +LDAP_PORT="${LDAP_PORT:-1389}" +LDAP_ADMIN_DN="cn=admin,dc=example,dc=org" +LDAP_ADMIN_PASSWORD="adminpassword" +LDAP_BASE_DN="dc=example,dc=org" + +compose_cmd() { + if [[ -n "$COMPOSE_PROJECT" ]]; then + docker compose -p "$COMPOSE_PROJECT" -f "$COMPOSE_FILE" "$@" + else + docker compose -f "$COMPOSE_FILE" "$@" + fi +} + +get_openldap_container_id() { + compose_cmd ps -q openldap | tr -d '[:space:]' +} + +wait_for_ldap() { + local timeout_seconds="${1:-60}" + local deadline=$((SECONDS + timeout_seconds)) + + while (( SECONDS < deadline )); do + if python - </dev/null 2>&1 +import socket + +with socket.create_connection(("${LDAP_HOST}", ${LDAP_PORT}), timeout=1): + pass +PY + then + return 0 + fi + sleep 1 + done + + return 1 +} + +wait_for_ldap_bind() { + local container_id="$1" + local timeout_seconds="${2:-60}" + local deadline=$((SECONDS + timeout_seconds)) + + while (( SECONDS < deadline )); do + if docker exec "$container_id" ldapsearch \ + -x \ + -H "ldap://127.0.0.1:389" \ + -D "$LDAP_ADMIN_DN" \ + -w "$LDAP_ADMIN_PASSWORD" \ + -b "$LDAP_BASE_DN" \ + -s base \ + "(objectclass=*)" dn >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + + return 1 +} + +ldap_entry_exists() { + local container_id="$1" + local dn="$2" + + docker exec "$container_id" ldapsearch \ + -x \ + -H "ldap://127.0.0.1:389" \ + -D "$LDAP_ADMIN_DN" \ + -w "$LDAP_ADMIN_PASSWORD" \ + -b "$dn" \ + -s base \ + "(objectclass=*)" dn >/dev/null 2>&1 +} + +ldap_add_if_missing() { + local container_id="$1" + local dn="$2" + local ldif="$3" + + if ldap_entry_exists "$container_id" "$dn"; then + return 0 + fi + + docker exec -i "$container_id" ldapadd \ + -x \ + -H "ldap://127.0.0.1:389" \ + -D "$LDAP_ADMIN_DN" \ + -w "$LDAP_ADMIN_PASSWORD" >/dev/null <&2 + exit 1 +fi +wait_for_ldap_bind "$CONTAINER_ID" 90 +seed_ldap_test_users "$CONTAINER_ID" docker ps diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 8852a31..3a3d290 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -261,7 +261,7 @@ LDAP Authenticator LDAP authenticator is designed for production deployments. The authenticator validates user login information (username/password) by communicating with LDAP server (e.g. active Directory server). The following example illustrates how to configure the server to -use demo OpenLDAP server running in docker container (run ``./start_LDAP.sh`` in the root +use demo OpenLDAP server running in docker container (run ``source continuous_integration/scripts/start_LDAP.sh`` in the root of the repository to start the server). The server is configured to authenticate two users: *'user01'* and *'user02'* with passwords *'password1'* and *'password2'* respectively. The configuration does not enable public access. :: diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index f09326a..079f0ed 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -128,4 +128,4 @@ locally, especially if the respective server code was not changed. The tests wil run on GitHub CI in properly configured environment and indicate if there is an issue. To run these tests locally, start OpenLDAP server in Docker container:: - $ source start_LDAP.sh + $ source continuous_integration/scripts/start_LDAP.sh diff --git a/scripts/run-ci-docker-worker-matrix.py b/scripts/run-ci-docker-worker-matrix.py new file mode 100755 index 0000000..562bde9 --- /dev/null +++ b/scripts/run-ci-docker-worker-matrix.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python3 +"""Run bluesky-httpserver CI-style checks using a docker client-worker model. + +This script is designed to mirror the project's GitHub Actions workflows locally, +with special focus on accelerating the pytest matrix by chunking tests and +dispatching chunks to worker containers. + +What it runs by default: +1. Style checks (black/isort/flake8/pre-commit) +2. Docs build check +3. Unit tests for Python 3.10/3.11/3.12/3.13 using worker containers + +Notes: +- The unit-test step follows `.github/workflows/testing.yml` dependency setup. +- Shared service containers (Redis, LDAP) are started once and reused. +- Unit tests are chunked by test file and distributed across workers per version. +""" + +from __future__ import annotations + +import argparse +import math +import os +import queue +import shutil +import subprocess +import sys +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, List, Sequence + +DEFAULT_PYTHON_VERSIONS = ["3.10", "3.11", "3.12", "3.13"] +DEFAULT_WORKERS_PER_VERSION = 2 + + +@dataclass +class ChunkResult: + python_version: str + worker_name: str + chunk_index: int + command: str + returncode: int + log_path: Path + + +def run( + command: Sequence[str], *, cwd: Path, env: dict | None = None, check: bool = True +) -> subprocess.CompletedProcess: + return subprocess.run(command, cwd=cwd, env=env, check=check, text=True) + + +def shell( + command: str, *, cwd: Path, env: dict | None = None, check: bool = True +) -> subprocess.CompletedProcess: + return subprocess.run(command, cwd=cwd, env=env, check=check, text=True, shell=True) + + +def chunk_items(items: List[str], chunks: int) -> List[List[str]]: + if chunks <= 0: + return [items] + if not items: + return [] + chunks = max(1, min(chunks, len(items))) + chunk_size = math.ceil(len(items) / chunks) + return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)] + + +def docker_cmd(*parts: str) -> List[str]: + return ["docker", *parts] + + +def ensure_docker_available(repo_root: Path) -> None: + run(["docker", "info"], cwd=repo_root) + + +def discover_test_files(repo_root: Path) -> List[str]: + test_dir = repo_root / "bluesky_httpserver" / "tests" + files = sorted( + str(path.relative_to(repo_root)) for path in test_dir.glob("test_*.py") + ) + return files + + +def start_redis_container(repo_root: Path, name: str) -> None: + run(docker_cmd("rm", "-f", name), cwd=repo_root, check=False) + run( + docker_cmd( + "run", + "-d", + "--rm", + "--name", + name, + "--network", + "host", + "redis:7", + ), + cwd=repo_root, + ) + + +def start_ldap_compose(repo_root: Path) -> None: + compose_file = ( + repo_root + / "continuous_integration" + / "docker-configs" + / "ldap-docker-compose.yml" + ) + env = os.environ.copy() + env["LDAP_COMPOSE_FILE"] = str(compose_file) + env["LDAP_COMPOSE_PROJECT"] = "bhs-ci-ldap" + env["LDAP_HOST"] = "127.0.0.1" + env["LDAP_PORT"] = "1389" + run( + [ + "bash", + str(repo_root / "continuous_integration" / "scripts" / "start_LDAP.sh"), + ], + cwd=repo_root, + env=env, + ) + + +def stop_ldap_compose(repo_root: Path) -> None: + compose_file = ( + repo_root + / "continuous_integration" + / "docker-configs" + / "ldap-docker-compose.yml" + ) + run( + [ + "docker", + "compose", + "-p", + "bhs-ci-ldap", + "-f", + str(compose_file), + "down", + "-v", + ], + cwd=repo_root, + check=False, + ) + + +def make_worker_name(python_version: str, index: int) -> str: + v = python_version.replace(".", "") + return f"bhs-ci-py{v}-worker{index}" + + +def start_worker_container( + repo_root: Path, python_version: str, worker_name: str +) -> None: + run(docker_cmd("rm", "-f", worker_name), cwd=repo_root, check=False) + run( + docker_cmd( + "run", + "-d", + "--rm", + "--name", + worker_name, + "--network", + "host", + "-v", + f"{repo_root}:/workspace", + "-w", + "/workspace", + f"python:{python_version}", + "bash", + "-lc", + "sleep infinity", + ), + cwd=repo_root, + ) + + +def exec_in_worker( + repo_root: Path, worker_name: str, command: str, *, log_path: Path +) -> int: + full_cmd = ["docker", "exec", worker_name, "bash", "-lc", command] + with log_path.open("w", encoding="utf-8") as log_file: + process = subprocess.run( + full_cmd, + cwd=repo_root, + stdout=log_file, + stderr=subprocess.STDOUT, + text=True, + ) + return process.returncode + + +def bootstrap_worker(repo_root: Path, worker_name: str) -> None: + # Mirrors `.github/workflows/testing.yml` install strategy as closely as possible. + install_cmd = " && ".join( + [ + "python -m pip install --upgrade pip setuptools numpy", + "pip install git+https://github.com/bluesky/bluesky-queueserver.git", + "pip install git+https://github.com/bluesky/bluesky-queueserver-api.git", + "pip install .", + "pip install -r requirements-dev.txt", + "pip list", + ] + ) + code = exec_in_worker( + repo_root, + worker_name, + install_cmd, + log_path=repo_root / ".ci-artifacts" / f"{worker_name}-bootstrap.log", + ) + if code != 0: + raise RuntimeError(f"Bootstrap failed for worker {worker_name}") + + +def run_style_and_docs(repo_root: Path, python_version: str) -> None: + worker_name = f"bhs-ci-style-docs-py{python_version.replace('.', '')}" + start_worker_container(repo_root, python_version, worker_name) + try: + bootstrap_worker(repo_root, worker_name) + steps = [ + ("black", "black . --check"), + ("isort", "isort . -c"), + ("flake8", "flake8"), + ("pre-commit", "pre-commit run --all-files"), + ("docs", "make -C docs/ html"), + ] + for label, cmd in steps: + log_path = repo_root / ".ci-artifacts" / f"{worker_name}-{label}.log" + code = exec_in_worker(repo_root, worker_name, cmd, log_path=log_path) + if code != 0: + raise RuntimeError(f"Step '{label}' failed. See {log_path}") + finally: + run(docker_cmd("rm", "-f", worker_name), cwd=repo_root, check=False) + + +def run_test_matrix( + repo_root: Path, + python_versions: Iterable[str], + workers_per_version: int, + include_pattern: str | None, + chunks_per_version: int | None, + tests_per_chunk: int | None, +) -> list[ChunkResult]: + artifacts_dir = repo_root / ".ci-artifacts" + test_files = discover_test_files(repo_root) + if include_pattern: + test_files = [f for f in test_files if include_pattern in f] + + if not test_files: + raise RuntimeError("No test files discovered.") + + all_results: list[ChunkResult] = [] + + for python_version in python_versions: + print(f"\n=== Python {python_version}: preparing workers ===", flush=True) + workers = [ + make_worker_name(python_version, i + 1) for i in range(workers_per_version) + ] + + for worker in workers: + start_worker_container(repo_root, python_version, worker) + try: + for worker in workers: + print(f"Bootstrapping {worker} ...", flush=True) + bootstrap_worker(repo_root, worker) + + if tests_per_chunk and tests_per_chunk > 0: + chunks = [ + test_files[i : i + tests_per_chunk] + for i in range(0, len(test_files), tests_per_chunk) + ] + else: + n_chunks = ( + chunks_per_version + if (chunks_per_version and chunks_per_version > 0) + else workers_per_version * 4 + ) + chunks = chunk_items(test_files, n_chunks) + work_queue: queue.Queue[tuple[int, list[str]]] = queue.Queue() + for idx, chunk in enumerate(chunks): + work_queue.put((idx, chunk)) + + results_lock = threading.Lock() + + def worker_loop(worker_name: str) -> None: + while True: + try: + chunk_index, chunk = work_queue.get_nowait() + except queue.Empty: + return + chunk_args = " ".join(chunk) + command = ( + "QSERVER_TEST_LDAP_HOST=localhost " + "QSERVER_TEST_LDAP_PORT=1389 " + f"coverage run -m pytest -vv {chunk_args}" + ) + log_path = ( + artifacts_dir / f"{worker_name}-chunk{chunk_index:03d}.log" + ) + rc = exec_in_worker( + repo_root, worker_name, command, log_path=log_path + ) + with results_lock: + all_results.append( + ChunkResult( + python_version=python_version, + worker_name=worker_name, + chunk_index=chunk_index, + command=command, + returncode=rc, + log_path=log_path, + ) + ) + work_queue.task_done() + + threads = [ + threading.Thread(target=worker_loop, args=(worker,), daemon=True) + for worker in workers + ] + for t in threads: + t.start() + for t in threads: + t.join() + + # Per-version coverage summary (best effort, non-fatal) + for worker in workers: + exec_in_worker( + repo_root, + worker, + "coverage report -m || true", + log_path=artifacts_dir / f"{worker}-coverage-report.log", + ) + finally: + for worker in workers: + run(docker_cmd("rm", "-f", worker), cwd=repo_root, check=False) + + return all_results + + +def parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--python-versions", + nargs="+", + default=DEFAULT_PYTHON_VERSIONS, + help="Python versions to test (default mirrors .github/workflows/testing.yml)", + ) + parser.add_argument( + "--workers-per-version", + type=int, + default=DEFAULT_WORKERS_PER_VERSION, + help="Worker containers per Python version", + ) + parser.add_argument( + "--chunks-per-version", + type=int, + default=None, + help="Total pytest chunks per Python version (default: workers_per_version * 4)", + ) + parser.add_argument( + "--tests-per-chunk", + type=int, + default=None, + help="Number of test files per chunk (overrides --chunks-per-version)", + ) + parser.add_argument( + "--include-pattern", + default=None, + help="Only run test files containing this substring", + ) + parser.add_argument( + "--skip-style-docs", + action="store_true", + help="Skip style/docs checks and run only the test matrix", + ) + parser.add_argument( + "--keep-artifacts", + action="store_true", + help="Keep .ci-artifacts from previous runs (default clears first)", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str]) -> int: + args = parse_args(argv) + repo_root = Path(__file__).resolve().parents[1] + artifacts_dir = repo_root / ".ci-artifacts" + + ensure_docker_available(repo_root) + + if artifacts_dir.exists() and not args.keep_artifacts: + shutil.rmtree(artifacts_dir) + artifacts_dir.mkdir(parents=True, exist_ok=True) + + redis_name = "bhs-ci-redis" + start_redis_container(repo_root, redis_name) + start_ldap_compose(repo_root) + + try: + if not args.skip_style_docs: + print( + "\n=== Running style/docs checks (CI parity for non-matrix workflows) ===", + flush=True, + ) + run_style_and_docs(repo_root, "3.12") + + print("\n=== Running chunked pytest matrix ===", flush=True) + results = run_test_matrix( + repo_root, + python_versions=args.python_versions, + workers_per_version=args.workers_per_version, + include_pattern=args.include_pattern, + chunks_per_version=args.chunks_per_version, + tests_per_chunk=args.tests_per_chunk, + ) + + failed = [r for r in results if r.returncode != 0] + print("\n=== Summary ===", flush=True) + print(f"Total chunks run: {len(results)}", flush=True) + print(f"Failed chunks: {len(failed)}", flush=True) + if failed: + for r in failed: + print( + f"[FAIL] py={r.python_version} worker={r.worker_name} chunk={r.chunk_index} " + f"log={r.log_path}", + flush=True, + ) + return 1 + print("All CI-equivalent checks passed.", flush=True) + return 0 + finally: + stop_ldap_compose(repo_root) + run(docker_cmd("rm", "-f", redis_name), cwd=repo_root, check=False) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/run-github-actions-local.sh b/scripts/run-github-actions-local.sh new file mode 100755 index 0000000..70f9741 --- /dev/null +++ b/scripts/run-github-actions-local.sh @@ -0,0 +1,379 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(git rev-parse --show-toplevel 2>/dev/null || true)" +if [[ -z "${ROOT_DIR}" ]]; then + echo "Error: not inside a git repository." >&2 + exit 1 +fi + +cd "${ROOT_DIR}" + +INSTALL_DEPS=0 +SKIP_LDAP=0 +TARGETS_RAW="all" +PYTHON_BIN="${PYTHON_BIN:-python}" +ENV_MODE="auto" # auto | uv | system +USE_UV=0 +UV_PROJECT_PYTHON=".venv/bin/python" +UV_PROJECT_ENVIRONMENT=".venv" + +LDAP_COMPOSE_FILE="continuous_integration/docker-configs/ldap-docker-compose.yml" + +print_help() { + cat <<'EOF' +Run local equivalents of the GitHub Actions checks. + +Usage: + ./scripts/run-github-actions-local.sh [options] + +Options: + --targets Comma-separated targets: + black,isort,flake8,pre-commit,unit,docs,all + (default: all) + --install-deps Install/upgrade dependencies before running + (auto-runs when required deps are missing) + --env Environment mode (default: auto) + auto: use uv if available, otherwise system + --skip-ldap Do not start LDAP for unit tests + --python Python executable (default: python) + -h, --help Show this help + +Examples: + ./scripts/run-github-actions-local.sh + ./scripts/run-github-actions-local.sh --targets black,isort,flake8,pre-commit + ./scripts/run-github-actions-local.sh --targets unit --install-deps + ./scripts/run-github-actions-local.sh --env uv --targets black,unit,docs +EOF +} + +log_section() { + echo + echo "==> $*" +} + +run_cmd() { + echo "+ $*" + "$@" +} + +run_local_cmd() { + if [[ ${USE_UV} -eq 1 ]]; then + run_cmd uv run --python "${UV_PROJECT_PYTHON}" "$@" + else + run_cmd "$@" + fi +} + +has_cmd() { + command -v "$1" >/dev/null 2>&1 +} + +install_dependencies() { + log_section "Installing dependencies" + if [[ ${USE_UV} -eq 1 ]]; then + if [[ ! -x "${UV_PROJECT_PYTHON}" ]]; then + echo "Error: uv mode requires an existing project environment at ${UV_PROJECT_PYTHON}." >&2 + echo "Create it once with: uv venv" >&2 + exit 1 + fi + run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" --upgrade pip setuptools numpy + run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" -e . + run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" -r requirements-dev.txt + else + run_cmd "${PYTHON_BIN}" -m pip install --upgrade pip setuptools numpy + run_cmd "${PYTHON_BIN}" -m pip install -e . + run_cmd "${PYTHON_BIN}" -m pip install -r requirements-dev.txt + fi +} + +start_ldap_if_needed() { + if [[ ${SKIP_LDAP} -eq 1 ]]; then + log_section "Skipping LDAP startup (--skip-ldap)" + return + fi + if [[ ! -f "${LDAP_COMPOSE_FILE}" ]]; then + echo "Warning: LDAP compose file not found at ${LDAP_COMPOSE_FILE}; continuing without startup." >&2 + return + fi + log_section "Starting LDAP service for LDAP-related tests" + run_cmd bash continuous_integration/scripts/start_LDAP.sh +} + +run_local_target() { + local target="$1" + case "${target}" in + black) + log_section "Running BLACK (local)" + run_local_cmd black . --check + ;; + isort) + log_section "Running ISORT (local)" + run_local_cmd isort . -c + ;; + flake8) + log_section "Running FLAKE8 (local)" + run_local_cmd flake8 + ;; + pre-commit) + log_section "Running pre-commit (local)" + run_local_cmd pre-commit run --all-files + ;; + unit) + log_section "Running unit tests (local)" + start_ldap_if_needed + run_local_cmd coverage run -m pytest -vv + run_local_cmd coverage report -m + ;; + docs) + log_section "Building docs (local)" + run_local_cmd make -C docs/ html + ;; + *) + echo "Error: unknown local target '${target}'" >&2 + exit 2 + ;; + esac +} + +normalize_targets() { + local raw="$1" + raw="${raw//testing/unit}" + if [[ "${raw}" == "all" ]]; then + echo "black isort flake8 pre-commit unit docs" + return + fi + + local csv="${raw//,/ }" + local out=() + local item + for item in ${csv}; do + case "${item}" in + black|isort|flake8|pre-commit|unit|docs) + out+=("${item}") + ;; + testing) + out+=("unit") + ;; + *) + echo "Error: unknown target '${item}'." >&2 + exit 2 + ;; + esac + done + + if [[ ${#out[@]} -eq 0 ]]; then + echo "Error: no targets specified." >&2 + exit 2 + fi + + echo "${out[*]}" +} + +detect_environment_mode() { + case "${ENV_MODE}" in + system) + USE_UV=0 + ;; + uv) + if ! has_cmd uv; then + echo "Error: --env uv requested but 'uv' is not installed or not on PATH." >&2 + exit 1 + fi + if [[ ! -x "${UV_PROJECT_PYTHON}" ]]; then + echo "Error: --env uv requested but ${UV_PROJECT_PYTHON} does not exist." >&2 + echo "Create it once with: uv venv" >&2 + exit 1 + fi + USE_UV=1 + ;; + auto) + if has_cmd uv; then + if [[ -x "${UV_PROJECT_PYTHON}" ]]; then + USE_UV=1 + else + USE_UV=0 + fi + else + USE_UV=0 + fi + ;; + *) + echo "Error: --env must be one of: auto, uv, system" >&2 + exit 2 + ;; + esac +} + +missing_local_tools() { + # In uv mode, commands are launched via 'uv run', so PATH checks are not useful. + if [[ ${USE_UV} -eq 1 ]]; then + return + fi + + local -a targets=("$@") + local -a required_tools=() + local target + + for target in "${targets[@]}"; do + case "${target}" in + black) + required_tools+=("black") + ;; + isort) + required_tools+=("isort") + ;; + flake8) + required_tools+=("flake8") + ;; + pre-commit) + required_tools+=("pre-commit") + ;; + unit) + required_tools+=("coverage" "pytest") + ;; + docs) + required_tools+=("make") + ;; + esac + done + + local -a missing=() + local tool + for tool in "${required_tools[@]}"; do + if ! has_cmd "${tool}"; then + missing+=("${tool}") + fi + done + + if [[ ${#missing[@]} -gt 0 ]]; then + echo "${missing[*]}" + fi +} + +uv_env_needs_bootstrap() { + local -a targets=("$@") + local module_check_script="import importlib.util as u\nmodules=[]\n" + + local target + for target in "${targets[@]}"; do + case "${target}" in + black) + module_check_script+="modules.append('black')\n" + ;; + isort) + module_check_script+="modules.append('isort')\n" + ;; + flake8) + module_check_script+="modules.append('flake8')\n" + ;; + pre-commit) + module_check_script+="modules.append('pre_commit')\n" + ;; + unit) + module_check_script+="modules.extend(['pytest', 'coverage'])\n" + ;; + docs) + module_check_script+="modules.append('sphinx')\n" + ;; + esac + done + module_check_script+="missing=[m for m in modules if u.find_spec(m) is None]\n" + module_check_script+="raise SystemExit(1 if missing else 0)\n" + + if [[ ! -x "${UV_PROJECT_PYTHON}" ]]; then + return 0 + fi + + if uv run --python "${UV_PROJECT_PYTHON}" python -c "${module_check_script}" >/dev/null 2>&1; then + return 1 + fi + return 0 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --targets) + TARGETS_RAW="$2" + shift 2 + ;; + --install-deps) + INSTALL_DEPS=1 + shift + ;; + --env) + ENV_MODE="$2" + shift 2 + ;; + --skip-ldap) + SKIP_LDAP=1 + shift + ;; + --python) + PYTHON_BIN="$2" + shift 2 + ;; + -h|--help) + print_help + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + print_help + exit 2 + ;; + esac +done + +IFS=' ' read -r -a TARGETS <<< "$(normalize_targets "${TARGETS_RAW}")" +detect_environment_mode + +log_section "Configuration" +echo "Repository: ${ROOT_DIR}" +echo "Mode: local" +echo "Targets: ${TARGETS[*]}" +if [[ ${USE_UV} -eq 1 ]]; then + export UV_PROJECT_ENVIRONMENT="${UV_PROJECT_ENVIRONMENT}" + echo "Environment: uv (${UV_PROJECT_PYTHON})" +else + echo "Environment: system (${PYTHON_BIN})" +fi + +if [[ ${INSTALL_DEPS} -eq 1 ]]; then + install_dependencies +fi + +if [[ ${USE_UV} -eq 1 ]]; then + if uv_env_needs_bootstrap "${TARGETS[@]}"; then + log_section "uv environment missing required packages" + echo "Auto-installing dependencies (equivalent to --install-deps)." + install_dependencies + fi +else + MISSING_TOOLS="$(missing_local_tools "${TARGETS[@]}" || true)" + if [[ -n "${MISSING_TOOLS}" ]]; then + log_section "Missing local tools detected" + echo "Missing: ${MISSING_TOOLS}" + echo "Auto-installing dependencies (equivalent to --install-deps)." + install_dependencies + fi +fi + +FAILURES=() + +for target in "${TARGETS[@]}"; do + if ! run_local_target "${target}"; then + FAILURES+=("${target}") + log_section "Target failed" + echo "Failed target: ${target}" + fi +done + +if [[ ${#FAILURES[@]} -gt 0 ]]; then + log_section "Summary" + echo "Completed with failures in: ${FAILURES[*]}" + exit 1 +fi + +log_section "Done" +echo "All requested checks completed." diff --git a/scripts/run_ci_docker_parallel.sh b/scripts/run_ci_docker_parallel.sh index c9caee7..efb6594 100755 --- a/scripts/run_ci_docker_parallel.sh +++ b/scripts/run_ci_docker_parallel.sh @@ -8,8 +8,10 @@ CHUNK_COUNT="" PYTHON_VERSIONS="latest" PYTEST_EXTRA_ARGS="" ARTIFACTS_DIR="$ROOT_DIR/.docker-test-artifacts" -DOCKER_NETWORK_NAME="bhs-ci-net" -LDAP_CONTAINER_NAME="bhs-ci-ldap" +LDAP_COMPOSE_FILE="$ROOT_DIR/continuous_integration/docker-configs/ldap-docker-compose.yml" +LDAP_COMPOSE_PROJECT="bhs-ci-ldap-parallel-$$" +LDAP_SERVICE_NAME="openldap" +DOCKER_NETWORK_NAME="${LDAP_COMPOSE_PROJECT}_default" SUMMARY_TSV="" SUMMARY_FAIL_LOGS="" @@ -154,46 +156,16 @@ normalize_python_versions() { echo "${normalized[@]}" } -ensure_ldap_image() { - local image_ref="bitnami/openldap:latest" - if docker image inspect "$image_ref" >/dev/null 2>&1; then - return - fi - - echo "LDAP image $image_ref not found locally; trying docker pull..." - if docker pull "$image_ref"; then - return - fi - - echo "docker pull failed; building bitnami/openldap:latest from source (CI fallback)." - local workdir="$ROOT_DIR/.docker-test-artifacts/bitnami-containers" - rm -rf "$workdir" - git clone --depth 1 https://github.com/bitnami/containers.git "$workdir" - (cd "$workdir/bitnami/openldap/2.6/debian-12" && docker build -t "$image_ref" .) -} - start_services() { - ensure_ldap_image - - docker network rm "$DOCKER_NETWORK_NAME" >/dev/null 2>&1 || true - docker network create "$DOCKER_NETWORK_NAME" >/dev/null - - docker rm -f "$LDAP_CONTAINER_NAME" >/dev/null 2>&1 || true - docker run -d --rm \ - --name "$LDAP_CONTAINER_NAME" \ - --network "$DOCKER_NETWORK_NAME" \ - -e LDAP_ADMIN_USERNAME=admin \ - -e LDAP_ADMIN_PASSWORD=adminpassword \ - -e LDAP_USERS=user01,user02 \ - -e LDAP_PASSWORDS=password1,password2 \ - bitnami/openldap:latest >/dev/null - - sleep 2 + LDAP_COMPOSE_FILE="$LDAP_COMPOSE_FILE" \ + LDAP_COMPOSE_PROJECT="$LDAP_COMPOSE_PROJECT" \ + LDAP_HOST="127.0.0.1" \ + LDAP_PORT="1389" \ + bash "$ROOT_DIR/continuous_integration/scripts/start_LDAP.sh" >/dev/null } stop_services() { - docker rm -f "$LDAP_CONTAINER_NAME" >/dev/null 2>&1 || true - docker network rm "$DOCKER_NETWORK_NAME" >/dev/null 2>&1 || true + docker compose -p "$LDAP_COMPOSE_PROJECT" -f "$LDAP_COMPOSE_FILE" down -v >/dev/null 2>&1 || true } cleanup() { @@ -385,8 +357,8 @@ run_chunk() { -e SHARD_COUNT="$CHUNK_COUNT" \ -e ARTIFACTS_DIR="/artifacts" \ -e PYTEST_EXTRA_ARGS="$PYTEST_EXTRA_ARGS" \ - -e QSERVER_TEST_LDAP_HOST="$LDAP_CONTAINER_NAME" \ - -e QSERVER_TEST_LDAP_PORT="1389" \ + -e QSERVER_TEST_LDAP_HOST="$LDAP_SERVICE_NAME" \ + -e QSERVER_TEST_LDAP_PORT="389" \ -e QSERVER_TEST_REDIS_ADDR="localhost" \ -e QSERVER_HTTP_TEST_BIND_HOST="127.0.0.1" \ -e QSERVER_HTTP_TEST_HOST="127.0.0.1" \ @@ -400,7 +372,7 @@ run_chunk() { } export -f run_chunk -export CHUNK_COUNT PYTEST_EXTRA_ARGS DOCKER_NETWORK_NAME LDAP_CONTAINER_NAME +export CHUNK_COUNT PYTEST_EXTRA_ARGS DOCKER_NETWORK_NAME LDAP_SERVICE_NAME for PYTHON_VERSION in "${SELECTED_PYTHON_VERSIONS[@]}"; do CURRENT_IMAGE_TAG="${IMAGE_TAG_BASE}-py${PYTHON_VERSION}" @@ -410,7 +382,7 @@ for PYTHON_VERSION in "${SELECTED_PYTHON_VERSIONS[@]}"; do echo "==> Building test image: $CURRENT_IMAGE_TAG (Python $PYTHON_VERSION)" docker build \ --build-arg PYTHON_VERSION="$PYTHON_VERSION" \ - -f "$ROOT_DIR/docker/test.Dockerfile" \ + -f "$ROOT_DIR/continuous_integration/dockerfiles/test.Dockerfile" \ -t "$CURRENT_IMAGE_TAG" \ "$ROOT_DIR" From 7a1b2900307f74a155a86b39c1d1f93d8d95917d Mon Sep 17 00:00:00 2001 From: David Pastl Date: Mon, 16 Mar 2026 11:43:30 -0600 Subject: [PATCH 04/15] Updating pyproject.toml and removing reqs This migrates to pyproject.toml which better supports the pixi and uv tools. Pip should work just fine as well. This removes the "python 2" support in setup.py but that was removed a long time ago already. --- .github/workflows/docs.yml | 2 +- .github/workflows/docs_publish.yml | 2 +- .github/workflows/pre-commit.yml | 2 +- .github/workflows/testing.yml | 2 +- MANIFEST.in | 2 - .../dockerfiles/test.Dockerfile | 4 +- docs/source/contributing.rst | 2 +- pyproject.toml | 73 +++++++++++++++++++ requirements-dev.txt | 23 ------ requirements.txt | 17 ----- scripts/run-ci-docker-worker-matrix.py | 2 +- scripts/run-github-actions-local.sh | 6 +- setup.py | 64 +--------------- 13 files changed, 84 insertions(+), 117 deletions(-) delete mode 100644 requirements-dev.txt delete mode 100644 requirements.txt diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 012a480..0cca78a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -33,7 +33,7 @@ jobs: popd pip install . - pip install -r requirements-dev.txt + pip install ".[dev]" pip list - name: Build Docs run: | diff --git a/.github/workflows/docs_publish.yml b/.github/workflows/docs_publish.yml index 49d73c3..4e6edae 100644 --- a/.github/workflows/docs_publish.yml +++ b/.github/workflows/docs_publish.yml @@ -45,7 +45,7 @@ jobs: popd pip install . - pip install -r requirements-dev.txt + pip install ".[dev]" pip list - name: Build Docs run: | diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 67916c4..696c03e 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -26,7 +26,7 @@ jobs: - name: Install dev dependencies run: | set -vxeuo pipefail - pip install -r requirements-dev.txt + pip install ".[dev]" python -m pip list - name: Run pre-commit diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 5f88dc1..a8a6ccc 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -58,7 +58,7 @@ jobs: pip install --upgrade pip pip install . - pip install -r requirements-dev.txt + pip install ".[dev]" # pip install "pydantic${{ matrix.pydantic-version }}" # pip install bluesky==1.11.0 diff --git a/MANIFEST.in b/MANIFEST.in index 16b309c..d9ffe21 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,8 +2,6 @@ include AUTHORS.rst include CONTRIBUTING.rst include LICENSE include README.rst -include requirements.txt - include bluesky_httpserver/database/alembic.ini.template include bluesky_httpserver/database/migrations/env.py include bluesky_httpserver/database/migrations/script.py.mako diff --git a/continuous_integration/dockerfiles/test.Dockerfile b/continuous_integration/dockerfiles/test.Dockerfile index 2e994cf..53288f3 100644 --- a/continuous_integration/dockerfiles/test.Dockerfile +++ b/continuous_integration/dockerfiles/test.Dockerfile @@ -13,15 +13,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /workspace -COPY requirements.txt requirements-dev.txt ./ COPY pyproject.toml setup.py setup.cfg MANIFEST.in versioneer.py README.rst AUTHORS.rst LICENSE ./ COPY bluesky_httpserver ./bluesky_httpserver RUN python -m pip install --upgrade pip setuptools wheel numpy && \ python -m pip install git+https://github.com/bluesky/bluesky-queueserver.git && \ python -m pip install git+https://github.com/bluesky/bluesky-queueserver-api.git && \ - python -m pip install -r requirements-dev.txt && \ - python -m pip install . + python -m pip install ".[dev]" COPY scripts/docker/run_shard_in_container.sh /usr/local/bin/run_shard_in_container.sh RUN chmod +x /usr/local/bin/run_shard_in_container.sh diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 079f0ed..a197a99 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -98,7 +98,7 @@ Install the HTTP Server in editable mode:: Install development dependencies:: - $ pip install -r requirements-dev.txt + $ pip install -e ".[dev]" Setting up `pre-commit` diff --git a/pyproject.toml b/pyproject.toml index d90939b..c7468a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,76 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "bluesky-httpserver" +dynamic = ["version"] +description = "HTTP Server for communicating with Bluesky Queue Server" +readme = "README.rst" +requires-python = ">=3.7" +license = "BSD-3-Clause" +authors = [{ name = "Brookhaven National Laboratory" }] +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Natural Language :: English", + "Programming Language :: Python :: 3", +] +dependencies = [ + "alembic", + "bluesky-queueserver", + "bluesky-queueserver-api", + "cachetools", + "fastapi", + "httpx", + "ldap3", + "orjson", + "pamela", + "pydantic", + "pydantic-settings", + "python-jose", + "pyzmq", + "sqlalchemy", + "starlette", + "typing-extensions; python_version < '3.8'", + "uvicorn", +] + +[project.optional-dependencies] +dev = [ + "black", + "codecov", + "coverage", + "cryptography", + "fastapi[all]", + "flake8", + "ipython", + "isort", + "matplotlib", + "numpydoc", + "pandas", + "pre-commit", + "py", + "pytest", + "pytest-asyncio", + "pytest-xprocess", + "requests", + "respx", + "sphinx", + "sphinx_rtd_theme", +] + +[project.scripts] +start-bluesky-httpserver = "bluesky_httpserver.server:start_server" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +exclude = ["docs", "tests"] + +[tool.setuptools.dynamic] +version = { attr = "bluesky_httpserver.__version__" } + [tool.black] line-length = 115 target-version = ['py37'] diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index e47dd72..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,23 +0,0 @@ -# These are required for developing the package (running the tests, building -# the documentation) but not necessarily required for _using_ it. -black -codecov -coverage -cryptography -fastapi[all] -flake8 -isort -pre-commit -pytest -pytest-asyncio -pytest-xprocess -py -respx -sphinx -ipython -numpydoc -matplotlib -pandas -sphinx -sphinx_rtd_theme -requests diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 1377ef0..0000000 --- a/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -alembic -bluesky-queueserver -bluesky-queueserver-api -cachetools -fastapi -httpx -ldap3 -orjson -pamela -pydantic -pydantic-settings -python-jose -pyzmq -sqlalchemy -starlette -typing-extensions;python_version<'3.8' -uvicorn diff --git a/scripts/run-ci-docker-worker-matrix.py b/scripts/run-ci-docker-worker-matrix.py index 562bde9..bd4caf4 100755 --- a/scripts/run-ci-docker-worker-matrix.py +++ b/scripts/run-ci-docker-worker-matrix.py @@ -199,7 +199,7 @@ def bootstrap_worker(repo_root: Path, worker_name: str) -> None: "pip install git+https://github.com/bluesky/bluesky-queueserver.git", "pip install git+https://github.com/bluesky/bluesky-queueserver-api.git", "pip install .", - "pip install -r requirements-dev.txt", + 'pip install ".[dev]"', "pip list", ] ) diff --git a/scripts/run-github-actions-local.sh b/scripts/run-github-actions-local.sh index 70f9741..8e67185 100755 --- a/scripts/run-github-actions-local.sh +++ b/scripts/run-github-actions-local.sh @@ -78,12 +78,10 @@ install_dependencies() { exit 1 fi run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" --upgrade pip setuptools numpy - run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" -e . - run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" -r requirements-dev.txt + run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" -e ".[dev]" else run_cmd "${PYTHON_BIN}" -m pip install --upgrade pip setuptools numpy - run_cmd "${PYTHON_BIN}" -m pip install -e . - run_cmd "${PYTHON_BIN}" -m pip install -r requirements-dev.txt + run_cmd "${PYTHON_BIN}" -m pip install -e ".[dev]" fi } diff --git a/setup.py b/setup.py index a32fa56..69f9b47 100644 --- a/setup.py +++ b/setup.py @@ -1,66 +1,6 @@ -import sys -from os import path - -from setuptools import find_packages, setup +from setuptools import setup import versioneer -# NOTE: This file must remain Python 2 compatible for the foreseeable future, -# to ensure that we error out properly for people with outdated setuptools -# and/or pip. -min_version = (3, 7) -if sys.version_info < min_version: - error = """ -bluesky-httpserver does not support Python {0}.{1}. -Python {2}.{3} and above is required. Check your Python version like so: - -python3 --version - -This may be due to an out-of-date pip. Make sure you have pip >= 9.0.1. -Upgrade pip like so: - -pip install --upgrade pip -""".format(*(sys.version_info[:2] + min_version)) - sys.exit(error) - -here = path.abspath(path.dirname(__file__)) - -with open(path.join(here, "README.rst"), encoding="utf-8") as readme_file: - readme = readme_file.read() - -with open(path.join(here, "requirements.txt")) as requirements_file: - # Parse requirements.txt, ignoring any commented-out lines. - requirements = [line for line in requirements_file.read().splitlines() if not line.startswith("#")] -setup( - name="bluesky-httpserver", - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), - description="HTTP Server for communicating with Bluesky Queue Server", - long_description=readme, - author="Brookhaven National Laboratory", - author_email="", - url="https://github.com/bluesky/bluesky-httpserver", - python_requires=">={}".format(".".join(str(n) for n in min_version)), - packages=find_packages(exclude=["docs", "tests"]), - entry_points={ - "console_scripts": [ - "start-bluesky-httpserver = bluesky_httpserver.server:start_server", - ], - }, - include_package_data=True, - package_data={ - "bluesky_httpserver": [ - # When adding files here, remember to update MANIFEST.in as well, - # or else they will not be included in the distribution on PyPI! - # 'path/to/data_file', - ] - }, - install_requires=requirements, - license="BSD (3-clause)", - classifiers=[ - "Development Status :: 2 - Pre-Alpha", - "Natural Language :: English", - "Programming Language :: Python :: 3", - ], -) +setup(cmdclass=versioneer.get_cmdclass()) From 1dc5c3d38dfdef7c66f15401dbb468a873ef582f Mon Sep 17 00:00:00 2001 From: David Pastl Date: Mon, 16 Mar 2026 13:36:29 -0600 Subject: [PATCH 05/15] Revert "Updating pyproject.toml and removing reqs" This reverts commit 7a1b2900307f74a155a86b39c1d1f93d8d95917d. --- .github/workflows/docs.yml | 2 +- .github/workflows/docs_publish.yml | 2 +- .github/workflows/pre-commit.yml | 2 +- .github/workflows/testing.yml | 2 +- MANIFEST.in | 2 + .../dockerfiles/test.Dockerfile | 4 +- docs/source/contributing.rst | 2 +- pyproject.toml | 73 ------------------- requirements-dev.txt | 23 ++++++ requirements.txt | 17 +++++ scripts/run-ci-docker-worker-matrix.py | 2 +- scripts/run-github-actions-local.sh | 6 +- setup.py | 64 +++++++++++++++- 13 files changed, 117 insertions(+), 84 deletions(-) create mode 100644 requirements-dev.txt create mode 100644 requirements.txt diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0cca78a..012a480 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -33,7 +33,7 @@ jobs: popd pip install . - pip install ".[dev]" + pip install -r requirements-dev.txt pip list - name: Build Docs run: | diff --git a/.github/workflows/docs_publish.yml b/.github/workflows/docs_publish.yml index 4e6edae..49d73c3 100644 --- a/.github/workflows/docs_publish.yml +++ b/.github/workflows/docs_publish.yml @@ -45,7 +45,7 @@ jobs: popd pip install . - pip install ".[dev]" + pip install -r requirements-dev.txt pip list - name: Build Docs run: | diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 696c03e..67916c4 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -26,7 +26,7 @@ jobs: - name: Install dev dependencies run: | set -vxeuo pipefail - pip install ".[dev]" + pip install -r requirements-dev.txt python -m pip list - name: Run pre-commit diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index a8a6ccc..5f88dc1 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -58,7 +58,7 @@ jobs: pip install --upgrade pip pip install . - pip install ".[dev]" + pip install -r requirements-dev.txt # pip install "pydantic${{ matrix.pydantic-version }}" # pip install bluesky==1.11.0 diff --git a/MANIFEST.in b/MANIFEST.in index d9ffe21..16b309c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,8 @@ include AUTHORS.rst include CONTRIBUTING.rst include LICENSE include README.rst +include requirements.txt + include bluesky_httpserver/database/alembic.ini.template include bluesky_httpserver/database/migrations/env.py include bluesky_httpserver/database/migrations/script.py.mako diff --git a/continuous_integration/dockerfiles/test.Dockerfile b/continuous_integration/dockerfiles/test.Dockerfile index 53288f3..2e994cf 100644 --- a/continuous_integration/dockerfiles/test.Dockerfile +++ b/continuous_integration/dockerfiles/test.Dockerfile @@ -13,13 +13,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /workspace +COPY requirements.txt requirements-dev.txt ./ COPY pyproject.toml setup.py setup.cfg MANIFEST.in versioneer.py README.rst AUTHORS.rst LICENSE ./ COPY bluesky_httpserver ./bluesky_httpserver RUN python -m pip install --upgrade pip setuptools wheel numpy && \ python -m pip install git+https://github.com/bluesky/bluesky-queueserver.git && \ python -m pip install git+https://github.com/bluesky/bluesky-queueserver-api.git && \ - python -m pip install ".[dev]" + python -m pip install -r requirements-dev.txt && \ + python -m pip install . COPY scripts/docker/run_shard_in_container.sh /usr/local/bin/run_shard_in_container.sh RUN chmod +x /usr/local/bin/run_shard_in_container.sh diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index a197a99..079f0ed 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -98,7 +98,7 @@ Install the HTTP Server in editable mode:: Install development dependencies:: - $ pip install -e ".[dev]" + $ pip install -r requirements-dev.txt Setting up `pre-commit` diff --git a/pyproject.toml b/pyproject.toml index c7468a1..d90939b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,76 +1,3 @@ -[build-system] -requires = ["setuptools>=64", "wheel"] -build-backend = "setuptools.build_meta" - -[project] -name = "bluesky-httpserver" -dynamic = ["version"] -description = "HTTP Server for communicating with Bluesky Queue Server" -readme = "README.rst" -requires-python = ">=3.7" -license = "BSD-3-Clause" -authors = [{ name = "Brookhaven National Laboratory" }] -classifiers = [ - "Development Status :: 2 - Pre-Alpha", - "Natural Language :: English", - "Programming Language :: Python :: 3", -] -dependencies = [ - "alembic", - "bluesky-queueserver", - "bluesky-queueserver-api", - "cachetools", - "fastapi", - "httpx", - "ldap3", - "orjson", - "pamela", - "pydantic", - "pydantic-settings", - "python-jose", - "pyzmq", - "sqlalchemy", - "starlette", - "typing-extensions; python_version < '3.8'", - "uvicorn", -] - -[project.optional-dependencies] -dev = [ - "black", - "codecov", - "coverage", - "cryptography", - "fastapi[all]", - "flake8", - "ipython", - "isort", - "matplotlib", - "numpydoc", - "pandas", - "pre-commit", - "py", - "pytest", - "pytest-asyncio", - "pytest-xprocess", - "requests", - "respx", - "sphinx", - "sphinx_rtd_theme", -] - -[project.scripts] -start-bluesky-httpserver = "bluesky_httpserver.server:start_server" - -[tool.setuptools] -include-package-data = true - -[tool.setuptools.packages.find] -exclude = ["docs", "tests"] - -[tool.setuptools.dynamic] -version = { attr = "bluesky_httpserver.__version__" } - [tool.black] line-length = 115 target-version = ['py37'] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..e47dd72 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,23 @@ +# These are required for developing the package (running the tests, building +# the documentation) but not necessarily required for _using_ it. +black +codecov +coverage +cryptography +fastapi[all] +flake8 +isort +pre-commit +pytest +pytest-asyncio +pytest-xprocess +py +respx +sphinx +ipython +numpydoc +matplotlib +pandas +sphinx +sphinx_rtd_theme +requests diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1377ef0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,17 @@ +alembic +bluesky-queueserver +bluesky-queueserver-api +cachetools +fastapi +httpx +ldap3 +orjson +pamela +pydantic +pydantic-settings +python-jose +pyzmq +sqlalchemy +starlette +typing-extensions;python_version<'3.8' +uvicorn diff --git a/scripts/run-ci-docker-worker-matrix.py b/scripts/run-ci-docker-worker-matrix.py index bd4caf4..562bde9 100755 --- a/scripts/run-ci-docker-worker-matrix.py +++ b/scripts/run-ci-docker-worker-matrix.py @@ -199,7 +199,7 @@ def bootstrap_worker(repo_root: Path, worker_name: str) -> None: "pip install git+https://github.com/bluesky/bluesky-queueserver.git", "pip install git+https://github.com/bluesky/bluesky-queueserver-api.git", "pip install .", - 'pip install ".[dev]"', + "pip install -r requirements-dev.txt", "pip list", ] ) diff --git a/scripts/run-github-actions-local.sh b/scripts/run-github-actions-local.sh index 8e67185..70f9741 100755 --- a/scripts/run-github-actions-local.sh +++ b/scripts/run-github-actions-local.sh @@ -78,10 +78,12 @@ install_dependencies() { exit 1 fi run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" --upgrade pip setuptools numpy - run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" -e ".[dev]" + run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" -e . + run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" -r requirements-dev.txt else run_cmd "${PYTHON_BIN}" -m pip install --upgrade pip setuptools numpy - run_cmd "${PYTHON_BIN}" -m pip install -e ".[dev]" + run_cmd "${PYTHON_BIN}" -m pip install -e . + run_cmd "${PYTHON_BIN}" -m pip install -r requirements-dev.txt fi } diff --git a/setup.py b/setup.py index 69f9b47..a32fa56 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,66 @@ -from setuptools import setup +import sys +from os import path + +from setuptools import find_packages, setup import versioneer +# NOTE: This file must remain Python 2 compatible for the foreseeable future, +# to ensure that we error out properly for people with outdated setuptools +# and/or pip. +min_version = (3, 7) +if sys.version_info < min_version: + error = """ +bluesky-httpserver does not support Python {0}.{1}. +Python {2}.{3} and above is required. Check your Python version like so: + +python3 --version + +This may be due to an out-of-date pip. Make sure you have pip >= 9.0.1. +Upgrade pip like so: + +pip install --upgrade pip +""".format(*(sys.version_info[:2] + min_version)) + sys.exit(error) + +here = path.abspath(path.dirname(__file__)) + +with open(path.join(here, "README.rst"), encoding="utf-8") as readme_file: + readme = readme_file.read() + +with open(path.join(here, "requirements.txt")) as requirements_file: + # Parse requirements.txt, ignoring any commented-out lines. + requirements = [line for line in requirements_file.read().splitlines() if not line.startswith("#")] -setup(cmdclass=versioneer.get_cmdclass()) +setup( + name="bluesky-httpserver", + version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), + description="HTTP Server for communicating with Bluesky Queue Server", + long_description=readme, + author="Brookhaven National Laboratory", + author_email="", + url="https://github.com/bluesky/bluesky-httpserver", + python_requires=">={}".format(".".join(str(n) for n in min_version)), + packages=find_packages(exclude=["docs", "tests"]), + entry_points={ + "console_scripts": [ + "start-bluesky-httpserver = bluesky_httpserver.server:start_server", + ], + }, + include_package_data=True, + package_data={ + "bluesky_httpserver": [ + # When adding files here, remember to update MANIFEST.in as well, + # or else they will not be included in the distribution on PyPI! + # 'path/to/data_file', + ] + }, + install_requires=requirements, + license="BSD (3-clause)", + classifiers=[ + "Development Status :: 2 - Pre-Alpha", + "Natural Language :: English", + "Programming Language :: Python :: 3", + ], +) From 1639a654058d9660405bcc5bb109fb888a4b6733 Mon Sep 17 00:00:00 2001 From: David Pastl Date: Mon, 16 Mar 2026 13:42:13 -0600 Subject: [PATCH 06/15] Fixing pre-commit checks There was an unused import. --- scripts/run-ci-docker-worker-matrix.py | 53 ++++++-------------------- 1 file changed, 11 insertions(+), 42 deletions(-) diff --git a/scripts/run-ci-docker-worker-matrix.py b/scripts/run-ci-docker-worker-matrix.py index 562bde9..6771c19 100755 --- a/scripts/run-ci-docker-worker-matrix.py +++ b/scripts/run-ci-docker-worker-matrix.py @@ -26,7 +26,6 @@ import subprocess import sys import threading -import time from dataclasses import dataclass from pathlib import Path from typing import Iterable, List, Sequence @@ -51,9 +50,7 @@ def run( return subprocess.run(command, cwd=cwd, env=env, check=check, text=True) -def shell( - command: str, *, cwd: Path, env: dict | None = None, check: bool = True -) -> subprocess.CompletedProcess: +def shell(command: str, *, cwd: Path, env: dict | None = None, check: bool = True) -> subprocess.CompletedProcess: return subprocess.run(command, cwd=cwd, env=env, check=check, text=True, shell=True) @@ -77,9 +74,7 @@ def ensure_docker_available(repo_root: Path) -> None: def discover_test_files(repo_root: Path) -> List[str]: test_dir = repo_root / "bluesky_httpserver" / "tests" - files = sorted( - str(path.relative_to(repo_root)) for path in test_dir.glob("test_*.py") - ) + files = sorted(str(path.relative_to(repo_root)) for path in test_dir.glob("test_*.py")) return files @@ -101,12 +96,7 @@ def start_redis_container(repo_root: Path, name: str) -> None: def start_ldap_compose(repo_root: Path) -> None: - compose_file = ( - repo_root - / "continuous_integration" - / "docker-configs" - / "ldap-docker-compose.yml" - ) + compose_file = repo_root / "continuous_integration" / "docker-configs" / "ldap-docker-compose.yml" env = os.environ.copy() env["LDAP_COMPOSE_FILE"] = str(compose_file) env["LDAP_COMPOSE_PROJECT"] = "bhs-ci-ldap" @@ -123,12 +113,7 @@ def start_ldap_compose(repo_root: Path) -> None: def stop_ldap_compose(repo_root: Path) -> None: - compose_file = ( - repo_root - / "continuous_integration" - / "docker-configs" - / "ldap-docker-compose.yml" - ) + compose_file = repo_root / "continuous_integration" / "docker-configs" / "ldap-docker-compose.yml" run( [ "docker", @@ -150,9 +135,7 @@ def make_worker_name(python_version: str, index: int) -> str: return f"bhs-ci-py{v}-worker{index}" -def start_worker_container( - repo_root: Path, python_version: str, worker_name: str -) -> None: +def start_worker_container(repo_root: Path, python_version: str, worker_name: str) -> None: run(docker_cmd("rm", "-f", worker_name), cwd=repo_root, check=False) run( docker_cmd( @@ -176,9 +159,7 @@ def start_worker_container( ) -def exec_in_worker( - repo_root: Path, worker_name: str, command: str, *, log_path: Path -) -> int: +def exec_in_worker(repo_root: Path, worker_name: str, command: str, *, log_path: Path) -> int: full_cmd = ["docker", "exec", worker_name, "bash", "-lc", command] with log_path.open("w", encoding="utf-8") as log_file: process = subprocess.run( @@ -254,9 +235,7 @@ def run_test_matrix( for python_version in python_versions: print(f"\n=== Python {python_version}: preparing workers ===", flush=True) - workers = [ - make_worker_name(python_version, i + 1) for i in range(workers_per_version) - ] + workers = [make_worker_name(python_version, i + 1) for i in range(workers_per_version)] for worker in workers: start_worker_container(repo_root, python_version, worker) @@ -266,10 +245,7 @@ def run_test_matrix( bootstrap_worker(repo_root, worker) if tests_per_chunk and tests_per_chunk > 0: - chunks = [ - test_files[i : i + tests_per_chunk] - for i in range(0, len(test_files), tests_per_chunk) - ] + chunks = [test_files[i : i + tests_per_chunk] for i in range(0, len(test_files), tests_per_chunk)] else: n_chunks = ( chunks_per_version @@ -295,12 +271,8 @@ def worker_loop(worker_name: str) -> None: "QSERVER_TEST_LDAP_PORT=1389 " f"coverage run -m pytest -vv {chunk_args}" ) - log_path = ( - artifacts_dir / f"{worker_name}-chunk{chunk_index:03d}.log" - ) - rc = exec_in_worker( - repo_root, worker_name, command, log_path=log_path - ) + log_path = artifacts_dir / f"{worker_name}-chunk{chunk_index:03d}.log" + rc = exec_in_worker(repo_root, worker_name, command, log_path=log_path) with results_lock: all_results.append( ChunkResult( @@ -314,10 +286,7 @@ def worker_loop(worker_name: str) -> None: ) work_queue.task_done() - threads = [ - threading.Thread(target=worker_loop, args=(worker,), daemon=True) - for worker in workers - ] + threads = [threading.Thread(target=worker_loop, args=(worker,), daemon=True) for worker in workers] for t in threads: t.start() for t in threads: From fd949ccde731b27271c6947c772f4bdd66a1c282 Mon Sep 17 00:00:00 2001 From: David Pastl Date: Tue, 17 Mar 2026 10:24:51 -0600 Subject: [PATCH 07/15] Attempting to harden unit test runners further --- .github/workflows/testing.yml | 11 +++++++ .../tests/test_core_api_main.py | 28 ++++++++++++---- continuous_integration/scripts/start_LDAP.sh | 33 +++++++++++++++++-- 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 5f88dc1..e9500c3 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -67,3 +67,14 @@ jobs: run: | coverage run -m pytest -vv coverage report -m + - name: Dump LDAP diagnostics on failure + if: failure() + run: | + docker ps + docker compose -f continuous_integration/docker-configs/ldap-docker-compose.yml ps + LDAP_CONTAINER_ID=$(docker compose -f continuous_integration/docker-configs/ldap-docker-compose.yml ps -q openldap | tr -d '[:space:]') + if [ -n "$LDAP_CONTAINER_ID" ]; then + docker logs --tail 200 "$LDAP_CONTAINER_ID" + else + docker compose -f continuous_integration/docker-configs/ldap-docker-compose.yml logs --tail 200 openldap + fi diff --git a/bluesky_httpserver/tests/test_core_api_main.py b/bluesky_httpserver/tests/test_core_api_main.py index b2b5140..7aa2e22 100644 --- a/bluesky_httpserver/tests/test_core_api_main.py +++ b/bluesky_httpserver/tests/test_core_api_main.py @@ -30,8 +30,17 @@ # Plans used in most of the tests: '_plan1' and '_plan2' are quickly executed '_plan3' runs for 5 seconds. _plan1 = {"name": "count", "args": [["det1", "det2"]], "item_type": "plan"} -_plan2 = {"name": "scan", "args": [["det1", "det2"], "motor", -1, 1, 10], "item_type": "plan"} -_plan3 = {"name": "count", "args": [["det1", "det2"]], "kwargs": {"num": 5, "delay": 1}, "item_type": "plan"} +_plan2 = { + "name": "scan", + "args": [["det1", "det2"], "motor", -1, 1, 10], + "item_type": "plan", +} +_plan3 = { + "name": "count", + "args": [["det1", "det2"]], + "kwargs": {"num": 5, "delay": 1}, + "item_type": "plan", +} _instruction_stop = {"name": "queue_stop", "item_type": "instruction"} @@ -515,8 +524,10 @@ def test_http_server_queue_item_update_2_fail(re_manager, fastapi_server, replac resp2 = request_to_json("post", "/queue/item/update", json=params) assert resp2["success"] is False - assert resp2["msg"] == "Failed to add an item: Failed to replace item: " \ - "Item with UID 'incorrect_uid' is not in the queue" + assert ( + resp2["msg"] == "Failed to add an item: Failed to replace item: " + "Item with UID 'incorrect_uid' is not in the queue" + ) resp3 = request_to_json("get", "/queue/get") assert resp3["items"] != [] @@ -1286,14 +1297,19 @@ def test_http_server_history_clear(re_manager, fastapi_server, clear_params, exp def test_http_server_manager_kill(re_manager, fastapi_server): # noqa F811 + timeout_variants = ( + "Request timeout: ZMQ communication error: timeout occurred", + "Request timeout: ZMQ communication error: Resource temporarily unavailable", + ) + request_to_json("post", "/environment/open") assert wait_for_environment_to_be_created(10), "Timeout" resp = request_to_json("post", "/test/manager/kill") assert "success" not in resp - assert "Request timeout: ZMQ communication error: timeout occurred" in resp["detail"] + assert any(_ in resp["detail"] for _ in timeout_variants) - ttime.sleep(10) + assert wait_for_manager_state_idle(20), "Timeout" resp = request_to_json("get", "/status") assert resp["msg"].startswith("RE Manager") diff --git a/continuous_integration/scripts/start_LDAP.sh b/continuous_integration/scripts/start_LDAP.sh index 897e9c6..ad10257 100755 --- a/continuous_integration/scripts/start_LDAP.sh +++ b/continuous_integration/scripts/start_LDAP.sh @@ -64,6 +64,23 @@ wait_for_ldap_bind() { return 1 } +print_ldap_diagnostics() { + local container_id="${1:-}" + + echo "LDAP startup diagnostics:" >&2 + compose_cmd ps >&2 || true + + if [[ -z "$container_id" ]]; then + container_id="$(get_openldap_container_id)" + fi + + if [[ -n "$container_id" ]]; then + docker logs --tail 200 "$container_id" >&2 || true + else + compose_cmd logs --tail 200 openldap >&2 || true + fi +} + ldap_entry_exists() { local container_id="$1" local dn="$2" @@ -120,12 +137,24 @@ userPassword: password2" # Start LDAP server in docker container compose_cmd up -d -wait_for_ldap 90 CONTAINER_ID="$(get_openldap_container_id)" if [[ -z "$CONTAINER_ID" ]]; then echo "Unable to determine LDAP container id from compose project." >&2 + print_ldap_diagnostics exit 1 fi -wait_for_ldap_bind "$CONTAINER_ID" 90 + +if ! wait_for_ldap 120; then + echo "LDAP port ${LDAP_HOST}:${LDAP_PORT} did not become reachable in time." >&2 + print_ldap_diagnostics "$CONTAINER_ID" + exit 1 +fi + +if ! wait_for_ldap_bind "$CONTAINER_ID" 120; then + echo "LDAP admin bind did not become ready in time." >&2 + print_ldap_diagnostics "$CONTAINER_ID" + exit 1 +fi + seed_ldap_test_users "$CONTAINER_ID" docker ps From 817bc5ced35d864282aad21dbf77399ffd69937d Mon Sep 17 00:00:00 2001 From: David Pastl Date: Tue, 17 Mar 2026 14:52:09 -0600 Subject: [PATCH 08/15] Next test of unit test runners --- bluesky_httpserver/tests/test_core_api_main.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/bluesky_httpserver/tests/test_core_api_main.py b/bluesky_httpserver/tests/test_core_api_main.py index 7aa2e22..0c471bd 100644 --- a/bluesky_httpserver/tests/test_core_api_main.py +++ b/bluesky_httpserver/tests/test_core_api_main.py @@ -1309,9 +1309,21 @@ def test_http_server_manager_kill(re_manager, fastapi_server): # noqa F811 assert "success" not in resp assert any(_ in resp["detail"] for _ in timeout_variants) - assert wait_for_manager_state_idle(20), "Timeout" + deadline = ttime.time() + 20 + last_status = None + while ttime.time() < deadline: + ttime.sleep(0.2) + last_status = request_to_json("get", "/status") + if ( + isinstance(last_status, dict) + and last_status.get("manager_state") == "idle" + and last_status.get("worker_environment_exists") is True + ): + break + else: + assert False, f"Timeout while waiting for manager recovery after kill. Last status: {last_status!r}" - resp = request_to_json("get", "/status") + resp = last_status assert resp["msg"].startswith("RE Manager") assert resp["manager_state"] == "idle" assert resp["items_in_queue"] == 0 From b7db23b0a74bde85666e468c87d84cf3c4d0368e Mon Sep 17 00:00:00 2001 From: David Pastl Date: Wed, 18 Mar 2026 09:35:07 -0600 Subject: [PATCH 09/15] Another test of fixing the runners --- .../tests/test_console_output.py | 127 ++++++++++++------ 1 file changed, 84 insertions(+), 43 deletions(-) diff --git a/bluesky_httpserver/tests/test_console_output.py b/bluesky_httpserver/tests/test_console_output.py index 1f089ec..052ba0d 100644 --- a/bluesky_httpserver/tests/test_console_output.py +++ b/bluesky_httpserver/tests/test_console_output.py @@ -3,6 +3,7 @@ import re import threading import time as ttime +from typing import Any import pytest import requests @@ -36,37 +37,42 @@ def __init__(self, api_key=API_KEY_FOR_TESTS, **kwargs): self._api_key = api_key def run(self): - kwargs = {"stream": True} + kwargs: dict[str, Any] = {"stream": True} if self._api_key: - auth = None headers = {"Authorization": f"ApiKey {self._api_key}"} - kwargs.update({"auth": auth, "headers": headers}) + kwargs.update({"headers": headers}) - with requests.get(f"http://{SERVER_ADDRESS}:{SERVER_PORT}/api/stream_console_output", **kwargs) as r: - r.encoding = "utf-8" + kwargs["timeout"] = (5, 1) - characters = [] - n_brackets = 0 + while not self._exit: + try: + with requests.get( + f"http://{SERVER_ADDRESS}:{SERVER_PORT}/api/stream_console_output", + **kwargs, + ) as r: + r.encoding = "utf-8" - for ch in r.iter_content(decode_unicode=True): - # Note, that some output must be received from the server before the loop exits - if self._exit: - break + characters = [] + n_brackets = 0 - characters.append(ch) - if ch == "{": - n_brackets += 1 - elif ch == "}": - n_brackets -= 1 + for ch in r.iter_content(decode_unicode=True): + if self._exit: + return - # If the received buffer ('characters') is not empty and the message contains - # equal number of opening and closing brackets then consider the message complete. - if characters and not n_brackets: - line = "".join(characters) - characters = [] + characters.append(ch) + if ch == "{": + n_brackets += 1 + elif ch == "}": + n_brackets -= 1 + + if characters and not n_brackets: + line = "".join(characters) + characters = [] - print(f"{line}") - self.received_data_buffer.append(json.loads(line)) + print(f"{line}") + self.received_data_buffer.append(json.loads(line)) + except requests.exceptions.ReadTimeout: + continue def stop(self): """ @@ -81,7 +87,10 @@ def __del__(self): @pytest.mark.parametrize("zmq_port", (None, 60619)) def test_http_server_stream_console_output_1( - monkeypatch, re_manager_cmd, fastapi_server_fs, zmq_port # noqa F811 + monkeypatch, + re_manager_cmd, + fastapi_server_fs, + zmq_port, # noqa F811 ): """ Test for ``stream_console_output`` API @@ -103,7 +112,9 @@ def test_http_server_stream_console_output_1( resp1 = request_to_json( "post", "/queue/item/add", - json={"item": {"name": "count", "args": [["det1", "det2"]], "item_type": "plan"}}, + json={ + "item": {"name": "count", "args": [["det1", "det2"]], "item_type": "plan"} + }, ) assert resp1["success"] is True assert resp1["qsize"] == 1 @@ -122,7 +133,10 @@ def test_http_server_stream_console_output_1( assert resp2["items"][0] == resp1["item"] assert resp2["running_item"] == {} - rsc.join() + rsc.join(timeout=10) + assert not rsc.is_alive(), ( + "Timed out waiting for stream_console_output thread to terminate" + ) assert len(rsc.received_data_buffer) >= 2, pprint.pformat(rsc.received_data_buffer) @@ -134,9 +148,9 @@ def test_http_server_stream_console_output_1( if emsg in msg["msg"]: expected_messages.remove(emsg) - assert ( - not expected_messages - ), f"Messages {expected_messages} were not found in captured output: {pprint.pformat(buffer)}" + assert not expected_messages, ( + f"Messages {expected_messages} were not found in captured output: {pprint.pformat(buffer)}" + ) _script1 = r""" @@ -160,7 +174,11 @@ def test_http_server_stream_console_output_1( @pytest.mark.parametrize("zmq_encoding", (None, "json", "msgpack")) @pytest.mark.parametrize("zmq_port", (None, 60619)) def test_http_server_console_output_1( - monkeypatch, re_manager_cmd, fastapi_server_fs, zmq_port, zmq_encoding # noqa F811 + monkeypatch, + re_manager_cmd, + fastapi_server_fs, + zmq_port, + zmq_encoding, # noqa F811 ): """ Test for ``console_output`` API (not a streaming version). @@ -238,7 +256,10 @@ def test_http_server_console_output_1( @pytest.mark.parametrize("zmq_port", (None, 60619)) def test_http_server_console_output_update_1( - monkeypatch, re_manager_cmd, fastapi_server_fs, zmq_port # noqa F811 + monkeypatch, + re_manager_cmd, + fastapi_server_fs, + zmq_port, # noqa F811 ): """ Test for ``console_output`` API (not a streaming version). @@ -270,7 +291,9 @@ def test_http_server_console_output_update_1( assert resp2a["console_output_msgs"] == [] # Download ALL existing messages - resp2b = request_to_json("get", "/console_output_update", json={"last_msg_uid": "ALL"}) + resp2b = request_to_json( + "get", "/console_output_update", json={"last_msg_uid": "ALL"} + ) assert resp2b["success"] is True assert resp2b["msg"] == "" assert resp2b["last_msg_uid"] == last_msg_uid_1 @@ -286,7 +309,9 @@ def test_http_server_console_output_update_1( ttime.sleep(3) assert wait_for_manager_state_idle(timeout=10) - resp4a = request_to_json("get", "/console_output_update", json={"last_msg_uid": last_msg_uid_1}) + resp4a = request_to_json( + "get", "/console_output_update", json={"last_msg_uid": last_msg_uid_1} + ) last_msg_uid_2 = resp4a["last_msg_uid"] assert last_msg_uid_2 != last_msg_uid_1 console_output = resp4a["console_output_msgs"] @@ -294,7 +319,9 @@ def test_http_server_console_output_update_1( assert expected_output in console_output_text assert console_output_text.count(expected_output) == 1 - resp4b = request_to_json("get", "/console_output_update", json={"last_msg_uid": "ALL"}) + resp4b = request_to_json( + "get", "/console_output_update", json={"last_msg_uid": "ALL"} + ) assert resp4b["last_msg_uid"] == last_msg_uid_2 console_output = resp4b["console_output_msgs"] console_output_text = "".join([_["msg"] for _ in console_output]) @@ -311,7 +338,9 @@ def test_http_server_console_output_update_1( assert wait_for_manager_state_idle(timeout=10) # Download the lastest updates - resp6a = request_to_json("get", "/console_output_update", json={"last_msg_uid": last_msg_uid_2}) + resp6a = request_to_json( + "get", "/console_output_update", json={"last_msg_uid": last_msg_uid_2} + ) last_msg_uid_3 = resp6a["last_msg_uid"] assert last_msg_uid_3 != last_msg_uid_2 console_output = resp6a["console_output_msgs"] @@ -321,7 +350,9 @@ def test_http_server_console_output_update_1( # Download the updates using 'old' UID. The script was uploaded twice, so the output should # contain two copies of the printed output - resp6b = request_to_json("get", "/console_output_update", json={"last_msg_uid": last_msg_uid_1}) + resp6b = request_to_json( + "get", "/console_output_update", json={"last_msg_uid": last_msg_uid_1} + ) assert resp6b["last_msg_uid"] == last_msg_uid_3 console_output = resp6b["console_output_msgs"] console_output_text = "".join([_["msg"] for _ in console_output]) @@ -329,7 +360,9 @@ def test_http_server_console_output_update_1( assert console_output_text.count(expected_output) == 2 # No updates are expected since last request - resp6c = request_to_json("get", "/console_output_update", json={"last_msg_uid": last_msg_uid_3}) + resp6c = request_to_json( + "get", "/console_output_update", json={"last_msg_uid": last_msg_uid_3} + ) assert resp6c["last_msg_uid"] == last_msg_uid_3 assert resp6c["console_output_msgs"] == [] @@ -379,7 +412,10 @@ def __del__(self): @pytest.mark.parametrize("zmq_port", (None, 60619)) def test_http_server_console_output_socket_1( - monkeypatch, re_manager_cmd, fastapi_server_fs, zmq_port # noqa F811 + monkeypatch, + re_manager_cmd, + fastapi_server_fs, + zmq_port, # noqa F811 ): """ Test for ``/console_output/ws`` websocket @@ -402,7 +438,9 @@ def test_http_server_console_output_socket_1( resp1 = request_to_json( "post", "/queue/item/add", - json={"item": {"name": "count", "args": [["det1", "det2"]], "item_type": "plan"}}, + json={ + "item": {"name": "count", "args": [["det1", "det2"]], "item_type": "plan"} + }, ) assert resp1["success"] is True assert resp1["qsize"] == 1 @@ -421,7 +459,10 @@ def test_http_server_console_output_socket_1( assert resp2["items"][0] == resp1["item"] assert resp2["running_item"] == {} - rsc.join() + rsc.join(timeout=10) + assert not rsc.is_alive(), ( + "Timed out waiting for console_output websocket thread to terminate" + ) assert len(rsc.received_data_buffer) >= 2, pprint.pformat(rsc.received_data_buffer) @@ -433,6 +474,6 @@ def test_http_server_console_output_socket_1( if emsg in msg["msg"]: expected_messages.remove(emsg) - assert ( - not expected_messages - ), f"Messages {expected_messages} were not found in captured output: {pprint.pformat(buffer)}" + assert not expected_messages, ( + f"Messages {expected_messages} were not found in captured output: {pprint.pformat(buffer)}" + ) From 9285c2bf4b5865fe5842a66179af124b6bdcf655 Mon Sep 17 00:00:00 2001 From: David Pastl Date: Wed, 18 Mar 2026 10:22:18 -0600 Subject: [PATCH 10/15] Fixing up pre-commit errors --- .../tests/test_console_output.py | 54 ++++++------------- 1 file changed, 16 insertions(+), 38 deletions(-) diff --git a/bluesky_httpserver/tests/test_console_output.py b/bluesky_httpserver/tests/test_console_output.py index 052ba0d..6193db0 100644 --- a/bluesky_httpserver/tests/test_console_output.py +++ b/bluesky_httpserver/tests/test_console_output.py @@ -7,14 +7,12 @@ import pytest import requests -from bluesky_queueserver.manager.tests.common import re_manager_cmd # noqa F401 from websockets.sync.client import connect from bluesky_httpserver.tests.conftest import ( # noqa F401 API_KEY_FOR_TESTS, SERVER_ADDRESS, SERVER_PORT, - fastapi_server_fs, request_to_json, set_qserver_zmq_encoding, wait_for_environment_to_be_closed, @@ -112,9 +110,7 @@ def test_http_server_stream_console_output_1( resp1 = request_to_json( "post", "/queue/item/add", - json={ - "item": {"name": "count", "args": [["det1", "det2"]], "item_type": "plan"} - }, + json={"item": {"name": "count", "args": [["det1", "det2"]], "item_type": "plan"}}, ) assert resp1["success"] is True assert resp1["qsize"] == 1 @@ -134,9 +130,7 @@ def test_http_server_stream_console_output_1( assert resp2["running_item"] == {} rsc.join(timeout=10) - assert not rsc.is_alive(), ( - "Timed out waiting for stream_console_output thread to terminate" - ) + assert not rsc.is_alive(), "Timed out waiting for stream_console_output thread to terminate" assert len(rsc.received_data_buffer) >= 2, pprint.pformat(rsc.received_data_buffer) @@ -148,9 +142,9 @@ def test_http_server_stream_console_output_1( if emsg in msg["msg"]: expected_messages.remove(emsg) - assert not expected_messages, ( - f"Messages {expected_messages} were not found in captured output: {pprint.pformat(buffer)}" - ) + assert ( + not expected_messages + ), f"Messages {expected_messages} were not found in captured output: {pprint.pformat(buffer)}" _script1 = r""" @@ -291,9 +285,7 @@ def test_http_server_console_output_update_1( assert resp2a["console_output_msgs"] == [] # Download ALL existing messages - resp2b = request_to_json( - "get", "/console_output_update", json={"last_msg_uid": "ALL"} - ) + resp2b = request_to_json("get", "/console_output_update", json={"last_msg_uid": "ALL"}) assert resp2b["success"] is True assert resp2b["msg"] == "" assert resp2b["last_msg_uid"] == last_msg_uid_1 @@ -309,9 +301,7 @@ def test_http_server_console_output_update_1( ttime.sleep(3) assert wait_for_manager_state_idle(timeout=10) - resp4a = request_to_json( - "get", "/console_output_update", json={"last_msg_uid": last_msg_uid_1} - ) + resp4a = request_to_json("get", "/console_output_update", json={"last_msg_uid": last_msg_uid_1}) last_msg_uid_2 = resp4a["last_msg_uid"] assert last_msg_uid_2 != last_msg_uid_1 console_output = resp4a["console_output_msgs"] @@ -319,9 +309,7 @@ def test_http_server_console_output_update_1( assert expected_output in console_output_text assert console_output_text.count(expected_output) == 1 - resp4b = request_to_json( - "get", "/console_output_update", json={"last_msg_uid": "ALL"} - ) + resp4b = request_to_json("get", "/console_output_update", json={"last_msg_uid": "ALL"}) assert resp4b["last_msg_uid"] == last_msg_uid_2 console_output = resp4b["console_output_msgs"] console_output_text = "".join([_["msg"] for _ in console_output]) @@ -338,9 +326,7 @@ def test_http_server_console_output_update_1( assert wait_for_manager_state_idle(timeout=10) # Download the lastest updates - resp6a = request_to_json( - "get", "/console_output_update", json={"last_msg_uid": last_msg_uid_2} - ) + resp6a = request_to_json("get", "/console_output_update", json={"last_msg_uid": last_msg_uid_2}) last_msg_uid_3 = resp6a["last_msg_uid"] assert last_msg_uid_3 != last_msg_uid_2 console_output = resp6a["console_output_msgs"] @@ -350,9 +336,7 @@ def test_http_server_console_output_update_1( # Download the updates using 'old' UID. The script was uploaded twice, so the output should # contain two copies of the printed output - resp6b = request_to_json( - "get", "/console_output_update", json={"last_msg_uid": last_msg_uid_1} - ) + resp6b = request_to_json("get", "/console_output_update", json={"last_msg_uid": last_msg_uid_1}) assert resp6b["last_msg_uid"] == last_msg_uid_3 console_output = resp6b["console_output_msgs"] console_output_text = "".join([_["msg"] for _ in console_output]) @@ -360,9 +344,7 @@ def test_http_server_console_output_update_1( assert console_output_text.count(expected_output) == 2 # No updates are expected since last request - resp6c = request_to_json( - "get", "/console_output_update", json={"last_msg_uid": last_msg_uid_3} - ) + resp6c = request_to_json("get", "/console_output_update", json={"last_msg_uid": last_msg_uid_3}) assert resp6c["last_msg_uid"] == last_msg_uid_3 assert resp6c["console_output_msgs"] == [] @@ -438,9 +420,7 @@ def test_http_server_console_output_socket_1( resp1 = request_to_json( "post", "/queue/item/add", - json={ - "item": {"name": "count", "args": [["det1", "det2"]], "item_type": "plan"} - }, + json={"item": {"name": "count", "args": [["det1", "det2"]], "item_type": "plan"}}, ) assert resp1["success"] is True assert resp1["qsize"] == 1 @@ -460,9 +440,7 @@ def test_http_server_console_output_socket_1( assert resp2["running_item"] == {} rsc.join(timeout=10) - assert not rsc.is_alive(), ( - "Timed out waiting for console_output websocket thread to terminate" - ) + assert not rsc.is_alive(), "Timed out waiting for console_output websocket thread to terminate" assert len(rsc.received_data_buffer) >= 2, pprint.pformat(rsc.received_data_buffer) @@ -474,6 +452,6 @@ def test_http_server_console_output_socket_1( if emsg in msg["msg"]: expected_messages.remove(emsg) - assert not expected_messages, ( - f"Messages {expected_messages} were not found in captured output: {pprint.pformat(buffer)}" - ) + assert ( + not expected_messages + ), f"Messages {expected_messages} were not found in captured output: {pprint.pformat(buffer)}" From 3e78a302b7acb4734394f8791ed959b7b9b40164 Mon Sep 17 00:00:00 2001 From: David Pastl Date: Wed, 18 Mar 2026 15:40:14 -0600 Subject: [PATCH 11/15] More unit test reliability changes --- .github/workflows/testing.yml | 4 ++- .../tests/test_console_output.py | 6 +++++ continuous_integration/scripts/start_LDAP.sh | 26 +++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e9500c3..d32c42c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -26,7 +26,7 @@ jobs: - name: Fetch tags run: git fetch --tags --prune --unshallow - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - uses: shogo82148/actions-setup-redis@v1 @@ -64,6 +64,8 @@ jobs: pip list - name: Test with pytest + env: + PYTEST_ADDOPTS: "--durations=20" run: | coverage run -m pytest -vv coverage report -m diff --git a/bluesky_httpserver/tests/test_console_output.py b/bluesky_httpserver/tests/test_console_output.py index 6193db0..a60a2cc 100644 --- a/bluesky_httpserver/tests/test_console_output.py +++ b/bluesky_httpserver/tests/test_console_output.py @@ -7,6 +7,7 @@ import pytest import requests +from bluesky_queueserver.manager.tests.common import re_manager_cmd as _re_manager_cmd from websockets.sync.client import connect from bluesky_httpserver.tests.conftest import ( # noqa F401 @@ -83,6 +84,11 @@ def __del__(self): self.stop() +@pytest.fixture +def re_manager_cmd(): + return _re_manager_cmd + + @pytest.mark.parametrize("zmq_port", (None, 60619)) def test_http_server_stream_console_output_1( monkeypatch, diff --git a/continuous_integration/scripts/start_LDAP.sh b/continuous_integration/scripts/start_LDAP.sh index ad10257..0f6a787 100755 --- a/continuous_integration/scripts/start_LDAP.sh +++ b/continuous_integration/scripts/start_LDAP.sh @@ -64,6 +64,25 @@ wait_for_ldap_bind() { return 1 } +wait_for_ldap_test_user_bind() { + local container_id="$1" + local timeout_seconds="${2:-60}" + local deadline=$((SECONDS + timeout_seconds)) + + while (( SECONDS < deadline )); do + if docker exec "$container_id" ldapwhoami \ + -x \ + -H "ldap://127.0.0.1:389" \ + -D "cn=user01,ou=users,$LDAP_BASE_DN" \ + -w "password1" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + + return 1 +} + print_ldap_diagnostics() { local container_id="${1:-}" @@ -157,4 +176,11 @@ if ! wait_for_ldap_bind "$CONTAINER_ID" 120; then fi seed_ldap_test_users "$CONTAINER_ID" + +if ! wait_for_ldap_test_user_bind "$CONTAINER_ID" 60; then + echo "LDAP test-user bind did not become ready in time." >&2 + print_ldap_diagnostics "$CONTAINER_ID" + exit 1 +fi + docker ps From ef413a30aafbf60adacc4c580cbb19e080649724 Mon Sep 17 00:00:00 2001 From: David Pastl Date: Thu, 19 Mar 2026 09:49:16 -0600 Subject: [PATCH 12/15] More attempts to get the unit tests reliable --- bluesky_httpserver/tests/conftest.py | 23 ++++++++++++++++--- .../tests/test_console_output.py | 7 ++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/bluesky_httpserver/tests/conftest.py b/bluesky_httpserver/tests/conftest.py index d5cafdb..8a81df9 100644 --- a/bluesky_httpserver/tests/conftest.py +++ b/bluesky_httpserver/tests/conftest.py @@ -4,6 +4,7 @@ import pytest import requests from bluesky_queueserver.manager.comms import zmq_single_request +from bluesky_queueserver.manager.tests.common import re_manager_cmd # noqa: F401 from bluesky_queueserver.manager.tests.common import set_qserver_zmq_encoding # noqa: F401 from xprocess import ProcessStarter @@ -60,7 +61,11 @@ def fastapi_server_fs(xprocess): to perform additional steps (such as setting environmental variables) before the server is started. """ - def start(http_server_host=SERVER_ADDRESS, http_server_port=SERVER_PORT, api_key=API_KEY_FOR_TESTS): + def start( + http_server_host=SERVER_ADDRESS, + http_server_port=SERVER_PORT, + api_key=API_KEY_FOR_TESTS, + ): class Starter(ProcessStarter): max_read_lines = 53 @@ -112,7 +117,12 @@ def add_plans_to_queue(): user_group = _user_group user = "HTTP unit test setup" - plan1 = {"name": "count", "args": [["det1", "det2"]], "kwargs": {"num": 10, "delay": 1}, "item_type": "plan"} + plan1 = { + "name": "count", + "args": [["det1", "det2"]], + "kwargs": {"num": 10, "delay": 1}, + "item_type": "plan", + } plan2 = {"name": "count", "args": [["det1", "det2"]], "item_type": "plan"} for plan in (plan1, plan2, plan2): resp2, _ = zmq_single_request("queue_item_add", {"item": plan, "user": user, "user_group": user_group}) @@ -120,7 +130,14 @@ def add_plans_to_queue(): def request_to_json( - request_type, path, *, request_prefix="/api", api_key=API_KEY_FOR_TESTS, token=None, login=None, **kwargs + request_type, + path, + *, + request_prefix="/api", + api_key=API_KEY_FOR_TESTS, + token=None, + login=None, + **kwargs, ): if login: auth = None diff --git a/bluesky_httpserver/tests/test_console_output.py b/bluesky_httpserver/tests/test_console_output.py index a60a2cc..557a973 100644 --- a/bluesky_httpserver/tests/test_console_output.py +++ b/bluesky_httpserver/tests/test_console_output.py @@ -21,6 +21,8 @@ wait_for_manager_state_idle, ) +re_manager_cmd = _re_manager_cmd + class _ReceiveStreamedConsoleOutput(threading.Thread): """ @@ -84,11 +86,6 @@ def __del__(self): self.stop() -@pytest.fixture -def re_manager_cmd(): - return _re_manager_cmd - - @pytest.mark.parametrize("zmq_port", (None, 60619)) def test_http_server_stream_console_output_1( monkeypatch, From a2909cdb4fd3c2064627972b7480c0df13236c81 Mon Sep 17 00:00:00 2001 From: David Pastl Date: Thu, 19 Mar 2026 15:11:35 -0600 Subject: [PATCH 13/15] Another attempt to make them reliable --- .../tests/test_console_output.py | 3 - bluesky_httpserver/tests/test_server.py | 26 +- docs/source/configuration.rst | 2 +- docs/source/contributing.rst | 2 +- docs/source/usage.rst | 4 +- scripts/run-ci-docker-worker-matrix.py | 407 ------------------ scripts/run-github-actions-local.sh | 379 ---------------- 7 files changed, 22 insertions(+), 801 deletions(-) delete mode 100755 scripts/run-ci-docker-worker-matrix.py delete mode 100755 scripts/run-github-actions-local.sh diff --git a/bluesky_httpserver/tests/test_console_output.py b/bluesky_httpserver/tests/test_console_output.py index 557a973..6193db0 100644 --- a/bluesky_httpserver/tests/test_console_output.py +++ b/bluesky_httpserver/tests/test_console_output.py @@ -7,7 +7,6 @@ import pytest import requests -from bluesky_queueserver.manager.tests.common import re_manager_cmd as _re_manager_cmd from websockets.sync.client import connect from bluesky_httpserver.tests.conftest import ( # noqa F401 @@ -21,8 +20,6 @@ wait_for_manager_state_idle, ) -re_manager_cmd = _re_manager_cmd - class _ReceiveStreamedConsoleOutput(threading.Thread): """ diff --git a/bluesky_httpserver/tests/test_server.py b/bluesky_httpserver/tests/test_server.py index 117f4df..33b82a2 100644 --- a/bluesky_httpserver/tests/test_server.py +++ b/bluesky_httpserver/tests/test_server.py @@ -27,8 +27,17 @@ # Plans used in most of the tests: '_plan1' and '_plan2' are quickly executed '_plan3' runs for 5 seconds. _plan1 = {"name": "count", "args": [["det1", "det2"]], "item_type": "plan"} -_plan2 = {"name": "scan", "args": [["det1", "det2"], "motor", -1, 1, 10], "item_type": "plan"} -_plan3 = {"name": "count", "args": [["det1", "det2"]], "kwargs": {"num": 5, "delay": 1}, "item_type": "plan"} +_plan2 = { + "name": "scan", + "args": [["det1", "det2"], "motor", -1, 1, 10], + "item_type": "plan", +} +_plan3 = { + "name": "count", + "args": [["det1", "det2"]], + "kwargs": {"num": 5, "delay": 1}, + "item_type": "plan", +} _config_public_key = """ @@ -122,7 +131,7 @@ def test_http_server_secure_1(monkeypatch, tmpdir, re_manager_cmd, fastapi_serve @pytest.mark.parametrize("option", ["ev", "cfg_file", "both"]) # fmt: on def test_http_server_set_zmq_address_1( - monkeypatch, tmpdir, re_manager_cmd, fastapi_server_fs, option # noqa: F811 + monkeypatch, tmpdir, re_manager_cmd, fastapi_server_fs, free_tcp_port_factory, option # noqa: F811 ): """ Test if ZMQ address of RE Manager is passed to the HTTP server using 'QSERVER_ZMQ_ADDRESS_CONTROL' @@ -130,11 +139,12 @@ def test_http_server_set_zmq_address_1( channel different from default address, add and execute a plan. """ - # Change ZMQ address to use port 60616 instead of the default port 60615. - zmq_control_address_server = "tcp://*:60616" - zmq_info_address_server = "tcp://*:60617" - zmq_control_address = "tcp://localhost:60616" - zmq_info_address = "tcp://localhost:60617" + zmq_control_port = free_tcp_port_factory() + zmq_info_port = free_tcp_port_factory() + zmq_control_address_server = f"tcp://*:{zmq_control_port}" + zmq_info_address_server = f"tcp://*:{zmq_info_port}" + zmq_control_address = f"tcp://localhost:{zmq_control_port}" + zmq_info_address = f"tcp://localhost:{zmq_info_port}" if option == "ev": monkeypatch.setenv("QSERVER_ZMQ_CONTROL_ADDRESS", zmq_control_address) monkeypatch.setenv("QSERVER_ZMQ_INFO_ADDRESS", zmq_info_address) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 3a3d290..8852a31 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -261,7 +261,7 @@ LDAP Authenticator LDAP authenticator is designed for production deployments. The authenticator validates user login information (username/password) by communicating with LDAP server (e.g. active Directory server). The following example illustrates how to configure the server to -use demo OpenLDAP server running in docker container (run ``source continuous_integration/scripts/start_LDAP.sh`` in the root +use demo OpenLDAP server running in docker container (run ``./start_LDAP.sh`` in the root of the repository to start the server). The server is configured to authenticate two users: *'user01'* and *'user02'* with passwords *'password1'* and *'password2'* respectively. The configuration does not enable public access. :: diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 079f0ed..f09326a 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -128,4 +128,4 @@ locally, especially if the respective server code was not changed. The tests wil run on GitHub CI in properly configured environment and indicate if there is an issue. To run these tests locally, start OpenLDAP server in Docker container:: - $ source continuous_integration/scripts/start_LDAP.sh + $ source start_LDAP.sh diff --git a/docs/source/usage.rst b/docs/source/usage.rst index bcae133..299bdcb 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -169,7 +169,7 @@ If you are already in a browser context, open: This redirects to the OIDC provider login page and then back to the server callback. -This can similarly be acheived using ``httpie`` by opening the URL in a browser after getting +This can similarly be acheived using ``httpie`` by opening the URL in a browser after getting the authorization URI from the server:: http POST http://localhost:60610/api/auth/provider/entra/authorize @@ -183,7 +183,7 @@ spawn a browser for the user to log in to the provider. CLI/device flow *************** -For terminal clients (i.e. no browser possible), start with +For terminal clients (i.e. no browser possible), start with ``POST /api/auth/provider//authorize``. The response includes: diff --git a/scripts/run-ci-docker-worker-matrix.py b/scripts/run-ci-docker-worker-matrix.py deleted file mode 100755 index 6771c19..0000000 --- a/scripts/run-ci-docker-worker-matrix.py +++ /dev/null @@ -1,407 +0,0 @@ -#!/usr/bin/env python3 -"""Run bluesky-httpserver CI-style checks using a docker client-worker model. - -This script is designed to mirror the project's GitHub Actions workflows locally, -with special focus on accelerating the pytest matrix by chunking tests and -dispatching chunks to worker containers. - -What it runs by default: -1. Style checks (black/isort/flake8/pre-commit) -2. Docs build check -3. Unit tests for Python 3.10/3.11/3.12/3.13 using worker containers - -Notes: -- The unit-test step follows `.github/workflows/testing.yml` dependency setup. -- Shared service containers (Redis, LDAP) are started once and reused. -- Unit tests are chunked by test file and distributed across workers per version. -""" - -from __future__ import annotations - -import argparse -import math -import os -import queue -import shutil -import subprocess -import sys -import threading -from dataclasses import dataclass -from pathlib import Path -from typing import Iterable, List, Sequence - -DEFAULT_PYTHON_VERSIONS = ["3.10", "3.11", "3.12", "3.13"] -DEFAULT_WORKERS_PER_VERSION = 2 - - -@dataclass -class ChunkResult: - python_version: str - worker_name: str - chunk_index: int - command: str - returncode: int - log_path: Path - - -def run( - command: Sequence[str], *, cwd: Path, env: dict | None = None, check: bool = True -) -> subprocess.CompletedProcess: - return subprocess.run(command, cwd=cwd, env=env, check=check, text=True) - - -def shell(command: str, *, cwd: Path, env: dict | None = None, check: bool = True) -> subprocess.CompletedProcess: - return subprocess.run(command, cwd=cwd, env=env, check=check, text=True, shell=True) - - -def chunk_items(items: List[str], chunks: int) -> List[List[str]]: - if chunks <= 0: - return [items] - if not items: - return [] - chunks = max(1, min(chunks, len(items))) - chunk_size = math.ceil(len(items) / chunks) - return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)] - - -def docker_cmd(*parts: str) -> List[str]: - return ["docker", *parts] - - -def ensure_docker_available(repo_root: Path) -> None: - run(["docker", "info"], cwd=repo_root) - - -def discover_test_files(repo_root: Path) -> List[str]: - test_dir = repo_root / "bluesky_httpserver" / "tests" - files = sorted(str(path.relative_to(repo_root)) for path in test_dir.glob("test_*.py")) - return files - - -def start_redis_container(repo_root: Path, name: str) -> None: - run(docker_cmd("rm", "-f", name), cwd=repo_root, check=False) - run( - docker_cmd( - "run", - "-d", - "--rm", - "--name", - name, - "--network", - "host", - "redis:7", - ), - cwd=repo_root, - ) - - -def start_ldap_compose(repo_root: Path) -> None: - compose_file = repo_root / "continuous_integration" / "docker-configs" / "ldap-docker-compose.yml" - env = os.environ.copy() - env["LDAP_COMPOSE_FILE"] = str(compose_file) - env["LDAP_COMPOSE_PROJECT"] = "bhs-ci-ldap" - env["LDAP_HOST"] = "127.0.0.1" - env["LDAP_PORT"] = "1389" - run( - [ - "bash", - str(repo_root / "continuous_integration" / "scripts" / "start_LDAP.sh"), - ], - cwd=repo_root, - env=env, - ) - - -def stop_ldap_compose(repo_root: Path) -> None: - compose_file = repo_root / "continuous_integration" / "docker-configs" / "ldap-docker-compose.yml" - run( - [ - "docker", - "compose", - "-p", - "bhs-ci-ldap", - "-f", - str(compose_file), - "down", - "-v", - ], - cwd=repo_root, - check=False, - ) - - -def make_worker_name(python_version: str, index: int) -> str: - v = python_version.replace(".", "") - return f"bhs-ci-py{v}-worker{index}" - - -def start_worker_container(repo_root: Path, python_version: str, worker_name: str) -> None: - run(docker_cmd("rm", "-f", worker_name), cwd=repo_root, check=False) - run( - docker_cmd( - "run", - "-d", - "--rm", - "--name", - worker_name, - "--network", - "host", - "-v", - f"{repo_root}:/workspace", - "-w", - "/workspace", - f"python:{python_version}", - "bash", - "-lc", - "sleep infinity", - ), - cwd=repo_root, - ) - - -def exec_in_worker(repo_root: Path, worker_name: str, command: str, *, log_path: Path) -> int: - full_cmd = ["docker", "exec", worker_name, "bash", "-lc", command] - with log_path.open("w", encoding="utf-8") as log_file: - process = subprocess.run( - full_cmd, - cwd=repo_root, - stdout=log_file, - stderr=subprocess.STDOUT, - text=True, - ) - return process.returncode - - -def bootstrap_worker(repo_root: Path, worker_name: str) -> None: - # Mirrors `.github/workflows/testing.yml` install strategy as closely as possible. - install_cmd = " && ".join( - [ - "python -m pip install --upgrade pip setuptools numpy", - "pip install git+https://github.com/bluesky/bluesky-queueserver.git", - "pip install git+https://github.com/bluesky/bluesky-queueserver-api.git", - "pip install .", - "pip install -r requirements-dev.txt", - "pip list", - ] - ) - code = exec_in_worker( - repo_root, - worker_name, - install_cmd, - log_path=repo_root / ".ci-artifacts" / f"{worker_name}-bootstrap.log", - ) - if code != 0: - raise RuntimeError(f"Bootstrap failed for worker {worker_name}") - - -def run_style_and_docs(repo_root: Path, python_version: str) -> None: - worker_name = f"bhs-ci-style-docs-py{python_version.replace('.', '')}" - start_worker_container(repo_root, python_version, worker_name) - try: - bootstrap_worker(repo_root, worker_name) - steps = [ - ("black", "black . --check"), - ("isort", "isort . -c"), - ("flake8", "flake8"), - ("pre-commit", "pre-commit run --all-files"), - ("docs", "make -C docs/ html"), - ] - for label, cmd in steps: - log_path = repo_root / ".ci-artifacts" / f"{worker_name}-{label}.log" - code = exec_in_worker(repo_root, worker_name, cmd, log_path=log_path) - if code != 0: - raise RuntimeError(f"Step '{label}' failed. See {log_path}") - finally: - run(docker_cmd("rm", "-f", worker_name), cwd=repo_root, check=False) - - -def run_test_matrix( - repo_root: Path, - python_versions: Iterable[str], - workers_per_version: int, - include_pattern: str | None, - chunks_per_version: int | None, - tests_per_chunk: int | None, -) -> list[ChunkResult]: - artifacts_dir = repo_root / ".ci-artifacts" - test_files = discover_test_files(repo_root) - if include_pattern: - test_files = [f for f in test_files if include_pattern in f] - - if not test_files: - raise RuntimeError("No test files discovered.") - - all_results: list[ChunkResult] = [] - - for python_version in python_versions: - print(f"\n=== Python {python_version}: preparing workers ===", flush=True) - workers = [make_worker_name(python_version, i + 1) for i in range(workers_per_version)] - - for worker in workers: - start_worker_container(repo_root, python_version, worker) - try: - for worker in workers: - print(f"Bootstrapping {worker} ...", flush=True) - bootstrap_worker(repo_root, worker) - - if tests_per_chunk and tests_per_chunk > 0: - chunks = [test_files[i : i + tests_per_chunk] for i in range(0, len(test_files), tests_per_chunk)] - else: - n_chunks = ( - chunks_per_version - if (chunks_per_version and chunks_per_version > 0) - else workers_per_version * 4 - ) - chunks = chunk_items(test_files, n_chunks) - work_queue: queue.Queue[tuple[int, list[str]]] = queue.Queue() - for idx, chunk in enumerate(chunks): - work_queue.put((idx, chunk)) - - results_lock = threading.Lock() - - def worker_loop(worker_name: str) -> None: - while True: - try: - chunk_index, chunk = work_queue.get_nowait() - except queue.Empty: - return - chunk_args = " ".join(chunk) - command = ( - "QSERVER_TEST_LDAP_HOST=localhost " - "QSERVER_TEST_LDAP_PORT=1389 " - f"coverage run -m pytest -vv {chunk_args}" - ) - log_path = artifacts_dir / f"{worker_name}-chunk{chunk_index:03d}.log" - rc = exec_in_worker(repo_root, worker_name, command, log_path=log_path) - with results_lock: - all_results.append( - ChunkResult( - python_version=python_version, - worker_name=worker_name, - chunk_index=chunk_index, - command=command, - returncode=rc, - log_path=log_path, - ) - ) - work_queue.task_done() - - threads = [threading.Thread(target=worker_loop, args=(worker,), daemon=True) for worker in workers] - for t in threads: - t.start() - for t in threads: - t.join() - - # Per-version coverage summary (best effort, non-fatal) - for worker in workers: - exec_in_worker( - repo_root, - worker, - "coverage report -m || true", - log_path=artifacts_dir / f"{worker}-coverage-report.log", - ) - finally: - for worker in workers: - run(docker_cmd("rm", "-f", worker), cwd=repo_root, check=False) - - return all_results - - -def parse_args(argv: Sequence[str]) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--python-versions", - nargs="+", - default=DEFAULT_PYTHON_VERSIONS, - help="Python versions to test (default mirrors .github/workflows/testing.yml)", - ) - parser.add_argument( - "--workers-per-version", - type=int, - default=DEFAULT_WORKERS_PER_VERSION, - help="Worker containers per Python version", - ) - parser.add_argument( - "--chunks-per-version", - type=int, - default=None, - help="Total pytest chunks per Python version (default: workers_per_version * 4)", - ) - parser.add_argument( - "--tests-per-chunk", - type=int, - default=None, - help="Number of test files per chunk (overrides --chunks-per-version)", - ) - parser.add_argument( - "--include-pattern", - default=None, - help="Only run test files containing this substring", - ) - parser.add_argument( - "--skip-style-docs", - action="store_true", - help="Skip style/docs checks and run only the test matrix", - ) - parser.add_argument( - "--keep-artifacts", - action="store_true", - help="Keep .ci-artifacts from previous runs (default clears first)", - ) - return parser.parse_args(argv) - - -def main(argv: Sequence[str]) -> int: - args = parse_args(argv) - repo_root = Path(__file__).resolve().parents[1] - artifacts_dir = repo_root / ".ci-artifacts" - - ensure_docker_available(repo_root) - - if artifacts_dir.exists() and not args.keep_artifacts: - shutil.rmtree(artifacts_dir) - artifacts_dir.mkdir(parents=True, exist_ok=True) - - redis_name = "bhs-ci-redis" - start_redis_container(repo_root, redis_name) - start_ldap_compose(repo_root) - - try: - if not args.skip_style_docs: - print( - "\n=== Running style/docs checks (CI parity for non-matrix workflows) ===", - flush=True, - ) - run_style_and_docs(repo_root, "3.12") - - print("\n=== Running chunked pytest matrix ===", flush=True) - results = run_test_matrix( - repo_root, - python_versions=args.python_versions, - workers_per_version=args.workers_per_version, - include_pattern=args.include_pattern, - chunks_per_version=args.chunks_per_version, - tests_per_chunk=args.tests_per_chunk, - ) - - failed = [r for r in results if r.returncode != 0] - print("\n=== Summary ===", flush=True) - print(f"Total chunks run: {len(results)}", flush=True) - print(f"Failed chunks: {len(failed)}", flush=True) - if failed: - for r in failed: - print( - f"[FAIL] py={r.python_version} worker={r.worker_name} chunk={r.chunk_index} " - f"log={r.log_path}", - flush=True, - ) - return 1 - print("All CI-equivalent checks passed.", flush=True) - return 0 - finally: - stop_ldap_compose(repo_root) - run(docker_cmd("rm", "-f", redis_name), cwd=repo_root, check=False) - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/run-github-actions-local.sh b/scripts/run-github-actions-local.sh deleted file mode 100755 index 70f9741..0000000 --- a/scripts/run-github-actions-local.sh +++ /dev/null @@ -1,379 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(git rev-parse --show-toplevel 2>/dev/null || true)" -if [[ -z "${ROOT_DIR}" ]]; then - echo "Error: not inside a git repository." >&2 - exit 1 -fi - -cd "${ROOT_DIR}" - -INSTALL_DEPS=0 -SKIP_LDAP=0 -TARGETS_RAW="all" -PYTHON_BIN="${PYTHON_BIN:-python}" -ENV_MODE="auto" # auto | uv | system -USE_UV=0 -UV_PROJECT_PYTHON=".venv/bin/python" -UV_PROJECT_ENVIRONMENT=".venv" - -LDAP_COMPOSE_FILE="continuous_integration/docker-configs/ldap-docker-compose.yml" - -print_help() { - cat <<'EOF' -Run local equivalents of the GitHub Actions checks. - -Usage: - ./scripts/run-github-actions-local.sh [options] - -Options: - --targets Comma-separated targets: - black,isort,flake8,pre-commit,unit,docs,all - (default: all) - --install-deps Install/upgrade dependencies before running - (auto-runs when required deps are missing) - --env Environment mode (default: auto) - auto: use uv if available, otherwise system - --skip-ldap Do not start LDAP for unit tests - --python Python executable (default: python) - -h, --help Show this help - -Examples: - ./scripts/run-github-actions-local.sh - ./scripts/run-github-actions-local.sh --targets black,isort,flake8,pre-commit - ./scripts/run-github-actions-local.sh --targets unit --install-deps - ./scripts/run-github-actions-local.sh --env uv --targets black,unit,docs -EOF -} - -log_section() { - echo - echo "==> $*" -} - -run_cmd() { - echo "+ $*" - "$@" -} - -run_local_cmd() { - if [[ ${USE_UV} -eq 1 ]]; then - run_cmd uv run --python "${UV_PROJECT_PYTHON}" "$@" - else - run_cmd "$@" - fi -} - -has_cmd() { - command -v "$1" >/dev/null 2>&1 -} - -install_dependencies() { - log_section "Installing dependencies" - if [[ ${USE_UV} -eq 1 ]]; then - if [[ ! -x "${UV_PROJECT_PYTHON}" ]]; then - echo "Error: uv mode requires an existing project environment at ${UV_PROJECT_PYTHON}." >&2 - echo "Create it once with: uv venv" >&2 - exit 1 - fi - run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" --upgrade pip setuptools numpy - run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" -e . - run_cmd uv pip install --python "${UV_PROJECT_PYTHON}" -r requirements-dev.txt - else - run_cmd "${PYTHON_BIN}" -m pip install --upgrade pip setuptools numpy - run_cmd "${PYTHON_BIN}" -m pip install -e . - run_cmd "${PYTHON_BIN}" -m pip install -r requirements-dev.txt - fi -} - -start_ldap_if_needed() { - if [[ ${SKIP_LDAP} -eq 1 ]]; then - log_section "Skipping LDAP startup (--skip-ldap)" - return - fi - if [[ ! -f "${LDAP_COMPOSE_FILE}" ]]; then - echo "Warning: LDAP compose file not found at ${LDAP_COMPOSE_FILE}; continuing without startup." >&2 - return - fi - log_section "Starting LDAP service for LDAP-related tests" - run_cmd bash continuous_integration/scripts/start_LDAP.sh -} - -run_local_target() { - local target="$1" - case "${target}" in - black) - log_section "Running BLACK (local)" - run_local_cmd black . --check - ;; - isort) - log_section "Running ISORT (local)" - run_local_cmd isort . -c - ;; - flake8) - log_section "Running FLAKE8 (local)" - run_local_cmd flake8 - ;; - pre-commit) - log_section "Running pre-commit (local)" - run_local_cmd pre-commit run --all-files - ;; - unit) - log_section "Running unit tests (local)" - start_ldap_if_needed - run_local_cmd coverage run -m pytest -vv - run_local_cmd coverage report -m - ;; - docs) - log_section "Building docs (local)" - run_local_cmd make -C docs/ html - ;; - *) - echo "Error: unknown local target '${target}'" >&2 - exit 2 - ;; - esac -} - -normalize_targets() { - local raw="$1" - raw="${raw//testing/unit}" - if [[ "${raw}" == "all" ]]; then - echo "black isort flake8 pre-commit unit docs" - return - fi - - local csv="${raw//,/ }" - local out=() - local item - for item in ${csv}; do - case "${item}" in - black|isort|flake8|pre-commit|unit|docs) - out+=("${item}") - ;; - testing) - out+=("unit") - ;; - *) - echo "Error: unknown target '${item}'." >&2 - exit 2 - ;; - esac - done - - if [[ ${#out[@]} -eq 0 ]]; then - echo "Error: no targets specified." >&2 - exit 2 - fi - - echo "${out[*]}" -} - -detect_environment_mode() { - case "${ENV_MODE}" in - system) - USE_UV=0 - ;; - uv) - if ! has_cmd uv; then - echo "Error: --env uv requested but 'uv' is not installed or not on PATH." >&2 - exit 1 - fi - if [[ ! -x "${UV_PROJECT_PYTHON}" ]]; then - echo "Error: --env uv requested but ${UV_PROJECT_PYTHON} does not exist." >&2 - echo "Create it once with: uv venv" >&2 - exit 1 - fi - USE_UV=1 - ;; - auto) - if has_cmd uv; then - if [[ -x "${UV_PROJECT_PYTHON}" ]]; then - USE_UV=1 - else - USE_UV=0 - fi - else - USE_UV=0 - fi - ;; - *) - echo "Error: --env must be one of: auto, uv, system" >&2 - exit 2 - ;; - esac -} - -missing_local_tools() { - # In uv mode, commands are launched via 'uv run', so PATH checks are not useful. - if [[ ${USE_UV} -eq 1 ]]; then - return - fi - - local -a targets=("$@") - local -a required_tools=() - local target - - for target in "${targets[@]}"; do - case "${target}" in - black) - required_tools+=("black") - ;; - isort) - required_tools+=("isort") - ;; - flake8) - required_tools+=("flake8") - ;; - pre-commit) - required_tools+=("pre-commit") - ;; - unit) - required_tools+=("coverage" "pytest") - ;; - docs) - required_tools+=("make") - ;; - esac - done - - local -a missing=() - local tool - for tool in "${required_tools[@]}"; do - if ! has_cmd "${tool}"; then - missing+=("${tool}") - fi - done - - if [[ ${#missing[@]} -gt 0 ]]; then - echo "${missing[*]}" - fi -} - -uv_env_needs_bootstrap() { - local -a targets=("$@") - local module_check_script="import importlib.util as u\nmodules=[]\n" - - local target - for target in "${targets[@]}"; do - case "${target}" in - black) - module_check_script+="modules.append('black')\n" - ;; - isort) - module_check_script+="modules.append('isort')\n" - ;; - flake8) - module_check_script+="modules.append('flake8')\n" - ;; - pre-commit) - module_check_script+="modules.append('pre_commit')\n" - ;; - unit) - module_check_script+="modules.extend(['pytest', 'coverage'])\n" - ;; - docs) - module_check_script+="modules.append('sphinx')\n" - ;; - esac - done - module_check_script+="missing=[m for m in modules if u.find_spec(m) is None]\n" - module_check_script+="raise SystemExit(1 if missing else 0)\n" - - if [[ ! -x "${UV_PROJECT_PYTHON}" ]]; then - return 0 - fi - - if uv run --python "${UV_PROJECT_PYTHON}" python -c "${module_check_script}" >/dev/null 2>&1; then - return 1 - fi - return 0 -} - -while [[ $# -gt 0 ]]; do - case "$1" in - --targets) - TARGETS_RAW="$2" - shift 2 - ;; - --install-deps) - INSTALL_DEPS=1 - shift - ;; - --env) - ENV_MODE="$2" - shift 2 - ;; - --skip-ldap) - SKIP_LDAP=1 - shift - ;; - --python) - PYTHON_BIN="$2" - shift 2 - ;; - -h|--help) - print_help - exit 0 - ;; - *) - echo "Unknown option: $1" >&2 - print_help - exit 2 - ;; - esac -done - -IFS=' ' read -r -a TARGETS <<< "$(normalize_targets "${TARGETS_RAW}")" -detect_environment_mode - -log_section "Configuration" -echo "Repository: ${ROOT_DIR}" -echo "Mode: local" -echo "Targets: ${TARGETS[*]}" -if [[ ${USE_UV} -eq 1 ]]; then - export UV_PROJECT_ENVIRONMENT="${UV_PROJECT_ENVIRONMENT}" - echo "Environment: uv (${UV_PROJECT_PYTHON})" -else - echo "Environment: system (${PYTHON_BIN})" -fi - -if [[ ${INSTALL_DEPS} -eq 1 ]]; then - install_dependencies -fi - -if [[ ${USE_UV} -eq 1 ]]; then - if uv_env_needs_bootstrap "${TARGETS[@]}"; then - log_section "uv environment missing required packages" - echo "Auto-installing dependencies (equivalent to --install-deps)." - install_dependencies - fi -else - MISSING_TOOLS="$(missing_local_tools "${TARGETS[@]}" || true)" - if [[ -n "${MISSING_TOOLS}" ]]; then - log_section "Missing local tools detected" - echo "Missing: ${MISSING_TOOLS}" - echo "Auto-installing dependencies (equivalent to --install-deps)." - install_dependencies - fi -fi - -FAILURES=() - -for target in "${TARGETS[@]}"; do - if ! run_local_target "${target}"; then - FAILURES+=("${target}") - log_section "Target failed" - echo "Failed target: ${target}" - fi -done - -if [[ ${#FAILURES[@]} -gt 0 ]]; then - log_section "Summary" - echo "Completed with failures in: ${FAILURES[*]}" - exit 1 -fi - -log_section "Done" -echo "All requested checks completed." From 210d12c68b8b1cc466697a7992a4913ef1d901d4 Mon Sep 17 00:00:00 2001 From: David Pastl Date: Thu, 19 Mar 2026 15:13:49 -0600 Subject: [PATCH 14/15] Fixing pre-commit --- docs/source/usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 299bdcb..bcae133 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -169,7 +169,7 @@ If you are already in a browser context, open: This redirects to the OIDC provider login page and then back to the server callback. -This can similarly be acheived using ``httpie`` by opening the URL in a browser after getting +This can similarly be acheived using ``httpie`` by opening the URL in a browser after getting the authorization URI from the server:: http POST http://localhost:60610/api/auth/provider/entra/authorize @@ -183,7 +183,7 @@ spawn a browser for the user to log in to the provider. CLI/device flow *************** -For terminal clients (i.e. no browser possible), start with +For terminal clients (i.e. no browser possible), start with ``POST /api/auth/provider//authorize``. The response includes: From 4de7d915b4c8e7330a89f5c7fa780ad1cdb0577c Mon Sep 17 00:00:00 2001 From: David Pastl Date: Fri, 20 Mar 2026 08:19:08 -0600 Subject: [PATCH 15/15] More changes --- .github/workflows/testing.yml | 2 +- continuous_integration/scripts/start_LDAP.sh | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d32c42c..adef4fc 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -37,7 +37,7 @@ jobs: # sudo apt install redis # Start LDAP - source continuous_integration/scripts/start_LDAP.sh + bash continuous_integration/scripts/start_LDAP.sh # These packages are installed in the base environment but may be older # versions. Explicitly upgrade them because they often create diff --git a/continuous_integration/scripts/start_LDAP.sh b/continuous_integration/scripts/start_LDAP.sh index 0f6a787..d2bd48d 100755 --- a/continuous_integration/scripts/start_LDAP.sh +++ b/continuous_integration/scripts/start_LDAP.sh @@ -46,16 +46,19 @@ wait_for_ldap_bind() { local container_id="$1" local timeout_seconds="${2:-60}" local deadline=$((SECONDS + timeout_seconds)) + local rc=0 while (( SECONDS < deadline )); do - if docker exec "$container_id" ldapsearch \ + rc=0 + docker exec "$container_id" ldapsearch \ -x \ -H "ldap://127.0.0.1:389" \ -D "$LDAP_ADMIN_DN" \ -w "$LDAP_ADMIN_PASSWORD" \ -b "$LDAP_BASE_DN" \ -s base \ - "(objectclass=*)" dn >/dev/null 2>&1; then + "(objectclass=*)" dn >/dev/null 2>&1 || rc=$? + if [[ "$rc" -eq 0 ]]; then return 0 fi sleep 1 @@ -68,13 +71,16 @@ wait_for_ldap_test_user_bind() { local container_id="$1" local timeout_seconds="${2:-60}" local deadline=$((SECONDS + timeout_seconds)) + local rc=0 while (( SECONDS < deadline )); do - if docker exec "$container_id" ldapwhoami \ + rc=0 + docker exec "$container_id" ldapwhoami \ -x \ -H "ldap://127.0.0.1:389" \ -D "cn=user01,ou=users,$LDAP_BASE_DN" \ - -w "password1" >/dev/null 2>&1; then + -w "password1" >/dev/null 2>&1 || rc=$? + if [[ "$rc" -eq 0 ]]; then return 0 fi sleep 1 @@ -169,6 +175,9 @@ if ! wait_for_ldap 120; then exit 1 fi +echo "LDAP port ${LDAP_HOST}:${LDAP_PORT} is reachable. Waiting for slapd initialization..." +sleep 3 + if ! wait_for_ldap_bind "$CONTAINER_ID" 120; then echo "LDAP admin bind did not become ready in time." >&2 print_ldap_diagnostics "$CONTAINER_ID"