diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml new file mode 100644 index 0000000..c4e3dd8 --- /dev/null +++ b/.github/workflows/ci_cd.yml @@ -0,0 +1,94 @@ +name: CI Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + release: + types: [published] + workflow_dispatch: + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Test (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + ninja-build \ + libopenmpi-dev \ + openmpi-bin + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Upgrade build tooling + run: python -m pip install --upgrade pip setuptools wheel + + - name: Build and install PyCAPIO from source + run: pip install . --verbose + + - name: Install test dependencies + run: | + pip install -r tests/requirements.txt + pip install pytest-cov + + - name: Run test suite with coverage + working-directory: tests + run: | + pytest -v \ + --cov=pycapio \ + --cov-report=xml \ + --cov-report=term-missing + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + files: tests/coverage.xml + flags: python-${{ matrix.python-version }} + name: pycapio-py${{ matrix.python-version }} + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + + build-sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Build sdist + run: | + python -m pip install --upgrade pip build twine + python -m build --sdist + python -m twine check dist/* + + - name: Upload sdist artifact + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..827ffd3 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,135 @@ +name: Release to PyPI + +on: + workflow_run: + workflows: ["CI Tests"] + branches: [main] + types: [completed] + workflow_dispatch: + +jobs: + check-version: + name: Check for Version Bump + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + outputs: + tag_exists: ${{ steps.check-tag.outputs.exists }} + version: ${{ steps.get-version.outputs.version }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Extract Version from CMakeLists.txt + id: get-version + run: | + VERSION=$(grep -E 'VERSION [0-9]+\.[0-9]+\.[0-9]+' CMakeLists.txt | awk '{print $2}') + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Check if git tag exists + id: check-tag + uses: mukunku/tag-exists-action@v1.7.0 + with: + tag: "v${{ steps.get-version.outputs.version }}" + + build-wheels: + name: Build wheels on ${{ matrix.os }} + needs: check-version + # GATEKEEPER: Only proceed if the tag DOES NOT exist + if: needs.check-version.outputs.tag_exists == 'false' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ['ubuntu-24.04', 'ubuntu-24.04-arm'] + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install cibuildwheel + run: python -m pip install --upgrade cibuildwheel + + - name: Build wheels + env: + CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-* cp314-*" + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 + MACOSX_DEPLOYMENT_TARGET: "15.0" + CIBW_BUILD_VERBOSITY: 1 + run: cibuildwheel --output-dir wheelhouse + + - name: Upload wheel artifacts + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }} + path: wheelhouse/*.whl + + build-sdist: + name: Build source distribution + needs: check-version + if: needs.check-version.outputs.tag_exists == 'false' + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Build sdist + run: | + python -m pip install --upgrade build + python -m build --sdist + + - name: Upload sdist artifact + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + + publish: + name: Release and Publish to PyPI + needs: [check-version, build-wheels, build-sdist] + if: needs.check-version.outputs.tag_exists == 'false' + runs-on: ubuntu-latest + permissions: + contents: write # Required for creating GitHub Releases + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Download all wheels + uses: actions/download-artifact@v4 + with: + pattern: wheels-* + merge-multiple: true + path: dist + + - name: Download sdist + uses: actions/download-artifact@v4 + with: + name: sdist + path: dist + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: "v${{ needs.check-version.outputs.version }}" + name: "v${{ needs.check-version.outputs.version }}" + generate_release_notes: true + files: dist/* + + - name: Publish to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_DEPLOY_KEY }} + run: | + python -m pip install --upgrade twine + twine check dist/* + twine upload dist/* \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 21d5c70..25e220d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,8 @@ include(GNUInstallDirs) option(CAPIO_LOG "Enable log capabilities within the CAPIO communication queues and libcapio adapter" OFF) +set(CAPIO_RELEASE_TAG "master" CACHE STRING "Default CAPIO tag used to compile PyCAPIO.") + set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -27,10 +29,12 @@ if (${CMAKE_BUILD_TYPE} STREQUAL "Debug") add_compile_definitions(CAPIO_LOG) endif () +message(STATUS "Using CAPIO on tag ${CAPIO_RELEASE_TAG}") + FetchContent_Declare( capio GIT_REPOSITORY https://github.com/High-Performance-IO/capio.git - GIT_TAG master + GIT_TAG ${CAPIO_RELEASE_TAG} CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} ) @@ -56,6 +60,11 @@ install( pybind11_add_module(_pycapio src/pycapio.cpp) +# patch server_println by copying server_println.hpp from server/utils/server_println.hpp to posix/utils/server_println.hpp +file(COPY "${capio_SOURCE_DIR}/capio/server/include/utils/server_println.hpp" + DESTINATION "${capio_SOURCE_DIR}/capio/posix/utils/") +message(WARNING "WARN: patch for server_println.hpp copied to posix has been applied!") + target_compile_definitions(_pycapio PRIVATE PYCAPIO_BINDINGS) message(STATUS "CAPIO_PREFIX = ${capio_SOURCE_DIR}") diff --git a/pyproject.toml b/pyproject.toml index 01eba5f..6bd9596 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ cmake.version = ">=3.15" wheel.packages = ["pycapio"] [tool.scikit-build.cmake.define] +CAPIO_RELEASE_TAG = "6b14036e85678f54cf9fa265141edb98b3394c8a" CMAKE_BUILD_TYPE = "Release" -CAPIO_LOG="OFF" -CAPIO_BUILD_POSIX="OFF" +CAPIO_LOG = "OFF" +CAPIO_BUILD_POSIX = "OFF" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..3663d94 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,79 @@ +import os +import socket + +import psutil +import pytest + +from utils import is_capio_running + + +def _terminate_capio_servers(): + killed = [] + for proc in psutil.process_iter(["name", "cmdline"]): + try: + name = (proc.info.get("name") or "").lower() + cmdline = " ".join(proc.info.get("cmdline") or []).lower() + if "capio_server" in name or "capio_server" in cmdline: + proc.terminate() + killed.append(proc) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + continue + if killed: + psutil.wait_procs(killed, timeout=10) + + +def _remove_files_location(): + registry = f"files_location_{socket.gethostname()}.txt" + if os.path.exists(registry): + try: + os.remove(registry) + except OSError: + pass + + +def force_cleanup(): + import atexit + + import pycapio + from pycapio.internals import pycapio_teardown + + try: + pycapio_teardown(True) + except Exception: + pass + + try: + atexit.unregister(pycapio_teardown) + except Exception: + pass + pycapio.py_capio_initialized = False + + _terminate_capio_servers() + _remove_files_location() + + +@pytest.fixture +def capio(): + from pycapio.internals import pycapio_init + + capio_dir = "/tmp" + + assert not is_capio_running() + pycapio_init( + CAPIO_DIR=capio_dir, + CAPIO_WORKFLOW_NAME="test", + CAPIO_APP_NAME="test", + ) + assert is_capio_running() + + try: + yield capio_dir + finally: + force_cleanup() + assert not is_capio_running() + + +@pytest.fixture(autouse=True) +def _ensure_clean_capio(): + yield + force_cleanup() \ No newline at end of file diff --git a/tests/test_context.py b/tests/test_context.py new file mode 100644 index 0000000..a0bc35a --- /dev/null +++ b/tests/test_context.py @@ -0,0 +1,55 @@ +import builtins +import io +import os + +import pycapio +from pycapio import CapioContext, PyCapioPath + + +def test_capio_context_patches_restores_and_streams(): + original_seen = {"builtins.open": builtins.open, + "os.mkdir": os.mkdir, + "os.makedirs": os.makedirs, + "os.scandir": os.scandir, + "io.open": io.open, + "os.listdir": os.listdir, + "os.path": os.path} + + seen = {} + + @CapioContext(capio_dir="/tmp", app_name="ctx", workflow_name="ctx_wf") + def do_io(): + seen["builtins.open"] = builtins.open + seen["os.mkdir"] = os.mkdir + seen["os.makedirs"] = os.makedirs + seen["os.scandir"] = os.scandir + seen["io.open"] = io.open + seen["os.listdir"] = os.listdir + seen["os.path"] = os.path + + path = "/tmp/capio_context.dat" + with open(path, "w") as f: + f.write("streamed via CapioContext") + with open(path, "r") as f: + return f.read() + + result = do_io() + + assert result == "streamed via CapioContext" + + assert seen["builtins.open"] is pycapio.open_proxy + assert seen["os.mkdir"] is pycapio.mkdir_proxy + assert seen["os.makedirs"] is pycapio.makedirs_proxy + assert seen["os.scandir"] is pycapio.scandir_proxy + assert seen["io.open"] is pycapio.open_proxy + assert seen["os.listdir"] is pycapio.listdir_proxy + assert seen["os.path"] is PyCapioPath + + assert builtins.open is original_seen["builtins.open"] + assert os.mkdir is original_seen["os.mkdir"] + assert os.makedirs is original_seen["os.makedirs"] + assert os.scandir is original_seen["os.scandir"] + assert io.open is original_seen["io.open"] + assert os.listdir is original_seen["os.listdir"] + assert os.path is original_seen["os.path"] + diff --git a/tests/test_os_path.py b/tests/test_os_path.py new file mode 100644 index 0000000..2df3c67 --- /dev/null +++ b/tests/test_os_path.py @@ -0,0 +1,65 @@ +import os + +from pycapio import PyCapioPath +from pycapio.internals import FILE_MODES, pycapio_open + +def test_basename(): + assert PyCapioPath.basename("/tmp/a/b.txt") == "b.txt" + + +def test_dirname(): + assert PyCapioPath.dirname("/tmp/a/b.txt") == "/tmp/a" + + +def test_join_basic(): + assert PyCapioPath.join("/tmp/a", "b.txt") == "/tmp/a/b.txt" + + +def test_join_empty_operands(): + assert PyCapioPath.join("", "") == "" + assert PyCapioPath.join("", "b") == "b" + assert PyCapioPath.join("a", "") == "a" + + +def test_splitext_returns_stem_and_extension(): + # wrapper returns the stem, not the full root + assert PyCapioPath.splitext("/tmp/a/b.txt") == ("b", ".txt") + assert PyCapioPath.splitext("/tmp/a/noext") == ("noext", "") + + +def test_isabs(): + assert PyCapioPath.isabs("/tmp/a") is True + assert PyCapioPath.isabs("a/b") is False + + +def test_normpath_collapses_dotdot(): + assert PyCapioPath.normpath("/tmp/a/../b") == "/tmp/b" + + +def test_split_into_head_and_tail(): + assert PyCapioPath.split("/tmp/a/b.txt") == ("/tmp/a", "b.txt") + + +def test_normcase_is_identity_on_unix(): + assert PyCapioPath.normcase("/Tmp/MixedCase") == "/Tmp/MixedCase" + + +def test_relpath(): + assert PyCapioPath.relpath("/tmp/a/b", "/tmp") == "a/b" + + +def test_exists_and_isfile_for_written_file(capio): + path = f"{capio}/ospath_file.dat" + + fd = pycapio_open(path, FILE_MODES["O_CREAT"], 0o777) + assert fd != -1 + from pycapio.internals import PyCapioTextIOWrapper + + wrapper = PyCapioTextIOWrapper(fd) + wrapper.write("some content") + del wrapper # flush + close + + assert PyCapioPath.exists(path) is True + assert PyCapioPath.isfile(path) is True + # the file is purely virtual: it never hits the real filesystem + assert not os.path.exists(path) \ No newline at end of file diff --git a/tests/utils.py b/tests/utils.py index c9ca83d..327c866 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -17,25 +17,37 @@ def check_shm_cleaned(): return True +def _is_capio_proc(proc): + try: + name = (proc.info.get("name") or "").lower() + if "capio_server" in name: + return True + cmdline = proc.info.get("cmdline") or [] + return any("capio_server" in part.lower() for part in cmdline) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + return False + + def is_capio_running(): - for proc in psutil.process_iter(['name']): - try: - if "capio_server" in proc.info['name'].lower(): - return True - except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): - continue + for proc in psutil.process_iter(["name", "cmdline"]): + if _is_capio_proc(proc): + return True return False def kill_capio_server(): procs_to_kill = [] - for proc in psutil.process_iter(['name']): - try: - if "capio_server" in proc.info['name'].lower(): + for proc in psutil.process_iter(["name", "cmdline"]): + if _is_capio_proc(proc): + try: proc.terminate() procs_to_kill.append(proc) - except (psutil.NoSuchProcess, psutil.AccessDenied): - continue + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue psutil.wait_procs(procs_to_kill) - os.remove(f"files_location_{socket.gethostname()}.txt") - assert check_shm_cleaned() + + registry = f"files_location_{socket.gethostname()}.txt" + if os.path.exists(registry): # guard: original crashed when absent + os.remove(registry) + + assert check_shm_cleaned() \ No newline at end of file