diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d18ec53dc..71150412bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -451,6 +451,12 @@ jobs: pyver: '3.13' do_binaries: false + - os: haiku + version: 'r1beta6' + display_name: Haiku + pyver: '3.14' + do_binaries: false + steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -474,7 +480,7 @@ jobs: # a normal boot takes < 1 minute; since 1.4.0 the action bounds # waiting for boot itself, this is just an additional safety net. timeout-minutes: 15 - uses: cross-platform-actions/action@24ef01df165c76df1ed2b9f9e9212e78dc2fc963 # v1.4.0 + uses: cross-platform-actions/action@faa0c6197e94aacf1c5956460152c8380d3560a5 # v1.5.0 with: operating_system: ${{ matrix.os }} version: ${{ matrix.version }} @@ -641,13 +647,11 @@ jobs: haiku) pkgman refresh - pkgman install -y git pkgconfig lz4 - pkgman install -y openssl3 - pkgman install -y rust_bin - pkgman install -y python3.11 + # already installed: + # pkgman install -y git pkgconfig lz4 openssl3 pkgman install -y lz4_devel openssl3_devel + pkgman install -y rust_bin - # there is no pkgman package for tox, so we install it into a venv python3 -m ensurepip --upgrade python3 -m pip install --upgrade pip wheel python3 -m venv .venv @@ -656,8 +660,31 @@ jobs: export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}" export BORG_LIBLZ4_PREFIX=/system/develop export BORG_OPENSSL_PREFIX=/system/develop - pip install -r requirements.d/development.lock.txt - pip install -e . + + # this VM corrupts data every now and then: rustc crashed on crate metadata it + # had written itself ("assertion failed: bytes[len] == STR_SENTINEL") and pip + # found sha256 mismatches reading big wheels back from its own cache. Build + # outside /boot/system/cache/tmp (haiku's /tmp) and run one rustc at a time, + # in case that corruption comes from disk or memory pressure. + export TMPDIR=/boot/home/borg-tmp + mkdir -p "$TMPDIR" + export CARGO_TARGET_DIR="$TMPDIR/cargo-target" + export CARGO_BUILD_JOBS=1 + + # retry a pip install that hit a corrupted file, dropping the cached (possibly + # corrupted) files first, so one of them does not fail the whole job - and does + # not end up in the actions cache, failing every following run, too. + pip_install() { + for attempt in 1 2 3; do + pip install "$@" && return 0 + echo "*** pip install failed (attempt $attempt), purging the pip cache and retrying ***" + pip cache purge || true + done + return 1 + } + + pip_install -r requirements.d/development.lock.txt + pip_install -e . # troubles with either tox or pytest xdist, so we run pytest manually: pytest -v -n auto -rs --cov=borg --cov-config=pyproject.toml --cov-report=xml --junitxml=test-results.xml --benchmark-skip -k "not remote and not socket" diff --git a/pyproject.toml b/pyproject.toml index a80a756a59..4ce900721d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,8 @@ dependencies = [ "blake3 >= 1.0; sys_platform != 'win32'", "blake3 >= 1.0,!=1.0.9; sys_platform == 'win32'", # 1.0.9 does not build on windows/msys2/mingw "tzdata; sys_platform == 'win32'", # zoneinfo has no system tz database on Windows (date: named zones) + # same on haiku (sys.platform is 'haiku1' on pythons older than 3.13) + "tzdata; sys_platform == 'haiku' or sys_platform == 'haiku1'", ] [project.optional-dependencies] diff --git a/src/borg/conftest.py b/src/borg/conftest.py index fd20dfe3f3..9925ee9d29 100644 --- a/src/borg/conftest.py +++ b/src/borg/conftest.py @@ -84,6 +84,10 @@ def clean_env(tmpdir_factory, monkeypatch): keys = [key for key in os.environ if key.startswith("BORG_") and key not in ("BORG_FUSE_IMPL",)] for key in keys: monkeypatch.delenv(key, raising=False) + # a "borg serve" started by a test must not mistake the ssh forced command of the session + # the tests happen to run in (e.g. a CI runner driven over ssh) for the command line of a + # borg client - it would fail to parse it and exit before serving anything. + monkeypatch.delenv("SSH_ORIGINAL_COMMAND", raising=False) # avoid that we access / modify the user's normal .config / .cache directory: base_dir = tmpdir_factory.mktemp("borg-base-dir") monkeypatch.setenv("BORG_BASE_DIR", str(base_dir)) diff --git a/src/borg/fslocking.py b/src/borg/fslocking.py index 5a7f241ea0..13bba12df2 100644 --- a/src/borg/fslocking.py +++ b/src/borg/fslocking.py @@ -220,6 +220,11 @@ def kill_stale_lock(self): return False except PermissionError: # win32 might throw this. return False + except OSError: + # we can not read the lock directory - e.g. haiku fails this with EBUSY or + # "bad data" while the directory is concurrently replaced via rename(). + # Not being able to look at the lock means we can not call it stale, see by_me(). + return False else: for name in names: try: diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py index e7a3d44ebf..3225bf8277 100644 --- a/src/borg/helpers/fs.py +++ b/src/borg/helpers/fs.py @@ -16,7 +16,7 @@ from .errors import Error from .process import prepare_subprocess_env -from ..platformflags import is_win32 +from ..platformflags import is_win32, is_haiku from ..constants import * # NOQA @@ -614,6 +614,13 @@ def read(self, size): data = os.read(self.fd, remaining) except BlockingIOError: data = None # a writer is connected, but no data is available right now + except OSError as err: + if is_haiku and err.errno == errno.ENOMEM: + # haiku fails a read from a fifo nobody has opened for writing with ENOMEM + # instead of returning b"" - it means the same here: no writer (yet). + data = b"" + else: + raise if data: parts.append(data) remaining -= len(data) diff --git a/src/borg/legacy/remote.py b/src/borg/legacy/remote.py index 93a5c9dbfb..cef1be02cc 100644 --- a/src/borg/legacy/remote.py +++ b/src/borg/legacy/remote.py @@ -29,7 +29,7 @@ from ..repository import Repository, StoreObjectNotFound from ..version import parse_version, format_version from ..helpers.datastruct import EfficientCollectionQueue -from ..platform import is_win32 +from ..platform import is_win32, is_haiku logger = create_logger(__name__) @@ -249,7 +249,10 @@ def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock self.stdout_fd = self.p.stdout.fileno() self.stderr_fd = self.p.stderr.fileno() self.r_fds = [self.stdout_fd, self.stderr_fd] - self.x_fds = [self.stdin_fd, self.stdout_fd, self.stderr_fd] + # haiku's select() reports pipes in the exceptional set although nothing is wrong, + # so do not watch for exceptional conditions there - a broken pipe still shows up + # as a read/write error below. + self.x_fds = [] if is_haiku else [self.stdin_fd, self.stdout_fd, self.stderr_fd] else: raise Error(f"Unsupported protocol {location.proto}") diff --git a/src/borg/testsuite/archiver/webdav_cmd_test.py b/src/borg/testsuite/archiver/webdav_cmd_test.py index 5457b41fd5..7b6f957265 100644 --- a/src/borg/testsuite/archiver/webdav_cmd_test.py +++ b/src/borg/testsuite/archiver/webdav_cmd_test.py @@ -84,15 +84,30 @@ def webdav_server(archiver): thread.join(timeout=10) +def urlopen(request, attempts=10): + """urllib.request.urlopen, but retry a connect that fails with EWOULDBLOCK. + + haiku fails connecting to a listening localhost socket with EWOULDBLOCK (rather than + waiting for the server to accept) when the server does not accept immediately. + """ + for attempt in range(attempts): + try: + return urllib.request.urlopen(request) + except urllib.error.URLError as err: + if not isinstance(err.reason, BlockingIOError) or attempt == attempts - 1: + raise + time.sleep(0.1) + + def get(url, headers=None): request = urllib.request.Request(url, headers=headers or {}) - with urllib.request.urlopen(request) as response: + with urlopen(request) as response: return response.status, dict(response.headers), response.read() def http_request(url, method, headers=None, body=None): req = urllib.request.Request(url, data=body, method=method, headers=headers or {}) - with urllib.request.urlopen(req) as response: + with urlopen(req) as response: return response.status, dict(response.headers), response.read() @@ -300,7 +315,7 @@ def test_webdav_errors(archivers, request): get(base_url + "/test/input/../input/file1") assert exc_info.value.code == 404 with pytest.raises(urllib.error.HTTPError) as exc_info: - urllib.request.urlopen(urllib.request.Request(base_url + "/", method="POST")) + urlopen(urllib.request.Request(base_url + "/", method="POST")) assert exc_info.value.code == 405 # writing/locking WebDAV methods are rejected, too for method in "PUT", "MKCOL", "LOCK", "PROPPATCH":