Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/usage/general/environment.rst.inc
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,8 @@ General:
0 means "always multi-threaded", a very large value effectively disables multi-threading.
BORG_ZSTD_MT_WORKERS
When set to a numeric value, use that many threads to zstd-compress a single chunk
(default: the cpu count, but at most 4). 0 or 1 means single-threaded compression.
(default: the cpu count, but at most 4). If it is unset or empty, the default is
used. 0 or 1 means single-threaded compression.
Only relevant when compressing with ``zstd``.
Chunks below 768KiB are always compressed single-threaded: libzstd will not use a
compression job smaller than 512KiB, so a small chunk gets split very unevenly and
Expand Down
3 changes: 2 additions & 1 deletion src/borg/archiver/help_cmd.py
Original file line number Diff line number Diff line change
Expand Up @@ -819,7 +819,8 @@ class HelpMixIn:
0 means "always multi-threaded", a very large value effectively disables multi-threading.
BORG_ZSTD_MT_WORKERS
When set to a numeric value, use that many threads to zstd-compress a single chunk
(default: the cpu count, but at most 4). 0 or 1 means single-threaded compression.
(default: the cpu count, but at most 4). If it is unset or empty, the default is
used. 0 or 1 means single-threaded compression.
Only relevant when compressing with ``zstd``.
Chunks below 768KiB are always compressed single-threaded: libzstd will not use a
compression job smaller than 512KiB, so a small chunk gets split very unevenly and
Expand Down
7 changes: 4 additions & 3 deletions src/borg/compress.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ _zstd_mt_workers = None # cached (chunk workers, stream workers)
def get_zstd_mt_workers(stream=False):
"""How many threads libzstd may use to compress a single chunk (or stream).

BORG_ZSTD_MT_WORKERS overrides the defaults below (for chunks and streams alike).
BORG_ZSTD_MT_WORKERS overrides the defaults below (for chunks and streams alike);
if it is unset or empty, the defaults are used.
0 or 1 means single-threaded compression, which also avoids the small loss of
compression ratio that splitting a chunk into jobs causes (measured at zstd,3:
+0.05% for a 1MiB chunk, +0.64% for an 8MiB one; higher levels lose a bit more as
Expand All @@ -84,9 +85,9 @@ def get_zstd_mt_workers(stream=False):
"""
global _zstd_mt_workers
if _zstd_mt_workers is None:
value = os.environ.get("BORG_ZSTD_MT_WORKERS")
value = os.environ.get("BORG_ZSTD_MT_WORKERS", "").strip()
cpus = os.cpu_count() or 1
if value is None:
if not value: # unset or empty: use the defaults
workers = (min(cpus, 4), cpus)
else:
try:
Expand Down
5 changes: 4 additions & 1 deletion src/borg/testsuite/compress_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -350,9 +350,12 @@ def workers_for(env_value, stream=False):
cpus = os.cpu_count() or 1
assert workers_for(None) == min(cpus, 4) # per-chunk default is capped
assert workers_for(None, stream=True) == cpus # stream default is not
for empty in ["", " "]: # an empty value is treated like an unset one
assert workers_for(empty) == min(cpus, 4)
assert workers_for(empty, stream=True) == cpus
for value, expected in [("0", 0), ("1", 1), ("4", 4), ("12", 12)]:
assert workers_for(value) == expected # the env var is not capped
assert workers_for(value, stream=True) == expected
for invalid in ["", "yes", "4x", "1.5", "-1"]:
for invalid in ["yes", "4x", "1.5", "-1"]:
with pytest.raises(Error):
workers_for(invalid)
Loading