From c719cca5b0611edc66b6bce0557b3caacd84ad37 Mon Sep 17 00:00:00 2001 From: Roman Prilipskii Date: Thu, 14 May 2026 19:19:52 -0400 Subject: [PATCH 1/2] CLOS-2816: Cap disk image size to filesystem file size limit On systems with large partitions (e.g. /home on a SAN with ~19 TB free), the target_userspace_creator actor tried to create a sparse disk image whose apparent size exceeded the 16 TiB maximum file size of ext4 (4K blocks), causing dd to fail with exit code 1. Add _get_max_diskimage_size_mibs() which uses os.pathconf's PC_FILESIZEBITS to determine the actual file size limit of the disk images filesystem, then cap each disk image size to that limit in _prepare_required_mounts(). Co-Authored-By: Claude Sonnet 4.6 --- .../common/libraries/overlaygen.py | 29 ++++ .../common/libraries/tests/test_overlaygen.py | 163 ++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 repos/system_upgrade/common/libraries/tests/test_overlaygen.py diff --git a/repos/system_upgrade/common/libraries/overlaygen.py b/repos/system_upgrade/common/libraries/overlaygen.py index 1bf8c68692..b168f9656f 100644 --- a/repos/system_upgrade/common/libraries/overlaygen.py +++ b/repos/system_upgrade/common/libraries/overlaygen.py @@ -141,6 +141,27 @@ def get_recommended_leapp_free_space(userspace_path=None): return prot_size +def _get_max_diskimage_size_mibs(directory): + """ + Return the maximum sparse file size in MiB supported by the filesystem at `directory`. + + Uses PC_FILESIZEBITS (number of bits in a file offset) to derive the limit. + Example: ext4 with 4K blocks reports 44 bits -> max 16 TiB. + + Returns None if the limit cannot be determined, in which case no capping is applied. + """ + try: + filesizebits = os.pathconf(directory, 'PC_FILESIZEBITS') + return (2 ** filesizebits - 1) // (1024 * 1024) + except (AttributeError, ValueError, OSError): + api.current_logger().warning( + 'Cannot determine filesystem file size limit for %s.' + ' Disk image sizes will not be capped by filesystem limit.', + directory + ) + return None + + def _get_fspace(path, convert_to_mibs=False, coefficient=1): """ Return the free disk space on given path. @@ -279,6 +300,7 @@ def _prepare_required_mounts(scratch_dir, mounts_dir, storage_info, scratch_rese # as disk images are cleaned in the end of this functions, # but we want to reserve some space in advance. scratch_disk_size = _get_fspace(scratch_dir, convert_to_mibs=True) - scratch_reserve + max_image_size_mibs = _get_max_diskimage_size_mibs(disk_images_directory) result = {} for mountpoint in mount_points: @@ -286,6 +308,13 @@ def _prepare_required_mounts(scratch_dir, mounts_dir, storage_info, scratch_rese disk_size = _get_fspace(mountpoint, convert_to_mibs=True, coefficient=0.95) if mountpoint == scratch_mp: disk_size = scratch_disk_size + if max_image_size_mibs is not None and disk_size > max_image_size_mibs: + api.current_logger().warning( + 'Disk image size for %s (%d MiB) exceeds filesystem file size limit (%d MiB).' + ' Capping to filesystem limit.', + mountpoint, disk_size, max_image_size_mibs + ) + disk_size = max_image_size_mibs image = _create_mount_disk_image(disk_images_directory, mountpoint, disk_size) result[mountpoint] = mounting.LoopMount( source=image, diff --git a/repos/system_upgrade/common/libraries/tests/test_overlaygen.py b/repos/system_upgrade/common/libraries/tests/test_overlaygen.py new file mode 100644 index 0000000000..f1b3389b13 --- /dev/null +++ b/repos/system_upgrade/common/libraries/tests/test_overlaygen.py @@ -0,0 +1,163 @@ +import os + +import pytest + +from leapp.libraries.common import overlaygen +from leapp.libraries.common.testutils import CurrentActorMocked, logger_mocked +from leapp.libraries.stdlib import api +from leapp.models import FstabEntry, StorageInfo + + +def make_storage_info(mountpoints): + entries = [ + FstabEntry( + fs_spec='/dev/sda1', + fs_file=mp, + fs_vfstype='ext4', + fs_mntops='defaults', + fs_freq='0', + fs_passno='0', + ) + for mp in mountpoints + ] + return StorageInfo(fstab=entries) + + +class TestGetMaxDiskimageSizeMibs(object): + def test_returns_correct_value_for_ext4_filesizebits(self, monkeypatch): + # ext4 with 4K blocks reports 44 bits -> max 2^44 - 1 bytes -> 16,777,215 MiB + monkeypatch.setattr(os, 'pathconf', lambda path, name: 44) + result = overlaygen._get_max_diskimage_size_mibs('/some/dir') + assert result == (2 ** 44 - 1) // (1024 * 1024) + assert result == 16777215 + + def test_returns_correct_value_for_xfs_filesizebits(self, monkeypatch): + # XFS reports 63 bits -> max 2^63 - 1 bytes + monkeypatch.setattr(os, 'pathconf', lambda path, name: 63) + result = overlaygen._get_max_diskimage_size_mibs('/some/dir') + assert result == (2 ** 63 - 1) // (1024 * 1024) + + def test_returns_none_on_oserror(self, monkeypatch): + def raise_oserror(path, name): + raise OSError('not supported') + + monkeypatch.setattr(os, 'pathconf', raise_oserror) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked()) + result = overlaygen._get_max_diskimage_size_mibs('/some/dir') + assert result is None + + def test_returns_none_when_pathconf_missing(self, monkeypatch): + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked()) + # Simulate AttributeError by making pathconf raise it + monkeypatch.setattr(os, 'pathconf', lambda path, name: (_ for _ in ()).throw(AttributeError())) + result = overlaygen._get_max_diskimage_size_mibs('/some/dir') + assert result is None + + +class TestPrepareRequiredMountsCapsDiskSize(object): + """Verify that _prepare_required_mounts caps disk image size to the filesystem limit.""" + + def _setup_mocks(self, monkeypatch, tmp_path, free_space_by_path, max_image_size_mibs): + """Common mock setup. Returns (scratch_dir, mounts_dir, created_images dict).""" + scratch_dir = str(tmp_path / 'scratch') + mounts_dir = str(tmp_path / 'mounts') + os.makedirs(scratch_dir) + os.makedirs(mounts_dir) + created_images = {} + + def fake_get_fspace(path, convert_to_mibs=False, coefficient=1): + size = free_space_by_path.get(path, 10240) + return int(size * coefficient) + + def fake_get_max(directory): + return max_image_size_mibs + + def fake_create_diskimages_dir(sd, diskimages_dir): + os.makedirs(diskimages_dir, exist_ok=True) + + def fake_ensure_space(space_needed, directory): + pass + + def fake_run(cmd): + pass + + def fake_create_mount_disk_image(disk_images_directory, path, disk_size): + created_images[path] = disk_size + image_path = os.path.join(disk_images_directory, 'root' + path.replace('/', '_')) + open(image_path, 'w').close() + return image_path + + monkeypatch.setattr(overlaygen, '_get_fspace', fake_get_fspace) + monkeypatch.setattr(overlaygen, '_get_max_diskimage_size_mibs', fake_get_max) + monkeypatch.setattr(overlaygen, '_create_diskimages_dir', fake_create_diskimages_dir) + monkeypatch.setattr(overlaygen, '_ensure_enough_diskimage_space', fake_ensure_space) + monkeypatch.setattr(overlaygen, '_create_mount_disk_image', fake_create_mount_disk_image) + monkeypatch.setattr(overlaygen, 'run', fake_run) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked()) + + return scratch_dir, mounts_dir, created_images + + def test_disk_size_capped_when_exceeds_fs_limit(self, monkeypatch, tmp_path): + """When /home free space exceeds the FS file size limit, the disk image is capped.""" + # Simulate CL7 cPanel server: / on ext4 (142 GiB free), /home on large SAN (19 TB free) + ext4_limit_mibs = 16777215 # 2^44 - 1 MiB (ext4 with 4K blocks) + scratch_dir = str(tmp_path / 'scratch') + + free_space = { + '/': 145408, + '/home': 19373090, # ~19 TB - exceeds ext4 limit + scratch_dir: 145408, + } + + scratch_dir, mounts_dir, created_images = self._setup_mocks( + monkeypatch, tmp_path, free_space, ext4_limit_mibs + ) + + storage_info = make_storage_info(['/', '/home']) + overlaygen._prepare_required_mounts(scratch_dir, mounts_dir, storage_info, scratch_reserve=0) + + assert created_images['/home'] == ext4_limit_mibs, ( + '/home disk image must be capped to ext4 file size limit' + ) + assert created_images['/'] <= ext4_limit_mibs + + def test_disk_size_not_capped_when_within_fs_limit(self, monkeypatch, tmp_path): + """When free space is within the FS limit, the disk image size is kept as-is.""" + ext4_limit_mibs = 16777215 + scratch_dir = str(tmp_path / 'scratch') + + free_space = { + '/': 145408, + '/home': 200000, # 200 GiB - well within ext4 limit + scratch_dir: 145408, + } + + scratch_dir, mounts_dir, created_images = self._setup_mocks( + monkeypatch, tmp_path, free_space, ext4_limit_mibs + ) + + storage_info = make_storage_info(['/', '/home']) + overlaygen._prepare_required_mounts(scratch_dir, mounts_dir, storage_info, scratch_reserve=0) + + # /home is within limit: 95% of 200000 = 190000 + assert created_images['/home'] == int(200000 * 0.95) + + def test_disk_size_not_capped_when_fs_limit_unknown(self, monkeypatch, tmp_path): + """When the FS limit cannot be determined (None), disk sizes are passed through unchanged.""" + scratch_dir = str(tmp_path / 'scratch') + large_size = 19373090 # ~19 TB + + free_space = { + '/': large_size, + scratch_dir: large_size, + } + + scratch_dir, mounts_dir, created_images = self._setup_mocks( + monkeypatch, tmp_path, free_space, None # unknown limit + ) + + storage_info = make_storage_info(['/']) + overlaygen._prepare_required_mounts(scratch_dir, mounts_dir, storage_info, scratch_reserve=0) + + # No cap applied - scratch_disk_size = large_size - 0 = large_size + assert created_images['/'] == large_size From 4db4b99e603778630ab163278b7c43c97ee5c954 Mon Sep 17 00:00:00 2001 From: Roman Prilipskii Date: Thu, 14 May 2026 19:19:52 -0400 Subject: [PATCH 2/2] fixup: remove unused imports in test_overlaygen Co-Authored-By: Claude Sonnet 4.6 --- .../common/libraries/tests/test_overlaygen.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/repos/system_upgrade/common/libraries/tests/test_overlaygen.py b/repos/system_upgrade/common/libraries/tests/test_overlaygen.py index f1b3389b13..e40c843ed5 100644 --- a/repos/system_upgrade/common/libraries/tests/test_overlaygen.py +++ b/repos/system_upgrade/common/libraries/tests/test_overlaygen.py @@ -1,9 +1,7 @@ import os -import pytest - from leapp.libraries.common import overlaygen -from leapp.libraries.common.testutils import CurrentActorMocked, logger_mocked +from leapp.libraries.common.testutils import CurrentActorMocked from leapp.libraries.stdlib import api from leapp.models import FstabEntry, StorageInfo @@ -46,10 +44,12 @@ def raise_oserror(path, name): result = overlaygen._get_max_diskimage_size_mibs('/some/dir') assert result is None - def test_returns_none_when_pathconf_missing(self, monkeypatch): + def test_returns_none_when_pathconf_raises_attribute_error(self, monkeypatch): + def raise_attribute_error(path, name): + raise AttributeError('not available') + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked()) - # Simulate AttributeError by making pathconf raise it - monkeypatch.setattr(os, 'pathconf', lambda path, name: (_ for _ in ()).throw(AttributeError())) + monkeypatch.setattr(os, 'pathconf', raise_attribute_error) result = overlaygen._get_max_diskimage_size_mibs('/some/dir') assert result is None