From f117d2aac5f275daedb67f87d2c30eff7468a87a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Hanko?= Date: Fri, 7 Aug 2026 09:37:32 +0200 Subject: [PATCH 1/3] conftest: re-allocate hugepages when requested amount not met hugepages_allocated() previously only checked that *some* hugepages were free (HugePages_Free != 0). If a user increased --suricata-hugepages on a machine that already had hugepages mounted, the new allocation request was silently ignored. Add _parse_size_to_bytes() helper and compare the currently allocated hugepage memory (HugePages_Total * Hugepagesize from /proc/meminfo) against the requested amount, re-running dpdk-hugepages.py --setup only when the mounted amount is lower. Also validate --suricata-hugepages in pytest_configure and raise a clean pytest.UsageError on invalid input (e.g. 6X or abc) instead of a raw ValueError traceback from the session fixture. docs: document hugepages re-allocation behavior Update README.md (note after DEFAULT_HUGEPAGES, binary-search setup step) and pytest_start.sh -sh help text to describe that hugepages are re-allocated when the currently mounted amount is lower than the requested --suricata-hugepages value. hugepages: simplify allocation check and robust size parsing - Read /proc/meminfo stdout directly instead of writing to a temp file and reading it back with a second cat command. - Parse size strings loosely (any trailing unit chars) and validate the suffix against the multiplier table, so the space-separated form from /proc/meminfo (e.g. '2048 kB') is accepted. - Catch KeyError from the deferred suffix validation in _validate_hugepages_option. rename this commit conftest: try dpdk-hugepages --reserve as last-ditch effort If dpdk-hugepages.py --setup fails, attempt --reserve before giving up so hugepages can still be pinned. --reserve still runs after a successful --setup so the pages are actually reserved for Suricata. hugepages: account for double allocation when checking; lower default descriptors docs: simplify hugepages help text; restore default rx/tx descriptors docs: clarify hugepages re-allocation behavior --- README.md | 5 ++ conftest.py | 153 ++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 137 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 3399711..987d35c 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,11 @@ DEFAULT_HUGEPAGES="6G" LOGLEVEL="INFO" ``` +**Note on hugepages:** `DEFAULT_HUGEPAGES` (or `--suricata-hugepages` when running pytest directly) +is the amount of RAM requested for hugepages on the Suricata server. If the machine already has +less hugepage memory mounted than requested, it is re-allocated up to the requested amount. +Note that this only ever increases the allocation — it never shrinks it back down. + Note that an empty string ("") in `-d` (or `DEFAULT_TESTS`) is a valid value for running all tests and that setting `DEFAULT_TESTS` will prevent you from doing so. diff --git a/conftest.py b/conftest.py index a6924c1..3922767 100644 --- a/conftest.py +++ b/conftest.py @@ -24,7 +24,7 @@ from dataclasses import dataclass from lbr_testsuite.executable import executable, remote_executor from lbr_trex_client.interactive import trex -from typing import Tuple, List +from typing import Tuple from pathlib import Path from itertools import product from param import filter @@ -93,10 +93,14 @@ def pytest_addoption(parser): ) parser.addoption( "--suricata-hugepages", - type=str, + type=_parse_size_to_bytes, default="6G", action="store", - help=("Specify amount of hugepages to be setup on remote machine. "), + help=( + "Specify amount of hugepages to be setup on remote machine. " + "If the machine already has less mounted, it is re-allocated " + "to this amount." + ), ) parser.addoption( "--suricata-cfg", @@ -475,29 +479,106 @@ def assert_available_machines(request) -> None: assert int(stdout) > 0, "Interface on host not found" +def _parse_size_to_bytes(size: str) -> int: + """Parse a size string like ``6G``, ``512M`` or ``1024`` into bytes. + + Supported suffixes are ``K``, ``M``, ``G``, ``T`` and ``P`` (binary + multiples). A bare number is interpreted as bytes. A unit is matched + loosely (any trailing characters) and validated against the multiplier + table below, so the space-separated form printed by ``/proc/meminfo`` + (e.g. ``2048 kB``) is also accepted. + """ + size = size.strip().upper() + match = re.fullmatch(r"(\d+)\s*(.+)?", size) + if not match: + raise ValueError(f"Couldn't parse byte count from {size!r}") + value = int(match.group(1)) + unit = match.group(2) or "" + # Normalize e.g. "KB" -> "K" (meminfo prints "kB"). + if unit.endswith("B") and len(unit) > 1: + unit = unit[:-1] + multipliers = { + "": 1, + "K": 1024, + "M": 1024**2, + "G": 1024**3, + "T": 1024**4, + "P": 1024**5, + } + if unit not in multipliers: + raise ValueError(f"Unknown size unit {unit!r} in {size!r}") + return value * multipliers[unit] + + +def _bytes_to_size(size: int) -> str: + """Format a byte count as a compact size string (e.g. ``6G``). + + Inverse of ``_parse_size_to_bytes``, used to hand the requested amount back + to ``dpdk-hugepages.py --setup``, which expects a size string rather than a + raw byte count. Picks the largest unit that divides the value evenly. + """ + if size == 0: + return "0" + units = [ + (1024**5, "P"), + (1024**4, "T"), + (1024**3, "G"), + (1024**2, "M"), + (1024, "K"), + ] + for factor, suffix in units: + if size % factor == 0: + return f"{size // factor}{suffix}" + return str(size) + + def hugepages_allocated(request) -> bool: + """Check whether the requested amount of hugepages is already allocated. + + The check compares the currently allocated hugepage memory + (``HugePages_Total`` * ``Hugepagesize`` from ``/proc/meminfo``) against the + amount requested via ``--suricata-hugepages``. It returns ``True`` only if + the allocated amount is at least the requested one. + + This ensures that increasing ``--suricata-hugepages`` on a machine that + already has hugepages mounted triggers a re-allocation instead of silently + ignoring the new request (the old implementation only checked that *some* + hugepages were free). + """ process_cat_hugepages_count = executable.Tool( - "cat /proc/meminfo | grep 'HugePages_Free:' > /tmp/hugepages_allocated_info", + "cat /proc/meminfo | grep -E 'HugePages_Total:|Hugepagesize:'", executor=get_suri_executor(request), sudo=True, ) - process_cat_hugepages_count.run() - - process_get_hugepages_count_str = executable.Tool( - "cat /tmp/hugepages_allocated_info", - sudo=True, - executor=get_suri_executor(request), - ) - stdout, stderr = process_get_hugepages_count_str.run() + stdout, stderr = process_cat_hugepages_count.run() assert stderr == "", ( f"Error while gathering information about allocated hugepages: {stderr}" ) - # huge_pages[0] == "HugePages_Free:", huge_pages[1] is some nubmer as `str` - huge_pages: List[str] = stdout.split() + # Parse HugePages_Total and Hugepagesize from the output, e.g.: + # HugePages_Total: 3072 + # Hugepagesize: 2048 kB + total_pages = 0 + page_size_kb = 0 + for line in stdout.splitlines(): + parts = line.split() + if len(parts) < 2: + continue + if parts[0] == "HugePages_Total:": + total_pages = int(parts[1]) + elif parts[0] == "Hugepagesize:": + page_size_kb = int(parts[1]) + + allocated_bytes = total_pages * page_size_kb * 1024 + # ``dpdk-hugepages.py --setup `` actually allocates double the + # requested amount (e.g. ``--setup 4G`` allocates 8G), so halve the read + # value to compare against the requested (single) amount. Without this, + # requesting 5G would see 8G already allocated and skip re-allocation. + allocated_bytes //= 2 + requested_bytes = request.config.getoption("--suricata-hugepages") - return huge_pages[1] != "0" + return allocated_bytes >= requested_bytes @pytest.fixture(scope="session", autouse=True) @@ -506,18 +587,48 @@ def check_hugepages(request) -> None: logger.info("Huge-pages already allocated") return - logger.info( - "Allocating huge-pages: %s", request.config.getoption("--suricata-hugepages") - ) + requested_bytes = request.config.getoption("--suricata-hugepages") + logger.info("Allocating huge-pages: %s", _bytes_to_size(requested_bytes)) process_set_hugepages = executable.Tool( - f"dpdk-hugepages.py --setup {request.config.getoption('--suricata-hugepages')}", + f"dpdk-hugepages.py --setup {_bytes_to_size(requested_bytes)}", + sudo=True, + executor=get_suri_executor(request), + ) + process_reserve_hugepages = executable.Tool( + f"dpdk-hugepages.py --reserve {_bytes_to_size(requested_bytes)}", sudo=True, executor=get_suri_executor(request), ) - _, stderr = process_set_hugepages.run() + stderr_parts = [] + try: + _, stderr = process_set_hugepages.run() + stderr_parts.append(stderr) + except executable.ExecutableProcessError as e: + logger.warning( + "dpdk-hugepages.py --setup failed (%s). Trying --reserve as a " + "last-ditch effort.", + e, + ) + try: + _, stderr = process_reserve_hugepages.run() + stderr_parts.append(stderr) + except executable.ExecutableProcessError as reserve_e: + logger.critical( + "Failed to allocate huge-pages (%s). Continuing with the " + "currently allocated huge-pages; tests that require more will " + "fail with a specific error.", + reserve_e, + ) + return + else: + # --setup succeeded; still reserve the pages so they are actually + # pinned for Suricata. + _, stderr = process_reserve_hugepages.run() + stderr_parts.append(stderr) - assert stderr == "", f"Error while allocating hugepages: {stderr}" + combined_stderr = "\n".join(part for part in stderr_parts if part) + assert combined_stderr == "", f"Error while allocating hugepages: {combined_stderr}" logger.info("Huge-pages allocated successfully") From 4278c148249d0955d76a557a4fe1c1a54405be67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Hanko?= Date: Thu, 20 Aug 2026 12:40:59 +0200 Subject: [PATCH 2/3] confest: re-allocate hugepages when requested amount not met --- README.md | 9 +++++---- conftest.py | 45 +++++++++------------------------------------ 2 files changed, 14 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 987d35c..24be933 100644 --- a/README.md +++ b/README.md @@ -151,10 +151,11 @@ DEFAULT_HUGEPAGES="6G" LOGLEVEL="INFO" ``` -**Note on hugepages:** `DEFAULT_HUGEPAGES` (or `--suricata-hugepages` when running pytest directly) -is the amount of RAM requested for hugepages on the Suricata server. If the machine already has -less hugepage memory mounted than requested, it is re-allocated up to the requested amount. -Note that this only ever increases the allocation — it never shrinks it back down. +**Note on hugepages:** `DEFAULT_HUGEPAGES` (or `--suricata-hugepages`/`-sh` in `pytest_start.sh`, +or `--suricata-hugepages` when running pytest directly) is the amount of RAM requested for +hugepages on the Suricata server. If the machine already has less hugepage memory mounted than +requested, it is re-allocated up to the requested amount. Note that this only ever increases the +allocation — it never shrinks it back down. Note that an empty string ("") in `-d` (or `DEFAULT_TESTS`) is a valid value for running all tests and that setting `DEFAULT_TESTS` will prevent you from doing so. diff --git a/conftest.py b/conftest.py index 3922767..e617339 100644 --- a/conftest.py +++ b/conftest.py @@ -494,9 +494,7 @@ def _parse_size_to_bytes(size: str) -> int: raise ValueError(f"Couldn't parse byte count from {size!r}") value = int(match.group(1)) unit = match.group(2) or "" - # Normalize e.g. "KB" -> "K" (meminfo prints "kB"). - if unit.endswith("B") and len(unit) > 1: - unit = unit[:-1] + unit = unit.removesuffix("B") multipliers = { "": 1, "K": 1024, @@ -510,28 +508,6 @@ def _parse_size_to_bytes(size: str) -> int: return value * multipliers[unit] -def _bytes_to_size(size: int) -> str: - """Format a byte count as a compact size string (e.g. ``6G``). - - Inverse of ``_parse_size_to_bytes``, used to hand the requested amount back - to ``dpdk-hugepages.py --setup``, which expects a size string rather than a - raw byte count. Picks the largest unit that divides the value evenly. - """ - if size == 0: - return "0" - units = [ - (1024**5, "P"), - (1024**4, "T"), - (1024**3, "G"), - (1024**2, "M"), - (1024, "K"), - ] - for factor, suffix in units: - if size % factor == 0: - return f"{size // factor}{suffix}" - return str(size) - - def hugepages_allocated(request) -> bool: """Check whether the requested amount of hugepages is already allocated. @@ -560,7 +536,7 @@ def hugepages_allocated(request) -> bool: # HugePages_Total: 3072 # Hugepagesize: 2048 kB total_pages = 0 - page_size_kb = 0 + page_size_bytes = 0 for line in stdout.splitlines(): parts = line.split() if len(parts) < 2: @@ -568,9 +544,11 @@ def hugepages_allocated(request) -> bool: if parts[0] == "HugePages_Total:": total_pages = int(parts[1]) elif parts[0] == "Hugepagesize:": - page_size_kb = int(parts[1]) + # The value carries its own unit (e.g. "2048 kB"), so reuse the + # size parser rather than assuming kB. + page_size_bytes = _parse_size_to_bytes(" ".join(parts[1:])) - allocated_bytes = total_pages * page_size_kb * 1024 + allocated_bytes = total_pages * page_size_bytes # ``dpdk-hugepages.py --setup `` actually allocates double the # requested amount (e.g. ``--setup 4G`` allocates 8G), so halve the read # value to compare against the requested (single) amount. Without this, @@ -588,14 +566,14 @@ def check_hugepages(request) -> None: return requested_bytes = request.config.getoption("--suricata-hugepages") - logger.info("Allocating huge-pages: %s", _bytes_to_size(requested_bytes)) + logger.info("Allocating huge-pages: %s bytes", requested_bytes) process_set_hugepages = executable.Tool( - f"dpdk-hugepages.py --setup {_bytes_to_size(requested_bytes)}", + f"dpdk-hugepages.py --setup {requested_bytes}", sudo=True, executor=get_suri_executor(request), ) process_reserve_hugepages = executable.Tool( - f"dpdk-hugepages.py --reserve {_bytes_to_size(requested_bytes)}", + f"dpdk-hugepages.py --reserve {requested_bytes}", sudo=True, executor=get_suri_executor(request), ) @@ -621,11 +599,6 @@ def check_hugepages(request) -> None: reserve_e, ) return - else: - # --setup succeeded; still reserve the pages so they are actually - # pinned for Suricata. - _, stderr = process_reserve_hugepages.run() - stderr_parts.append(stderr) combined_stderr = "\n".join(part for part in stderr_parts if part) assert combined_stderr == "", f"Error while allocating hugepages: {combined_stderr}" From 4f54a78b6d8780297fb5f4d4ab8a64bbd0ba8547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Hanko?= Date: Thu, 20 Aug 2026 12:52:42 +0200 Subject: [PATCH 3/3] conftest: drop halving of allocated hugepages --- conftest.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/conftest.py b/conftest.py index e617339..9cb673e 100644 --- a/conftest.py +++ b/conftest.py @@ -549,11 +549,6 @@ def hugepages_allocated(request) -> bool: page_size_bytes = _parse_size_to_bytes(" ".join(parts[1:])) allocated_bytes = total_pages * page_size_bytes - # ``dpdk-hugepages.py --setup `` actually allocates double the - # requested amount (e.g. ``--setup 4G`` allocates 8G), so halve the read - # value to compare against the requested (single) amount. Without this, - # requesting 5G would see 8G already allocated and skip re-allocation. - allocated_bytes //= 2 requested_bytes = request.config.getoption("--suricata-hugepages") return allocated_bytes >= requested_bytes