Merge pull request #93 from pythonbpf/fix/verifier-level-tests

add verifier-level tests

Author: r41k0u

Makefile

@@ -10,12 +10,13 @@ test:
 	pytest tests/ -W ignore::DeprecationWarning -v --tb=short -m "not verifier"
 
 test-cov:
-	pytest tests/ -v --tb=short -m "not verifier" \
+	pytest tests/ -W ignore::DeprecationWarning -v --tb=short -m "not verifier" \
 		--cov=pythonbpf --cov-report=term-missing --cov-report=html
 
 test-verifier:
-	@echo "NOTE: verifier tests require sudo and bpftool. Uses sudo .venv/bin/python3."
-	pytest tests/test_verifier.py -v --tb=short -m verifier
+	@echo "NOTE: verifier tests shell out to 'sudo bpftool'; run 'sudo -v' first so"
+	@echo "      the timestamp does not lapse mid-run. bpftool must be installed."
+	pytest tests/test_verifier.py -W ignore::DeprecationWarning -v --tb=short -m verifier
 
 all: clean install
 

tests/README.md

@@ -69,8 +69,16 @@ Known-broken tests are declared in `tests/test_config.toml`:
 "failing_tests/my_test.py" = {reason = "...", level = "ir"}
 ```
 
-- `level = "ir"` — fails during IR generation; both IR and LLC tests are marked xfail.
-- `level = "llc"` — IR generates fine but `llc` rejects it; only the LLC test is marked xfail.
+- `level = "ir"` — fails during IR generation.
+- `level = "llc"` — IR generates fine but `llc` rejects it.
+- `level = "verifier"` — IR and `llc` both succeed, but the kernel verifier rejects the object.
+
+A failure at one level implies failure at every later one, so the declared level marks
+that level **and all later ones** xfail. An `"ir"` entry is xfail at all three levels; a
+`"verifier"` entry is xfail at level 3 only and must still pass levels 1 and 2.
+
+Every test file runs at every level, including the ones declared here — level 3 does not
+skip declared failures, it reports them as expected ones.
 
 All xfails use `strict = True`: if a test starts **passing** it shows up as **XPASS** and is treated as a test failure. This is intentional — it means the bug was fixed and the test should be promoted to `passing_tests/`.
 

tests/conftest.py

@@ -19,6 +19,7 @@
 
 import pytest
 
+from tests.framework.bpf_test_case import level_index
 from tests.framework.collector import collect_all_test_files
 
 # ── vmlinux availability ────────────────────────────────────────────────────
@@ -70,14 +71,19 @@ def pytest_collection_modifyitems(items):
 
         # xfail (strict: XPASS counts as a test failure, alerting us to fixed bugs)
         if case.is_expected_fail:
-            # Level "ir" → fails at IR generation: xfail both IR and LLC tests
-            # Level "llc" → IR succeeds but LLC fails: only xfail the LLC test
-            is_llc_test = item.nodeid.startswith("tests/test_llc_compilation.py")
-
-            apply_xfail = (case.xfail_level == "ir") or (
-                case.xfail_level == "llc" and is_llc_test
-            )
-            if apply_xfail:
+            # A failure at one level implies failure at every later one, so mark
+            # this item xfail whenever the declared level is at or before it:
+            #   "ir"       → IR, LLC and verifier
+            #   "llc"      → LLC and verifier (IR is expected to succeed)
+            #   "verifier" → verifier only
+            if item.nodeid.startswith("tests/test_verifier.py"):
+                item_level = "verifier"
+            elif item.nodeid.startswith("tests/test_llc_compilation.py"):
+                item_level = "llc"
+            else:
+                item_level = "ir"
+
+            if level_index(case.xfail_level) <= level_index(item_level):
                 item.add_marker(
                     pytest.mark.xfail(
                         reason=case.xfail_reason,

tests/failing_tests/if.py

@@ -3,7 +3,7 @@
 
 
 @bpf
-@section("sometag1")
+@section("tracepoint/syscalls/sys_enter_execve")
 def sometag(ctx: c_void_p) -> c_int64:
     if 3 + 2 == 5:
         return c_int64(5)

tests/framework/bpf_test_case.py

@@ -1,14 +1,28 @@
 from dataclasses import dataclass
 from pathlib import Path
 
+# The three test levels, in pipeline order. A test declared as failing at one
+# level is also expected to fail at every later level: a program that cannot
+# generate IR cannot reach llc, and one that llc rejects never reaches the
+# kernel. Used by conftest to decide which items to mark xfail.
+LEVELS = ("ir", "llc", "verifier")
+
+
+def level_index(level: str) -> int:
+    """Position of a level in the pipeline. Unknown levels sort first ("ir")."""
+    try:
+        return LEVELS.index(level)
+    except ValueError:
+        return 0
+
 
 @dataclass
 class BpfTestCase:
     path: Path
     rel_path: str
     is_expected_fail: bool = False
     xfail_reason: str = ""
-    xfail_level: str = "ir"  # "ir" or "llc"
+    xfail_level: str = "ir"  # one of LEVELS
     needs_vmlinux: bool = False
     skip_reason: str = ""
 

tests/passing_tests/return.py

@@ -3,7 +3,7 @@
 
 
 @bpf
-@section("sometag1")
+@section("tracepoint/syscalls/sys_enter_execve")
 def sometag(ctx: c_void_p) -> c_int64:
     return c_int64(1 - 1)
 

tests/passing_tests/var_rval.py

@@ -5,7 +5,7 @@
 
 
 @bpf
-@section("sometag1")
+@section("tracepoint/syscalls/sys_enter_execve")
 def sometag(ctx: c_void_p) -> c_int64:
     a = 1 - 1
     return c_int64(a)

tests/test_verifier.py

@@ -24,12 +24,12 @@
 from tests.framework.verifier import verify_object
 
 
-def _passing_test_files():
-    return [c.path for c in collect_all_test_files() if not c.is_expected_fail]
+def _verifier_test_files():
+    return [c.path for c in collect_all_test_files()]
 
 
-def _passing_test_ids():
-    return [c.rel_path for c in collect_all_test_files() if not c.is_expected_fail]
+def _verifier_test_ids():
+    return [c.rel_path for c in collect_all_test_files()]
 
 
 def _get_rejection_reason(verifier_test_file: Path, output) -> str:
@@ -43,11 +43,16 @@ def _get_rejection_reason(verifier_test_file: Path, output) -> str:
     return errstr
 
 
+# Every test file runs at this level, including the ones declared in
+# test_config.toml. conftest marks those xfail, so an "ir"- or "llc"-level
+# failure is still reported as an expected failure rather than being silently
+# dropped from the level-3 run — and a "verifier"-level entry becomes possible
+# at all.
 @pytest.mark.verifier
 @pytest.mark.parametrize(
     "verifier_test_file",
-    _passing_test_files(),
-    ids=_passing_test_ids(),
+    _verifier_test_files(),
+    ids=_verifier_test_ids(),
 )
 def test_kernel_verifier(verifier_test_file: Path, tmp_path, caplog):
     """Compile the BPF test and verify it passes the kernel verifier."""