Merge pull request #109 from pythonbpf/tests/loop-selftests

Port selftests for loop (non-volatile) which are feasible now

Author: r41k0u

tests/framework/verifier.py

@@ -3,25 +3,34 @@
 from collections import namedtuple
 from pathlib import Path
 
+Output = namedtuple("Output", ["stdout", "stderr"])
 
-def verify_object(obj_path: Path) -> tuple[bool, str]:
-    """Run bpftool prog load -d to verify a BPF object file against the kernel verifier.
+
+def verify_object(obj_path: Path) -> tuple[bool, Output]:
+    """Run bpftool prog load to verify a BPF object file against the kernel verifier.
 
     Pins the program temporarily at /sys/fs/bpf/bpf_prog_test_<uuid>, then removes it.
-    Returns (success, combined_output). Requires sudo / root.
+    Returns (success, Output(stdout, stderr)). Requires sudo / root.
+
+    No -d: it asks the kernel for the full instruction-level log (log level
+    1+2+4) on every load. For a program that makes the verifier walk a loop that
+    log outgrows libbpf's buffer, the kernel then fails the load with ENOSPC, and
+    libbpf doubles the buffer and verifies the whole program again, until the
+    load takes longer than the timeout. A rejection still comes with its log:
+    libbpf retries a failed load at log level 1 and prints the log as a warning.
     """
     pin_path = f"/sys/fs/bpf/bpf_prog_test_{uuid.uuid4().hex[:8]}"
     try:
         result = subprocess.run(
-            ["sudo", "bpftool", "prog", "load", "-d", str(obj_path), pin_path],
+            ["sudo", "bpftool", "prog", "load", str(obj_path), pin_path],
             capture_output=True,
             text=True,
             timeout=30,
         )
-        Output = namedtuple("Output", ["stdout", "stderr"])
-        output = Output(stdout=result.stdout, stderr=result.stderr)
-        return result.returncode == 0, output
+        return result.returncode == 0, Output(
+            stdout=result.stdout, stderr=result.stderr
+        )
     except subprocess.TimeoutExpired:
-        return False, "bpftool timed out after 30s"
+        return False, Output(stdout="", stderr="bpftool timed out after 30s")
     finally:
         subprocess.run(["sudo", "rm", "-f", pin_path], check=False, capture_output=True)

tests/kernel_selftest_equivalent/PORTING-NOTES.md

@@ -200,6 +200,41 @@ Globals no longer appear as a blocker at all. The next unlocks by count are help
 long tail, but `bpf_get_current_task`, `bpf_ktime_get_boot_ns` and the `bpf_probe_read_user*`
 family recur), array maps, and typed program arguments.
 
+## Fourth batch: loops
+
+With `for`/`while` lowering (#106) the audit no longer counts a plain loop as a hard
+blocker; the bpf_for/bpf_repeat iterators and the `bpf_loop` helper still are (38
+programs), and `do { } while (0)` macros are no longer mistaken for loops. Against
+bpf-next `5714ca8` that takes the programs with no hard blocker from 29 to 39. Four of
+the ten are ported, all from `prog_tests/bpf_verif_scale.c`:
+
+| Port | Upstream | Section | Upstream BPF insns | Port BPF insns |
+|---|---|---|---|---|
+| `vmlinux/loop1.py` | `loop1.c` | `raw_tracepoint/kfree_skb` | 21 | 20 |
+| `vmlinux/loop2.py` | `loop2.c` | `raw_tracepoint/consume_skb` | 12 | 2 |
+| `vmlinux/loop4.py` | `loop4.c` | `socket` | 14 | 97 |
+| `vmlinux/loop5.py` | `loop5.c` | `socket` | 18 | 13 |
+
+The loop lowering matches clang's: with `volatile`, `barrier()` and the no-unroll pragma
+taken out of the C, clang emits the same 2 and 97 instructions for `loop2` and `loop4`.
+Those three are what the ports cannot say, and they are the point of the upstream tests,
+which exist to make the verifier walk a loop. `loop2` folds to `return 42` and `loop4`
+unrolls, so those two ports compile, load and verify but no longer exercise what their
+upstream counterparts do. Each says so in its header; they are the first tests to
+tighten once volatile reads have a spelling. `loop1` and `loop5` keep their loops.
+
+The other six stay unported:
+
+- `loop3.c` is a negative test (the verifier must give up on a 2^32-iteration loop), and
+  without `volatile` LLVM replaces that loop with a closed form.
+- `test_skb_ctx.c` needs a subscript on a context array (`skb->cb[i]`) and writes to
+  context fields. Today the first fails with a bare `TypeError` in `handle_cond` and the
+  second with a `KeyError` in `handle_struct_field_assignment`.
+- `test_verif_scale1.c`, `2` and `3` contain no loop, only a `do { } while (0)` macro;
+  they need `jhash` over packet bytes through casts, and `scale1` a `noinline` subprogram.
+- `test_parse_tcp_hdr_opt.c` reads packet bytes through casts and passes a struct pointer
+  to a static function.
+
 ## 3. Incidental findings
 
 - **Nested struct field access fails with a misleading error.** `ctx.regs.ip` reports

tests/kernel_selftest_equivalent/vmlinux/loop1.py

@@ -0,0 +1,56 @@
+# Ported from Linux tools/testing/selftests/bpf/progs/loop1.c
+#
+# A verifier-scale test (prog_tests/bpf_verif_scale.c, test_verif_scale_loop1):
+# two nested bounded loops, the inner one bounded by the outer counter, so the
+# verifier has to prove termination of ~45000 iterations. Upstream:
+#
+#     SEC("raw_tracepoint/kfree_skb")
+#     int nested_loops(volatile struct pt_regs* ctx)
+#     {
+#             int i, j, sum = 0, m;
+#
+#             for (j = 0; j < 300; j++)
+#                     for (i = 0; i < j; i++) {
+#                             if (j & 1)
+#                                     m = PT_REGS_RC(ctx);
+#                             else
+#                                     m = j;
+#                             sum += i * m;
+#                     }
+#
+#             return sum;
+#     }
+#
+# PT_REGS_RC is ctx->ax on x86. The `volatile` on ctx, which forces a reload of
+# ctx->ax on every iteration, has no PythonBPF spelling yet; without it LLVM may
+# hoist the load out of the loops. Here it does not (the load sits in a branch),
+# and the object keeps upstream's nested-loop shape: 20 instructions against
+# clang's 21, with 64-bit loop counters where C's are 32-bit.
+
+from pythonbpf import bpf, section, bpfglobal, compile
+from vmlinux import struct_pt_regs
+from ctypes import c_int32
+
+
+@bpf
+@section("raw_tracepoint/kfree_skb")
+def nested_loops(ctx: struct_pt_regs) -> c_int32:
+    total: c_int32 = 0
+    m: c_int32 = 0
+    for j in range(300):
+        for i in range(j):
+            if j & 1:
+                m = ctx.ax
+            else:
+                m = j
+            total += i * m
+    return total
+
+
+@bpf
+@bpfglobal
+def LICENSE() -> str:
+    return "GPL"
+
+
+compile()

tests/kernel_selftest_equivalent/vmlinux/loop2.py

@@ -0,0 +1,57 @@
+# Ported from Linux tools/testing/selftests/bpf/progs/loop2.c
+#
+# A verifier-scale test (prog_tests/bpf_verif_scale.c, test_verif_scale_loop2):
+# an unbounded-looking `while (true)` that only terminates through a `break`,
+# with a step that depends on the context. Upstream:
+#
+#     SEC("raw_tracepoint/consume_skb")
+#     int while_true(volatile struct pt_regs* ctx)
+#     {
+#             int i = 0;
+#
+#             while (true) {
+#                     if (PT_REGS_RC(ctx) & 1)
+#                             i += 3;
+#                     else
+#                             i += 7;
+#                     if (i > 40)
+#                             break;
+#             }
+#
+#             return i;
+#     }
+#
+# PT_REGS_RC is ctx->ax on x86.
+#
+# NOTE: this port does not test what upstream tests. `volatile` has no
+# PythonBPF spelling yet, so ctx->ax is loaded once and LLVM sees that both
+# paths end at 42 (3 * 14 and 7 * 6): the program compiles to `return 42` and
+# the verifier never sees a loop. Clang does the same to the C with `volatile`
+# removed. Revisit when volatile reads land.
+
+from pythonbpf import bpf, section, bpfglobal, compile
+from vmlinux import struct_pt_regs
+from ctypes import c_int32
+
+
+@bpf
+@section("raw_tracepoint/consume_skb")
+def while_true(ctx: struct_pt_regs) -> c_int32:
+    i: c_int32 = 0
+    while True:
+        if ctx.ax & 1:
+            i += 3
+        else:
+            i += 7
+        if i > 40:
+            break
+    return i
+
+
+@bpf
+@bpfglobal
+def LICENSE() -> str:
+    return "GPL"
+
+
+compile()

tests/kernel_selftest_equivalent/vmlinux/loop4.py

@@ -0,0 +1,47 @@
+# Ported from Linux tools/testing/selftests/bpf/progs/loop4.c
+#
+# A verifier-scale test (prog_tests/bpf_verif_scale.c, test_verif_scale_loop4):
+# a 20-iteration loop whose body branches on a context field, so the verifier
+# sees 2^20 paths unless it prunes states. Upstream:
+#
+#     SEC("socket")
+#     int combinations(volatile struct __sk_buff* skb)
+#     {
+#             int ret = 0, i;
+#
+#             __pragma_loop_no_unroll
+#             for (i = 0; i < 20; i++)
+#                     if (skb->len)
+#                             ret |= 1 << i;
+#             return ret;
+#     }
+#
+# NOTE: this port does not test what upstream tests. `volatile` and the
+# no-unroll pragma have no PythonBPF spelling, so skb->len is loaded once and
+# LLVM fully unrolls the loop (97 instructions against upstream's 14): the
+# verifier walks straight-line code instead of pruning states across a loop.
+# Clang does the same to the C with `volatile` and the pragma removed. Revisit
+# when volatile reads land.
+
+from pythonbpf import bpf, section, bpfglobal, compile
+from vmlinux import struct___sk_buff
+from ctypes import c_int32
+
+
+@bpf
+@section("socket")
+def combinations(skb: struct___sk_buff) -> c_int32:
+    ret: c_int32 = 0
+    for i in range(20):
+        if skb.len:
+            ret |= 1 << i
+    return ret
+
+
+@bpf
+@bpfglobal
+def LICENSE() -> str:
+    return "GPL"
+
+
+compile()

tests/kernel_selftest_equivalent/vmlinux/loop5.py

@@ -0,0 +1,68 @@
+# Ported from Linux tools/testing/selftests/bpf/progs/loop5.c
+#
+# A verifier-scale test (prog_tests/bpf_verif_scale.c, test_verif_scale_loop5):
+# `while (1)` with several exits, where whether an exit is ever reached depends
+# on a context field. Upstream:
+#
+#     SEC("socket")
+#     int while_true(volatile struct __sk_buff* skb)
+#     {
+#             int i = 0;
+#
+#             while (1) {
+#                     if (skb->len)
+#                             i += 3;
+#                     else
+#                             i += 7;
+#                     if (i == 9)
+#                             break;
+#                     barrier();
+#                     if (i == 10)
+#                             break;
+#                     barrier();
+#                     if (i == 13)
+#                             break;
+#                     barrier();
+#                     if (i == 14)
+#                             break;
+#             }
+#             return i;
+#     }
+#
+# barrier() is an empty `asm volatile("" ::: "memory")` that stops clang from
+# merging the exits; PythonBPF has no compiler barrier, and LLVM merges the four
+# exits into one bitmask test. The loop survives, so the verifier still has to
+# prove it terminates.
+
+from pythonbpf import bpf, section, bpfglobal, compile
+from vmlinux import struct___sk_buff
+from ctypes import c_int32
+
+
+@bpf
+@section("socket")
+def while_true(skb: struct___sk_buff) -> c_int32:
+    i: c_int32 = 0
+    while True:
+        if skb.len:
+            i += 3
+        else:
+            i += 7
+        if i == 9:
+            break
+        if i == 10:
+            break
+        if i == 13:
+            break
+        if i == 14:
+            break
+    return i
+
+
+@bpf
+@bpfglobal
+def LICENSE() -> str:
+    return "GPL"
+
+
+compile()

tests/test_verifier.py

@@ -2,7 +2,7 @@
 Level 3 — Kernel verifier tests.
 
 For every passing BPF test file, compiles to a .o and runs:
-    sudo bpftool prog load -d <file.o> /sys/fs/bpf/bpf_prog_test_<id>
+    sudo bpftool prog load <file.o> /sys/fs/bpf/bpf_prog_test_<id>
 
 These tests are opt-in: they require sudo and kernel access, and are gated
 behind the `verifier` pytest mark. Run with:

tools/selftest-audit.py

@@ -15,7 +15,7 @@
 construct lists below) must be kept in step with the compiler by hand. Re-run
 it after a feature lands to see what the feature unlocked. The lists were last
 reconciled against pythonbpf/helper and pythonbpf/maps when scalar @bpfglobal
-support and integer signedness merged.
+support, integer signedness and loops merged.
 """
 
 import argparse
@@ -70,7 +70,10 @@
 # cost a careful rewrite can absorb). Order does not matter.
 PATTERNS = [
     # control flow
-    ("loop", "hard", r"\b(for|while)\s*\(|\bbpf_for\b|\bbpf_repeat\b|\bbpf_loop\s*\("),
+    # plain for/while lower since loops landed (#106); `do { } while (0)` is a
+    # macro idiom, not a loop. Iterator loops and the bpf_loop helper are not.
+    ("open_coded_loop", "hard", r"\bbpf_for(_each)?\b|\bbpf_repeat\b|\bbpf_loop\s*\("),
+    ("do_while", "soft", r"\}\s*while\s*\((?!\s*0\s*\))"),
     ("goto", "hard", r"\bgoto\s+\w+"),
     ("switch", "hard", r"\bswitch\s*\("),
     ("ternary", "soft", r"\?[^?:]*:"),