Add soak testing and refactor system test harness

Makefile

@@ -11,6 +11,7 @@ DOCS_DIR := docs
 
 PKG := pyrobusta
 
+PROJECT_ROOT := $(shell pwd)
 MICROPY_ROOT := external/micropython
 MPY_CROSS := $(MICROPY_ROOT)/mpy-cross/build/mpy-cross
 MICROPYTHON := $(MICROPY_ROOT)/ports/unix/build-standard/micropython
@@ -253,7 +254,7 @@ black:
 .PHONY: unit-test
 unit-test:
 	@echo "Running unit tests"
-	@python3 -m unittest -v
+	@python3 -m unittest discover -s tests/unit -v
 
 # -----------------------------
 # Run all static checkers
@@ -310,23 +311,49 @@ test-device: stage-test #clean-device upload
 # Performance testing
 # ================================================
 
+# -----------------------------
+# Run HTTP soak tests
+# -----------------------------
+.PHONY: perf-test-http-soak
+perf-test-http-soak:
+	@mpremote $(DEVICE) soft-reset
+	mpremote $(DEVICE) cp $(PT_DIR)/app_base.py :app_base.py
+	mpremote $(DEVICE) cp $(PT_DIR)/app_multipart.py :app_multipart.py
+	mpremote $(DEVICE) cp $(PT_DIR)/http_soak/boot.py :boot.py
+	cd $(PT_DIR) && python -m http_soak.test \
+		"$(DEVICE)" "$(DEVICE_IP)" "$(DEVICE_NAME)" \
+		"$(PROJECT_ROOT)/$(DOCS_DIR)/soak"
+
 # -----------------------------
 # Run HTTP dimensioning tests
 # -----------------------------
 .PHONY: perf-test-http-dimensioning
 perf-test-http-dimensioning:
 	@mpremote $(DEVICE) soft-reset
-	mpremote $(DEVICE) cp $(PT_DIR)/http_dimensioning/app_base.py :app_base.py
-	mpremote $(DEVICE) cp $(PT_DIR)/http_dimensioning/app_multipart.py :app_multipart.py
+	mpremote $(DEVICE) cp $(PT_DIR)/app_base.py :app_base.py
+	mpremote $(DEVICE) cp $(PT_DIR)/app_multipart.py :app_multipart.py
 	mpremote $(DEVICE) cp $(PT_DIR)/http_dimensioning/boot.py :boot.py
-	@mpremote $(DEVICE) reset
-	$(PT_DIR)/http_dimensioning/test.py "$(DEVICE)" "$(DEVICE_IP)" "$(DEVICE_NAME)"
+	cd $(PT_DIR) && python -m http_dimensioning.test \
+		"$(DEVICE)" "$(DEVICE_IP)" "$(DEVICE_NAME)" \
+		"$(PROJECT_ROOT)/$(DOCS_DIR)/dimensioning"
 
 # -----------------------------
 # Run all performance tests
 # -----------------------------
 .PHONY: perf-test-device
-perf-test-device: perf-test-http-dimensioning
+perf-test-device: perf-test-http-dimensioning perf-test-http-soak
+
+# -----------------------------
+# Regenerate plots
+# -----------------------------
+.PHONY: perf-test-regenerate-plots
+perf-test-regenerate-plots:
+	@for type in dimensioning soak; do \
+		[ -d docs/$$type/esp32_c3 ] && \
+			python3 tests/system/summary.py ESP32-C3 docs/$$type; \
+		[ -d docs/$$type/esp32_s3 ] && \
+			python3 tests/system/summary.py ESP32-S3 docs/$$type; \
+	done
 
 # ================================================
 # Utilities for TLS

tests/.pylintrc

@@ -6,3 +6,5 @@ disable=W0212,
         R0904,
         R0902,
         R0801
+
+init-hook='import os, sys; tests = os.path.join(os.getcwd(), "tests"); [sys.path.append(d) for d, _, _ in os.walk(tests)];'

tests/system/http_dimensioning/app_base.py → tests/system/app_base.py

@@ -1,10 +1,12 @@
+# pylint: disable=E0401
 from pyrobusta.protocol.http import HttpEngine
 
 
-def chunk_handler(_, chunk):
+def chunk_handler(_, chunk: bytes):
     if not chunk:  # Received terminating chunk/part
         return "text/plain", "OK"
-    pass  # process chunk data as needed
+    # <process chunk data>
+    return None
 
 
 def load():

tests/system/http_dimensioning/app_multipart.py → tests/system/app_multipart.py

@@ -1,7 +1,8 @@
+# pylint: disable=E0401
 from pyrobusta.protocol.http import HttpEngine
 
 
-def multipart_response(num_responses, part_size):
+def multipart_response(num_responses: int, part_size: int):
     i = 0
 
     def response_generator():
@@ -14,19 +15,20 @@ def response_generator():
     return response_generator
 
 
-def multipart_producer(http_ctx, _):
+def multipart_producer(http_ctx: HttpEngine, _):
     part_count = int(http_ctx.headers.get("x-part-count", 1))
     part_size = int(http_ctx.headers.get("x-part-size", 1024))
     return "multipart/form-data", multipart_response(part_count, part_size)
 
 
-def multipart_handler(http_ctx, _):
+def multipart_handler(http_ctx: HttpEngine, _):
     if (
         http_ctx.headers.get("content-type", "").startswith("multipart/form-data")
         and http_ctx.mp_is_last
     ):
         return "text/plain", "OK"
-    pass  # process part data as needed
+    # <process part data>
+    return None
 
 
 def load():

tests/system/device.py

@@ -0,0 +1,127 @@
+import subprocess
+import tempfile
+import socket
+import time
+
+
+class Device:
+    def __init__(
+        self, device_id: str, device_ip: str, device_name: str, base_config: dict
+    ):
+        self.device_id = device_id
+        self.device_ip = device_ip
+        self.device_name = device_name
+        self.base_config = base_config
+        self.current_config = {}
+
+        self.validate_device_ip()
+
+    def apply_base_config(self):
+        self.apply_config(self.base_config)
+
+    def apply_config(self, config: dict):
+        """
+        Apply device configuration with mpremote
+        """
+        self.current_config = dict(config)
+
+        subprocess.run(["mpremote", self.device_id, "soft-reset"], check=True)
+
+        with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as tmp:
+            config_lines = subprocess.run(
+                ["mpremote", self.device_id, "cat", ":/pyrobusta.env"],
+                check=True,
+                stdout=subprocess.PIPE,
+                text=True,
+            ).stdout.splitlines()
+
+            current_config = {}
+
+            for line in config_lines:
+                line = line.rstrip("\r\n").split("#")[0]
+                if not line.strip():
+                    continue
+                parts = line.split("=", 1)
+                key = parts[0].strip()
+                value = parts[1].strip().strip("'").strip('"')
+                current_config[key] = value
+
+            current_config.update(config)
+
+            tmp.write(
+                "\n".join([f"{key}={value}" for key, value in current_config.items()])
+                + "\n"
+            )
+            tmp.flush()
+
+            subprocess.run(
+                ["mpremote", self.device_id, "cp", tmp.name, ":/pyrobusta.env"],
+                check=True,
+            )
+            subprocess.run(["mpremote", self.device_id, "reset"], check=True)
+            time.sleep(5)  # Allow the device to initialize
+
+    def get_mem_params(self):
+        """
+        Determine SRAM and buffer settings of a device
+        """
+        sram_bytes = int(
+            subprocess.run(
+                [
+                    "mpremote",
+                    self.device_id,
+                    "exec",
+                    "import gc\nprint(gc.mem_free() + gc.mem_alloc())",
+                ],
+                check=True,
+                stdout=subprocess.PIPE,
+                text=True,
+            ).stdout
+        )
+
+        (
+            send_buf_min_bytes,
+            send_buf_max_bytes,
+            recv_buf_min_bytes,
+            recv_buf_max_bytes,
+            con_overhead,
+        ) = [
+            int(i)
+            for i in subprocess.run(
+                [
+                    "mpremote",
+                    self.device_id,
+                    "exec",
+                    (
+                        "from pyrobusta.server.http_server import HttpServer\n"
+                        "print( HttpServer.SEND_BUF_MIN_BYTES,\n"
+                        "HttpServer.SEND_BUF_MAX_BYTES,\n"
+                        "HttpServer.RECV_BUF_MIN_BYTES,\n"
+                        "HttpServer.RECV_BUF_MAX_BYTES,\n"
+                        "HttpServer.CON_OVERHEAD_BYTES,\n"
+                        ")"
+                    ),
+                ],
+                check=True,
+                stdout=subprocess.PIPE,
+                text=True,
+            ).stdout.split()
+        ]
+
+        buffer_small = send_buf_min_bytes + recv_buf_min_bytes + con_overhead
+        buffer_large = send_buf_max_bytes + recv_buf_max_bytes + con_overhead
+        return sram_bytes, buffer_small, buffer_large
+
+    def validate_device_ip(self):
+        """
+        Check if a device IP is valid
+        """
+        try:
+            socket.inet_aton(self.device_ip)
+        except socket.error as exc:
+            raise ValueError(f"Invalid device address: {self.device_ip}") from exc
+
+    def get_host(self):
+        proto = "https" if self.current_config["tls"] else "http"
+        port = 4443 if self.current_config["tls"] else 8080
+        return f"{proto}://{self.device_ip}:{port}"

tests/system/http_dimensioning/boot.py

@@ -1,31 +1,59 @@
+# pylint: disable=E0401,C0415,E1101,E0611
 # This file is executed on every boot (including wake-boot from deepsleep)
-import machine
 import asyncio
+import os
+import time
 from gc import mem_alloc, collect
+import machine
 
-import pyrobusta.server.http_server as http_server
+from pyrobusta.server import http_server
 from pyrobusta.protocol.http import HttpEngine
 from pyrobusta.connectivity import wifi
 
 from pyrobusta.utils.config import get_config, CONF_HTTP_MULTIPART
 
-TS_DURATION = 100
-MEM_TIME_SERIES = [0] * TS_DURATION
+LOG_FILE = "heap_usage.csv"
+
+SAMPLE_PERIOD = 5  # seconds
+FLUSH_PERIOD = 12  # Flush every minute
 
 
 @HttpEngine.route("/heap/time-series", "GET")
-def time_series(*_):
-    return "application/json", MEM_TIME_SERIES
+def time_series(http_ctx: HttpEngine, _):
+    http_ctx.set_response_header(
+        b"content-length", str(os.stat(LOG_FILE)[6]).encode("ascii")
+    )
+    http_ctx.set_response_header(b"content-type", b"text/csv")
+    http_ctx.terminate(200)
+    # pylint: disable=R1732
+    http_ctx.resp_handler = open(LOG_FILE, "rb", encoding="utf-8")
 
 
 async def mem_usage():
-    i = 0
+    flush_counter = 0
+    try:
+        os.remove(LOG_FILE)
+    except OSError:
+        pass
     collect()
-    while True:
-        collect()
-        MEM_TIME_SERIES[i] = mem_alloc()
-        await asyncio.sleep(1)
-        i = (i + 1) % TS_DURATION
+    with open(LOG_FILE, "a", encoding="utf-8") as log:
+        start_ts = time.ticks_ms()
+        log.write(f"0,{mem_alloc()}\n")
+
+        next_sample = time.ticks_add(start_ts, SAMPLE_PERIOD * 1000)
+
+        while True:
+            delay = time.ticks_diff(next_sample, time.ticks_ms())
+            if delay > 0:
+                await asyncio.sleep_ms(delay)
+
+            collect()
+            elapsed = time.ticks_diff(time.ticks_ms(), start_ts) // 1000
+            log.write(f"{elapsed},{mem_alloc()}\n")
+            flush_counter = (flush_counter + 1) % FLUSH_PERIOD
+            if not flush_counter:
+                log.flush()
+            next_sample = time.ticks_add(next_sample, SAMPLE_PERIOD * 1000)
 
 
 async def main():