refactor: index tests numerically and simplify makefile

.github/workflows/pr.yml

@@ -21,7 +21,7 @@ jobs:
       ENABLE_KVM: "false"
       KVM_DEVICE: /dev/null
       KUBECONFIG: ${{ github.workspace }}/metal-lab/kubeconfig.yaml
-      COMPATIBILITY_VALUES: ${{ github.workspace }}/metal-lab/values-containerlab-node1.yaml
+      VALUES: ${{ github.workspace }}/metal-lab/values-containerlab-node1.yaml
     steps:
       - name: Checkout test framework
         uses: actions/checkout@v4
@@ -57,10 +57,10 @@ jobs:
             fedhcp-deploy-wait tftp-deploy-wait
           sudo chown -R "$USER:$USER" .
 
-      - name: Run compatibility suite
+      - name: Run test suite
         run: |
           mkdir -p "${{ github.workspace }}/artifacts"
-          make test-compatibility-all \
+          make test \
             ASSERT_TIMEOUT=25m \
             CHAINSAW_EXTRA_FLAGS="--report-format JSON --report-name chainsaw-report --report-path ${{ github.workspace }}/artifacts"
 

Makefile

@@ -1,97 +1,48 @@
 CHAINSAW ?= chainsaw
 
-COMPATIBILITY_TEST_DIR := tests/compatibility
-COMPATIBILITY_VALUES ?= infra/kind/values-basic-go.yaml
+TEST_DIR := tests
+VALUES ?= infra/kind/values-basic-go.yaml
 ASSERT_TIMEOUT ?= 15m
 CHAINSAW_EXTRA_FLAGS ?=
 
+CHAINSAW_RUN = $(CHAINSAW) test --values $(VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS)
+
+TESTS := \
+	01-bmc-registration \
+	02-discovery \
+	03-power-annotation \
+	04-bmc-reset \
+	05-indicator-led \
+	06-biossettings-noreboot \
+	07-biossettings-reboot \
+	08-bmcsettings \
+	09-persistent-boot-order
+
 .PHONY: help
 help: ## Show available targets
 	@echo "test framework targets:"
 	@echo ""
-	@echo "  compatibility tests:"
-	@echo "    * values via COMPATIBILITY_VALUES (default infra/kind/values-basic-go.yaml)"
+	@echo "  tests:"
+	@echo "    * values via VALUES (default infra/kind/values-basic-go.yaml)"
 	@echo "    * assert timeout via ASSERT_TIMEOUT (default 15m; e.g. ASSERT_TIMEOUT=5m)"
 	@echo "    * extra chainsaw flags via CHAINSAW_EXTRA_FLAGS (e.g. CHAINSAW_EXTRA_FLAGS=\"--skip-delete -v\")"
 	@echo ""
-	@echo "    test-compatibility           (A1, A2)"
-	@echo "    test-compatibility-a1"
-	@echo "    test-compatibility-a2"
-	@echo "    test-compatibility-b         (B1-B3)"
-	@echo "    test-compatibility-b1        (power ops via operation annotation)"
-	@echo "    test-compatibility-b2        (BMC reset)"
-	@echo "    test-compatibility-b3        (indicator LED)"
-	@echo "    test-compatibility-d         (D1-D3)"
-	@echo "    test-compatibility-d1        (BIOSSettings, non-reboot setting)"
-	@echo "    test-compatibility-d2        (BIOSSettings, reboot-required setting)"
-	@echo "    test-compatibility-d3        (BMCSettings, Manager attribute; needs a Dell BMC)"
-	@echo "    test-compatibility-f         (F3)"
-	@echo "    test-compatibility-f3        (persistent boot order)"
+	@echo "    test                             (run every test)"
+	@echo "    test/01-bmc-registration         (BMC registration)"
+	@echo "    test/02-discovery                (server discovery and inventory)"
+	@echo "    test/03-power-annotation         (power ops via operation annotation)"
+	@echo "    test/04-bmc-reset                (BMC reset)"
+	@echo "    test/05-indicator-led            (indicator LED)"
+	@echo "    test/06-biossettings-noreboot    (BIOSSettings, non-reboot setting)"
+	@echo "    test/07-biossettings-reboot      (BIOSSettings, reboot-required setting)"
+	@echo "    test/08-bmcsettings              (BMCSettings, Manager attribute; needs a Dell BMC)"
+	@echo "    test/09-persistent-boot-order    (persistent boot order)"
 	@echo ""
-	@echo "    test-compatibility-all       (everything: A1,A2,B1-3,D1-3,F3)"
-	@echo ""
-
-.PHONY: test-compatibility test-compatibility-a1 test-compatibility-a2
-test-compatibility: ## Run A compatibility chainsaw tests (A1, A2)
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \
-		$(COMPATIBILITY_TEST_DIR)/a1-bmc-registration \
-		$(COMPATIBILITY_TEST_DIR)/a2-discovery
-
-test-compatibility-a1: ## Run A1 BMC registration
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/a1-bmc-registration
-
-test-compatibility-a2: ## Run A2 discovery
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/a2-discovery
-
-.PHONY: test-compatibility-b test-compatibility-b1 test-compatibility-b2 test-compatibility-b3
-test-compatibility-b: ## Run all B power-management tests (B1-B3)
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \
-		$(COMPATIBILITY_TEST_DIR)/b1-power-annotation \
-		$(COMPATIBILITY_TEST_DIR)/b2-bmc-reset \
-		$(COMPATIBILITY_TEST_DIR)/b3-indicator-led
-
-test-compatibility-b1: ## Run B1 power operations via operation annotation
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/b1-power-annotation
-
-test-compatibility-b2: ## Run B2 BMC reset
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/b2-bmc-reset
-
-test-compatibility-b3: ## Run B3 indicator LED
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/b3-indicator-led
-
-.PHONY: test-compatibility-d test-compatibility-d1 test-compatibility-d2 test-compatibility-d3
-test-compatibility-d: ## Run all D settings tests (D1-D3)
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \
-		$(COMPATIBILITY_TEST_DIR)/d1-biossettings-noreboot \
-		$(COMPATIBILITY_TEST_DIR)/d2-biossettings-reboot \
-		$(COMPATIBILITY_TEST_DIR)/d3-bmcsettings
-
-test-compatibility-d1: ## Run D1 BIOSSettings, non-reboot setting
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/d1-biossettings-noreboot
-
-test-compatibility-d2: ## Run D2 BIOSSettings, reboot-required setting
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/d2-biossettings-reboot
-
-test-compatibility-d3: ## Run D3 BMCSettings, attribute change
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/d3-bmcsettings
 
-.PHONY: test-compatibility-f test-compatibility-f3
-test-compatibility-f: ## Run all F boot-control tests (F3)
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \
-		$(COMPATIBILITY_TEST_DIR)/f3-persistent-boot-order
+.PHONY: test $(addprefix test/,$(TESTS))
 
-test-compatibility-f3: ## Run F3 persistent boot order
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/f3-persistent-boot-order
+test: ## Run every test
+	$(CHAINSAW_RUN) $(addprefix $(TEST_DIR)/,$(TESTS))
 
-.PHONY: test-compatibility-all
-test-compatibility-all: ## Run every compatibility case (A1,A2,B1-3,D1-3,F3)
-	$(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \
-		$(COMPATIBILITY_TEST_DIR)/a1-bmc-registration \
-		$(COMPATIBILITY_TEST_DIR)/a2-discovery \
-		$(COMPATIBILITY_TEST_DIR)/b1-power-annotation \
-		$(COMPATIBILITY_TEST_DIR)/b2-bmc-reset \
-		$(COMPATIBILITY_TEST_DIR)/b3-indicator-led \
-		$(COMPATIBILITY_TEST_DIR)/d1-biossettings-noreboot \
-		$(COMPATIBILITY_TEST_DIR)/d2-biossettings-reboot \
-		$(COMPATIBILITY_TEST_DIR)/d3-bmcsettings \
-		$(COMPATIBILITY_TEST_DIR)/f3-persistent-boot-order
+$(addprefix test/,$(TESTS)): test/%: ## Run a single test
+	$(CHAINSAW_RUN) $(TEST_DIR)/$*

README.md

@@ -1,20 +1,19 @@
 # metal-operator test framework
 
 ## Test suite
-The `tests/compatibility` directory contains a suite of compatibility tests based on [chainsaw](https://kyverno.github.io/chainsaw/latest/). Every test case creates k8s resources in steps and asserts their status before proceeding to the next steps and implements common metal-operator workflows. Tests are independent from the infrastructure they run on and respect `KUBECONFIG`.
+The `tests` directory contains a suite of tests based on [chainsaw](https://kyverno.github.io/chainsaw/latest/). Every test case creates k8s resources in steps and asserts their status before proceeding to the next steps and implements common metal-operator workflows. Tests are independent from the infrastructure they run on and respect `KUBECONFIG`.
 
 ### Requirements
 * [chainsaw](https://kyverno.github.io/chainsaw/latest/)
 * A metal-operator installation and BMC to run tests against. This repository usually uses the locally virtualised [metal-lab](https://github.com/simontesar/metal-lab).
 
 ### Usage
-The server to run a test against is configured by passing a values file to chainsaw. The default file is `infra/kind/values-basic-go.yaml` that points to a redfish mock setup in the `kind` environment and can be overridden via `COMPATIBILITY_VALUES`.
+The server to run a test against is configured by passing a values file to chainsaw. The default file is `infra/kind/values-basic-go.yaml` that points to a redfish mock setup in the `kind` environment and can be overridden via `VALUES`.
 
 ```bash
-make test-compatibility                                                                    # Run all tests
-make test-compatibility-b                                                                  # Run a specific category of tests
-make test-compatibility-a1                                                                 # Run a specific case
-make test-compatibility-b1 COMPATIBILITY_VALUES=/path/to/metal-lab/values-containerlab-node1.yaml # Run against a specific BMC.
+make test                                                                    # Run all tests
+make test/01-bmc-registration                                                # Run a specific test
+make test/03-power-annotation VALUES=/path/to/metal-lab/values-containerlab-node1.yaml # Run against a specific BMC.
 ```
 
 ### Predefined values

infra/kind/README.md

@@ -64,7 +64,7 @@ Run a test against the basic go instance:
 ```shell
 $ cd ../..
 $ export KUBECONFIG=$PWD/kubeconfig.yaml
-$ make test-compatibility-b1 COMPATIBILITY_VALUES=infra/kind/values-basic-go.yaml CHAINSAW_EXTRA_FLAGS="--pause-on-failure"
+$ make test/03-power-annotation VALUES=infra/kind/values-basic-go.yaml CHAINSAW_EXTRA_FLAGS="--pause-on-failure"
 ```
 
 Tear down:

tests/compatibility/a1-bmc-registration/chainsaw-test.yaml → tests/01-bmc-registration/chainsaw-test.yaml

@@ -1,9 +1,9 @@
-# A1 — BMC registration
+# 01 — BMC registration
 # Asserts model/firmwareVersion/powerState
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: a1-bmc-registration
+  name: 01-bmc-registration
 spec:
   concurrent: false
   steps:

tests/compatibility/a2-discovery/chainsaw-test.yaml → tests/02-discovery/chainsaw-test.yaml

@@ -1,8 +1,8 @@
-# A2 — Server discovery and inventory
+# 02 — Server discovery and inventory
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: a2-discovery
+  name: 02-discovery
 spec:
   concurrent: false
   steps:

tests/compatibility/b1-power-annotation/chainsaw-test.yaml → tests/03-power-annotation/chainsaw-test.yaml

@@ -1,8 +1,8 @@
-# B1 — Power operations via operation annotation
+# 03 — Power operations via operation annotation
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: b1-power-annotation
+  name: 03-power-annotation
 spec:
   concurrent: false
   steps:

tests/compatibility/b2-bmc-reset/chainsaw-test.yaml → tests/04-bmc-reset/chainsaw-test.yaml

@@ -1,9 +1,9 @@
-# B2 — BMC reset (see cases.md)
+# 04 — BMC reset
 # - Should be extended to asser status.lastResetTime once testing agains physical servers
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: b2-bmc-reset
+  name: 04-bmc-reset
 spec:
   concurrent: false
   steps:

tests/compatibility/b3-indicator-led/chainsaw-test.yaml → tests/05-indicator-led/chainsaw-test.yaml

@@ -1,9 +1,9 @@
-# B3 — Indicator LED
+# 05 — Indicator LED
 # - Kept as a separate test to be skippable for models that do not support LED control
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: b3-indicator-led
+  name: 05-indicator-led
 spec:
   concurrent: false
   steps:

tests/compatibility/d1-biossettings-noreboot/chainsaw-test.yaml → tests/06-biossettings-noreboot/chainsaw-test.yaml

@@ -1,7 +1,7 @@
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: d1-biossettings-noreboot
+  name: 06-biossettings-noreboot
 spec:
   concurrent: false
   steps:

tests/compatibility/d2-biossettings-reboot/chainsaw-test.yaml → tests/07-biossettings-reboot/chainsaw-test.yaml

@@ -1,7 +1,7 @@
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: d2-biossettings-reboot
+  name: 07-biossettings-reboot
 spec:
   concurrent: false
   steps:

tests/compatibility/d3-bmcsettings/chainsaw-test.yaml → tests/08-bmcsettings/chainsaw-test.yaml

@@ -1,7 +1,7 @@
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: d3-bmcsettings
+  name: 08-bmcsettings
 spec:
   concurrent: false
   steps:

tests/compatibility/f3-persistent-boot-order/chainsaw-test.yaml → tests/09-persistent-boot-order/chainsaw-test.yaml

@@ -1,7 +1,7 @@
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: f3-persistent-boot-order
+  name: 09-persistent-boot-order
 spec:
   concurrent: false
   steps: