From 840d9866dd45d65940a28fdf916c2f883374024b Mon Sep 17 00:00:00 2001 From: simontesar Date: Fri, 4 Sep 2026 08:03:14 +0200 Subject: [PATCH 1/2] refactor: index tests numerically Signed-off-by: simontesar --- Makefile | 137 ++++++++---------- README.md | 7 +- infra/kind/README.md | 2 +- .../chainsaw-test.yaml | 4 +- .../chainsaw-test.yaml | 4 +- .../chainsaw-test.yaml | 4 +- .../ignitionsecret.yaml | 0 .../serverclaim.yaml | 0 .../chainsaw-test.yaml | 4 +- .../chainsaw-test.yaml | 4 +- .../biossettings-noreboot.yaml | 0 .../chainsaw-test.yaml | 2 +- .../biossettings-reboot.yaml | 0 .../chainsaw-test.yaml | 2 +- .../bmcsettings.yaml | 0 .../chainsaw-test.yaml | 2 +- .../bootorder-check-job.yaml | 0 .../chainsaw-test.yaml | 2 +- 18 files changed, 76 insertions(+), 98 deletions(-) rename tests/compatibility/{a1-bmc-registration => 01-bmc-registration}/chainsaw-test.yaml (93%) rename tests/compatibility/{a2-discovery => 02-discovery}/chainsaw-test.yaml (95%) rename tests/compatibility/{b1-power-annotation => 03-power-annotation}/chainsaw-test.yaml (98%) rename tests/compatibility/{b1-power-annotation => 03-power-annotation}/ignitionsecret.yaml (100%) rename tests/compatibility/{b1-power-annotation => 03-power-annotation}/serverclaim.yaml (100%) rename tests/compatibility/{b2-bmc-reset => 04-bmc-reset}/chainsaw-test.yaml (96%) rename tests/compatibility/{b3-indicator-led => 05-indicator-led}/chainsaw-test.yaml (97%) rename tests/compatibility/{d1-biossettings-noreboot => 06-biossettings-noreboot}/biossettings-noreboot.yaml (100%) rename tests/compatibility/{d1-biossettings-noreboot => 06-biossettings-noreboot}/chainsaw-test.yaml (98%) rename tests/compatibility/{d2-biossettings-reboot => 07-biossettings-reboot}/biossettings-reboot.yaml (100%) rename tests/compatibility/{d2-biossettings-reboot => 07-biossettings-reboot}/chainsaw-test.yaml (98%) rename tests/compatibility/{d3-bmcsettings => 08-bmcsettings}/bmcsettings.yaml (100%) rename tests/compatibility/{d3-bmcsettings => 08-bmcsettings}/chainsaw-test.yaml (98%) rename tests/compatibility/{f3-persistent-boot-order => 09-persistent-boot-order}/bootorder-check-job.yaml (100%) rename tests/compatibility/{f3-persistent-boot-order => 09-persistent-boot-order}/chainsaw-test.yaml (98%) diff --git a/Makefile b/Makefile index 9127c21..5148f26 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,19 @@ COMPATIBILITY_VALUES ?= infra/kind/values-basic-go.yaml ASSERT_TIMEOUT ?= 15m CHAINSAW_EXTRA_FLAGS ?= +CHAINSAW_RUN = $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) + +COMPATIBILITY_CASES := \ + 01-bmc-registration \ + 02-discovery \ + 03-power-annotation \ + 04-bmc-reset \ + 05-indicator-led \ + 06-biossettings-noreboot \ + 07-biossettings-reboot \ + 08-bmcsettings \ + 09-persistent-boot-order + .PHONY: help help: ## Show available targets @echo "test framework targets:" @@ -14,84 +27,50 @@ help: ## Show available targets @echo " * assert timeout via ASSERT_TIMEOUT (default 15m; e.g. ASSERT_TIMEOUT=5m)" @echo " * extra chainsaw flags via CHAINSAW_EXTRA_FLAGS (e.g. CHAINSAW_EXTRA_FLAGS=\"--skip-delete -v\")" @echo "" - @echo " test-compatibility (A1, A2)" - @echo " test-compatibility-a1" - @echo " test-compatibility-a2" - @echo " test-compatibility-b (B1-B3)" - @echo " test-compatibility-b1 (power ops via operation annotation)" - @echo " test-compatibility-b2 (BMC reset)" - @echo " test-compatibility-b3 (indicator LED)" - @echo " test-compatibility-d (D1-D3)" - @echo " test-compatibility-d1 (BIOSSettings, non-reboot setting)" - @echo " test-compatibility-d2 (BIOSSettings, reboot-required setting)" - @echo " test-compatibility-d3 (BMCSettings, Manager attribute; needs a Dell BMC)" - @echo " test-compatibility-f (F3)" - @echo " test-compatibility-f3 (persistent boot order)" - @echo "" - @echo " test-compatibility-all (everything: A1,A2,B1-3,D1-3,F3)" + @echo " test-compatibility (run every case)" + @echo " test-compatibility-all (alias of test-compatibility)" + @echo " test-compatibility-01 (BMC registration)" + @echo " test-compatibility-02 (server discovery and inventory)" + @echo " test-compatibility-03 (power ops via operation annotation)" + @echo " test-compatibility-04 (BMC reset)" + @echo " test-compatibility-05 (indicator LED)" + @echo " test-compatibility-06 (BIOSSettings, non-reboot setting)" + @echo " test-compatibility-07 (BIOSSettings, reboot-required setting)" + @echo " test-compatibility-08 (BMCSettings, Manager attribute; needs a Dell BMC)" + @echo " test-compatibility-09 (persistent boot order)" @echo "" -.PHONY: test-compatibility test-compatibility-a1 test-compatibility-a2 -test-compatibility: ## Run A compatibility chainsaw tests (A1, A2) - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \ - $(COMPATIBILITY_TEST_DIR)/a1-bmc-registration \ - $(COMPATIBILITY_TEST_DIR)/a2-discovery - -test-compatibility-a1: ## Run A1 BMC registration - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/a1-bmc-registration - -test-compatibility-a2: ## Run A2 discovery - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/a2-discovery - -.PHONY: test-compatibility-b test-compatibility-b1 test-compatibility-b2 test-compatibility-b3 -test-compatibility-b: ## Run all B power-management tests (B1-B3) - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \ - $(COMPATIBILITY_TEST_DIR)/b1-power-annotation \ - $(COMPATIBILITY_TEST_DIR)/b2-bmc-reset \ - $(COMPATIBILITY_TEST_DIR)/b3-indicator-led - -test-compatibility-b1: ## Run B1 power operations via operation annotation - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/b1-power-annotation - -test-compatibility-b2: ## Run B2 BMC reset - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/b2-bmc-reset - -test-compatibility-b3: ## Run B3 indicator LED - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/b3-indicator-led - -.PHONY: test-compatibility-d test-compatibility-d1 test-compatibility-d2 test-compatibility-d3 -test-compatibility-d: ## Run all D settings tests (D1-D3) - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \ - $(COMPATIBILITY_TEST_DIR)/d1-biossettings-noreboot \ - $(COMPATIBILITY_TEST_DIR)/d2-biossettings-reboot \ - $(COMPATIBILITY_TEST_DIR)/d3-bmcsettings - -test-compatibility-d1: ## Run D1 BIOSSettings, non-reboot setting - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/d1-biossettings-noreboot - -test-compatibility-d2: ## Run D2 BIOSSettings, reboot-required setting - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/d2-biossettings-reboot - -test-compatibility-d3: ## Run D3 BMCSettings, attribute change - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/d3-bmcsettings - -.PHONY: test-compatibility-f test-compatibility-f3 -test-compatibility-f: ## Run all F boot-control tests (F3) - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \ - $(COMPATIBILITY_TEST_DIR)/f3-persistent-boot-order - -test-compatibility-f3: ## Run F3 persistent boot order - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/f3-persistent-boot-order - -.PHONY: test-compatibility-all -test-compatibility-all: ## Run every compatibility case (A1,A2,B1-3,D1-3,F3) - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \ - $(COMPATIBILITY_TEST_DIR)/a1-bmc-registration \ - $(COMPATIBILITY_TEST_DIR)/a2-discovery \ - $(COMPATIBILITY_TEST_DIR)/b1-power-annotation \ - $(COMPATIBILITY_TEST_DIR)/b2-bmc-reset \ - $(COMPATIBILITY_TEST_DIR)/b3-indicator-led \ - $(COMPATIBILITY_TEST_DIR)/d1-biossettings-noreboot \ - $(COMPATIBILITY_TEST_DIR)/d2-biossettings-reboot \ - $(COMPATIBILITY_TEST_DIR)/d3-bmcsettings \ - $(COMPATIBILITY_TEST_DIR)/f3-persistent-boot-order +.PHONY: test-compatibility test-compatibility-all +test-compatibility: ## Run every compatibility case + $(CHAINSAW_RUN) $(addprefix $(COMPATIBILITY_TEST_DIR)/,$(COMPATIBILITY_CASES)) + +test-compatibility-all: test-compatibility ## Alias of test-compatibility + +.PHONY: test-compatibility-01 test-compatibility-02 test-compatibility-03 test-compatibility-04 test-compatibility-05 test-compatibility-06 test-compatibility-07 test-compatibility-08 test-compatibility-09 + +test-compatibility-01: ## Run 01 BMC registration + $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/01-bmc-registration + +test-compatibility-02: ## Run 02 server discovery and inventory + $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/02-discovery + +test-compatibility-03: ## Run 03 power operations via operation annotation + $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/03-power-annotation + +test-compatibility-04: ## Run 04 BMC reset + $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/04-bmc-reset + +test-compatibility-05: ## Run 05 indicator LED + $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/05-indicator-led + +test-compatibility-06: ## Run 06 BIOSSettings, non-reboot setting + $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/06-biossettings-noreboot + +test-compatibility-07: ## Run 07 BIOSSettings, reboot-required setting + $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/07-biossettings-reboot + +test-compatibility-08: ## Run 08 BMCSettings, attribute change + $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/08-bmcsettings + +test-compatibility-09: ## Run 09 persistent boot order + $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/09-persistent-boot-order diff --git a/README.md b/README.md index 5886431..c12025e 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,9 @@ The `tests/compatibility` directory contains a suite of compatibility tests base The server to run a test against is configured by passing a values file to chainsaw. The default file is `infra/kind/values-basic-go.yaml` that points to a redfish mock setup in the `kind` environment and can be overridden via `COMPATIBILITY_VALUES`. ```bash -make test-compatibility # Run all tests -make test-compatibility-b # Run a specific category of tests -make test-compatibility-a1 # Run a specific case -make test-compatibility-b1 COMPATIBILITY_VALUES=/path/to/metal-lab/values-containerlab-node1.yaml # Run against a specific BMC. +make test-compatibility # Run all cases +make test-compatibility-01 # Run a specific case +make test-compatibility-03 COMPATIBILITY_VALUES=/path/to/metal-lab/values-containerlab-node1.yaml # Run against a specific BMC. ``` ### Predefined values diff --git a/infra/kind/README.md b/infra/kind/README.md index 63c1198..46cf9b9 100644 --- a/infra/kind/README.md +++ b/infra/kind/README.md @@ -64,7 +64,7 @@ Run a test against the basic go instance: ```shell $ cd ../.. $ export KUBECONFIG=$PWD/kubeconfig.yaml -$ make test-compatibility-b1 COMPATIBILITY_VALUES=infra/kind/values-basic-go.yaml CHAINSAW_EXTRA_FLAGS="--pause-on-failure" +$ make test-compatibility-03 COMPATIBILITY_VALUES=infra/kind/values-basic-go.yaml CHAINSAW_EXTRA_FLAGS="--pause-on-failure" ``` Tear down: diff --git a/tests/compatibility/a1-bmc-registration/chainsaw-test.yaml b/tests/compatibility/01-bmc-registration/chainsaw-test.yaml similarity index 93% rename from tests/compatibility/a1-bmc-registration/chainsaw-test.yaml rename to tests/compatibility/01-bmc-registration/chainsaw-test.yaml index 73e292e..da58ea2 100644 --- a/tests/compatibility/a1-bmc-registration/chainsaw-test.yaml +++ b/tests/compatibility/01-bmc-registration/chainsaw-test.yaml @@ -1,9 +1,9 @@ -# A1 — BMC registration +# 01 — BMC registration # Asserts model/firmwareVersion/powerState apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: a1-bmc-registration + name: 01-bmc-registration spec: concurrent: false steps: diff --git a/tests/compatibility/a2-discovery/chainsaw-test.yaml b/tests/compatibility/02-discovery/chainsaw-test.yaml similarity index 95% rename from tests/compatibility/a2-discovery/chainsaw-test.yaml rename to tests/compatibility/02-discovery/chainsaw-test.yaml index 0ba5e79..988f59f 100644 --- a/tests/compatibility/a2-discovery/chainsaw-test.yaml +++ b/tests/compatibility/02-discovery/chainsaw-test.yaml @@ -1,8 +1,8 @@ -# A2 — Server discovery and inventory +# 02 — Server discovery and inventory apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: a2-discovery + name: 02-discovery spec: concurrent: false steps: diff --git a/tests/compatibility/b1-power-annotation/chainsaw-test.yaml b/tests/compatibility/03-power-annotation/chainsaw-test.yaml similarity index 98% rename from tests/compatibility/b1-power-annotation/chainsaw-test.yaml rename to tests/compatibility/03-power-annotation/chainsaw-test.yaml index be8d598..b386597 100644 --- a/tests/compatibility/b1-power-annotation/chainsaw-test.yaml +++ b/tests/compatibility/03-power-annotation/chainsaw-test.yaml @@ -1,8 +1,8 @@ -# B1 — Power operations via operation annotation +# 03 — Power operations via operation annotation apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: b1-power-annotation + name: 03-power-annotation spec: concurrent: false steps: diff --git a/tests/compatibility/b1-power-annotation/ignitionsecret.yaml b/tests/compatibility/03-power-annotation/ignitionsecret.yaml similarity index 100% rename from tests/compatibility/b1-power-annotation/ignitionsecret.yaml rename to tests/compatibility/03-power-annotation/ignitionsecret.yaml diff --git a/tests/compatibility/b1-power-annotation/serverclaim.yaml b/tests/compatibility/03-power-annotation/serverclaim.yaml similarity index 100% rename from tests/compatibility/b1-power-annotation/serverclaim.yaml rename to tests/compatibility/03-power-annotation/serverclaim.yaml diff --git a/tests/compatibility/b2-bmc-reset/chainsaw-test.yaml b/tests/compatibility/04-bmc-reset/chainsaw-test.yaml similarity index 96% rename from tests/compatibility/b2-bmc-reset/chainsaw-test.yaml rename to tests/compatibility/04-bmc-reset/chainsaw-test.yaml index 70b9f61..a6b5973 100644 --- a/tests/compatibility/b2-bmc-reset/chainsaw-test.yaml +++ b/tests/compatibility/04-bmc-reset/chainsaw-test.yaml @@ -1,9 +1,9 @@ -# B2 — BMC reset (see cases.md) +# 04 — BMC reset # - Should be extended to asser status.lastResetTime once testing agains physical servers apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: b2-bmc-reset + name: 04-bmc-reset spec: concurrent: false steps: diff --git a/tests/compatibility/b3-indicator-led/chainsaw-test.yaml b/tests/compatibility/05-indicator-led/chainsaw-test.yaml similarity index 97% rename from tests/compatibility/b3-indicator-led/chainsaw-test.yaml rename to tests/compatibility/05-indicator-led/chainsaw-test.yaml index 7ba8124..571713b 100644 --- a/tests/compatibility/b3-indicator-led/chainsaw-test.yaml +++ b/tests/compatibility/05-indicator-led/chainsaw-test.yaml @@ -1,9 +1,9 @@ -# B3 — Indicator LED +# 05 — Indicator LED # - Kept as a separate test to be skippable for models that do not support LED control apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: b3-indicator-led + name: 05-indicator-led spec: concurrent: false steps: diff --git a/tests/compatibility/d1-biossettings-noreboot/biossettings-noreboot.yaml b/tests/compatibility/06-biossettings-noreboot/biossettings-noreboot.yaml similarity index 100% rename from tests/compatibility/d1-biossettings-noreboot/biossettings-noreboot.yaml rename to tests/compatibility/06-biossettings-noreboot/biossettings-noreboot.yaml diff --git a/tests/compatibility/d1-biossettings-noreboot/chainsaw-test.yaml b/tests/compatibility/06-biossettings-noreboot/chainsaw-test.yaml similarity index 98% rename from tests/compatibility/d1-biossettings-noreboot/chainsaw-test.yaml rename to tests/compatibility/06-biossettings-noreboot/chainsaw-test.yaml index 0d23f3d..73b93ad 100644 --- a/tests/compatibility/d1-biossettings-noreboot/chainsaw-test.yaml +++ b/tests/compatibility/06-biossettings-noreboot/chainsaw-test.yaml @@ -1,7 +1,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: d1-biossettings-noreboot + name: 06-biossettings-noreboot spec: concurrent: false steps: diff --git a/tests/compatibility/d2-biossettings-reboot/biossettings-reboot.yaml b/tests/compatibility/07-biossettings-reboot/biossettings-reboot.yaml similarity index 100% rename from tests/compatibility/d2-biossettings-reboot/biossettings-reboot.yaml rename to tests/compatibility/07-biossettings-reboot/biossettings-reboot.yaml diff --git a/tests/compatibility/d2-biossettings-reboot/chainsaw-test.yaml b/tests/compatibility/07-biossettings-reboot/chainsaw-test.yaml similarity index 98% rename from tests/compatibility/d2-biossettings-reboot/chainsaw-test.yaml rename to tests/compatibility/07-biossettings-reboot/chainsaw-test.yaml index cd73911..6568c22 100644 --- a/tests/compatibility/d2-biossettings-reboot/chainsaw-test.yaml +++ b/tests/compatibility/07-biossettings-reboot/chainsaw-test.yaml @@ -1,7 +1,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: d2-biossettings-reboot + name: 07-biossettings-reboot spec: concurrent: false steps: diff --git a/tests/compatibility/d3-bmcsettings/bmcsettings.yaml b/tests/compatibility/08-bmcsettings/bmcsettings.yaml similarity index 100% rename from tests/compatibility/d3-bmcsettings/bmcsettings.yaml rename to tests/compatibility/08-bmcsettings/bmcsettings.yaml diff --git a/tests/compatibility/d3-bmcsettings/chainsaw-test.yaml b/tests/compatibility/08-bmcsettings/chainsaw-test.yaml similarity index 98% rename from tests/compatibility/d3-bmcsettings/chainsaw-test.yaml rename to tests/compatibility/08-bmcsettings/chainsaw-test.yaml index ae4e914..43613e0 100644 --- a/tests/compatibility/d3-bmcsettings/chainsaw-test.yaml +++ b/tests/compatibility/08-bmcsettings/chainsaw-test.yaml @@ -1,7 +1,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: d3-bmcsettings + name: 08-bmcsettings spec: concurrent: false steps: diff --git a/tests/compatibility/f3-persistent-boot-order/bootorder-check-job.yaml b/tests/compatibility/09-persistent-boot-order/bootorder-check-job.yaml similarity index 100% rename from tests/compatibility/f3-persistent-boot-order/bootorder-check-job.yaml rename to tests/compatibility/09-persistent-boot-order/bootorder-check-job.yaml diff --git a/tests/compatibility/f3-persistent-boot-order/chainsaw-test.yaml b/tests/compatibility/09-persistent-boot-order/chainsaw-test.yaml similarity index 98% rename from tests/compatibility/f3-persistent-boot-order/chainsaw-test.yaml rename to tests/compatibility/09-persistent-boot-order/chainsaw-test.yaml index d692235..423d419 100644 --- a/tests/compatibility/f3-persistent-boot-order/chainsaw-test.yaml +++ b/tests/compatibility/09-persistent-boot-order/chainsaw-test.yaml @@ -1,7 +1,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: f3-persistent-boot-order + name: 09-persistent-boot-order spec: concurrent: false steps: From 1aca26db5fb588e8b8e5852cb38d59bbef5852bd Mon Sep 17 00:00:00 2001 From: simontesar Date: Fri, 4 Sep 2026 11:21:25 +0200 Subject: [PATCH 2/2] refactor: remove unnecessary 'compatibility' prefixes in test names Signed-off-by: simontesar --- .github/workflows/pr.yml | 6 +- Makefile | 70 ++++++------------- README.md | 10 +-- infra/kind/README.md | 2 +- .../01-bmc-registration/chainsaw-test.yaml | 0 .../02-discovery/chainsaw-test.yaml | 0 .../03-power-annotation/chainsaw-test.yaml | 0 .../03-power-annotation/ignitionsecret.yaml | 0 .../03-power-annotation/serverclaim.yaml | 0 .../04-bmc-reset/chainsaw-test.yaml | 0 .../05-indicator-led/chainsaw-test.yaml | 0 .../biossettings-noreboot.yaml | 0 .../chainsaw-test.yaml | 0 .../biossettings-reboot.yaml | 0 .../07-biossettings-reboot/chainsaw-test.yaml | 0 .../08-bmcsettings/bmcsettings.yaml | 0 .../08-bmcsettings/chainsaw-test.yaml | 0 .../bootorder-check-job.yaml | 0 .../chainsaw-test.yaml | 0 tests/{compatibility => }/shared/bmc.yaml | 0 .../{compatibility => }/shared/bmcsecret.yaml | 0 21 files changed, 30 insertions(+), 58 deletions(-) rename tests/{compatibility => }/01-bmc-registration/chainsaw-test.yaml (100%) rename tests/{compatibility => }/02-discovery/chainsaw-test.yaml (100%) rename tests/{compatibility => }/03-power-annotation/chainsaw-test.yaml (100%) rename tests/{compatibility => }/03-power-annotation/ignitionsecret.yaml (100%) rename tests/{compatibility => }/03-power-annotation/serverclaim.yaml (100%) rename tests/{compatibility => }/04-bmc-reset/chainsaw-test.yaml (100%) rename tests/{compatibility => }/05-indicator-led/chainsaw-test.yaml (100%) rename tests/{compatibility => }/06-biossettings-noreboot/biossettings-noreboot.yaml (100%) rename tests/{compatibility => }/06-biossettings-noreboot/chainsaw-test.yaml (100%) rename tests/{compatibility => }/07-biossettings-reboot/biossettings-reboot.yaml (100%) rename tests/{compatibility => }/07-biossettings-reboot/chainsaw-test.yaml (100%) rename tests/{compatibility => }/08-bmcsettings/bmcsettings.yaml (100%) rename tests/{compatibility => }/08-bmcsettings/chainsaw-test.yaml (100%) rename tests/{compatibility => }/09-persistent-boot-order/bootorder-check-job.yaml (100%) rename tests/{compatibility => }/09-persistent-boot-order/chainsaw-test.yaml (100%) rename tests/{compatibility => }/shared/bmc.yaml (100%) rename tests/{compatibility => }/shared/bmcsecret.yaml (100%) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 544c33c..d62f97d 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -21,7 +21,7 @@ jobs: ENABLE_KVM: "false" KVM_DEVICE: /dev/null KUBECONFIG: ${{ github.workspace }}/metal-lab/kubeconfig.yaml - COMPATIBILITY_VALUES: ${{ github.workspace }}/metal-lab/values-containerlab-node1.yaml + VALUES: ${{ github.workspace }}/metal-lab/values-containerlab-node1.yaml steps: - name: Checkout test framework uses: actions/checkout@v4 @@ -57,10 +57,10 @@ jobs: fedhcp-deploy-wait tftp-deploy-wait sudo chown -R "$USER:$USER" . - - name: Run compatibility suite + - name: Run test suite run: | mkdir -p "${{ github.workspace }}/artifacts" - make test-compatibility-all \ + make test \ ASSERT_TIMEOUT=25m \ CHAINSAW_EXTRA_FLAGS="--report-format JSON --report-name chainsaw-report --report-path ${{ github.workspace }}/artifacts" diff --git a/Makefile b/Makefile index 5148f26..c7d0a29 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ CHAINSAW ?= chainsaw -COMPATIBILITY_TEST_DIR := tests/compatibility -COMPATIBILITY_VALUES ?= infra/kind/values-basic-go.yaml +TEST_DIR := tests +VALUES ?= infra/kind/values-basic-go.yaml ASSERT_TIMEOUT ?= 15m CHAINSAW_EXTRA_FLAGS ?= -CHAINSAW_RUN = $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) +CHAINSAW_RUN = $(CHAINSAW) test --values $(VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) -COMPATIBILITY_CASES := \ +TESTS := \ 01-bmc-registration \ 02-discovery \ 03-power-annotation \ @@ -22,55 +22,27 @@ COMPATIBILITY_CASES := \ help: ## Show available targets @echo "test framework targets:" @echo "" - @echo " compatibility tests:" - @echo " * values via COMPATIBILITY_VALUES (default infra/kind/values-basic-go.yaml)" + @echo " tests:" + @echo " * values via VALUES (default infra/kind/values-basic-go.yaml)" @echo " * assert timeout via ASSERT_TIMEOUT (default 15m; e.g. ASSERT_TIMEOUT=5m)" @echo " * extra chainsaw flags via CHAINSAW_EXTRA_FLAGS (e.g. CHAINSAW_EXTRA_FLAGS=\"--skip-delete -v\")" @echo "" - @echo " test-compatibility (run every case)" - @echo " test-compatibility-all (alias of test-compatibility)" - @echo " test-compatibility-01 (BMC registration)" - @echo " test-compatibility-02 (server discovery and inventory)" - @echo " test-compatibility-03 (power ops via operation annotation)" - @echo " test-compatibility-04 (BMC reset)" - @echo " test-compatibility-05 (indicator LED)" - @echo " test-compatibility-06 (BIOSSettings, non-reboot setting)" - @echo " test-compatibility-07 (BIOSSettings, reboot-required setting)" - @echo " test-compatibility-08 (BMCSettings, Manager attribute; needs a Dell BMC)" - @echo " test-compatibility-09 (persistent boot order)" + @echo " test (run every test)" + @echo " test/01-bmc-registration (BMC registration)" + @echo " test/02-discovery (server discovery and inventory)" + @echo " test/03-power-annotation (power ops via operation annotation)" + @echo " test/04-bmc-reset (BMC reset)" + @echo " test/05-indicator-led (indicator LED)" + @echo " test/06-biossettings-noreboot (BIOSSettings, non-reboot setting)" + @echo " test/07-biossettings-reboot (BIOSSettings, reboot-required setting)" + @echo " test/08-bmcsettings (BMCSettings, Manager attribute; needs a Dell BMC)" + @echo " test/09-persistent-boot-order (persistent boot order)" @echo "" -.PHONY: test-compatibility test-compatibility-all -test-compatibility: ## Run every compatibility case - $(CHAINSAW_RUN) $(addprefix $(COMPATIBILITY_TEST_DIR)/,$(COMPATIBILITY_CASES)) +.PHONY: test $(addprefix test/,$(TESTS)) -test-compatibility-all: test-compatibility ## Alias of test-compatibility +test: ## Run every test + $(CHAINSAW_RUN) $(addprefix $(TEST_DIR)/,$(TESTS)) -.PHONY: test-compatibility-01 test-compatibility-02 test-compatibility-03 test-compatibility-04 test-compatibility-05 test-compatibility-06 test-compatibility-07 test-compatibility-08 test-compatibility-09 - -test-compatibility-01: ## Run 01 BMC registration - $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/01-bmc-registration - -test-compatibility-02: ## Run 02 server discovery and inventory - $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/02-discovery - -test-compatibility-03: ## Run 03 power operations via operation annotation - $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/03-power-annotation - -test-compatibility-04: ## Run 04 BMC reset - $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/04-bmc-reset - -test-compatibility-05: ## Run 05 indicator LED - $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/05-indicator-led - -test-compatibility-06: ## Run 06 BIOSSettings, non-reboot setting - $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/06-biossettings-noreboot - -test-compatibility-07: ## Run 07 BIOSSettings, reboot-required setting - $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/07-biossettings-reboot - -test-compatibility-08: ## Run 08 BMCSettings, attribute change - $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/08-bmcsettings - -test-compatibility-09: ## Run 09 persistent boot order - $(CHAINSAW_RUN) $(COMPATIBILITY_TEST_DIR)/09-persistent-boot-order +$(addprefix test/,$(TESTS)): test/%: ## Run a single test + $(CHAINSAW_RUN) $(TEST_DIR)/$* diff --git a/README.md b/README.md index c12025e..e1ef1e9 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,19 @@ # metal-operator test framework ## Test suite -The `tests/compatibility` directory contains a suite of compatibility tests based on [chainsaw](https://kyverno.github.io/chainsaw/latest/). Every test case creates k8s resources in steps and asserts their status before proceeding to the next steps and implements common metal-operator workflows. Tests are independent from the infrastructure they run on and respect `KUBECONFIG`. +The `tests` directory contains a suite of tests based on [chainsaw](https://kyverno.github.io/chainsaw/latest/). Every test case creates k8s resources in steps and asserts their status before proceeding to the next steps and implements common metal-operator workflows. Tests are independent from the infrastructure they run on and respect `KUBECONFIG`. ### Requirements * [chainsaw](https://kyverno.github.io/chainsaw/latest/) * A metal-operator installation and BMC to run tests against. This repository usually uses the locally virtualised [metal-lab](https://github.com/simontesar/metal-lab). ### Usage -The server to run a test against is configured by passing a values file to chainsaw. The default file is `infra/kind/values-basic-go.yaml` that points to a redfish mock setup in the `kind` environment and can be overridden via `COMPATIBILITY_VALUES`. +The server to run a test against is configured by passing a values file to chainsaw. The default file is `infra/kind/values-basic-go.yaml` that points to a redfish mock setup in the `kind` environment and can be overridden via `VALUES`. ```bash -make test-compatibility # Run all cases -make test-compatibility-01 # Run a specific case -make test-compatibility-03 COMPATIBILITY_VALUES=/path/to/metal-lab/values-containerlab-node1.yaml # Run against a specific BMC. +make test # Run all tests +make test/01-bmc-registration # Run a specific test +make test/03-power-annotation VALUES=/path/to/metal-lab/values-containerlab-node1.yaml # Run against a specific BMC. ``` ### Predefined values diff --git a/infra/kind/README.md b/infra/kind/README.md index 46cf9b9..7d78ac7 100644 --- a/infra/kind/README.md +++ b/infra/kind/README.md @@ -64,7 +64,7 @@ Run a test against the basic go instance: ```shell $ cd ../.. $ export KUBECONFIG=$PWD/kubeconfig.yaml -$ make test-compatibility-03 COMPATIBILITY_VALUES=infra/kind/values-basic-go.yaml CHAINSAW_EXTRA_FLAGS="--pause-on-failure" +$ make test/03-power-annotation VALUES=infra/kind/values-basic-go.yaml CHAINSAW_EXTRA_FLAGS="--pause-on-failure" ``` Tear down: diff --git a/tests/compatibility/01-bmc-registration/chainsaw-test.yaml b/tests/01-bmc-registration/chainsaw-test.yaml similarity index 100% rename from tests/compatibility/01-bmc-registration/chainsaw-test.yaml rename to tests/01-bmc-registration/chainsaw-test.yaml diff --git a/tests/compatibility/02-discovery/chainsaw-test.yaml b/tests/02-discovery/chainsaw-test.yaml similarity index 100% rename from tests/compatibility/02-discovery/chainsaw-test.yaml rename to tests/02-discovery/chainsaw-test.yaml diff --git a/tests/compatibility/03-power-annotation/chainsaw-test.yaml b/tests/03-power-annotation/chainsaw-test.yaml similarity index 100% rename from tests/compatibility/03-power-annotation/chainsaw-test.yaml rename to tests/03-power-annotation/chainsaw-test.yaml diff --git a/tests/compatibility/03-power-annotation/ignitionsecret.yaml b/tests/03-power-annotation/ignitionsecret.yaml similarity index 100% rename from tests/compatibility/03-power-annotation/ignitionsecret.yaml rename to tests/03-power-annotation/ignitionsecret.yaml diff --git a/tests/compatibility/03-power-annotation/serverclaim.yaml b/tests/03-power-annotation/serverclaim.yaml similarity index 100% rename from tests/compatibility/03-power-annotation/serverclaim.yaml rename to tests/03-power-annotation/serverclaim.yaml diff --git a/tests/compatibility/04-bmc-reset/chainsaw-test.yaml b/tests/04-bmc-reset/chainsaw-test.yaml similarity index 100% rename from tests/compatibility/04-bmc-reset/chainsaw-test.yaml rename to tests/04-bmc-reset/chainsaw-test.yaml diff --git a/tests/compatibility/05-indicator-led/chainsaw-test.yaml b/tests/05-indicator-led/chainsaw-test.yaml similarity index 100% rename from tests/compatibility/05-indicator-led/chainsaw-test.yaml rename to tests/05-indicator-led/chainsaw-test.yaml diff --git a/tests/compatibility/06-biossettings-noreboot/biossettings-noreboot.yaml b/tests/06-biossettings-noreboot/biossettings-noreboot.yaml similarity index 100% rename from tests/compatibility/06-biossettings-noreboot/biossettings-noreboot.yaml rename to tests/06-biossettings-noreboot/biossettings-noreboot.yaml diff --git a/tests/compatibility/06-biossettings-noreboot/chainsaw-test.yaml b/tests/06-biossettings-noreboot/chainsaw-test.yaml similarity index 100% rename from tests/compatibility/06-biossettings-noreboot/chainsaw-test.yaml rename to tests/06-biossettings-noreboot/chainsaw-test.yaml diff --git a/tests/compatibility/07-biossettings-reboot/biossettings-reboot.yaml b/tests/07-biossettings-reboot/biossettings-reboot.yaml similarity index 100% rename from tests/compatibility/07-biossettings-reboot/biossettings-reboot.yaml rename to tests/07-biossettings-reboot/biossettings-reboot.yaml diff --git a/tests/compatibility/07-biossettings-reboot/chainsaw-test.yaml b/tests/07-biossettings-reboot/chainsaw-test.yaml similarity index 100% rename from tests/compatibility/07-biossettings-reboot/chainsaw-test.yaml rename to tests/07-biossettings-reboot/chainsaw-test.yaml diff --git a/tests/compatibility/08-bmcsettings/bmcsettings.yaml b/tests/08-bmcsettings/bmcsettings.yaml similarity index 100% rename from tests/compatibility/08-bmcsettings/bmcsettings.yaml rename to tests/08-bmcsettings/bmcsettings.yaml diff --git a/tests/compatibility/08-bmcsettings/chainsaw-test.yaml b/tests/08-bmcsettings/chainsaw-test.yaml similarity index 100% rename from tests/compatibility/08-bmcsettings/chainsaw-test.yaml rename to tests/08-bmcsettings/chainsaw-test.yaml diff --git a/tests/compatibility/09-persistent-boot-order/bootorder-check-job.yaml b/tests/09-persistent-boot-order/bootorder-check-job.yaml similarity index 100% rename from tests/compatibility/09-persistent-boot-order/bootorder-check-job.yaml rename to tests/09-persistent-boot-order/bootorder-check-job.yaml diff --git a/tests/compatibility/09-persistent-boot-order/chainsaw-test.yaml b/tests/09-persistent-boot-order/chainsaw-test.yaml similarity index 100% rename from tests/compatibility/09-persistent-boot-order/chainsaw-test.yaml rename to tests/09-persistent-boot-order/chainsaw-test.yaml diff --git a/tests/compatibility/shared/bmc.yaml b/tests/shared/bmc.yaml similarity index 100% rename from tests/compatibility/shared/bmc.yaml rename to tests/shared/bmc.yaml diff --git a/tests/compatibility/shared/bmcsecret.yaml b/tests/shared/bmcsecret.yaml similarity index 100% rename from tests/compatibility/shared/bmcsecret.yaml rename to tests/shared/bmcsecret.yaml