diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 544c33c..d62f97d 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -21,7 +21,7 @@ jobs: ENABLE_KVM: "false" KVM_DEVICE: /dev/null KUBECONFIG: ${{ github.workspace }}/metal-lab/kubeconfig.yaml - COMPATIBILITY_VALUES: ${{ github.workspace }}/metal-lab/values-containerlab-node1.yaml + VALUES: ${{ github.workspace }}/metal-lab/values-containerlab-node1.yaml steps: - name: Checkout test framework uses: actions/checkout@v4 @@ -57,10 +57,10 @@ jobs: fedhcp-deploy-wait tftp-deploy-wait sudo chown -R "$USER:$USER" . - - name: Run compatibility suite + - name: Run test suite run: | mkdir -p "${{ github.workspace }}/artifacts" - make test-compatibility-all \ + make test \ ASSERT_TIMEOUT=25m \ CHAINSAW_EXTRA_FLAGS="--report-format JSON --report-name chainsaw-report --report-path ${{ github.workspace }}/artifacts" diff --git a/Makefile b/Makefile index 9127c21..c7d0a29 100644 --- a/Makefile +++ b/Makefile @@ -1,97 +1,48 @@ CHAINSAW ?= chainsaw -COMPATIBILITY_TEST_DIR := tests/compatibility -COMPATIBILITY_VALUES ?= infra/kind/values-basic-go.yaml +TEST_DIR := tests +VALUES ?= infra/kind/values-basic-go.yaml ASSERT_TIMEOUT ?= 15m CHAINSAW_EXTRA_FLAGS ?= +CHAINSAW_RUN = $(CHAINSAW) test --values $(VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) + +TESTS := \ + 01-bmc-registration \ + 02-discovery \ + 03-power-annotation \ + 04-bmc-reset \ + 05-indicator-led \ + 06-biossettings-noreboot \ + 07-biossettings-reboot \ + 08-bmcsettings \ + 09-persistent-boot-order + .PHONY: help help: ## Show available targets @echo "test framework targets:" @echo "" - @echo " compatibility tests:" - @echo " * values via COMPATIBILITY_VALUES (default infra/kind/values-basic-go.yaml)" + @echo " tests:" + @echo " * values via VALUES (default infra/kind/values-basic-go.yaml)" @echo " * assert timeout via ASSERT_TIMEOUT (default 15m; e.g. ASSERT_TIMEOUT=5m)" @echo " * extra chainsaw flags via CHAINSAW_EXTRA_FLAGS (e.g. CHAINSAW_EXTRA_FLAGS=\"--skip-delete -v\")" @echo "" - @echo " test-compatibility (A1, A2)" - @echo " test-compatibility-a1" - @echo " test-compatibility-a2" - @echo " test-compatibility-b (B1-B3)" - @echo " test-compatibility-b1 (power ops via operation annotation)" - @echo " test-compatibility-b2 (BMC reset)" - @echo " test-compatibility-b3 (indicator LED)" - @echo " test-compatibility-d (D1-D3)" - @echo " test-compatibility-d1 (BIOSSettings, non-reboot setting)" - @echo " test-compatibility-d2 (BIOSSettings, reboot-required setting)" - @echo " test-compatibility-d3 (BMCSettings, Manager attribute; needs a Dell BMC)" - @echo " test-compatibility-f (F3)" - @echo " test-compatibility-f3 (persistent boot order)" + @echo " test (run every test)" + @echo " test/01-bmc-registration (BMC registration)" + @echo " test/02-discovery (server discovery and inventory)" + @echo " test/03-power-annotation (power ops via operation annotation)" + @echo " test/04-bmc-reset (BMC reset)" + @echo " test/05-indicator-led (indicator LED)" + @echo " test/06-biossettings-noreboot (BIOSSettings, non-reboot setting)" + @echo " test/07-biossettings-reboot (BIOSSettings, reboot-required setting)" + @echo " test/08-bmcsettings (BMCSettings, Manager attribute; needs a Dell BMC)" + @echo " test/09-persistent-boot-order (persistent boot order)" @echo "" - @echo " test-compatibility-all (everything: A1,A2,B1-3,D1-3,F3)" - @echo "" - -.PHONY: test-compatibility test-compatibility-a1 test-compatibility-a2 -test-compatibility: ## Run A compatibility chainsaw tests (A1, A2) - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \ - $(COMPATIBILITY_TEST_DIR)/a1-bmc-registration \ - $(COMPATIBILITY_TEST_DIR)/a2-discovery - -test-compatibility-a1: ## Run A1 BMC registration - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/a1-bmc-registration - -test-compatibility-a2: ## Run A2 discovery - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/a2-discovery - -.PHONY: test-compatibility-b test-compatibility-b1 test-compatibility-b2 test-compatibility-b3 -test-compatibility-b: ## Run all B power-management tests (B1-B3) - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \ - $(COMPATIBILITY_TEST_DIR)/b1-power-annotation \ - $(COMPATIBILITY_TEST_DIR)/b2-bmc-reset \ - $(COMPATIBILITY_TEST_DIR)/b3-indicator-led - -test-compatibility-b1: ## Run B1 power operations via operation annotation - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/b1-power-annotation - -test-compatibility-b2: ## Run B2 BMC reset - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/b2-bmc-reset - -test-compatibility-b3: ## Run B3 indicator LED - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/b3-indicator-led - -.PHONY: test-compatibility-d test-compatibility-d1 test-compatibility-d2 test-compatibility-d3 -test-compatibility-d: ## Run all D settings tests (D1-D3) - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \ - $(COMPATIBILITY_TEST_DIR)/d1-biossettings-noreboot \ - $(COMPATIBILITY_TEST_DIR)/d2-biossettings-reboot \ - $(COMPATIBILITY_TEST_DIR)/d3-bmcsettings - -test-compatibility-d1: ## Run D1 BIOSSettings, non-reboot setting - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/d1-biossettings-noreboot - -test-compatibility-d2: ## Run D2 BIOSSettings, reboot-required setting - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/d2-biossettings-reboot - -test-compatibility-d3: ## Run D3 BMCSettings, attribute change - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/d3-bmcsettings -.PHONY: test-compatibility-f test-compatibility-f3 -test-compatibility-f: ## Run all F boot-control tests (F3) - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \ - $(COMPATIBILITY_TEST_DIR)/f3-persistent-boot-order +.PHONY: test $(addprefix test/,$(TESTS)) -test-compatibility-f3: ## Run F3 persistent boot order - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/f3-persistent-boot-order +test: ## Run every test + $(CHAINSAW_RUN) $(addprefix $(TEST_DIR)/,$(TESTS)) -.PHONY: test-compatibility-all -test-compatibility-all: ## Run every compatibility case (A1,A2,B1-3,D1-3,F3) - $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \ - $(COMPATIBILITY_TEST_DIR)/a1-bmc-registration \ - $(COMPATIBILITY_TEST_DIR)/a2-discovery \ - $(COMPATIBILITY_TEST_DIR)/b1-power-annotation \ - $(COMPATIBILITY_TEST_DIR)/b2-bmc-reset \ - $(COMPATIBILITY_TEST_DIR)/b3-indicator-led \ - $(COMPATIBILITY_TEST_DIR)/d1-biossettings-noreboot \ - $(COMPATIBILITY_TEST_DIR)/d2-biossettings-reboot \ - $(COMPATIBILITY_TEST_DIR)/d3-bmcsettings \ - $(COMPATIBILITY_TEST_DIR)/f3-persistent-boot-order +$(addprefix test/,$(TESTS)): test/%: ## Run a single test + $(CHAINSAW_RUN) $(TEST_DIR)/$* diff --git a/README.md b/README.md index 5886431..e1ef1e9 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,19 @@ # metal-operator test framework ## Test suite -The `tests/compatibility` directory contains a suite of compatibility tests based on [chainsaw](https://kyverno.github.io/chainsaw/latest/). Every test case creates k8s resources in steps and asserts their status before proceeding to the next steps and implements common metal-operator workflows. Tests are independent from the infrastructure they run on and respect `KUBECONFIG`. +The `tests` directory contains a suite of tests based on [chainsaw](https://kyverno.github.io/chainsaw/latest/). Every test case creates k8s resources in steps and asserts their status before proceeding to the next steps and implements common metal-operator workflows. Tests are independent from the infrastructure they run on and respect `KUBECONFIG`. ### Requirements * [chainsaw](https://kyverno.github.io/chainsaw/latest/) * A metal-operator installation and BMC to run tests against. This repository usually uses the locally virtualised [metal-lab](https://github.com/simontesar/metal-lab). ### Usage -The server to run a test against is configured by passing a values file to chainsaw. The default file is `infra/kind/values-basic-go.yaml` that points to a redfish mock setup in the `kind` environment and can be overridden via `COMPATIBILITY_VALUES`. +The server to run a test against is configured by passing a values file to chainsaw. The default file is `infra/kind/values-basic-go.yaml` that points to a redfish mock setup in the `kind` environment and can be overridden via `VALUES`. ```bash -make test-compatibility # Run all tests -make test-compatibility-b # Run a specific category of tests -make test-compatibility-a1 # Run a specific case -make test-compatibility-b1 COMPATIBILITY_VALUES=/path/to/metal-lab/values-containerlab-node1.yaml # Run against a specific BMC. +make test # Run all tests +make test/01-bmc-registration # Run a specific test +make test/03-power-annotation VALUES=/path/to/metal-lab/values-containerlab-node1.yaml # Run against a specific BMC. ``` ### Predefined values diff --git a/infra/kind/README.md b/infra/kind/README.md index 63c1198..7d78ac7 100644 --- a/infra/kind/README.md +++ b/infra/kind/README.md @@ -64,7 +64,7 @@ Run a test against the basic go instance: ```shell $ cd ../.. $ export KUBECONFIG=$PWD/kubeconfig.yaml -$ make test-compatibility-b1 COMPATIBILITY_VALUES=infra/kind/values-basic-go.yaml CHAINSAW_EXTRA_FLAGS="--pause-on-failure" +$ make test/03-power-annotation VALUES=infra/kind/values-basic-go.yaml CHAINSAW_EXTRA_FLAGS="--pause-on-failure" ``` Tear down: diff --git a/tests/compatibility/a1-bmc-registration/chainsaw-test.yaml b/tests/01-bmc-registration/chainsaw-test.yaml similarity index 93% rename from tests/compatibility/a1-bmc-registration/chainsaw-test.yaml rename to tests/01-bmc-registration/chainsaw-test.yaml index 73e292e..da58ea2 100644 --- a/tests/compatibility/a1-bmc-registration/chainsaw-test.yaml +++ b/tests/01-bmc-registration/chainsaw-test.yaml @@ -1,9 +1,9 @@ -# A1 — BMC registration +# 01 — BMC registration # Asserts model/firmwareVersion/powerState apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: a1-bmc-registration + name: 01-bmc-registration spec: concurrent: false steps: diff --git a/tests/compatibility/a2-discovery/chainsaw-test.yaml b/tests/02-discovery/chainsaw-test.yaml similarity index 95% rename from tests/compatibility/a2-discovery/chainsaw-test.yaml rename to tests/02-discovery/chainsaw-test.yaml index 0ba5e79..988f59f 100644 --- a/tests/compatibility/a2-discovery/chainsaw-test.yaml +++ b/tests/02-discovery/chainsaw-test.yaml @@ -1,8 +1,8 @@ -# A2 — Server discovery and inventory +# 02 — Server discovery and inventory apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: a2-discovery + name: 02-discovery spec: concurrent: false steps: diff --git a/tests/compatibility/b1-power-annotation/chainsaw-test.yaml b/tests/03-power-annotation/chainsaw-test.yaml similarity index 98% rename from tests/compatibility/b1-power-annotation/chainsaw-test.yaml rename to tests/03-power-annotation/chainsaw-test.yaml index be8d598..b386597 100644 --- a/tests/compatibility/b1-power-annotation/chainsaw-test.yaml +++ b/tests/03-power-annotation/chainsaw-test.yaml @@ -1,8 +1,8 @@ -# B1 — Power operations via operation annotation +# 03 — Power operations via operation annotation apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: b1-power-annotation + name: 03-power-annotation spec: concurrent: false steps: diff --git a/tests/compatibility/b1-power-annotation/ignitionsecret.yaml b/tests/03-power-annotation/ignitionsecret.yaml similarity index 100% rename from tests/compatibility/b1-power-annotation/ignitionsecret.yaml rename to tests/03-power-annotation/ignitionsecret.yaml diff --git a/tests/compatibility/b1-power-annotation/serverclaim.yaml b/tests/03-power-annotation/serverclaim.yaml similarity index 100% rename from tests/compatibility/b1-power-annotation/serverclaim.yaml rename to tests/03-power-annotation/serverclaim.yaml diff --git a/tests/compatibility/b2-bmc-reset/chainsaw-test.yaml b/tests/04-bmc-reset/chainsaw-test.yaml similarity index 96% rename from tests/compatibility/b2-bmc-reset/chainsaw-test.yaml rename to tests/04-bmc-reset/chainsaw-test.yaml index 70b9f61..a6b5973 100644 --- a/tests/compatibility/b2-bmc-reset/chainsaw-test.yaml +++ b/tests/04-bmc-reset/chainsaw-test.yaml @@ -1,9 +1,9 @@ -# B2 — BMC reset (see cases.md) +# 04 — BMC reset # - Should be extended to asser status.lastResetTime once testing agains physical servers apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: b2-bmc-reset + name: 04-bmc-reset spec: concurrent: false steps: diff --git a/tests/compatibility/b3-indicator-led/chainsaw-test.yaml b/tests/05-indicator-led/chainsaw-test.yaml similarity index 97% rename from tests/compatibility/b3-indicator-led/chainsaw-test.yaml rename to tests/05-indicator-led/chainsaw-test.yaml index 7ba8124..571713b 100644 --- a/tests/compatibility/b3-indicator-led/chainsaw-test.yaml +++ b/tests/05-indicator-led/chainsaw-test.yaml @@ -1,9 +1,9 @@ -# B3 — Indicator LED +# 05 — Indicator LED # - Kept as a separate test to be skippable for models that do not support LED control apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: b3-indicator-led + name: 05-indicator-led spec: concurrent: false steps: diff --git a/tests/compatibility/d1-biossettings-noreboot/biossettings-noreboot.yaml b/tests/06-biossettings-noreboot/biossettings-noreboot.yaml similarity index 100% rename from tests/compatibility/d1-biossettings-noreboot/biossettings-noreboot.yaml rename to tests/06-biossettings-noreboot/biossettings-noreboot.yaml diff --git a/tests/compatibility/d1-biossettings-noreboot/chainsaw-test.yaml b/tests/06-biossettings-noreboot/chainsaw-test.yaml similarity index 98% rename from tests/compatibility/d1-biossettings-noreboot/chainsaw-test.yaml rename to tests/06-biossettings-noreboot/chainsaw-test.yaml index 0d23f3d..73b93ad 100644 --- a/tests/compatibility/d1-biossettings-noreboot/chainsaw-test.yaml +++ b/tests/06-biossettings-noreboot/chainsaw-test.yaml @@ -1,7 +1,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: d1-biossettings-noreboot + name: 06-biossettings-noreboot spec: concurrent: false steps: diff --git a/tests/compatibility/d2-biossettings-reboot/biossettings-reboot.yaml b/tests/07-biossettings-reboot/biossettings-reboot.yaml similarity index 100% rename from tests/compatibility/d2-biossettings-reboot/biossettings-reboot.yaml rename to tests/07-biossettings-reboot/biossettings-reboot.yaml diff --git a/tests/compatibility/d2-biossettings-reboot/chainsaw-test.yaml b/tests/07-biossettings-reboot/chainsaw-test.yaml similarity index 98% rename from tests/compatibility/d2-biossettings-reboot/chainsaw-test.yaml rename to tests/07-biossettings-reboot/chainsaw-test.yaml index cd73911..6568c22 100644 --- a/tests/compatibility/d2-biossettings-reboot/chainsaw-test.yaml +++ b/tests/07-biossettings-reboot/chainsaw-test.yaml @@ -1,7 +1,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: d2-biossettings-reboot + name: 07-biossettings-reboot spec: concurrent: false steps: diff --git a/tests/compatibility/d3-bmcsettings/bmcsettings.yaml b/tests/08-bmcsettings/bmcsettings.yaml similarity index 100% rename from tests/compatibility/d3-bmcsettings/bmcsettings.yaml rename to tests/08-bmcsettings/bmcsettings.yaml diff --git a/tests/compatibility/d3-bmcsettings/chainsaw-test.yaml b/tests/08-bmcsettings/chainsaw-test.yaml similarity index 98% rename from tests/compatibility/d3-bmcsettings/chainsaw-test.yaml rename to tests/08-bmcsettings/chainsaw-test.yaml index ae4e914..43613e0 100644 --- a/tests/compatibility/d3-bmcsettings/chainsaw-test.yaml +++ b/tests/08-bmcsettings/chainsaw-test.yaml @@ -1,7 +1,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: d3-bmcsettings + name: 08-bmcsettings spec: concurrent: false steps: diff --git a/tests/compatibility/f3-persistent-boot-order/bootorder-check-job.yaml b/tests/09-persistent-boot-order/bootorder-check-job.yaml similarity index 100% rename from tests/compatibility/f3-persistent-boot-order/bootorder-check-job.yaml rename to tests/09-persistent-boot-order/bootorder-check-job.yaml diff --git a/tests/compatibility/f3-persistent-boot-order/chainsaw-test.yaml b/tests/09-persistent-boot-order/chainsaw-test.yaml similarity index 98% rename from tests/compatibility/f3-persistent-boot-order/chainsaw-test.yaml rename to tests/09-persistent-boot-order/chainsaw-test.yaml index d692235..423d419 100644 --- a/tests/compatibility/f3-persistent-boot-order/chainsaw-test.yaml +++ b/tests/09-persistent-boot-order/chainsaw-test.yaml @@ -1,7 +1,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: f3-persistent-boot-order + name: 09-persistent-boot-order spec: concurrent: false steps: diff --git a/tests/compatibility/shared/bmc.yaml b/tests/shared/bmc.yaml similarity index 100% rename from tests/compatibility/shared/bmc.yaml rename to tests/shared/bmc.yaml diff --git a/tests/compatibility/shared/bmcsecret.yaml b/tests/shared/bmcsecret.yaml similarity index 100% rename from tests/compatibility/shared/bmcsecret.yaml rename to tests/shared/bmcsecret.yaml