From 1fee9fbfb5b41b6648f7dfa927fc58b515eeeeb0 Mon Sep 17 00:00:00 2001 From: simontesar Date: Wed, 2 Sep 2026 12:06:22 +0200 Subject: [PATCH 01/11] feat: add target to run all tests Signed-off-by: simontesar --- Makefile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Makefile b/Makefile index e569458..9127c21 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,8 @@ help: ## Show available targets @echo " test-compatibility-f (F3)" @echo " test-compatibility-f3 (persistent boot order)" @echo "" + @echo " test-compatibility-all (everything: A1,A2,B1-3,D1-3,F3)" + @echo "" .PHONY: test-compatibility test-compatibility-a1 test-compatibility-a2 test-compatibility: ## Run A compatibility chainsaw tests (A1, A2) @@ -80,3 +82,16 @@ test-compatibility-f: ## Run all F boot-control tests (F3) test-compatibility-f3: ## Run F3 persistent boot order $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) $(COMPATIBILITY_TEST_DIR)/f3-persistent-boot-order + +.PHONY: test-compatibility-all +test-compatibility-all: ## Run every compatibility case (A1,A2,B1-3,D1-3,F3) + $(CHAINSAW) test --values $(COMPATIBILITY_VALUES) --parallel 1 --assert-timeout $(ASSERT_TIMEOUT) $(CHAINSAW_EXTRA_FLAGS) \ + $(COMPATIBILITY_TEST_DIR)/a1-bmc-registration \ + $(COMPATIBILITY_TEST_DIR)/a2-discovery \ + $(COMPATIBILITY_TEST_DIR)/b1-power-annotation \ + $(COMPATIBILITY_TEST_DIR)/b2-bmc-reset \ + $(COMPATIBILITY_TEST_DIR)/b3-indicator-led \ + $(COMPATIBILITY_TEST_DIR)/d1-biossettings-noreboot \ + $(COMPATIBILITY_TEST_DIR)/d2-biossettings-reboot \ + $(COMPATIBILITY_TEST_DIR)/d3-bmcsettings \ + $(COMPATIBILITY_TEST_DIR)/f3-persistent-boot-order From a736c72226647ab6aa948a5d75ef70f8cf0f0444 Mon Sep 17 00:00:00 2001 From: simontesar Date: Wed, 2 Sep 2026 12:06:32 +0200 Subject: [PATCH 02/11] ci: add inital pipeline Signed-off-by: simontesar --- .github/workflows/e2e.yml | 103 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..3f4220e --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,103 @@ +name: e2e compatibility + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + +concurrency: + group: e2e-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + e2e: + # Gate: manual dispatch, or PR labelled 'ok-to-test' + if: >- + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'ok-to-test') + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + METAL_LAB_REF: main + ENABLE_KVM: "false" # consumed by metal-lab/infra.clab.yaml + KVM_DEVICE: /dev/null # ditto - avoids binding a missing /dev/kvm + KUBECONFIG: ${{ github.workspace }}/metal-lab/kubeconfig.yaml + COMPATIBILITY_VALUES: ${{ github.workspace }}/metal-lab/values-containerlab-node1.yaml + steps: + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + large-packages: false + + - name: Checkout test framework + uses: actions/checkout@v4 + + - name: Checkout metal-lab + uses: actions/checkout@v4 + with: + repository: simontesar/metal-lab + ref: ${{ env.METAL_LAB_REF }} + path: metal-lab + + - name: Install kubectl + uses: azure/setup-kubectl@v4 + + - name: Install kustomize + run: | + curl -sSL "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash + sudo mv kustomize /usr/local/bin/ + + - name: Install kind + run: | + curl -sSLo kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64 + sudo install -m 0755 kind /usr/local/bin/kind + + - name: Install containerlab + uses: containerlab/setup-clab@main + + - name: Install chainsaw + uses: kyverno/action-install-chainsaw@v0.2.12 + + - name: Bring up metal-lab (TCG, both nodes) + working-directory: metal-lab + run: | + touch kubeconfig.yaml + sudo -E make deploy \ + metal-operator-deploy-wait boot-operator-deploy-wait \ + fedhcp-deploy-wait tftp-deploy-wait + sudo chown -R "$USER:$USER" . + + - name: Run compatibility suite + run: | + make test-compatibility-all \ + ASSERT_TIMEOUT=25m \ + CHAINSAW_EXTRA_FLAGS="--report-format JSON --report-name chainsaw-report --report-path ${{ github.workspace }}/artifacts" + + - name: Collect diagnostics + if: always() + run: | + mkdir -p artifacts + kubectl get all -A > artifacts/k8s-all.txt 2>&1 || true + kubectl get bmc,server,serverclaim,biossettings,bmcsettings -A -o yaml > artifacts/metal-crs.yaml 2>&1 || true + kubectl -n metal-operator-system logs deploy/metal-operator-controller-manager --tail=-1 > artifacts/metal-operator.log 2>&1 || true + kubectl -n boot-operator-system logs deploy/boot-operator-controller-manager --tail=-1 > artifacts/boot-operator.log 2>&1 || true + cp metal-lab/vm/node1/console.log artifacts/node1-console.log 2>/dev/null || true + cp metal-lab/vm/node2/console.log artifacts/node2-console.log 2>/dev/null || true + sudo -E clab inspect -t metal-lab/infra.clab.yaml > artifacts/clab-inspect.txt 2>&1 || true + + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-artifacts + path: artifacts/ + if-no-files-found: warn + + - name: Tear down + if: always() + working-directory: metal-lab + run: sudo -E make destroy clean-disks || true From 42f8ef18064c989beb696a72af112bc993e44fbb Mon Sep 17 00:00:00 2001 From: simontesar Date: Wed, 2 Sep 2026 13:04:18 +0200 Subject: [PATCH 03/11] ci: clean pipeline Signed-off-by: simontesar --- .github/workflows/e2e.yml | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3f4220e..30dcb3c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -22,20 +22,13 @@ jobs: timeout-minutes: 120 env: METAL_LAB_REF: main - ENABLE_KVM: "false" # consumed by metal-lab/infra.clab.yaml - KVM_DEVICE: /dev/null # ditto - avoids binding a missing /dev/kvm + ENABLE_KVM: "false" + KVM_DEVICE: /dev/null KUBECONFIG: ${{ github.workspace }}/metal-lab/kubeconfig.yaml COMPATIBILITY_VALUES: ${{ github.workspace }}/metal-lab/values-containerlab-node1.yaml steps: - - name: Free disk space - uses: jlumbroso/free-disk-space@main - with: - tool-cache: false - large-packages: false - - name: Checkout test framework uses: actions/checkout@v4 - - name: Checkout metal-lab uses: actions/checkout@v4 with: @@ -45,20 +38,16 @@ jobs: - name: Install kubectl uses: azure/setup-kubectl@v4 - - name: Install kustomize run: | curl -sSL "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash sudo mv kustomize /usr/local/bin/ - - name: Install kind run: | curl -sSLo kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64 sudo install -m 0755 kind /usr/local/bin/kind - - name: Install containerlab uses: containerlab/setup-clab@main - - name: Install chainsaw uses: kyverno/action-install-chainsaw@v0.2.12 @@ -89,13 +78,13 @@ jobs: cp metal-lab/vm/node2/console.log artifacts/node2-console.log 2>/dev/null || true sudo -E clab inspect -t metal-lab/infra.clab.yaml > artifacts/clab-inspect.txt 2>&1 || true - - name: Upload artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: e2e-artifacts - path: artifacts/ - if-no-files-found: warn + # - name: Upload artifacts + # if: always() + # uses: actions/upload-artifact@v4 + # with: + # name: e2e-artifacts + # path: artifacts/ + # if-no-files-found: warn - name: Tear down if: always() From e4b3fb488543799ddce508330cc57a6a13913b26 Mon Sep 17 00:00:00 2001 From: simontesar Date: Wed, 2 Sep 2026 13:08:35 +0200 Subject: [PATCH 04/11] ci: fix containerlab installation Signed-off-by: simontesar --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 30dcb3c..80cc66d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -47,7 +47,7 @@ jobs: curl -sSLo kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64 sudo install -m 0755 kind /usr/local/bin/kind - name: Install containerlab - uses: containerlab/setup-clab@main + run: bash -c "$(curl -sL https://get.containerlab.dev)" -- -v 0.78.2 - name: Install chainsaw uses: kyverno/action-install-chainsaw@v0.2.12 From 3e2a5be15c7bd485b1bfdad6f3b269717a062989 Mon Sep 17 00:00:00 2001 From: simontesar Date: Wed, 2 Sep 2026 18:11:23 +0200 Subject: [PATCH 05/11] ci: fix kind setup to latest Signed-off-by: simontesar --- .github/workflows/e2e.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 80cc66d..629d146 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -42,10 +42,11 @@ jobs: run: | curl -sSL "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash sudo mv kustomize /usr/local/bin/ - - name: Install kind + - name: Install the latest version of kind run: | - curl -sSLo kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64 - sudo install -m 0755 kind /usr/local/bin/kind + curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind - name: Install containerlab run: bash -c "$(curl -sL https://get.containerlab.dev)" -- -v 0.78.2 - name: Install chainsaw From 402eb1248c1fc1cdbee155c3189663e5447e8083 Mon Sep 17 00:00:00 2001 From: simontesar Date: Thu, 3 Sep 2026 06:06:59 +0200 Subject: [PATCH 06/11] ci: rename workflow Signed-off-by: simontesar --- .github/workflows/{e2e.yml => pr.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{e2e.yml => pr.yml} (100%) diff --git a/.github/workflows/e2e.yml b/.github/workflows/pr.yml similarity index 100% rename from .github/workflows/e2e.yml rename to .github/workflows/pr.yml From 778a385c827f77c8a03bf25451844c2e33ee952d Mon Sep 17 00:00:00 2001 From: simontesar Date: Thu, 3 Sep 2026 06:09:01 +0200 Subject: [PATCH 07/11] ci: renames in workflow file Signed-off-by: simontesar --- .github/workflows/pr.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 629d146..e54c25b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,4 +1,4 @@ -name: e2e compatibility +name: PR test run on: pull_request: @@ -6,14 +6,14 @@ on: workflow_dispatch: concurrency: - group: e2e-${{ github.event.pull_request.number || github.ref }} + group: test-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true permissions: contents: read jobs: - e2e: + test: # Gate: manual dispatch, or PR labelled 'ok-to-test' if: >- github.event_name == 'workflow_dispatch' || From 374103866b9bfde860468abdce736fee3f1f91f2 Mon Sep 17 00:00:00 2001 From: simontesar Date: Thu, 3 Sep 2026 06:09:17 +0200 Subject: [PATCH 08/11] ci: enable artifact upload Signed-off-by: simontesar --- .github/workflows/pr.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index e54c25b..ffafa3a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -79,13 +79,13 @@ jobs: cp metal-lab/vm/node2/console.log artifacts/node2-console.log 2>/dev/null || true sudo -E clab inspect -t metal-lab/infra.clab.yaml > artifacts/clab-inspect.txt 2>&1 || true - # - name: Upload artifacts - # if: always() - # uses: actions/upload-artifact@v4 - # with: - # name: e2e-artifacts - # path: artifacts/ - # if-no-files-found: warn + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-artifacts + path: artifacts/ + if-no-files-found: warn - name: Tear down if: always() From d6b2efed0d54c24ec3c9d9f58e5d553f56972cb0 Mon Sep 17 00:00:00 2001 From: simontesar Date: Thu, 3 Sep 2026 12:18:21 +0200 Subject: [PATCH 09/11] ci: fix: create artifacts directory Signed-off-by: simontesar --- .github/workflows/pr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ffafa3a..8092c91 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -52,7 +52,7 @@ jobs: - name: Install chainsaw uses: kyverno/action-install-chainsaw@v0.2.12 - - name: Bring up metal-lab (TCG, both nodes) + - name: Bring up metal-lab working-directory: metal-lab run: | touch kubeconfig.yaml @@ -63,6 +63,7 @@ jobs: - name: Run compatibility suite run: | + mkdir -p "${{ github.workspace }}/artifacts" make test-compatibility-all \ ASSERT_TIMEOUT=25m \ CHAINSAW_EXTRA_FLAGS="--report-format JSON --report-name chainsaw-report --report-path ${{ github.workspace }}/artifacts" From c6c27dd2829e54737ff114e7b852952cc7e02a35 Mon Sep 17 00:00:00 2001 From: simontesar Date: Thu, 3 Sep 2026 13:34:17 +0200 Subject: [PATCH 10/11] ci: remove gate and decrease timeout Signed-off-by: simontesar --- .github/workflows/pr.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8092c91..3618956 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -14,12 +14,8 @@ permissions: jobs: test: - # Gate: manual dispatch, or PR labelled 'ok-to-test' - if: >- - github.event_name == 'workflow_dispatch' || - contains(github.event.pull_request.labels.*.name, 'ok-to-test') runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 30 env: METAL_LAB_REF: main ENABLE_KVM: "false" From 481cfa14f4023ccf2e55bbda9ec3876d0b64032d Mon Sep 17 00:00:00 2001 From: simontesar Date: Thu, 3 Sep 2026 13:34:36 +0200 Subject: [PATCH 11/11] ci: get serverbootconfigurations in output artifacts Signed-off-by: simontesar --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3618956..544c33c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -69,7 +69,7 @@ jobs: run: | mkdir -p artifacts kubectl get all -A > artifacts/k8s-all.txt 2>&1 || true - kubectl get bmc,server,serverclaim,biossettings,bmcsettings -A -o yaml > artifacts/metal-crs.yaml 2>&1 || true + kubectl get bmc,server,serverclaim,biossettings,bmcsettings,server,serverbootconfigurations -A -o yaml > artifacts/metal-crs.yaml 2>&1 || true kubectl -n metal-operator-system logs deploy/metal-operator-controller-manager --tail=-1 > artifacts/metal-operator.log 2>&1 || true kubectl -n boot-operator-system logs deploy/boot-operator-controller-manager --tail=-1 > artifacts/boot-operator.log 2>&1 || true cp metal-lab/vm/node1/console.log artifacts/node1-console.log 2>/dev/null || true