From d5cd429ce3b271792194793fa837ea845e608dbb Mon Sep 17 00:00:00 2001 From: Joe Beda Date: Tue, 27 Oct 2026 13:16:25 -0700 Subject: [PATCH] fix(ci): run browser tests in pinned Playwright containers Avoid runtime apt mirror dependency installation in SDK browser jobs. Keep image and npm versions aligned, retain failure diagnostics, and use the same provisioning for Studio. Co-authored-by: Mecatl --- .github/scripts/playwright-ci_test.py | 85 +++++++++++++++++++++++++++ .github/workflows/ci.yml | 54 +++++++++++------ Taskfile.yml | 1 + renovate.json | 11 ++++ 4 files changed, 132 insertions(+), 19 deletions(-) create mode 100644 .github/scripts/playwright-ci_test.py diff --git a/.github/scripts/playwright-ci_test.py b/.github/scripts/playwright-ci_test.py new file mode 100644 index 0000000000..dd313d551b --- /dev/null +++ b/.github/scripts/playwright-ci_test.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Offline contracts for the two Playwright job containers and coordinated updates.""" + +import json +from pathlib import Path +import re +import unittest + +ROOT = Path(__file__).resolve().parents[2] +JOBS = {"sdk-browser": "sdk/typescript", "studio-browser": "apps"} + + +def validate(workflow, packages, renovate): + images = [] + for job, package in JOBS.items(): + block = re.search(rf"^ {job}:\n(.*?)(?=^ [\w-]+:|\Z)", workflow, re.M | re.S) + assert block, f"missing {job}" + block = block[1] + image = re.search(r"^ image: (\S+)$", block, re.M) + assert image, f"{job}: missing container image" + version = re.fullmatch( + r"mcr\.microsoft\.com/playwright:v(\d+\.\d+\.\d+)-noble@sha256:[0-9a-f]{64}", + image[1], + ) + assert version, f"{job}: image must have an exact version and digest" + assert version[1] == packages[package]["devDependencies"]["@playwright/test"], ( + f"{job}: image and @playwright/test versions differ" + ) + images.append(image[1]) + assert "options: --user 1001:1001 --init --ipc=host" in block, job + assert "shell: bash" in block, job + assert "timeout-minutes: 20" in block, job + assert "playwright install" not in block and "browser:install" not in block, job + assert "playwright-cache" not in block and "~/.cache/ms-playwright" not in block, job + assert "if: failure()" in block and "include-hidden-files: true" in block, job + assert f"name: {job}-${{{{ matrix.project }}}}" in block, job + assert f"path: .scratch/{'sdk' if job == 'sdk-browser' else 'studio'}-browser-playwright/" in block, job + assert images[0] == images[1], "browser jobs must use the same image" + rule = next(r for r in renovate["packageRules"] if r.get("groupName") == "playwright") + assert set(rule["matchPackageNames"]) == { + "@playwright/test", "playwright", "playwright-core", "mcr.microsoft.com/playwright" + }, "Playwright npm packages and Docker image must update together" + + +class PlaywrightCI(unittest.TestCase): + def setUp(self): + self.workflow = (ROOT / ".github/workflows/ci.yml").read_text() + self.packages = {p: json.loads((ROOT / p / "package.json").read_text()) for p in JOBS.values()} + self.renovate = json.loads((ROOT / "renovate.json").read_text()) + + def check(self, workflow=None): + validate(self.workflow if workflow is None else workflow, self.packages, self.renovate) + + def test_repository(self): + self.check() + + def test_rejects_version_skew(self): + for package in self.packages.values(): + with self.subTest(package=package["name"]): + version = package["devDependencies"]["@playwright/test"] + package["devDependencies"]["@playwright/test"] = "0.0.0" + with self.assertRaisesRegex(AssertionError, "versions differ"): + self.check() + package["devDependencies"]["@playwright/test"] = version + + def test_rejects_missing_pins_and_runtime_installs(self): + for before, after in [ + (r"@sha256:[0-9a-f]{64}", ""), + (r"( sdk-browser:\n)", r"\1 # playwright install --with-deps\n"), + (r"( studio-browser:\n)", r"\1 # task studio:browser:install\n"), + (r"include-hidden-files: true", "include-hidden-files: false"), + (r"--user 1001:1001 ", ""), + ]: + with self.subTest(before=before), self.assertRaises(AssertionError): + self.check(re.sub(before, after, self.workflow)) + + def test_rejects_split_update_groups(self): + rule = next(r for r in self.renovate["packageRules"] if r.get("groupName") == "playwright") + rule["matchPackageNames"].remove("mcr.microsoft.com/playwright") + with self.assertRaisesRegex(AssertionError, "update together"): + self.check() + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e926aeeb1..ecb3b169b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -184,6 +184,7 @@ jobs: bash .github/scripts/macos-relevant-workflow_test.sh bash .github/scripts/relevant-changes_test.sh bash .github/scripts/relevant-changes-workflow_test.sh + python3 .github/scripts/playwright-ci_test.py sdk: name: TypeScript SDK (Node ${{ matrix.node-version }}) @@ -422,6 +423,14 @@ jobs: # changes are relevant. if: needs.changes.outputs.docs_only != 'true' && (needs.changes.outputs.go_relevant == 'true' || needs.changes.outputs.sdk_relevant == 'true') runs-on: ${{ matrix.runner }} + # Official image includes browsers and OS libraries; avoid apt mirror setup. + # Keep its version aligned with both @playwright/test pins (checked below). + container: + image: mcr.microsoft.com/playwright:v1.64.1-noble@sha256:47b510dd9230bf0b9bfc9b20102241e577ea7a99334ab4c3ca19f19770e9b8ea + options: --user 1001:1001 --init --ipc=host + defaults: + run: + shell: bash timeout-minutes: 20 env: MECATL_PREBUILT_MECATED: '1' @@ -471,25 +480,19 @@ jobs: - name: Install SDK dependencies (frozen) run: task sdk:install - - name: Resolve pinned Playwright version - id: playwright-version - working-directory: sdk/typescript - run: node -e "process.stdout.write('version=' + require('@playwright/test/package.json').version + '\\n')" >> "$GITHUB_OUTPUT" - - - name: Cache pinned Playwright browser - id: playwright-cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/ms-playwright - key: playwright-${{ matrix.runner }}-${{ runner.arch }}-${{ matrix.browser }}-${{ steps.playwright-version.outputs.version }} - - - name: Ensure pinned Playwright browser and host dependencies - working-directory: sdk/typescript - run: pnpm exec playwright install --with-deps ${{ matrix.browser }} - - name: Run browser project run: task sdk:browser:project PROJECT="${{ matrix.project }}" + - name: Upload browser failure diagnostics + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: sdk-browser-${{ matrix.project }} + path: .scratch/sdk-browser-playwright/ + include-hidden-files: true + retention-days: 7 + if-no-files-found: ignore + sdk-macos-spawn: name: TypeScript SDK macOS spawn (macos-14, Node 22) needs: changes @@ -856,6 +859,12 @@ jobs: needs: [changes, mecated] if: needs.changes.outputs.docs_only != 'true' && (needs.changes.outputs.studio_relevant == 'true' || needs.changes.outputs.go_relevant == 'true') runs-on: ubuntu-24.04 + container: + image: mcr.microsoft.com/playwright:v1.64.1-noble@sha256:47b510dd9230bf0b9bfc9b20102241e577ea7a99334ab4c3ca19f19770e9b8ea + options: --user 1001:1001 --init --ipc=host + defaults: + run: + shell: bash timeout-minutes: 20 env: MECATL_PREBUILT_MECATED: '1' @@ -901,12 +910,19 @@ jobs: - name: Install Studio dependencies (frozen) run: task studio:install - - name: Install Chromium for Studio browser journeys - run: task studio:browser:install - - name: Test Studio browser journeys (Playwright, offline) run: task studio:browser -- --project=${{ matrix.project }} + - name: Upload browser failure diagnostics + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: studio-browser-${{ matrix.project }} + path: .scratch/studio-browser-playwright/ + include-hidden-files: true + retention-days: 7 + if-no-files-found: ignore + studio-gate: name: Mecatl Studio (gate) needs: [changes, studio, studio-browser] diff --git a/Taskfile.yml b/Taskfile.yml index cb07a20e19..1a3a0655a4 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1180,6 +1180,7 @@ tasks: - sh .github/scripts/release-tag-guard_test.sh - sh .github/scripts/resolve-main-build_test.sh - python3 .github/scripts/instruction-files_test.py + - python3 .github/scripts/playwright-ci_test.py lint:action-templates: desc: | diff --git a/renovate.json b/renovate.json index 75c5110ae8..78f91404d3 100644 --- a/renovate.json +++ b/renovate.json @@ -100,6 +100,17 @@ "groupName": "javascript dependencies", "groupSlug": "javascript-dependencies" }, + { + "description": "Keep Playwright npm packages and the matching browser container in one update, including major upgrades", + "matchPackageNames": [ + "@playwright/test", + "playwright", + "playwright-core", + "mcr.microsoft.com/playwright" + ], + "groupName": "playwright", + "groupSlug": "playwright" + }, { "description": "Keep approval-gated runner upgrades out of the routine batch so they cannot hold it up", "matchManagers": ["github-actions"],