From ec12768bd104f8dc3bc30a6415a82acad2592c55 Mon Sep 17 00:00:00 2001 From: Deven Ducommun <155489027+DevenDucommun@users.noreply.github.com> Date: Thu, 16 Jul 2026 09:02:53 -0700 Subject: [PATCH 1/2] =?UTF-8?q?ci:=20tiered=20integration=20testing=20?= =?UTF-8?q?=E2=80=94=20per-PR=20mock=20tier=20+=20nightly=20device=20tier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits the 21 integration flows into a mockable per-PR tier (13 flows, headless, no router) and a device-bound nightly tier (8 flows, lab router via integration_web.sh). Classification and rationale in integration_test/FLOW-TIERS.md. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci-integration-device.yml | 65 +++++++++++++++ .github/workflows/ci-integration-mock.yml | 91 +++++++++++++++++++++ integration_test/FLOW-TIERS.md | 69 ++++++++++++++++ 3 files changed, 225 insertions(+) create mode 100644 .github/workflows/ci-integration-device.yml create mode 100644 .github/workflows/ci-integration-mock.yml create mode 100644 integration_test/FLOW-TIERS.md diff --git a/.github/workflows/ci-integration-device.yml b/.github/workflows/ci-integration-device.yml new file mode 100644 index 000000000..a63c7bd33 --- /dev/null +++ b/.github/workflows/ci-integration-device.yml @@ -0,0 +1,65 @@ +# Nightly device-bound integration flows on the lab router (tier: device). +# Classification and rationale: integration_test/FLOW-TIERS.md +# RUNNER: waits on a self-hosted runner (label: lab-lan) on the QA lab LAN with +# the lab router reachable at 192.168.1.1. A Jetson box is being provisioned +# for this. Router admin password comes from the ROUTER_ADMIN_PASSWORD secret. +# NOTE: this tier uses the repo's own harness (integration_web.sh + +# shell_scripts/*) which handles factory reset, PnP skip, and device-mode waits. +name: Integration (device tier, nightly) + +on: + schedule: + - cron: '30 18 * * *' # 02:30 Taipei / 18:30 UTC, outside CBT lab hours + workflow_dispatch: + inputs: + tags: + description: 'test_meta tags to run (default = device-bound set)' + required: false + default: 'pnp,wifi,internet_settings,speed_test' + +concurrency: + group: integration-device + cancel-in-progress: false # never interrupt a run that may have factory-reset the router + +jobs: + device-flows: + runs-on: [self-hosted, lab-lan] + timeout-minutes: 240 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Flutter + uses: subosito/flutter-action@v2 + with: + channel: stable + cache: true + + - name: Install Dependencies + run: flutter pub get + + - name: Preflight - router reachable + run: | + curl -sk --max-time 10 https://192.168.1.1/ > /dev/null + bash integration_test/shell_scripts/get_device_info.sh + + # Device-bound flows per integration_test/FLOW-TIERS.md. + # integration_web.sh resolves tags via test_meta/*.json and runs each flow's + # declared setUp (factory_reset, skip_setup, pre_paired) before flutter drive. + - name: Run device-bound flows + env: + ROUTER_ADMIN_PASSWORD: ${{ secrets.ROUTER_ADMIN_PASSWORD }} + run: | + bash integration_web.sh -t "${{ inputs.tags || 'pnp,wifi,internet_settings,speed_test' }}" + + - name: Generate report + if: always() + run: bash integration_test/shell_scripts/generate_integration_report.sh || true + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: device-tier-report-${{ github.run_number }} + path: build/integration_test/ + retention-days: 30 diff --git a/.github/workflows/ci-integration-mock.yml b/.github/workflows/ci-integration-mock.yml new file mode 100644 index 000000000..0b8bfe03e --- /dev/null +++ b/.github/workflows/ci-integration-mock.yml @@ -0,0 +1,91 @@ +# Per-PR mockable integration flows (tier: mock). +# Classification and rationale: integration_test/FLOW-TIERS.md +# TODO(UI team): confirm the mock/demo JNAP flag used by demo builds +# (referenced below as --dart-define=FORCE_COMMAND_TYPE=local) and that the +# headless chromedriver invocation matches integration_web.sh conventions. +name: Integration (mock tier, per-PR) + +on: + pull_request: + branches: [main, 'dev-1.*'] # integration_test/ exists on the 1.x line only + paths: + - 'lib/**' + - 'integration_test/**' + - 'pubspec.yaml' + +concurrency: + group: integration-mock-${{ github.ref }} + cancel-in-progress: true + +jobs: + mock-flows: + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + # 13 mockable flows per integration_test/FLOW-TIERS.md + flow: + - administration_test + - advanced_routing_test + - apps_and_gaming_test + - dashboard_home_test + - dhcp_reservations_test + - dmz_test + - firewall_test + - instant_admin_test + - instant_privacy_test + - instant_safety_test + - local_network_settings_test + - menu_test + - recovery_and_login_test + max-parallel: 4 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Flutter + uses: subosito/flutter-action@v2 + with: + channel: stable + cache: true + + - name: Setup SSH for Private Deps + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.UI_KIT_DEPLOY_KEY }} + + - name: Configure Git SSH + run: git config --global url."git@github.com:".insteadOf "https://github.com/" + + - name: Setup Environment + run: cp assets/agents/env.template assets/agents/.env + + - name: Install Dependencies + run: flutter pub get + + - name: Setup Chrome + chromedriver + uses: browser-actions/setup-chrome@v1 + with: + install-chromedriver: true + + - name: Start chromedriver + run: chromedriver --port=4444 & + + - name: Run ${{ matrix.flow }} (mocked JNAP) + run: | + flutter drive \ + --driver=test_driver/integration_test.dart \ + --target=integration_test/${{ matrix.flow }}.dart \ + --dart-define=FORCE_COMMAND_TYPE=local \ + -d web-server --headless \ + --browser-name=chrome --driver-port=4444 + + - name: Upload failure artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: mock-flow-failure-${{ matrix.flow }} + path: | + build/integration_test/ + retention-days: 7 diff --git a/integration_test/FLOW-TIERS.md b/integration_test/FLOW-TIERS.md new file mode 100644 index 000000000..bb67ed21b --- /dev/null +++ b/integration_test/FLOW-TIERS.md @@ -0,0 +1,69 @@ +# Integration Flow Tiers: Mockable vs Device-Bound + +> Proposal from QA (Deven). Splits the 21 integration flows into a per-PR mock tier +> and a nightly device tier so the suite runs automatically instead of manually. +> Companion workflows: `.github/workflows/ci-integration-mock.yml` and +> `ci-integration-device.yml`. + +## Classification principle + +Every flow can technically run with mocked JNAP. The split is about where the +regression value lives: + +- **MOCKABLE**: the flow asserts UI behavior and the UI-to-JNAP contract (forms, + validation, CRUD echo, navigation). A mocked JNAP layer preserves nearly all of its + regression value. Run per-PR, headless, no router. +- **DEVICE-BOUND**: the flow's value depends on real device behavior (radio apply, + WAN state machines, device-mode transitions, real measurements). Run nightly on the + lab router via `integration_web.sh`. + +## The split (13 mockable / 8 device-bound) + +### Mockable, per-PR tier + +| Flow | Why mockable | +|---|---| +| `administration_test` | Toggle persistence is a JNAP Set/Get echo, no HW behavior asserted | +| `advanced_routing_test` | Route CRUD plus validation error rendering | +| `apps_and_gaming_test` | DDNS / port-forwarding / triggering form CRUD | +| `dashboard_home_test` | Outer-layer cards and navigation; speed-test card stubbed | +| `dhcp_reservations_test` | Add/edit reservation UI plus Get-back verification | +| `dmz_test` | Enable switch plus IP/MAC form management | +| `firewall_test` | Switch states plus IPv6 port-service CRUD | +| `instant_admin_test` | Password/timezone forms; manual-FW-update entry stubbed | +| `instant_privacy_test` | UI contract is mockable. Live runs also risk kicking the test client off WiFi (MAC filter), an extra reason to keep it in the mock tier | +| `instant_safety_test` | Enable flow, JNAP contract only | +| `local_network_settings_test` | Hostname/LAN IP/DHCP-pool forms; mock avoids the real re-IP dance | +| `menu_test` | Pure navigation, every card and button on the dashboard menu | +| `recovery_and_login_test` | Login and forgot-password flow against mocked auth responses | + +### Device-bound, nightly tier + +| Flow | Why device-bound | +|---|---| +| `factory_reset_setup_test` | Full PnP on a genuinely reset device; device-mode state machine | +| `prepaired_reset_setup_test` | Pre-paired PnP path; mesh pairing state is real-device-only | +| `incredible_wifi_test` | Band settings must apply to real radios | +| `incredible_wifi_advanced_and_mac_filtering_test` | Advanced radio switches, MLO alert, MAC filtering against real clients | +| `internet_settings_set_1_test` | IPv4 DHCP/Static/PPPoE, real WAN state transitions | +| `internet_settings_set_2_test` | PPTP/L2TP/Bridge mode, WAN mode switching | +| `internet_settings_set_3_test` | IPv6 DHCP/Static/PPPoE, dual-stack behavior | +| `speed_test_test` | Real measurement; meta itself says only run on devices supporting speed test | + +## Open items for review + +1. **Mock flag**: the mock workflow assumes `--dart-define=FORCE_COMMAND_TYPE=local` + activates the mocked JNAP layer the demo builds use. Please correct if the flag or + mechanism differs. +2. **Two rows worth a second opinion**: `dashboard_home_test` (speed-test card + interaction depth) and `local_network_settings_test` (whether it asserts post-re-IP + reachability). +3. **`tier` field proposal**: add `"tier": "mock" | "device"` to `test_meta/*.json` so + `integration_web.sh -t` and CI can select by tier instead of hardcoded lists. Happy + to do this as a follow-up once the classification is agreed. +4. **Model genericity**: flows must stay model-generic across M60/M61/M62. Known model + difference: band count (6 GHz coverage in `incredible_wifi_test` is conditional on + tri-band hardware). Proposal is to data-drive model differences, never branch test + logic on model. +5. **2.x line**: `integration_test/` exists on `main` and the 1.x line only. Should the + suite (or the mock tier at least) be ported to the `usp`/2.x line? From 4e13f92036999d9f6e01be5a350c04fceb684e6a Mon Sep 17 00:00:00 2001 From: Deven Ducommun <155489027+DevenDucommun@users.noreply.github.com> Date: Thu, 16 Jul 2026 09:37:52 -0700 Subject: [PATCH 2/2] =?UTF-8?q?ci:=20address=20review=20round=201=20?= =?UTF-8?q?=E2=80=94=20correct=20mock=20flag,=20harden=20workflows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - mock: --dart-define=force=local (C-1, matches build_config.dart 'force' key and integration_web.sh:219), env.template guard (W-1), chromedriver readiness wait, permissions block - device: SSH-agent + env setup steps (W-2/W-3), tags via env var not inline interpolation (W-4), permissions block, TLS + password notes - FLOW-TIERS: open item 1 resolved, mock-tier self-containment caveat (W-6), cross-tier tag warning (W-7), dashboard_home rationale honesty (W-8), W-5 password threading recorded as open item Co-Authored-By: Claude Fable 5 --- .github/workflows/ci-integration-device.yml | 27 +++++++++++++++- .github/workflows/ci-integration-mock.yml | 28 ++++++++++++---- integration_test/FLOW-TIERS.md | 36 +++++++++++++++------ 3 files changed, 74 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci-integration-device.yml b/.github/workflows/ci-integration-device.yml index a63c7bd33..e4af70693 100644 --- a/.github/workflows/ci-integration-device.yml +++ b/.github/workflows/ci-integration-device.yml @@ -3,6 +3,8 @@ # RUNNER: waits on a self-hosted runner (label: lab-lan) on the QA lab LAN with # the lab router reachable at 192.168.1.1. A Jetson box is being provisioned # for this. Router admin password comes from the ROUTER_ADMIN_PASSWORD secret. +# OPEN (review W-5): integration_web.sh does not read that env var yet; the +# password must be threaded into the harness config before first live run. # NOTE: this tier uses the repo's own harness (integration_web.sh + # shell_scripts/*) which handles factory reset, PnP skip, and device-mode waits. name: Integration (device tier, nightly) @@ -17,6 +19,9 @@ on: required: false default: 'pnp,wifi,internet_settings,speed_test' +permissions: + contents: read + concurrency: group: integration-device cancel-in-progress: false # never interrupt a run that may have factory-reset the router @@ -35,9 +40,27 @@ jobs: channel: stable cache: true + - name: Setup SSH for Private Deps + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.UI_KIT_DEPLOY_KEY }} + + - name: Configure Git SSH + run: git config --global url."git@github.com:".insteadOf "https://github.com/" + + - name: Setup Environment + run: | + mkdir -p assets/agents + if [ -f assets/agents/env.template ]; then + cp assets/agents/env.template assets/agents/.env + else + touch assets/agents/.env + fi + - name: Install Dependencies run: flutter pub get + # -k is deliberate: the lab router serves a self-signed cert on an isolated LAN. - name: Preflight - router reachable run: | curl -sk --max-time 10 https://192.168.1.1/ > /dev/null @@ -46,11 +69,13 @@ jobs: # Device-bound flows per integration_test/FLOW-TIERS.md. # integration_web.sh resolves tags via test_meta/*.json and runs each flow's # declared setUp (factory_reset, skip_setup, pre_paired) before flutter drive. + # inputs.tags goes through an env var, never inline interpolation (review W-4). - name: Run device-bound flows env: ROUTER_ADMIN_PASSWORD: ${{ secrets.ROUTER_ADMIN_PASSWORD }} + TAGS_INPUT: ${{ inputs.tags || 'pnp,wifi,internet_settings,speed_test' }} run: | - bash integration_web.sh -t "${{ inputs.tags || 'pnp,wifi,internet_settings,speed_test' }}" + bash integration_web.sh -t "$TAGS_INPUT" - name: Generate report if: always() diff --git a/.github/workflows/ci-integration-mock.yml b/.github/workflows/ci-integration-mock.yml index 0b8bfe03e..a24a3c091 100644 --- a/.github/workflows/ci-integration-mock.yml +++ b/.github/workflows/ci-integration-mock.yml @@ -1,8 +1,7 @@ # Per-PR mockable integration flows (tier: mock). # Classification and rationale: integration_test/FLOW-TIERS.md -# TODO(UI team): confirm the mock/demo JNAP flag used by demo builds -# (referenced below as --dart-define=FORCE_COMMAND_TYPE=local) and that the -# headless chromedriver invocation matches integration_web.sh conventions. +# Mock JNAP layer flag confirmed in review round 1: the app reads the 'force' +# dart-define (lib/constants/build_config.dart), same as integration_web.sh. name: Integration (mock tier, per-PR) on: @@ -13,6 +12,9 @@ on: - 'integration_test/**' - 'pubspec.yaml' +permissions: + contents: read + concurrency: group: integration-mock-${{ github.ref }} cancel-in-progress: true @@ -58,8 +60,15 @@ jobs: - name: Configure Git SSH run: git config --global url."git@github.com:".insteadOf "https://github.com/" + # env.template is not tracked in git; fall back to an empty .env (review W-1). - name: Setup Environment - run: cp assets/agents/env.template assets/agents/.env + run: | + mkdir -p assets/agents + if [ -f assets/agents/env.template ]; then + cp assets/agents/env.template assets/agents/.env + else + touch assets/agents/.env + fi - name: Install Dependencies run: flutter pub get @@ -70,14 +79,21 @@ jobs: install-chromedriver: true - name: Start chromedriver - run: chromedriver --port=4444 & + run: | + chromedriver --port=4444 & + for i in $(seq 1 30); do + curl -sf http://127.0.0.1:4444/status > /dev/null && break + sleep 1 + done + # NOTE: this bypasses integration_web.sh, so test_meta setUp/tearDown hooks do + # NOT run. Mock-tier flows must be self-contained (see FLOW-TIERS.md caveat). - name: Run ${{ matrix.flow }} (mocked JNAP) run: | flutter drive \ --driver=test_driver/integration_test.dart \ --target=integration_test/${{ matrix.flow }}.dart \ - --dart-define=FORCE_COMMAND_TYPE=local \ + --dart-define=force=local \ -d web-server --headless \ --browser-name=chrome --driver-port=4444 diff --git a/integration_test/FLOW-TIERS.md b/integration_test/FLOW-TIERS.md index bb67ed21b..bb6ecb57e 100644 --- a/integration_test/FLOW-TIERS.md +++ b/integration_test/FLOW-TIERS.md @@ -17,6 +17,16 @@ regression value lives: WAN state machines, device-mode transitions, real measurements). Run nightly on the lab router via `integration_web.sh`. +**Mock-tier caveat (review W-6):** the mock workflow calls `flutter drive` directly, +so the `setUp`/`tearDown` hooks declared in `test_meta/*.json` do NOT run in the mock +tier. Mock-tier flows must therefore be self-contained against the mocked JNAP state. +Whether all 13 actually are is a first-run validation item. + +**Tag caveat (review W-7):** `test_meta` tags cross tiers (e.g. `menu` is on the +mockable `menu_test` AND device-bound wifi/speed flows). Do not use bare tags to +select a tier with `integration_web.sh -t`; the `tier` field (open item 3) is the +real fix. + ## The split (13 mockable / 8 device-bound) ### Mockable, per-PR tier @@ -26,14 +36,14 @@ regression value lives: | `administration_test` | Toggle persistence is a JNAP Set/Get echo, no HW behavior asserted | | `advanced_routing_test` | Route CRUD plus validation error rendering | | `apps_and_gaming_test` | DDNS / port-forwarding / triggering form CRUD | -| `dashboard_home_test` | Outer-layer cards and navigation; speed-test card stubbed | +| `dashboard_home_test` | Outer-layer cards and navigation. OPEN (review W-8): test_meta says the flow includes Speed Test interaction; whether that triggers a real measurement (which a mock run cannot answer) or only renders the card needs confirming before this row is trusted | | `dhcp_reservations_test` | Add/edit reservation UI plus Get-back verification | | `dmz_test` | Enable switch plus IP/MAC form management | | `firewall_test` | Switch states plus IPv6 port-service CRUD | | `instant_admin_test` | Password/timezone forms; manual-FW-update entry stubbed | | `instant_privacy_test` | UI contract is mockable. Live runs also risk kicking the test client off WiFi (MAC filter), an extra reason to keep it in the mock tier | | `instant_safety_test` | Enable flow, JNAP contract only | -| `local_network_settings_test` | Hostname/LAN IP/DHCP-pool forms; mock avoids the real re-IP dance | +| `local_network_settings_test` | Hostname/LAN IP/DHCP-pool forms; mock avoids the real re-IP dance. Open: confirm the test does not assert post-re-IP reachability (mock cannot answer that) | | `menu_test` | Pure navigation, every card and button on the dashboard menu | | `recovery_and_login_test` | Login and forgot-password flow against mocked auth responses | @@ -52,18 +62,24 @@ regression value lives: ## Open items for review -1. **Mock flag**: the mock workflow assumes `--dart-define=FORCE_COMMAND_TYPE=local` - activates the mocked JNAP layer the demo builds use. Please correct if the flag or - mechanism differs. -2. **Two rows worth a second opinion**: `dashboard_home_test` (speed-test card - interaction depth) and `local_network_settings_test` (whether it asserts post-re-IP - reachability). +1. ~~Mock flag~~ **RESOLVED (review round 1, C-1):** the app reads the `force` + dart-define (`lib/constants/build_config.dart`); the workflow now passes + `--dart-define=force=local`, matching `integration_web.sh:219`. +2. **Two rows worth a second opinion**: `dashboard_home_test` (Speed Test interaction + depth, see W-8 note in its row) and `local_network_settings_test` (post-re-IP + reachability assertion). 3. **`tier` field proposal**: add `"tier": "mock" | "device"` to `test_meta/*.json` so - `integration_web.sh -t` and CI can select by tier instead of hardcoded lists. Happy - to do this as a follow-up once the classification is agreed. + selection is data-driven instead of hardcoded lists, and cross-tier tags (W-7) stop + being a foot-gun. Happy to do this as a follow-up once the classification is agreed. 4. **Model genericity**: flows must stay model-generic across M60/M61/M62. Known model difference: band count (6 GHz coverage in `incredible_wifi_test` is conditional on tri-band hardware). Proposal is to data-drive model differences, never branch test logic on model. 5. **2.x line**: `integration_test/` exists on `main` and the 1.x line only. Should the suite (or the mock tier at least) be ported to the `usp`/2.x line? +6. **Admin password threading (review W-5)**: the device workflow exports + `ROUTER_ADMIN_PASSWORD`, but `integration_web.sh` never reads it; the harness gets + the password from `is_default_admin_password_or_get_password()` (default or config + JSON). Before the first live nightly run, thread the secret into the harness config + (or agree the lab device stays on the label-default password, which factory resets + restore anyway).