Diagnose intermittent CI crashes and website-data test aborts

.github/scripts/ci-crash-diagnosis.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# Capture evidence before android-emulator-runner shuts the emulator down.
+set -euo pipefail
+
+project=${DIAGNOSTIC_PROJECT:-.build/plugins/outputs/skip-web/SkipWebTests/destination/skipstone}
+output=${DIAGNOSTIC_DIR:?Set DIAGNOSTIC_DIR to a directory outside the checkout}
+scope=${DIAGNOSTIC_SCOPE:-suite}
+repetitions=${DIAGNOSTIC_REPETITIONS:-3}
+if [[ ! "$repetitions" =~ ^([1-9]|10)$ ]] || [[ "$scope" != suite && "$scope" != popup ]]; then
+    echo 'Expected DIAGNOSTIC_REPETITIONS=1..10 and DIAGNOSTIC_SCOPE=suite|popup' >&2
+    exit 64
+fi
+test -f "$project/settings.gradle.kts"
+mkdir -p "$output"
+adb shell dumpsys webviewupdate > "$output/webview-provider.txt" 2>&1
+adb shell getprop ro.build.fingerprint > "$output/android-build.txt"
+adb logcat -b all -v threadtime > "$output/logcat.txt" 2>&1 &
+logcat_pid=$!
+
+finish() {
+    result=$?
+    trap - EXIT
+    set +e
+    adb logcat -b crash -d -v threadtime > "$output/crash-buffer.txt" 2>&1
+    adb shell dumpsys meminfo > "$output/memory.txt" 2>&1
+    if [ "$result" -ne 0 ]; then
+        python3 - "$output/bugreport.zip" > "$output/bugreport-command.txt" 2>&1 <<'PY'
+import subprocess
+import sys
+try:
+    subprocess.run(['adb', 'bugreport', sys.argv[1]], timeout=120, check=False)
+except subprocess.TimeoutExpired:
+    print('Bug report timed out after 120 seconds; other diagnostic files are retained.')
+PY
+    fi
+    kill "$logcat_pid" 2>/dev/null
+    wait "$logcat_pid" 2>/dev/null
+    exit "$result"
+}
+trap finish EXIT
+
+gradle_args=(:SkipWeb:connectedDebugAndroidTest --project-dir "$project"
+    -PbuildDir=.build/SkipWeb --rerun-tasks --console=plain --stacktrace)
+if [[ "$scope" == popup ]]; then
+    gradle_args+=('-Pandroid.testInstrumentationRunnerArguments.class=skip.web.AndroidPopupStateIsolationTests')
+fi
+
+failed=0
+for ((attempt = 1; attempt <= repetitions; attempt++)); do
+    attempt_dir="$output/attempt-$attempt"
+    mkdir -p "$attempt_dir"
+    echo "Running $scope tests, attempt $attempt/$repetitions"
+    adb shell log -t SkipWebCI "BEGIN attempt=$attempt scope=$scope"
+    set +e
+    gradle "${gradle_args[@]}" 2>&1 | tee "$attempt_dir/gradle.log"
+    result=${PIPESTATUS[0]}
+    set -e
+    echo "$result" > "$attempt_dir/exit-code.txt"
+    adb shell log -t SkipWebCI "END attempt=$attempt exit=$result" || true
+    # Copy reports before the next run overwrites them. Include failed/partial runs.
+    build="$project/SkipWeb/.build/SkipWeb"
+    for report in outputs/androidTest-results reports/androidTests; do
+        if [[ -d "$build/$report" ]]; then
+            cp -R "$build/$report" "$attempt_dir/"
+        fi
+    done
+    if [[ "$result" -ne 0 ]]; then
+        failed=1
+        # Keep running after a test-process crash to measure intermittency.
+        # The final nonzero result still makes CI fail even if later attempts pass.
+    fi
+done
+exit "$failed"

.github/workflows/ci-crash-diagnosis.yml

@@ -0,0 +1,113 @@
+name: CI crash diagnosis
+
+on:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      scope:
+        description: Run the full suite or only the popup tests
+        type: choice
+        options: [suite, popup]
+        default: suite
+      repetitions:
+        description: Number of runs per compiler version (1–10)
+        type: string
+        default: '3'
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  android:
+    if: github.event_name == 'workflow_dispatch' || github.head_ref == 'ci-crash-diagnosis'
+    runs-on: macos-26-intel
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        skip: ['1.9.19', '1.9.20']
+    env:
+      SKIP_BRIDGE: '0'
+      ANDROID_SERIAL: emulator-5554
+      DIAGNOSTIC_SCOPE: ${{ inputs.scope || 'suite' }}
+      DIAGNOSTIC_REPETITIONS: ${{ inputs.repetitions || '3' }}
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set diagnostic output directory
+        run: echo "DIAGNOSTIC_DIR=$RUNNER_TEMP/ci-crash-diagnosis" >> "$GITHUB_ENV"
+      - uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: '21'
+      - uses: gradle/actions/setup-gradle@v5
+        with:
+          gradle-version: '9.8.2'
+
+      # SwiftPM downloads the compiler bundled with the selected Skip package.
+      # No separately installed Skip CLI can override this comparison.
+      - name: Pin the diagnostic dependency graph
+        env:
+          DIAGNOSTIC_SKIP_VERSION: ${{ matrix.skip }}
+        run: |
+          mkdir -p "$DIAGNOSTIC_DIR"
+          git rev-parse HEAD > "$DIAGNOSTIC_DIR/commit.txt"
+          cp Package.swift "$DIAGNOSTIC_DIR/Package.original.swift"
+          python3 - <<'PY'
+          import os
+          from pathlib import Path
+          path = Path('Package.swift')
+          source = path.read_text()
+          versions = {'skip': os.environ['DIAGNOSTIC_SKIP_VERSION'], 'skip-ui': '1.65.0'}
+          minimums = {'skip': '1.8.9', 'skip-ui': '1.54.0'}
+          for package, version in versions.items():
+              old = f'.package(url: "https://github.com/skiptools/{package}.git", from: "{minimums[package]}")'
+              new = f'.package(url: "https://github.com/skiptools/{package}.git", exact: "{version}")'
+              assert source.count(old) == 1, f'Unexpected manifest entry for {package}'
+              source = source.replace(old, new)
+          # Keep the remaining transpiled dependencies at the failing run's versions.
+          pins = {'skip-lib': '1.4.4', 'skip-foundation': '1.4.8', 'skip-model': '1.8.1', 'skip-unit': '1.7.2'}
+          extra = ''.join(f'        .package(url: "https://github.com/skiptools/{name}.git", exact: "{version}"),\n' for name, version in pins.items())
+          source = source.replace('    dependencies: [\n', '    dependencies: [\n' + extra, 1)
+          path.write_text(source)
+          PY
+          swift --version | tee "$DIAGNOSTIC_DIR/swift-version.txt"
+          java -version 2> "$DIAGNOSTIC_DIR/java-version.txt"
+          gradle --version > "$DIAGNOSTIC_DIR/gradle-version.txt"
+
+      - name: Generate and build the test sources
+        run: swift build --build-tests 2>&1 | tee "$DIAGNOSTIC_DIR/swift-build.log"
+
+      - name: Repeat Android tests with native crash capture
+        uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2, same as the failed run
+        with:
+          api-level: 28
+          target: default
+          arch: x86_64
+          profile: Nexus 4
+          disk-size: 4096M
+          disable-animations: true
+          emulator-options: -no-snapshot -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          script: bash .github/scripts/ci-crash-diagnosis.sh
+
+      - name: Retain compiler inputs and generated sources
+        if: always()
+        run: |
+          mkdir -p "$DIAGNOSTIC_DIR"
+          cp Package.swift "$DIAGNOSTIC_DIR/Package.diagnostic.swift"
+          if [ -f Package.resolved ]; then cp Package.resolved "$DIAGNOSTIC_DIR/"; fi
+          project=.build/plugins/outputs/skip-web/SkipWebTests/destination/skipstone
+          if [ -d "$project/SkipWeb/src" ]; then
+            cp -R "$project/SkipWeb/src" "$DIAGNOSTIC_DIR/generated-sources"
+            cp "$project/settings.gradle.kts" "$DIAGNOSTIC_DIR/"
+            cp "$project/SkipWeb/build.gradle.kts" "$DIAGNOSTIC_DIR/"
+          fi
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: android-crash-skip-${{ matrix.skip }}
+          path: ${{ runner.temp }}/ci-crash-diagnosis
+          retention-days: 14

Tests/SkipWebTests/AndroidPopupStateIsolationTests.swift

@@ -11,6 +11,19 @@ final class AndroidPopupStateIsolationTests: XCTestCase {
     @get:org.junit.Rule
     val composeRule = androidx.compose.ui.test.junit4.createComposeRule()
 
+    @get:org.junit.Rule
+    val diagnosticRule = object : org.junit.rules.TestWatcher() {
+        override fun starting(description: org.junit.runner.Description) {
+            android.util.Log.i("SkipWebPopup", "BEGIN ${description.methodName}")
+        }
+        override fun failed(error: Throwable, description: org.junit.runner.Description) {
+            android.util.Log.e("SkipWebPopup", "FAILED ${description.methodName}", error)
+        }
+        override fun finished(description: org.junit.runner.Description) {
+            android.util.Log.i("SkipWebPopup", "END ${description.methodName}")
+        }
+    }
+
     @Test
     fun testBlankPopupNavigationDoesNotOverwriteParentStateOrNotifyParent() {
         composeRule.runOnIdle {
@@ -68,6 +81,7 @@ final class AndroidPopupStateIsolationTests: XCTestCase {
                 }
             }
             composeRule.runOnIdle {
+                popup.trace("mounted callbacks begin")
                 org.junit.Assert.assertTrue(popup.childView.isAttachedToWindow)
                 mountedEvents.clear()
                 popup.childView.documentURL = "https://example.test/allowed-popup"
@@ -80,6 +94,7 @@ final class AndroidPopupStateIsolationTests: XCTestCase {
                 org.junit.Assert.assertEquals(listOf("committed", "finished"), mountedEvents)
                 popup.assertParentUnchanged()
                 org.junit.Assert.assertEquals(emptyList<String>(), popup.parentEvents)
+                popup.trace("mounted callbacks end")
             }
         } finally {
             composeRule.runOnIdle { visible.value = false }
@@ -93,10 +108,12 @@ final class AndroidPopupStateIsolationTests: XCTestCase {
             val popup = PopupFixture()
             try {
                 val client = popup.childView.webViewClient
+                popup.trace("navigation policy callbacks begin")
                 org.junit.Assert.assertTrue(client.shouldOverrideUrlLoading(popup.childView, Request("https://example.test/blocked")))
                 org.junit.Assert.assertFalse(client.shouldOverrideUrlLoading(popup.childView, Request("https://example.test/allowed")))
                 org.junit.Assert.assertEquals(listOf("https://example.test/blocked", "https://example.test/allowed"), popup.policyURLs)
                 popup.assertParentUnchanged()
+                popup.trace("navigation policy callbacks end")
             } finally {
                 popup.close()
             }
@@ -125,14 +142,15 @@ final class AndroidPopupStateIsolationTests: XCTestCase {
     }
 
     private class PopupFixture {
+        private val diagnosticID = System.identityHashCode(this)
         val parentState = WebViewState()
         val parentEvents = mutableListOf<String>()
         val childEvents = mutableListOf<String>()
         val downloads = mutableListOf<WebDownloadRequest>()
         val policyURLs = mutableListOf<String>()
         val parentURL = skip.foundation.URL(string = "https://example.test/article")
-        val parentEngine = WebEngine()
-        val childView = PopupPlatformView(parentEngine.webView.context)
+        val parentEngine = WebEngine().also { trace("parent engine created") }
+        val childView = PopupPlatformView(parentEngine.webView.context).also { trace("child view created") }
         val childDelegate = object: SkipWebNavigationDelegate {
             override fun webEngineDidStartProvisionalNavigation(engine: WebEngine) { childEvents.add("started") }
             override fun webEngineDidCommitNavigation(engine: WebEngine) { childEvents.add("committed") }
@@ -142,6 +160,8 @@ final class AndroidPopupStateIsolationTests: XCTestCase {
         val owner: WebView
 
         init {
+            val provider = android.webkit.WebView.getCurrentWebViewPackage()
+            trace("provider=${provider?.packageName} version=${provider?.versionName} api=${android.os.Build.VERSION.SDK_INT}")
             parentState.url = parentURL
             parentState.pageTitle = "Parent article"
             parentState.canGoBack = true
@@ -166,8 +186,10 @@ final class AndroidPopupStateIsolationTests: XCTestCase {
             val transport = parentEngine.webView.WebViewTransport()
             val message = android.os.Message.obtain(android.os.Handler(android.os.Looper.getMainLooper()))
             message.obj = transport
+            trace("onCreateWindow begin")
             org.junit.Assert.assertTrue(chrome.onCreateWindow(parentEngine.webView, false, true, message))
             org.junit.Assert.assertSame(childView, transport.webView)
+            trace("onCreateWindow end")
         }
 
         fun assertParentUnchanged() {
@@ -181,8 +203,15 @@ final class AndroidPopupStateIsolationTests: XCTestCase {
         }
 
         fun close() {
+            trace("child destroy begin attached=${childView.isAttachedToWindow}")
             childView.destroy()
+            trace("child destroy end; parent destroy begin attached=${parentEngine.webView.isAttachedToWindow}")
             parentEngine.webView.destroy()
+            trace("parent destroy end")
+        }
+
+        fun trace(stage: String) {
+            android.util.Log.i("SkipWebPopup", "fixture=$diagnosticID uptimeMs=${android.os.SystemClock.uptimeMillis()} thread=${Thread.currentThread().name} $stage")
         }
     }
     */

Tests/SkipWebTests/WebsiteDataTestServer.swift

@@ -225,14 +225,17 @@ final class WebsiteDataTestServer: @unchecked Sendable {
       localStorage.setItem('fixture', value);
       sessionStorage.setItem('fixture', value);
       const db = await fixtureAwait('database open', () => database());
-      await fixtureAwait('seed transaction complete', () => new Promise((resolve, reject) => {
-        const tx = db.transaction('values', 'readwrite');
-        tx.objectStore('values').put(value, 'fixture');
-        tx.oncomplete = resolve; tx.onerror = () => reject(tx.error);
-        tx.onabort = () => fixtureEvent('seed transaction aborted: ' + String(tx.error));
-      }));
-      db.close();
-      fixtureEvent('database close');
+      try {
+        await fixtureAwait('seed transaction complete', () => new Promise((resolve, reject) => {
+          const tx = db.transaction('values', 'readwrite');
+          tx.objectStore('values').put(value, 'fixture');
+          tx.oncomplete = resolve; tx.onerror = () => reject(tx.error);
+          tx.onabort = () => reject(tx.error || new Error('seed transaction aborted without an error'));
+        }));
+      } finally {
+        db.close();
+        fixtureEvent('database close');
+      }
       await cachedValue(key);
     }
     async function cachedValue(key) {
@@ -242,14 +245,18 @@ final class WebsiteDataTestServer: @unchecked Sendable {
     }
     async function read(key) {
       const db = await fixtureAwait('database open', () => database());
-      const indexed = await fixtureAwait('read indexed value', () => new Promise((resolve, reject) => {
-        const tx = db.transaction('values');
-        tx.onabort = () => fixtureEvent('read transaction aborted: ' + String(tx.error));
-        const request = tx.objectStore('values').get('fixture');
-        request.onsuccess = () => resolve(request.result || '');
-        request.onerror = () => reject(request.error);
-      }));
-      db.close();
+      let indexed;
+      try {
+        indexed = await fixtureAwait('read indexed value', () => new Promise((resolve, reject) => {
+          const tx = db.transaction('values');
+          tx.onabort = () => reject(tx.error || new Error('read transaction aborted without an error'));
+          const request = tx.objectStore('values').get('fixture');
+          request.onsuccess = () => resolve(request.result || '');
+          request.onerror = () => reject(request.error);
+        }));
+      } finally {
+        db.close();
+      }
       fixtureEvent('database close; read cookie/localStorage/sessionStorage');
       return { cookie: document.cookie, local: localStorage.getItem('fixture') || '',
         session: sessionStorage.getItem('fixture') || '', indexed: indexed,