diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index eb0cea96a7c..3f8ca060f8f 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -6,6 +6,7 @@ import os import glob import re +import tempfile import time import json @@ -14,7 +15,7 @@ from logging import getLogger from argparse import ArgumentParser from argparse import RawTextHelpFormatter -from shutil import rmtree +from shutil import make_archive, rmtree from typing import Optional from shared.androidhelper import AndroidHelper from shared.androidinstrumentation import AndroidInstrumentationHelper @@ -767,6 +768,43 @@ def run(self): RunCommand(installCmd, verbose=True).run() getLogger().info("Completed install.") + # The Mac.iPhone.17.Perf devices in the Helix pool are shared across many test + # runs over days, and iOS retains crash reports in /var/mobile/Library/Logs/ + # CrashReporter/ until rotated out. To avoid re-uploading that historical + # backlog on a kill failure, snapshot the device's current crash report set + # now and below upload only reports that appeared since this snapshot. + # This mirrors XHarness's CrashSnapshotReporter pattern (it's a copy, not a + # move — device state is unchanged). + def listDeviceCrashReports(): + """Return the set of crash report identifiers currently on the device, + or None if listing failed.""" + listFilePath = None + try: + with tempfile.NamedTemporaryFile(mode='w', suffix='.list', delete=False) as listFile: + listFilePath = listFile.name + listCmd = xharnesscommand() + [ + 'apple', 'mlaunch', '--', + f'--list-crash-reports={listFilePath}', + '--devname', deviceUDID, + ] + RunCommand(listCmd, verbose=True).run() + with open(listFilePath) as f: + return {line.strip() for line in f if line.strip()} + except Exception as listEx: + getLogger().warning(f"Failed to list device crash reports: {listEx}") + return None + finally: + if listFilePath and os.path.exists(listFilePath): + try: + os.remove(listFilePath) + except OSError: + pass + + getLogger().info("Snapshotting existing crash reports on device.") + initial_device_crashes = listDeviceCrashReports() + if initial_device_crashes is not None: + getLogger().info(f"Found {len(initial_device_crashes)} pre-existing crash report(s).") + allResults = [] timeToFirstDrawEventEndDateTime = datetime.now() + timedelta(minutes=-10) # This is used to keep track of the latest time to draw end event, we use this to calculate time to draw and also as a reference point for the next iteration log time. for i in range(self.startupiterations + 1): # adding one iteration to account for the warmup iteration @@ -843,7 +881,62 @@ def run(self): '--devname', deviceUDID ] killCmdCommand = RunCommand(killCmd, verbose=True) - killCmdCommand.run() + try: + killCmdCommand.run() + except CalledProcessError as ex: + # The kill is cleanup-only; the measurement data is already in the .logarchive above. + # devicectl returns non-zero when the app process is already gone (e.g. iOS terminated + # it, the app crashed, or it self-exited). Upload the .logarchive AND any device-side + # crash reports for this bundle to the Helix results container so we can diagnose + # why the app was already gone before re-raising. + getLogger().warning(f"App kill failed (app may have already exited): {ex}") + upload_root = os.environ.get('HELIX_WORKITEM_UPLOAD_ROOT') + if upload_root: + if os.path.exists(logarchive_filename): + archive_base = os.path.join(upload_root, f'iteration{i}.logarchive') + try: + getLogger().info(f"Saving {logarchive_filename} to {archive_base}.zip for diagnosis.") + make_archive(archive_base, 'zip', root_dir=logarchive_filename) + except Exception as upload_ex: + getLogger().warning(f"Failed to save logarchive for diagnosis: {upload_ex}") + # Take a final snapshot and download only crash reports that appeared + # since the initial snapshot taken before the iteration loop. This + # matches XHarness's CrashSnapshotReporter pattern and avoids uploading + # the historical backlog of unrelated crashes the shared device retains. + # iOS may take a few seconds to finish writing a crash report after the + # process dies, so poll the snapshot for up to 60s waiting for new + # entries to appear (matches CrashSnapshotReporter.EndCaptureAsync). + if initial_device_crashes is None: + getLogger().info("Skipping device crash log download (initial snapshot unavailable).") + else: + crash_wait_deadline = time.time() + 60 + final_device_crashes = listDeviceCrashReports() + new_crashes = sorted(final_device_crashes - initial_device_crashes) if final_device_crashes is not None else [] + while final_device_crashes is not None and not new_crashes and time.time() < crash_wait_deadline: + time.sleep(1) + final_device_crashes = listDeviceCrashReports() + new_crashes = sorted(final_device_crashes - initial_device_crashes) if final_device_crashes is not None else [] + if final_device_crashes is None: + getLogger().warning("Skipping device crash log download (final snapshot failed).") + elif not new_crashes: + getLogger().info("No new crash reports on device for this test run.") + else: + crash_dest = os.path.join(upload_root, f'iteration{i}_crashlogs') + os.makedirs(crash_dest, exist_ok=True) + getLogger().info(f"Downloading {len(new_crashes)} new crash report(s) to {crash_dest}.") + for crash_id in new_crashes: + dst = os.path.join(crash_dest, os.path.basename(crash_id)) + dlCmd = xharnesscommand() + [ + 'apple', 'mlaunch', '--', + f'--download-crash-report={crash_id}', + f'--download-crash-report-to={dst}', + '--devname', deviceUDID, + ] + try: + RunCommand(dlCmd, verbose=True).run() + except Exception as dlEx: + getLogger().warning(f"Failed to download crash report {crash_id}: {dlEx}") + raise # Process Data