diff --git a/backends/arm/test/conftest.py b/backends/arm/test/conftest.py index 55a7451c128..5e9b3268427 100644 --- a/backends/arm/test/conftest.py +++ b/backends/arm/test/conftest.py @@ -26,7 +26,13 @@ def pytest_configure(config): pytest._test_options["llama_inputs"] = config.option.llama_inputs # type: ignore[attr-defined] logging.basicConfig(stream=sys.stdout) - _set_random_seed() + seed, seed_label = _setup_random_seed() + config._test_seed = seed + config._test_seed_label = seed_label + + +def pytest_report_header(config): + return config._test_seed_label def pytest_collection_modifyitems(config, items): @@ -64,42 +70,45 @@ def pytest_sessionfinish(session, exitstatus): @pytest.fixture(autouse=True) -def set_random_seed(): +def set_random_seed(request): """Control random numbers in Arm test suite. Default behavior is to use a fixed seed (0), which ensures reproducible tests. Use the env variable - ARM_TEST_SEED to set a custom seed, or set it to RANDOM for random seed + TEST_SEED to set a custom seed, or set it to RANDOM for random seed behavior. Examples: As default use fixed seed (0) for reproducible tests pytest --config-file=/dev/null --verbose -s --color=yes backends/arm/test/ops/test_avg_pool.py -k Use a random seed for each test - ARM_TEST_SEED=RANDOM pytest --config-file=/dev/null --verbose -s --color=yes backends/arm/test/ops/test_avg_pool.py -k + TEST_SEED=RANDOM pytest --config-file=/dev/null --verbose -s --color=yes backends/arm/test/ops/test_avg_pool.py -k Rerun with a specific seed - ARM_TEST_SEED=3478246 pytest --config-file=/dev/null --verbose -s --color=yes backends/arm/test/ops/test_avg_pool.py -k + TEST_SEED=3478246 pytest --config-file=/dev/null --verbose -s --color=yes backends/arm/test/ops/test_avg_pool.py -k """ - _set_random_seed() + _set_random_seed(request.config._test_seed) -def _set_random_seed(): - import torch - - seed_env = os.environ.get("ARM_TEST_SEED", "0") +def _setup_random_seed(): + seed_env = os.environ.get("TEST_SEED", "0") if seed_env == "RANDOM": random.seed() # reset seed, in case any other test has fiddled with it seed = random.randint(0, 2**32 - 1) # nosec B311 - non-crypto seed for tests - torch.manual_seed(seed) - print(f" ARM_TEST_SEED=RANDOM using:{seed} ", end=" ") + seed_label = f"TEST_SEED=RANDOM using:{seed}" elif str.isdigit(seed_env): seed = int(seed_env) - random.seed(seed) - torch.manual_seed(seed) - print(f" ARM_TEST_SEED={seed} ", end=" ") + seed_label = f"TEST_SEED={seed}" else: - raise TypeError( - "ARM_TEST_SEED env variable must be integers or the string RANDOM" - ) + raise TypeError("TEST_SEED env variable must be integers or the string RANDOM") + + _set_random_seed(seed) + return seed, seed_label + + +def _set_random_seed(seed): + import torch + + random.seed(seed) + torch.manual_seed(seed) # ==== End of Pytest fixtures ===== diff --git a/backends/cortex_m/test/conftest.py b/backends/cortex_m/test/conftest.py index bf39d295c26..6bcb0a8dae4 100644 --- a/backends/cortex_m/test/conftest.py +++ b/backends/cortex_m/test/conftest.py @@ -1,12 +1,17 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import os +import random + import pytest from executorch.backends.cortex_m.target_config import CortexMTargetConfig + _DEFAULT_TARGET = "cortex-m55" @@ -29,7 +34,7 @@ def _selected_targets(config) -> list[str]: def pytest_report_header(config): - return f"cortex-m op-test targets: {', '.join(_selected_targets(config))}" + return f"cortex-m op-test targets: {', '.join(_selected_targets(config))} {config._test_seed_label}" def pytest_generate_tests(metafunc): @@ -46,3 +51,43 @@ def cortex_m_target(request) -> CortexMTargetConfig: the AoT target config (and, for implementation tests, the matching prebuilt FVP runner).""" return CortexMTargetConfig.from_target_string(request.param) + + +def pytest_configure(config): + seed, seed_label = _setup_random_seed() + config._test_seed = seed + config._test_seed_label = seed_label + + +@pytest.fixture(autouse=True) +def set_random_seed(request): + """Control random numbers in the Cortex-M test suite. + + By default this uses a fixed seed (0) for reproducible tests. Use + TEST_SEED to set a custom seed, or set it to RANDOM for random seed + behavior. + """ + _set_random_seed(request.config._test_seed) + + +def _setup_random_seed(): + seed_env = os.environ.get("TEST_SEED", "0") + if seed_env == "RANDOM": + random.seed() # reset seed, in case any other test has fiddled with it + seed = random.randint(0, 2**32 - 1) # nosec B311 - non-crypto seed for tests + seed_label = f"TEST_SEED=RANDOM using:{seed}" + elif str.isdigit(seed_env): + seed = int(seed_env) + seed_label = f"TEST_SEED={seed}" + else: + raise TypeError("TEST_SEED env variable must be integers or the string RANDOM") + + _set_random_seed(seed) + return seed, seed_label + + +def _set_random_seed(seed): + import torch + + random.seed(seed) + torch.manual_seed(seed)