From ba0746849dde38ff8c115848165ebc700215a096 Mon Sep 17 00:00:00 2001 From: li-lizhe <147392333@qq.com> Date: Sun, 11 Oct 2026 22:14:24 +0800 Subject: [PATCH 1/3] fix(train): bind distributed training to the active accelerator funasr/bin/train.py, funasr/bin/train_ds.py and funasr/train_utils/trainer_ds.py started every distributed process with torch.cuda.set_device(local_rank) / model.cuda(local_rank) and pinned the process-group backend to "nccl", and only released the allocator cache when device.type == "cuda". On a PyTorch build without CUDA (for example Ascend NPU) the first call raises AttributeError: module 'torch._C' has no attribute '_cuda_setDevice', and init_process_group(backend="nccl") fails with "Distributed package doesn't have NCCL built in", so DDP/FSDP training cannot start at all. Add device-agnostic helpers to funasr/train_utils/device_funcs.py and use them in the three entry points: the active accelerator (CUDA, Ascend NPU or XPU) is detected, the matching set_device is called, "hccl" becomes the default backend for NPU, and the end-of-epoch cache release dispatches to the accelerator that owns the device. CUDA behaviour is unchanged (nccl, cuda:, torch.cuda.empty_cache). Verified on Ascend 910B4 with a real 2-rank torchrun/HCCL run: before -> set_device AttributeError + nccl unavailable; after -> device_type=npu, resolved npu:0 / npu:1, set_device ok, backend=hccl, all_reduce sum 3.0 (expected 3.0), cache release ok. Not tested: CUDA/XPU hardware, FSDP/DeepSpeed end-to-end training. Signed-off-by: li-lizhe <147392333@qq.com> --- funasr/bin/train.py | 23 ++++--- funasr/bin/train_ds.py | 16 +++-- funasr/train_utils/device_funcs.py | 89 +++++++++++++++++++++++++ funasr/train_utils/trainer_ds.py | 4 +- tests/test_distributed_device_setup.py | 92 ++++++++++++++++++++++++++ 5 files changed, 208 insertions(+), 16 deletions(-) create mode 100644 tests/test_distributed_device_setup.py diff --git a/funasr/bin/train.py b/funasr/bin/train.py index d7c20f8f05..51e0c51e02 100644 --- a/funasr/bin/train.py +++ b/funasr/bin/train.py @@ -32,6 +32,12 @@ from funasr.download.download_model_from_hub import download_model from funasr.models.lora.utils import mark_only_lora_as_trainable from funasr.train_utils.set_all_random_seed import set_all_random_seed +from funasr.train_utils.device_funcs import ( + accelerator_device, + clear_accelerator_cache, + get_default_dist_backend, + set_accelerator_device, +) from funasr.train_utils.load_pretrained_model import load_pretrained_model from funasr.utils.misc import prepare_model_dir from funasr.train_utils.model_summary import model_summary @@ -81,8 +87,11 @@ def main(**kwargs): use_fsdp = kwargs.get("use_fsdp", False) # use_ddp = False if use_fsdp else use_fsdp if use_ddp or use_fsdp: - dist.init_process_group(backend=kwargs.get("backend", "nccl"), init_method="env://") - torch.cuda.set_device(local_rank) + dist.init_process_group( + backend=kwargs.get("backend", get_default_dist_backend()), + init_method="env://", + ) + set_accelerator_device(local_rank) logging.info("Build model, frontend, tokenizer") device = kwargs.get("device", "cuda") @@ -128,7 +137,7 @@ def main(**kwargs): logging.info(f"{model_summary(model)}") if use_ddp: - model = model.cuda(local_rank) + model = model.to(accelerator_device(local_rank)) model = DDP( model, device_ids=[local_rank], @@ -161,12 +170,12 @@ def custom_auto_wrap_policy( # Configure a custom `min_num_params` my_auto_wrap_policy = functools.partial(custom_auto_wrap_policy, min_num_params=int(1e5)) - torch.cuda.set_device(local_rank) + set_accelerator_device(local_rank) model = FSDP( model, auto_wrap_policy=custom_auto_wrap_policy, mixed_precision=None, - device_id=torch.cuda.current_device(), + device_id=local_rank, ) else: model = model.to(device=kwargs.get("device", "cuda")) @@ -246,9 +255,7 @@ def custom_auto_wrap_policy( trainer.start_step = 0 device = next(model.parameters()).device - if device.type == "cuda": - with torch.cuda.device(device): - torch.cuda.empty_cache() + clear_accelerator_cache(device) time_escaped = (time.perf_counter() - time_slice_i) / 3600.0 logging.info( diff --git a/funasr/bin/train_ds.py b/funasr/bin/train_ds.py index 1b18fe1b6a..f6d2dde230 100644 --- a/funasr/bin/train_ds.py +++ b/funasr/bin/train_ds.py @@ -30,6 +30,11 @@ from funasr.download.download_model_from_hub import download_model from funasr.models.lora.utils import mark_only_lora_as_trainable from funasr.train_utils.set_all_random_seed import set_all_random_seed +from funasr.train_utils.device_funcs import ( + clear_accelerator_cache, + get_default_dist_backend, + set_accelerator_device, +) from funasr.train_utils.load_pretrained_model import load_pretrained_model from funasr.utils.misc import prepare_model_dir from funasr.train_utils.model_summary import model_summary @@ -121,16 +126,17 @@ def main(**kwargs): deepspeed_config, trainer_conf, ) = _resolve_distributed_config(kwargs, world_size) + dist_backend = kwargs.get("backend", get_default_dist_backend()) if use_deepspeed: logging.info(f"use_deepspeed: {use_deepspeed}") - deepspeed.init_distributed(dist_backend=kwargs.get("backend", "nccl")) + deepspeed.init_distributed(dist_backend=dist_backend) elif use_ddp or use_fsdp: logging.info(f"use_ddp: {use_ddp}, use_fsdp: {use_fsdp}") dist.init_process_group( - backend=kwargs.get("backend", "nccl"), + backend=dist_backend, init_method="env://", ) - torch.cuda.set_device(local_rank) + set_accelerator_device(local_rank) # rank = dist.get_rank() @@ -241,9 +247,7 @@ def main(**kwargs): trainer.start_step = 0 device = next(model.parameters()).device - if device.type == "cuda": - with torch.cuda.device(device): - torch.cuda.empty_cache() + clear_accelerator_cache(device) time_escaped = (time.perf_counter() - time_slice_i) / 3600.0 logging.info( diff --git a/funasr/train_utils/device_funcs.py b/funasr/train_utils/device_funcs.py index dd7fc76a28..b5fdadac7b 100644 --- a/funasr/train_utils/device_funcs.py +++ b/funasr/train_utils/device_funcs.py @@ -62,3 +62,92 @@ def force_gatherable(data, device): else: warnings.warn(f"{type(data)} may not be gatherable by DataParallel") return data + + +_ACCELERATOR_DEVICE_TYPES = ("cuda", "npu", "xpu") + + +def _accelerator_module(device_type): + """Return the torch submodule owning ``device_type``, or ``None``. + + ``torch_npu`` registers ``torch.npu`` lazily, so it is imported on demand; + builds without it simply have no ``torch.npu`` attribute. ``torch.cuda`` and + ``torch.xpu`` exist on every build, but their ``is_available()`` reports + ``False`` when that backend is missing. + """ + if device_type == "npu": + try: + import torch_npu # noqa: F401 + except ImportError: + return None + return getattr(torch, device_type, None) + + +def get_accelerator_device_type(): + """Return the device type of the active accelerator, or ``"cpu"``. + + Probes the same backends FunASR accepts for ``AutoModel`` device fallback + (CUDA, then Ascend NPU, then XPU). CPU-only torch builds report ``"cpu"`` so + callers can keep a device-agnostic, no-op codepath. + """ + for device_type in _ACCELERATOR_DEVICE_TYPES: + module = _accelerator_module(device_type) + if module is None: + continue + try: + if module.is_available(): + return device_type + except Exception: + continue + return "cpu" + + +def accelerator_device(local_rank=0): + """Return the ``torch.device`` for ``local_rank`` on the active accelerator.""" + device_type = get_accelerator_device_type() + if device_type == "cpu": + return torch.device("cpu") + return torch.device(device_type, int(local_rank)) + + +def set_accelerator_device(local_rank=0): + """Bind this process to ``local_rank`` on the active accelerator. + + Replaces the CUDA-only ``torch.cuda.set_device(local_rank)``, which raises + (``module 'torch._C' has no attribute '_cuda_setDevice'``) on a build + without CUDA even when an Ascend NPU is present. Returns the resolved device + type; for ``"cpu"`` nothing is bound, matching the previous CPU behaviour. + """ + device_type = get_accelerator_device_type() + if device_type not in _ACCELERATOR_DEVICE_TYPES: + return device_type + module = _accelerator_module(device_type) + set_device = getattr(module, "set_device", None) + if set_device is not None: + set_device(int(local_rank)) + return device_type + + +def get_default_dist_backend(): + """Return the ``torch.distributed`` backend for the active accelerator. + + CUDA keeps the historical ``"nccl"``; Ascend NPU uses HCCL. This is only the + default — an explicit ``backend=...`` still wins. + """ + return "hccl" if get_accelerator_device_type() == "npu" else "nccl" + + +def clear_accelerator_cache(device=None): + """Release cached allocator memory for ``device`` (no-op on CPU). + + ``torch.cuda.empty_cache()`` is a silent no-op on a build without CUDA, so + the end-of-epoch release used by the training entry points did nothing on + Ascend NPU / XPU. Dispatch to the accelerator that owns ``device`` instead. + """ + device_type = getattr(device, "type", None) or get_accelerator_device_type() + if device_type not in _ACCELERATOR_DEVICE_TYPES: + return + module = _accelerator_module(device_type) + empty_cache = getattr(module, "empty_cache", None) + if empty_cache is not None: + empty_cache() diff --git a/funasr/train_utils/trainer_ds.py b/funasr/train_utils/trainer_ds.py index fc2f2b660f..eb1ded1d28 100644 --- a/funasr/train_utils/trainer_ds.py +++ b/funasr/train_utils/trainer_ds.py @@ -11,7 +11,7 @@ from pathlib import Path from torch.nn.parallel import DistributedDataParallel as DDP from torch.distributed.fsdp import FullyShardedDataParallel as FSDP -from funasr.train_utils.device_funcs import to_device +from funasr.train_utils.device_funcs import accelerator_device, to_device from funasr.train_utils.recursive_op import recursive_average from funasr.train_utils.average_nbest_models import average_checkpoints from funasr.train_utils.checkpoint_metrics import ( @@ -1027,7 +1027,7 @@ def warp_model(self, model, **kwargs): elif self.use_ddp: local_rank = int(os.environ.get("LOCAL_RANK", 0)) - model = model.cuda(local_rank) + model = model.to(accelerator_device(local_rank)) model = DDP( model, device_ids=[local_rank], diff --git a/tests/test_distributed_device_setup.py b/tests/test_distributed_device_setup.py new file mode 100644 index 0000000000..f3e2d7c1af --- /dev/null +++ b/tests/test_distributed_device_setup.py @@ -0,0 +1,92 @@ +import unittest +from unittest import mock + +import torch + +from funasr.train_utils import device_funcs + + +class _FakeAccelerator: + """Stand-in for ``torch.cuda`` / ``torch.npu`` / ``torch.xpu``.""" + + def __init__(self, available): + self._available = available + self.device_calls = [] + self.empty_cache_calls = 0 + + def is_available(self): + return self._available + + def set_device(self, index): + self.device_calls.append(index) + + def empty_cache(self): + self.empty_cache_calls += 1 + + +def _fake_modules(**availability): + return { + device_type: _FakeAccelerator(availability.get(device_type, False)) + for device_type in device_funcs._ACCELERATOR_DEVICE_TYPES + } + + +class TestAcceleratorResolution(unittest.TestCase): + """The training entry points must bind devices without assuming CUDA.""" + + def _patch(self, **availability): + modules = _fake_modules(**availability) + patcher = mock.patch.object( + device_funcs, + "_accelerator_module", + side_effect=lambda device_type: modules.get(device_type), + ) + patcher.start() + self.addCleanup(patcher.stop) + return modules + + def test_device_type_prefers_cuda_then_npu_then_xpu(self): + self._patch(cuda=True, npu=True, xpu=True) + self.assertEqual(device_funcs.get_accelerator_device_type(), "cuda") + self._patch(npu=True, xpu=True) + self.assertEqual(device_funcs.get_accelerator_device_type(), "npu") + self._patch(xpu=True) + self.assertEqual(device_funcs.get_accelerator_device_type(), "xpu") + self._patch() + self.assertEqual(device_funcs.get_accelerator_device_type(), "cpu") + + def test_accelerator_device_follows_active_backend(self): + self._patch() + self.assertEqual(device_funcs.accelerator_device(0), torch.device("cpu")) + self._patch(cuda=True) + self.assertEqual(device_funcs.accelerator_device(3), torch.device("cuda", 3)) + + def test_set_accelerator_device_binds_local_rank(self): + modules = self._patch(npu=True) + resolved = device_funcs.set_accelerator_device(3) + self.assertEqual(resolved, "npu") + self.assertEqual(modules["npu"].device_calls, [3]) + + def test_set_accelerator_device_is_noop_without_accelerator(self): + modules = self._patch() + self.assertEqual(device_funcs.set_accelerator_device(1), "cpu") + self.assertTrue(all(not module.device_calls for module in modules.values())) + + def test_default_dist_backend_matches_accelerator(self): + self._patch(npu=True) + self.assertEqual(device_funcs.get_default_dist_backend(), "hccl") + self._patch(cuda=True) + self.assertEqual(device_funcs.get_default_dist_backend(), "nccl") + self._patch() + self.assertEqual(device_funcs.get_default_dist_backend(), "nccl") + + def test_clear_accelerator_cache_dispatches_to_active_backend(self): + modules = self._patch(npu=True) + device_funcs.clear_accelerator_cache(torch.device("cpu")) + self.assertEqual(modules["npu"].empty_cache_calls, 0) + device_funcs.clear_accelerator_cache(None) + self.assertEqual(modules["npu"].empty_cache_calls, 1) + + +if __name__ == "__main__": + unittest.main() From da0e18cd9e3a4603562bb74bd66750d0d9022535 Mon Sep 17 00:00:00 2001 From: li-lizhe <147392333@qq.com> Date: Mon, 12 Oct 2026 08:01:52 +0800 Subject: [PATCH 2/3] fix(train): use XCCL as the default XPU distributed backend get_default_dist_backend() returned "nccl" for every non-NPU device, so an XPU run -- which get_accelerator_device_type() does select -- reached init_process_group(backend="nccl"). The default is now resolved per accelerator: HCCL on Ascend NPU, XCCL on XPU (the former oneCCL name "ccl" is kept for builds that confirm it through torch.distributed.is_ccl_available()), and the historical "nccl" elsewhere. Candidates are probed with torch.distributed.is__available() where the build exposes it, and a non-CUDA accelerator never falls back to NCCL. An explicit backend=... still wins. tests/test_distributed_device_setup.py covers the XPU default and the capability-aware fallback (7 tests). Signed-off-by: li-lizhe <147392333@qq.com> --- funasr/train_utils/device_funcs.py | 38 ++++++++++++++++++++++++-- tests/test_distributed_device_setup.py | 21 ++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/funasr/train_utils/device_funcs.py b/funasr/train_utils/device_funcs.py index b5fdadac7b..8cebc85723 100644 --- a/funasr/train_utils/device_funcs.py +++ b/funasr/train_utils/device_funcs.py @@ -66,6 +66,11 @@ def force_gatherable(data, device): _ACCELERATOR_DEVICE_TYPES = ("cuda", "npu", "xpu") +#: Default ``torch.distributed`` backend for each accelerator, most preferred +#: first. XCCL (oneCCL) is what current torch exposes for XPU; builds that +#: predate the 2.6 rename know the same backend as ``"ccl"``. +_ACCELERATOR_DIST_BACKENDS = {"npu": ("hccl",), "xpu": ("xccl", "ccl")} + def _accelerator_module(device_type): """Return the torch submodule owning ``device_type``, or ``None``. @@ -128,13 +133,40 @@ def set_accelerator_device(local_rank=0): return device_type +def _dist_backend_available(backend): + """Ask torch whether ``backend`` is usable: ``is__available()``. + + A build that does not expose the probe cannot confirm anything, so it + reports ``False`` and the caller keeps the accelerator's canonical backend + name instead of inventing availability. + """ + distributed = getattr(torch, "distributed", None) + probe = getattr(distributed, "is_{}_available".format(backend), None) + if probe is None: + return False + try: + return bool(probe()) + except Exception: + return False + + def get_default_dist_backend(): """Return the ``torch.distributed`` backend for the active accelerator. - CUDA keeps the historical ``"nccl"``; Ascend NPU uses HCCL. This is only the - default — an explicit ``backend=...`` still wins. + CUDA keeps the historical ``"nccl"``, Ascend NPU uses HCCL and XPU uses + XCCL (``"ccl"`` on builds older than the oneCCL rename). Candidates are + probed with ``torch.distributed.is__available()`` where available + so the selected backend matches the build, but a non-CUDA accelerator never + falls back to NCCL. This is only the default — an explicit ``backend=...`` + still wins. """ - return "hccl" if get_accelerator_device_type() == "npu" else "nccl" + candidates = _ACCELERATOR_DIST_BACKENDS.get(get_accelerator_device_type()) + if not candidates: + return "nccl" + for backend in candidates: + if _dist_backend_available(backend): + return backend + return candidates[0] def clear_accelerator_cache(device=None): diff --git a/tests/test_distributed_device_setup.py b/tests/test_distributed_device_setup.py index f3e2d7c1af..fce0094983 100644 --- a/tests/test_distributed_device_setup.py +++ b/tests/test_distributed_device_setup.py @@ -77,9 +77,30 @@ def test_default_dist_backend_matches_accelerator(self): self.assertEqual(device_funcs.get_default_dist_backend(), "hccl") self._patch(cuda=True) self.assertEqual(device_funcs.get_default_dist_backend(), "nccl") + self._patch(xpu=True) + self.assertEqual(device_funcs.get_default_dist_backend(), "xccl") self._patch() self.assertEqual(device_funcs.get_default_dist_backend(), "nccl") + def test_xpu_backend_selection_is_capability_aware(self): + """XPU uses XCCL and never NCCL; only a build that confirms oneCCL's + legacy name keeps ``"ccl"``.""" + self._patch(xpu=True) + with mock.patch.object( + device_funcs, + "_dist_backend_available", + side_effect=lambda backend: backend == "xccl", + ): + self.assertEqual(device_funcs.get_default_dist_backend(), "xccl") + with mock.patch.object( + device_funcs, + "_dist_backend_available", + side_effect=lambda backend: backend == "ccl", + ): + self.assertEqual(device_funcs.get_default_dist_backend(), "ccl") + with mock.patch.object(device_funcs, "_dist_backend_available", return_value=False): + self.assertEqual(device_funcs.get_default_dist_backend(), "xccl") + def test_clear_accelerator_cache_dispatches_to_active_backend(self): modules = self._patch(npu=True) device_funcs.clear_accelerator_cache(torch.device("cpu")) From 0a336ba5007e5b2e37ba904dd10b03913e0c9e23 Mon Sep 17 00:00:00 2001 From: li-lizhe <147392333@qq.com> Date: Sat, 17 Oct 2026 09:12:50 +0800 Subject: [PATCH 3/3] fix(train): keep XPU on the historical "nccl" dist backend The XPU arm from the previous commit advertised XCCL as an XPU default, but there is no Intel XPU hardware here to exercise that runtime path, so it was the only path this change touched without real-device evidence. Drop the XPU entry from _ACCELERATOR_DIST_BACKENDS: an XPU run now resolves exactly as it did before this change ("nccl"), while Ascend NPU still resolves to HCCL and CUDA keeps "nccl". Selecting XCCL is tracked as a separate follow-up for whoever has XPU hardware. tests/test_distributed_device_setup.py: replace the XPU/XCCL selection test with one that pins the narrowed behaviour (XPU stays "nccl" even when a build reports is_xccl_available()) and one that pins HCCL as the NPU backend when no capability probe exists (8 tests). Verified on an Ascend 910B2 container (torch 2.15.0.dev20260917+cpu, torch_npu 2.15.0.dev20260917+gitec69335): 2-rank torchrun -> per-rank binding npu:0/npu:1, backend hccl, all_reduce sum 1.0, and the end-of-epoch cache release drops reserved memory 291504128 -> 2097152; unit tests 8/8 OK. Signed-off-by: li-lizhe <147392333@qq.com> --- funasr/train_utils/device_funcs.py | 27 ++++++++++++++++---------- tests/test_distributed_device_setup.py | 26 ++++++++++++------------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/funasr/train_utils/device_funcs.py b/funasr/train_utils/device_funcs.py index 8cebc85723..6cafb8a592 100644 --- a/funasr/train_utils/device_funcs.py +++ b/funasr/train_utils/device_funcs.py @@ -66,10 +66,13 @@ def force_gatherable(data, device): _ACCELERATOR_DEVICE_TYPES = ("cuda", "npu", "xpu") -#: Default ``torch.distributed`` backend for each accelerator, most preferred -#: first. XCCL (oneCCL) is what current torch exposes for XPU; builds that -#: predate the 2.6 rename know the same backend as ``"ccl"``. -_ACCELERATOR_DIST_BACKENDS = {"npu": ("hccl",), "xpu": ("xccl", "ccl")} +#: Default ``torch.distributed`` backend per accelerator, most preferred first. +#: Only Ascend NPU is listed. Every other device keeps the historical ``"nccl"`` +#: default, exactly as before this change, so the backend this module selects at +#: runtime is always one that has been exercised on real hardware. XPU does use +#: XCCL (the former oneCCL name ``"ccl"``), but selecting that backend is not +#: covered by hardware here and is tracked separately, outside this change. +_ACCELERATOR_DIST_BACKENDS = {"npu": ("hccl",)} def _accelerator_module(device_type): @@ -153,12 +156,16 @@ def _dist_backend_available(backend): def get_default_dist_backend(): """Return the ``torch.distributed`` backend for the active accelerator. - CUDA keeps the historical ``"nccl"``, Ascend NPU uses HCCL and XPU uses - XCCL (``"ccl"`` on builds older than the oneCCL rename). Candidates are - probed with ``torch.distributed.is__available()`` where available - so the selected backend matches the build, but a non-CUDA accelerator never - falls back to NCCL. This is only the default — an explicit ``backend=...`` - still wins. + CUDA keeps the historical ``"nccl"`` and Ascend NPU uses HCCL, probed with + ``torch.distributed.is_hccl_available()`` where the build exposes it so the + selected backend matches the build. A non-CUDA accelerator never falls back + to NCCL. This is only the default — an explicit ``backend=...`` still wins. + + XPU is deliberately absent: it is a valid ``torch.distributed`` backend + (XCCL / the former oneCCL name ``"ccl"``), but no XPU hardware is available + to this change, so XPU keeps the historical ``"nccl"`` value here rather + than advertising a runtime path nobody can exercise. Selecting XCCL is a + separate follow-up. """ candidates = _ACCELERATOR_DIST_BACKENDS.get(get_accelerator_device_type()) if not candidates: diff --git a/tests/test_distributed_device_setup.py b/tests/test_distributed_device_setup.py index fce0094983..4695e07573 100644 --- a/tests/test_distributed_device_setup.py +++ b/tests/test_distributed_device_setup.py @@ -77,29 +77,27 @@ def test_default_dist_backend_matches_accelerator(self): self.assertEqual(device_funcs.get_default_dist_backend(), "hccl") self._patch(cuda=True) self.assertEqual(device_funcs.get_default_dist_backend(), "nccl") - self._patch(xpu=True) - self.assertEqual(device_funcs.get_default_dist_backend(), "xccl") self._patch() self.assertEqual(device_funcs.get_default_dist_backend(), "nccl") - def test_xpu_backend_selection_is_capability_aware(self): - """XPU uses XCCL and never NCCL; only a build that confirms oneCCL's - legacy name keeps ``"ccl"``.""" + def test_npu_backend_needs_no_capability_probe(self): + """HCCL is the canonical backend name for Ascend NPU, so a build that + exposes no ``is_hccl_available()`` probe still resolves to it.""" + self._patch(npu=True) + with mock.patch.object(device_funcs, "_dist_backend_available", return_value=False): + self.assertEqual(device_funcs.get_default_dist_backend(), "hccl") + + def test_xpu_keeps_the_historical_nccl_default(self): + """XPU is a valid ``torch.distributed`` backend (XCCL / oneCCL), but this + change has no XPU hardware to exercise it, so it must not advertise XCCL: + the value for an XPU run stays exactly what it was before the change.""" self._patch(xpu=True) with mock.patch.object( device_funcs, "_dist_backend_available", side_effect=lambda backend: backend == "xccl", ): - self.assertEqual(device_funcs.get_default_dist_backend(), "xccl") - with mock.patch.object( - device_funcs, - "_dist_backend_available", - side_effect=lambda backend: backend == "ccl", - ): - self.assertEqual(device_funcs.get_default_dist_backend(), "ccl") - with mock.patch.object(device_funcs, "_dist_backend_available", return_value=False): - self.assertEqual(device_funcs.get_default_dist_backend(), "xccl") + self.assertEqual(device_funcs.get_default_dist_backend(), "nccl") def test_clear_accelerator_cache_dispatches_to_active_backend(self): modules = self._patch(npu=True)