diff --git a/.github/workflows/cutile-rs.yml b/.github/workflows/cutile-rs.yml index 4329f2b827..ec856b5785 100644 --- a/.github/workflows/cutile-rs.yml +++ b/.github/workflows/cutile-rs.yml @@ -289,7 +289,7 @@ jobs: - name: Native f16 DeviceCopy working-directory: ${{ github.workspace }} - run: cargo test -p cuda-core --features f16 --test simt_device_copy_impls + run: cargo test -p cuda-core --features f16 --test device_copy_impls - name: loom model check (interleavings + weak memory) working-directory: ${{ github.workspace }} diff --git a/cuda-async/README.md b/cuda-async/README.md index d9cc2c6dc5..47581cbc6e 100644 --- a/cuda-async/README.md +++ b/cuda-async/README.md @@ -17,7 +17,7 @@ until `.sync()`, `.sync_on()`, or `.await` is called. use cutile::prelude::*; fn main() -> Result<(), DeviceError> { - let device = cuda_core::Device::new(0)?; + let device = cuda_core::Context::new(0)?; let stream = device.new_stream()?; let mut z = api::zeros::(&[16, 16]).sync_on(&stream)?; diff --git a/cuda-async/src/cuda_graph.rs b/cuda-async/src/cuda_graph.rs index fc163050dc..582c35076a 100644 --- a/cuda-async/src/cuda_graph.rs +++ b/cuda-async/src/cuda_graph.rs @@ -7,7 +7,7 @@ use crate::device_context::with_default_device_policy; use crate::device_future::DeviceFuture; use crate::device_operation::{DeviceOp, ExecutionContext, GraphNode, ReplayResource}; use crate::error::DeviceError; -use cuda_core::{sys, Device, IntoResult, Stream}; +use cuda_core::{sys, Context, IntoResult, Stream}; use std::collections::HashMap; use std::future::IntoFuture; use std::mem::MaybeUninit; @@ -67,7 +67,7 @@ struct GraphExecHandle { /// Contexts of `update` ops not yet consumed by a replay. pending: Mutex>, /// Bound before destruction: the driver needs the owning context current. - device: Arc, + device: Arc, cu_graph: sys::CUgraph, cu_graph_exec: sys::CUgraphExec, } @@ -85,7 +85,7 @@ impl GraphExecHandle { /// instantiation fails, and owned by the returned handle otherwise /// (whose `Drop` also covers an upload failure). fn instantiate( - device: Arc, + device: Arc, cu_graph: sys::CUgraph, stream: &Stream, ) -> Result { @@ -165,7 +165,7 @@ fn capture_on( stream: &Arc, record: impl FnOnce() -> Result, ) -> Result<(R, Arc), DeviceError> { - let device = stream.device().clone(); + let device = stream.context().clone(); device.bind_to_thread()?; unsafe { @@ -204,7 +204,7 @@ fn capture_on( let exec = GraphExecHandle::instantiate(device, cu_graph, stream)?; // Drain the upload. The recorded work itself has *not* run. - unsafe { stream.synchronize() }?; + stream.synchronize()?; Ok((output, Arc::new(exec))) } diff --git a/cuda-async/src/device_buffer.rs b/cuda-async/src/device_buffer.rs index a6da6f5a75..4238a1f3ad 100644 --- a/cuda-async/src/device_buffer.rs +++ b/cuda-async/src/device_buffer.rs @@ -3,61 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -//! Wrappers around CUDA device pointers: owned (freed on drop), foreign (kept -//! alive by an external owner via a liveness token), and raw-borrowed (an -//! `unsafe` escape that does nothing on drop). +//! Shared allocation ownership and typed device addresses. -use crate::device_context::with_deallocator_stream; -use cuda_core::free_async; use cuda_core::sys::CUdeviceptr; +pub use cuda_core::{DeviceAllocation, RawDeviceBuffer as DeviceBuffer}; use std::marker::PhantomData; -use std::sync::Arc; - -/// A CUDA device allocation owned by an external framework (cudarc, torch, a VMM -/// range, ...). -/// -/// A borrowed [`DeviceBuffer`] can hold an `Arc` as a -/// *liveness token*: while the buffer — and anything derived from it — is alive, -/// the owner's refcount is nonzero, so the memory cannot be freed out from under -/// cutile. This turns liveness from an ongoing caller promise into a fact -/// enforced at construction. -/// -/// # Safety -/// Implementors must guarantee that, for as long as the implementor is alive: -/// - `device_ptr()` returns a valid CUDA device pointer on `device_id()`, -/// - the allocation there is at least `len_bytes()` bytes, and -/// - all three getters are **stable**: they return the same values on every -/// call for the implementor's entire lifetime. Consumers read them once at -/// construction and cache the results (`DeviceBuffer::foreign`), so an -/// implementor whose pointer moves would silently diverge from the cached -/// copy. This is the same stability `std`'s allocator traits demand of a -/// currently-allocated block. -/// -/// This positional obligation is discharged **once, in the `impl`** — not at -/// each borrow. The temporal obligation (nobody else touches the bytes while -/// cutile work is in flight) cannot live here: it is per-borrow, and is why -/// `Tensor::from_foreign` remains an `unsafe` call. -pub unsafe trait DeviceAllocation: Send + Sync + 'static { - /// The device pointer of the allocation. - fn device_ptr(&self) -> CUdeviceptr; - /// The size of the allocation in bytes. - fn len_bytes(&self) -> usize; - /// The device ordinal the allocation lives on. - fn device_id(&self) -> usize; -} - -/// How a [`DeviceBuffer`]'s memory is owned. -enum Owner { - /// cutile allocated the memory; free it asynchronously on drop. - Owned, - /// A raw, unowned borrow (via `borrowed_from_raw_parts`); do nothing on drop. - /// Liveness is the caller's unverified obligation. - Borrowed, - /// A foreign allocation kept alive by its owner; do nothing on drop, and - /// hold the owner so the memory provably outlives this buffer. The `Arc` is - /// held purely for that RAII liveness effect — never read directly. - Foreign(#[allow(dead_code)] Arc), -} /// A non-owning, copyable handle to a typed device memory address. #[derive(Debug, Copy, Clone)] @@ -85,141 +35,3 @@ impl DevicePointer { } } } - -/// A type-erased handle to a CUDA device memory allocation. -/// -/// `DeviceBuffer` manages a raw byte buffer on the GPU. It stores only the device pointer, -/// byte length, and device id — all type information lives in higher-level wrappers -/// such as `Tensor`. -/// -/// An *owned* buffer (the default) is freed asynchronously on drop. A *foreign* -/// buffer (see [`foreign`](Self::foreign)) wraps memory owned by an external -/// allocator (cudarc, torch, a VMM range, ...), holds a liveness token, and is a -/// no-op on drop. A raw *borrowed* buffer -/// (see [`borrowed_from_raw_parts`](Self::borrowed_from_raw_parts)) is the same -/// no-op-on-drop but without a token — an `unsafe` escape whose liveness is the -/// caller's responsibility. -pub struct DeviceBuffer { - device_id: usize, - cudptr: CUdeviceptr, - len: usize, - owner: Owner, -} - -impl std::fmt::Debug for DeviceBuffer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let kind = match self.owner { - Owner::Owned => "owned", - Owner::Borrowed => "borrowed", - Owner::Foreign(_) => "foreign", - }; - f.debug_struct("DeviceBuffer") - .field("device_id", &self.device_id) - .field("cudptr", &self.cudptr) - .field("len", &self.len) - .field("owner", &kind) - .finish() - } -} - -unsafe impl Send for DeviceBuffer {} -unsafe impl Sync for DeviceBuffer {} - -impl Drop for DeviceBuffer { - fn drop(&mut self) { - // Only owned buffers are freed here. Borrowed/foreign buffers wrap memory - // owned elsewhere; freeing it would be a double free (or invalid, e.g. - // for a VMM-mapped range). A `Foreign` owner's `Arc` drops with `self`. - if !matches!(self.owner, Owner::Owned) { - return; - } - // A destructor has no way to report failure, and a panic here would - // abort the process if this drop runs during an unwind. Both a missing - // device context and a refused `cuMemFreeAsync` are therefore ignored: - // the allocation leaks until the primary context is released, which - // is the least bad outcome for memory the driver would not free. - // SAFETY: the pointer was allocated stream-ordered by this crate, and - // the driver completes queued work on the deallocator stream before - // the memory is handed out again. - let _ = unsafe { - with_deallocator_stream(self.device_id, |stream| free_async(self.cudptr, stream)) - }; - } -} - -impl DeviceBuffer { - /// Constructs an owned `DeviceBuffer` from a raw device pointer, byte length, and device id. - /// - /// The buffer takes ownership of the allocation and frees it asynchronously on drop. - /// - /// # Safety - /// The caller must ensure `dptr` points to a valid device allocation of at least - /// `len_bytes` bytes that is safe to free via the deallocator stream. - pub unsafe fn from_raw_parts(dptr: CUdeviceptr, len_bytes: usize, device_id: usize) -> Self { - Self { - cudptr: dptr, - len: len_bytes, - device_id, - owner: Owner::Owned, - } - } - - /// Wraps `len_bytes` of a foreign allocation, holding `owner` alive so the - /// memory outlives this buffer. Dropping it does **not** free the memory. - /// - /// `len_bytes` is the borrowing view's **logical** size (what the tensor - /// invariant checks), not the owner's allocation size — the owner may be - /// larger, and the addressable-extent-vs-owner check happens in - /// `Tensor::from_foreign` before this is called. Liveness is enforced by - /// `owner`'s refcount; pointer validity was asserted once when `owner`'s - /// type implemented [`DeviceAllocation`]. - pub fn foreign(owner: Arc, len_bytes: usize) -> Self { - Self { - cudptr: owner.device_ptr(), - len: len_bytes, - device_id: owner.device_id(), - owner: Owner::Foreign(owner), - } - } - - /// Constructs a borrowed `DeviceBuffer` that wraps memory owned by an external - /// allocator. Dropping it does **not** free the allocation. - /// - /// # Safety - /// This borrows, rather than takes ownership of, `dptr`; dropping the buffer - /// never frees it. The caller must ensure `dptr` points to a valid device - /// allocation of at least `len_bytes` bytes, and must keep that allocation - /// mapped — never freed, reallocated, or resized — for the entire lifetime of - /// this buffer and every value derived from it. When wrapped in a tensor used - /// as a mutable output, the owner must additionally not access the same memory - /// concurrently; see `Tensor::borrow_raw_parts` in the `cutile` crate for the - /// full obligations. - pub unsafe fn borrowed_from_raw_parts( - dptr: CUdeviceptr, - len_bytes: usize, - device_id: usize, - ) -> Self { - Self { - cudptr: dptr, - len: len_bytes, - device_id, - owner: Owner::Borrowed, - } - } - /// Returns whether the allocation is empty (zero bytes). - pub fn is_empty(&self) -> bool { - self.len_bytes() == 0 - } - /// Returns the byte length of the allocation. - pub fn len_bytes(&self) -> usize { - self.len - } - /// Returns the raw CUDA device pointer. - pub fn cu_deviceptr(&self) -> CUdeviceptr { - self.cudptr - } - /// Returns the device id this allocation belongs to. - pub fn device_id(&self) -> usize { - self.device_id - } -} diff --git a/cuda-async/src/device_context.rs b/cuda-async/src/device_context.rs index a119beae28..e9846e9359 100644 --- a/cuda-async/src/device_context.rs +++ b/cuda-async/src/device_context.rs @@ -7,7 +7,7 @@ //! //! ## Architecture //! -//! - **Global (process-wide)**: one [`Device`] per device id, shared across all threads +//! - **Global (process-wide)**: one [`Context`] per device id, shared across all threads //! via [`OnceLock`]. //! //! - **Per-thread**: Scheduling policy and deallocator stream remain thread-local, since @@ -18,7 +18,7 @@ use crate::error::{device_assert, device_error, DeviceError}; use crate::scheduling_policies::{SchedulingPolicy, StreamPoolRoundRobin}; -use cuda_core::{Device, Function, MemPool, Module, Stream}; +use cuda_core::{Context, Function, MemPool, Module, Stream}; use std::cell::Cell; use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; @@ -89,28 +89,28 @@ pub struct Validator { pub launch_checks: Vec, } -// ── Global Device (process-wide, per-device singleton) ───────────────────── +// ── Global Context (process-wide, per-device singleton) ───────────────────── /// Global per-device handles. Shared across all threads so that /// `Module`/`Function` loaded against a device can be used from any thread. -static DEVICES: OnceLock>>> = OnceLock::new(); +static DEVICES: OnceLock>>> = OnceLock::new(); -fn devices() -> &'static Mutex>> { +fn devices() -> &'static Mutex>> { DEVICES.get_or_init(|| Mutex::new(HashMap::new())) } -/// Get or create the global [`Device`] for a device ordinal. +/// Get or create the global [`Context`] for a device ordinal. /// /// The first call for a given `device_id` creates the device handle; subsequent -/// calls return the same `Arc`. -fn get_or_init_device(device_id: usize) -> Result, DeviceError> { +/// calls return the same `Arc`. +fn get_or_init_device(device_id: usize) -> Result, DeviceError> { let mut devices = devices() .lock() .map_err(|_| device_error(device_id, "device map lock poisoned"))?; if let Some(device) = devices.get(&device_id) { return Ok(Arc::clone(device)); } - let device = Device::new(device_id)?; + let device = Context::new(device_id)?; devices.insert(device_id, Arc::clone(&device)); Ok(device) } @@ -243,7 +243,7 @@ pub fn init_device( ) -> Result<(), DeviceError> { let device_context = new_device_context(device_id, policy)?; let pred = hashmap.insert(device_id, device_context).is_none(); - device_assert(device_id, pred, "Device is already initialized.") + device_assert(device_id, pred, "Context is already initialized.") } pub fn init_with_default_policy( @@ -260,7 +260,7 @@ pub fn init_with_default_policy( pool: None, }; let pred = hashmap.insert(device_id, device_context).is_none(); - device_assert(device_id, pred, "Device is already initialized.") + device_assert(device_id, pred, "Context is already initialized.") } /// Holds the thread's context map while a closure runs against it, and puts it @@ -371,10 +371,10 @@ where }) } -/// Run a closure with a reference to the [`Device`] for `device_id`. +/// Run a closure with a reference to the [`Context`] for `device_id`. pub fn with_device(device_id: usize, f: F) -> Result where - F: FnOnce(&Arc) -> R, + F: FnOnce(&Arc) -> R, { let device = get_or_init_device(device_id)?; Ok(f(&device)) @@ -429,7 +429,7 @@ pub fn set_default_device(default_device_id: usize) { /// Returns [`DeviceError::Context`](crate::error::DeviceError::Context) if /// `pool` was created on a different device than `device_id`. pub fn set_device_pool(device_id: usize, pool: Arc) -> Result<(), DeviceError> { - let pool_device = pool.device().ordinal(); + let pool_device = pool.context().ordinal(); device_assert( device_id, pool_device == device_id, @@ -467,7 +467,10 @@ pub fn get_device_pool(device_id: usize) -> Result>, DeviceE /// Errors from the device-context lookup are downgraded to `None`; this is the /// single choke-point for that decision so callers don't each re-derive it. pub fn pool_for_stream(stream: &Arc) -> Option> { - get_device_pool(stream.device().ordinal()).ok().flatten() + get_device_pool(stream.context().ordinal()) + .ok() + .flatten() + .filter(|pool| pool.context().as_ref() == stream.context().as_ref()) } /// Run a closure with the scheduling policy of the current thread's default device. @@ -495,12 +498,12 @@ pub fn load_module_from_file(filename: &str, device_id: usize) -> Result StreamHealth { return StreamHealth::Capturing; } // SAFETY: see above; not capturing, so a query is permitted. - match unsafe { stream.query() } { + match stream.query() { Ok(true) => StreamHealth::Idle, Ok(false) => StreamHealth::Busy, Err(e) => StreamHealth::Faulted(e), @@ -334,11 +334,11 @@ impl> DeviceFuture { })?; // The drop may run on an executor thread that never touched // CUDA; the query and synchronize below need a current context. - stream.device().bind_to_thread()?; + stream.context().bind_to_thread()?; match probe_stream(&stream) { StreamHealth::Idle => Ok(()), // SAFETY: the context was bound above; the stream is valid. - StreamHealth::Busy => unsafe { stream.synchronize() }.map_err(DeviceError::Driver), + StreamHealth::Busy => stream.synchronize().map_err(DeviceError::Driver), StreamHealth::Capturing => Err(DeviceError::Internal( "the future's stream is recording a graph; it cannot be synchronized".into(), )), @@ -470,7 +470,7 @@ impl> Future for DeviceFuture { let deadline = std::time::Instant::now() + std::time::Duration::from_micros(inline_spin_budget_us()); loop { - match unsafe { ctx.get_cuda_stream().query() } { + match ctx.get_cuda_stream().query() { Ok(true) => break 'spin Ok(true), Ok(false) => {} Err(e) => break 'spin Err(e), @@ -526,7 +526,7 @@ impl> Future for DeviceFuture { Some(ctx) => { // The poll may run on an executor thread that never // touched CUDA; the probe needs a current context. - let _ = ctx.device().bind_to_thread(); + let _ = ctx.context().bind_to_thread(); probe_stream(ctx.get_cuda_stream()) } None => StreamHealth::Busy, diff --git a/cuda-async/src/device_operation.rs b/cuda-async/src/device_operation.rs index 9c0c51d173..dc4c8ba986 100644 --- a/cuda-async/src/device_operation.rs +++ b/cuda-async/src/device_operation.rs @@ -9,7 +9,7 @@ use crate::device_context::{pool_for_stream, with_default_device_policy}; use crate::device_future::DeviceFuture; use crate::error::DeviceError; use crate::scheduling_policies::SchedulingPolicy; -use cuda_core::{Device, Event, MemPool, Stream}; +use cuda_core::{Context, Event, MemPool, Stream}; use std::cell::Cell; use std::fmt::Debug; use std::future::IntoFuture; @@ -107,7 +107,7 @@ pub trait ReplayResource: Send + Sync { pub struct ExecutionContext { ordinal: DeviceOrdinal, cuda_stream: Arc, - device: Arc, + device: Arc, pool: Option>, submission: Arc, recording: bool, @@ -115,7 +115,7 @@ pub struct ExecutionContext { impl ExecutionContext { pub fn new(cuda_stream: Arc) -> Self { - let device = cuda_stream.device().clone(); + let device = cuda_stream.context().clone(); let ordinal = device.ordinal(); let pool = pool_for_stream(&cuda_stream); Self { @@ -186,7 +186,7 @@ impl ExecutionContext { pub(crate) unsafe fn complete(&self) { self.submission.complete(); } - pub fn device(&self) -> &Arc { + pub fn context(&self) -> &Arc { &self.device } pub fn get_device_id(&self) -> DeviceOrdinal { @@ -516,7 +516,7 @@ pub trait DeviceOp: let _execution_lock = acquire_execution_lock()?; let ctx = ExecutionContext::new(stream.clone()); let res = unsafe { self.execute(&ctx) }; - let sync_res = unsafe { stream.synchronize() }; + let sync_res = stream.synchronize(); sync_res?; unsafe { ctx.complete() }; res @@ -600,7 +600,7 @@ impl Producer { /// context's stream. fn record(ctx: &ExecutionContext) -> Result { let stream = Arc::clone(ctx.get_cuda_stream()); - let event = stream.device().new_event()?; + let event = stream.context().new_event()?; event.record(&stream)?; Ok(Self { stream, event }) } diff --git a/cuda-async/src/launch.rs b/cuda-async/src/launch.rs index d948cb0a85..f47cb85c3b 100644 --- a/cuda-async/src/launch.rs +++ b/cuda-async/src/launch.rs @@ -264,11 +264,17 @@ impl AsyncKernelLaunch { /// # Safety /// The caller must ensure the kernel arguments and launch config are valid. unsafe fn launch(mut self, stream: &Arc) -> Result<(), DeviceError> { + if self.func.context().as_ref() != stream.context().as_ref() { + return Err(DeviceError::Driver(cuda_core::DriverError( + cuda_core::sys::cudaError_enum_CUDA_ERROR_INVALID_CONTEXT, + ))); + } + stream.context().bind_to_thread()?; let cfg = self.cfg.ok_or_else(|| { DeviceError::Launch("Await called before launching the kernel.".to_string()) })?; let launch = if self.programmatic_dependent_launch { - let architecture = cuda_core::get_device_sm_name(stream.device().cu_device())?; + let architecture = cuda_core::get_device_sm_name(stream.context().cu_device())?; let sm: u32 = architecture .strip_prefix("sm_") .and_then(|s| s.trim_end_matches(['a', 'f']).parse().ok()) diff --git a/cuda-async/src/reactor.rs b/cuda-async/src/reactor.rs index 3a29ab208b..30c9501b9c 100644 --- a/cuda-async/src/reactor.rs +++ b/cuda-async/src/reactor.rs @@ -216,7 +216,7 @@ fn probe_stale_slots( /// slot's flag write can no longer land — the error is the context's sticky /// fault — so recycling the slot cannot be clobbered by a late device write. fn stream_is_faulted(stream: &Stream) -> bool { - if stream.device().bind_to_thread().is_err() { + if stream.context().bind_to_thread().is_err() { return false; } matches!(probe_stream(stream), StreamHealth::Faulted(_)) diff --git a/cuda-async/src/scheduling_policies.rs b/cuda-async/src/scheduling_policies.rs index 5bcc0cac79..5de27cac0a 100644 --- a/cuda-async/src/scheduling_policies.rs +++ b/cuda-async/src/scheduling_policies.rs @@ -6,7 +6,7 @@ //! Stream scheduling policies that control how operations are assigned to CUDA streams. use crate::error::DeviceError; -use cuda_core::{Device, Stream}; +use cuda_core::{Context, Stream}; use std::sync::atomic::AtomicUsize; use std::sync::Arc; @@ -39,7 +39,7 @@ pub struct StreamPoolRoundRobin { impl StreamPoolRoundRobin { /// Creates a round-robin pool with `num_streams` streams on the given device. - pub fn new(device: &Arc, num_streams: usize) -> Result { + pub fn new(device: &Arc, num_streams: usize) -> Result { let mut stream_pool = Vec::with_capacity(num_streams); for _ in 0..num_streams { stream_pool.push(device.new_stream()?); @@ -68,7 +68,7 @@ pub struct SingleStream { impl SingleStream { /// Creates a single-stream policy on the given device. - pub fn new(device: &Arc) -> Result { + pub fn new(device: &Arc) -> Result { Ok(Self { stream: device.new_stream()?, }) diff --git a/cuda-async/src/simt/device_box.rs b/cuda-async/src/simt/device_box.rs index 941f201778..bde5b94b34 100644 --- a/cuda-async/src/simt/device_box.rs +++ b/cuda-async/src/simt/device_box.rs @@ -15,13 +15,9 @@ //! rather than blocking with `cuMemFree`. Callers **must** synchronize all //! streams that reference the allocation before dropping the box. -use crate::simt::device_context::with_deallocator_stream; -use crate::simt::error::DeviceError; use crate::simt::launch::{AsyncKernelLaunchBuilder, KernelArgument}; use cuda_bindings::CUdeviceptr; -use cuda_core::simt::memory::free_async; -use std::io::{self, Write}; -use std::marker::PhantomData; +use std::{marker::PhantomData, sync::Arc}; /// Non-owning, `Copy` handle to a typed device allocation. /// @@ -54,19 +50,12 @@ impl KernelArgument for DevicePointer { } } -/// Owning wrapper around a device allocation that frees memory asynchronously -/// on drop. -/// -/// `DeviceBox<[DType]>` represents a contiguous device buffer of `len` -/// elements. The allocation is freed via `cuMemFreeAsync` on the device's -/// deallocator stream when the box is dropped. -/// -/// # Safety contract -/// -/// All streams that reference this allocation **must** be synchronized before -/// the box is dropped. Violating this causes use-after-free on the device. +/// Owning wrapper backed by the shared CUDA allocation owner and access tracker. +/// Generated launch adapters retain storage until GPU completion. Raw-pointer +/// callers must independently retain storage and exclude conflicting accesses. #[derive(Debug)] pub struct DeviceBox { + storage: cuda_core::RawDeviceBuffer, /// Ordinal of the device that owns this allocation. device_id: usize, /// Raw CUDA device pointer to the start of the allocation. @@ -89,44 +78,30 @@ unsafe impl Send for DeviceBox {} /// value, which is safe to read from any thread. unsafe impl Sync for DeviceBox {} -/// Best-effort enqueue of an asynchronous free on the device's deallocator -/// stream. -/// -/// Drop cannot return errors, so failures to access the deallocator stream or -/// enqueue the free are reported to stderr instead of panicking. -impl Drop for DeviceBox { - fn drop(&mut self) { - let result = unsafe { - with_deallocator_stream(self.device_id, |stream| { - free_async(self.cudptr, stream.cu_stream()).map_err(DeviceError::Driver) - }) - }; - - match result { - Ok(Ok(())) => {} - Ok(Err(err)) | Err(err) => { - let mut stderr = io::stderr().lock(); - let _ = writeln!( - stderr, - "cuda-async: failed to enqueue async free for device pointer on device_id={}: {}", - self.device_id, err - ); - } - } - } -} - impl DeviceBox<[DType]> { /// Constructs a `DeviceBox<[DType]>` from a raw device pointer, element - /// count, and device ordinal. + /// count, and allocating stream. /// /// # Safety /// /// * `dptr` must point to a valid device allocation of at least - /// `len * size_of::()` bytes on device `device_id`. + /// `len * size_of::()` bytes, allocated asynchronously on `stream`. + /// * Every element must be valid for its device-side type before use. + /// * The caller must order uses after allocation and track all raw accesses. /// * Ownership of the allocation transfers to the returned `DeviceBox`. - pub unsafe fn from_raw_parts(dptr: CUdeviceptr, len: usize, device_id: usize) -> Self { + pub unsafe fn from_raw_parts( + dptr: CUdeviceptr, + len: usize, + stream: Arc, + ) -> Self { + let bytes = len + .checked_mul(std::mem::size_of::()) + .expect("device allocation size overflow"); + let device_id = stream.context().ordinal(); + let storage = + unsafe { cuda_core::RawDeviceBuffer::from_raw_parts_async(dptr, bytes, stream) }; Self { + storage, _marker: PhantomData, cudptr: dptr, len, @@ -134,6 +109,10 @@ impl DeviceBox<[DType]> { } } + pub fn storage(&self) -> &cuda_core::RawDeviceBuffer { + &self.storage + } + /// Returns `true` if the buffer contains zero elements. pub fn is_empty(&self) -> bool { self.len == 0 diff --git a/cuda-async/src/simt/launch.rs b/cuda-async/src/simt/launch.rs index de151fe3cd..ba252e3de9 100644 --- a/cuda-async/src/simt/launch.rs +++ b/cuda-async/src/simt/launch.rs @@ -23,7 +23,7 @@ use crate::simt::device_future::DeviceFuture; use crate::simt::device_operation::{DeviceOperation, ExecutionContext}; use crate::simt::error::DeviceError; use crate::simt::scheduling_policies::SchedulingPolicy; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaFunction, CudaStream}; use std::ffi::c_void; use std::future::IntoFuture; @@ -61,6 +61,7 @@ use std::sync::Arc; pub struct AsyncKernelLaunchBuilder<'a> { /// Handle to the compiled device function. func: Arc, + resources: Vec<(cuda_core::RawDeviceBuffer, bool)>, /// Heap-allocated, type-erased argument storage passed to the CUDA driver. args: KernelArgStorage, /// Optional thread-block cluster dimensions for `cuLaunchKernelEx`. @@ -82,7 +83,7 @@ pub struct AsyncKernelLaunchBuilder<'a> { /// /// ```compile_fail,E0599 /// use cuda_async::simt::launch::AsyncKernelLaunch; -/// use cuda_core::simt::LaunchConfig; +/// use cuda_core::LaunchConfig; /// /// fn cannot_reconfigure(launch: &mut AsyncKernelLaunch<'_>, config: LaunchConfig) { /// launch.set_launch_config(config); @@ -92,6 +93,7 @@ pub struct AsyncKernelLaunchBuilder<'a> { pub struct AsyncKernelLaunch<'a> { /// Handle to the compiled device function. func: Arc, + resources: Vec<(cuda_core::RawDeviceBuffer, bool)>, /// Heap-allocated, type-erased argument storage passed to the CUDA driver. args: KernelArgStorage, /// Complete grid/block dimensions and dynamic shared-memory size. @@ -157,6 +159,7 @@ impl<'a> AsyncKernelLaunchBuilder<'a> { pub fn new(func: Arc) -> Self { Self { func, + resources: Vec::new(), args: KernelArgStorage::default(), cluster_dim: None, cooperative: false, @@ -164,6 +167,11 @@ impl<'a> AsyncKernelLaunchBuilder<'a> { } } + /// Retains a shared allocation and records its access mode for submission. + pub fn retain_buffer(&mut self, storage: cuda_core::RawDeviceBuffer, write: bool) { + self.resources.push((storage, write)); + } + /// Appends a by-value `Copy` argument to the kernel launch packet. /// /// This is the typed-module path used for scalar, raw-pointer, custom @@ -261,7 +269,7 @@ impl<'a> AsyncKernelLaunchBuilder<'a> { /// /// ```compile_fail,E0133 /// use cuda_async::simt::launch::AsyncKernelLaunchBuilder; - /// use cuda_core::simt::LaunchConfig; + /// use cuda_core::LaunchConfig; /// /// fn raw_finalize_requires_unsafe( /// builder: AsyncKernelLaunchBuilder<'static>, @@ -273,6 +281,7 @@ impl<'a> AsyncKernelLaunchBuilder<'a> { pub unsafe fn finalize_unchecked(self, cfg: LaunchConfig) -> AsyncKernelLaunch<'a> { AsyncKernelLaunch { func: self.func, + resources: self.resources, args: self.args, cfg, cluster_dim: self.cluster_dim, @@ -302,6 +311,17 @@ impl<'a> AsyncKernelLaunch<'a> { /// Returns [`DeviceError::Driver`] if context binding or launch submission /// fails. unsafe fn launch(mut self, stream: &Arc) -> Result<(), DeviceError> { + if self.func.context().as_ref() != stream.context().as_ref() { + return Err(DeviceError::Driver(cuda_core::DriverError( + cuda_core::sys::cudaError_enum_CUDA_ERROR_INVALID_CONTEXT, + ))); + } + let leases = self + .resources + .iter() + .map(|(storage, write)| storage.acquire(stream, *write, false)) + .collect::, _>>() + .map_err(DeviceError::Driver)?; let cfg = self.cfg; let result = match (self.cluster_dim, self.cooperative) { (Some(cluster_dim), true) => unsafe { @@ -347,7 +367,20 @@ impl<'a> AsyncKernelLaunch<'a> { ) }, }; + // The legacy future may hold only borrowed output or (). Retain the + // allocations, access leases and module independently of that future. + let retained = cuda_core::completion::retain_until_complete(self.func.clone(), stream); + let mut access_error = None; + for lease in leases { + if let Err(error) = lease.retain_until_complete(stream) { + access_error = Some(error); + } + } result.map_err(DeviceError::Driver)?; + retained.map_err(DeviceError::Driver)?; + if let Some(error) = access_error { + return Err(DeviceError::Driver(error)); + } Ok(()) } } diff --git a/cuda-async/src/submission.rs b/cuda-async/src/submission.rs index 8a2e3c890a..e37846c9da 100644 --- a/cuda-async/src/submission.rs +++ b/cuda-async/src/submission.rs @@ -153,11 +153,11 @@ impl Drop for Submission { .unwrap_or_else(|e| e.into_inner()) .release(|| { let result = (|| { - self.stream.device().bind_to_thread()?; + self.stream.context().bind_to_thread()?; match probe_stream(&self.stream) { StreamHealth::Idle => Ok(()), StreamHealth::Busy => { - unsafe { self.stream.synchronize() }.map_err(DeviceError::Driver) + self.stream.synchronize().map_err(DeviceError::Driver) } StreamHealth::Faulted(e) => Err(DeviceError::Driver(e)), StreamHealth::Capturing => Err(DeviceError::Internal( diff --git a/cuda-async/tests/concurrent_capture.rs b/cuda-async/tests/concurrent_capture.rs index e574ac2e8d..93c6403c5d 100644 --- a/cuda-async/tests/concurrent_capture.rs +++ b/cuda-async/tests/concurrent_capture.rs @@ -10,7 +10,7 @@ use cuda_async::device_operation::value; use cuda_async::error::DeviceError; fn has_gpu() -> bool { - cuda_core::Device::device_count() + cuda_core::Context::device_count() .map(|n| n > 0) .unwrap_or(false) } @@ -23,7 +23,7 @@ fn concurrent_capture_same_context() { return; } - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream_a = device.new_stream().unwrap(); let stream_b = device.new_stream().unwrap(); @@ -66,7 +66,7 @@ fn new_stream_during_capture_on_another_thread() { return; } - let device_a = cuda_core::Device::new(0).unwrap(); + let device_a = cuda_core::Context::new(0).unwrap(); let stream_a = device_a.new_stream().unwrap(); let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); @@ -86,7 +86,7 @@ fn new_stream_during_capture_on_another_thread() { let handle_b = std::thread::spawn(move || -> Result<(), DeviceError> { barrier_b.wait(); - let device_b = cuda_core::Device::new(0).unwrap(); + let device_b = cuda_core::Context::new(0).unwrap(); let _stream = device_b.new_stream()?; Ok(()) }); diff --git a/cuda-async/tests/cuda_graph.rs b/cuda-async/tests/cuda_graph.rs index 04aafe04ed..ff7544dcdf 100644 --- a/cuda-async/tests/cuda_graph.rs +++ b/cuda-async/tests/cuda_graph.rs @@ -13,7 +13,7 @@ use cuda_async::error::DeviceError; use std::future::IntoFuture; fn has_gpu() -> bool { - cuda_core::Device::device_count() + cuda_core::Context::device_count() .map(|n| n > 0) .unwrap_or(false) } @@ -28,7 +28,7 @@ fn scope_empty_closure() { return; } on_fresh_thread(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let graph = CudaGraph::scope(&stream, |_s| Ok(())).unwrap(); @@ -42,7 +42,7 @@ fn scope_records_value_ops() { return; } on_fresh_thread(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let mut recorded = Vec::new(); @@ -66,7 +66,7 @@ fn scope_error_propagation() { return; } on_fresh_thread(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let result = CudaGraph::scope(&stream, |_s| { @@ -93,7 +93,7 @@ fn scope_panic_safety() { return; } let result = std::thread::spawn(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { @@ -103,7 +103,7 @@ fn scope_panic_safety() { })); // Stream should still be usable after the panic. - unsafe { stream.synchronize() }.unwrap(); + stream.synchronize().unwrap(); }) .join(); @@ -119,7 +119,7 @@ fn scope_multiple_launches() { return; } on_fresh_thread(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let graph = CudaGraph::scope(&stream, |_s| Ok(())).unwrap(); @@ -140,7 +140,7 @@ fn scope_nested_execution_rejected() { return; } on_fresh_thread(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let other_stream = device.new_stream().unwrap(); @@ -179,7 +179,7 @@ fn launch_outlives_its_graph() { return; } on_fresh_thread(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let graph = CudaGraph::scope(&stream, |s| { @@ -214,7 +214,7 @@ fn capture_rejects_nested_execution() { return; } on_fresh_thread(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let mut graph = CudaGraph::capture( @@ -242,7 +242,7 @@ fn capture_panic_ends_capture_and_releases_lock() { return; } on_fresh_thread(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { @@ -253,7 +253,9 @@ fn capture_panic_ends_capture_and_releases_lock() { })); assert!(panicked.is_err(), "the panic must propagate"); - unsafe { stream.synchronize() }.expect("stream must not be left in capture mode"); + stream + .synchronize() + .expect("stream must not be left in capture mode"); assert_eq!(value(2).sync_on(&stream).expect("lock must be free"), 2); let mut graph = CudaGraph::capture(stream.clone(), value(4)).expect("recapture"); assert_eq!(graph.take_output(), Some(4)); @@ -284,7 +286,7 @@ fn capture_error_ends_capture_and_releases_lock() { return; } on_fresh_thread(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let err = match CudaGraph::capture(stream.clone(), FailingOp) { @@ -296,7 +298,9 @@ fn capture_error_ends_capture_and_releases_lock() { "op error must propagate unchanged, got {err:?}" ); - unsafe { stream.synchronize() }.expect("stream must not be left in capture mode"); + stream + .synchronize() + .expect("stream must not be left in capture mode"); assert_eq!(value(2).sync_on(&stream).expect("lock must be free"), 2); CudaGraph::capture(stream.clone(), value(4)).expect("recapture"); }); @@ -310,7 +314,7 @@ fn graph_combinators_capture_and_replay() { return; } on_fresh_thread(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let mut graph = value(5).graph_on(stream.clone()).expect("graph_on"); @@ -327,7 +331,9 @@ fn graph_combinators_capture_and_replay() { .graph_on(stream.clone()) })); assert!(panicked.is_err()); - unsafe { stream.synchronize() }.expect("stream must not be left in capture mode"); + stream + .synchronize() + .expect("stream must not be left in capture mode"); assert_eq!(value(7).sync_on(&stream).expect("lock must be free"), 7); }); } @@ -345,7 +351,7 @@ fn update_runs_unit_graph_nodes() { return; } on_fresh_thread(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let graph = CudaGraph::scope(&stream, |_s| Ok(())).unwrap(); diff --git a/cuda-async/tests/drop_in_flight.rs b/cuda-async/tests/drop_in_flight.rs index 55464399f8..0e311efded 100644 --- a/cuda-async/tests/drop_in_flight.rs +++ b/cuda-async/tests/drop_in_flight.rs @@ -192,7 +192,7 @@ impl DeviceOp for SlowOp { ))); } } - let event = stream.device().new_event()?; + let event = stream.context().new_event()?; event.record(stream)?; Ok(Tracked { event, diff --git a/cuda-async/tests/error_handling.rs b/cuda-async/tests/error_handling.rs index 0513bec421..e1618dcda9 100644 --- a/cuda-async/tests/error_handling.rs +++ b/cuda-async/tests/error_handling.rs @@ -176,7 +176,7 @@ fn new_device_context_with_invalid_device_returns_driver_error() { // Device ordinal 9999 should not exist on any reasonable system. // Device::new(9999) will fail with a driver error, which propagates // through new_device_context. - let result = cuda_core::Device::new(9999); + let result = cuda_core::Context::new(9999); let result = result.map_err(DeviceError::Driver); match result { Err(DeviceError::Driver(_)) => { /* expected */ } diff --git a/cuda-async/tests/execution_lock.rs b/cuda-async/tests/execution_lock.rs index 0a28cc44b4..81f507f1d1 100644 --- a/cuda-async/tests/execution_lock.rs +++ b/cuda-async/tests/execution_lock.rs @@ -48,7 +48,7 @@ fn then_closure_cannot_nest_execution() { fn then_unchecked_closure_can_nest_sync_sync_on_and_await() { on_fresh_thread(|| { init_device_contexts(0, 1).expect("init failed (requires GPU)"); - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let other_stream = device.new_stream().unwrap(); let chain = unsafe { @@ -140,7 +140,7 @@ fn panic_inside_future_poll_releases_lock() { #[test] fn panic_inside_scope_releases_lock() { on_fresh_thread(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { CudaGraph::scope(&stream, |_s| panic!("intentional panic in scope")) diff --git a/cuda-async/tests/pool_allocation.rs b/cuda-async/tests/pool_allocation.rs index 37b8fdb228..c1bb9ea932 100644 --- a/cuda-async/tests/pool_allocation.rs +++ b/cuda-async/tests/pool_allocation.rs @@ -416,7 +416,7 @@ fn mem_stats_track_alloc_and_free() { let dptr = unsafe { cuda_core::malloc_from_pool_async(N, &pool, &stream) } .expect("pool allocation failed"); assert!(dptr != 0); - unsafe { stream.synchronize() }.expect("stream sync failed"); + stream.synchronize().expect("stream sync failed"); let stats = pool.mem_stats().expect("read stats failed"); assert!( @@ -431,7 +431,7 @@ fn mem_stats_track_alloc_and_free() { ); unsafe { cuda_core::free_async(dptr, &stream) }.expect("free failed"); - unsafe { stream.synchronize() }.expect("stream sync failed"); + stream.synchronize().expect("stream sync failed"); let stats = pool.mem_stats().expect("read stats failed"); assert_eq!( @@ -467,7 +467,7 @@ fn reset_used_high_collapses_watermark_to_current() { let dptr = unsafe { cuda_core::malloc_from_pool_async(N, &pool, &stream) } .expect("pool allocation failed"); unsafe { cuda_core::free_async(dptr, &stream) }.expect("free failed"); - unsafe { stream.synchronize() }.expect("stream sync failed"); + stream.synchronize().expect("stream sync failed"); let before = pool.mem_stats().expect("read stats failed"); assert_eq!(before.used_current, 0); diff --git a/cuda-core/README.md b/cuda-core/README.md index 2814763b15..cc0547a310 100644 --- a/cuda-core/README.md +++ b/cuda-core/README.md @@ -1,7 +1,43 @@ -# CUDA +# CUDA host runtime `cuda-core` wraps `cuda-bindings` with the CUDA types and functions used by -the rest of the workspace. +the rest of the workspace. Tile and SIMT use the same handles and allocation +owners; there is no `simt` module in this crate. + +`Device` identifies a physical GPU. `Context` owns or borrows a CUDA execution +context on that GPU. `Stream`, `Event`, `Module`, and `Function` retain their +actual context, and compatibility checks compare context handles rather than +device ordinals. The `CudaContext`, `CudaStream`, `CudaEvent`, `CudaModule`, and +`CudaFunction` spellings are aliases of those shared types. + +`DeviceBuffer` provides typed storage. `RawDeviceBuffer` shares allocation +ownership, byte offsets, context identity, and access tracking with typed +buffers and Tile tensors. Owned synchronous allocations, stream-ordered +allocations, borrowed pointers, and foreign owners use this same representation. +Foreign `DeviceAllocation` implementations supply their actual context. + +Safe submissions retain allocation access leases independently of their future. +Conflicting accesses on different streams return `CUDA_ERROR_NOT_READY` until +completion; accesses in one stream remain ordered by CUDA. Buffer and stream +destructors retire resources without waiting for GPU work on the dropping +thread. If completion cannot be established, retirement retains the resource +instead of freeing or reusing it. Unsafe raw-pointer users must establish their +own lifetime, initialization, and ordering guarantees. + +`DeviceCopy` permits byte transfers of valid values. `DType` describes Tile +element types. Neither trait implies the other; types supported by both models +implement both. Safe casts to another representation additionally require +`AnyBitPattern`, while `DeviceRepr` supports ownership-preserving wrappers such +as device atomics. A `DeviceBuffer` remains supported, but arbitrary bytes +cannot safely be cast to `bool`. + +For migration, replace `cuda_core::simt::...` imports with `cuda_core::...`. +Code using the old context-owning `Device` should use `Context`; stream and pool +context accessors are named `context()`. With cuTile, use `Tensor::from_buffer` +and `Tensor::into_buffer` for ownership transfer without copying device memory, +or `Tensor::as_buffer` / `TensorView::as_buffer` for contiguous borrowed kernel +arguments. Ownership transfer requires unique storage; borrowed slices preserve +their byte offset and reject strided layouts. # Testing diff --git a/cuda-core/src/allocation.rs b/cuda-core/src/allocation.rs new file mode 100644 index 0000000000..11358744fb --- /dev/null +++ b/cuda-core/src/allocation.rs @@ -0,0 +1,436 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Shared allocation ownership, byte views, and in-flight access tracking. + +use crate::{Context, DriverError, Stream}; +use cuda_bindings::CUdeviceptr; +use std::sync::{Arc, Mutex}; + +/// A foreign allocation whose owner keeps both its memory and CUDA context alive. +/// +/// # Safety +/// The pointer, byte extent, and context must remain stable and valid for the +/// owner's lifetime. The pointer must address at least `len_bytes` bytes in that +/// context. Per-operation exclusion from foreign accesses remains the obligation +/// of callers constructing typed views over this memory. +pub unsafe trait DeviceAllocation: Send + Sync + 'static { + fn device_ptr(&self) -> CUdeviceptr; + fn len_bytes(&self) -> usize; + fn context(&self) -> Arc; + fn device_id(&self) -> usize { + self.context().ordinal() + } +} + +struct Access { + id: u64, + stream: usize, + write: bool, + recording: bool, + completion: Option>, +} + +#[derive(Default)] +struct Accesses { + next_id: u64, + active: Vec, +} + +impl Accesses { + fn register( + &mut self, + stream: usize, + write: bool, + recording: bool, + ) -> Result { + // A synchronous wait on another thread may have completed a retired + // operation before the worker got to its lease. Do not report a stale + // conflict in that case. Errors leave the exclusion intact. + self.active + .retain(|a| !matches!(a.completion.as_ref().map(|e| e.query()), Some(Ok(true)))); + if !recording + && self + .active + .iter() + .any(|a| !a.recording && a.stream != stream && (write || a.write)) + { + return Err(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_NOT_READY, + )); + } + let id = self.next_id.checked_add(1).ok_or(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE, + ))?; + self.next_id = id; + self.active.push(Access { + id, + stream, + write, + recording, + completion: None, + }); + Ok(id) + } +} + +enum Owner { + Synchronous, + StreamOrdered(Arc), + Borrowed, + Foreign(#[allow(dead_code)] Arc), +} + +struct Allocation { + ptr: CUdeviceptr, + bytes: usize, + context: Arc, + owner: Owner, + accesses: Mutex, +} + +impl Drop for Allocation { + fn drop(&mut self) { + if self.ptr == 0 || matches!(self.owner, Owner::Borrowed | Owner::Foreign(_)) { + return; + } + let ptr = self.ptr; + let context = self.context.clone(); + let allocation_stream = match &self.owner { + Owner::StreamOrdered(s) => Some(s.clone()), + _ => None, + }; + // All safe users hold an independent access lease until completion. + // Driver teardown is retired on a worker, never on the dropping thread. + crate::completion::defer(move || { + if let Err(error) = context.bind_to_thread() { + context.record_err(Err::<(), _>(error)); + std::mem::forget((context, allocation_stream)); + return; + } + let result = if let Some(stream) = &allocation_stream { + // SAFETY: the allocation is no longer accessed and the retained + // stream is the one on which its allocation became valid. + unsafe { crate::memory::free_async(ptr, stream.cu_stream()) } + } else { + // SAFETY: ownership of this synchronous allocation is unique. + unsafe { crate::memory::free_sync(ptr) } + }; + if let Err(error) = result { + context.record_err(Err::<(), _>(error)); + std::mem::forget((context, allocation_stream)); + } + }); + } +} + +/// A byte view sharing an allocation owner and access tracker with every derived +/// typed buffer, tensor, and sliced view. Cloning does not copy device memory. +#[derive(Clone)] +pub struct RawDeviceBuffer { + allocation: Arc, + // Only user-visible views share this token. Completion leases retain the + // allocation directly, so creating/dropping them cannot affect uniqueness. + views: Arc<()>, + offset: usize, + len: usize, +} + +impl std::fmt::Debug for RawDeviceBuffer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RawDeviceBuffer") + .field("ptr", &self.cu_deviceptr()) + .field("len", &self.len) + .field("context", &self.allocation.context) + .finish_non_exhaustive() + } +} + +impl RawDeviceBuffer { + fn new(ptr: CUdeviceptr, bytes: usize, context: Arc, owner: Owner) -> Self { + assert!( + ptr.checked_add(bytes as u64).is_some(), + "device allocation extent overflows" + ); + Self { + allocation: Arc::new(Allocation { + ptr, + bytes, + context, + owner, + accesses: Mutex::default(), + }), + views: Arc::new(()), + offset: 0, + len: bytes, + } + } + + /// Adopts a synchronous CUDA allocation. + /// + /// # Safety + /// `ptr` must uniquely own `bytes` bytes allocated by `cuMemAlloc` in + /// `context`, with no outstanding untracked accesses. + pub unsafe fn from_raw_parts(ptr: CUdeviceptr, bytes: usize, context: Arc) -> Self { + Self::new(ptr, bytes, context, Owner::Synchronous) + } + + /// Adopts a stream-ordered CUDA allocation. + /// + /// # Safety + /// `ptr` must uniquely own `bytes` bytes from `cuMemAllocAsync` or its pool + /// variant on `stream`. The caller must establish allocation ordering for + /// every use and retain the allocation through outstanding work. + pub unsafe fn from_raw_parts_async( + ptr: CUdeviceptr, + bytes: usize, + stream: Arc, + ) -> Self { + Self::new( + ptr, + bytes, + stream.context().clone(), + Owner::StreamOrdered(stream), + ) + } + + /// Borrows externally managed memory without taking ownership. + /// + /// # Safety + /// Memory and context must outlive every derived view and queued use. No + /// external access may conflict with accesses through this allocation. + pub unsafe fn borrowed_from_raw_parts( + ptr: CUdeviceptr, + bytes: usize, + context: Arc, + ) -> Self { + Self::new(ptr, bytes, context, Owner::Borrowed) + } + + /// Retains an external owner and limits the logical view to `bytes` bytes. + /// No CUDA calls are made, permitting metadata validation before submission. + pub fn foreign(owner: Arc, bytes: usize) -> Self { + assert!( + bytes <= owner.len_bytes(), + "view exceeds foreign allocation" + ); + Self::new( + owner.device_ptr(), + bytes, + owner.context(), + Owner::Foreign(owner), + ) + } + + pub fn cu_deviceptr(&self) -> CUdeviceptr { + self.allocation.ptr + self.offset as u64 + } + pub fn len_bytes(&self) -> usize { + self.len + } + pub fn is_empty(&self) -> bool { + self.len == 0 + } + pub fn context(&self) -> &Arc { + &self.allocation.context + } + pub(crate) fn allocation_stream(&self) -> Option<&Arc> { + match &self.allocation.owner { + Owner::StreamOrdered(stream) => Some(stream), + _ => None, + } + } + pub fn device_id(&self) -> usize { + self.context().ordinal() + } + /// Identity of the owner/access tracker, unchanged across byte views. + pub fn allocation_id(&self) -> usize { + Arc::as_ptr(&self.allocation) as usize + } + + /// Creates a bounds-checked view without changing allocation identity. + pub fn slice_bytes(&self, offset: usize, bytes: usize) -> Result { + if offset.checked_add(bytes).is_none_or(|end| end > self.len) { + return Err(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE, + )); + } + Ok(Self { + allocation: self.allocation.clone(), + views: self.views.clone(), + offset: self.offset + offset, + len: bytes, + }) + } + + /// Whether only this user-visible view owns the allocation. Completion leases + /// do not count as user aliases; they still enforce in-flight exclusion. + pub fn is_unique(&self) -> bool { + Arc::strong_count(&self.views) == 1 + } + + /// Acquires an allocation-wide access, rejecting cross-stream conflicts. + /// The returned lease must be retained until the submitted work completes. + pub fn acquire( + &self, + stream: &Stream, + write: bool, + recording: bool, + ) -> Result { + if self.context().as_ref() != stream.context().as_ref() { + return Err(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_CONTEXT, + )); + } + self.acquire_on_stream(stream.cu_stream() as usize, write, recording) + } + + /// Registers bookkeeping for a known stream handle. This does not submit GPU + /// work or grant permission to dereference pointers. Runtime adapters must + /// separately validate the stream's context before submitting work. + #[doc(hidden)] + pub fn acquire_on_stream( + &self, + stream: usize, + write: bool, + recording: bool, + ) -> Result { + let mut accesses = self + .allocation + .accesses + .lock() + .unwrap_or_else(|e| e.into_inner()); + let id = accesses.register(stream, write, recording)?; + Ok(AllocationLease { + allocation: self.allocation.clone(), + id, + }) + } + + pub(crate) fn into_owned_parts( + self, + ) -> (CUdeviceptr, usize, Arc, Option>) { + assert_eq!(self.offset, 0, "cannot detach a sliced allocation"); + assert_eq!( + self.len, self.allocation.bytes, + "cannot detach a partial allocation" + ); + let mut allocation = Arc::try_unwrap(self.allocation) + .unwrap_or_else(|_| panic!("cannot detach a shared or in-flight allocation")); + let stream = match &allocation.owner { + Owner::Synchronous => None, + Owner::StreamOrdered(stream) => Some(stream.clone()), + _ => panic!("cannot detach an allocation owned externally"), + }; + let ptr = std::mem::replace(&mut allocation.ptr, 0); + (ptr, allocation.bytes, allocation.context.clone(), stream) + } +} + +/// Keeps an allocation alive and excludes conflicting accesses until dropped. +/// A runtime must hold this beyond enqueue, until completion has been proven. +pub struct AllocationLease { + allocation: Arc, + id: u64, +} + +impl Drop for AllocationLease { + fn drop(&mut self) { + let mut accesses = self + .allocation + .accesses + .lock() + .unwrap_or_else(|e| e.into_inner()); + if let Some(pos) = accesses.active.iter().position(|a| a.id == self.id) { + accesses.active.swap_remove(pos); + } + } +} + +impl AllocationLease { + /// Transfers this lease to completion-bound retirement after submission. + pub fn retain_until_complete(self, stream: &Stream) -> Result<(), DriverError> { + let event = match stream.record_event(None) { + Ok(event) => Arc::new(event), + Err(error) => { + std::mem::forget(self); + return Err(error); + } + }; + { + let mut accesses = self + .allocation + .accesses + .lock() + .unwrap_or_else(|e| e.into_inner()); + if let Some(access) = accesses.active.iter_mut().find(|a| a.id == self.id) { + access.completion = Some(event.clone()); + } + } + crate::completion::retain_until_event(self, event); + Ok(()) + } +} + +impl std::fmt::Debug for AllocationLease { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AllocationLease") + .field("id", &self.id) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicBool, Ordering}; + + fn metadata() -> RawDeviceBuffer { + // SAFETY: bookkeeping only. These borrowed handles are never passed to + // CUDA, dereferenced, or freed by their destructors. + unsafe { + RawDeviceBuffer::borrowed_from_raw_parts( + 0x1000, + 128, + Context::borrow_raw(0x2000 as *mut _, 0, 0), + ) + } + } + + #[test] + fn uniqueness_counts_byte_views_but_not_completion_leases() { + let raw = metadata(); + let view = raw.slice_bytes(16, 32).unwrap(); + let clone = raw.clone(); + let lease = raw.acquire_on_stream(1, true, false).unwrap(); + assert!(!raw.is_unique()); + drop(view); + assert!(!raw.is_unique()); + drop(clone); + assert!(raw.is_unique()); + assert!(raw.acquire_on_stream(2, false, false).is_err()); + drop(lease); + assert!(raw.acquire_on_stream(2, false, false).is_ok()); + } + + #[test] + fn concurrent_lease_retirement_cannot_change_user_uniqueness() { + let raw = metadata(); + let stop = AtomicBool::new(false); + let mut false_negatives = 0; + std::thread::scope(|scope| { + scope.spawn(|| { + while !stop.load(Ordering::Relaxed) { + drop(raw.acquire_on_stream(1, false, false).unwrap()); + } + }); + for _ in 0..100_000 { + false_negatives += usize::from(!raw.is_unique()); + } + stop.store(true, Ordering::Relaxed); + }); + assert_eq!(false_negatives, 0); + } +} diff --git a/cuda-core/src/api.rs b/cuda-core/src/api.rs index 0b6ea8a9bf..4802bf5dca 100644 --- a/cuda-core/src/api.rs +++ b/cuda-core/src/api.rs @@ -171,6 +171,7 @@ pub unsafe fn malloc_async( num_bytes: usize, stream: &Arc, ) -> Result { + stream.context().bind_to_thread()?; crate::cudarc_shim::memory::malloc_async(stream.cu_stream(), num_bytes) } @@ -185,6 +186,7 @@ pub unsafe fn malloc_from_pool_async( pool: &Arc, stream: &Arc, ) -> Result { + stream.context().bind_to_thread()?; crate::cudarc_shim::pool::malloc_from_pool_async(pool.cu_pool(), stream.cu_stream(), num_bytes) } @@ -199,6 +201,7 @@ pub unsafe fn memset_d8_async( num_bytes: usize, stream: &Arc, ) -> Result<(), DriverError> { + stream.context().bind_to_thread()?; crate::cudarc_shim::memory::memset_d8_async(dptr, value, num_bytes, stream.cu_stream()) } @@ -209,6 +212,7 @@ pub unsafe fn memset_d8_async( /// [`malloc_from_pool_async`], `stream` must be ordered after every use of /// the allocation, and `dptr` must not be used after this call. pub unsafe fn free_async(dptr: sys::CUdeviceptr, stream: &Arc) -> Result<(), DriverError> { + stream.context().bind_to_thread()?; crate::cudarc_shim::memory::free_async(dptr, stream.cu_stream()) } @@ -222,6 +226,7 @@ pub unsafe fn memcpy_htod_async( num_elements: usize, stream: &Arc, ) -> Result<(), DriverError> { + stream.context().bind_to_thread()?; let num_bytes = num_elements * mem::size_of::(); unsafe { crate::cudarc_shim::memory::memcpy_htod_async(dst, src, num_bytes, stream.cu_stream()) @@ -238,6 +243,7 @@ pub unsafe fn memcpy_dtoh_async( num_elements: usize, stream: &Arc, ) -> Result<(), DriverError> { + stream.context().bind_to_thread()?; let num_bytes = num_elements * mem::size_of::(); unsafe { crate::cudarc_shim::memory::memcpy_dtoh_async(dst, src, num_bytes, stream.cu_stream()) @@ -254,6 +260,7 @@ pub unsafe fn memcpy_dtod_async( num_elements: usize, stream: &Arc, ) -> Result<(), DriverError> { + stream.context().bind_to_thread()?; let num_bytes = num_elements * mem::size_of::(); unsafe { crate::cudarc_shim::memory::memcpy_dtod_async(dst, src, num_bytes, stream.cu_stream()) diff --git a/cuda-core/src/completion.rs b/cuda-core/src/completion.rs new file mode 100644 index 0000000000..c6acd9b982 --- /dev/null +++ b/cuda-core/src/completion.rs @@ -0,0 +1,107 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Completion-bound retirement without waiting in destructors. + +use crate::{DriverError, Event, Stream}; +use std::sync::{mpsc, Arc, OnceLock}; +use std::time::Duration; + +type Cleanup = Box; +struct Pending { + event: Option>, + cleanup: Cleanup, +} + +fn enqueue(pending: Pending) { + static QUEUE: OnceLock>> = OnceLock::new(); + let queue = QUEUE.get_or_init(|| { + let (sender, receiver) = mpsc::channel::(); + std::thread::Builder::new() + .name("cuda-resource-retirement".into()) + .spawn(move || { + let mut pending = Vec::::new(); + loop { + match receiver.recv_timeout(Duration::from_millis(1)) { + Ok(item) => pending.push(item), + Err(mpsc::RecvTimeoutError::Disconnected) => break, + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + pending.extend(receiver.try_iter()); + let mut i = 0; + while i < pending.len() { + let ready = pending[i] + .event + .as_ref() + .map_or(Ok(true), |event| event.query()); + match ready { + Ok(false) => i += 1, + Ok(true) => { + let item = pending.swap_remove(i); + // Preserve the event's context until cleanup has finished. + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe( + item.cleanup, + )); + } + Err(_) => { + // Completion is unknown: leaking the owner and its access + // lease is safer than freeing memory or allowing conflicting work. + std::mem::forget(pending.swap_remove(i)); + } + } + } + } + for item in pending { + std::mem::forget(item); + } + }) + .ok() + .map(|_| sender) + }); + if let Some(queue) = queue { + if let Err(error) = queue.send(pending) { + std::mem::forget(error.0); + } + } else { + std::mem::forget(pending); + } +} + +pub(crate) fn defer(cleanup: impl FnOnce() + Send + 'static) { + enqueue(Pending { + event: None, + cleanup: Box::new(cleanup), + }); +} + +/// Keeps an owner or access lease alive until work already queued on `stream` +/// completes. Dropping the caller's future cannot shorten this lifetime. +/// +/// Failure to record or observe completion intentionally leaks the owner; +/// releasing it could invalidate work that the driver has already accepted. +pub fn retain_until_complete( + owner: impl Send + 'static, + stream: &Stream, +) -> Result<(), DriverError> { + let event = match stream.record_event(None) { + Ok(event) => event, + Err(error) => { + std::mem::forget(owner); + return Err(error); + } + }; + enqueue(Pending { + event: Some(Arc::new(event)), + cleanup: Box::new(move || drop(owner)), + }); + Ok(()) +} + +pub(crate) fn retain_until_event(owner: impl Send + 'static, event: Arc) { + enqueue(Pending { + event: Some(event), + cleanup: Box::new(move || drop(owner)), + }); +} diff --git a/cuda-core/src/simt/context.rs b/cuda-core/src/context.rs similarity index 68% rename from cuda-core/src/simt/context.rs rename to cuda-core/src/context.rs index 2a76c5a389..5d8270254d 100644 --- a/cuda-core/src/simt/context.rs +++ b/cuda-core/src/context.rs @@ -3,93 +3,18 @@ * SPDX-License-Identifier: Apache-2.0 */ -//! CUDA context management (primary context, RAII). -//! -//! [`CudaContext`] retains the **primary context** for a given device ordinal -//! via `cuDevicePrimaryCtxRetain` and releases it on [`Drop`]. The primary -//! context is shared across the process; multiple `CudaContext` instances for -//! the same device share the same underlying `CUcontext`. -//! -//! # Thread binding -//! -//! CUDA driver calls are context-scoped and thread-local. [`CudaContext`] -//! transparently calls `cuCtxSetCurrent` before any driver operation, so -//! callers do not need to manage the context stack manually. +//! Context queries and configuration shared by both programming models. -use crate::error::{DriverError, IntoResult}; -use crate::simt::launch::DeviceLaunchLimits; -use crate::simt::stream::CudaStream; -use std::ffi::c_int; -use std::mem::MaybeUninit; -use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; -use std::sync::Arc; - -/// Owns a reference to a CUDA device's primary context. -/// -/// Created via [`CudaContext::new`] and typically held in an `Arc` so streams, -/// events, and modules can share the same context. Dropping the last reference -/// releases the primary context (`cuDevicePrimaryCtxRelease`). -/// -/// Tracks live stream count and accumulated error state atomically for -/// cross-thread diagnostics. -#[derive(Debug)] -pub struct CudaContext { - /// Raw CUDA device handle (`CUdevice`). - pub(crate) cu_device: cuda_bindings::CUdevice, - /// Raw CUDA context handle (`CUcontext`). Set to null on drop. - pub(crate) cu_ctx: cuda_bindings::CUcontext, - /// Zero-based device ordinal passed to [`CudaContext::new`]. - pub(crate) ordinal: usize, - /// Number of live [`CudaStream`] instances sharing this context. - pub(crate) num_streams: AtomicUsize, - /// When `true`, the first [`new_stream`](CudaContext::new_stream) call - /// synchronizes the context to establish a clean ordering baseline. - pub(crate) event_tracking: AtomicBool, - /// Sticky error state recorded by [`record_err`](CudaContext::record_err). - /// Stores the raw `CUresult` value, or `0` if no error. - pub(crate) error_state: AtomicU32, -} - -/// # Safety -/// -/// `CUdevice` and `CUcontext` are process-wide handles. All mutable state -/// (`num_streams`, `event_tracking`, `error_state`) uses atomics. The CUDA -/// driver itself is thread-safe for distinct contexts, and the -/// [`bind_to_thread`](CudaContext::bind_to_thread) mechanism ensures the -/// correct context is current before each call. -unsafe impl Send for CudaContext {} -/// See [`Send`] impl. -unsafe impl Sync for CudaContext {} - -/// Releases the primary context on drop. -/// -/// Binds the context to the current thread first (required by -/// `cuDevicePrimaryCtxRelease`). Errors during teardown are recorded via -/// [`record_err`](CudaContext::record_err) rather than panicking. -impl Drop for CudaContext { - fn drop(&mut self) { - self.record_err(self.bind_to_thread()); - let ctx = std::mem::replace(&mut self.cu_ctx, std::ptr::null_mut()); - if !ctx.is_null() { - self.record_err(unsafe { - cuda_bindings::cuDevicePrimaryCtxRelease_v2(self.cu_device).result() - }); - } - } -} - -/// Equality is based on device handle, context handle, and ordinal. -impl PartialEq for CudaContext { - fn eq(&self, other: &Self) -> bool { - self.cu_device == other.cu_device - && self.cu_ctx == other.cu_ctx - && self.ordinal == other.ordinal - } -} -impl Eq for CudaContext {} +pub use crate::runtime::{Context, Context as CudaContext}; +use crate::{DeviceLaunchLimits, DriverError, IntoResult, KeepAlive, Stream}; +use std::{ + ffi::c_int, + mem::MaybeUninit, + sync::{atomic::Ordering, Arc}, +}; /// The device's meaningful stream priorities, from -/// [`CudaContext::stream_priority_range`]. +/// [`Context::stream_priority_range`]. /// /// CUDA orders priorities the opposite way round to intuition: **a lower /// number is a higher priority**, so the range runs from @@ -142,17 +67,17 @@ impl StreamPriorityRange { /// The value the driver will actually apply for a request of `priority`. /// /// Answers the silent clamp in - /// [`CudaContext::new_stream_with_priority`] ahead of the call. + /// [`Context::new_stream_with_priority`] ahead of the call. pub fn clamp(&self, priority: i32) -> i32 { priority.clamp(self.greatest, self.least) } } /// A per-context device limit (`CU_LIMIT_*`), read with -/// [`CudaContext::limit`] and written with [`CudaContext::set_limit`]. +/// [`Context::limit`] and written with [`Context::set_limit`]. /// /// Every limit is state on the device's **primary** context, so it is shared -/// by every [`CudaContext`] for that device, by any runtime-API user of the +/// by every [`Context`] for that device, by any runtime-API user of the /// same device in this process, and across library boundaries. A limit set /// here is not scoped to the handle that set it. /// @@ -254,8 +179,8 @@ impl ContextLimit { /// waits when the calling host thread blocks on GPU work (a stream or /// context synchronize). /// -/// Set via [`CudaContext::set_sync_policy`], read via -/// [`CudaContext::sync_policy`]. The trade-off is host CPU against wake +/// Set via [`Context::set_sync_policy`], read via +/// [`Context::sync_policy`]. The trade-off is host CPU against wake /// latency: [`Spin`](Self::Spin) burns a full core for the lowest latency, /// [`BlockingSync`](Self::BlockingSync) frees the core but wakes later. On a /// host with spare cores this trade rarely matters; on a CPU-constrained one @@ -302,7 +227,7 @@ impl SyncPolicy { } } -impl CudaContext { +impl Context { fn device_attribute( &self, attribute: cuda_bindings::CUdevice_attribute, @@ -316,173 +241,6 @@ impl CudaContext { .map_err(|_| DriverError(cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE)) } } - - /// Creates a new context for the device at `ordinal`. - /// - /// Calls [`cuInit`](crate::init), obtains the device handle, retains the - /// primary context, and binds it to the calling thread. Returns the context - /// wrapped in an `Arc` for shared ownership across streams, events, and - /// modules. - pub fn new(ordinal: usize) -> Result, DriverError> { - unsafe { crate::init(0)? }; - - let cu_device = unsafe { - let mut cu_device = MaybeUninit::uninit(); - cuda_bindings::cuDeviceGet(cu_device.as_mut_ptr(), ordinal as c_int).result()?; - cu_device.assume_init() - }; - - let cu_ctx = unsafe { - let mut cu_ctx = MaybeUninit::uninit(); - cuda_bindings::cuDevicePrimaryCtxRetain(cu_ctx.as_mut_ptr(), cu_device).result()?; - cu_ctx.assume_init() - }; - - let ctx = Arc::new(CudaContext { - cu_device, - cu_ctx, - ordinal, - num_streams: AtomicUsize::new(0), - event_tracking: AtomicBool::new(true), - error_state: AtomicU32::new(0), - }); - ctx.bind_to_thread()?; - Ok(ctx) - } - - /// Returns the zero-based device ordinal. - pub fn ordinal(&self) -> usize { - self.ordinal - } - - /// Returns the raw `CUdevice` handle. - pub fn cu_device(&self) -> cuda_bindings::CUdevice { - self.cu_device - } - - /// Returns the raw `CUcontext` handle. - pub fn cu_ctx(&self) -> cuda_bindings::CUcontext { - self.cu_ctx - } - - /// Binds this context to the calling thread if not already current. - /// - /// Checks [`check_err`](Self::check_err) first and propagates any sticky - /// error. Skips the `cuCtxSetCurrent` call when the context is already - /// bound, avoiding an unnecessary driver round-trip. - /// - /// Most methods on [`CudaStream`], [`CudaEvent`](crate::CudaEvent), and - /// [`CudaModule`](crate::CudaModule) call this internally. - /// - /// CUcontext is the backing runtime object, and CUmodule / CUfunction / CUstream - /// are opaque handles to objects created under that context. bind_to_thread() - /// makes that context, the one, the current host thread is operating against. - /// If the thread is currently bound to some other context, using those handles - /// can fail. - pub fn bind_to_thread(&self) -> Result<(), DriverError> { - self.check_err()?; - let mut current = MaybeUninit::uninit(); - unsafe { - cuda_bindings::cuCtxGetCurrent(current.as_mut_ptr()).result()?; - let current = current.assume_init(); - if current.is_null() || current != self.cu_ctx { - cuda_bindings::cuCtxSetCurrent(self.cu_ctx).result()?; - } - } - Ok(()) - } - - /// Blocks the calling thread until all preceding work in this context - /// completes. - /// - /// Binds the context first, then calls `cuCtxSynchronize`. - pub fn synchronize(&self) -> Result<(), DriverError> { - self.bind_to_thread()?; - unsafe { cuda_bindings::cuCtxSynchronize() }.result() - } - - /// Returns a handle to the per-context default stream (stream `0`). - /// - /// The default stream implicitly synchronizes with all blocking streams in - /// the same context. The returned [`CudaStream`] holds a null `CUstream` - /// pointer, which the driver interprets as the default stream. - pub fn default_stream(self: &Arc) -> Arc { - Arc::new(CudaStream { - cu_stream: std::ptr::null_mut(), - ctx: self.clone(), - }) - } - - /// Creates a new non-blocking stream in this context. - /// - /// The stream is created with `CU_STREAM_NON_BLOCKING`, so it does not - /// implicitly synchronize with the default stream. - /// - /// On the first call (when `num_streams` transitions from 0 to 1), the - /// context is synchronized to establish a clean ordering baseline if - /// `event_tracking` is enabled. - pub fn new_stream(self: &Arc) -> Result, DriverError> { - self.create_stream(None) - } - - /// Creates a new non-blocking stream at `priority` in this context. - /// - /// As [`new_stream`](Self::new_stream), with the stream created through - /// `cuStreamCreateWithPriority`. - /// - /// **Lower numbers are higher priorities**, and a value outside the - /// device's range is **clamped silently** to the nearest end of it rather - /// than refused, so passing [`i32::MIN`] yields the greatest priority the - /// device supports and reports nothing. Read - /// [`CudaStream::priority`] back to see what the driver applied, or - /// consult [`stream_priority_range`](Self::stream_priority_range) first. - /// - /// Priority orders **compute kernels only**. Host-to-device and - /// device-to-host transfers are unaffected, so a high-priority stream does - /// not jump a queue of copies. On a device without priority support every - /// priority collapses to the single value 0 and the stream still works. - pub fn new_stream_with_priority( - self: &Arc, - priority: i32, - ) -> Result, DriverError> { - self.create_stream(Some(priority)) - } - - /// Shared body of [`new_stream`](Self::new_stream) and - /// [`new_stream_with_priority`](Self::new_stream_with_priority). - /// - /// `None` takes `cuStreamCreate`, which is what the driver documents as - /// equivalent to a priority of 0, rather than passing 0 explicitly: on a - /// device whose range does not contain 0 those are different requests. - fn create_stream( - self: &Arc, - priority: Option, - ) -> Result, DriverError> { - self.bind_to_thread()?; - let prev = self.num_streams.fetch_add(1, Ordering::Relaxed); - if prev == 0 && self.event_tracking.load(Ordering::Relaxed) { - self.synchronize()?; - } - let flags = cuda_bindings::CUstream_flags_enum_CU_STREAM_NON_BLOCKING; - let mut cu_stream = MaybeUninit::uninit(); - let cu_stream = unsafe { - match priority { - Some(priority) => cuda_bindings::cuStreamCreateWithPriority( - cu_stream.as_mut_ptr(), - flags, - priority, - ), - None => cuda_bindings::cuStreamCreate(cu_stream.as_mut_ptr(), flags), - } - .result()?; - cu_stream.assume_init() - }; - Ok(Arc::new(CudaStream { - cu_stream, - ctx: self.clone(), - })) - } - /// Returns the device's meaningful stream priority range. /// /// Wraps `cuCtxGetStreamPriorityRange`. See [`StreamPriorityRange`] for @@ -501,7 +259,6 @@ impl CudaContext { }) } } - /// Queries the device's marketing name (e.g. `"NVIDIA GeForce RTX 5090"`). /// /// Wraps `cuDeviceGetName` with a 256-byte buffer (driver guarantees @@ -523,7 +280,6 @@ impl CudaContext { let end = buf.iter().position(|&c| c == 0).unwrap_or(buf.len()); Ok(String::from_utf8_lossy(&buf[..end]).into_owned()) } - /// Queries the compute capability (SM version) of the device. /// /// Returns `(major, minor)` -- e.g. `(9, 0)` for Hopper H100. @@ -547,7 +303,6 @@ impl CudaContext { Ok((major.assume_init(), minor.assume_init())) } } - /// Queries dimension, thread-count, and portable shared-memory launch /// limits for this device. /// @@ -586,7 +341,6 @@ impl CudaContext { )?, }) } - /// Queries the non-portable opt-in shared-memory limit per block. /// /// Typed launch preparation calls this only when static plus dynamic @@ -596,7 +350,6 @@ impl CudaContext { cuda_bindings::CUdevice_attribute_enum_CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN, ) } - /// Returns whether this device supports cooperative kernel launches. pub fn supports_cooperative_launch(&self) -> Result { self.device_attribute( @@ -604,7 +357,6 @@ impl CudaContext { ) .map(|value| value != 0) } - /// Returns whether this device supports thread-block cluster launches. pub fn supports_cluster_launch(&self) -> Result { self.device_attribute( @@ -612,14 +364,12 @@ impl CudaContext { ) .map(|value| value != 0) } - /// Returns the number of streaming multiprocessors on this device. pub fn multiprocessor_count(&self) -> Result { self.device_attribute( cuda_bindings::CUdevice_attribute_enum_CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, ) } - /// Returns the maximum number of threads resident on one streaming /// multiprocessor. /// @@ -631,7 +381,6 @@ impl CudaContext { cuda_bindings::CUdevice_attribute_enum_CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR, ) } - /// Reads a device limit off this context. /// /// Wraps `cuCtxGetLimit`. See [`ContextLimit`] for what each limit means, @@ -645,7 +394,6 @@ impl CudaContext { Ok(value.assume_init()) } } - /// Writes a device limit on this context. /// /// Wraps `cuCtxSetLimit`. The limit is state on the device's primary @@ -662,7 +410,6 @@ impl CudaContext { self.bind_to_thread()?; unsafe { cuda_bindings::cuCtxSetLimit(limit.to_raw(), value) }.result() } - /// Queries the per-thread device stack size, in bytes. /// /// Equivalent to [`limit`](Self::limit) with @@ -671,7 +418,6 @@ impl CudaContext { pub fn stack_size(&self) -> Result { self.limit(ContextLimit::StackSize) } - /// Sets the per-thread device stack size, in bytes. /// /// Equivalent to [`set_limit`](Self::set_limit) with @@ -688,19 +434,18 @@ impl CudaContext { pub fn set_stack_size(&self, bytes: usize) -> Result<(), DriverError> { self.set_limit(ContextLimit::StackSize, bytes) } - /// Sets the context's scheduling policy (`CU_CTX_SCHED_*`). /// /// Wraps `cuDevicePrimaryCtxSetFlags_v2`, scoped to this context's /// device rather than `cuCtxSetFlags`'s "whatever is current on this - /// thread": [`CudaContext`] retains and owns a specific device's primary + /// thread": [`Context`] retains and owns a specific device's primary /// context, so the device-scoped call is the one that matches what this /// type actually holds. Read-modify-write: only the 3 scheduling bits /// (`CU_CTX_SCHED_MASK`) are replaced, any other flag bit the process has /// set independently (e.g. `CU_CTX_MAP_HOST`) is preserved. /// /// The policy is process-wide state on the device's primary context: - /// it affects every [`CudaContext`] clone of this device, and any + /// it affects every [`Context`] clone of this device, and any /// runtime-API user of the same device in this process, not just this /// handle. The read-modify-write over `CU_CTX_SCHED_MASK` is also not /// atomic: a concurrent flag writer on the same device can interleave @@ -727,7 +472,6 @@ impl CudaContext { cuda_bindings::cuDevicePrimaryCtxSetFlags_v2(self.cu_device, new_flags).result() } } - /// Returns the context's current scheduling policy (`CU_CTX_SCHED_*`). /// /// `None` if the driver reports a scheduling value this enum does not @@ -747,7 +491,6 @@ impl CudaContext { Ok(SyncPolicy::from_raw(flags.assume_init())) } } - /// Atomically reads and clears the sticky error state. /// /// Returns `Ok(())` if no error was recorded, or the stored @@ -760,7 +503,6 @@ impl CudaContext { Err(DriverError(error_state)) } } - /// Records a driver error into the sticky error state. /// /// Used during [`Drop`] paths where returning a `Result` is not possible. @@ -772,4 +514,35 @@ impl CudaContext { self.error_state.store(err.0, Ordering::Relaxed) } } + /// Returns the context's legacy default stream (stream zero). + pub fn default_stream(self: &Arc) -> Arc { + Arc::new(Stream { + cu_stream: std::ptr::null_mut(), + context: self.clone(), + owned: false, + _keep_alive: KeepAlive::none(), + }) + } + /// Creates a nonblocking stream with the requested CUDA scheduling priority. + pub fn new_stream_with_priority( + self: &Arc, + priority: i32, + ) -> Result, DriverError> { + self.bind_to_thread()?; + let mut stream = std::ptr::null_mut(); + unsafe { + cuda_bindings::cuStreamCreateWithPriority( + &mut stream, + cuda_bindings::CUstream_flags_enum_CU_STREAM_NON_BLOCKING, + priority, + ) + .result()?; + } + Ok(Arc::new(Stream { + cu_stream: stream, + context: self.clone(), + owned: true, + _keep_alive: KeepAlive::none(), + })) + } } diff --git a/cuda-core/src/simt/device_buffer.rs b/cuda-core/src/device_buffer.rs similarity index 67% rename from cuda-core/src/simt/device_buffer.rs rename to cuda-core/src/device_buffer.rs index 69caab9868..e9249273ea 100644 --- a/cuda-core/src/simt/device_buffer.rs +++ b/cuda-core/src/device_buffer.rs @@ -37,10 +37,10 @@ use bytemuck::AnyBitPattern; use cuda_bindings::CUdeviceptr; use cuda_repr::DeviceRepr; +use crate::context::Context; use crate::error::DriverError; -use crate::simt::context::CudaContext; -use crate::simt::pinned_host_buffer::PinnedHostBuffer; -use crate::simt::stream::CudaStream; +use crate::pinned_host_buffer::PinnedHostBuffer; +use crate::stream::Stream; /// Marker trait for values that can be safely copied between host and device /// memory as raw bytes. @@ -133,36 +133,42 @@ unsafe impl DeviceCopy for f16 {} unsafe impl DeviceCopy for half::bf16 {} unsafe impl DeviceCopy for half::f16 {} +// SAFETY: these transparent numeric storage wrappers own no resources, have +// valid all-zero representations, and can be duplicated byte-for-byte. Packed +// FP4 remains one byte holding two logical values; copying does not unpack it. +impl_device_copy!( + crate::tf32, + crate::f8e4m3fn, + crate::f8e5m2, + crate::f8e8m0fnu, + crate::f8e5m3fnu, + crate::f4e2m1fnx2 +); + /// Owning handle to a contiguous device allocation of `T` elements. /// -/// Holds a raw device pointer, element count, and a reference-counted -/// context that keeps the CUDA context alive. Synchronous allocations are -/// freed with `cuMemFree`. Dropping a stream-ordered allocation synchronizes -/// its context before enqueueing `cuMemFreeAsync` on its retained allocation -/// stream. Use the unsafe [`DeviceBuffer::drop_async`] when the caller can -/// provide explicit stream ordering and must avoid that context-wide -/// synchronization. +/// Shares an allocation owner, its CUDA context, and an access tracker with +/// tensor and byte views. Safe submissions retain independent access leases +/// until completion, including when a launch future is dropped or forgotten. +/// Drop retires deallocation on a worker without waiting on the caller's +/// thread. Unsafe raw-pointer users must retain storage through their own work. /// /// Device buffers may only transfer plain device-copyable values. Owning host /// types such as [`String`] are rejected because copying their bytes to and /// from device memory would not preserve Rust ownership invariants. /// /// ```compile_fail -/// # use cuda_core::{CudaStream, DeviceBuffer}; -/// # fn rejects_non_device_copy(stream: &CudaStream) { +/// # use cuda_core::{Stream, DeviceBuffer}; +/// # fn rejects_non_device_copy(stream: &Stream) { /// let _ = DeviceBuffer::::zeroed(stream, 1); /// # } /// ``` pub struct DeviceBuffer { + storage: crate::RawDeviceBuffer, + shared_access_writes: bool, ptr: CUdeviceptr, len: usize, num_bytes: usize, - ctx: Arc, - /// Retains the allocation stream for a stream-ordered (`cuMemAllocAsync`) - /// allocation. Ordinary `Drop` first synchronizes the context so work - /// submitted on any stream has completed, then frees on this stream. - /// `None` identifies a synchronous (`cuMemAlloc`) allocation. - dealloc_stream: Option>, _marker: PhantomData, } @@ -173,28 +179,90 @@ unsafe impl Send for DeviceBuffer {} // return Copy values. No interior mutability. unsafe impl Sync for DeviceBuffer {} -impl Drop for DeviceBuffer { - fn drop(&mut self) { - if self.ptr != 0 { - self.ctx.record_err(self.ctx.bind_to_thread()); - // Safe buffer operations can enqueue work on any stream in this - // context. Synchronize all of them before implicitly freeing a - // stream-ordered allocation. - let result = match &self.dealloc_stream { - Some(stream) => match self.ctx.synchronize() { - Ok(()) => unsafe { - crate::simt::memory::free_async(self.ptr, stream.cu_stream()) - }, - Err(error) => Err(error), - }, - None => unsafe { crate::simt::memory::free_sync(self.ptr) }, - }; - self.ctx.record_err(result); +impl DeviceBuffer { + fn enqueue_access( + &self, + stream: &Stream, + write: bool, + enqueue: impl FnOnce() -> Result<(), DriverError>, + ) -> Result<(), DriverError> { + let lease = self.storage.acquire(stream, write, false)?; + self.storage.context().bind_to_thread()?; + let submitted = enqueue(); + let retained = lease.retain_until_complete(stream); + submitted.and(retained) + } + + /// The shared byte storage and allocation identity underlying this view. + pub fn storage(&self) -> &crate::RawDeviceBuffer { + &self.storage + } + /// Whether a shared kernel reference may mutate elements through interior + /// mutability. Representation wrappers and unclassified raw types are + /// conservative; ordinary DeviceCopy values can share read accesses. + #[doc(hidden)] + pub fn shared_access_writes(&self) -> bool { + self.shared_access_writes + } + + /// Erases the element type without copying or releasing storage. + pub fn into_storage(self) -> crate::RawDeviceBuffer { + self.storage + } + + /// Constructs a typed view over shared byte storage. + /// + /// # Safety + /// Every initialized element must be a valid `T`; the caller must preserve + /// element validity across all aliases (including future writes through + /// aliases with another element type). The storage must be initialized before + /// any read, and raw foreign accesses must obey its access-exclusion contract. + pub unsafe fn from_storage( + storage: crate::RawDeviceBuffer, + len: usize, + ) -> Result { + let num_bytes = allocation_size::(len)?; + let ptr = storage.cu_deviceptr(); + if num_bytes != storage.len_bytes() + || (num_bytes != 0 && !(ptr as usize).is_multiple_of(std::mem::align_of::())) + { + return Err(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE, + )); + } + Ok(Self { + ptr, + len, + num_bytes, + storage, + shared_access_writes: true, + _marker: PhantomData, + }) + } + + /// Borrows a contiguous range of elements while retaining the same storage + /// owner and access tracker. No device memory is copied. + pub fn slice( + &self, + range: std::ops::Range, + ) -> Result, DriverError> { + if range.start > range.end || range.end > self.len { + return Err(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE, + )); } + let storage = self.storage.slice_bytes( + allocation_size::(range.start)?, + allocation_size::(range.end - range.start)?, + )?; + Ok(DeviceBufferView { + storage, + shared_access_writes: self.shared_access_writes, + len: range.end - range.start, + _borrow: PhantomData, + }) } -} -impl DeviceBuffer { /// Returns the raw `CUdeviceptr` for use in kernel argument lists. #[inline] pub fn cu_deviceptr(&self) -> CUdeviceptr { @@ -221,8 +289,8 @@ impl DeviceBuffer { /// Returns a reference to the owning context. #[inline] - pub fn context(&self) -> &Arc { - &self.ctx + pub fn context(&self) -> &Arc { + self.storage.context() } /// Constructs a `DeviceBuffer` from pre-existing raw parts. @@ -240,7 +308,7 @@ impl DeviceBuffer { /// # Panics /// /// Panics if `len * size_of::()` overflows `usize`. - pub unsafe fn from_raw_parts(ptr: CUdeviceptr, len: usize, ctx: Arc) -> Self { + pub unsafe fn from_raw_parts(ptr: CUdeviceptr, len: usize, ctx: Arc) -> Self { // SAFETY: `from_raw_parts` has the same raw-allocation safety contract, // with no stream-ordered deallocation metadata attached. unsafe { Self::from_raw_parts_with_dealloc_stream(ptr, len, ctx, None) } @@ -249,17 +317,22 @@ impl DeviceBuffer { unsafe fn from_raw_parts_with_dealloc_stream( ptr: CUdeviceptr, len: usize, - ctx: Arc, - dealloc_stream: Option>, + ctx: Arc, + dealloc_stream: Option>, ) -> Self { let num_bytes = allocation_size::(len).expect("DeviceBuffer::from_raw_parts byte size overflow"); + let storage = if let Some(stream) = &dealloc_stream { + unsafe { crate::RawDeviceBuffer::from_raw_parts_async(ptr, num_bytes, stream.clone()) } + } else { + unsafe { crate::RawDeviceBuffer::from_raw_parts(ptr, num_bytes, ctx.clone()) } + }; Self { + storage, + shared_access_writes: true, ptr, len, num_bytes, - ctx, - dealloc_stream, _marker: PhantomData, } } @@ -270,30 +343,15 @@ impl DeviceBuffer { /// allocator that matches how it was created. For stream-ordered /// allocations, this does not return the stored deallocation stream; the /// caller must already know which stream to use for `cuMemFreeAsync`. - pub fn into_raw_parts(self) -> (CUdeviceptr, usize, Arc) { + pub fn into_raw_parts(self) -> (CUdeviceptr, usize, Arc) { let (ptr, len, ctx, _dealloc_stream) = self.into_all_raw_parts(); (ptr, len, ctx) } - fn into_all_raw_parts( - self, - ) -> ( - CUdeviceptr, - usize, - Arc, - Option>, - ) { - // Suppress the buffer's `Drop` (which would free `ptr`) while still - // moving out the heap-owned fields. Callers that do not need - // `dealloc_stream` can drop it after this helper returns. - let this = std::mem::ManuallyDrop::new(self); - let ptr = this.ptr; - let len = this.len; - // SAFETY: `this` is `ManuallyDrop` and is never used again, so reading - // out its non-`Copy` fields takes ownership without a double drop. - let ctx = unsafe { std::ptr::read(&this.ctx) }; - let dealloc_stream = unsafe { std::ptr::read(&this.dealloc_stream) }; - (ptr, len, ctx, dealloc_stream) + fn into_all_raw_parts(self) -> (CUdeviceptr, usize, Arc, Option>) { + let Self { storage, len, .. } = self; + let (ptr, _, ctx, stream) = storage.into_owned_parts(); + (ptr, len, ctx, stream) } /// Reinterpret the element type of this buffer as `A`. @@ -313,7 +371,9 @@ impl DeviceBuffer { pub fn cast_elem(self) -> DeviceBuffer { // SAFETY: every bit pattern is a valid `A`, so the bytes of any `T` // are valid `A` values; `relabel` checks the layout. - unsafe { self.relabel() } + let mut buffer = unsafe { self.relabel() }; + buffer.shared_access_writes = false; + buffer } /// View this buffer through a `#[repr(transparent)]` wrapper of `T`. @@ -352,7 +412,9 @@ impl DeviceBuffer { { // SAFETY: DeviceRepr guarantees every initialized T representation // is valid as I; relabel preserves ownership and checks the layout. - unsafe { self.relabel() } + let mut buffer = unsafe { self.relabel() }; + buffer.shared_access_writes = false; + buffer } /// Hand this allocation to a `DeviceBuffer` of the same length. @@ -372,16 +434,13 @@ impl DeviceBuffer { std::mem::align_of::(), "element casts require the same element alignment" ); - let (ptr, len, ctx, dealloc_stream) = self.into_all_raw_parts(); - // SAFETY: `ptr` came from a valid `DeviceBuffer` allocation of `len` - // elements; `A` has identical size and alignment, so the same allocation - // is a valid `DeviceBuffer` of the same length and the same byte - // extent, and the caller guarantees the values are valid `A`s. - // Ownership transfers; the original buffer's `Drop` is suppressed by - // `into_all_raw_parts`, and the allocation metadata is preserved for - // the new element type. - unsafe { - DeviceBuffer::::from_raw_parts_with_dealloc_stream(ptr, len, ctx, dealloc_stream) + DeviceBuffer { + storage: self.storage, + shared_access_writes: true, + ptr: self.ptr, + len: self.len, + num_bytes: self.num_bytes, + _marker: PhantomData, } } @@ -440,14 +499,13 @@ impl DeviceBuffer { ) else { return Err(self); }; - let (ptr, _len, ctx, dealloc_stream) = self.into_all_raw_parts(); - // SAFETY: `ptr` came from a valid allocation of `bytes` bytes, checked - // above to be exactly `new_len` elements of `A` and to be aligned for - // `A`. The allocation and its ownership are unchanged; only the element - // type and the count describing the same bytes change. `Drop` on the - // original is suppressed by `into_all_raw_parts`. - Ok(unsafe { - DeviceBuffer::::from_raw_parts_with_dealloc_stream(ptr, new_len, ctx, dealloc_stream) + Ok(DeviceBuffer { + storage: self.storage, + shared_access_writes: false, + ptr: self.ptr, + len: new_len, + num_bytes: self.num_bytes, + _marker: PhantomData, }) } @@ -469,6 +527,28 @@ impl DeviceBuffer { } impl DeviceBuffer { + fn empty_copy(len: usize, ctx: Arc) -> Self { + // SAFETY: a null pointer with zero bytes is never dereferenced + // and Drop/drop_async ignore it. DeviceCopy excludes interior mutability. + let mut buf = unsafe { Self::from_raw_parts(0, len, ctx) }; + buf.shared_access_writes = false; + buf + } + + /// Constructs a plain-value typed view of shared byte storage. + /// + /// # Safety + /// The initialization, validity, and aliasing requirements of + /// `from_storage` apply. DeviceCopy additionally excludes interior mutability. + pub unsafe fn from_storage_copy( + storage: crate::RawDeviceBuffer, + len: usize, + ) -> Result { + let mut buffer = unsafe { Self::from_storage(storage, len)? }; + buffer.shared_access_writes = false; + Ok(buffer) + } + /// Allocates device memory, copies `data` from the host on `stream`, and /// synchronizes `stream` before returning. /// @@ -487,7 +567,7 @@ impl DeviceBuffer { /// stream synchronization run. If either step fails, the early return /// drops the buffer and its `Drop` impl frees the allocation, so no /// device memory is leaked. - pub fn from_host(stream: &CudaStream, data: &[T]) -> Result { + pub fn from_host(stream: &Stream, data: &[T]) -> Result { let ctx = stream.context().clone(); let len = data.len(); let num_bytes = allocation_size::(len)?; @@ -495,27 +575,22 @@ impl DeviceBuffer { // cuMemAlloc rejects zero-byte requests with CUDA_ERROR_INVALID_VALUE, // so represent an empty buffer as a null pointer (Drop skips it). if num_bytes == 0 { - // SAFETY: a null pointer with zero bytes is never dereferenced - // and Drop ignores it. - return Ok(unsafe { Self::from_raw_parts(0, len, ctx) }); + return Ok(Self::empty_copy(len, ctx)); } - let ptr = unsafe { crate::simt::memory::malloc_sync(num_bytes)? }; + ctx.bind_to_thread()?; + let ptr = unsafe { crate::memory::malloc_sync(num_bytes)? }; // SAFETY: `ptr` was just allocated with `num_bytes` bytes in the // stream's context; ownership transfers to `buf` here so any early // return below frees it through the buffer's own `Drop`. - let buf = unsafe { Self::from_raw_parts(ptr, len, ctx) }; + let mut buf = unsafe { Self::from_raw_parts(ptr, len, ctx) }; let enqueue_result = unsafe { - crate::simt::memory::memcpy_htod_async( - buf.ptr, - data.as_ptr(), - num_bytes, - stream.cu_stream(), - ) + crate::memory::memcpy_htod_async(buf.ptr, data.as_ptr(), num_bytes, stream.cu_stream()) }; let sync_result = stream.synchronize(); enqueue_result?; sync_result?; + buf.shared_access_writes = false; Ok(buf) } @@ -528,9 +603,9 @@ impl DeviceBuffer { /// still be reading from `data` after the borrow is released. The caller /// must ensure `data` is not dropped, freed, mutated, or aliased until the /// enqueued copy has completed, typically after the next - /// [`CudaStream::synchronize`] call or a stream-ordered event wait. + /// [`Stream::synchronize`] call or a stream-ordered event wait. pub unsafe fn from_host_async_unchecked( - stream: &CudaStream, + stream: &Stream, data: &[T], ) -> Result { let ctx = stream.context().clone(); @@ -540,24 +615,19 @@ impl DeviceBuffer { // cuMemAlloc rejects zero-byte requests with CUDA_ERROR_INVALID_VALUE, // so represent an empty buffer as a null pointer (Drop skips it). if num_bytes == 0 { - // SAFETY: a null pointer with zero bytes is never dereferenced - // and Drop ignores it. - return Ok(unsafe { Self::from_raw_parts(0, len, ctx) }); + return Ok(Self::empty_copy(len, ctx)); } - let ptr = unsafe { crate::simt::memory::malloc_sync(num_bytes)? }; + ctx.bind_to_thread()?; + let ptr = unsafe { crate::memory::malloc_sync(num_bytes)? }; // SAFETY: `ptr` was just allocated with `num_bytes` bytes in the // stream's context; ownership transfers to `buf` here so any early // return below frees it through the buffer's own `Drop`. - let buf = unsafe { Self::from_raw_parts(ptr, len, ctx) }; - unsafe { - crate::simt::memory::memcpy_htod_async( - buf.ptr, - data.as_ptr(), - num_bytes, - stream.cu_stream(), - )?; - } + let mut buf = unsafe { Self::from_raw_parts(ptr, len, ctx) }; + buf.enqueue_access(stream, true, || unsafe { + crate::memory::memcpy_htod_async(buf.ptr, data.as_ptr(), num_bytes, stream.cu_stream()) + })?; + buf.shared_access_writes = false; Ok(buf) } @@ -570,8 +640,8 @@ impl DeviceBuffer { /// /// `PinnedHostBuffer` currently uses `cuMemAllocHost` without the /// `PORTABLE` flag, so the allocation is only pinned in the context that - /// created it. In debug builds this asserts that `data` and `stream` - /// share the same [`CudaContext`]. + /// created it. This checks that `data` and `stream` share the same + /// CUDA context. /// /// The device-to-host counterparts are [`Self::copy_to_pinned_host`] /// (blocking) and [`Self::copy_to_pinned_host_async`] (non-blocking). To @@ -585,18 +655,19 @@ impl DeviceBuffer { /// after this function returns. The caller is responsible for ensuring /// `data` is not dropped, freed, mutated, or aliased until the enqueued /// copy has completed, typically after the next - /// [`CudaStream::synchronize`] call or a stream-ordered event wait. + /// [`Stream::synchronize`] call or a stream-ordered event wait. /// Dropping `data` before that synchronization point calls /// `cuMemFreeHost` while the in-flight transfer is still reading the /// buffer, which is undefined behavior. pub unsafe fn from_pinned_host( - stream: &CudaStream, + stream: &Stream, data: &PinnedHostBuffer, ) -> Result { - debug_assert!( - Arc::ptr_eq(data.context(), stream.context()), - "pinned host buffer and stream must belong to the same CUDA context" - ); + if data.context().as_ref() != stream.context().as_ref() { + return Err(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_CONTEXT, + )); + } // SAFETY: this method's safety contract requires the caller to keep // the pinned source valid until the enqueued copy completes. unsafe { Self::from_host_async_unchecked(stream, data.as_slice()) } @@ -619,27 +690,27 @@ impl DeviceBuffer { /// `memset_d8_async` enqueue and stream synchronization run. If either /// fails, the early return drops the buffer and its `Drop` impl frees the /// allocation, so no device memory is leaked. - pub fn zeroed(stream: &CudaStream, len: usize) -> Result { + pub fn zeroed(stream: &Stream, len: usize) -> Result { let ctx = stream.context().clone(); let num_bytes = allocation_size::(len)?; // cuMemAlloc rejects zero-byte requests with CUDA_ERROR_INVALID_VALUE, // so represent an empty buffer as a null pointer (Drop skips it). if num_bytes == 0 { - // SAFETY: a null pointer with zero bytes is never dereferenced - // and Drop ignores it. - return Ok(unsafe { Self::from_raw_parts(0, len, ctx) }); + return Ok(Self::empty_copy(len, ctx)); } - let ptr = unsafe { crate::simt::memory::malloc_sync(num_bytes)? }; + ctx.bind_to_thread()?; + let ptr = unsafe { crate::memory::malloc_sync(num_bytes)? }; // SAFETY: `ptr` was just allocated with `num_bytes` bytes in the // stream's context; ownership transfers to `buf` here so any early // return below frees it through the buffer's own `Drop`. - let buf = unsafe { Self::from_raw_parts(ptr, len, ctx) }; + let mut buf = unsafe { Self::from_raw_parts(ptr, len, ctx) }; unsafe { - crate::simt::memory::memset_d8_async(buf.ptr, 0, num_bytes, stream.cu_stream())?; + crate::memory::memset_d8_async(buf.ptr, 0, num_bytes, stream.cu_stream())?; } stream.synchronize()?; + buf.shared_access_writes = false; Ok(buf) } @@ -647,18 +718,35 @@ impl DeviceBuffer { /// /// Synchronizes on `stream` before returning so the host vector is safe /// to read immediately. - pub fn to_host_vec(&self, stream: &CudaStream) -> Result, DriverError> { - let mut host = Vec::with_capacity(self.len); - unsafe { - crate::simt::memory::memcpy_dtoh_async( + pub fn to_host_vec(&self, stream: &Stream) -> Result, DriverError> { + let mut host = Vec::::with_capacity(self.len); + if self.num_bytes() == 0 { + // SAFETY: empty buffers or zero-sized DeviceCopy values need no bytes. + unsafe { + host.set_len(self.len); + } + return Ok(host); + } + let lease = self.storage.acquire(stream, false, false)?; + self.storage.context().bind_to_thread()?; + let submitted = unsafe { + crate::memory::memcpy_dtoh_async( host.as_mut_ptr(), self.ptr, self.num_bytes(), stream.cu_stream(), - )?; + ) + }; + let completed = stream.synchronize(); + if let Err(error) = completed { + // The driver has not established that it stopped writing host memory. + std::mem::forget((host, lease)); + return Err(error); + } + submitted?; + unsafe { + host.set_len(self.len); } - stream.synchronize()?; - unsafe { host.set_len(self.len) }; Ok(host) } @@ -666,22 +754,26 @@ impl DeviceBuffer { /// /// Synchronizes on `stream` before returning. Panics if /// `dst.len() < self.len()`. - pub fn copy_to_host(&self, stream: &CudaStream, dst: &mut [T]) -> Result<(), DriverError> { - assert!( - dst.len() >= self.len, - "destination slice too small: {} < {}", - dst.len(), - self.len - ); - unsafe { - crate::simt::memory::memcpy_dtoh_async( + pub fn copy_to_host(&self, stream: &Stream, dst: &mut [T]) -> Result<(), DriverError> { + assert!(dst.len() >= self.len(), "destination host slice too small"); + if self.num_bytes() == 0 { + return Ok(()); + } + let lease = self.storage.acquire(stream, false, false)?; + self.storage.context().bind_to_thread()?; + let submitted = unsafe { + crate::memory::memcpy_dtoh_async( dst.as_mut_ptr(), self.ptr, self.num_bytes(), stream.cu_stream(), - )?; + ) + }; + let completed = stream.synchronize(); + if completed.is_err() { + std::mem::forget(lease); } - stream.synchronize() + submitted.and(completed) } /// Copies the buffer contents into an existing pinned host buffer and @@ -695,7 +787,7 @@ impl DeviceBuffer { /// synchronize the stream later. pub fn copy_to_pinned_host( &self, - stream: &CudaStream, + stream: &Stream, dst: &mut PinnedHostBuffer, ) -> Result<(), DriverError> { // SAFETY: we synchronize the stream below before returning, so the @@ -713,7 +805,7 @@ impl DeviceBuffer { /// `PinnedHostBuffer` currently uses `cuMemAllocHost` without the /// `PORTABLE` flag, so the allocation is only pinned in the context that /// created it. In debug builds this asserts that `dst` and `stream` - /// share the same [`CudaContext`]. + /// share the same [`Context`]. /// /// # Safety /// @@ -721,33 +813,35 @@ impl DeviceBuffer { /// returns; CUDA may still be writing into `dst`'s pinned pointer long /// after this function returns. The caller is responsible for ensuring /// `dst` is not dropped, freed, read, or aliased until the enqueued copy - /// has completed, typically after the next [`CudaStream::synchronize`] + /// has completed, typically after the next [`Stream::synchronize`] /// call or a stream-ordered event wait. Dropping `dst` before that /// synchronization point calls `cuMemFreeHost` while the in-flight /// transfer is still writing the buffer, which is undefined behavior. pub unsafe fn copy_to_pinned_host_async( &self, - stream: &CudaStream, + stream: &Stream, dst: &mut PinnedHostBuffer, ) -> Result<(), DriverError> { - debug_assert!( - Arc::ptr_eq(dst.context(), stream.context()), - "pinned host buffer and stream must belong to the same CUDA context" - ); assert!( - dst.len() >= self.len, - "destination pinned host buffer too small: {} < {}", - dst.len(), - self.len + dst.len() >= self.len(), + "destination pinned host buffer too small" ); - unsafe { - crate::simt::memory::memcpy_dtoh_async( + if dst.context().as_ref() != self.storage.context().as_ref() { + return Err(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_CONTEXT, + )); + } + if self.num_bytes() == 0 { + return Ok(()); + } + self.enqueue_access(stream, false, || unsafe { + crate::memory::memcpy_dtoh_async( dst.as_mut_ptr(), self.ptr, self.num_bytes(), stream.cu_stream(), ) - } + }) } /// Enqueues a host-to-device copy from a pinned host buffer into this @@ -763,8 +857,8 @@ impl DeviceBuffer { /// /// `PinnedHostBuffer` currently uses `cuMemAllocHost` without the /// `PORTABLE` flag, so the allocation is only pinned in the context that - /// created it. In debug builds this asserts that `src` and `stream` - /// share the same [`CudaContext`]. + /// created it. This checks that `src` and `stream` share the same + /// CUDA context. /// /// # Safety /// @@ -773,34 +867,35 @@ impl DeviceBuffer { /// after this function returns. The caller is responsible for ensuring /// `src` is not dropped, freed, mutated, or aliased until the enqueued /// copy has completed, typically after the next - /// [`CudaStream::synchronize`] call or a stream-ordered event wait. + /// [`Stream::synchronize`] call or a stream-ordered event wait. /// Dropping `src` before that synchronization point calls /// `cuMemFreeHost` while the in-flight transfer is still reading the /// buffer, which is undefined behavior. pub unsafe fn copy_from_pinned_host_async( &mut self, - stream: &CudaStream, + stream: &Stream, src: &PinnedHostBuffer, ) -> Result<(), DriverError> { - debug_assert!( - Arc::ptr_eq(src.context(), stream.context()), - "pinned host buffer and stream must belong to the same CUDA context" - ); assert!( - src.len() <= self.len, - "source pinned host buffer too large: {} > {}", - src.len(), - self.len + src.len() <= self.len(), + "source pinned host buffer too large" ); - let num_bytes = src.num_bytes(); - unsafe { - crate::simt::memory::memcpy_htod_async( + if src.context().as_ref() != self.storage.context().as_ref() { + return Err(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_CONTEXT, + )); + } + if src.num_bytes() == 0 { + return Ok(()); + } + self.enqueue_access(stream, true, || unsafe { + crate::memory::memcpy_htod_async( self.ptr, src.as_ptr(), - num_bytes, + src.num_bytes(), stream.cu_stream(), ) - } + }) } /// Allocates `len` elements of uninitialized device memory, enqueued on @@ -810,35 +905,30 @@ impl DeviceBuffer { /// the returned buffer are undefined until the caller writes them. /// /// The buffer co-owns `stream` (via the `Arc`) so its implicit `Drop` can - /// release the stream-ordered allocation after synchronizing the context. - /// Call the unsafe [`Self::drop_async`] to free explicitly on a chosen - /// stream without a context-wide synchronization. + /// retire the stream-ordered allocation after tracked accesses complete. + /// Unsafe callers must establish allocation and use ordering. /// /// # Safety /// /// Reading from the returned buffer before any kernel or memcpy has /// written it is undefined behavior. pub unsafe fn uninitialized_async( - stream: &Arc, + stream: &Arc, len: usize, ) -> Result { let ctx = stream.context().clone(); let num_bytes = allocation_size::(len)?; if num_bytes == 0 { - // SAFETY: a null pointer with zero bytes is never dereferenced - // and Drop/drop_async ignore it. - return Ok(unsafe { Self::from_raw_parts(0, len, ctx) }); + return Ok(Self::empty_copy(len, ctx)); } - let ptr = unsafe { crate::simt::memory::malloc_async(stream.cu_stream(), num_bytes)? }; - Ok(Self { - ptr, - len, - num_bytes, - ctx, - dealloc_stream: Some(stream.clone()), - _marker: PhantomData, - }) + ctx.bind_to_thread()?; + let ptr = unsafe { crate::memory::malloc_async(stream.cu_stream(), num_bytes)? }; + let mut buffer = unsafe { + Self::from_raw_parts_with_dealloc_stream(ptr, len, ctx, Some(stream.clone())) + }; + buffer.shared_access_writes = false; + Ok(buffer) } /// Copies `other` into `self` device-to-device, enqueued on `stream`. @@ -847,24 +937,39 @@ impl DeviceBuffer { pub fn copy_from_device_async( &mut self, other: &DeviceBuffer, - stream: &CudaStream, + stream: &Stream, ) -> Result<(), DriverError> { - assert_eq!( - self.len, other.len, - "device-to-device copy length mismatch: dst {} != src {}", - self.len, other.len - ); + assert_eq!(self.len, other.len, "device-to-device copy length mismatch"); if self.num_bytes() == 0 { return Ok(()); } - unsafe { - crate::simt::memory::memcpy_dtod_async( - self.ptr, - other.ptr, - self.num_bytes(), - stream.cu_stream(), - ) + if self.storage.context().as_ref() != stream.context().as_ref() + || other.storage.context().as_ref() != self.storage.context().as_ref() + { + return Err(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_CONTEXT, + )); + } + if self.ptr == other.ptr { + return Ok(()); + } + if self.ptr < other.ptr + other.num_bytes as u64 + && other.ptr < self.ptr + self.num_bytes as u64 + { + return Err(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE, + )); } + self.enqueue_access(stream, true, || { + other.enqueue_access(stream, false, || unsafe { + crate::memory::memcpy_dtod_async( + self.ptr, + other.ptr, + self.num_bytes(), + stream.cu_stream(), + ) + }) + }) } /// Copies `src` into `self` host-to-device on `stream` and synchronizes @@ -873,7 +978,7 @@ impl DeviceBuffer { /// The synchronization keeps this safe for borrowed host slices: `src` /// may be dropped, reused, or mutated immediately after this function /// returns. Panics if `src.len() != self.len()`. - pub fn copy_from_host(&mut self, stream: &CudaStream, src: &[T]) -> Result<(), DriverError> { + pub fn copy_from_host(&mut self, stream: &Stream, src: &[T]) -> Result<(), DriverError> { // SAFETY: this safe wrapper synchronizes `stream` before returning, // so the borrowed host slice cannot be used by CUDA after this call. let enqueue_result = unsafe { self.copy_from_host_async_unchecked(stream, src) }; @@ -895,31 +1000,25 @@ impl DeviceBuffer { /// still be reading from `src` after the borrow is released. The caller /// must ensure `src` is not dropped, freed, mutated, or aliased until the /// enqueued copy has completed, typically after the next - /// [`CudaStream::synchronize`] call or a stream-ordered event wait. + /// [`Stream::synchronize`] call or a stream-ordered event wait. /// Panics if `src.len() != self.len()`. pub unsafe fn copy_from_host_async_unchecked( &mut self, - stream: &CudaStream, + stream: &Stream, src: &[T], ) -> Result<(), DriverError> { - assert_eq!( - self.len, - src.len(), - "host-to-device copy length mismatch: dst {} != src {}", - self.len, - src.len() - ); + assert_eq!(self.len, src.len(), "host-to-device copy length mismatch"); if self.num_bytes() == 0 { return Ok(()); } - unsafe { - crate::simt::memory::memcpy_htod_async( + self.enqueue_access(stream, true, || unsafe { + crate::memory::memcpy_htod_async( self.ptr, src.as_ptr(), self.num_bytes(), stream.cu_stream(), ) - } + }) } /// Consumes the buffer and frees it asynchronously on `stream`. @@ -939,42 +1038,33 @@ impl DeviceBuffer { /// /// The caller must ensure that every stream other than the allocation /// stream that has pending work touching this buffer is ordered before - /// `stream` (for example via [`CudaStream::join`]). Only the allocation + /// `stream` (for example via [`Stream::join`]). Only the allocation /// stream is joined automatically; an unordered third stream still /// racing the free is a driver-level use-after-free. This is the same /// deferred-use contract as [`Self::copy_from_host_async_unchecked`]: - /// ordinary [`Drop`] is the safe alternative and synchronizes the whole - /// context first. - pub unsafe fn drop_async(mut self, stream: &CudaStream) -> Result<(), DriverError> { - if self.ctx.as_ref() != stream.context().as_ref() { + /// ordinary [`Drop`] relies on completion leases acquired by safe APIs. + /// Unsafe raw-pointer launches must retain their own completion leases. + pub unsafe fn drop_async(self, stream: &Stream) -> Result<(), DriverError> { + if self.context().as_ref() != stream.context().as_ref() { return Err(DriverError( cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_CONTEXT, )); } - if self.ptr == 0 { - return Ok(()); + stream.context().bind_to_thread()?; + if let Some(allocation_stream) = self.storage.allocation_stream() { + stream.join(allocation_stream)?; } - - self.ctx.bind_to_thread()?; - if let Some(allocation_stream) = &self.dealloc_stream { - if allocation_stream.as_ref() != stream { - stream.join(allocation_stream)?; - } - } - - let ptr = self.ptr; - self.ptr = 0; - unsafe { crate::simt::memory::free_async(ptr, stream.cu_stream()) } + crate::completion::retain_until_complete(self.storage, stream) } /// Zeroes every byte in the buffer asynchronously on `stream`. - pub fn zero_async(&mut self, stream: &CudaStream) -> Result<(), DriverError> { + pub fn zero_async(&mut self, stream: &Stream) -> Result<(), DriverError> { if self.num_bytes() == 0 { return Ok(()); } - unsafe { - crate::simt::memory::memset_d8_async(self.ptr, 0, self.num_bytes(), stream.cu_stream()) - } + self.enqueue_access(stream, true, || unsafe { + crate::memory::memset_d8_async(self.ptr, 0, self.num_bytes(), stream.cu_stream()) + }) } } @@ -1003,6 +1093,69 @@ fn chunk_cast_len(bytes: usize, addr: usize, elem_size: usize, align: usize) -> Some(bytes / elem_size) } +/// A borrowed contiguous typed view sharing its parent's allocation and access tracker. +pub struct DeviceBufferView<'a, T> { + storage: crate::RawDeviceBuffer, + shared_access_writes: bool, + len: usize, + _borrow: PhantomData<&'a DeviceBuffer>, +} +impl DeviceBufferView<'_, T> { + pub fn cu_deviceptr(&self) -> CUdeviceptr { + self.storage.cu_deviceptr() + } + pub fn len(&self) -> usize { + self.len + } + pub fn is_empty(&self) -> bool { + self.len == 0 + } + pub fn num_bytes(&self) -> usize { + self.storage.len_bytes() + } + pub fn context(&self) -> &Arc { + self.storage.context() + } + pub fn storage(&self) -> &crate::RawDeviceBuffer { + &self.storage + } + /// Whether a shared kernel reference may mutate elements through interior + /// mutability. Representation wrappers and unclassified raw types are + /// conservative; ordinary DeviceCopy values can share read accesses. + #[doc(hidden)] + pub fn shared_access_writes(&self) -> bool { + self.shared_access_writes + } +} + +impl<'a, T: DeviceCopy> DeviceBufferView<'a, T> { + /// Borrows initialized elements from shared byte storage without a copy. + /// + /// # Safety + /// The range must contain valid `T` values. All aliases must preserve that + /// validity and use the same access tracker for every device access. + pub unsafe fn borrow_storage( + storage: &'a crate::RawDeviceBuffer, + offset: usize, + len: usize, + ) -> Result { + let storage = storage.slice_bytes(offset, allocation_size::(len)?)?; + if !storage.is_empty() + && !(storage.cu_deviceptr() as usize).is_multiple_of(std::mem::align_of::()) + { + return Err(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE, + )); + } + Ok(Self { + storage, + len, + shared_access_writes: false, + _borrow: PhantomData, + }) + } +} + #[cfg(test)] mod chunk_cast_tests { use super::chunk_cast_len; diff --git a/cuda-core/src/simt/embedded.rs b/cuda-core/src/embedded.rs similarity index 96% rename from cuda-core/src/simt/embedded.rs rename to cuda-core/src/embedded.rs index c146f58753..1bb8565184 100644 --- a/cuda-core/src/simt/embedded.rs +++ b/cuda-core/src/embedded.rs @@ -5,7 +5,7 @@ //! Loading CUDA modules from embedded device artifact bundles. -use crate::{CudaContext, CudaModule, DriverError}; +use crate::{Context, DriverError, Module}; use oxide_artifacts::ArtifactError; pub use oxide_artifacts::{ ArtifactCompileOptions, ArtifactDebugPolicy, ArtifactPayloadKind, OwnedArtifactBundle, @@ -43,7 +43,7 @@ impl EmbeddedModule { self.bundle.payload(kind) } - pub fn load(&self, ctx: &Arc) -> Result, EmbeddedModuleError> { + pub fn load(&self, ctx: &Arc) -> Result, EmbeddedModuleError> { let image = loadable_payload(&self.bundle).expect("EmbeddedModule always has a loadable payload"); ctx.load_module_from_image(image) @@ -78,9 +78,9 @@ pub fn embedded_modules_from_current_exe() -> Result, Embedd } pub fn load_embedded_module( - ctx: &Arc, + ctx: &Arc, name: &str, -) -> Result, EmbeddedModuleError> { +) -> Result, EmbeddedModuleError> { let module = embedded_modules_from_current_exe()? .into_iter() .find(|module| module.name() == name) @@ -90,9 +90,7 @@ pub fn load_embedded_module( module.load(ctx) } -pub fn load_first_embedded_module( - ctx: &Arc, -) -> Result, EmbeddedModuleError> { +pub fn load_first_embedded_module(ctx: &Arc) -> Result, EmbeddedModuleError> { let module = embedded_modules_from_current_exe()? .into_iter() .next() diff --git a/cuda-core/src/event.rs b/cuda-core/src/event.rs new file mode 100644 index 0000000000..93de0bddc2 --- /dev/null +++ b/cuda-core/src/event.rs @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Event timing and configuration helpers. + +pub use crate::runtime::{Event, Event as CudaEvent}; +use crate::{Context, DriverError, IntoResult}; +use std::{mem::MaybeUninit, sync::Arc}; + +impl Context { + /// Creates a new CUDA event in this context. + /// + /// `flags` defaults to `CU_EVENT_DISABLE_TIMING` when `None`. Use + /// `Some(CU_EVENT_DEFAULT)` to enable timing queries via + /// [`Event::elapsed_ms`]. + pub fn new_event_with_flags( + self: &Arc, + flags: Option, + ) -> Result { + let flags = flags.unwrap_or(cuda_bindings::CUevent_flags_enum_CU_EVENT_DISABLE_TIMING); + self.bind_to_thread()?; + let mut cu_event = MaybeUninit::uninit(); + let cu_event = unsafe { + cuda_bindings::cuEventCreate(cu_event.as_mut_ptr(), flags).result()?; + cu_event.assume_init() + }; + Ok(Event { + cu_event, + context: self.clone(), + }) + } +} +impl Event { + /// Returns the raw `CUevent` handle. + pub fn cu_event(&self) -> cuda_bindings::CUevent { + self.cu_event + } + /// Returns the parent [`Context`]. + pub fn context(&self) -> &Arc { + &self.context + } + /// Returns the elapsed time in milliseconds between `self` (start) and + /// `end`. + /// + /// Both events are synchronized before querying. Both events must have + /// been created **without** `CU_EVENT_DISABLE_TIMING`; otherwise the + /// driver returns `CUDA_ERROR_INVALID_HANDLE`. + /// + /// `self` must have been recorded before `end` in wall-clock time. + pub fn elapsed_ms(&self, end: &Self) -> Result { + self.synchronize()?; + end.synchronize()?; + let mut ms: f32 = 0.0; + unsafe { + cuda_bindings::cu_event_elapsed_time(&mut ms as *mut _, self.cu_event, end.cu_event) + .result()?; + } + Ok(ms) + } +} diff --git a/cuda-core/src/simt/launch.rs b/cuda-core/src/launch.rs similarity index 99% rename from cuda-core/src/simt/launch.rs rename to cuda-core/src/launch.rs index ff1f37684b..a3a870ebc8 100644 --- a/cuda-core/src/simt/launch.rs +++ b/cuda-core/src/launch.rs @@ -10,7 +10,7 @@ //! proving that it matches a kernel is unsafe. Use a rank-preserving config and //! [`PreparedLaunch`] for the checked path. -use crate::{CudaFunction, CudaStream, DriverError}; +use crate::{DriverError, Function, Stream}; use std::error::Error; use std::fmt::{self, Display, Formatter}; use std::marker::PhantomData; @@ -867,7 +867,7 @@ impl From for LaunchContractError { /// value performs no contract query; it only compares the stream's context /// handle before the normal launch path binds that context. pub struct PreparedLaunch { - function: CudaFunction, + function: Function, config: LaunchConfig, _contract: PhantomData C>, } @@ -897,7 +897,7 @@ impl PreparedLaunch { /// relationship. #[doc(hidden)] pub unsafe fn __prepare( - function: CudaFunction, + function: Function, config: C::Config, ) -> Result { let raw = config.__raw(); @@ -1043,7 +1043,7 @@ impl PreparedLaunch { } /// Returns the prepared CUDA function. - pub fn function(&self) -> &CudaFunction { + pub fn function(&self) -> &Function { &self.function } @@ -1057,7 +1057,7 @@ impl PreparedLaunch { } /// Rejects a stream from a different context without making a driver call. - pub fn validate_stream(&self, stream: &CudaStream) -> Result<(), LaunchContractError> { + pub fn validate_stream(&self, stream: &Stream) -> Result<(), LaunchContractError> { let function_context = self.function.context(); let stream_context = stream.context(); if function_context.cu_ctx() == stream_context.cu_ctx() { diff --git a/cuda-core/src/simt/mod.rs b/cuda-core/src/launch_raw.rs similarity index 80% rename from cuda-core/src/simt/mod.rs rename to cuda-core/src/launch_raw.rs index 7f07b4c361..53c0b0f1fc 100644 --- a/cuda-core/src/simt/mod.rs +++ b/cuda-core/src/launch_raw.rs @@ -3,83 +3,20 @@ * SPDX-License-Identifier: Apache-2.0 */ -//! The cuda-oxide host surface. -//! -//! Everything under this module is the cuda-oxide host API, brought over -//! as-is for the shared host-crate migration rather than reconciled: -//! [`CudaContext`] lands next to this crate's own `Device`, and -//! [`LaunchConfig`] next to this crate's own `LaunchConfig`. The two surfaces -//! get reconciled after the repository migration, with both models in one -//! tree, and this module is deletable as a unit once that happens. -//! -//! Naming: everything cuda-oxide exposed at its crate root is re-exported -//! here, so `cuda_core::simt::X` is oxide's `cuda_core::X`. The crate root -//! additionally re-exports every name that does not collide with the -//! existing surface; the one collision is [`LaunchConfig`], which is -//! reachable only through this module. -//! -//! Differences from the cuda-oxide original, recorded for the -//! reconciliation: -//! -//! - `DeviceCopy` for the unstable `f16` primitive sits behind the -//! default-off `f16` cargo feature, since it is nightly-only; the -//! `half::f16` and `half::bf16` impls are always present. -//! - `#[derive(DeviceCopy)]` comes from the sibling `cuda-core-derive` -//! crate, extracted from cuda-oxide's `cuda-macros` so it is publishable. -//! - `vmm` and `peer` are copied like the rest; #202 remains the idiomatic -//! port of the VMM wrappers onto this crate's own runtime types. -//! - `init` and `launch_kernel` are not copied: the crate root already -//! exposes identical ones, re-exported below. -//! - `oxide-artifacts` comes from crates.io (cuda-oxide publishes it) and is -//! re-exported as [`artifacts`]. +//! Context-aware raw CUDA launch helpers. -pub mod context; -pub mod device_buffer; -pub mod embedded; -pub mod event; -pub mod launch; -pub mod memory; -pub mod module; -pub mod peer; -pub mod pinned_host_buffer; -pub mod stream; -pub mod vmm; +use crate::{launch_kernel, DriverError, Function, IntoResult, Stream}; -pub use context::{ContextLimit, CudaContext, StreamPriorityRange, SyncPolicy}; -/// `#[derive(DeviceCopy)]`, re-exported next to the trait so -/// `use cuda_core::DeviceCopy;` brings both into scope (serde pattern). -pub use cuda_core_derive::DeviceCopy; -pub use device_buffer::{DeviceBuffer, DeviceCopy}; -pub use embedded::{EmbeddedModule, EmbeddedModuleError}; -pub use event::CudaEvent; -pub use launch::{ - BlockRequirement, DeviceLaunchLimits, DynamicSharedMemoryRequirement, KernelLaunchConfig, - KernelLaunchContract, LaunchAxis, LaunchConfig, LaunchConfig1D, LaunchConfig2D, LaunchConfig3D, - LaunchContractError, LaunchContractSpec, LaunchDimension, PreparedLaunch, -}; -pub use module::{ConstantHandle, CudaFunction, CudaModule}; -pub use pinned_host_buffer::PinnedHostBuffer; -pub use stream::CudaStream; - -// The remaining items of oxide's crate root are identical on both surfaces; -// they are re-exported so `cuda_core::simt` mirrors it completely. -pub use crate::error::{DriverError, IntoResult}; -pub use crate::init; -pub use crate::launch_kernel; -pub use cuda_bindings as sys; -/// Artifact-bundle metadata, from the published crate cuda-oxide releases. -pub use oxide_artifacts as artifacts; - -/// Launches a CUDA kernel on a specific [`CudaStream`], binding its context first. +/// Launches a CUDA kernel on a specific [`Stream`], binding its context first. /// /// This is the usual host-side helper for `cuda_launch!` and async launches. /// It ensures the stream's owning context is current before calling the raw /// [`launch_kernel`] entry point, so callers do not need to manually call -/// [`CudaContext::bind_to_thread`] before every launch. +/// [`crate::Context::bind_to_thread`] before every launch. /// /// Unlike [`launch_kernel`], this helper works with typed wrappers rather than /// raw driver handles. It is therefore the preferred API whenever you already -/// have a [`CudaFunction`] and [`CudaStream`]. +/// have a [`Function`] and [`Stream`]. /// /// # Safety /// @@ -100,11 +37,11 @@ pub use oxide_artifacts as artifacts; /// `cuLaunchKernel` call rejects the launch. #[inline] pub unsafe fn launch_kernel_on_stream( - func: &CudaFunction, + func: &Function, grid_dim: (u32, u32, u32), block_dim: (u32, u32, u32), shared_mem_bytes: u32, - stream: &CudaStream, + stream: &Stream, kernel_params: &mut [*mut std::ffi::c_void], ) -> Result<(), DriverError> { stream.context().bind_to_thread()?; @@ -218,12 +155,12 @@ pub unsafe fn launch_kernel_ex( /// `cuLaunchKernelEx` call rejects the launch. #[inline] pub unsafe fn launch_kernel_ex_on_stream( - func: &CudaFunction, + func: &Function, grid_dim: (u32, u32, u32), block_dim: (u32, u32, u32), shared_mem_bytes: u32, cluster_dim: (u32, u32, u32), - stream: &CudaStream, + stream: &Stream, kernel_params: &mut [*mut std::ffi::c_void], ) -> Result<(), DriverError> { stream.context().bind_to_thread()?; @@ -330,11 +267,11 @@ pub unsafe fn launch_kernel_cooperative( /// `cuLaunchKernelEx` call rejects the launch. #[inline] pub unsafe fn launch_kernel_cooperative_on_stream( - func: &CudaFunction, + func: &Function, grid_dim: (u32, u32, u32), block_dim: (u32, u32, u32), shared_mem_bytes: u32, - stream: &CudaStream, + stream: &Stream, kernel_params: &mut [*mut std::ffi::c_void], ) -> Result<(), DriverError> { stream.context().bind_to_thread()?; @@ -447,12 +384,12 @@ pub unsafe fn launch_kernel_ex_cooperative( /// `cuLaunchKernelEx` call rejects the launch. #[inline] pub unsafe fn launch_kernel_ex_cooperative_on_stream( - func: &CudaFunction, + func: &Function, grid_dim: (u32, u32, u32), block_dim: (u32, u32, u32), shared_mem_bytes: u32, cluster_dim: (u32, u32, u32), - stream: &CudaStream, + stream: &Stream, kernel_params: &mut [*mut std::ffi::c_void], ) -> Result<(), DriverError> { stream.context().bind_to_thread()?; diff --git a/cuda-core/src/lib.rs b/cuda-core/src/lib.rs index 73cf8e529f..9d48e11132 100644 --- a/cuda-core/src/lib.rs +++ b/cuda-core/src/lib.rs @@ -7,12 +7,25 @@ #![cfg_attr(feature = "f16", feature(f16))] +pub mod allocation; mod api; +pub use allocation::{AllocationLease, DeviceAllocation, RawDeviceBuffer}; +pub mod completion; +pub mod context; pub(crate) mod cudarc_shim; +pub mod device_buffer; mod dtype; +pub mod embedded; mod error; +pub mod event; +pub mod launch; +mod launch_raw; +pub mod memory; +pub mod module; +pub mod peer; +pub mod pinned_host_buffer; mod runtime; -pub mod simt; +pub mod stream; pub mod vmm; pub use api::*; @@ -26,19 +39,18 @@ pub use dtype::*; pub use error::*; pub use runtime::*; -// The cuda-oxide surface, re-exported at the root where its consumers -// expect it. `simt::LaunchConfig` and `simt::vmm` are deliberately absent: -// they collide with this crate's own `LaunchConfig` and with the reviewed -// `vmm` module above (both forks of the same cuda-oxide ancestor), and stay -// reachable only through `simt::`. -pub use simt::embedded; -pub use simt::{ - launch_kernel_cooperative, launch_kernel_cooperative_on_stream, launch_kernel_ex, - launch_kernel_ex_cooperative, launch_kernel_ex_cooperative_on_stream, - launch_kernel_ex_on_stream, launch_kernel_on_stream, BlockRequirement, ConstantHandle, - ContextLimit, CudaContext, CudaEvent, CudaFunction, CudaModule, CudaStream, DeviceBuffer, - DeviceCopy, DeviceLaunchLimits, DynamicSharedMemoryRequirement, EmbeddedModule, - EmbeddedModuleError, KernelLaunchConfig, KernelLaunchContract, LaunchAxis, LaunchConfig1D, - LaunchConfig2D, LaunchConfig3D, LaunchContractError, LaunchContractSpec, LaunchDimension, - PinnedHostBuffer, PreparedLaunch, StreamPriorityRange, SyncPolicy, +pub use context::{ContextLimit, StreamPriorityRange, SyncPolicy}; +pub use cuda_core_derive::DeviceCopy; +pub use device_buffer::{DeviceBuffer, DeviceBufferView, DeviceCopy}; +pub use embedded::{EmbeddedModule, EmbeddedModuleError}; +pub use launch::*; +pub use launch_raw::*; +pub use module::ConstantHandle; +pub use oxide_artifacts as artifacts; +pub use pinned_host_buffer::PinnedHostBuffer; + +// Compatibility spellings are aliases of the shared handles, not separate runtimes. +pub use runtime::{ + Context as CudaContext, Event as CudaEvent, Function as CudaFunction, Module as CudaModule, + Stream as CudaStream, }; diff --git a/cuda-core/src/simt/memory.rs b/cuda-core/src/memory.rs similarity index 100% rename from cuda-core/src/simt/memory.rs rename to cuda-core/src/memory.rs diff --git a/cuda-core/src/simt/module.rs b/cuda-core/src/module.rs similarity index 64% rename from cuda-core/src/simt/module.rs rename to cuda-core/src/module.rs index a2b4cb646f..952a8a5f88 100644 --- a/cuda-core/src/simt/module.rs +++ b/cuda-core/src/module.rs @@ -3,98 +3,18 @@ * SPDX-License-Identifier: Apache-2.0 */ -//! CUDA module and function management (RAII, PTX/cubin loading). -//! -//! A [`CudaModule`] wraps a `CUmodule` loaded from PTX source or a cubin file. -//! [`CudaFunction`] extracts a kernel entry point from a loaded module by -//! symbol name. Both types are reference-counted and tie their lifetime to the -//! parent [`CudaContext`] / [`CudaModule`] respectively. -//! -//! # Typical workflow -//! -//! ```ignore -//! let ctx = CudaContext::new(0)?; -//! let module = ctx.load_module_from_ptx_src(ptx)?; -//! let kernel = module.load_function("my_kernel")?; -//! ``` -//! -//! # Raw CUDA interop -//! -//! Most users should load kernels with [`CudaModule::load_function`] and -//! launch them through cuda-oxide's typed launch helpers. Some CUDA-adjacent -//! libraries need the underlying driver handle to inspect or register -//! module-scope device state before launch. For those cases, -//! [`CudaModule::cu_module`] exposes a non-owning raw `CUmodule` handle under -//! an explicit `unsafe` contract. - -use crate::error::{DriverError, IntoResult}; -use crate::simt::context::CudaContext; -use std::borrow::Cow; -use std::ffi::{c_void, CString}; -use std::mem::MaybeUninit; -use std::sync::Arc; - -/// An RAII wrapper around a `CUmodule` handle. -/// -/// Holds an `Arc` to ensure the context outlives the module. -/// Unloaded automatically via `cuModuleUnload` on [`Drop`]. -#[derive(Debug)] -pub struct CudaModule { - /// Raw CUDA module handle. - pub(crate) cu_module: cuda_bindings::CUmodule, - /// Owning context. Kept alive for the lifetime of this module. - pub(crate) ctx: Arc, -} - -/// # Safety -/// -/// `CUmodule` handles are not thread-local. The CUDA driver permits querying -/// functions from a module on any thread, provided the owning context is bound. -unsafe impl Send for CudaModule {} -/// See [`Send`] impl. -unsafe impl Sync for CudaModule {} - -/// Unloads the module on drop. -/// -/// Binds the context to the current thread first (required by -/// `cuModuleUnload`). Errors are recorded on the context rather than -/// panicking. -impl Drop for CudaModule { - fn drop(&mut self) { - self.ctx.record_err(self.ctx.bind_to_thread()); - self.ctx - .record_err(unsafe { cuda_bindings::cuModuleUnload(self.cu_module).result() }); - } -} - -impl CudaContext { - /// JIT-compiles PTX source and loads the resulting module into this - /// context. - /// - /// `ptx_src` must be a valid, null-terminator-free PTX string. The driver - /// performs JIT compilation targeting the current device architecture. - /// - /// # Panics - /// - /// Panics if `ptx_src` contains interior null bytes. - pub fn load_module_from_ptx_src( - self: &Arc, - ptx_src: &str, - ) -> Result, DriverError> { - self.bind_to_thread()?; - let c_src = CString::new(ptx_src).unwrap(); - let cu_module = unsafe { - let mut cu_module = MaybeUninit::uninit(); - cuda_bindings::cuModuleLoadData(cu_module.as_mut_ptr(), c_src.as_ptr() as *const _) - .result()?; - cu_module.assume_init() - }; - Ok(Arc::new(CudaModule { - cu_module, - ctx: self.clone(), - })) - } - +//! Module globals and function resource queries. + +pub use crate::runtime::{Function, Module}; +use crate::{Context, DriverError, IntoResult}; +use std::{ + borrow::Cow, + ffi::{c_void, CString}, + mem::MaybeUninit, + sync::Arc, +}; + +impl Context { /// Loads a CUDA module from an in-memory image. /// /// `image` may be PTX source bytes, a cubin, or a fatbin. PTX text is @@ -104,7 +24,7 @@ impl CudaContext { pub fn load_module_from_image( self: &Arc, image: &[u8], - ) -> Result, DriverError> { + ) -> Result, DriverError> { self.bind_to_thread()?; let image = null_terminated_image(image); let cu_module = unsafe { @@ -113,151 +33,24 @@ impl CudaContext { .result()?; cu_module.assume_init() }; - Ok(Arc::new(CudaModule { + Ok(Arc::new(Module { cu_module, - ctx: self.clone(), - })) - } - - /// Loads a module from a cubin or PTX file on disk. - /// - /// `filename` is the filesystem path. The driver selects the loader based - /// on file contents (PTX text or cubin ELF). - /// - /// # Panics - /// - /// Panics if `filename` contains interior null bytes. - pub fn load_module_from_file( - self: &Arc, - filename: &str, - ) -> Result, DriverError> { - self.bind_to_thread()?; - let c_str = CString::new(filename).unwrap(); - let mut cu_module = MaybeUninit::uninit(); - let cu_module = unsafe { - cuda_bindings::cuModuleLoad(cu_module.as_mut_ptr(), c_str.as_ptr()).result()?; - cu_module.assume_init() - }; - Ok(Arc::new(CudaModule { - cu_module, - ctx: self.clone(), + context: self.clone(), + owned: true, })) } } - -fn null_terminated_image(image: &[u8]) -> Cow<'_, [u8]> { - if image.last() == Some(&0) { - Cow::Borrowed(image) - } else { - let mut owned = Vec::with_capacity(image.len() + 1); - owned.extend_from_slice(image); - owned.push(0); - Cow::Owned(owned) - } -} - -/// A handle to a device kernel entry point within a loaded [`CudaModule`]. -/// -/// Holds an `Arc` so the module (and transitively the context) -/// remains loaded for the lifetime of this handle. Cloning is cheap (just an -/// `Arc` bump). -#[derive(Debug, Clone)] -pub struct CudaFunction { - /// Raw CUDA function handle. - pub(crate) cu_function: cuda_bindings::CUfunction, - /// Owning module. Prevents unloading while this function handle exists. - #[allow(unused)] - pub(crate) module: Arc, -} - -/// # Safety -/// -/// `CUfunction` handles are derived from a `CUmodule` and valid in any thread -/// that has the owning context bound. -unsafe impl Send for CudaFunction {} -/// See [`Send`] impl. -unsafe impl Sync for CudaFunction {} - -impl CudaModule { - /// Returns the parent [`CudaContext`]. +impl Module { + /// Returns the parent [`Context`]. /// /// This is mainly useful when interoperating with raw CUDA driver APIs: - /// call [`CudaContext::bind_to_thread`] on this context before passing raw + /// call [`Context::bind_to_thread`] on this context before passing raw /// module/function handles to APIs that require the owning context to be /// current on the calling host thread. - pub fn context(&self) -> &Arc { - &self.ctx - } - - /// Returns the raw `CUmodule` handle owned by this wrapper. - /// - /// This is an escape hatch for CUDA driver interop libraries that need to - /// inspect or register a loaded module directly. For example, NVSHMEM uses - /// module-level state and may need the raw `CUmodule` before launching a - /// kernel that calls into NVSHMEM device code. - /// - /// The returned handle is copied by value, but it is non-owning. - /// cuda-oxide still owns the module and will unload it when the last - /// [`Arc`] owning this module is dropped. - /// - /// # Safety - /// - /// - The returned handle is valid only while this [`CudaModule`] remains - /// alive. If the handle is stored outside the immediate call, the caller - /// must keep an [`Arc`] to this module alive for at least as long. - /// - The caller must not unload the module through the raw handle, transfer - /// ownership of it, or pass it to any API that may invalidate module, - /// function, or global handles owned by cuda-oxide. - /// - Before passing the handle to CUDA driver APIs or interop libraries - /// that make driver calls, the caller must ensure this module's owning - /// context is current on the calling host thread, for example with - /// [`CudaModule::context`] followed by - /// [`CudaContext::bind_to_thread`]. - /// - Any foreign library that retains this handle must obey the same - /// lifetime and context-current requirements. cuda-oxide cannot enforce - /// those requirements once the raw handle leaves Rust's type system. - pub unsafe fn cu_module(&self) -> cuda_bindings::CUmodule { - self.cu_module - } - - /// Looks up a kernel entry point by `fn_name` in this module. - /// - /// The returned [`CudaFunction`] holds an `Arc` back to this module, - /// preventing unloading while the handle is live. - /// - /// This method first binds the module's owning context to the calling - /// thread, then performs `cuModuleGetFunction`. That makes it safe to look - /// up functions from any host thread, provided the module and its context - /// are still alive. - /// - /// # Errors - /// - /// Returns an error if binding the module's context fails or if - /// `cuModuleGetFunction` cannot resolve `fn_name` in this module. - /// - /// # Panics - /// - /// Panics if `fn_name` contains interior null bytes. - pub fn load_function(self: &Arc, fn_name: &str) -> Result { - self.ctx.bind_to_thread()?; - let c_name = CString::new(fn_name).unwrap(); - let cu_function = unsafe { - let mut cu_function = MaybeUninit::uninit(); - cuda_bindings::cuModuleGetFunction( - cu_function.as_mut_ptr(), - self.cu_module, - c_name.as_ptr(), - ) - .result()?; - cu_function.assume_init() - }; - Ok(CudaFunction { - cu_function, - module: self.clone(), - }) + pub fn context(&self) -> &Arc { + &self.context } } - /// A resolved handle to a `#[constant]` device global. Macro-generated /// `set_` methods resolve these lazily on first use and cache the /// handle on the `LoadedModule` struct. Callers pass `size_of::()` on @@ -270,7 +63,7 @@ pub struct ConstantHandle { impl ConstantHandle { /// Construct from a raw device pointer. Used by macro-generated - /// `LoadedModule` initializers after [`CudaModule::get_global`] has + /// `LoadedModule` initializers after [`Module::get_global`] has /// resolved the symbol and the size has been asserted against /// `size_of::()`. /// @@ -293,14 +86,12 @@ impl ConstantHandle { /// - The bytes must have a layout compatible with the device-side type. pub unsafe fn write_async( &self, - stream: &crate::CudaStream, + stream: &crate::Stream, src: *const u8, num_bytes: usize, ) -> Result<(), DriverError> { stream.context().bind_to_thread()?; - unsafe { - crate::simt::memory::memcpy_htod_async(self.dptr, src, num_bytes, stream.cu_stream()) - } + unsafe { crate::memory::memcpy_htod_async(self.dptr, src, num_bytes, stream.cu_stream()) } } /// Stream-ordered `cuMemcpyHtoDAsync` from owned host bytes into the @@ -311,7 +102,7 @@ impl ConstantHandle { /// caller passes a temporary such as `&3.0`. pub fn write_async_staged( &self, - stream: &crate::CudaStream, + stream: &crate::Stream, bytes: Box<[MaybeUninit]>, ) -> Result<(), DriverError> { stream.context().bind_to_thread()?; @@ -321,7 +112,7 @@ impl ConstantHandle { } unsafe { - crate::simt::memory::memcpy_htod_async( + crate::memory::memcpy_htod_async( self.dptr, bytes.as_ptr() as *const u8, num_bytes, @@ -370,7 +161,7 @@ impl ConstantHandle { /// Same contract as [`write_async`](Self::write_async). pub unsafe fn write_blocking( &self, - module: &Arc, + module: &Arc, src: *const u8, num_bytes: usize, ) -> Result<(), DriverError> { @@ -378,7 +169,7 @@ impl ConstantHandle { } } -impl CudaModule { +impl Module { /// Resolves a device global by name and returns its device pointer and /// size in bytes. /// @@ -402,7 +193,7 @@ impl CudaModule { self: &Arc, name: &str, ) -> Result<(cuda_bindings::CUdeviceptr, usize), DriverError> { - self.ctx.bind_to_thread()?; + self.context.bind_to_thread()?; let c_name = CString::new(name).unwrap(); let mut dptr = MaybeUninit::::uninit(); let mut size = MaybeUninit::::uninit(); @@ -439,12 +230,12 @@ impl CudaModule { src: *const u8, num_bytes: usize, ) -> Result<(), DriverError> { - self.ctx.bind_to_thread()?; - unsafe { crate::simt::memory::memcpy_htod_sync(dptr, src, num_bytes) } + self.context.bind_to_thread()?; + unsafe { crate::memory::memcpy_htod_sync(dptr, src, num_bytes) } } } -impl CudaFunction { +impl Function { fn attribute( &self, attribute: cuda_bindings::CUfunction_attribute, @@ -460,7 +251,7 @@ impl CudaFunction { } /// Returns the context that owns this function. - pub fn context(&self) -> &Arc { + pub fn context(&self) -> &Arc { self.module.context() } @@ -497,7 +288,7 @@ impl CudaFunction { /// bytes. /// /// This is the per-thread quantity that - /// [`CudaContext::set_stack_size`](crate::simt::context::CudaContext::set_stack_size) + /// [`Context::set_stack_size`](crate::context::Context::set_stack_size) /// budgets for: the driver reserves the stack limit for every resident /// thread on the device, so a kernel with a large frame can reserve /// device memory far in excess of anything it allocates explicitly. @@ -681,20 +472,15 @@ impl CudaFunction { .map_err(|_| DriverError(cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE)) } } +} - /// Returns the raw `CUfunction` handle. - /// - /// # Safety - /// - /// The returned handle is copied by value, but it is non-owning. It is - /// invalidated if the parent [`CudaModule`] is dropped. - /// - /// Because [`CudaFunction`] holds an [`Arc`] to its parent module, the - /// module cannot be unloaded while `self` is alive. If the raw handle is - /// stored outside the immediate call, the caller must keep this - /// [`CudaFunction`] or another [`Arc`] owning the parent module alive for - /// at least as long as the raw handle is used. - pub unsafe fn cu_function(&self) -> cuda_bindings::CUfunction { - self.cu_function +fn null_terminated_image(image: &[u8]) -> Cow<'_, [u8]> { + if image.last() == Some(&0) { + Cow::Borrowed(image) + } else { + let mut owned = Vec::with_capacity(image.len() + 1); + owned.extend_from_slice(image); + owned.push(0); + Cow::Owned(owned) } } diff --git a/cuda-core/src/simt/peer.rs b/cuda-core/src/peer.rs similarity index 89% rename from cuda-core/src/simt/peer.rs rename to cuda-core/src/peer.rs index 4b67f79337..076207697e 100644 --- a/cuda-core/src/simt/peer.rs +++ b/cuda-core/src/peer.rs @@ -12,8 +12,8 @@ //! P2P access depends on hardware topology. Use `can_access_peer` to query //! support before calling `enable_peer_access`. +use crate::context::Context; use crate::error::{DriverError, IntoResult}; -use crate::simt::context::CudaContext; use std::mem::MaybeUninit; use std::sync::Arc; @@ -25,7 +25,7 @@ use std::sync::Arc; /// /// This is a query only -- it does not enable access. Call `enable_peer_access` /// to actually enable it. -pub fn can_access_peer(from: &CudaContext, to: &CudaContext) -> Result { +pub fn can_access_peer(from: &Context, to: &Context) -> Result { let mut can_access = MaybeUninit::uninit(); unsafe { cuda_bindings::cuDeviceCanAccessPeer( @@ -49,10 +49,7 @@ pub fn can_access_peer(from: &CudaContext, to: &CudaContext) -> Result, - to: &Arc, -) -> Result<(), DriverError> { +pub fn enable_peer_access(from: &Arc, to: &Arc) -> Result<(), DriverError> { from.bind_to_thread()?; let result = unsafe { cuda_bindings::cuCtxEnablePeerAccess(to.cu_ctx(), 0) }; match result { @@ -67,10 +64,7 @@ pub fn enable_peer_access( /// After this call, direct memory access from `from` to `to` is no longer /// possible. Any in-flight operations accessing peer memory must have completed /// before calling this. -pub fn disable_peer_access( - from: &Arc, - to: &Arc, -) -> Result<(), DriverError> { +pub fn disable_peer_access(from: &Arc, to: &Arc) -> Result<(), DriverError> { from.bind_to_thread()?; let result = unsafe { cuda_bindings::cuCtxDisablePeerAccess(to.cu_ctx()) }; match result { diff --git a/cuda-core/src/simt/pinned_host_buffer.rs b/cuda-core/src/pinned_host_buffer.rs similarity index 91% rename from cuda-core/src/simt/pinned_host_buffer.rs rename to cuda-core/src/pinned_host_buffer.rs index 1795eaaff5..5ed2382fb1 100644 --- a/cuda-core/src/simt/pinned_host_buffer.rs +++ b/cuda-core/src/pinned_host_buffer.rs @@ -18,9 +18,9 @@ use std::ptr::NonNull; use std::slice; use std::sync::Arc; +use crate::context::Context; +use crate::device_buffer::DeviceCopy; use crate::error::DriverError; -use crate::simt::context::CudaContext; -use crate::simt::device_buffer::DeviceCopy; /// Owned page-locked host buffer. /// @@ -33,8 +33,8 @@ use crate::simt::device_buffer::DeviceCopy; /// as [`String`] are rejected. /// /// ```compile_fail -/// # use cuda_core::{CudaContext, PinnedHostBuffer}; -/// # fn rejects_non_device_copy(ctx: &std::sync::Arc) { +/// # use cuda_core::{Context, PinnedHostBuffer}; +/// # fn rejects_non_device_copy(ctx: &std::sync::Arc) { /// let _ = PinnedHostBuffer::::zeroed(ctx, 1); /// # } /// ``` @@ -44,7 +44,7 @@ pub struct PinnedHostBuffer { allocation: Option>, len: usize, num_bytes: usize, - ctx: Arc, + ctx: Arc, _marker: PhantomData, } @@ -56,7 +56,7 @@ unsafe impl Sync for PinnedHostBuffer {} impl PinnedHostBuffer { /// Allocates a pinned host buffer and fills it with zero bytes. - pub fn zeroed(ctx: &Arc, len: usize) -> Result { + pub fn zeroed(ctx: &Arc, len: usize) -> Result { let mut buffer = Self::allocate(ctx, len)?; if buffer.num_bytes != 0 { unsafe { @@ -68,7 +68,7 @@ impl PinnedHostBuffer { } /// Allocates a pinned host buffer and copies `data` into it. - pub fn from_slice(ctx: &Arc, data: &[T]) -> Result { + pub fn from_slice(ctx: &Arc, data: &[T]) -> Result { let buffer = Self::allocate(ctx, data.len())?; if !data.is_empty() { unsafe { @@ -98,7 +98,7 @@ impl PinnedHostBuffer { /// Returns the CUDA context used to allocate this buffer. #[inline] - pub fn context(&self) -> &Arc { + pub fn context(&self) -> &Arc { &self.ctx } @@ -126,13 +126,13 @@ impl PinnedHostBuffer { unsafe { slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } } - fn allocate(ctx: &Arc, len: usize) -> Result { + fn allocate(ctx: &Arc, len: usize) -> Result { let (num_bytes, allocation_bytes) = allocation_sizes::(len)?; let (ptr, allocation) = if num_bytes == 0 { (NonNull::dangling(), None) } else { ctx.bind_to_thread()?; - let raw = unsafe { crate::simt::memory::malloc_host(allocation_bytes)? }; + let raw = unsafe { crate::memory::malloc_host(allocation_bytes)? }; let allocation = NonNull::new(raw.cast::()).ok_or(DriverError( cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE, ))?; @@ -162,7 +162,7 @@ impl Drop for PinnedHostBuffer { if let Some(allocation) = self.allocation { self.ctx.record_err(self.ctx.bind_to_thread()); self.ctx - .record_err(unsafe { crate::simt::memory::free_host(allocation.as_ptr().cast()) }); + .record_err(unsafe { crate::memory::free_host(allocation.as_ptr().cast()) }); } } } diff --git a/cuda-core/src/runtime.rs b/cuda-core/src/runtime.rs index b1893e92b5..07200048cf 100644 --- a/cuda-core/src/runtime.rs +++ b/cuda-core/src/runtime.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -//! CUDA runtime types: Device, Stream, Module, Function, LaunchConfig. +//! CUDA runtime types: Context, Stream, Module, Function, LaunchConfig. //! //! These are the public API of `cuda-core`. They wrap raw CUDA driver //! handles with RAII lifetimes and provide `borrow_raw` constructors @@ -16,17 +16,6 @@ use crate::cudarc_shim::{ctx, device, module, pool, primary_ctx, stream}; use crate::error::*; use crate::init; -/// Kernel launch configuration specifying grid, block, and shared memory sizes. -#[derive(Clone, Copy, Debug)] -pub struct LaunchConfig { - /// Grid dimensions `(x, y, z)` in thread blocks. - pub grid_dim: (u32, u32, u32), - /// Block dimensions `(x, y, z)` in threads. - pub block_dim: (u32, u32, u32), - /// Bytes of dynamic shared memory per block. - pub shared_mem_bytes: u32, -} - /// Anything that owns an external CUDA resource. A borrowed handle can hold an /// `Arc` as a *liveness token*: while the handle (and anything /// derived from it) is alive, the token's refcount is nonzero, so the external @@ -72,38 +61,88 @@ impl PartialEq for KeepAlive { } impl Eq for KeepAlive {} -/// A GPU device handle wrapping a CUDA primary context. +/// A physical CUDA device identity, independent of any CUDA context. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Device { + cu_device: cuda_bindings::CUdevice, + ordinal: usize, +} + +impl Device { + /// Looks up a device without retaining its primary context. + pub fn new(ordinal: usize) -> Result { + let raw = ordinal_to_c_int(ordinal)?; + unsafe { init(0)? }; + Ok(Self { + cu_device: device::get(raw)?, + ordinal, + }) + } + /// Returns the physical device ordinal. + pub fn ordinal(&self) -> usize { + self.ordinal + } + /// Returns the raw physical device handle. + pub fn cu_device(&self) -> cuda_bindings::CUdevice { + self.cu_device + } + /// Returns the device name. + pub fn name(&self) -> Result { + device::get_name(self.cu_device) + } + /// Retains this device's primary context for resource creation and submission. + pub fn primary_context(&self) -> Result, DriverError> { + Context::new(self.ordinal) + } + /// Returns the number of CUDA devices. + pub fn device_count() -> Result { + Context::device_count() + } +} + +impl Context { + /// The physical device hosting this context. Equal devices do not imply equal contexts. + pub fn device(&self) -> Device { + Device { + cu_device: self.cu_device, + ordinal: self.ordinal, + } + } +} + +/// A CUDA execution context, distinct from its physical [`Device`]. /// -/// Can be **owned** (created via [`Device::new`], releases the primary context -/// on drop), **borrowed** (created via [`Device::borrow_raw`], does NOT release -/// on drop), or **foreign** (created via [`Device::borrow_with_owner`], holds a +/// Can be **owned** (created via [`Context::new`], releases the primary context +/// on drop), **borrowed** (created via [`Context::borrow_raw`], does NOT release +/// on drop), or **foreign** (created via [`Context::borrow_with_owner`], holds a /// liveness token so the external owner outlives it). #[derive(Debug)] -pub struct Device { +pub struct Context { pub(crate) cu_device: cuda_bindings::CUdevice, pub(crate) cu_ctx: cuda_bindings::CUcontext, pub(crate) ordinal: usize, - owned: bool, - _keep_alive: KeepAlive, + pub(crate) owned: bool, + pub(crate) _keep_alive: KeepAlive, + pub(crate) error_state: std::sync::atomic::AtomicU32, } -unsafe impl Send for Device {} -unsafe impl Sync for Device {} +unsafe impl Send for Context {} +unsafe impl Sync for Context {} -impl Drop for Device { +impl Drop for Context { fn drop(&mut self) { if !self.owned { return; } let _guard = teardown_lock(); - // Streams hold an Arc, so by the time the last device handle + // Streams hold an Arc, so by the time the last device handle // for this ordinal drops, every pooled handle is idle; the context // release below reclaims them. Discard so a later re-retain cannot // pop a handle from a torn-down context. stream_pool() .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) - .remove(&self.ordinal); + .remove(&(self.cu_ctx as usize)); let _ = self.bind_to_thread(); let ctx = std::mem::replace(&mut self.cu_ctx, std::ptr::null_mut()); if !ctx.is_null() { @@ -112,14 +151,14 @@ impl Drop for Device { } } -impl PartialEq for Device { +impl PartialEq for Context { fn eq(&self, other: &Self) -> bool { self.cu_device == other.cu_device && self.cu_ctx == other.cu_ctx && self.ordinal == other.ordinal } } -impl Eq for Device {} +impl Eq for Context {} /// The driver indexes devices with a C `int`. An ordinal that does not fit is /// not a device at all; `as c_int` would wrap it onto some other device's @@ -129,7 +168,7 @@ fn ordinal_to_c_int(ordinal: usize) -> Result { .map_err(|_| DriverError(cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_DEVICE)) } -impl Device { +impl Context { /// Creates a new owned device on the specified ordinal. /// /// Errors with `CUDA_ERROR_INVALID_DEVICE` (before touching the driver) @@ -139,15 +178,16 @@ impl Device { unsafe { init(0)? }; let cu_device = device::get(cu_ordinal)?; let cu_ctx = unsafe { primary_ctx::retain(cu_device) }?; - let device = Arc::new(Device { + let context = Arc::new(Context { cu_device, cu_ctx, ordinal, owned: true, _keep_alive: KeepAlive::none(), + error_state: std::sync::atomic::AtomicU32::new(0), }); - device.bind_to_thread()?; - Ok(device) + context.bind_to_thread()?; + Ok(context) } /// Wraps externally-owned CUDA handles without taking ownership. @@ -162,7 +202,7 @@ impl Device { /// /// The caller must ensure: /// - `cu_ctx` points to a valid retained `CUcontext` for `cu_device` - /// - The handles outlive the returned `Device` **and everything derived + /// - The handles outlive the returned `Context` **and everything derived /// from it**. Streams from [`new_stream`](Self::new_stream) hold the /// device alive and, because the device is borrowed, are never parked in /// the process-wide stream pool: each is synchronized and destroyed @@ -170,12 +210,13 @@ impl Device { /// therefore still be valid when the last such stream drops. /// - No concurrent destruction of the handles pub unsafe fn borrow_raw(cu_ctx: *mut c_void, cu_device: c_int, ordinal: usize) -> Arc { - Arc::new(Device { + Arc::new(Context { cu_device: cu_device as cuda_bindings::CUdevice, cu_ctx: cu_ctx as cuda_bindings::CUcontext, ordinal, owned: false, _keep_alive: KeepAlive::none(), + error_state: std::sync::atomic::AtomicU32::new(0), }) } @@ -184,8 +225,8 @@ impl Device { /// /// Same as [`borrow_raw`](Self::borrow_raw), but the liveness obligation is /// discharged by construction: `owner` is whatever owns the context (a - /// cudarc device, a torch context handle, ...), and holding it here - /// guarantees the handles stay valid as long as this `Device` — or anything + /// cudarc context, a torch context handle, ...), and holding it here + /// guarantees the handles stay valid as long as this `Context` — or anything /// derived from it — lives. Only the point-in-time validity of the handles /// remains a caller assertion. /// @@ -200,12 +241,13 @@ impl Device { ordinal: usize, owner: Arc, ) -> Arc { - Arc::new(Device { + Arc::new(Context { cu_device: cu_device as cuda_bindings::CUdevice, cu_ctx: cu_ctx as cuda_bindings::CUcontext, ordinal, owned: false, _keep_alive: KeepAlive::owner(owner), + error_state: std::sync::atomic::AtomicU32::new(0), }) } @@ -216,7 +258,7 @@ impl Device { } /// Returns the raw `CUdevice` handle for a given ordinal without - /// creating a full `Device` (no context retained). Rejects an ordinal + /// creating a full `Context` (no context retained). Rejects an ordinal /// that does not fit `c_int` like [`new`](Self::new) does. pub fn raw_device(ordinal: usize) -> Result { let cu_ordinal = ordinal_to_c_int(ordinal)?; @@ -229,7 +271,7 @@ impl Device { self.ordinal } - /// Get the name of this device. + /// Get the name of this context. pub fn name(&self) -> Result { device::get_name(self.cu_device) } @@ -246,6 +288,7 @@ impl Device { /// Binds this context to the calling thread if not already current. pub fn bind_to_thread(&self) -> Result<(), DriverError> { + self.check_err()?; if match ctx::get_current()? { Some(curr_ctx) => curr_ctx != self.cu_ctx, None => true, @@ -257,18 +300,17 @@ impl Device { /// Blocks until all work on this device's context is complete. /// - /// # Safety - /// The caller must ensure this device's context is current on the - /// calling thread (via [`bind_to_thread`](Device::bind_to_thread)). - pub unsafe fn synchronize(&self) -> Result<(), DriverError> { + /// Binds this context to the calling thread before waiting. + pub fn synchronize(&self) -> Result<(), DriverError> { + self.bind_to_thread()?; ctx::synchronize() } - /// Creates a new non-blocking CUDA stream on this device. + /// Creates a new non-blocking CUDA stream on this context. /// /// On an owned device the handle may be one parked by an earlier stream's /// drop (see [`Stream`]). On a borrowed device it is always freshly - /// created: the pool is keyed by ordinal alone and holds handles from the + /// created: the pool is keyed by context identity and holds handles from the /// primary contexts this crate retains, which need not be the context the /// external owner handed to [`borrow_raw`](Self::borrow_raw). pub fn new_stream(self: &Arc) -> Result, DriverError> { @@ -277,7 +319,7 @@ impl Device { stream_pool() .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) - .get_mut(&self.ordinal) + .get_mut(&(self.cu_ctx as usize)) .and_then(Vec::pop) } else { None @@ -288,7 +330,7 @@ impl Device { }; Ok(Arc::new(Stream { cu_stream, - device: self.clone(), + context: self.clone(), owned: true, _keep_alive: KeepAlive::none(), })) @@ -306,7 +348,7 @@ impl Device { }?; Ok(Arc::new(Module { cu_module, - device: self.clone(), + context: self.clone(), owned: true, })) } @@ -320,7 +362,7 @@ impl Device { let cu_module = { module::load(filename) }?; Ok(Arc::new(Module { cu_module, - device: self.clone(), + context: self.clone(), owned: true, })) } @@ -364,12 +406,12 @@ impl Device { let cu_module = unsafe { module::load_data(image.as_ptr().cast()) }?; Ok(Arc::new(Module { cu_module, - device: self.clone(), + context: self.clone(), owned: true, })) } - /// Creates a new memory pool on this device. + /// Creates a new memory pool on this context. /// /// The returned pool is owned — it will be destroyed when the last `Arc` /// is dropped. Pair with [`MemPool::set_release_threshold`] to control @@ -384,21 +426,21 @@ impl Device { let cu_pool = unsafe { pool::create(&props) }?; Ok(Arc::new(MemPool { cu_pool, - device: self.clone(), + context: self.clone(), owned: true, })) } - /// Returns the driver-owned default memory pool for this device. + /// Returns the driver-owned default memory pool for this context. /// /// The returned wrapper is **not owned** — dropping it does not destroy the - /// default pool, which is shared across all users of the device. + /// default pool, which is shared across all users of the context. pub fn default_mem_pool(self: &Arc) -> Result, DriverError> { self.bind_to_thread()?; let cu_pool = unsafe { pool::get_default(self.cu_device) }?; Ok(Arc::new(MemPool { cu_pool, - device: self.clone(), + context: self.clone(), owned: false, })) } @@ -406,17 +448,17 @@ impl Device { /// A CUDA memory pool handle. /// -/// Can be either **owned** (created via [`Device::new_mem_pool`], destroyed on -/// drop) or **borrowed** (created via [`Device::default_mem_pool`], does NOT +/// Can be either **owned** (created via [`Context::new_mem_pool`], destroyed on +/// drop) or **borrowed** (created via [`Context::default_mem_pool`], does NOT /// destroy on drop). /// /// Used by async tensor allocation via `cuMemAllocFromPoolAsync` when a pool -/// is registered via `cuda_async::device_context::set_device_pool`. +/// is registered via `cuda_async::device_device::set_device_pool`. #[derive(Debug)] pub struct MemPool { pub(crate) cu_pool: cuda_bindings::CUmemoryPool, - pub(crate) device: Arc, - owned: bool, + pub(crate) context: Arc, + pub(crate) owned: bool, } unsafe impl Send for MemPool {} @@ -427,7 +469,7 @@ impl Drop for MemPool { if !self.owned { return; } - let _ = self.device.bind_to_thread(); + let _ = self.context.bind_to_thread(); let _ = unsafe { pool::destroy(self.cu_pool) }; } } @@ -438,9 +480,9 @@ impl MemPool { self.cu_pool } - /// Returns a reference to the parent device. - pub fn device(&self) -> &Arc { - &self.device + /// Returns a reference to the parent context. + pub fn context(&self) -> &Arc { + &self.context } /// Sets the release threshold for this pool. @@ -449,7 +491,7 @@ impl MemPool { /// below this threshold. Use `u64::MAX` to prevent the OS from reclaiming /// pool memory (useful for inference workloads with stable memory footprints). pub fn set_release_threshold(&self, threshold: u64) -> Result<(), DriverError> { - self.device.bind_to_thread()?; + self.context.bind_to_thread()?; unsafe { pool::set_release_threshold(self.cu_pool, threshold) } } @@ -460,7 +502,7 @@ impl MemPool { /// a synchronously consistent reading. For stable values, sync the streams /// that allocate from this pool first. pub fn mem_stats(&self) -> Result { - self.device.bind_to_thread()?; + self.context.bind_to_thread()?; unsafe { Ok(PoolMemStats { used_current: pool::get_attribute_u64( @@ -485,7 +527,7 @@ impl MemPool { /// Resets `used_high` to the current `used_current`. pub fn reset_used_high(&self) -> Result<(), DriverError> { - self.device.bind_to_thread()?; + self.context.bind_to_thread()?; unsafe { pool::reset_high_watermark( self.cu_pool, @@ -496,7 +538,7 @@ impl MemPool { /// Resets `reserved_high` to the current `reserved_current`. pub fn reset_reserved_high(&self) -> Result<(), DriverError> { - self.device.bind_to_thread()?; + self.context.bind_to_thread()?; unsafe { pool::reset_high_watermark( self.cu_pool, @@ -523,41 +565,32 @@ pub struct PoolMemStats { /// A CUDA stream handle. /// -/// Can be **owned** (created via [`Device::new_stream`]), **borrowed** (created +/// Can be **owned** (created via [`Context::new_stream`]), **borrowed** (created /// via [`Stream::borrow_raw`], does NOT destroy on drop), or **foreign** /// (created via [`Stream::borrow_with_owner`], holds a liveness token so the /// external owner outlives it). /// -/// Dropping an owned stream blocks in `cuStreamSynchronize` until the work -/// enqueued on it has drained. What happens to the handle next depends on the -/// device it was created on: -/// -/// - On an **owned** device the handle is *not* destroyed: it is parked in a -/// process-wide, per-ordinal pool and handed out again by the next -/// [`Device::new_stream`] on that ordinal (see `stream_pool` for why). The -/// last owned `Device` for the ordinal discards the pool entry, and the -/// primary-context release reclaims the handles. -/// - On a **borrowed** device ([`Device::borrow_raw`] / -/// [`Device::borrow_with_owner`]) the handle is destroyed, under the -/// teardown lock. It must never enter the pool: the pool is keyed by -/// ordinal only, so an owned `Device` created later for the same ordinal -/// would pop a handle belonging to a context the external owner may already -/// have destroyed. +/// Drop returns without waiting for queued GPU work. An event-backed retirement +/// queue retains the context and foreign owner until completion. Idle default +/// priority streams on owned contexts enter a pool keyed by CUDA context +/// identity; streams on borrowed contexts and priority streams are destroyed +/// after completion. Completion errors retain resources rather than reuse a +/// handle that may still be active. #[derive(Debug, PartialEq, Eq)] pub struct Stream { pub(crate) cu_stream: cuda_bindings::CUstream, - pub(crate) device: Arc, - owned: bool, - _keep_alive: KeepAlive, + pub(crate) context: Arc, + pub(crate) owned: bool, + pub(crate) _keep_alive: KeepAlive, } -/// Per-device pool of idle stream handles (all created `NonBlocking` by -/// [`Device::new_stream`]). `cuStreamDestroy` racing concurrent driver +/// Per-context pool of idle stream handles (all created `NonBlocking` by +/// [`Context::new_stream`]). `cuStreamDestroy` racing concurrent driver /// activity from other threads segfaults inside libcuda (reproducible in /// parallel test runs; a leak-streams experiment ran 10/10 clean where /// destroy-paths crashed ~half the runs), so owned streams are never /// destroyed: drops return the handle here, `new_stream` reuses it, and the -/// last `Device` drop for an ordinal discards the pooled handles — the +/// last `Context` drop for an ordinal discards the pooled handles — the /// primary-context release reclaims them. fn stream_pool() -> &'static std::sync::Mutex>> { static POOL: std::sync::OnceLock< @@ -590,8 +623,8 @@ unsafe impl Sync for Stream {} /// early is safe). An event is bound to its device at construction; /// recording it on a stream of another device is rejected. pub struct Event { - cu_event: cuda_bindings::CUevent, - device: Arc, + pub(crate) cu_event: cuda_bindings::CUevent, + pub(crate) context: Arc, } unsafe impl Send for Event {} @@ -602,20 +635,21 @@ impl Event { /// /// Errors with `CUDA_ERROR_INVALID_VALUE` if the stream belongs to a /// different device than the one this event was created on. - pub fn record(&self, stream: &Arc) -> Result<(), DriverError> { - if stream.device().ordinal() != self.device.ordinal() { + pub fn record(&self, stream: &Stream) -> Result<(), DriverError> { + if stream.context().as_ref() != self.context.as_ref() { return Err(DriverError( cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE, )); } - // Safety: both handles are valid by construction (RAII wrappers), - // and the same-device check above pins them to one context. + self.context.bind_to_thread()?; + // SAFETY: the handles have equal context identity and that context is current. unsafe { crate::cudarc_shim::event::record(self.cu_event, stream.cu_stream()) } } /// Blocks the calling thread until this event has completed. pub fn synchronize(&self) -> Result<(), DriverError> { - // Safety: the handle is valid by construction. + self.context.bind_to_thread()?; + // SAFETY: the handle is valid and its context is current. unsafe { crate::cudarc_shim::event::synchronize(self.cu_event) } } @@ -624,7 +658,8 @@ impl Event { /// (or the event was never recorded), `Ok(false)` while it is still in /// flight. Any other driver result is returned as the error. pub fn query(&self) -> Result { - // Safety: the handle is valid by construction. + self.context.bind_to_thread()?; + // SAFETY: the handle is valid and its context is current. match unsafe { crate::cudarc_shim::event::query(self.cu_event) } { Ok(()) => Ok(true), Err(DriverError(cuda_bindings::cudaError_enum_CUDA_ERROR_NOT_READY)) => Ok(false), @@ -640,27 +675,34 @@ impl Event { /// call [`synchronize`](Self::synchronize) on it first, or the driver /// reports not-ready. pub fn elapsed_time(&self, end: &Event) -> Result { - // Safety: both handles are valid by construction. + if self.context.as_ref() != end.context.as_ref() { + return Err(DriverError( + cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_CONTEXT, + )); + } + self.context.bind_to_thread()?; + // SAFETY: both handles belong to the current context. unsafe { crate::cudarc_shim::event::elapsed(self.cu_event, end.cu_event) } } } impl Drop for Event { fn drop(&mut self) { + self.context.record_err(self.context.bind_to_thread()); // Safety: owned handle; the driver defers destruction while in use. let _ = unsafe { crate::cudarc_shim::event::destroy(self.cu_event) }; } } -impl Device { - /// Creates a timing-enabled event on this device. +impl Context { + /// Creates a timing-enabled event on this context. pub fn new_event(self: &Arc) -> Result { self.bind_to_thread()?; let cu_event = crate::cudarc_shim::event::create(cuda_bindings::CUevent_flags_enum_CU_EVENT_DEFAULT)?; Ok(Event { cu_event, - device: self.clone(), + context: self.clone(), }) } @@ -681,31 +723,58 @@ impl Device { } } -impl Drop for Stream { +// Retains both context and foreign liveness tokens while a stream drains. +struct RetiredStream { + handle: usize, + context: Arc, + owned: bool, + reusable: bool, + _keep_alive: KeepAlive, +} + +impl Drop for RetiredStream { fn drop(&mut self) { - if !self.owned || self.cu_stream.is_null() { + if !self.owned || self.handle == 0 { return; } - let _ = self.device.bind_to_thread(); - if self.device.owned { - // Never destroyed (see `stream_pool`): drain, then return the - // handle for reuse by the next `new_stream` on this device. - let _ = unsafe { stream::synchronize(self.cu_stream) }; + if self.reusable { stream_pool() .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .entry(self.device.ordinal) + .unwrap_or_else(|e| e.into_inner()) + .entry(self.context.cu_ctx() as usize) .or_default() - .push(self.cu_stream as usize); + .push(self.handle); } else { - // Borrowed device: the pool is keyed by ordinal only, so parking - // this handle would let an owned `Device` created later for the - // same ordinal pop a stream from a context the external owner may - // by then have destroyed (reproduced as a segfault). Destroy it - // instead, serialized with the other destructive driver calls. let _guard = teardown_lock(); - let _ = unsafe { stream::synchronize(self.cu_stream) }; - let _ = unsafe { stream::destroy(self.cu_stream) }; + if self.context.bind_to_thread().is_err() { + // Preserve the context rather than releasing a resource whose + // teardown the driver could not perform safely. + std::mem::forget(self.context.clone()); + return; + } + self.context + .record_err(unsafe { stream::destroy(self.handle as cuda_bindings::CUstream) }); + } + } +} + +impl Drop for Stream { + fn drop(&mut self) { + let retired = RetiredStream { + handle: self.cu_stream as usize, + context: self.context.clone(), + owned: self.owned, + // Preserve priority semantics: only default-priority streams enter + // the pool used by Context::new_stream. + reusable: self.context.owned && self.priority().ok() == Some(0), + _keep_alive: self._keep_alive.clone(), + }; + match self.query() { + Ok(true) => drop(retired), + Ok(false) => { + let _ = crate::completion::retain_until_complete(retired, self); + } + Err(_) => std::mem::forget(retired), } } } @@ -714,7 +783,7 @@ impl Stream { /// Wraps an externally-owned CUDA stream without taking ownership. /// /// `cu_stream` is the raw `CUstream` opaque pointer. See - /// [`Device::borrow_raw`] for why this is a `*mut c_void` rather than a + /// [`Context::borrow_raw`] for why this is a `*mut c_void` rather than a /// `cuda_bindings::CUstream` typedef. /// /// # Safety @@ -723,10 +792,10 @@ impl Stream { /// - `cu_stream` points to a valid CUDA stream on `device` /// - The stream outlives the returned `Stream` /// - No concurrent destruction of the stream - pub unsafe fn borrow_raw(cu_stream: *mut c_void, device: &Arc) -> Arc { + pub unsafe fn borrow_raw(cu_stream: *mut c_void, context: &Arc) -> Arc { Arc::new(Stream { cu_stream: cu_stream as cuda_bindings::CUstream, - device: device.clone(), + context: context.clone(), owned: false, _keep_alive: KeepAlive::none(), }) @@ -747,12 +816,12 @@ impl Stream { /// that stream (i.e. `owner` really is what keeps it alive). pub unsafe fn borrow_with_owner( cu_stream: *mut c_void, - device: &Arc, + context: &Arc, owner: Arc, ) -> Arc { Arc::new(Stream { cu_stream: cu_stream as cuda_bindings::CUstream, - device: device.clone(), + context: context.clone(), owned: false, _keep_alive: KeepAlive::owner(owner), }) @@ -763,28 +832,26 @@ impl Stream { self.cu_stream } - /// Returns a reference to the parent device. - pub fn device(&self) -> &Arc { - &self.device + /// Returns a reference to the parent context. + pub fn context(&self) -> &Arc { + &self.context } /// Blocks until all work on this stream is complete. /// - /// # Safety - /// The caller must ensure the parent device's context is current on - /// the calling thread. - pub unsafe fn synchronize(&self) -> Result<(), DriverError> { - stream::synchronize(self.cu_stream) + /// Binds the parent context to the calling thread before submitting. + pub fn synchronize(&self) -> Result<(), DriverError> { + self.context.bind_to_thread()?; + unsafe { stream::synchronize(self.cu_stream) } } /// Queries stream completion without blocking: `Ok(true)` when all prior /// work has completed, `Ok(false)` when work is still in flight. /// - /// # Safety - /// The caller must ensure the parent device's context is current on - /// the calling thread. - pub unsafe fn query(&self) -> Result { - stream::query(self.cu_stream) + /// Binds the parent context to the calling thread before submitting. + pub fn query(&self) -> Result { + self.context.bind_to_thread()?; + unsafe { stream::query(self.cu_stream) } } /// Makes all work subsequently enqueued on this stream wait until @@ -795,13 +862,13 @@ impl Stream { /// Errors with `CUDA_ERROR_INVALID_VALUE` if the event belongs to a /// different device than this stream. pub fn wait_event(&self, event: &Event) -> Result<(), DriverError> { - if event.device.ordinal() != self.device.ordinal() { + if event.context.as_ref() != self.context.as_ref() { return Err(DriverError( cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE, )); } - // Safety: both handles are valid by construction (RAII wrappers), - // and the same-device check above pins them to one context. + self.context.bind_to_thread()?; + // SAFETY: the handles have equal context identity and that context is current. unsafe { stream::wait_event( self.cu_stream, @@ -831,13 +898,12 @@ impl Stream { /// callback fires; on `Err` the driver never invokes the callback, so the /// box is reclaimed here and the closure's captures are dropped. /// - /// # Safety - /// The caller must ensure the parent device's context is current on - /// the calling thread. - pub unsafe fn launch_host_function( + /// Binds the parent context to the calling thread before submitting. + pub fn launch_host_function( &self, host_func: F, ) -> Result<(), DriverError> { + self.context.bind_to_thread()?; Self::enqueue_boxed(host_func, |func, arg| unsafe { stream::launch_host_function(self.cu_stream, func, arg) }) @@ -849,14 +915,13 @@ impl Stream { /// rules: no CUDA calls inside `host_func`, panics are caught and /// discarded, and a refused launch reclaims the boxed closure. /// - /// # Safety - /// The caller must ensure the parent device's context is current on - /// the calling thread. - pub unsafe fn launch_host_function_with_sync_mode( + /// Binds the parent context to the calling thread before submitting. + pub fn launch_host_function_with_sync_mode( &self, host_func: F, sync_mode: ::core::ffi::c_uint, ) -> Result<(), DriverError> { + self.context.bind_to_thread()?; Self::enqueue_boxed(host_func, |func, arg| unsafe { stream::launch_host_function_v2(self.cu_stream, func, arg, sync_mode) }) @@ -922,14 +987,14 @@ impl Stream { /// A loaded CUDA module (PTX/cubin). /// -/// Can be either **owned** (created via [`Device::load_module_from_ptx_src`] -/// / [`Device::load_module_from_file`], unloads on drop) or **borrowed** +/// Can be either **owned** (created via [`Context::load_module_from_ptx_src`] +/// / [`Context::load_module_from_file`], unloads on drop) or **borrowed** /// (created via [`Module::borrow_raw`], does NOT unload on drop). #[derive(Debug)] pub struct Module { pub(crate) cu_module: cuda_bindings::CUmodule, - pub(crate) device: Arc, - owned: bool, + pub(crate) context: Arc, + pub(crate) owned: bool, } unsafe impl Send for Module {} @@ -941,7 +1006,7 @@ impl Drop for Module { return; } let _guard = teardown_lock(); - let _ = self.device.bind_to_thread(); + let _ = self.context.bind_to_thread(); let _ = unsafe { module::unload(self.cu_module) }; } } @@ -950,7 +1015,7 @@ impl Module { /// Wraps an externally-owned CUDA module without taking ownership. /// /// `cu_module` is the raw `CUmodule` opaque pointer. See - /// [`Device::borrow_raw`] for why this is a `*mut c_void`. + /// [`Context::borrow_raw`] for why this is a `*mut c_void`. /// /// # Safety /// @@ -958,10 +1023,10 @@ impl Module { /// - `cu_module` points to a valid module loaded on `device` /// - The module outlives the returned `Module` /// - No concurrent unload of the module - pub unsafe fn borrow_raw(cu_module: *mut c_void, device: &Arc) -> Arc { + pub unsafe fn borrow_raw(cu_module: *mut c_void, context: &Arc) -> Arc { Arc::new(Module { cu_module: cu_module as cuda_bindings::CUmodule, - device: device.clone(), + context: context.clone(), owned: false, }) } @@ -973,6 +1038,7 @@ impl Module { /// Looks up a device function by name within this module. pub fn load_function(self: &Arc, fn_name: &str) -> Result { + self.context.bind_to_thread()?; let cu_function = unsafe { module::get_function(self.cu_module, fn_name) }?; Ok(Function { cu_function, @@ -1104,33 +1170,33 @@ mod tests { } fn has_gpu() -> bool { - Device::device_count().map(|n| n > 0).unwrap_or(false) + Context::device_count().map(|n| n > 0).unwrap_or(false) } - /// The stream pool is process-wide and keyed by ordinal, so the tests that - /// assert on its contents for ordinal 0 must not overlap: a concurrent - /// test's `new_stream` pops, and its `Device` drop discards the whole + /// The stream pool is process-wide and keyed by context, so the tests that + /// assert on the primary context pool must not overlap: a concurrent + /// test's `new_stream` pops, and its `Context` drop discards the whole /// entry. Nothing else in this test binary touches the pool. fn pool_tests_lock() -> std::sync::MutexGuard<'static, ()> { static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) } - /// Snapshot of the parked handles for `ordinal`. - fn pooled_handles(ordinal: usize) -> Vec { + /// Snapshot of parked handles for this CUDA context. + fn pooled_handles(context: &Context) -> Vec { stream_pool() .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) - .get(&ordinal) + .get(&(context.cu_ctx() as usize)) .cloned() .unwrap_or_default() } - fn borrow(owner: &Arc) -> Arc { + fn borrow(owner: &Arc) -> Arc { // SAFETY: `owner` retains the primary context for the duration of // every test below, and nothing destroys it concurrently. unsafe { - Device::borrow_raw( + Context::borrow_raw( owner.cu_ctx().cast(), owner.cu_device() as c_int, owner.ordinal(), @@ -1142,11 +1208,11 @@ mod tests { fn ordinals_beyond_c_int_are_rejected_not_truncated() { let invalid = DriverError(cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_DEVICE); // Checked before `cuInit`, so this holds with or without a driver. - assert_eq!(Device::new(usize::MAX).err(), Some(invalid)); - assert_eq!(Device::raw_device(usize::MAX).err(), Some(invalid)); + assert_eq!(Context::new(usize::MAX).err(), Some(invalid)); + assert_eq!(Context::raw_device(usize::MAX).err(), Some(invalid)); // The first ordinal past `c_int` would have wrapped to device 0. let wraps_to_zero = (c_int::MAX as usize) + 1 + (c_int::MAX as usize) + 1; - assert_eq!(Device::new(wraps_to_zero).err(), Some(invalid)); + assert_eq!(Context::new(wraps_to_zero).err(), Some(invalid)); assert_eq!(ordinal_to_c_int(0), Ok(0)); assert_eq!(ordinal_to_c_int(c_int::MAX as usize), Ok(c_int::MAX)); } @@ -1157,12 +1223,12 @@ mod tests { return; } let _serialized = pool_tests_lock(); - let owner = Device::new(0).unwrap(); + let owner = Context::new(0).unwrap(); let first = owner.new_stream().unwrap(); let handle = first.cu_stream() as usize; drop(first); assert!( - pooled_handles(0).contains(&handle), + pooled_handles(&owner).contains(&handle), "an owned device's stream is parked, not destroyed" ); let second = owner.new_stream().unwrap(); @@ -1175,14 +1241,14 @@ mod tests { return; } let _serialized = pool_tests_lock(); - let owner = Device::new(0).unwrap(); + let owner = Context::new(0).unwrap(); // Seed the pool with a handle from the owned device so that a // borrowed `new_stream` that consulted the pool would be caught. let parked = { let seed = owner.new_stream().unwrap(); seed.cu_stream() as usize }; - assert!(pooled_handles(0).contains(&parked)); + assert!(pooled_handles(&owner).contains(&parked)); let borrowed = borrow(&owner); let stream = borrowed.new_stream().unwrap(); @@ -1193,18 +1259,18 @@ mod tests { belong to contexts this crate retained, not to the borrowed one" ); assert!( - pooled_handles(0).contains(&parked), + pooled_handles(&owner).contains(&parked), "and leaves the pool as it was" ); - unsafe { stream.synchronize() }.unwrap(); + stream.synchronize().unwrap(); drop(stream); assert!( - !pooled_handles(0).contains(&handle), + !pooled_handles(&owner).contains(&handle), "a borrowed device's stream must be destroyed on drop, never parked \ - where a later owned Device for this ordinal could pop it" + where a later owned Context for this ordinal could pop it" ); - assert!(pooled_handles(0).contains(&parked)); + assert!(pooled_handles(&owner).contains(&parked)); } #[test] diff --git a/cuda-core/src/simt/event.rs b/cuda-core/src/simt/event.rs deleted file mode 100644 index 0ef390afa3..0000000000 --- a/cuda-core/src/simt/event.rs +++ /dev/null @@ -1,144 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -//! CUDA event management (RAII, timing, synchronization). -//! -//! A [`CudaEvent`] wraps a `CUevent` handle and ties its lifetime to its -//! parent [`CudaContext`]. Events are the fundamental synchronization -//! primitive between CUDA streams: record an event on one stream, then wait -//! on it from another to establish ordering. -//! -//! # Timing -//! -//! By default, events are created with `CU_EVENT_DISABLE_TIMING` for lower -//! overhead. Pass `Some(CU_EVENT_DEFAULT)` to -//! [`CudaContext::new_event`] or [`CudaStream::record_event`] if you -//! need [`elapsed_ms`](CudaEvent::elapsed_ms). - -use crate::error::{DriverError, IntoResult}; -use crate::simt::context::CudaContext; -use crate::simt::stream::CudaStream; -use std::mem::MaybeUninit; -use std::sync::Arc; - -/// An RAII wrapper around a `CUevent` handle. -/// -/// Holds an `Arc` to ensure the context outlives the event. -/// Destroyed automatically via `cuEventDestroy` on [`Drop`]. -#[derive(Debug)] -pub struct CudaEvent { - /// Raw CUDA event handle. - pub(crate) cu_event: cuda_bindings::CUevent, - /// Owning context. Kept alive for the lifetime of this event. - pub(crate) ctx: Arc, -} - -/// # Safety -/// -/// `CUevent` handles are not thread-local. The CUDA driver permits recording -/// and waiting on events from any thread, provided the owning context is bound. -unsafe impl Send for CudaEvent {} -/// See [`Send`] impl. -unsafe impl Sync for CudaEvent {} - -/// Destroys the underlying `CUevent` on drop. -/// -/// Binds the context to the current thread first (required by -/// `cuEventDestroy`). Errors are recorded on the context rather than -/// panicking. -impl Drop for CudaEvent { - fn drop(&mut self) { - self.ctx.record_err(self.ctx.bind_to_thread()); - self.ctx - .record_err(unsafe { cuda_bindings::cuEventDestroy_v2(self.cu_event).result() }); - } -} - -impl CudaContext { - /// Creates a new CUDA event in this context. - /// - /// `flags` defaults to `CU_EVENT_DISABLE_TIMING` when `None`. Use - /// `Some(CU_EVENT_DEFAULT)` to enable timing queries via - /// [`CudaEvent::elapsed_ms`]. - pub fn new_event( - self: &Arc, - flags: Option, - ) -> Result { - let flags = flags.unwrap_or(cuda_bindings::CUevent_flags_enum_CU_EVENT_DISABLE_TIMING); - self.bind_to_thread()?; - let mut cu_event = MaybeUninit::uninit(); - let cu_event = unsafe { - cuda_bindings::cuEventCreate(cu_event.as_mut_ptr(), flags).result()?; - cu_event.assume_init() - }; - Ok(CudaEvent { - cu_event, - ctx: self.clone(), - }) - } -} - -impl CudaEvent { - /// Returns the raw `CUevent` handle. - pub fn cu_event(&self) -> cuda_bindings::CUevent { - self.cu_event - } - - /// Returns the parent [`CudaContext`]. - pub fn context(&self) -> &Arc { - &self.ctx - } - - /// Records this event on `stream`. - /// - /// The event captures the point in the stream's work queue at the time of - /// the call. A subsequent [`CudaStream::wait`] on this event will block - /// the waiting stream until all work prior to the record point completes. - pub fn record(&self, stream: &CudaStream) -> Result<(), DriverError> { - self.ctx.bind_to_thread()?; - unsafe { cuda_bindings::cuEventRecord(self.cu_event, stream.cu_stream()).result() } - } - - /// Blocks the calling thread until this event has been recorded and all - /// preceding stream work has completed. - pub fn synchronize(&self) -> Result<(), DriverError> { - self.ctx.bind_to_thread()?; - unsafe { cuda_bindings::cuEventSynchronize(self.cu_event).result() } - } - - /// Returns `true` when all work captured by this event has completed, - /// `false` when it is still in flight. Never blocks. - /// - /// Wraps `cuEventQuery`, mapping `CUDA_SUCCESS` to `Ok(true)` and - /// `CUDA_ERROR_NOT_READY` to `Ok(false)`. Any other code is a real - /// driver error. - pub fn query(&self) -> Result { - self.ctx.bind_to_thread()?; - match unsafe { cuda_bindings::cuEventQuery(self.cu_event) } { - cuda_bindings::cudaError_enum_CUDA_SUCCESS => Ok(true), - cuda_bindings::cudaError_enum_CUDA_ERROR_NOT_READY => Ok(false), - err => Err(DriverError(err)), - } - } - - /// Returns the elapsed time in milliseconds between `self` (start) and - /// `end`. - /// - /// Both events are synchronized before querying. Both events must have - /// been created **without** `CU_EVENT_DISABLE_TIMING`; otherwise the - /// driver returns `CUDA_ERROR_INVALID_HANDLE`. - /// - /// `self` must have been recorded before `end` in wall-clock time. - pub fn elapsed_ms(&self, end: &Self) -> Result { - self.synchronize()?; - end.synchronize()?; - let mut ms: f32 = 0.0; - unsafe { - cuda_bindings::cu_event_elapsed_time(&mut ms as *mut _, self.cu_event, end.cu_event) - .result()?; - } - Ok(ms) - } -} diff --git a/cuda-core/src/simt/stream.rs b/cuda-core/src/simt/stream.rs deleted file mode 100644 index 8a97417cc4..0000000000 --- a/cuda-core/src/simt/stream.rs +++ /dev/null @@ -1,252 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -//! CUDA stream management (RAII, host callbacks, fork/join). -//! -//! A [`CudaStream`] wraps a `CUstream` handle and ties its lifetime to its -//! parent [`CudaContext`]. Streams are created via -//! [`CudaContext::new_stream`] or [`CudaStream::fork`] and destroyed -//! automatically on [`Drop`]. -//! -//! # Ordering model -//! -//! Operations enqueued on the same stream execute in FIFO order. Operations on -//! different streams may overlap. Use [`fork`](CudaStream::fork) / -//! [`join`](CudaStream::join) or explicit events ([`record_event`](CudaStream::record_event), -//! [`wait`](CudaStream::wait)) to establish cross-stream ordering. -//! -//! # Host callbacks -//! -//! [`launch_host_function`](CudaStream::launch_host_function) enqueues a -//! host-side closure that the driver invokes after all preceding stream work -//! completes. This is the primary bridge between CUDA stream completion and -//! Rust `async` futures. - -use crate::error::{DriverError, IntoResult}; -use crate::simt::context::CudaContext; -use crate::simt::event::CudaEvent; -use std::ffi::c_void; -use std::mem::MaybeUninit; -use std::sync::atomic::Ordering; -use std::sync::Arc; - -/// An RAII wrapper around a `CUstream` handle. -/// -/// Holds an `Arc` to ensure the context outlives the stream. -/// A null `cu_stream` represents the per-context default stream (stream 0). -#[derive(Debug, PartialEq, Eq)] -pub struct CudaStream { - /// Raw CUDA stream handle. Null for the default stream. - pub(crate) cu_stream: cuda_bindings::CUstream, - /// Owning context. Kept alive for the lifetime of this stream. - pub(crate) ctx: Arc, -} - -/// # Safety -/// -/// `CUstream` handles are not thread-local. The CUDA driver permits enqueuing -/// work onto a stream from any thread, provided the owning context is bound -/// (which [`bind_to_thread`](CudaContext::bind_to_thread) ensures). -unsafe impl Send for CudaStream {} -/// See [`Send`] impl. -unsafe impl Sync for CudaStream {} - -/// Destroys the underlying `CUstream` on drop and decrements the context's -/// live stream count. -/// -/// The default stream (null handle) is never destroyed. Errors during -/// teardown are recorded on the context rather than panicking. -impl Drop for CudaStream { - fn drop(&mut self) { - self.ctx.record_err(self.ctx.bind_to_thread()); - if !self.cu_stream.is_null() { - self.ctx.num_streams.fetch_sub(1, Ordering::Relaxed); - self.ctx - .record_err(unsafe { cuda_bindings::cuStreamDestroy_v2(self.cu_stream).result() }); - } - } -} - -impl CudaStream { - /// Returns the raw `CUstream` handle (null for the default stream). - pub fn cu_stream(&self) -> cuda_bindings::CUstream { - self.cu_stream - } - - /// Returns the parent [`CudaContext`]. - pub fn context(&self) -> &Arc { - &self.ctx - } - - /// Returns the priority this stream actually runs at. - /// - /// Wraps `cuStreamGetPriority`. This is the value to read after - /// [`CudaContext::new_stream_with_priority`](crate::simt::context::CudaContext::new_stream_with_priority), - /// which clamps an out-of-range request to the device's range without - /// reporting it. Lower numbers are higher priorities. The default stream - /// and any stream from [`CudaContext::new_stream`](crate::simt::context::CudaContext::new_stream) - /// report `0`. - pub fn priority(&self) -> Result { - self.ctx.bind_to_thread()?; - let mut priority = MaybeUninit::uninit(); - unsafe { - cuda_bindings::cuStreamGetPriority(self.cu_stream, priority.as_mut_ptr()).result()?; - Ok(priority.assume_init()) - } - } - - /// Blocks the calling thread until all work enqueued on this stream - /// completes. - pub fn synchronize(&self) -> Result<(), DriverError> { - self.ctx.bind_to_thread()?; - unsafe { cuda_bindings::cuStreamSynchronize(self.cu_stream) }.result() - } - - /// Returns `true` when every operation enqueued on this stream has - /// completed, `false` while any is still in flight. Never blocks. - /// - /// Wraps `cuStreamQuery`, mapping `CUDA_SUCCESS` to `Ok(true)` and - /// `CUDA_ERROR_NOT_READY` to `Ok(false)`, as - /// [`CudaEvent::query`](crate::simt::event::CudaEvent::query) does for - /// `cuEventQuery`. Any other code is a real driver error. - /// - /// This is the completion half of the async bridge that - /// [`launch_host_function`](Self::launch_host_function) opens: a - /// `Future::poll` implementation needs to ask whether work has finished - /// without parking the executor's thread, which - /// [`synchronize`](Self::synchronize) cannot answer. - /// - /// A sticky error from earlier work on the stream surfaces here rather - /// than as `Ok(false)`, the same way it would from - /// [`synchronize`](Self::synchronize). - /// - /// On the default stream this reports the completion of every blocking - /// stream in the context, since the default stream orders against them - /// all. - pub fn query(&self) -> Result { - self.ctx.bind_to_thread()?; - match unsafe { cuda_bindings::cuStreamQuery(self.cu_stream) } { - cuda_bindings::cudaError_enum_CUDA_SUCCESS => Ok(true), - cuda_bindings::cudaError_enum_CUDA_ERROR_NOT_READY => Ok(false), - err => Err(DriverError(err)), - } - } - - /// Creates a new non-blocking stream that waits on all prior work in - /// `self` before executing its own. - /// - /// Semantically equivalent to creating a new stream and calling - /// [`join`](Self::join) on it with `self`, establishing a fork point - /// in the stream DAG. - pub fn fork(&self) -> Result, DriverError> { - self.ctx.bind_to_thread()?; - self.ctx.num_streams.fetch_add(1, Ordering::Relaxed); - let mut cu_stream = MaybeUninit::uninit(); - let cu_stream = unsafe { - cuda_bindings::cuStreamCreate( - cu_stream.as_mut_ptr(), - cuda_bindings::CUstream_flags_enum_CU_STREAM_NON_BLOCKING, - ) - .result()?; - cu_stream.assume_init() - }; - let stream = Arc::new(CudaStream { - cu_stream, - ctx: self.ctx.clone(), - }); - stream.join(self)?; - Ok(stream) - } - - /// Makes `self` wait on all prior work in `other`. - /// - /// Records an event on `other` and enqueues a wait on `self`. This is the - /// join side of the fork/join pattern: after this call, work enqueued on - /// `self` is guaranteed to observe all side effects of prior work on - /// `other`. - pub fn join(&self, other: &CudaStream) -> Result<(), DriverError> { - self.wait(&other.record_event(None)?) - } - - /// Records an event on this stream and returns it. - /// - /// `flags` defaults to `CU_EVENT_DISABLE_TIMING` when `None`, which is - /// cheaper than a timing-enabled event. Pass - /// `Some(CU_EVENT_DEFAULT)` if you need [`CudaEvent::elapsed_ms`]. - pub fn record_event( - &self, - flags: Option, - ) -> Result { - let event = self.ctx.new_event(flags)?; - event.record(self)?; - Ok(event) - } - - /// Enqueues a wait on `event` into this stream. - /// - /// All work enqueued on `self` after this call will not begin until - /// `event` has been recorded (i.e., all prior work on the stream that - /// recorded `event` has completed). - pub fn wait(&self, event: &CudaEvent) -> Result<(), DriverError> { - self.ctx.bind_to_thread()?; - unsafe { - cuda_bindings::cuStreamWaitEvent( - self.cu_stream, - event.cu_event(), - cuda_bindings::CUevent_wait_flags_enum_CU_EVENT_WAIT_DEFAULT, - ) - .result() - } - } - - /// Enqueues a host-side callback that the driver invokes after all prior - /// work on this stream completes. - /// - /// This is the bridge between CUDA stream ordering and Rust async: wrap a - /// `oneshot::Sender::send` or `Waker::wake` in `host_func` to unblock a - /// future when GPU work finishes. - /// - /// `host_func` is boxed, leaked into a raw pointer, and passed as user - /// data to `cuLaunchHostFunc`. The driver calls - /// `callback_wrapper` on a driver-internal - /// thread, which reclaims the box and invokes the closure. - /// - /// Panics inside the closure are caught and discarded to prevent unwinding - /// across the FFI boundary. - pub fn launch_host_function( - &self, - host_func: F, - ) -> Result<(), DriverError> { - let boxed = Box::new(host_func); - unsafe { - cuda_bindings::cuLaunchHostFunc( - self.cu_stream, - Some(Self::callback_wrapper::), - Box::into_raw(boxed) as *mut c_void, - ) - .result() - } - } - - /// `extern "C"` trampoline invoked by the CUDA driver on a driver-internal - /// thread when a host function callback fires. - /// - /// Reconstructs the `Box` from the raw pointer and calls the closure. - /// Panics are caught with `catch_unwind` to prevent unwinding across the - /// C ABI boundary. - /// - /// # Safety - /// - /// - `callback` must be a pointer produced by `Box::into_raw(Box::new(f))` - /// where `f: F`. - /// - Must be called exactly once per enqueued callback (double-free - /// otherwise). - unsafe extern "C" fn callback_wrapper(callback: *mut c_void) { - let _ = std::panic::catch_unwind(|| { - let callback: Box = unsafe { Box::from_raw(callback as *mut F) }; - callback(); - }); - } -} diff --git a/cuda-core/src/simt/vmm.rs b/cuda-core/src/simt/vmm.rs deleted file mode 100644 index 4784fc6099..0000000000 --- a/cuda-core/src/simt/vmm.rs +++ /dev/null @@ -1,519 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -//! CUDA Virtual Memory Management (VMM) API wrappers. -//! -//! The VMM APIs provide fine-grained control over physical memory allocation, -//! virtual address reservation, and mapping. Unlike `cuMemAlloc`, which bundles -//! all three steps, VMM separates them so that physical memory from one device -//! can be mapped into another device's virtual address space -- the foundation -//! for P2P symmetric heaps. -//! -//! All handle types are RAII: `PhysicalAllocation` releases via `cuMemRelease`, -//! `VirtualReservation` frees via `cuMemAddressFree`, and `Mapping` unmaps -//! via `cuMemUnmap`. Drop order matters -- mappings must be dropped before the -//! physical allocation or virtual reservation they reference. -//! -//! The module also wraps CUDA's multicast objects (`cuMulticast*`, CUDA -//! 12.1+): `MulticastObject` (compiled out on pre-12.1 toolkits, see the -//! build script probe) builds an NVLink SHARP (NVLS) team whose -//! mapped VA ranges respond to device-side `multimem.*` instructions with -//! switch-side reduction and broadcast across every bound GPU. See -//! `tests/vmm_multicast.rs` and the `nvls_all_reduce` example. - -use crate::error::{DriverError, IntoResult}; -use cuda_bindings::CUdeviceptr; -use std::mem::MaybeUninit; - -/// Sets the device ordinal on a `CUmemLocation_st`. -/// -/// CUDA 13.2 wraps `id` in an anonymous union (`__bindgen_anon_1.id`), while -/// older versions expose it directly. The memory layout is identical -- `id` is -/// always at offset 4 (after the `type_` enum). Writing via pointer works across -/// both layouts. -unsafe fn set_mem_location_device( - loc: &mut cuda_bindings::CUmemLocation_st, - device: cuda_bindings::CUdevice, -) { - loc.type_ = cuda_bindings::CUmemLocationType_enum_CU_MEM_LOCATION_TYPE_DEVICE; - unsafe { - let base = loc as *mut _ as *mut u8; - (base.add(4) as *mut i32).write(device); - } -} - -/// A physical memory allocation created by `cuMemCreate`. -/// -/// Owns the underlying `CUmemGenericAllocationHandle`. The allocation lives on -/// a specific device and can be mapped into any device's VA space that has been -/// granted access. -/// -/// Dropping this releases the physical memory. All `Mapping`s referencing this -/// allocation must be dropped first. -pub struct PhysicalAllocation { - handle: cuda_bindings::CUmemGenericAllocationHandle, - size: usize, - device: cuda_bindings::CUdevice, -} - -impl PhysicalAllocation { - /// Allocates `size` bytes of physical memory on `device`. - /// - /// `size` must be a multiple of the allocation granularity for the device - /// (query via [`allocation_granularity`]). - pub fn new(device: cuda_bindings::CUdevice, size: usize) -> Result { - let mut prop: cuda_bindings::CUmemAllocationProp_st = unsafe { std::mem::zeroed() }; - prop.type_ = cuda_bindings::CUmemAllocationType_enum_CU_MEM_ALLOCATION_TYPE_PINNED; - unsafe { set_mem_location_device(&mut prop.location, device) }; - - let mut handle = MaybeUninit::uninit(); - unsafe { - cuda_bindings::cuMemCreate(handle.as_mut_ptr(), size, &prop, 0).result()?; - Ok(Self { - handle: handle.assume_init(), - size, - device, - }) - } - } - - /// Returns the raw `CUmemGenericAllocationHandle`. - pub fn handle(&self) -> cuda_bindings::CUmemGenericAllocationHandle { - self.handle - } - - /// Returns the allocation size in bytes. - pub fn size(&self) -> usize { - self.size - } - - /// Returns the device this allocation lives on. - pub fn device(&self) -> cuda_bindings::CUdevice { - self.device - } -} - -impl Drop for PhysicalAllocation { - fn drop(&mut self) { - unsafe { - let _ = cuda_bindings::cuMemRelease(self.handle).result(); - } - } -} - -/// A reserved virtual address range created by `cuMemAddressReserve`. -/// -/// Owns a contiguous VA range `[base, base + size)`. Physical memory can be -/// mapped into this range via [`Mapping::new`]. The range is freed on drop. -/// -/// All `Mapping`s within this range must be dropped before the reservation. -pub struct VirtualReservation { - base: CUdeviceptr, - size: usize, -} - -impl VirtualReservation { - /// Reserves `size` bytes of virtual address space. - /// - /// The driver chooses the base address. `size` must be a multiple of the - /// allocation granularity. `alignment` can be 0 to let the driver choose. - pub fn new(size: usize, alignment: usize) -> Result { - let mut base = MaybeUninit::uninit(); - unsafe { - cuda_bindings::cuMemAddressReserve(base.as_mut_ptr(), size, alignment, 0, 0) - .result()?; - Ok(Self { - base: base.assume_init(), - size, - }) - } - } - - /// Returns the base device pointer of the reserved range. - pub fn base(&self) -> CUdeviceptr { - self.base - } - - /// Returns the reserved size in bytes. - pub fn size(&self) -> usize { - self.size - } -} - -impl Drop for VirtualReservation { - fn drop(&mut self) { - unsafe { - let _ = cuda_bindings::cuMemAddressFree(self.base, self.size).result(); - } - } -} - -/// A mapping of physical memory into a virtual address range. -/// -/// Created by [`Mapping::new`], which calls `cuMemMap` to bind a -/// `PhysicalAllocation` (or a portion of it) to a region within a -/// `VirtualReservation`. Dropped via `cuMemUnmap`. -pub struct Mapping { - va: CUdeviceptr, - size: usize, -} - -impl Mapping { - /// Maps `size` bytes of `phys` at `offset` into virtual address `va`. - /// - /// `va` must lie within a `VirtualReservation`. `offset` is the byte - /// offset into the physical allocation (typically 0 for full mappings). - /// `size` must be a multiple of the allocation granularity. - pub fn new( - va: CUdeviceptr, - size: usize, - phys: &PhysicalAllocation, - offset: usize, - ) -> Result { - unsafe { - cuda_bindings::cuMemMap(va, size, offset, phys.handle(), 0).result()?; - } - Ok(Self { va, size }) - } - - /// Maps `size` bytes of a multicast object at `offset` into virtual - /// address `va`. - /// - /// The resulting VA is a *multicast* view: `multimem.*` PTX instructions - /// issued against it operate on every copy bound to the object (see - /// [`MulticastObject`]). Like [`Mapping::new`], the mapping is not - /// accessible until [`set_access`] grants the accessing device permission. - /// - /// All devices must have been added to the multicast object (via - /// [`MulticastObject::add_device`]) before mapping it. - #[cfg(cuda_has_multicast)] - pub fn new_multicast( - va: CUdeviceptr, - size: usize, - multicast: &MulticastObject, - offset: usize, - ) -> Result { - unsafe { - cuda_bindings::cuMemMap(va, size, offset, multicast.handle(), 0).result()?; - } - Ok(Self { va, size }) - } - - /// Returns the virtual address this mapping occupies. - pub fn va(&self) -> CUdeviceptr { - self.va - } - - /// Returns the mapped size in bytes. - pub fn size(&self) -> usize { - self.size - } -} - -impl Drop for Mapping { - fn drop(&mut self) { - unsafe { - let _ = cuda_bindings::cuMemUnmap(self.va, self.size).result(); - } - } -} - -/// Sets read/write access on a virtual address range for one or more devices. -/// -/// After calling `cuMemMap`, the mapping is not yet accessible. This function -/// grants the specified `devices` read/write permission on the range -/// `[va, va + size)`. -/// -/// Typically called once after all mappings within a reservation are established. -pub fn set_access( - va: CUdeviceptr, - size: usize, - devices: &[cuda_bindings::CUdevice], -) -> Result<(), DriverError> { - let descs: Vec = devices - .iter() - .map(|&dev| { - let mut desc: cuda_bindings::CUmemAccessDesc_st = unsafe { std::mem::zeroed() }; - unsafe { set_mem_location_device(&mut desc.location, dev) }; - desc.flags = cuda_bindings::CUmemAccess_flags_enum_CU_MEM_ACCESS_FLAGS_PROT_READWRITE; - desc - }) - .collect(); - - unsafe { cuda_bindings::cuMemSetAccess(va, size, descs.as_ptr(), descs.len()) }.result() -} - -/// Queries the minimum allocation granularity for VMM operations on `device`. -/// -/// All sizes passed to [`PhysicalAllocation::new`], [`VirtualReservation::new`], -/// and [`Mapping::new`] must be multiples of this value. -pub fn allocation_granularity(device: cuda_bindings::CUdevice) -> Result { - let mut prop: cuda_bindings::CUmemAllocationProp_st = unsafe { std::mem::zeroed() }; - prop.type_ = cuda_bindings::CUmemAllocationType_enum_CU_MEM_ALLOCATION_TYPE_PINNED; - unsafe { set_mem_location_device(&mut prop.location, device) }; - - let mut granularity = MaybeUninit::uninit(); - unsafe { - cuda_bindings::cuMemGetAllocationGranularity( - granularity.as_mut_ptr(), - &prop, - cuda_bindings::CUmemAllocationGranularity_flags_enum_CU_MEM_ALLOC_GRANULARITY_MINIMUM, - ) - .result()?; - Ok(granularity.assume_init()) - } -} - -/// Rounds `size` up to the nearest multiple of `granularity`. -pub fn align_size(size: usize, granularity: usize) -> usize { - (size + granularity - 1) & !(granularity - 1) -} - -/// Granularity flavor for [`multicast_granularity`]. -#[cfg(cuda_has_multicast)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum MulticastGranularity { - /// Minimum required granularity for multicast sizes and offsets. - Minimum, - /// Recommended granularity for best performance. - Recommended, -} - -#[cfg(cuda_has_multicast)] -impl MulticastGranularity { - fn to_flag(self) -> cuda_bindings::CUmulticastGranularity_flags { - match self { - MulticastGranularity::Minimum => { - cuda_bindings::CUmulticastGranularity_flags_enum_CU_MULTICAST_GRANULARITY_MINIMUM - } - MulticastGranularity::Recommended => { - cuda_bindings::CUmulticastGranularity_flags_enum_CU_MULTICAST_GRANULARITY_RECOMMENDED - } - } - } -} - -/// Fills a `CUmulticastObjectProp_st` for a single-process team of -/// `num_devices` GPUs binding up to `size` bytes each. -/// -/// `handleTypes` is left at 0: the object cannot be exported to other -/// processes. Multi-process teams (exporting the handle over a POSIX file -/// descriptor or fabric handle) are out of scope for these wrappers. -#[cfg(cuda_has_multicast)] -fn multicast_prop(num_devices: u32, size: usize) -> cuda_bindings::CUmulticastObjectProp_st { - let mut prop: cuda_bindings::CUmulticastObjectProp_st = unsafe { std::mem::zeroed() }; - prop.numDevices = num_devices; - prop.size = size; - prop -} - -/// Returns whether `device` supports switch multicast and reduction -/// operations (`CU_DEVICE_ATTRIBUTE_MULTICAST_SUPPORTED`). -/// -/// Multicast requires an NVLink-switch-connected system (e.g. HGX/DGX -/// H100 or B200) and CUDA 12.1+. -#[cfg(cuda_has_multicast)] -pub fn multicast_supported(device: cuda_bindings::CUdevice) -> Result { - let mut value = MaybeUninit::uninit(); - let status = unsafe { - cuda_bindings::cuDeviceGetAttribute( - value.as_mut_ptr(), - cuda_bindings::CUdevice_attribute_enum_CU_DEVICE_ATTRIBUTE_MULTICAST_SUPPORTED, - device, - ) - }; - // Drivers older than CUDA 12.1 do not know this attribute and answer - // CUDA_ERROR_INVALID_VALUE. That means "no multicast on this system", - // not a failure, so callers can probe-and-skip uniformly. - if status == cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE { - return Ok(false); - } - status.result()?; - Ok(unsafe { value.assume_init() } != 0) -} - -/// Built against a pre-12.1 CUDA toolkit whose headers lack the multicast -/// API: no device can be used for multicast through these wrappers. -#[cfg(not(cuda_has_multicast))] -pub fn multicast_supported(_device: cuda_bindings::CUdevice) -> Result { - Ok(false) -} - -/// Queries the multicast size/offset granularity for a team of -/// `num_devices` GPUs binding `size` bytes each. -/// -/// All sizes and offsets passed to [`MulticastObject::new`] and -/// [`MulticastObject::bind_mem`] must be multiples of the -/// [`MulticastGranularity::Minimum`] value; use -/// [`MulticastGranularity::Recommended`] for best performance. -#[cfg(cuda_has_multicast)] -pub fn multicast_granularity( - num_devices: u32, - size: usize, - granularity: MulticastGranularity, -) -> Result { - let prop = multicast_prop(num_devices, size); - let mut value = MaybeUninit::uninit(); - unsafe { - cuda_bindings::cuMulticastGetGranularity(value.as_mut_ptr(), &prop, granularity.to_flag()) - .result()?; - Ok(value.assume_init()) - } -} - -/// A multicast object created by `cuMulticastCreate`: one virtual "team" -/// handle backed by up to one physical allocation per participating GPU. -/// -/// Once every team device is added ([`add_device`](Self::add_device)) and has -/// bound physical memory ([`bind_mem`](Self::bind_mem)), the object can be -/// mapped into each device's VA space with [`Mapping::new_multicast`]. -/// Device-side `multimem.ld_reduce` / `multimem.st` / `multimem.red` -/// instructions issued against that mapping operate on *all* bound copies at -/// once -- the NVSwitch performs the reduction/broadcast in the fabric -/// (NVLink SHARP, the mechanism behind NCCL's NVLS algorithm). -/// -/// Lifecycle rules, in order: -/// 1. [`MulticastObject::new`] with the final team size. -/// 2. [`add_device`](Self::add_device) exactly `num_devices` times. -/// 3. [`bind_mem`](Self::bind_mem) per device (after ALL devices are added). -/// 4. [`Mapping::new_multicast`] + [`set_access`] per device. -/// -/// Teardown reverses it: mappings first, then bindings ([`MulticastBinding`]), -/// then this object, then the physical allocations. -/// -/// Dropping releases the handle via `cuMemRelease` (the documented release -/// path for multicast objects). -#[cfg(cuda_has_multicast)] -pub struct MulticastObject { - handle: cuda_bindings::CUmemGenericAllocationHandle, - size: usize, - num_devices: u32, -} - -#[cfg(cuda_has_multicast)] -impl MulticastObject { - /// Creates a multicast object for a team of `num_devices` GPUs binding - /// up to `size` bytes each. - /// - /// `size` must be a multiple of the minimum multicast granularity - /// (query via [`multicast_granularity`]). The object is single-process - /// only (no exportable handle types). - pub fn new(num_devices: u32, size: usize) -> Result { - let prop = multicast_prop(num_devices, size); - let mut handle = MaybeUninit::uninit(); - unsafe { - cuda_bindings::cuMulticastCreate(handle.as_mut_ptr(), &prop).result()?; - Ok(Self { - handle: handle.assume_init(), - size, - num_devices, - }) - } - } - - /// Adds `device` to the multicast team. - /// - /// Must be called exactly [`num_devices`](Self::num_devices) times, once - /// per device, before any memory is bound. - pub fn add_device(&self, device: cuda_bindings::CUdevice) -> Result<(), DriverError> { - unsafe { cuda_bindings::cuMulticastAddDevice(self.handle, device).result() } - } - - /// Binds `size` bytes of `phys` (starting at `mem_offset`) to this - /// multicast object at `mc_offset`. - /// - /// The bound device is taken from the physical allocation. All offsets - /// and `size` must be multiples of the minimum multicast granularity, - /// and every team device must already have been added. A CUDA context - /// for the bound device must be current. - /// - /// The returned [`MulticastBinding`] unbinds on drop; it must be dropped - /// before this object and before `phys`. - pub fn bind_mem( - &self, - mc_offset: usize, - phys: &PhysicalAllocation, - mem_offset: usize, - size: usize, - ) -> Result { - unsafe { - cuda_bindings::cuMulticastBindMem( - self.handle, - mc_offset, - phys.handle(), - mem_offset, - size, - 0, - ) - .result()?; - } - Ok(MulticastBinding { - mc_handle: self.handle, - device: phys.device(), - mc_offset, - size, - }) - } - - /// Returns the raw multicast `CUmemGenericAllocationHandle`. - pub fn handle(&self) -> cuda_bindings::CUmemGenericAllocationHandle { - self.handle - } - - /// Returns the per-device bind capacity in bytes. - pub fn size(&self) -> usize { - self.size - } - - /// Returns the team size this object was created for. - pub fn num_devices(&self) -> u32 { - self.num_devices - } -} - -#[cfg(cuda_has_multicast)] -impl Drop for MulticastObject { - fn drop(&mut self) { - unsafe { - let _ = cuda_bindings::cuMemRelease(self.handle).result(); - } - } -} - -/// One device's physical memory bound into a [`MulticastObject`]. -/// -/// Created by [`MulticastObject::bind_mem`]; unbinds via `cuMulticastUnbind` -/// on drop. Must be dropped before the multicast object and before the -/// physical allocation it binds. -#[cfg(cuda_has_multicast)] -pub struct MulticastBinding { - mc_handle: cuda_bindings::CUmemGenericAllocationHandle, - device: cuda_bindings::CUdevice, - mc_offset: usize, - size: usize, -} - -#[cfg(cuda_has_multicast)] -impl MulticastBinding { - /// Returns the device whose memory is bound. - pub fn device(&self) -> cuda_bindings::CUdevice { - self.device - } -} - -#[cfg(cuda_has_multicast)] -impl Drop for MulticastBinding { - fn drop(&mut self) { - unsafe { - let _ = cuda_bindings::cuMulticastUnbind( - self.mc_handle, - self.device, - self.mc_offset, - self.size, - ) - .result(); - } - } -} diff --git a/cuda-core/src/stream.rs b/cuda-core/src/stream.rs new file mode 100644 index 0000000000..ee807b775f --- /dev/null +++ b/cuda-core/src/stream.rs @@ -0,0 +1,70 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Stream ordering and priority helpers. + +pub use crate::runtime::{Stream, Stream as CudaStream}; +use crate::{DriverError, Event, IntoResult}; +use std::{mem::MaybeUninit, sync::Arc}; + +impl Stream { + /// Returns the priority this stream actually runs at. + /// + /// Wraps `cuStreamGetPriority`. This is the value to read after + /// [`Context::new_stream_with_priority`](crate::context::Context::new_stream_with_priority), + /// which clamps an out-of-range request to the device's range without + /// reporting it. Lower numbers are higher priorities. The default stream + /// and any stream from [`Context::new_stream`](crate::context::Context::new_stream) + /// report `0`. + pub fn priority(&self) -> Result { + self.context.bind_to_thread()?; + let mut priority = MaybeUninit::uninit(); + unsafe { + cuda_bindings::cuStreamGetPriority(self.cu_stream, priority.as_mut_ptr()).result()?; + Ok(priority.assume_init()) + } + } + /// Creates a new non-blocking stream that waits on all prior work in + /// `self` before executing its own. + /// + /// Semantically equivalent to creating a new stream and calling + /// [`join`](Self::join) on it with `self`, establishing a fork point + /// in the stream DAG. + pub fn fork(&self) -> Result, DriverError> { + let stream = self.context.new_stream()?; + stream.join(self)?; + Ok(stream) + } + /// Makes `self` wait on all prior work in `other`. + /// + /// Records an event on `other` and enqueues a wait on `self`. This is the + /// join side of the fork/join pattern: after this call, work enqueued on + /// `self` is guaranteed to observe all side effects of prior work on + /// `other`. + pub fn join(&self, other: &Stream) -> Result<(), DriverError> { + self.wait(&other.record_event(None)?) + } + /// Records an event on this stream and returns it. + /// + /// `flags` defaults to `CU_EVENT_DISABLE_TIMING` when `None`, which is + /// cheaper than a timing-enabled event. Pass + /// `Some(CU_EVENT_DEFAULT)` if you need [`Event::elapsed_ms`]. + pub fn record_event( + &self, + flags: Option, + ) -> Result { + let event = self.context.new_event_with_flags(flags)?; + event.record(self)?; + Ok(event) + } + /// Enqueues a wait on `event` into this stream. + /// + /// All work enqueued on `self` after this call will not begin until + /// `event` has been recorded (i.e., all prior work on the stream that + /// recorded `event` has completed). + pub fn wait(&self, event: &Event) -> Result<(), DriverError> { + self.wait_event(event) + } +} diff --git a/cuda-core/tests/borrow_raw.rs b/cuda-core/tests/borrow_raw.rs index b2713d4b3e..6834d9bc3e 100644 --- a/cuda-core/tests/borrow_raw.rs +++ b/cuda-core/tests/borrow_raw.rs @@ -3,13 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -//! Verifies the ownership contract of `Device::borrow_raw` and +//! Verifies the ownership contract of `Context::borrow_raw` and //! `Stream::borrow_raw`: dropping a borrowed wrapper must not release the //! primary context or destroy the stream, so the source handles keep //! working afterward. use core::ffi::{c_int, c_void}; -use cuda_core::{Device, Function, Module, Stream}; +use cuda_core::{Context, Function, Module, Stream}; const NOOP_PTX: &str = "\ .version 7.0 @@ -23,12 +23,12 @@ const NOOP_PTX: &str = "\ "; fn has_gpu() -> bool { - Device::device_count().map(|n| n > 0).unwrap_or(false) + Context::device_count().map(|n| n > 0).unwrap_or(false) } -fn borrow_device(src: &Device) -> std::sync::Arc { +fn borrow_device(src: &Context) -> std::sync::Arc { unsafe { - Device::borrow_raw( + Context::borrow_raw( src.cu_ctx() as *mut c_void, src.cu_device() as c_int, src.ordinal(), @@ -36,7 +36,7 @@ fn borrow_device(src: &Device) -> std::sync::Arc { } } -fn borrow_stream(src: &Stream, dev: &std::sync::Arc) -> std::sync::Arc { +fn borrow_stream(src: &Stream, dev: &std::sync::Arc) -> std::sync::Arc { unsafe { Stream::borrow_raw(src.cu_stream() as *mut c_void, dev) } } @@ -45,16 +45,16 @@ fn borrowed_drop_leaves_source_device_usable() { if !has_gpu() { return; } - let source = Device::new(0).unwrap(); + let source = Context::new(0).unwrap(); { let borrowed = borrow_device(&source); borrowed.bind_to_thread().unwrap(); - unsafe { borrowed.synchronize() }.unwrap(); + borrowed.synchronize().unwrap(); } source.bind_to_thread().unwrap(); - unsafe { source.synchronize() }.unwrap(); + source.synchronize().unwrap(); let _stream = source.new_stream().unwrap(); } @@ -63,16 +63,16 @@ fn borrowed_drop_leaves_source_stream_usable() { if !has_gpu() { return; } - let source_dev = Device::new(0).unwrap(); + let source_dev = Context::new(0).unwrap(); let source_stream = source_dev.new_stream().unwrap(); { let borrowed_dev = borrow_device(&source_dev); let borrowed_stream = borrow_stream(&source_stream, &borrowed_dev); - unsafe { borrowed_stream.synchronize() }.unwrap(); + borrowed_stream.synchronize().unwrap(); } - unsafe { source_stream.synchronize() }.unwrap(); + source_stream.synchronize().unwrap(); } #[test] @@ -80,7 +80,7 @@ fn borrowed_module_drop_does_not_unload() { if !has_gpu() { return; } - let source_dev = Device::new(0).unwrap(); + let source_dev = Context::new(0).unwrap(); let source_module = source_dev.load_module_from_ptx_src(NOOP_PTX).unwrap(); { @@ -100,7 +100,7 @@ fn borrowed_function_uses_parent_module() { if !has_gpu() { return; } - let source_dev = Device::new(0).unwrap(); + let source_dev = Context::new(0).unwrap(); let source_module = source_dev.load_module_from_ptx_src(NOOP_PTX).unwrap(); let source_fn = source_module.load_function("noop").unwrap(); @@ -116,15 +116,15 @@ fn many_borrowed_drops_do_not_invalidate_source() { if !has_gpu() { return; } - let source_dev = Device::new(0).unwrap(); + let source_dev = Context::new(0).unwrap(); let source_stream = source_dev.new_stream().unwrap(); for _ in 0..8 { let borrowed_dev = borrow_device(&source_dev); let borrowed_stream = borrow_stream(&source_stream, &borrowed_dev); - unsafe { borrowed_stream.synchronize() }.unwrap(); + borrowed_stream.synchronize().unwrap(); } - unsafe { source_stream.synchronize() }.unwrap(); + source_stream.synchronize().unwrap(); let _another = source_dev.new_stream().unwrap(); } diff --git a/cuda-core/tests/simt_context_limits.rs b/cuda-core/tests/context_limits.rs similarity index 100% rename from cuda-core/tests/simt_context_limits.rs rename to cuda-core/tests/context_limits.rs diff --git a/cuda-core/tests/simt_context_sync_policy.rs b/cuda-core/tests/context_sync_policy.rs similarity index 100% rename from cuda-core/tests/simt_context_sync_policy.rs rename to cuda-core/tests/context_sync_policy.rs diff --git a/cuda-core/tests/simt_device_buffer.rs b/cuda-core/tests/device_buffer.rs similarity index 93% rename from cuda-core/tests/simt_device_buffer.rs rename to cuda-core/tests/device_buffer.rs index 6ebf9ace90..1e2ec8b925 100644 --- a/cuda-core/tests/simt_device_buffer.rs +++ b/cuda-core/tests/device_buffer.rs @@ -187,6 +187,10 @@ fn owned_wrapper_roundtrip_preserves_allocation_and_stream() { assert_eq!(words.to_host_vec(&stream).unwrap(), values); drop(words); stream.synchronize().unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while Arc::strong_count(&stream) != stream_refs - 1 && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } assert_eq!(Arc::strong_count(&stream), stream_refs - 1); } @@ -267,7 +271,7 @@ fn device_buffer_async_compat_methods_roundtrip() { } #[test] -fn async_allocation_ordinary_drop_waits_for_cross_stream_work() { +fn async_allocation_ordinary_drop_retires_cross_stream_work() { let _gated_test_guard = lock_gated_test(); let ctx = CudaContext::new(0).expect("failed to create CUDA context"); let allocation_stream = ctx @@ -295,13 +299,12 @@ fn async_allocation_ordinary_drop_waits_for_cross_stream_work() { .send(()) .expect("failed to send drop completion"); }); - finish_gated_worker( - "ordinary async buffer drop", - release_gate, - started_rx, - completion_rx, - drop_thread, - ); + started_rx.recv_timeout(COMPLETION_TIMEOUT).unwrap(); + let completed = completion_rx.recv_timeout(COMPLETION_TIMEOUT); + // Release the gate even on failure so the test cannot strand the driver. + release_gate.send(()).unwrap(); + drop_thread.join().unwrap(); + completed.expect("ordinary buffer drop must not wait for GPU work"); ctx.synchronize().expect("context cleanup failed"); } @@ -408,11 +411,19 @@ fn drop_async_bind_error_preserves_ordinary_cleanup() { ctx.bind_to_thread() .expect("ordinary drop should leave the context usable"); - assert_eq!( - unsafe { cuda_bindings::cuMemGetAddressRange_v2(&mut base, &mut size, ptr) }, - cuda_bindings::cudaError_enum_CUDA_ERROR_NOT_FOUND, - "ordinary drop must reclaim the still-armed allocation" - ); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let status = unsafe { cuda_bindings::cuMemGetAddressRange_v2(&mut base, &mut size, ptr) }; + if status == cuda_bindings::cudaError_enum_CUDA_ERROR_NOT_FOUND { + break; + } + assert_eq!(status, cuda_bindings::cudaError_enum_CUDA_SUCCESS); + assert!( + std::time::Instant::now() < deadline, + "retirement failed to reclaim the allocation" + ); + std::thread::sleep(Duration::from_millis(5)); + } } #[test] @@ -458,11 +469,11 @@ fn copy_from_host_with_pinned_source_allows_source_reuse_after_return() { // Dangerous-path regression: a stream-ordered (`cuMemAllocAsync`) buffer that // is dropped *implicitly* while a copy is still in flight, with no explicit -// synchronization by the caller. Ordinary `Drop` synchronizes the context -// before enqueueing the free, so the in-flight copy cannot race deallocation. +// synchronization by the caller. Completion leases retain the allocation +// until the in-flight copy finishes, so it cannot race deallocation. // Run under `compute-sanitizer --tool memcheck` to catch a regression. #[test] -fn uninitialized_async_implicit_drop_waits_for_pending_work() { +fn uninitialized_async_implicit_drop_retains_pending_work() { let ctx = CudaContext::new(0).expect("failed to create CUDA context"); let stream = ctx.new_stream().expect("failed to create CUDA stream"); diff --git a/cuda-core/tests/simt_device_buffer_casts.rs b/cuda-core/tests/device_buffer_casts.rs similarity index 93% rename from cuda-core/tests/simt_device_buffer_casts.rs rename to cuda-core/tests/device_buffer_casts.rs index fed5848915..c2f80df130 100644 --- a/cuda-core/tests/simt_device_buffer_casts.rs +++ b/cuda-core/tests/device_buffer_casts.rs @@ -9,7 +9,7 @@ //! turn byte storage into a `DeviceBuffer`. These cases only need to //! compile or fail to compile; none touches a GPU. Regenerate the //! expectations with `TRYBUILD=overwrite cargo test -p cuda-core --test -//! simt_device_buffer_casts` after an intentional diagnostic change. +//! device_buffer_casts` after an intentional diagnostic change. #[test] fn device_buffer_cast_bounds() { diff --git a/cuda-core/tests/device_buffer_casts/fail_cast_chunks_bool.stderr b/cuda-core/tests/device_buffer_casts/fail_cast_chunks_bool.stderr index 6701a8e469..bade663b28 100644 --- a/cuda-core/tests/device_buffer_casts/fail_cast_chunks_bool.stderr +++ b/cuda-core/tests/device_buffer_casts/fail_cast_chunks_bool.stderr @@ -16,7 +16,7 @@ error[E0277]: the trait bound `bool: AnyBitPattern` is not satisfied and $N others = note: required for `bool` to implement `AnyBitPattern` note: required by a bound in `DeviceBuffer::::cast_chunks` - --> src/simt/device_buffer.rs + --> src/device_buffer.rs | | pub fn cast_chunks(self) -> Result, Self> { | ^^^^^^^^^^^^^ required by this bound in `DeviceBuffer::::cast_chunks` diff --git a/cuda-core/tests/device_buffer_casts/fail_cast_elem_bool.stderr b/cuda-core/tests/device_buffer_casts/fail_cast_elem_bool.stderr index 64b7a30ea1..ca875fc475 100644 --- a/cuda-core/tests/device_buffer_casts/fail_cast_elem_bool.stderr +++ b/cuda-core/tests/device_buffer_casts/fail_cast_elem_bool.stderr @@ -16,7 +16,7 @@ error[E0277]: the trait bound `bool: AnyBitPattern` is not satisfied and $N others = note: required for `bool` to implement `AnyBitPattern` note: required by a bound in `DeviceBuffer::::cast_elem` - --> src/simt/device_buffer.rs + --> src/device_buffer.rs | | pub fn cast_elem(self) -> DeviceBuffer { | ^^^^^^^^^^^^^ required by this bound in `DeviceBuffer::::cast_elem` diff --git a/cuda-core/tests/device_buffer_casts/fail_wrap_elem_bool.stderr b/cuda-core/tests/device_buffer_casts/fail_wrap_elem_bool.stderr index cdcbd14208..58cf4ebdef 100644 --- a/cuda-core/tests/device_buffer_casts/fail_wrap_elem_bool.stderr +++ b/cuda-core/tests/device_buffer_casts/fail_wrap_elem_bool.stderr @@ -10,7 +10,7 @@ help: the trait `DeviceRepr` is implemented for `Wrapping` | unsafe impl DeviceRepr for core::num::Wrapping {} | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ note: required by a bound in `DeviceBuffer::::wrap_elem` - --> src/simt/device_buffer.rs + --> src/device_buffer.rs | | pub fn wrap_elem>(self) -> DeviceBuffer { | ^^^^^^^^^^^^^ required by this bound in `DeviceBuffer::::wrap_elem` diff --git a/cuda-core/tests/simt_device_buffer_leaks.rs b/cuda-core/tests/device_buffer_leaks.rs similarity index 89% rename from cuda-core/tests/simt_device_buffer_leaks.rs rename to cuda-core/tests/device_buffer_leaks.rs index 2e51c654ea..01d604eb74 100644 --- a/cuda-core/tests/simt_device_buffer_leaks.rs +++ b/cuda-core/tests/device_buffer_leaks.rs @@ -22,6 +22,13 @@ use std::sync::Arc; use cuda_core::{CudaContext, DeviceBuffer}; +fn wait_for_retirement(ctx: &Arc, baseline: usize) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while Arc::strong_count(ctx) != baseline && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(5)); + } +} + /// A live buffer must hold exactly one extra context strong count, and the /// count must return to baseline once the buffer drops. A construction /// helper that `mem::forget`s a context clone (as an arm/disarm pointer @@ -44,6 +51,7 @@ fn ctx_strong_count_returns_to_baseline_after_buffer_lifecycle() { drop(buf); } + wait_for_retirement(&ctx, baseline); assert_eq!( Arc::strong_count(&ctx), baseline, @@ -55,6 +63,7 @@ fn ctx_strong_count_returns_to_baseline_after_buffer_lifecycle() { drop(buf); } + wait_for_retirement(&ctx, baseline); assert_eq!( Arc::strong_count(&ctx), baseline, @@ -81,11 +90,14 @@ fn vram_returns_to_baseline_after_buffer_cycles() { free } + let baseline = Arc::strong_count(&ctx); + // Warm up driver allocator caches so the measured window is stable. for _ in 0..4 { let b = DeviceBuffer::::zeroed(&stream, 1 << 20).expect("warmup alloc failed"); drop(b); } + wait_for_retirement(&ctx, baseline); ctx.synchronize().expect("sync failed"); let before = free_mem(); @@ -93,6 +105,7 @@ fn vram_returns_to_baseline_after_buffer_cycles() { let b = DeviceBuffer::::zeroed(&stream, 1 << 20).expect("cycle alloc failed"); drop(b); } + wait_for_retirement(&ctx, baseline); ctx.synchronize().expect("sync failed"); let after = free_mem(); @@ -129,6 +142,7 @@ fn zero_length_construction_succeeds_for_both_constructors() { drop(zeroed); drop(from_host); + wait_for_retirement(&ctx, baseline); assert_eq!( Arc::strong_count(&ctx), baseline, diff --git a/cuda-core/tests/simt_device_copy_derive.rs b/cuda-core/tests/device_copy_derive.rs similarity index 100% rename from cuda-core/tests/simt_device_copy_derive.rs rename to cuda-core/tests/device_copy_derive.rs diff --git a/cuda-core/tests/simt_device_copy_impls.rs b/cuda-core/tests/device_copy_impls.rs similarity index 77% rename from cuda-core/tests/simt_device_copy_impls.rs rename to cuda-core/tests/device_copy_impls.rs index e3a3c1980b..68a17a70b6 100644 --- a/cuda-core/tests/simt_device_copy_impls.rs +++ b/cuda-core/tests/device_copy_impls.rs @@ -17,6 +17,16 @@ fn device_copy_supports_half_types() { assert_device_copy::(); } +#[test] +fn tile_storage_types_also_implement_device_copy() { + assert_device_copy::(); + assert_device_copy::(); + assert_device_copy::(); + assert_device_copy::(); + assert_device_copy::(); + assert_device_copy::(); +} + #[cfg(feature = "f16")] #[test] fn device_copy_supports_native_f16() { diff --git a/cuda-core/tests/simt_launch_contract_types.rs b/cuda-core/tests/launch_contract_types.rs similarity index 100% rename from cuda-core/tests/simt_launch_contract_types.rs rename to cuda-core/tests/launch_contract_types.rs diff --git a/cuda-core/tests/simt_vmm_multicast.rs b/cuda-core/tests/nvlink_multicast.rs similarity index 96% rename from cuda-core/tests/simt_vmm_multicast.rs rename to cuda-core/tests/nvlink_multicast.rs index a5716b9860..eb9220a4d3 100644 --- a/cuda-core/tests/simt_vmm_multicast.rs +++ b/cuda-core/tests/nvlink_multicast.rs @@ -20,8 +20,8 @@ //! (CUDA 12.1); see cuda-core's build script probe. #![cfg(cuda_has_multicast)] -use cuda_core::simt::context::CudaContext; -use cuda_core::simt::vmm; +use cuda_core::context::CudaContext; +use cuda_core::vmm; use cuda_core::IntoResult; use std::mem::MaybeUninit; @@ -115,7 +115,7 @@ fn multicast_two_gpu_team_roundtrip() { let byte_len = pattern.len() * std::mem::size_of::(); let stream0 = ctx0.default_stream(); unsafe { - cuda_core::simt::memory::memcpy_htod_async( + cuda_core::memory::memcpy_htod_async( mappings[0].0.base(), pattern.as_ptr(), byte_len, @@ -127,7 +127,7 @@ fn multicast_two_gpu_team_roundtrip() { let mut readback = vec![0u32; 256]; unsafe { - cuda_core::simt::memory::memcpy_dtoh_async( + cuda_core::memory::memcpy_dtoh_async( readback.as_mut_ptr(), mappings[0].0.base(), byte_len, diff --git a/cuda-core/tests/simt_pinned_host_buffer.rs b/cuda-core/tests/pinned_host_buffer.rs similarity index 100% rename from cuda-core/tests/simt_pinned_host_buffer.rs rename to cuda-core/tests/pinned_host_buffer.rs diff --git a/cuda-core/tests/shared_allocation.rs b/cuda-core/tests/shared_allocation.rs new file mode 100644 index 0000000000..9d3e35a2e1 --- /dev/null +++ b/cuda-core/tests/shared_allocation.rs @@ -0,0 +1,151 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use cuda_core::{Context, DeviceBuffer, RawDeviceBuffer}; +use std::sync::{mpsc, Arc}; +use std::time::{Duration, Instant}; + +fn metadata(context: usize) -> RawDeviceBuffer { + // SAFETY: these fake handles are used only for bookkeeping; no CUDA API + // sees them and borrowed allocation/context destruction performs no calls. + unsafe { + RawDeviceBuffer::borrowed_from_raw_parts( + 0x1000, + 128, + Context::borrow_raw(context as *mut _, 0, 0), + ) + } +} + +#[test] +fn byte_views_share_extent_identity_and_access_exclusion() { + let raw = metadata(0x2000); + let view = raw.slice_bytes(16, 32).unwrap(); + assert_eq!(view.cu_deviceptr(), raw.cu_deviceptr() + 16); + assert_eq!(view.allocation_id(), raw.allocation_id()); + assert!(raw.slice_bytes(120, 9).is_err()); + assert!(raw.slice_bytes(usize::MAX, 1).is_err()); + assert!(!raw.is_unique()); + let read = raw.acquire_on_stream(1, false, false).unwrap(); + let second_read = view.acquire_on_stream(2, false, false).unwrap(); + assert!(view.acquire_on_stream(2, true, false).is_err()); + drop((read, second_read)); + let write = view.acquire_on_stream(2, true, false).unwrap(); + assert!(raw.acquire_on_stream(1, false, false).is_err()); + assert!(raw.acquire_on_stream(2, true, false).is_ok()); + drop(view); + assert!(raw.is_unique(), "completion leases are not user aliases"); + drop(write); +} + +#[test] +fn same_ordinal_does_not_mean_same_context() { + let raw = metadata(0x2000); + let other = metadata(0x3000); + assert_eq!(raw.device_id(), other.device_id()); + assert_ne!(raw.context(), other.context()); + // SAFETY: this fake borrowed stream is never submitted or destroyed: forget + // it after testing the context check, before Drop could query the driver. + let stream = unsafe { cuda_core::Stream::borrow_raw(0x4000 as *mut _, other.context()) }; + let result = raw.acquire(&stream, false, false); + std::mem::forget(stream); + assert!(result.is_err()); +} + +struct Gate(mpsc::Sender<()>); +impl Gate { + fn arm(stream: &cuda_core::Stream) -> Self { + let (tx, rx) = mpsc::channel(); + stream + .launch_host_function(move || { + let _ = rx.recv_timeout(Duration::from_secs(10)); + }) + .unwrap(); + Self(tx) + } +} +impl Drop for Gate { + fn drop(&mut self) { + let _ = self.0.send(()); + } +} + +#[test] +fn pending_copy_excludes_other_stream_and_outlives_dropped_buffer() { + let context = Context::new(0).unwrap(); + let stream = context.new_stream().unwrap(); + let other = context.new_stream().unwrap(); + let mut buffer = DeviceBuffer::from_host(&stream, &[7u32; 32]).unwrap(); + let storage = buffer.storage().clone(); + let gate = Gate::arm(&stream); + buffer.zero_async(&stream).unwrap(); + assert!(storage.acquire(&other, false, false).is_err()); + let before = Instant::now(); + drop(buffer); + assert!( + before.elapsed() < Duration::from_secs(2), + "buffer drop waited for GPU work" + ); + assert!(storage.acquire(&other, false, false).is_err()); + drop(gate); + stream.synchronize().unwrap(); + // SAFETY: the zero fill completed, u32 accepts every representation, and + // this test is the only remaining user of the allocation. + let buffer = unsafe { DeviceBuffer::::from_storage_copy(storage, 32).unwrap() }; + assert_eq!(buffer.to_host_vec(&other).unwrap(), vec![0; 32]); +} + +#[test] +fn stream_drop_is_nonblocking_and_holds_context_until_completion() { + let context = Context::new(0).unwrap(); + let stream = context.new_stream().unwrap(); + let gate = Gate::arm(&stream); + let weak = Arc::downgrade(&context); + drop(context); + let before = Instant::now(); + drop(stream); + assert!( + before.elapsed() < Duration::from_secs(2), + "stream drop waited for GPU work" + ); + assert!(weak.upgrade().is_some()); + drop(gate); + let deadline = Instant::now() + Duration::from_secs(5); + while weak.upgrade().is_some() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!( + weak.upgrade().is_none(), + "retirement leaked an idle stream/context" + ); +} + +#[test] +fn fresh_thread_allocation_binds_its_context() { + let context = Context::new(0).unwrap(); + let stream = context.new_stream().unwrap(); + std::thread::spawn(move || { + let buffer = DeviceBuffer::from_host(&stream, &[11u32, 29]).unwrap(); + assert_eq!(buffer.to_host_vec(&stream).unwrap(), [11, 29]); + }) + .join() + .unwrap(); +} + +#[test] +fn empty_and_peeled_buffers_preserve_shared_read_access() { + let context = Context::new(0).unwrap(); + let stream = context.new_stream().unwrap(); + let empty = DeviceBuffer::::from_host(&stream, &[]).unwrap(); + assert!(!empty.shared_access_writes()); + let zeroed_empty = DeviceBuffer::::zeroed(&stream, 0).unwrap(); + assert!(!zeroed_empty.shared_access_writes()); + + let words = DeviceBuffer::from_host(&stream, &[1u32, 2, 3]).unwrap(); + let wrapped = words.wrap_elem::>(); + assert!(wrapped.shared_access_writes()); + let peeled = wrapped.peel_elem::(); + assert!(!peeled.shared_access_writes()); +} diff --git a/cuda-core/tests/simt_stream_priority.rs b/cuda-core/tests/stream_priority.rs similarity index 100% rename from cuda-core/tests/simt_stream_priority.rs rename to cuda-core/tests/stream_priority.rs diff --git a/cuda-core/tests/simt_stream_query.rs b/cuda-core/tests/stream_query.rs similarity index 100% rename from cuda-core/tests/simt_stream_query.rs rename to cuda-core/tests/stream_query.rs diff --git a/cuda-core/tests/vmm.rs b/cuda-core/tests/vmm.rs index ac36a4d42c..b27c6333c8 100644 --- a/cuda-core/tests/vmm.rs +++ b/cuda-core/tests/vmm.rs @@ -13,7 +13,7 @@ //! //! Skips when no GPU is present. -use cuda_core::{vmm, Device, IntoResult}; +use cuda_core::{vmm, Context, IntoResult}; use std::mem::MaybeUninit; #[test] @@ -58,7 +58,7 @@ fn vmm_single_gpu_roundtrip() { return; } - let device = Device::new(0).expect("GPU 0 device"); + let device = Context::new(0).expect("GPU 0 device"); let cu_device = device.cu_device(); let granularity = vmm::allocation_granularity(cu_device).expect("allocation granularity query"); diff --git a/cuda-core/tests/vmm_multicast.rs b/cuda-core/tests/vmm_multicast.rs index 0c8cb398d5..29cbbc198b 100644 --- a/cuda-core/tests/vmm_multicast.rs +++ b/cuda-core/tests/vmm_multicast.rs @@ -17,7 +17,7 @@ //! //! Requires an NVLink-switch system (e.g. HGX H100/B200); skips elsewhere. -use cuda_core::{vmm, Device, IntoResult}; +use cuda_core::{vmm, Context, IntoResult}; use std::mem::MaybeUninit; fn gpu_count() -> Result { @@ -37,8 +37,8 @@ fn multicast_two_gpu_team_roundtrip() { return; } - let ctx0 = Device::new(0).expect("GPU 0 device"); - let ctx1 = Device::new(1).expect("GPU 1 device"); + let ctx0 = Context::new(0).expect("GPU 0 device"); + let ctx1 = Context::new(1).expect("GPU 1 device"); let devices = [ctx0.cu_device(), ctx1.cu_device()]; for (i, &dev) in devices.iter().enumerate() { diff --git a/cuda-core/tests/simt_vmm_p2p.rs b/cuda-core/tests/vmm_p2p.rs similarity index 95% rename from cuda-core/tests/simt_vmm_p2p.rs rename to cuda-core/tests/vmm_p2p.rs index 8adca11b01..c8fa45c202 100644 --- a/cuda-core/tests/simt_vmm_p2p.rs +++ b/cuda-core/tests/vmm_p2p.rs @@ -23,9 +23,9 @@ //! //! Requires a system with at least 2 CUDA-capable GPUs with P2P support. -use cuda_core::simt::context::CudaContext; -use cuda_core::simt::peer; -use cuda_core::simt::vmm; +use cuda_core::context::CudaContext; +use cuda_core::peer; +use cuda_core::vmm; use cuda_core::IntoResult; use std::mem::MaybeUninit; @@ -62,7 +62,7 @@ fn vmm_alloc_map_set_access_roundtrip() { let stream = ctx.default_stream(); let pattern: Vec = (0..64).map(|i| (i * 3 + 7) as u8).collect(); unsafe { - cuda_core::simt::memory::memcpy_htod_async( + cuda_core::memory::memcpy_htod_async( va.base(), pattern.as_ptr(), pattern.len(), @@ -74,7 +74,7 @@ fn vmm_alloc_map_set_access_roundtrip() { let mut readback = vec![0u8; 64]; unsafe { - cuda_core::simt::memory::memcpy_dtoh_async( + cuda_core::memory::memcpy_dtoh_async( readback.as_mut_ptr(), va.base(), readback.len(), @@ -142,7 +142,7 @@ fn p2p_vmm_cross_gpu_access() { let stream0 = ctx0.default_stream(); ctx0.bind_to_thread().expect("bind ctx0"); unsafe { - cuda_core::simt::memory::memcpy_htod_async( + cuda_core::memory::memcpy_htod_async( va0.base(), pattern.as_ptr(), byte_len, @@ -157,7 +157,7 @@ fn p2p_vmm_cross_gpu_access() { let stream1 = ctx1.default_stream(); ctx1.bind_to_thread().expect("bind ctx1"); unsafe { - cuda_core::simt::memory::memcpy_dtoh_async( + cuda_core::memory::memcpy_dtoh_async( readback.as_mut_ptr(), va1.base(), byte_len, diff --git a/cuda-oxide/README.md b/cuda-oxide/README.md index ed157bd6c0..f69a45570c 100644 --- a/cuda-oxide/README.md +++ b/cuda-oxide/README.md @@ -31,7 +31,7 @@ Please see [CONTRIBUTING.md](../CONTRIBUTING.md) if you're interested in contrib ```rust use cuda_device::{cuda_module, kernel, thread, DisjointSlice}; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; // Device: generic kernel that applies any function to each element. @@ -306,7 +306,7 @@ cargo oxide run gemm_sol_final | `cuda-host` | Typed module loading, launch helpers, LTOIR loader | | `cuda-macros` | Proc macros (`#[cuda_module]`, `#[kernel]`, `gpu_printf!`) | | `cuda-bindings` | Raw `bindgen` FFI bindings to `cuda.h` (shared with cutile-rs) | -| `cuda-core` | Safe RAII wrappers (`CudaContext`, `DeviceBuffer`, ...); SIMT API under `cuda_core::simt` | +| `cuda-core` | Safe RAII wrappers (`CudaContext`, `DeviceBuffer`, ...); one host API shared by SIMT and Tile | | `cuda-async` | Async layer (`DeviceOperation`, `DeviceBox`, ...); SIMT API under `cuda_async::simt` | | `libnvvm-sys` | `dlopen` bindings to libNVVM (used by `cuda-host::ltoir`) | | `cuda-target-spec` | Shared CUDA target parsing and recorded LLVM PTX-floor policy | diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/scaffold.rs b/cuda-oxide/crates/cargo-oxide/src/commands/scaffold.rs index db77c02d29..28566eaa47 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/scaffold.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/scaffold.rs @@ -148,7 +148,7 @@ fn scaffold_main_rs(async_mode: bool) -> String { if async_mode { r#"use cuda_async::simt::device_context::{init_device_contexts, with_cuda_context}; use cuda_async::simt::device_operation::DeviceOperation; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; @@ -169,7 +169,7 @@ mod kernels { #[tokio::main] async fn main() -> Result<(), Box> { use cuda_async::simt::device_box::DeviceBox; - use cuda_core::simt::memory::{malloc_async, memcpy_dtoh_async, memcpy_htod_async}; + use cuda_core::memory::{malloc_async, memcpy_dtoh_async, memcpy_htod_async}; use std::mem; init_device_contexts(0, 1)?; @@ -190,9 +190,9 @@ async fn main() -> Result<(), Box> { memcpy_htod_async(b, b_host.as_ptr(), num_bytes, stream.cu_stream()).unwrap(); stream.synchronize().unwrap(); ( - DeviceBox::<[f32]>::from_raw_parts(a, N, 0), - DeviceBox::<[f32]>::from_raw_parts(b, N, 0), - DeviceBox::<[f32]>::from_raw_parts(c, N, 0), + DeviceBox::<[f32]>::from_raw_parts(a, N, stream.clone()), + DeviceBox::<[f32]>::from_raw_parts(b, N, stream.clone()), + DeviceBox::<[f32]>::from_raw_parts(c, N, stream.clone()), ) } })?; diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/tests.rs b/cuda-oxide/crates/cargo-oxide/src/commands/tests.rs index 2ddcaeb911..3c95706098 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/tests.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/tests.rs @@ -5653,7 +5653,7 @@ fn scaffold_async_template_keeps_async_deps_and_docs() { assert!(files.cargo_toml.contains("cuda-async = { git = \"")); assert!(!files.cargo_toml.contains("cuda-async = \"")); assert!(!files.cargo_toml.contains("cuda-bindings")); - assert!(files.main_rs.contains("use cuda_core::simt::LaunchConfig;")); + assert!(files.main_rs.contains("use cuda_core::LaunchConfig;")); assert!( files .main_rs diff --git a/cuda-oxide/crates/cuda-host/README.md b/cuda-oxide/crates/cuda-host/README.md index 7d921a8bb6..fccf549af0 100644 --- a/cuda-oxide/crates/cuda-host/README.md +++ b/cuda-oxide/crates/cuda-host/README.md @@ -8,7 +8,7 @@ keep `#[kernel]` on the actual GPU entry points, then load the embedded device artifact as a typed Rust value. ```rust -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/cuda-host/src/launch.rs b/cuda-oxide/crates/cuda-host/src/launch.rs index c3a2266731..f39e9e9519 100644 --- a/cuda-oxide/crates/cuda-host/src/launch.rs +++ b/cuda-oxide/crates/cuda-host/src/launch.rs @@ -466,6 +466,14 @@ pub fn push_kernel_row_width_device_slice( /// `UnsafeCell`-based or atomic element types under their synchronization /// contract. /// +/// `storage()` must return the allocation covering this exact view, with the +/// same CUDA context and shared access tracker as every other safe alias. +/// When it returns `None`, the implementation must independently retain the +/// allocation and exclude conflicting accesses until GPU completion, even +/// when the launch future is dropped or forgotten. Returning `false` from +/// `shared_access_writes()` additionally promises that the shared reference +/// cannot write any device memory in this range. +/// /// Implementing this trait is unsafe because generated launch methods trust /// the pointer and length without inspecting the allocation: /// @@ -491,6 +499,20 @@ pub unsafe trait KernelSliceArg { /// Returns the number of `T` elements in the allocation. fn len(&self) -> usize; + /// An independently retained allocation and access tracker for this argument. + /// Implementations returning None must themselves keep memory and exclusion + /// valid through GPU completion, including dropped and forgotten futures. + fn storage(&self) -> Option { + None + } + + /// Shared references to interior-mutable elements (including device atomics) + /// can write. Such arguments require exclusive tracking against ordinary + /// reads/copies; the conservative default also covers unknown custom types. + fn shared_access_writes(&self) -> bool { + true + } + /// Returns true when the allocation has zero elements. fn is_empty(&self) -> bool { self.len() == 0 @@ -514,6 +536,13 @@ pub unsafe trait KernelSliceArgMut: KernelSliceArg {} // SAFETY: DeviceBuffer owns the reported allocation and keeps its CUDA context // alive; its pointer and length accessors describe that allocation exactly. unsafe impl KernelSliceArg for cuda_core::DeviceBuffer { + fn storage(&self) -> Option { + Some(self.storage().clone()) + } + fn shared_access_writes(&self) -> bool { + self.shared_access_writes() + } + type Elem = T; fn cu_deviceptr(&self) -> cuda_core::sys::CUdeviceptr { @@ -534,6 +563,10 @@ unsafe impl KernelSliceArgMut for cuda_core::DeviceBuffer {} // SAFETY: DeviceBox owns the reported allocation and its raw constructor // requires the pointer, element count, and device ordinal to be truthful. unsafe impl KernelSliceArg for cuda_async::simt::device_box::DeviceBox<[T]> { + fn storage(&self) -> Option { + Some(self.storage().clone()) + } + type Elem = T; fn cu_deviceptr(&self) -> cuda_core::sys::CUdeviceptr { @@ -557,6 +590,13 @@ unsafe impl KernelSliceArg for Arc where B: KernelSliceArg + ?Sized, { + fn storage(&self) -> Option { + self.as_ref().storage() + } + fn shared_access_writes(&self) -> bool { + self.as_ref().shared_access_writes() + } + type Elem = B::Elem; fn cu_deviceptr(&self) -> cuda_core::sys::CUdeviceptr { @@ -568,6 +608,25 @@ where } } +#[cfg(feature = "async")] +// SAFETY: views retain their parent's storage and preserve typed byte extents; +// launch retention protects the allocation independently of the Rust borrow. +unsafe impl KernelSliceArg for cuda_core::DeviceBufferView<'_, T> { + type Elem = T; + fn cu_deviceptr(&self) -> cuda_core::sys::CUdeviceptr { + self.cu_deviceptr() + } + fn len(&self) -> usize { + self.len() + } + fn storage(&self) -> Option { + Some(self.storage().clone()) + } + fn shared_access_writes(&self) -> bool { + self.shared_access_writes() + } +} + #[doc(hidden)] #[cfg(feature = "async")] pub fn new_async_kernel_launch_builder<'a>( @@ -801,6 +860,9 @@ pub fn push_async_read_only_device_slice( ) where B: KernelSliceArg + ?Sized, { + if let Some(storage) = buffer.storage() { + launch.retain_buffer(storage, buffer.shared_access_writes()); + } let len = buffer.len(); launch.push_scalar_arg(kernel_slice_device_ptr::( buffer.cu_deviceptr(), @@ -824,6 +886,9 @@ pub fn push_async_writable_device_slice( ) where B: KernelSliceArgMut + ?Sized, { + if let Some(storage) = buffer.storage() { + launch.retain_buffer(storage, true); + } let len = buffer.len(); launch.push_scalar_arg(kernel_slice_device_ptr::( buffer.cu_deviceptr(), @@ -848,6 +913,7 @@ pub fn push_async_row_width_device_slice( launch: &mut cuda_async::simt::launch::AsyncKernelLaunchBuilder<'_>, bound: RowWidth<'_, T>, ) { + launch.retain_buffer(bound.buffer.storage().clone(), true); let (ptr, len, width) = row_width_device_buffer_arg(bound); launch.push_scalar_arg(ptr); launch.push_scalar_arg(len); @@ -869,6 +935,9 @@ pub fn push_async_owned_row_width_device_slice( ) where B: KernelSliceArgMut, { + if let Some(storage) = bound.buffer.storage() { + launch.retain_buffer(storage, true); + } let len = bound.buffer.len(); launch.push_scalar_arg(kernel_slice_device_ptr::( bound.buffer.cu_deviceptr(), diff --git a/cuda-oxide/crates/cuda-host/src/lib.rs b/cuda-oxide/crates/cuda-host/src/lib.rs index 1fa90924a7..6de2486788 100644 --- a/cuda-oxide/crates/cuda-host/src/lib.rs +++ b/cuda-oxide/crates/cuda-host/src/lib.rs @@ -51,7 +51,7 @@ //! ```ignore //! use cuda_device::{kernel, thread, DisjointSlice}; //! use cuda_host::cuda_module; -//! use cuda_core::simt::LaunchConfig; +//! use cuda_core::LaunchConfig; //! use cuda_core::{CudaContext, DeviceBuffer}; //! //! #[cuda_module] diff --git a/cuda-oxide/crates/cuda-host/tests/cuda_module_api.rs b/cuda-oxide/crates/cuda-host/tests/cuda_module_api.rs index e41e9c70d7..c61036465e 100644 --- a/cuda-oxide/crates/cuda-host/tests/cuda_module_api.rs +++ b/cuda-oxide/crates/cuda-host/tests/cuda_module_api.rs @@ -5,7 +5,7 @@ #![allow(dead_code, non_upper_case_globals, clippy::needless_lifetimes)] -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaStream, DeviceBuffer, LaunchConfig1D}; #[cfg(feature = "async")] use cuda_host::cuda_async::simt::device_box::DeviceBox; diff --git a/cuda-oxide/crates/cuda-host/tests/shared_launch_lifetimes.rs b/cuda-oxide/crates/cuda-host/tests/shared_launch_lifetimes.rs new file mode 100644 index 0000000000..b8f137f249 --- /dev/null +++ b/cuda-oxide/crates/cuda-host/tests/shared_launch_lifetimes.rs @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#![cfg(feature = "async")] + +use cuda_async::simt::{ + device_operation::DeviceOperation, launch::AsyncKernelLaunchBuilder, + scheduling_policies::SingleStream, +}; +use cuda_core::{Context, DeviceBuffer, LaunchConfig}; +use std::{ + future::Future, + pin::Pin, + sync::{Arc, mpsc}, + task::{Context as TaskContext, Waker}, + time::Duration, +}; + +const PTX: &str = ".version 7.0\n.target sm_52\n.address_size 64\n.visible .entry use_slice(.param .u64 ptr, .param .u64 len) { ret; }"; + +struct Gate(mpsc::Sender<()>); +impl Drop for Gate { + fn drop(&mut self) { + let _ = self.0.send(()); + } +} + +fn pending_launch_keeps_access(forget: bool, atomic: bool) { + let context = Context::new(0).unwrap(); + let stream = context.new_stream().unwrap(); + let other = context.new_stream().unwrap(); + let function = Arc::new( + context + .load_module_from_ptx_src(PTX) + .unwrap() + .load_function("use_slice") + .unwrap(), + ); + let buffer = DeviceBuffer::from_host(&stream, &[1u32; 32]).unwrap(); + let storage = buffer.storage().clone(); + let mut builder = AsyncKernelLaunchBuilder::new(function); + if atomic { + let buffer = buffer.wrap_elem::(); + cuda_host::push_async_read_only_device_slice(&mut builder, &buffer); + drop(buffer); + } else { + let mut buffer = buffer; + cuda_host::push_async_writable_device_slice(&mut builder, &mut buffer); + drop(buffer); + } + let (tx, rx) = mpsc::channel(); + stream + .launch_host_function(move || { + let _ = rx.recv_timeout(Duration::from_secs(10)); + }) + .unwrap(); + let gate = Gate(tx); + // SAFETY: PTX declares exactly the pointer/length packet above and no + // memory accesses. The descriptors still model a writable kernel argument, + // so this test can observe exclusion without deliberately racing GPU memory. + let launch = unsafe { + builder.finalize_unchecked(LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (1, 1, 1), + shared_mem_bytes: 0, + }) + }; + let mut future = launch + .schedule(&SingleStream { + stream: Some(stream.clone()), + }) + .unwrap(); + assert!( + Pin::new(&mut future) + .poll(&mut TaskContext::from_waker(Waker::noop())) + .is_pending() + ); + if forget { + std::mem::forget(future); + } else { + drop(future); + } + assert!( + storage.acquire(&other, false, false).is_err(), + "early future release lost a writable argument's access lease" + ); + drop(gate); + stream.synchronize().unwrap(); + assert!( + storage.acquire(&other, false, false).is_ok(), + "completed work must release exclusion" + ); +} + +#[test] +#[ignore = "requires a live CUDA GPU"] +fn dropped_borrowed_launch_retains_buffers_and_accesses() { + pending_launch_keeps_access(false, false); +} +#[test] +#[ignore = "requires a live CUDA GPU"] +fn forgotten_borrowed_launch_retains_buffers_and_accesses() { + pending_launch_keeps_access(true, false); +} +#[test] +#[ignore = "requires a live CUDA GPU"] +fn shared_atomic_arguments_exclude_ordinary_reads_after_drop() { + pending_launch_keeps_access(false, true); +} diff --git a/cuda-oxide/crates/cuda-macros/src/cuda_module/launchers.rs b/cuda-oxide/crates/cuda-macros/src/cuda_module/launchers.rs index 279a15cce4..d8603f5130 100644 --- a/cuda-oxide/crates/cuda-macros/src/cuda_module/launchers.rs +++ b/cuda-oxide/crates/cuda-macros/src/cuda_module/launchers.rs @@ -407,7 +407,7 @@ fn generate_cuda_module_legacy_launch_method(kernel: &CudaModuleKernel) -> Token #vis unsafe fn #fn_name #impl_generics ( &self, #stream: &::cuda_core::CudaStream, - #config: ::cuda_core::simt::LaunchConfig, + #config: ::cuda_core::LaunchConfig, #(#params),* ) -> ::core::result::Result<(), ::cuda_core::DriverError> #where_clause @@ -493,7 +493,7 @@ fn generate_cuda_module_prepared_launch_method(kernel: &CudaModuleKernel) -> Tok #vis unsafe fn #unchecked_name #impl_generics ( &self, #stream: &::cuda_core::CudaStream, - #config: ::cuda_core::simt::LaunchConfig, + #config: ::cuda_core::LaunchConfig, #(#params),* ) -> ::core::result::Result<(), ::cuda_core::DriverError> #where_clause @@ -556,7 +556,7 @@ fn generate_cuda_module_legacy_async_launch_method(kernel: &CudaModuleKernel) -> #[allow(clippy::multiple_bound_locations, clippy::too_many_arguments)] #vis unsafe fn #fn_name #impl_generics ( &self, - #config: ::cuda_core::simt::LaunchConfig, + #config: ::cuda_core::LaunchConfig, #(#params),* ) -> ::core::result::Result<::cuda_host::AsyncKernelLaunch<#async_lifetime>, ::cuda_core::DriverError> #where_clause @@ -681,7 +681,7 @@ fn generate_cuda_module_prepared_async_launch_method(kernel: &CudaModuleKernel) #[allow(clippy::multiple_bound_locations, clippy::too_many_arguments)] #vis unsafe fn #unchecked_name #impl_generics ( &self, - #config: ::cuda_core::simt::LaunchConfig, + #config: ::cuda_core::LaunchConfig, #(#params),* ) -> ::core::result::Result< ::cuda_host::AsyncKernelLaunch<#async_lifetime>, @@ -784,7 +784,7 @@ fn generate_cuda_module_legacy_owned_async_launch_method( #[allow(clippy::multiple_bound_locations, clippy::too_many_arguments)] #vis unsafe fn #fn_name #impl_generics ( &self, - #config: ::cuda_core::simt::LaunchConfig, + #config: ::cuda_core::LaunchConfig, #(#params),* ) -> ::core::result::Result<::cuda_host::OwnedAsyncKernelLaunch<#resources_ty>, ::cuda_core::DriverError> #where_clause @@ -955,7 +955,7 @@ fn generate_cuda_module_prepared_owned_async_launch_method( #[allow(clippy::multiple_bound_locations, clippy::too_many_arguments)] #vis unsafe fn #unchecked_name #impl_generics ( &self, - #config: ::cuda_core::simt::LaunchConfig, + #config: ::cuda_core::LaunchConfig, #(#params),* ) -> ::core::result::Result< ::cuda_host::OwnedAsyncKernelLaunch<#resources_ty>, diff --git a/cuda-oxide/crates/cuda-macros/src/lib.rs b/cuda-oxide/crates/cuda-macros/src/lib.rs index 0a3d738b55..50e4b423d4 100644 --- a/cuda-oxide/crates/cuda-macros/src/lib.rs +++ b/cuda-oxide/crates/cuda-macros/src/lib.rs @@ -1247,7 +1247,7 @@ pub fn cuda_launch(input: TokenStream) -> TokenStream { /// /// ```ignore /// use cuda_host::cuda_launch_async; -/// use cuda_core::simt::LaunchConfig; +/// use cuda_core::LaunchConfig; /// /// // SAFETY: ABI, lifetimes, geometry, and resources match vecadd. /// let op = unsafe { diff --git a/cuda-oxide/crates/cuda-macros/src/tests/cuda_module.rs b/cuda-oxide/crates/cuda-macros/src/tests/cuda_module.rs index be7a69ac79..3502c5a7f2 100644 --- a/cuda-oxide/crates/cuda-macros/src/tests/cuda_module.rs +++ b/cuda-oxide/crates/cuda-macros/src/tests/cuda_module.rs @@ -711,7 +711,7 @@ fn launch_contract_generates_prepared_and_unsafe_raw_paths() { expanded.contains("__cuda_oxide_prepared:&::cuda_core::PreparedLaunch<__map_CudaKernel>") ); assert!(expanded.contains("unsafefnmap_unchecked(")); - assert!(expanded.contains("__cuda_oxide_config:::cuda_core::simt::LaunchConfig")); + assert!(expanded.contains("__cuda_oxide_config:::cuda_core::LaunchConfig")); assert!(expanded.contains("min_alignment:128u32")); assert!(expanded.contains("with_min_compute_capability(8u32,0u32)")); assert!(expanded.contains("with_u32_coordinates()")); diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/async_launch_macro_requires_unsafe.rs b/cuda-oxide/crates/cuda-macros/tests/compile_fail/async_launch_macro_requires_unsafe.rs index a80b8e0c6f..2d0d3cd16a 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/async_launch_macro_requires_unsafe.rs +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/async_launch_macro_requires_unsafe.rs @@ -22,7 +22,7 @@ mod cuda_async { pub mod simt { pub mod launch { use cuda_core::CudaFunction; - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use std::sync::Arc; pub struct AsyncKernelLaunchBuilder; @@ -42,7 +42,7 @@ fn launch_without_unsafe(module: &FakeModule) { let _ = cuda_launch_async! { kernel: raw, module: module, - config: cuda_core::simt::LaunchConfig { + config: cuda_core::LaunchConfig { grid_dim: (1, 2, 1), block_dim: (64, 1, 1), shared_mem_bytes: 0, diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/async_launch_macro_requires_unsafe.stderr b/cuda-oxide/crates/cuda-macros/tests/compile_fail/async_launch_macro_requires_unsafe.stderr index 98269f9792..4901480d60 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/async_launch_macro_requires_unsafe.stderr +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/async_launch_macro_requires_unsafe.stderr @@ -5,7 +5,7 @@ error[E0133]: call to unsafe function `cuda_async::simt::launch::AsyncKernelLaun | _____________^ 43 | | kernel: raw, 44 | | module: module, -45 | | config: cuda_core::simt::LaunchConfig { +45 | | config: cuda_core::LaunchConfig { ... | 50 | | args: [], 51 | | }; diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/contract_unchecked_requires_unsafe.rs b/cuda-oxide/crates/cuda-macros/tests/compile_fail/contract_unchecked_requires_unsafe.rs index 402f69f667..6c3734e051 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/contract_unchecked_requires_unsafe.rs +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/contract_unchecked_requires_unsafe.rs @@ -22,7 +22,7 @@ fn unchecked_without_unsafe( module .contracted_unchecked( stream, - cuda_core::simt::LaunchConfig { + cuda_core::LaunchConfig { grid_dim: (1, 1, 1), block_dim: (64, 1, 1), shared_mem_bytes: 0, diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/contract_unchecked_requires_unsafe.stderr b/cuda-oxide/crates/cuda-macros/tests/compile_fail/contract_unchecked_requires_unsafe.stderr index 4a90e0a54a..5d059edbc3 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/contract_unchecked_requires_unsafe.stderr +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/contract_unchecked_requires_unsafe.stderr @@ -4,7 +4,7 @@ error[E0133]: call to unsafe function `LoadedModule::contracted_unchecked` is un 22 | / module 23 | | .contracted_unchecked( 24 | | stream, -25 | | cuda_core::simt::LaunchConfig { +25 | | cuda_core::LaunchConfig { ... | 30 | | input, 31 | | ) diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/cuda_module_include_kernel_boundary.rs b/cuda-oxide/crates/cuda-macros/tests/compile_fail/cuda_module_include_kernel_boundary.rs index 1293feb7a5..aa8aca52c5 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/cuda_module_include_kernel_boundary.rs +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/cuda_module_include_kernel_boundary.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use cuda_core::CudaStream; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_macros::cuda_module; #[cuda_module] diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/cuda_module_nested_type_mismatch.rs b/cuda-oxide/crates/cuda-macros/tests/compile_fail/cuda_module_nested_type_mismatch.rs index 6e43cd2287..59575c3e65 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/cuda_module_nested_type_mismatch.rs +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/cuda_module_nested_type_mismatch.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use cuda_core::CudaStream; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_macros::cuda_module; #[cuda_module] diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/cuda_module_pub_super_scope.rs b/cuda-oxide/crates/cuda-macros/tests/compile_fail/cuda_module_pub_super_scope.rs index 5fedcf6534..b29ea97625 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/cuda_module_pub_super_scope.rs +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/cuda_module_pub_super_scope.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use cuda_core::CudaStream; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_macros::cuda_module; #[cuda_module] diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_alias_fakes_row_width.rs b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_alias_fakes_row_width.rs index a66be86723..57896d07f0 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_alias_fakes_row_width.rs +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_alias_fakes_row_width.rs @@ -32,7 +32,7 @@ fn launch( // The spelling chose the three-word (ptr, len, width) marshalling, but // the resolved device slice is two words. The ABI bound must reject the // call before a mis-shaped packet can exist. - let cfg = cuda_core::simt::LaunchConfig { + let cfg = cuda_core::LaunchConfig { grid_dim: (1, 1, 1), block_dim: (64, 1, 1), shared_mem_bytes: 0, diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_alias_hides_row_width.rs b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_alias_hides_row_width.rs index fbbcaff005..88445a9f9f 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_alias_hides_row_width.rs +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_alias_hides_row_width.rs @@ -32,7 +32,7 @@ fn launch( // The spelling chose the two-word (ptr, len) marshalling, so this raw // launch would push two kernel parameters for a three-parameter kernel. // The ABI bound must reject the call. - let cfg = cuda_core::simt::LaunchConfig { + let cfg = cuda_core::LaunchConfig { grid_dim: (1, 1, 1), block_dim: (64, 1, 1), shared_mem_bytes: 0, diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_requires_unsafe.rs b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_requires_unsafe.rs index 112ee2f037..6edd83f0b3 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_requires_unsafe.rs +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_requires_unsafe.rs @@ -28,7 +28,7 @@ fn launch_without_unsafe( kernel: dummy, stream: stream, module: module, - config: cuda_core::simt::LaunchConfig::for_num_elems(1), + config: cuda_core::LaunchConfig::for_num_elems(1), args: [x] } .unwrap(); diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_requires_unsafe.stderr b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_requires_unsafe.stderr index f63438e3eb..fe6ee6e6b3 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_requires_unsafe.stderr +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_requires_unsafe.stderr @@ -5,7 +5,7 @@ error[E0133]: call to unsafe function `launch_kernel_on_stream` is unsafe and re 28 | | kernel: dummy, 29 | | stream: stream, 30 | | module: module, -31 | | config: cuda_core::simt::LaunchConfig::for_num_elems(1), +31 | | config: cuda_core::LaunchConfig::for_num_elems(1), 32 | | args: [x] 33 | | } | |_____^ call to unsafe function diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/uncontracted_launch_requires_unsafe.rs b/cuda-oxide/crates/cuda-macros/tests/compile_fail/uncontracted_launch_requires_unsafe.rs index 5a1fa7d265..b2394780a5 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/uncontracted_launch_requires_unsafe.rs +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/uncontracted_launch_requires_unsafe.rs @@ -20,7 +20,7 @@ fn launch_without_unsafe( module .uncontracted( stream, - cuda_core::simt::LaunchConfig { + cuda_core::LaunchConfig { grid_dim: (1, 2, 1), block_dim: (64, 1, 1), shared_mem_bytes: 0, diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/uncontracted_launch_requires_unsafe.stderr b/cuda-oxide/crates/cuda-macros/tests/compile_fail/uncontracted_launch_requires_unsafe.stderr index 19a3595186..ea7d2fd9a1 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/uncontracted_launch_requires_unsafe.stderr +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/uncontracted_launch_requires_unsafe.stderr @@ -4,7 +4,7 @@ error[E0133]: call to unsafe function `LoadedModule::uncontracted` is unsafe and 20 | / module 21 | | .uncontracted( 22 | | stream, -23 | | cuda_core::simt::LaunchConfig { +23 | | cuda_core::LaunchConfig { ... | 28 | | 7, 29 | | ) diff --git a/cuda-oxide/crates/cuda-macros/tests/pass/async_launch_macro_in_unsafe.rs b/cuda-oxide/crates/cuda-macros/tests/pass/async_launch_macro_in_unsafe.rs index c983dd75f5..d0b5a1933b 100644 --- a/cuda-oxide/crates/cuda-macros/tests/pass/async_launch_macro_in_unsafe.rs +++ b/cuda-oxide/crates/cuda-macros/tests/pass/async_launch_macro_in_unsafe.rs @@ -22,7 +22,7 @@ mod cuda_async { pub mod simt { pub mod launch { use cuda_core::CudaFunction; - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use std::sync::Arc; pub struct AsyncKernelLaunchBuilder; @@ -43,7 +43,7 @@ fn launch_with_unsafe(module: &FakeModule) { cuda_launch_async! { kernel: raw, module: module, - config: cuda_core::simt::LaunchConfig { + config: cuda_core::LaunchConfig { grid_dim: (1, 2, 1), block_dim: (64, 1, 1), shared_mem_bytes: 0, diff --git a/cuda-oxide/crates/cuda-macros/tests/pass/const_generic_hygiene.rs b/cuda-oxide/crates/cuda-macros/tests/pass/const_generic_hygiene.rs index 0ede621af9..85fb0a1bca 100644 --- a/cuda-oxide/crates/cuda-macros/tests/pass/const_generic_hygiene.rs +++ b/cuda-oxide/crates/cuda-macros/tests/pass/const_generic_hygiene.rs @@ -3,7 +3,7 @@ #![allow(dead_code, non_upper_case_globals)] -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaModule, CudaStream}; use cuda_macros::{cuda_launch, kernel}; use std::sync::Arc; diff --git a/cuda-oxide/crates/cuda-macros/tests/pass/cuda_module_inline_namespaces.rs b/cuda-oxide/crates/cuda-macros/tests/pass/cuda_module_inline_namespaces.rs index c964a060d1..aa475b823e 100644 --- a/cuda-oxide/crates/cuda-macros/tests/pass/cuda_module_inline_namespaces.rs +++ b/cuda-oxide/crates/cuda-macros/tests/pass/cuda_module_inline_namespaces.rs @@ -5,7 +5,7 @@ #![allow(dead_code)] use cuda_core::CudaStream; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_macros::{cuda_module, kernel}; #[derive(Clone, Copy)] diff --git a/cuda-oxide/crates/cuda-macros/tests/pass/grid_constant_arguments.rs b/cuda-oxide/crates/cuda-macros/tests/pass/grid_constant_arguments.rs index 29cd8e0a86..fe07e8383c 100644 --- a/cuda-oxide/crates/cuda-macros/tests/pass/grid_constant_arguments.rs +++ b/cuda-oxide/crates/cuda-macros/tests/pass/grid_constant_arguments.rs @@ -42,7 +42,7 @@ fn typed_launches( descriptor: Descriptor, typed_descriptor: TensorMap, ) { - let config = cuda_core::simt::LaunchConfig::for_num_elems(1); + let config = cuda_core::LaunchConfig::for_num_elems(1); // SAFETY: this function only type-checks the by-value host ABI; it is never run. unsafe { module.plain(stream, config, descriptor).unwrap(); diff --git a/cuda-oxide/crates/cuda-macros/tests/pass/uncontracted_launch_in_unsafe.rs b/cuda-oxide/crates/cuda-macros/tests/pass/uncontracted_launch_in_unsafe.rs index 5fc62ce57c..0fa24556be 100644 --- a/cuda-oxide/crates/cuda-macros/tests/pass/uncontracted_launch_in_unsafe.rs +++ b/cuda-oxide/crates/cuda-macros/tests/pass/uncontracted_launch_in_unsafe.rs @@ -21,7 +21,7 @@ fn launch_with_unsafe( module .uncontracted( stream, - cuda_core::simt::LaunchConfig { + cuda_core::LaunchConfig { grid_dim: (1, 2, 1), block_dim: (64, 1, 1), shared_mem_bytes: 0, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/abi_hmm/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/abi_hmm/src/main.rs index f3dab8edbd..20c46ab9c7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/abi_hmm/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/abi_hmm/src/main.rs @@ -37,7 +37,7 @@ //! ``` use cuda_core::CudaContext; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_device::{cuda_module, kernel, thread}; // ============================================================================= diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/addressof_sharedarray/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/addressof_sharedarray/src/main.rs index c2c124323e..9052739af7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/addressof_sharedarray/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/addressof_sharedarray/src/main.rs @@ -47,7 +47,7 @@ #![allow(static_mut_refs)] #![allow(clippy::assign_op_pattern)] // Expanded assignment preserves the addressof repro CFG. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, SharedArray, device, kernel, thread}; use cuda_host::{cuda_module, ltoir}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_loads/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_loads/src/main.rs index de2bd536ea..2b5b6bd30a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_loads/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_loads/src/main.rs @@ -26,7 +26,7 @@ //! //! Run: `cargo oxide run aligned_field_loads` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_stores/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_stores/src/main.rs index 2bae17b7e0..a7948e54ad 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_stores/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_stores/src/main.rs @@ -33,7 +33,7 @@ //! //! Run: `cargo oxide run aligned_field_stores` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_constants/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_constants/src/main.rs index 686f91339e..3f2a47b3dc 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_constants/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_constants/src/main.rs @@ -32,7 +32,7 @@ //! ./crates/rustc-codegen-cuda/examples/array_constants/verify-code-shape.sh use core::mem::MaybeUninit; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_for_loop/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_for_loop/src/main.rs index 9b7a1c2917..d430175d91 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_for_loop/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_for_loop/src/main.rs @@ -26,7 +26,7 @@ //! //! Run: cargo oxide run array_for_loop -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_index/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_index/src/main.rs index 3a56f6fe09..31e610a29f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_index/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_index/src/main.rs @@ -24,7 +24,7 @@ //! //! Once array index writes are implemented, all tests should pass. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_init/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_init/src/main.rs index 8e47e960a2..707cc6aed1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_init/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_init/src/main.rs @@ -15,7 +15,7 @@ //! //! Run: cargo oxide run array_init -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/asin_acos_smoke/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/asin_acos_smoke/src/main.rs index c8773d936d..72d92b7e66 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/asin_acos_smoke/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/asin_acos_smoke/src/main.rs @@ -34,7 +34,7 @@ //! //! Exits 0 on PASS, 1 on FAIL. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::{cuda_module, ltoir}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/assume_init_uninhabited_trap/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/assume_init_uninhabited_trap/src/main.rs index df1a653686..02c41b3909 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/assume_init_uninhabited_trap/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/assume_init_uninhabited_trap/src/main.rs @@ -10,7 +10,7 @@ //! ("attempted to instantiate uninhabited type"), so the kernel must trap. //! If the launch succeeds instead, the panic path was compiled away. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/async_mlp/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/async_mlp/src/main.rs index de892af8f0..2372cba196 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/async_mlp/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/async_mlp/src/main.rs @@ -39,10 +39,8 @@ use cuda_async::simt::device_box::DeviceBox; use cuda_async::simt::device_context::init_device_contexts; use cuda_async::simt::device_operation::{self, DeviceOperation, Zippable, value}; use cuda_async::zip; -use cuda_core::simt::LaunchConfig; -use cuda_core::simt::memory::{ - malloc_async, memcpy_dtoh_async, memcpy_htod_async, memset_d8_async, -}; +use cuda_core::LaunchConfig; +use cuda_core::memory::{malloc_async, memcpy_dtoh_async, memcpy_htod_async, memset_d8_async}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; use std::future::IntoFuture; @@ -142,7 +140,7 @@ fn h2d(host_data: Vec) -> impl DeviceOperation> { unsafe { let dptr = malloc_async(stream.cu_stream(), num_bytes).unwrap(); memcpy_htod_async(dptr, host_data.as_ptr(), num_bytes, stream.cu_stream()).unwrap(); - value(DeviceBox::from_raw_parts(dptr, n, ctx.get_device_id())) + value(DeviceBox::from_raw_parts(dptr, n, stream.clone())) } }) } @@ -156,7 +154,7 @@ fn zeros(n: usize) -> impl DeviceOperation> { unsafe { let dptr = malloc_async(stream.cu_stream(), num_bytes).unwrap(); memset_d8_async(dptr, 0, num_bytes, stream.cu_stream()).unwrap(); - value(DeviceBox::from_raw_parts(dptr, n, ctx.get_device_id())) + value(DeviceBox::from_raw_parts(dptr, n, stream.clone())) } }) } diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/async_vecadd/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/async_vecadd/src/main.rs index d79daadf61..32a2c37c84 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/async_vecadd/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/async_vecadd/src/main.rs @@ -46,8 +46,8 @@ async fn main() -> Result<(), Box> { use cuda_async::simt::device_box::DeviceBox; use cuda_async::simt::device_context::init_device_contexts; use cuda_async::simt::device_operation::DeviceOperation; - use cuda_core::simt::LaunchConfig; - use cuda_core::simt::memory::{malloc_async, memcpy_dtoh_async, memcpy_htod_async}; + use cuda_core::LaunchConfig; + use cuda_core::memory::{malloc_async, memcpy_dtoh_async, memcpy_htod_async}; use std::mem; println!("=== Async Vector Addition (cuda-async) ===\n"); @@ -83,9 +83,9 @@ async fn main() -> Result<(), Box> { stream.synchronize().unwrap(); - let a_dev = DeviceBox::<[f32]>::from_raw_parts(a_dptr, N, 0); - let b_dev = DeviceBox::<[f32]>::from_raw_parts(b_dptr, N, 0); - let c_dev = DeviceBox::<[f32]>::from_raw_parts(c_dptr, N, 0); + let a_dev = DeviceBox::<[f32]>::from_raw_parts(a_dptr, N, stream.clone()); + let b_dev = DeviceBox::<[f32]>::from_raw_parts(b_dptr, N, stream.clone()); + let c_dev = DeviceBox::<[f32]>::from_raw_parts(c_dptr, N, stream.clone()); (a_dev, b_dev, c_dev) } })?; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/atomic_f16/src/bin/bench.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/atomic_f16/src/bin/bench.rs index ba9f8d35c8..3a24a09025 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/atomic_f16/src/bin/bench.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/atomic_f16/src/bin/bench.rs @@ -12,7 +12,7 @@ use std::sync::Arc; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::atomic::{AtomicOrdering, DeviceAtomicF16, DeviceAtomicF32}; use cuda_device::{DisjointSlice, kernel, thread}; @@ -210,7 +210,7 @@ fn reset( stream: &CudaStream, ) -> Result<(), Box> { unsafe { - cuda_core::simt::memory::memset_d8_async( + cuda_core::memory::memset_d8_async( buffer.cu_deviceptr(), 0, buffer.num_bytes(), diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/atomic_f16/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/atomic_f16/src/main.rs index 94a77623bb..83939b562e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/atomic_f16/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/atomic_f16/src/main.rs @@ -9,7 +9,7 @@ #![feature(f16)] -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::atomic::{AtomicOrdering, BlockAtomicF16, DeviceAtomicF16, SystemAtomicF16}; use cuda_device::{DisjointSlice, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/atomics/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/atomics/src/main.rs index 774cd77142..453c2e8a39 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/atomics/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/atomics/src/main.rs @@ -46,7 +46,7 @@ //! cargo oxide run atomics use core::sync::atomic::Ordering; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::atomic::{ AtomicOrdering, BlockAtomicU32, DeviceAtomicF32, DeviceAtomicF64, DeviceAtomicI32, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/barrier/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/barrier/src/main.rs index f450bff9f0..3ebcec49f7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/barrier/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/barrier/src/main.rs @@ -15,7 +15,7 @@ //! Build and run with: //! cargo oxide run barrier -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::barrier::{ Barrier, mbarrier_arrive, mbarrier_arrive_no_complete, mbarrier_init, mbarrier_inval, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_arith/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_arith/src/main.rs index 52f20059ce..f993c4dc52 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_arith/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_arith/src/main.rs @@ -3,7 +3,7 @@ //! End-to-end example exercising all eight packed bf16x2 arithmetic intrinsics. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::bf16x2::{ abs_bf16x2, add_bf16x2, fma_relu_bf16x2, max_bf16x2, min_bf16x2, mul_bf16x2, neg_bf16x2, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_fma/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_fma/src/main.rs index cf0a5af7bc..50486794a7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_fma/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_fma/src/main.rs @@ -3,7 +3,7 @@ //! Behavior example for the packed bf16x2 FMA intrinsic. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::bf16x2::fma_bf16x2; use cuda_device::convert::cvt_bf16x2_f32; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/bool_phi_cmp/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/bool_phi_cmp/src/main.rs index 6225168543..ea5f112864 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/bool_phi_cmp/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/bool_phi_cmp/src/main.rs @@ -24,7 +24,7 @@ * cargo oxide run bool_phi_cmp */ -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/carrying_mul_add/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/carrying_mul_add/src/main.rs index 3c91191070..31a6c243ce 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/carrying_mul_add/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/carrying_mul_add/src/main.rs @@ -30,7 +30,7 @@ #![feature(signed_bigint_helpers)] -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_chunks/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_chunks/src/main.rs index 80b013c00b..3e61123662 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_chunks/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_chunks/src/main.rs @@ -15,7 +15,7 @@ //! //! Run: `cargo oxide run cast_chunks` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_tests/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_tests/src/main.rs index 0fc21de2dd..1f907f8260 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_tests/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_tests/src/main.rs @@ -24,7 +24,7 @@ //! the correct LLVM instruction. Struct↔ptr conversions (fat pointers, newtypes) //! are handled generically via `emit_pointer_cast` in mir-lower. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cbrt_smoke/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cbrt_smoke/src/main.rs index 4c84f7469a..4a8507f41b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cbrt_smoke/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cbrt_smoke/src/main.rs @@ -33,7 +33,7 @@ //! //! Exits 0 on PASS, 1 on FAIL. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::{cuda_module, ltoir}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/checked_arith/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/checked_arith/src/main.rs index dfdc160be1..50c7e5dfe0 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/checked_arith/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/checked_arith/src/main.rs @@ -21,7 +21,7 @@ //! cargo oxide run checked_arith //! CUDA_OXIDE_NO_OPT=1 cargo oxide run checked_arith -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/clc/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/clc/src/main.rs index ddd6d92684..eb8c4191ef 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/clc/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/clc/src/main.rs @@ -20,7 +20,7 @@ //! **Hardware Requirements:** Blackwell (B200, GB200) with SM 100+ use core::ptr::addr_of_mut; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::barrier::{ Barrier, fence_proxy_async_shared_cta, mbarrier_arrive_expect_tx, mbarrier_init, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cluster/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cluster/src/main.rs index d9d2a5c18f..4a0f9ed9ad 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cluster/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cluster/src/main.rs @@ -274,7 +274,7 @@ mod kernels { // ============================================================================ fn main() { - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; println!("=== Thread Block Cluster Tests (sm_90+) ===\n"); diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/compiler_features/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/compiler_features/src/main.rs index b774c7856b..7e5981601d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/compiler_features/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/compiler_features/src/main.rs @@ -21,7 +21,7 @@ //! //! Run: cargo oxide run compiler_features -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::shared::cvta_generic_to_shared_offset; use cuda_device::{DisjointSlice, SharedArray, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/complex_mul/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/complex_mul/src/main.rs index dd9af04d1b..314b037234 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/complex_mul/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/complex_mul/src/main.rs @@ -17,7 +17,7 @@ //! //! Run: cargo oxide run complex_mul -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_aggregate/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_aggregate/src/main.rs index fbbe5150b6..c16dcef2b1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_aggregate/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_aggregate/src/main.rs @@ -21,7 +21,7 @@ //! //! Run: cargo oxide run const_aggregate -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_bool_dead_branch/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_bool_dead_branch/src/main.rs index 3e46631a87..a2d4b18efc 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_bool_dead_branch/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_bool_dead_branch/src/main.rs @@ -4,7 +4,7 @@ //! Regression for generic branches controlled by an associated constant. use core::marker::PhantomData; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer, DeviceCopy}; use cuda_device::{DisjointSlice, DynamicSharedArray, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_generic/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_generic/src/main.rs index 05f649dde7..2afea93825 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_generic/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_generic/src/main.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{cuda_module, device, kernel, launch_bounds, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_table_lookup/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_table_lookup/src/main.rs index dae9eaa082..4e156013a9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_table_lookup/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_table_lookup/src/main.rs @@ -45,7 +45,7 @@ //! //! Run: `cargo oxide run const_table_lookup` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_tuple_table_lookup/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_tuple_table_lookup/src/main.rs index 464417d905..9d2f0044f0 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_tuple_table_lookup/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_tuple_table_lookup/src/main.rs @@ -37,7 +37,7 @@ //! //! Run: `cargo oxide run const_tuple_table_lookup` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_index_from_end/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_index_from_end/src/main.rs index 13699d90db..974c80fb92 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_index_from_end/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_index_from_end/src/main.rs @@ -19,7 +19,7 @@ //! Run with: //! CUDA_OXIDE_DUMP_MIR=1 cargo oxide run constant_index_from_end -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory/src/main.rs index f575c5f970..87e7b1807e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory/src/main.rs @@ -14,7 +14,7 @@ //! Build and run with: //! cargo oxide run constant_memory -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{ConstantMemory, DisjointSlice, constant, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_coeffs/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_coeffs/src/main.rs index 5cc830e458..e065d0b3fd 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_coeffs/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_coeffs/src/main.rs @@ -21,7 +21,7 @@ //! Build and run with: //! cargo oxide run constant_memory_coeffs -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{ConstantMemory, DisjointSlice, constant, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_simple/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_simple/src/main.rs index 4bb64fe912..45b8ffdb3d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_simple/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_simple/src/main.rs @@ -8,7 +8,7 @@ //! Build and run with: //! cargo oxide run constant_memory_simple -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{ConstantMemory, DisjointSlice, constant, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_table/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_table/src/main.rs index 5a8d5783fc..f61e5716b1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_table/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_table/src/main.rs @@ -42,7 +42,7 @@ //! //! Run: `cargo oxide run constant_memory_table` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/coop_groups_demo/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/coop_groups_demo/src/main.rs index 1b5cf1e66a..47ef78eb45 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/coop_groups_demo/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/coop_groups_demo/src/main.rs @@ -20,7 +20,7 @@ //! op/type matrix. The generated `coop_groups_demo.ptx` is also handy //! for inspecting how each kernel actually lowers. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer, LaunchConfig1D}; use cuda_device::cooperative_groups::{ ThreadGroup, WarpCollective, block_reduce, block_scan, coalesced_threads, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_aggregate_borrow/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_aggregate_borrow/src/main.rs index 2017fb65ed..b0d73e86bf 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_aggregate_borrow/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_aggregate_borrow/src/main.rs @@ -9,7 +9,7 @@ //! read-only methods. Runtime indexing into its small array fields must not //! force the aggregate into NVPTX local memory. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_nonoverlapping/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_nonoverlapping/src/main.rs index ac8bda154d..90efeb06ca 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_nonoverlapping/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_nonoverlapping/src/main.rs @@ -19,7 +19,7 @@ //! Usage: //! cargo oxide run copy_nonoverlapping -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, SharedArray, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/counted_barrier/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/counted_barrier/src/main.rs index 088cb66596..8ec3f3a380 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/counted_barrier/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/counted_barrier/src/main.rs @@ -26,7 +26,7 @@ //! Build and run with: //! cargo oxide run counted_barrier -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::barrier::{barrier_cta_arrive, barrier_cta_sync}; use cuda_device::{DisjointSlice, SharedArray, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_small/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_small/src/main.rs index ee84d89c02..dd4fe23738 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_small/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_small/src/main.rs @@ -14,7 +14,7 @@ //! Build and run with: //! cargo oxide run cp_async_small -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::async_copy::{ cp_async_ca_4, cp_async_ca_8, cp_async_ca_16, cp_async_ca_zfill_4, cp_async_ca_zfill_8, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_zfill/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_zfill/src/main.rs index b2a9d6b174..bb7c873a5d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_zfill/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_zfill/src/main.rs @@ -17,7 +17,7 @@ //! Build and run with: //! cargo oxide run cp_async_zfill -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::async_copy::{cp_async_ca_zfill_4, cp_async_commit_group, cp_async_wait_all}; use cuda_device::{DisjointSlice, SharedArray, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_embedded/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_embedded/src/main.rs index a1de359f63..e62b3144b5 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_embedded/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_embedded/src/main.rs @@ -22,7 +22,7 @@ //! //! Run: cargo oxide run cross_crate_embedded -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use embedded_kernel_lib::kernels; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/src/main.rs index 36388bf446..34d9b76a7f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/src/main.rs @@ -32,7 +32,7 @@ //! cargo oxide run cross_crate_kernel //! ``` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; // Import the public kernel functions and their generated `_ptx_name` diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_recursion/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_recursion/src/main.rs index b10a5fdbd0..0de522d2d5 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_recursion/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_recursion/src/main.rs @@ -10,7 +10,7 @@ //! defined LOCALLY in this crate always compiled, which isolates the axis to "cross-crate + not //! inlinable", not the function body. See PR "pass -Zalways-encode-mir ...". -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_in_lib/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_in_lib/src/main.rs index 5150484d3b..cff252211b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_in_lib/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_in_lib/src/main.rs @@ -48,7 +48,7 @@ //! cargo oxide run cuda_module_in_lib //! ``` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; use module_kernels::kernels; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_nested/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_nested/src/main.rs index ecdacf1082..d1162c1de3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_nested/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_nested/src/main.rs @@ -17,7 +17,7 @@ //! Build and run with: //! cargo oxide run cuda_module_nested -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_f16x2/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_f16x2/src/main.rs index 3152f7fae6..87a695465e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_f16x2/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_f16x2/src/main.rs @@ -15,7 +15,7 @@ //! //! Run: cargo oxide run cvt_f16x2 -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::convert::cvt_f16x2_f32; use cuda_device::{DisjointSlice, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_packed/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_packed/src/main.rs index f2fe30c8fd..b9a825ecfb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_packed/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_packed/src/main.rs @@ -31,7 +31,7 @@ //! //! Run: cargo oxide run cvt_packed -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::convert::{ cvt_f16x2_f32, cvt_f32_bf16x2_hi, cvt_f32_bf16x2_lo, cvt_f32_f16x2_hi, cvt_f32_f16x2_lo, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/debug-tests/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/debug-tests/src/main.rs index deff084d07..e1beda3ea9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/debug-tests/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/debug-tests/src/main.rs @@ -41,7 +41,7 @@ //! ./debug-tests oob_index triggers a Rust bounds-check trap //! ./debug-tests null_deref triggers a null-pointer dereference -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{ ConstantMemory, DisjointSlice, SharedArray, constant, debug, gpu_assert, gpu_printf, kernel, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/debug/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/debug/src/main.rs index deceab0b39..9d0d9053a9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/debug/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/debug/src/main.rs @@ -170,7 +170,7 @@ mod kernels { // ============================================================================= fn main() -> Result<(), Box> { - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; println!("=== GPU Debug & Utility Intrinsics Test (Unified) ===\n"); diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/debug_pointer_locals/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/debug_pointer_locals/src/main.rs index bab5eab83c..88fcdee2af 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/debug_pointer_locals/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/debug_pointer_locals/src/main.rs @@ -8,7 +8,7 @@ //! Run under cuda-gdb with: //! `CUDA_OXIDE_DEBUG=full cargo oxide debug debug_pointer_locals` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_closures/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_closures/src/main.rs index 5b4f1fe1cc..38b14208d5 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_closures/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_closures/src/main.rs @@ -322,7 +322,7 @@ mod kernels { fn main() { println!("=== Unified Compilation: Closures ===\n"); - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; let ctx = CudaContext::new(0).expect("Failed to create CUDA context"); diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs index 0deb545121..b12586cbc0 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs @@ -511,7 +511,7 @@ fn build_pipeline() -> Result { // Uses cuda-driver to load the merged cubin, launch kernels, and verify results. // ============================================================================= -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use std::sync::Arc; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_global/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_global/src/main.rs index f3703af6cb..0583a5fdc6 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_global/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_global/src/main.rs @@ -8,7 +8,7 @@ //! Build and run with: //! cargo oxide run device_global -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::kernel; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_panic_trap/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_panic_trap/src/main.rs index fa446fe363..60f511b016 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_panic_trap/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_panic_trap/src/main.rs @@ -18,7 +18,7 @@ //! driver reports as a launch failure. A launch that instead *succeeds* would //! mean the panic path fell through rather than trapping. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_from_raw_parts/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_from_raw_parts/src/main.rs index 11b1f27fee..1bb917f4c9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_from_raw_parts/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_from_raw_parts/src/main.rs @@ -24,7 +24,7 @@ //! wrote a constant, or that addressed the wrong element, fails the check //! rather than passing quietly. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, Runtime2DIndex, cuda_module, device, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_slice_len/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_slice_len/src/main.rs index 85625fe267..76b37495e9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_slice_len/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_slice_len/src/main.rs @@ -15,7 +15,7 @@ //! ``` //! //! Run: cargo oxide run disjoint_slice_len -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/dotprod/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/dotprod/src/main.rs index e502cda0a1..c48212973d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/dotprod/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/dotprod/src/main.rs @@ -25,7 +25,7 @@ //! Build and run with: //! cargo oxide run dotprod --arch sm_80 -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::dotprod::{dp2a_s32, dp2a_u32, dp4a_s32, dp4a_u32}; use cuda_device::{DisjointSlice, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/dpx_minmax_chains/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/dpx_minmax_chains/src/main.rs index d59612e2d3..974d89d346 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/dpx_minmax_chains/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/dpx_minmax_chains/src/main.rs @@ -14,7 +14,7 @@ //! //! Run: cargo oxide run dpx_minmax_chains -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/drop_glue/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/drop_glue/src/main.rs index 316fd51308..38296aab33 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/drop_glue/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/drop_glue/src/main.rs @@ -22,7 +22,7 @@ use core::mem::{ManuallyDrop, MaybeUninit}; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/dst_metadata/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/dst_metadata/src/main.rs index 0960a045ae..d752a6de32 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/dst_metadata/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/dst_metadata/src/main.rs @@ -14,7 +14,7 @@ //! Usage: //! cargo oxide run dst_metadata -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/dynamic_smem/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/dynamic_smem/src/main.rs index b0603a0ed2..3df807c48f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/dynamic_smem/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/dynamic_smem/src/main.rs @@ -26,7 +26,7 @@ //! Build and run with: //! cargo oxide run dynamic_smem -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, DynamicSharedArray, gpu_printf, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/elect_leader/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/elect_leader/src/main.rs index a1a83d0b50..0eb94779f9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/elect_leader/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/elect_leader/src/main.rs @@ -74,7 +74,7 @@ mod kernels { // ============================================================================= fn main() { - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; println!("=== elect.sync warp leader election (sm_90+) ===\n"); diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_abi_multi_payload/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_abi_multi_payload/src/main.rs index 785853b78b..4507a2610e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_abi_multi_payload/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_abi_multi_payload/src/main.rs @@ -31,7 +31,7 @@ //! Run with: //! cargo oxide run enum_abi_multi_payload -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer, DeviceCopy}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_array_match/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_array_match/src/main.rs index 7d3d200096..ef6ec3c203 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_array_match/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_array_match/src/main.rs @@ -34,7 +34,7 @@ //! Run with: //! cargo oxide run enum_array_match -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, device, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_constant_provenance/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_constant_provenance/src/main.rs index 941a825e11..40049127ef 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_constant_provenance/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_constant_provenance/src/main.rs @@ -9,7 +9,7 @@ //! static subobject, direct-tagged enums, and relocation-carrying enums nested //! inside struct, tuple, and array constants. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::kernel; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_payload_addr/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_payload_addr/src/main.rs index 348dafc884..0239ba88b5 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_payload_addr/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_payload_addr/src/main.rs @@ -34,7 +34,7 @@ //! Each kernel reads its value back after mutating, so a write that landed in //! a copy shows up as an unchanged element rather than passing quietly. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, device, kernel, thread}; use std::time::Instant; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_state_transitions/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_state_transitions/src/main.rs index 0b20e5abe7..b5339d52ae 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_state_transitions/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_state_transitions/src/main.rs @@ -13,7 +13,7 @@ //! cargo oxide run enum_state_transitions //! CUDA_OXIDE_NO_OPT=1 cargo oxide run enum_state_transitions -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_table_lookup/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_table_lookup/src/main.rs index 4e37292a26..a101ec7f1a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_table_lookup/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_table_lookup/src/main.rs @@ -41,7 +41,7 @@ //! //! Run: `cargo oxide run enum_table_lookup` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ex2_approx_f16/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/ex2_approx_f16/src/main.rs index 83538ee75e..5239cf4ee4 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ex2_approx_f16/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ex2_approx_f16/src/main.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::float::ex2_approx_f16; use cuda_device::{DisjointSlice, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/exact_div_chunks/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/exact_div_chunks/src/main.rs index 1003adab19..c8751b906f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/exact_div_chunks/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/exact_div_chunks/src/main.rs @@ -21,7 +21,7 @@ #![feature(core_intrinsics)] #![allow(internal_features)] -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/export_name_policy/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/export_name_policy/src/main.rs index 49ba3eb1ea..320096b78c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/export_name_policy/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/export_name_policy/src/main.rs @@ -16,7 +16,7 @@ //! not be classified as generic merely because the MIR `FnDef` operand carries //! substitutions or because the FQDN contains `<`, `>`, or `::`. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_crate_closure/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_crate_closure/src/main.rs index 4850e874fb..75400d096b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_crate_closure/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_crate_closure/src/main.rs @@ -20,7 +20,7 @@ //! //! Run: cargo oxide run extern_crate_closure -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_libdevice/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_libdevice/src/main.rs index 309ce7548d..0dbde672b9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_libdevice/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_libdevice/src/main.rs @@ -24,7 +24,7 @@ //! Exits 0 on SUCCESS, 1 on FAIL. use core::marker::PhantomData; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, device, kernel, thread}; use cuda_host::{cuda_module, ltoir}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/f16_stress/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/f16_stress/src/main.rs index 0d78067872..4cd16438ef 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/f16_stress/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/f16_stress/src/main.rs @@ -12,7 +12,7 @@ //! //! Run: cargo oxide run f16_stress -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/f16x2_arith/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/f16x2_arith/src/main.rs index bc20264607..b1f34607de 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/f16x2_arith/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/f16x2_arith/src/main.rs @@ -3,7 +3,7 @@ //! End-to-end example for packed f16x2 arithmetic intrinsics. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::f16x2::{ abs_f16x2, add_f16x2, fma_f16x2, fma_ftz_f16x2, fma_ftz_relu_f16x2, fma_ftz_sat_f16x2, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/f32x2_arith/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/f32x2_arith/src/main.rs index b40854d4c0..49cf92233f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/f32x2_arith/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/f32x2_arith/src/main.rs @@ -3,7 +3,7 @@ //! End-to-end example for the SM100 packed f32x2 arithmetic family. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::f32x2::{ add_f32x2, add_ftz_f32x2, fma_f32x2, fma_ftz_f32x2, mul_f32x2, mul_ftz_f32x2, pack_f32x2, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/fcmp_ne_nan_smoke/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/fcmp_ne_nan_smoke/src/main.rs index f6c46412ac..40df592afe 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/fcmp_ne_nan_smoke/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/fcmp_ne_nan_smoke/src/main.rs @@ -6,7 +6,7 @@ // Minimal repro: float `!=` should be UNORDERED (x != x is TRUE for NaN). // Rust PartialEq::ne on floats is unordered. cuda-oxide lowered it to fcmp ONE // (ordered, FALSE for NaN), so x != x folded to false -> NaN handling broken. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/field_array_assign/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/field_array_assign/src/main.rs index ddbe90fad2..531e83470d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/field_array_assign/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/field_array_assign/src/main.rs @@ -16,7 +16,7 @@ //! //! Run: cargo oxide run field_array_assign -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/float_rounding/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/float_rounding/src/main.rs index a8f31d8d14..22eee8a9d2 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/float_rounding/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/float_rounding/src/main.rs @@ -31,7 +31,7 @@ //! //! Exits 0 on PASS, 1 on FAIL. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/fmaxmin_smoke/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/fmaxmin_smoke/src/main.rs index 958a72a1a5..392fcdc79f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/fmaxmin_smoke/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/fmaxmin_smoke/src/main.rs @@ -29,7 +29,7 @@ //! Build and run with: //! cargo oxide run fmaxmin_smoke -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::{cuda_module, ltoir}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/fn_ptr_naming/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/fn_ptr_naming/src/main.rs index f726fb5862..a1e592d7a0 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/fn_ptr_naming/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/fn_ptr_naming/src/main.rs @@ -17,7 +17,7 @@ //! Run with: //! cargo oxide run fn_ptr_naming -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/fp_special_literal_smoke/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/fp_special_literal_smoke/src/main.rs index 09d2a2fb32..a8f158a27d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/fp_special_literal_smoke/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/fp_special_literal_smoke/src/main.rs @@ -31,7 +31,7 @@ //! Build and run with: //! cargo oxide run fp_special_literal_smoke -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::{cuda_module, ltoir}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/function_item_call/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/function_item_call/src/main.rs index 08e0784125..e6d7150675 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/function_item_call/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/function_item_call/src/main.rs @@ -18,7 +18,7 @@ //! `call_once_decoy` proves that an ordinary function whose name contains //! trait-like text is not mistaken for a callable-trait shim. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, device, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/future_apis/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/future_apis/src/main.rs index 55426c2f0a..19f6030320 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/future_apis/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/future_apis/src/main.rs @@ -304,7 +304,7 @@ mod kernels { // ============================================================================= fn main() -> Result<(), Box> { - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; println!("=== Future APIs Test (Unified) ===\n"); diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm/src/main.rs index 23f1f9108a..248d5fe16f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm/src/main.rs @@ -13,7 +13,7 @@ //! Build and run with: //! cargo oxide run gemm -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol/src/main.rs index f8e9a8d45b..577b817a93 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol/src/main.rs @@ -36,7 +36,7 @@ //! Build and run with: //! cargo oxide run gemm_sol -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::atomic::{AtomicOrdering, DeviceAtomicU32}; use cuda_device::barrier::{ @@ -5821,7 +5821,7 @@ fn run_benchmark_persistent( for _ in 0..WARMUP { let z = 0u32; unsafe { - cuda_core::simt::memory::memcpy_htod_async( + cuda_core::memory::memcpy_htod_async( dev_tile_counter.cu_deviceptr(), &z as *const u32, std::mem::size_of::(), @@ -5851,7 +5851,7 @@ fn run_benchmark_persistent( for _ in 0..ITERS { let z = 0u32; unsafe { - cuda_core::simt::memory::memcpy_htod_async( + cuda_core::memory::memcpy_htod_async( dev_tile_counter.cu_deviceptr(), &z as *const u32, std::mem::size_of::(), diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol_final/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol_final/src/main.rs index 0749d8c40a..8511930604 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol_final/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol_final/src/main.rs @@ -22,7 +22,7 @@ //! - B: N×K f16, row-major (transposed storage, K contiguous) //! - C: M×N bf16 output, row-major (packed as u32 pairs) -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::barrier::{ Barrier, fence_proxy_async_shared_cta, mbarrier_arrive, mbarrier_arrive_cluster, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics/src/main.rs index 8c659d964d..f772e29912 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics/src/main.rs @@ -8,7 +8,7 @@ //! The kernel calls generated coordinate, lane-mask, vote, and shuffle intrinsics //! directly. This covers the raw path instead of only `cuda-device` wrappers. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel}; use cuda_device::{ diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics_blackwell/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics_blackwell/src/main.rs index 0d29f842d9..25dd4dcc60 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics_blackwell/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics_blackwell/src/main.rs @@ -17,7 +17,7 @@ //! //! cargo oxide run generated_intrinsics_blackwell -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{ DisjointSlice, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_ldmatrix/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_ldmatrix/src/main.rs index 9bd7a2f03f..3267e9e6a1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_ldmatrix/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_ldmatrix/src/main.rs @@ -18,7 +18,7 @@ //! Build and run with: //! cargo oxide run generated_ldmatrix -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::shared::cvta_generic_to_shared_u32; use cuda_device::wmma::{ diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/generic/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/generic/src/main.rs index 013cdd4baa..543133fead 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/generic/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/generic/src/main.rs @@ -22,7 +22,7 @@ //! //! We should see a PTX entry point for `scale` (or `scale_f32` if we add type info). -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; use std::ops::{Add, Mul}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap/src/main.rs index 3a96e26c2d..cdaff2722c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap/src/main.rs @@ -27,7 +27,7 @@ use std::sync::Arc; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::atomic::{AtomicOrdering, DeviceAtomicU64}; use cuda_device::{DisjointSlice, kernel, thread}; @@ -273,7 +273,7 @@ impl GpuHashMap { let slots = DeviceBuffer::::zeroed(stream, capacity)?; unsafe { - cuda_core::simt::memory::memset_d8_async( + cuda_core::memory::memset_d8_async( slots.cu_deviceptr(), 0xFF, slots.num_bytes(), diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v2/src/bin/bench.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v2/src/bin/bench.rs index ad66f6359a..77e5fe3c2c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v2/src/bin/bench.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v2/src/bin/bench.rs @@ -36,7 +36,7 @@ use std::sync::Arc; use std::time::Instant; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use hashbrown::HashMap as HbMap; use hashmap_v2::*; @@ -96,13 +96,13 @@ unsafe fn reset_table_async( stream: &Arc, ) -> Result<(), Box> { unsafe { - cuda_core::simt::memory::memset_d8_async( + cuda_core::memory::memset_d8_async( map.ctrl.cu_deviceptr(), 0xFF, map.ctrl.num_bytes(), stream.cu_stream(), )?; - cuda_core::simt::memory::memset_d8_async( + cuda_core::memory::memset_d8_async( map.slots.cu_deviceptr(), 0xFF, map.slots.num_bytes(), diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v2/src/lib.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v2/src/lib.rs index 424e895fdc..678581d375 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v2/src/lib.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v2/src/lib.rs @@ -61,7 +61,7 @@ use std::sync::Arc; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaStream, DeviceBuffer}; use cuda_device::atomic::{AtomicOrdering, DeviceAtomicU32, DeviceAtomicU64}; use cuda_device::cooperative_groups::{ThreadGroup, WarpCollective, this_thread_block}; @@ -1142,13 +1142,13 @@ impl GpuSwissMap { let ctrl = DeviceBuffer::::zeroed(stream, capacity / GROUP)?; let slots = DeviceBuffer::::zeroed(stream, capacity)?; unsafe { - cuda_core::simt::memory::memset_d8_async( + cuda_core::memory::memset_d8_async( ctrl.cu_deviceptr(), 0xFF, ctrl.num_bytes(), stream.cu_stream(), )?; - cuda_core::simt::memory::memset_d8_async( + cuda_core::memory::memset_d8_async( slots.cu_deviceptr(), 0xFF, slots.num_bytes(), diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v3/src/bin/bench.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v3/src/bin/bench.rs index 7a8e5945de..2c65bd8b58 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v3/src/bin/bench.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v3/src/bin/bench.rs @@ -41,7 +41,7 @@ use std::sync::Arc; use std::time::Instant; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use hashbrown::HashMap as HbMap; use hashmap_v3::*; @@ -101,13 +101,13 @@ unsafe fn reset_table_async( stream: &Arc, ) -> Result<(), Box> { unsafe { - cuda_core::simt::memory::memset_d8_async( + cuda_core::memory::memset_d8_async( map.ctrl.cu_deviceptr(), 0xFF, map.ctrl.num_bytes(), stream.cu_stream(), )?; - cuda_core::simt::memory::memset_d8_async( + cuda_core::memory::memset_d8_async( map.slots.cu_deviceptr(), 0xFF, map.slots.num_bytes(), diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v3/src/lib.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v3/src/lib.rs index 22f0969e7c..6298d91977 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v3/src/lib.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v3/src/lib.rs @@ -57,7 +57,7 @@ use std::sync::Arc; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaStream, DeviceBuffer}; use cuda_device::atomic::{AtomicOrdering, DeviceAtomicU32, DeviceAtomicU64}; use cuda_device::cooperative_groups::{ThreadGroup, WarpCollective, this_thread_block}; @@ -1069,13 +1069,13 @@ impl GpuSwissMap { let ctrl = DeviceBuffer::::zeroed(stream, capacity / GROUP)?; let slots = DeviceBuffer::::zeroed(stream, capacity)?; unsafe { - cuda_core::simt::memory::memset_d8_async( + cuda_core::memory::memset_d8_async( ctrl.cu_deviceptr(), 0xFF, ctrl.num_bytes(), stream.cu_stream(), )?; - cuda_core::simt::memory::memset_d8_async( + cuda_core::memory::memset_d8_async( slots.cu_deviceptr(), 0xFF, slots.num_bytes(), @@ -1379,13 +1379,13 @@ impl GpuSwissMap { let new_ctrl = DeviceBuffer::::zeroed(stream, new_capacity / GROUP)?; let new_slots = DeviceBuffer::::zeroed(stream, new_capacity)?; unsafe { - cuda_core::simt::memory::memset_d8_async( + cuda_core::memory::memset_d8_async( new_ctrl.cu_deviceptr(), 0xFF, new_ctrl.num_bytes(), stream.cu_stream(), )?; - cuda_core::simt::memory::memset_d8_async( + cuda_core::memory::memset_d8_async( new_slots.cu_deviceptr(), 0xFF, new_slots.num_bytes(), diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/hello_constant/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/hello_constant/src/main.rs index 4fbb97cfb8..fcb76b45c5 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/hello_constant/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/hello_constant/src/main.rs @@ -11,7 +11,7 @@ //! Build and run with: //! cargo oxide run hello_constant -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::kernel; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/helper_fn/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/helper_fn/src/main.rs index 8dafaf4c98..75b82d0ad2 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/helper_fn/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/helper_fn/src/main.rs @@ -12,7 +12,7 @@ //! Build and run with: //! cargo oxide run helper_fn -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, device, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure/src/main.rs index 1e267a6268..3c5946cb7f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure/src/main.rs @@ -31,7 +31,7 @@ //! (`#[repr(C)]`) struct captures. use cuda_async::simt::device_operation::DeviceOperation; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::{cuda_launch, cuda_launch_async, cuda_module, load_kernel_module}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure_target_feature/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure_target_feature/src/main.rs index 5d5989b2f4..8c1c5576a3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure_target_feature/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure_target_feature/src/main.rs @@ -23,7 +23,7 @@ use std::sync::Arc; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::{cuda_module, load_kernel_module}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/iket_trace/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/iket_trace/src/main.rs index 3106b31652..e5d51c2ff9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/iket_trace/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/iket_trace/src/main.rs @@ -11,7 +11,7 @@ //! cargo oxide build iket_trace --arch sm_120 //! ``` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, iket, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/index2d_const/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/index2d_const/src/main.rs index 5888f2b09b..aed40c7828 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/index2d_const/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/index2d_const/src/main.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/index_field_assign/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/index_field_assign/src/main.rs index 1557e57f06..38c794f608 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/index_field_assign/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/index_field_assign/src/main.rs @@ -31,7 +31,7 @@ //! Usage: //! cargo oxide run index_field_assign -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_intent/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_intent/src/main.rs index 037b086542..c9b22840d9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_intent/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_intent/src/main.rs @@ -24,7 +24,7 @@ //! reaches LLVM as its own definition. Otherwise rustc folds the small ones //! into the kernel before any attribute could be observed. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, device, kernel}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_ptx/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_ptx/src/main.rs index 18238fd1f7..48795d93f7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_ptx/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_ptx/src/main.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, ptx_asm}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/integer_minmax/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/integer_minmax/src/main.rs index c8e331e578..abc9883fb9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/integer_minmax/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/integer_minmax/src/main.rs @@ -5,7 +5,7 @@ //! End-to-end example for scalar and packed integer min/max intrinsics (sm_90+). -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::i16x2::{ max_relu_s16x2, max_s16x2, max_u16x2, min_relu_s16x2, min_s16x2, min_u16x2, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/interior_mutability/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/interior_mutability/src/main.rs index adf0c23f65..140955dcd3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/interior_mutability/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/interior_mutability/src/main.rs @@ -13,7 +13,7 @@ //! cargo oxide run interior_mutability //! CUDA_OXIDE_NO_OPT=1 cargo oxide run interior_mutability -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/lanemask_scan/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/lanemask_scan/src/main.rs index 3c0fd45658..d77b163752 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/lanemask_scan/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/lanemask_scan/src/main.rs @@ -114,7 +114,7 @@ mod kernels { // ============================================================================= fn main() { - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; println!("=== Lane-Position Masks & Warp Prefix Sum ===\n"); diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_fadd/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_fadd/src/main.rs index b9fc05e8cf..e65167feb8 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_fadd/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_fadd/src/main.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{ DisjointSlice, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_rmw_cas/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_rmw_cas/src/main.rs index be0858fa1a..74eb386556 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_rmw_cas/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_rmw_cas/src/main.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{ DisjointSlice, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_nvvm_pointer_shapes/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_nvvm_pointer_shapes/src/main.rs index c2ab6b9545..f79fa3613a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_nvvm_pointer_shapes/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_nvvm_pointer_shapes/src/main.rs @@ -10,7 +10,7 @@ //! and `u32`. Together, these cases check that legacy typed pointers work //! throughout a function, not only at loads and stores. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/libdevice_math/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/libdevice_math/src/main.rs index 1c97759edd..8cc396fa5b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/libdevice_math/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/libdevice_math/src/main.rs @@ -9,7 +9,7 @@ //! example checks SwiGLU plus `sin`, `exp`, `sqrt`, `atan`, `atan2`, `acos`, //! and `tan` on both legacy and modern NVVM targets. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/libm_math/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/libm_math/src/main.rs index aeebfe45ac..8b4bd44e53 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/libm_math/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/libm_math/src/main.rs @@ -23,7 +23,7 @@ //! Run with: //! cargo oxide run libm_math -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_generic/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_generic/src/main.rs index 12222c65d3..a17ad8431c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_generic/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_generic/src/main.rs @@ -9,7 +9,7 @@ //! instead of `#[cuda_module]`. It keeps the explicit API covered while the //! typed embedded-module API is the preferred ergonomic path. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::{cuda_launch, load_kernel_module}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_libdevice/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_libdevice/src/main.rs index 43856a4012..9b249fb53d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_libdevice/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_libdevice/src/main.rs @@ -5,7 +5,7 @@ //! Manual low-level launch API regression test for NVVM IR/libdevice artifacts. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::{cuda_launch, load_kernel_module}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/map_sum/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/map_sum/src/main.rs index 29d7d1ed7a..76e1ff293b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/map_sum/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/map_sum/src/main.rs @@ -32,7 +32,7 @@ //! Usage: //! cargo oxide run map_sum -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_atan/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_atan/src/main.rs index 5c6f7e97df..27e4f4185f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_atan/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_atan/src/main.rs @@ -15,7 +15,7 @@ //! //! Exits 0 on PASS, 1 on FAIL. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::{cuda_module, ltoir}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_hyperbolic/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_hyperbolic/src/main.rs index 85131b60aa..8714b5d7a1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_hyperbolic/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_hyperbolic/src/main.rs @@ -32,7 +32,7 @@ // style lints that do not improve a numeric smoke test. #![allow(clippy::too_many_arguments, clippy::type_complexity)] -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::{cuda_module, ltoir}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_no_mir_inline/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_no_mir_inline/src/main.rs index 78faf4af2e..5893ec0b3a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_no_mir_inline/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_no_mir_inline/src/main.rs @@ -35,7 +35,7 @@ //! //! Exits 0 on SUCCESS, 1 on FAIL. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::{cuda_module, ltoir}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_tan/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_tan/src/main.rs index 89544b519a..51e7953ee5 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_tan/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_tan/src/main.rs @@ -23,7 +23,7 @@ //! //! Exits 0 on PASS, 1 on FAIL. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::{cuda_module, ltoir}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/mathdx_ffi_test/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/mathdx_ffi_test/src/main.rs index 9848dabb43..335448298d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/mathdx_ffi_test/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/mathdx_ffi_test/src/main.rs @@ -873,7 +873,7 @@ fn build_pipeline(mathdx_root: &Path) -> Result { // Host-side test infrastructure // ============================================================================= -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use std::sync::Arc; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/mcast_barrier_test/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/mcast_barrier_test/src/main.rs index 90cb8dee3d..3c83ce4c84 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/mcast_barrier_test/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/mcast_barrier_test/src/main.rs @@ -13,7 +13,7 @@ //! Build and run: //! cargo oxide run mcast_barrier_test -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::barrier::{ Barrier, fence_proxy_async_shared_cta, mbarrier_arrive_cluster, mbarrier_init, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/mem_swap/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/mem_swap/src/main.rs index d5effc1497..ab6bb8bf59 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/mem_swap/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/mem_swap/src/main.rs @@ -20,7 +20,7 @@ //! cargo oxide run mem_swap //! CUDA_OXIDE_NO_OPT=1 cargo oxide run mem_swap -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/mir_projected_call_destination/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/mir_projected_call_destination/src/main.rs index 4a1ccfe81a..d927a1f103 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/mir_projected_call_destination/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/mir_projected_call_destination/src/main.rs @@ -45,7 +45,7 @@ //! cargo oxide run mir_projected_call_destination use core::intrinsics::mir::*; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/mma_mxf8f6f4/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/mma_mxf8f6f4/src/main.rs index 9c52f8c0fd..852eff1a8b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/mma_mxf8f6f4/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/mma_mxf8f6f4/src/main.rs @@ -26,7 +26,7 @@ //! Build and run with: //! cargo oxide run mma_mxf8f6f4 --arch sm_120a -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread, wmma}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ne_bytes_transmute/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/ne_bytes_transmute/src/main.rs index ffb8bad124..d007f4f61d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ne_bytes_transmute/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ne_bytes_transmute/src/main.rs @@ -13,7 +13,7 @@ // Kernels lock in both directions for u32 and f32, plus `f32::to_bits`, // a same-width scalar transmute that must stay on the plain bitcast path. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/nested_index_assignment/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/nested_index_assignment/src/main.rs index 5472665507..223fbf5123 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/nested_index_assignment/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/nested_index_assignment/src/main.rs @@ -12,7 +12,7 @@ //! through it instead of rejecting the assignment before the generic //! projection walker can handle it. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/numeric_stress/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/numeric_stress/src/main.rs index 3eb12d458d..06901ef3e9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/numeric_stress/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/numeric_stress/src/main.rs @@ -14,7 +14,7 @@ //! //! Run: cargo oxide run numeric_stress -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/nvls_all_reduce/README.md b/cuda-oxide/crates/rustc-codegen-cuda/examples/nvls_all_reduce/README.md index 2a11e96506..6a544416aa 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/nvls_all_reduce/README.md +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/nvls_all_reduce/README.md @@ -1,7 +1,7 @@ # nvls_all_reduce Single-process NVLS (NVLink SHARP) all-reduce across every multicast-capable -GPU in the machine, built on the `cuda_core::simt::vmm` multicast wrappers +GPU in the machine, built on the `cuda_core::vmm` multicast wrappers (`MulticastObject`, `Mapping::new_multicast`). ## What it demonstrates @@ -54,7 +54,7 @@ integers, so the sums are exact and order-independent). ## See also -- `cuda-core/tests/simt_vmm_multicast.rs` in cutile-rs: host-side team plumbing test +- `cuda-core/tests/vmm_multicast.rs` in cutile-rs: host-side team plumbing test (bind/map/unbind lifecycle) that runs without issuing `multimem`. - PTX ISA section 9.7.13.4 (`multimem.ld_reduce`, `multimem.st`, `multimem.red`). diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/nvls_all_reduce/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/nvls_all_reduce/src/main.rs index a38077d7c3..7c8771b558 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/nvls_all_reduce/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/nvls_all_reduce/src/main.rs @@ -26,8 +26,8 @@ use cuda_core::CudaContext; use cuda_core::IntoResult; -use cuda_core::simt::LaunchConfig; -use cuda_core::simt::vmm; +use cuda_core::LaunchConfig; +use cuda_core::vmm; use cuda_device::{cuda_module, kernel, ptx_asm}; use std::mem::MaybeUninit; use std::sync::Arc; @@ -149,7 +149,7 @@ fn main() { let input: Vec = (0..ELEMS).map(|i| input_value(g, i)).collect(); let stream = ctx.default_stream(); unsafe { - cuda_core::simt::memory::memcpy_htod_async( + cuda_core::memory::memcpy_htod_async( views[g].0.base(), input.as_ptr(), bytes, @@ -197,7 +197,7 @@ fn main() { let mut result = vec![0f32; ELEMS]; let stream = ctx.default_stream(); unsafe { - cuda_core::simt::memory::memcpy_dtoh_async( + cuda_core::memory::memcpy_dtoh_async( result.as_mut_ptr(), views[g].0.base(), bytes, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_transmute/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_transmute/src/main.rs index 4267dcb23c..57c45246b7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_transmute/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_transmute/src/main.rs @@ -25,7 +25,7 @@ use core::mem; use core::ptr; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_unwrap_or/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_unwrap_or/src/main.rs index 155eee9460..f2283c1ea9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_unwrap_or/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_unwrap_or/src/main.rs @@ -30,7 +30,7 @@ //! Run with: //! cargo oxide run option_ref_unwrap_or -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/option_unwrap_trap/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/option_unwrap_trap/src/main.rs index 658c5fa9d5..dae19c9b04 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/option_unwrap_trap/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/option_unwrap_trap/src/main.rs @@ -9,7 +9,7 @@ //! which on the GPU means the kernel must trap and the launch must fail. //! If the launch succeeds instead, the panic path was compiled away. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ord_cmp/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/ord_cmp/src/main.rs index c8019f6b74..ec4d88d8c3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ord_cmp/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ord_cmp/src/main.rs @@ -21,7 +21,7 @@ //! //! Run: cargo oxide run ord_cmp -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_aggregate_abi/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_aggregate_abi/src/main.rs index 0b857b8fbb..ef18af3bd7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_aggregate_abi/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_aggregate_abi/src/main.rs @@ -28,7 +28,7 @@ //! `b` at byte offset 2. The padding byte itself is intentionally not checked: //! Rust does not guarantee a stable value for padding bytes. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{SharedArray, cuda_module, device, kernel}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_atomic_add/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_atomic_add/src/main.rs index 17eae1e0a8..520d69662c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_atomic_add/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_atomic_add/src/main.rs @@ -10,7 +10,7 @@ //! old values for each 16-bit half independently because PTX does not promise //! that the two returned halves form one coherent 32-bit snapshot. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::atomic::{atom_add_bf16x2, atom_add_f16x2}; use cuda_device::{kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/padded_constants/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/padded_constants/src/main.rs index ff825f45fc..229cc59b56 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/padded_constants/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/padded_constants/src/main.rs @@ -27,7 +27,7 @@ //! Run with: //! cargo oxide run padded_constants -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/pinned_overlap/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/pinned_overlap/src/main.rs index efccbe1b28..34195693cc 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/pinned_overlap/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/pinned_overlap/src/main.rs @@ -12,7 +12,7 @@ use std::error::Error; use std::sync::Arc; use std::time::{Duration, Instant}; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaEvent, CudaStream, DeviceBuffer, PinnedHostBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/place_read_fallbacks/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/place_read_fallbacks/src/main.rs index 6bc95a1793..500ac71691 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/place_read_fallbacks/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/place_read_fallbacks/src/main.rs @@ -18,7 +18,7 @@ //! //! Run: cargo oxide run place_read_fallbacks -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, device, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/primitive_stress/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/primitive_stress/src/main.rs index 0d5d196cbc..bc2521704b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/primitive_stress/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/primitive_stress/src/main.rs @@ -25,7 +25,7 @@ //! //! Run: `cargo oxide run primitive_stress` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/printf/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/printf/src/main.rs index 5900191f64..7891e0621a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/printf/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/printf/src/main.rs @@ -204,7 +204,7 @@ mod kernels { // ============================================================================= fn main() -> Result<(), Box> { - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; println!("=== GPU Printf Test (Unified) ===\n"); diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/projected_place_read/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/projected_place_read/src/main.rs index 1f0cc06a07..731f1b01a8 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/projected_place_read/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/projected_place_read/src/main.rs @@ -14,7 +14,7 @@ //! //! Run: cargo oxide run projected_place_read -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, device, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_copy/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_copy/src/main.rs index f80c309259..75e4cb7a90 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_copy/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_copy/src/main.rs @@ -18,7 +18,7 @@ //! cargo oxide run ptr_copy //! CUDA_OXIDE_NO_OPT=1 cargo oxide run ptr_copy -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_offset_from/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_offset_from/src/main.rs index 5da1ff8bb5..62ce6bacc6 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_offset_from/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_offset_from/src/main.rs @@ -13,7 +13,7 @@ //! offsets by the pointee type, retain pointer mutability through libcore's cast, //! and leave zero-sized pointees stationary. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_without_provenance/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_without_provenance/src/main.rs index e3fba3047f..88d591bcaf 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_without_provenance/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_without_provenance/src/main.rs @@ -8,7 +8,7 @@ //! raw address exactly; an old Debug-string byte parser mis-decoded these //! partially-initialized constant bytes. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/reborrow/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/reborrow/src/main.rs index eb1675311b..81e50df688 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/reborrow/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/reborrow/src/main.rs @@ -35,7 +35,7 @@ #![feature(reborrow)] -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_f32/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_f32/src/main.rs index 420de7f079..1cefb4e5fa 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_f32/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_f32/src/main.rs @@ -77,7 +77,7 @@ mod kernels { // ============================================================================= fn main() { - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; println!("=== redux.sync f32 family (Blackwell family 10x) ===\n"); diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_minmax/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_minmax/src/main.rs index 6bdc88c801..b7ec14f342 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_minmax/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_minmax/src/main.rs @@ -126,7 +126,7 @@ mod kernels { // ============================================================================= fn main() { - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; println!("=== redux.sync integer family (sm_80+) ===\n"); diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_sum/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_sum/src/main.rs index fdce5eeddd..4604acb2ad 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_sum/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_sum/src/main.rs @@ -74,7 +74,7 @@ mod kernels { // ============================================================================= fn main() { - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; println!("=== redux.sync.add Warp Reduction (sm_80+) ===\n"); diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_index_projections/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_index_projections/src/main.rs index aefcae951f..cdabb3d269 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_index_projections/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_index_projections/src/main.rs @@ -32,7 +32,7 @@ * projections, returning the field's base address). */ -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread, warp}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_operand_mul/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_operand_mul/src/main.rs index 5bc36bed83..bc50a9120d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_operand_mul/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_operand_mul/src/main.rs @@ -22,7 +22,7 @@ //! //! Run: cargo oxide run ref_operand_mul -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_host::cuda_module; use std::sync::Arc; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_align_block_args/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_align_block_args/src/main.rs index 36f3678093..2568dfe2a9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_align_block_args/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_align_block_args/src/main.rs @@ -18,7 +18,7 @@ //! //! Run: `cargo oxide run repr_align_block_args` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_transparent_abi/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_transparent_abi/src/main.rs index 7ff826c025..0ef5c34b07 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_transparent_abi/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_transparent_abi/src/main.rs @@ -13,7 +13,7 @@ use core::marker::PhantomData; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{Uniform, cuda_module, device, kernel}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/src/main.rs index d71f86e0c5..a268c968ec 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/src/main.rs @@ -52,7 +52,7 @@ * cargo oxide run repr_u32_enum_stride */ -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer, DeviceCopy}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/row_width_slice/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/row_width_slice/src/main.rs index ff14f911b2..02449c809e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/row_width_slice/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/row_width_slice/src/main.rs @@ -31,7 +31,7 @@ //! //! Run: cargo oxide run row_width_slice -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/rustlantis-smoke/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/rustlantis-smoke/src/main.rs index bb566b6bc5..46a5a3fc25 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/rustlantis-smoke/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/rustlantis-smoke/src/main.rs @@ -34,7 +34,7 @@ //! cargo oxide run rustlantis-smoke use core::intrinsics::mir::*; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/scoped_atomic_load_store/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/scoped_atomic_load_store/src/main.rs index 8674c63f1c..c3c937cbc2 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/scoped_atomic_load_store/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/scoped_atomic_load_store/src/main.rs @@ -34,7 +34,7 @@ //! Build and run with: //! cargo oxide run scoped_atomic_load_store --materialize-cubin -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/select_unpredictable/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/select_unpredictable/src/main.rs index b62ec4c4d1..f614ec9cc3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/select_unpredictable/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/select_unpredictable/src/main.rs @@ -24,7 +24,7 @@ //! Usage: //! cargo oxide run select_unpredictable -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant/src/main.rs index ed742c6bf6..81744c4132 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant/src/main.rs @@ -17,7 +17,7 @@ #![allow(internal_features)] use core::intrinsics::mir::*; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant_niche/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant_niche/src/main.rs index aab836ad43..b003b6946f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant_niche/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant_niche/src/main.rs @@ -16,7 +16,7 @@ use core::intrinsics::mir::*; use core::num::NonZeroU32; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer, DeviceCopy}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_debug/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_debug/src/main.rs index a5ae803dc3..002d1f3b1a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_debug/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_debug/src/main.rs @@ -7,7 +7,7 @@ //! Persistent AS3 debug-info fixture for cuda-gdb/T27. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::barrier::Barrier; use cuda_device::{DisjointSlice, SharedArray, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_memory_helper/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_memory_helper/src/main.rs index 6b25119cb0..0dd24011ab 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_memory_helper/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_memory_helper/src/main.rs @@ -7,7 +7,7 @@ //! `inline(never)` requests a MIR call boundary; inspect generated code to //! determine whether the backend also keeps the calls. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DynamicSharedArray, SharedPtr, barrier::Barrier}; use cuda_device::{cuda_module, kernel, launch_bounds, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_api/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_api/src/main.rs index 12ba608b2b..8378f0c4f2 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_api/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_api/src/main.rs @@ -30,7 +30,7 @@ //! Usage: //! cargo oxide run shared_ptr_api -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::shared::cvta_generic_to_shared_u32; use cuda_device::{DynamicSharedArray, SharedArray, SharedPtr, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_transmute/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_transmute/src/main.rs index bc9bbac296..e0d8362744 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_transmute/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_transmute/src/main.rs @@ -45,7 +45,7 @@ #![allow(static_mut_refs)] -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, SharedArray, cuda_module, device, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_slice_unsize/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_slice_unsize/src/main.rs index d70b97e58d..c402529f99 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_slice_unsize/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_slice_unsize/src/main.rs @@ -22,7 +22,7 @@ #![allow(static_mut_refs)] -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, SharedArray, device, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_stage_select/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_stage_select/src/main.rs index edadf18425..1941d2d3a1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_stage_select/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_stage_select/src/main.rs @@ -24,7 +24,7 @@ //! //! Run: `cargo oxide run shared_stage_select` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, SharedArray, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/sharedmem/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/sharedmem/src/main.rs index 7852b13c0f..1f82d5374d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/sharedmem/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/sharedmem/src/main.rs @@ -24,7 +24,7 @@ //! Build and run with: //! cargo oxide run sharedmem -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, SharedArray, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shuffle_64/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/shuffle_64/src/main.rs index c25cc2b641..2f3e326536 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shuffle_64/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shuffle_64/src/main.rs @@ -126,7 +126,7 @@ mod kernels { const WARP: usize = 32; fn main() { - use cuda_core::simt::LaunchConfig; + use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; println!("=== 64-bit warp shuffle (u64 / f64) ===\n"); diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_from_raw_parts_cast/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_from_raw_parts_cast/src/main.rs index 99fa6d3b48..d912a46282 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_from_raw_parts_cast/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_from_raw_parts_cast/src/main.rs @@ -20,7 +20,7 @@ //! Usage: //! cargo oxide run slice_from_raw_parts_cast -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, SharedArray, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_get_mut/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_get_mut/src/main.rs index 1f8b0a9b52..66c6f7ccda 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_get_mut/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_get_mut/src/main.rs @@ -42,7 +42,7 @@ * address through fat-pointer deref"). */ -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_index_bounds_check/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_index_bounds_check/src/main.rs index a7d7ca1c3b..fff2c5c707 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_index_bounds_check/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_index_bounds_check/src/main.rs @@ -14,7 +14,7 @@ //! or the launch must fail (thread trapped). //! If the sentinel is overwritten, the kernel performed an out-of-bounds global read. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_iterators/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_iterators/src/main.rs index 5256f4f31f..4fab01a5c9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_iterators/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_iterators/src/main.rs @@ -20,7 +20,7 @@ //! cargo oxide run slice_iterators //! CUDA_OXIDE_NO_OPT=1 cargo oxide run slice_iterators -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_reslice/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_reslice/src/main.rs index 9dd1471cd8..c2ffe061cb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_reslice/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_reslice/src/main.rs @@ -12,7 +12,7 @@ // pointer with `Rvalue::Aggregate(AggregateKind::RawPtr(..), [data_ptr, len])`. // The importer had no arm for that aggregate kind, so any kernel that passed // a sub-slice to a helper function failed to compile. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/small_type_ffi_test/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/small_type_ffi_test/src/main.rs index 02f3142588..d48de0e790 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/small_type_ffi_test/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/small_type_ffi_test/src/main.rs @@ -336,7 +336,7 @@ fn build_pipeline() -> Result { // the results. // ============================================================================= -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use std::sync::Arc; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_fp8/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_fp8/src/main.rs index 45c9b4ba13..4216c4f43f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_fp8/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_fp8/src/main.rs @@ -8,7 +8,7 @@ //! See the PTX ISA's "Matrix Fragments for sparse mma.m16n8k64" for the layout. //! `--probe` additionally compares candidate layouts on the current GPU. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread, wmma}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_ordered_float/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_ordered_float/src/main.rs index 7b02c14e88..3e939edca8 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_ordered_float/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_ordered_float/src/main.rs @@ -21,7 +21,7 @@ //! lane's metadata register also changes the result; //! - values stay small (A<=7, B<=8, C<=31, sums < 1024) so f16, bf16, and //! tf32 all accumulate exactly and the host reference is integer math. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread, wmma}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/split_at_mut/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/split_at_mut/src/main.rs index 4cdde3d73a..58e7a5b3de 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/split_at_mut/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/split_at_mut/src/main.rs @@ -21,7 +21,7 @@ //! Usage: //! cargo oxide run split_at_mut -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_addend/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_addend/src/main.rs index ea50b15c3f..c1b2c09974 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_addend/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_addend/src/main.rs @@ -11,7 +11,7 @@ //! //! Run: `cargo oxide run static_slice_addend` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::kernel; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_unsize/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_unsize/src/main.rs index 0443d1661b..158eada8e4 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_unsize/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_unsize/src/main.rs @@ -10,7 +10,7 @@ //! //! Run: `cargo oxide run static_slice_unsize` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::kernel; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/step_by/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/step_by/src/main.rs index 544c5e9714..e189c7782b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/step_by/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/step_by/src/main.rs @@ -23,7 +23,7 @@ //! //! Run: cargo oxide run step_by -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_constant_provenance/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_constant_provenance/src/main.rs index 6a3bdca1b0..9c0acc0e8b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_constant_provenance/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_constant_provenance/src/main.rs @@ -9,7 +9,7 @@ //! metadata word. The importer must preserve both pieces when the constant is //! materialized for GPU code. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_field_layout/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_field_layout/src/main.rs index c94cc5edf4..d317d8dc13 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_field_layout/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_field_layout/src/main.rs @@ -25,7 +25,7 @@ //! //! Run: `cargo oxide run struct_field_layout` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/subslice_projection/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/subslice_projection/src/main.rs index f05dd15c0c..6c87da4f20 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/subslice_projection/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/subslice_projection/src/main.rs @@ -22,7 +22,7 @@ * cargo oxide pipeline subslice_projection */ -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/switchint_128bit/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/switchint_128bit/src/main.rs index 8fb1f1e5d4..f8cc1a2e68 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/switchint_128bit/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/switchint_128bit/src/main.rs @@ -25,7 +25,7 @@ //! Usage: //! cargo oxide run switchint_128bit -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/swizzle_smem/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/swizzle_smem/src/main.rs index 93b01ee4b4..bf9dc33c5f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/swizzle_smem/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/swizzle_smem/src/main.rs @@ -12,7 +12,7 @@ //! Build and run with: //! cargo oxide run swizzle_smem -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, SharedArray, kernel, swizzle::Swizzle, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05/src/main.rs index acb06ddc36..c0a8b831c0 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05/src/main.rs @@ -32,7 +32,7 @@ //! Build and run with: //! cargo oxide run tcgen05 -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer, sys}; use cuda_device::barrier::Barrier; use cuda_device::shared::{SharedArray, cvta_generic_to_shared_offset}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05_matmul/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05_matmul/src/main.rs index 88fd1e2778..b1a8b21e82 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05_matmul/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05_matmul/src/main.rs @@ -10,7 +10,7 @@ //! Build and run with: //! cargo oxide run tcgen05_matmul -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{ CudaContext, CudaStream, DeviceBuffer, sys::{ diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tiled_gemm/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/tiled_gemm/src/main.rs index 87fa66fa63..d0a37f3383 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tiled_gemm/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tiled_gemm/src/main.rs @@ -15,7 +15,7 @@ //! Build and run with: //! cargo oxide run tiled_gemm -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, SharedArray, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_copy/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_copy/src/main.rs index aa21c06446..e0028246fa 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_copy/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_copy/src/main.rs @@ -22,7 +22,7 @@ //! Build and run with: //! cargo oxide run tma_copy -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{ CudaContext, CudaStream, DeviceBuffer, sys::{ diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_multicast/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_multicast/src/main.rs index 5a1e325a5d..58805648dc 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_multicast/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_multicast/src/main.rs @@ -28,7 +28,7 @@ //! Build and run with: //! cargo oxide run tma_multicast -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{ CudaContext, CudaStream, DeviceBuffer, sys::{ diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/transmute_scalar_struct/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/transmute_scalar_struct/src/main.rs index 2c3dbba3c6..fde67b4e40 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/transmute_scalar_struct/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/transmute_scalar_struct/src/main.rs @@ -18,7 +18,7 @@ //! //! Run: cargo oxide run transmute_scalar_struct -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_array_provenance/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_array_provenance/src/main.rs index e95958c280..7cced6c01b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_array_provenance/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_array_provenance/src/main.rs @@ -10,7 +10,7 @@ //! //! Run: `cargo oxide run tuple_array_provenance` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_constant_provenance/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_constant_provenance/src/main.rs index e167273513..49f046b9c1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_constant_provenance/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_constant_provenance/src/main.rs @@ -3,7 +3,7 @@ //! Regression for pointer provenance in a direct tuple constant. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_field_lookup/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_field_lookup/src/main.rs index 4fd3d31fab..6148e7b040 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_field_lookup/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_field_lookup/src/main.rs @@ -28,7 +28,7 @@ //! //! Run: `cargo oxide run tuple_field_lookup` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_return/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_return/src/main.rs index 733e2f727d..e9a75c72e1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_return/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_return/src/main.rs @@ -20,7 +20,7 @@ //! Build and run with: //! cargo oxide run tuple_return -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/unaligned_memory/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/unaligned_memory/src/main.rs index 50c2279e20..cdc68b231b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/unaligned_memory/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/unaligned_memory/src/main.rs @@ -12,7 +12,7 @@ //! cargo oxide run unaligned_memory //! CUDA_OXIDE_NO_OPT=1 cargo oxide run unaligned_memory -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/unchecked_indexing/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/unchecked_indexing/src/main.rs index 89e954f0f1..760f1a27ea 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/unchecked_indexing/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/unchecked_indexing/src/main.rs @@ -39,7 +39,7 @@ //! Build and run with: //! cargo oxide run unchecked_indexing -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/uninit_const/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/uninit_const/src/main.rs index 6e68befde0..8e8514c3de 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/uninit_const/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/uninit_const/src/main.rs @@ -14,7 +14,7 @@ //! Run: cargo oxide run uninit_const use core::mem::MaybeUninit; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/union_aggregate/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/union_aggregate/src/main.rs index b3d340c3a1..d563abae45 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/union_aggregate/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/union_aggregate/src/main.rs @@ -11,7 +11,7 @@ //! passing a union through an ordinary device function. It also checks that //! an over-aligned union keeps its size and array stride. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_bounds_check/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_bounds_check/src/main.rs index 37ecf93b63..2277bcdc2a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_bounds_check/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_bounds_check/src/main.rs @@ -6,7 +6,7 @@ //! `#[unroll]` must preserve checked indexing and division-by-zero checks. //! Run: cargo oxide run unroll_bounds_check -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_smoke/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_smoke/src/main.rs index 2469ed630a..9265bf3acf 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_smoke/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_smoke/src/main.rs @@ -27,7 +27,7 @@ //! //! Run: cargo oxide run unroll_smoke -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/vecadd/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/vecadd/src/main.rs index 01fb68414b..75e740b7cc 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/vecadd/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/vecadd/src/main.rs @@ -20,7 +20,7 @@ // No #![cfg_attr(cuda_device, no_std)] - this compiles as ONE unit! -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/vectorization/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/vectorization/src/main.rs index 255b16ace7..9861b39880 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/vectorization/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/vectorization/src/main.rs @@ -14,7 +14,7 @@ //! //! Run: `cargo oxide run vectorization` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{access, cuda_module}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/volatile_memory/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/volatile_memory/src/main.rs index 1e70397774..1e80bd6c30 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/volatile_memory/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/volatile_memory/src/main.rs @@ -5,7 +5,7 @@ //! //! Run: `cargo oxide run volatile_memory` -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/warp_reduce/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/warp_reduce/src/main.rs index 3cf6f60042..034265603c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/warp_reduce/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/warp_reduce/src/main.rs @@ -14,7 +14,7 @@ //! Build and run with: //! cargo oxide run warp_reduce -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::cooperative_groups::{ ThreadGroup, block_reduce, block_scan, ops::Sum, this_thread_block, diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/wgmma/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/wgmma/src/main.rs index faa4e622a9..090ab40ee5 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/wgmma/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/wgmma/src/main.rs @@ -16,7 +16,7 @@ //! Build and run with: //! cargo oxide run wgmma -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cuda_device::shared::SharedArray; use cuda_device::wgmma::{make_smem_desc, wgmma_commit_group, wgmma_fence, wgmma_wait_group}; diff --git a/cuda-oxide/cuda-oxide-book/appendix/api-quick-reference.md b/cuda-oxide/cuda-oxide-book/appendix/api-quick-reference.md index cc90fe89a5..a502b77123 100644 --- a/cuda-oxide/cuda-oxide-book/appendix/api-quick-reference.md +++ b/cuda-oxide/cuda-oxide-book/appendix/api-quick-reference.md @@ -524,7 +524,7 @@ N_COLS must be a power of 2 in the range [32, 512]. ### Typed Synchronous ```rust -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; let ctx = CudaContext::new(0).unwrap(); diff --git a/cuda-oxide/cuda-oxide-book/async-programming/the-device-operation-model.md b/cuda-oxide/cuda-oxide-book/async-programming/the-device-operation-model.md index 1a6d6e14db..31e9492de7 100644 --- a/cuda-oxide/cuda-oxide-book/async-programming/the-device-operation-model.md +++ b/cuda-oxide/cuda-oxide-book/async-programming/the-device-operation-model.md @@ -75,7 +75,7 @@ returns a recipe instead of cooking immediately: ```rust use cuda_async::simt::device_context::init_device_contexts; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; // One-time setup: create a stream pool for scheduling init_device_contexts(0, 1)?; @@ -298,7 +298,7 @@ until the `ExecutionContext` is available: ```rust use cuda_async::simt::device_operation::{value, with_context}; -use cuda_core::simt::memory::{malloc_async, memcpy_htod_async}; +use cuda_core::memory::{malloc_async, memcpy_htod_async}; fn h2d(host_data: Vec) -> impl DeviceOperation> { with_context(move |ctx| { diff --git a/cuda-oxide/cuda-oxide-book/compiler/architecture-overview.md b/cuda-oxide/cuda-oxide-book/compiler/architecture-overview.md index 64dc2797df..35ad4c2e3d 100644 --- a/cuda-oxide/cuda-oxide-book/compiler/architecture-overview.md +++ b/cuda-oxide/cuda-oxide-book/compiler/architecture-overview.md @@ -143,7 +143,7 @@ cuda-oxide is split into focused crates. Here is every one and its role: | `cute-rs` | Typed CuTe tensors, tiles, copies, pipelines, and MMA operations | | `cuda-macros` | Proc macros: `#[kernel]`, `#[device]` | | `cuda-host` | Host-side typed module loading and launch helpers | -| `cuda-core` | Shared with cutile-rs: safe Driver API bindings; SIMT surface under `cuda_core::simt` | +| `cuda-core` | Shared with cutile-rs: safe Driver API bindings; shared context, stream and buffer types | | `cuda-core-derive` | Shared with cutile-rs: `#[derive(DeviceCopy)]` proc-macro for `cuda-core` | | `cuda-async` | Shared with cutile-rs: `DeviceOperation` model and combinators under `cuda_async::simt` | | `cuda-bindings` | Shared with cutile-rs: FFI to the CUDA driver; `libcuda` is loaded at run time | diff --git a/cuda-oxide/cuda-oxide-book/getting-started/hello-gpu.md b/cuda-oxide/cuda-oxide-book/getting-started/hello-gpu.md index 3a4ff4ec5e..f3fb9bcefe 100644 --- a/cuda-oxide/cuda-oxide-book/getting-started/hello-gpu.md +++ b/cuda-oxide/cuda-oxide-book/getting-started/hello-gpu.md @@ -213,7 +213,7 @@ Here's the generated async vecadd template (with minor formatting edits for read ```rust use cuda_async::simt::device_context::init_device_contexts; use cuda_async::simt::device_operation::DeviceOperation; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_device::{DisjointSlice, kernel, thread}; use cuda_host::cuda_module; @@ -234,7 +234,7 @@ mod kernels { #[tokio::main] async fn main() -> Result<(), Box> { use cuda_async::simt::device_box::DeviceBox; - use cuda_core::simt::memory::{malloc_async, memcpy_dtoh_async, memcpy_htod_async}; + use cuda_core::memory::{malloc_async, memcpy_dtoh_async, memcpy_htod_async}; use std::mem; // 1. Initialize the device context map (default device 0, 1 device). diff --git a/cuda-oxide/cuda-oxide-book/gpu-programming/closures-and-generics.md b/cuda-oxide/cuda-oxide-book/gpu-programming/closures-and-generics.md index 7055e8100c..e63121abb7 100644 --- a/cuda-oxide/cuda-oxide-book/gpu-programming/closures-and-generics.md +++ b/cuda-oxide/cuda-oxide-book/gpu-programming/closures-and-generics.md @@ -63,7 +63,7 @@ forces the concrete instantiation and lets the loader look up the matching PTX entry point: ```rust -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; // SAFETY: config is 1D and both buffers cover every launched index. unsafe { diff --git a/cuda-oxide/cuda-oxide-book/gpu-programming/execution-model.md b/cuda-oxide/cuda-oxide-book/gpu-programming/execution-model.md index 7694d271f5..476b5bb6bb 100644 --- a/cuda-oxide/cuda-oxide-book/gpu-programming/execution-model.md +++ b/cuda-oxide/cuda-oxide-book/gpu-programming/execution-model.md @@ -160,7 +160,7 @@ for architecture-specific SM resource limits and occupancy calculations. On the host side, `LaunchConfig` tells the runtime how to shape the grid: ```rust -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; // Quick 1D launch: 256 threads per block, enough blocks to cover N elements let cfg = LaunchConfig::for_num_elems(N as u32); diff --git a/cuda-oxide/cuda-oxide-book/gpu-programming/launching-kernels.md b/cuda-oxide/cuda-oxide-book/gpu-programming/launching-kernels.md index 8a6b94844f..ba0bc04941 100644 --- a/cuda-oxide/cuda-oxide-book/gpu-programming/launching-kernels.md +++ b/cuda-oxide/cuda-oxide-book/gpu-programming/launching-kernels.md @@ -53,7 +53,7 @@ PreparedLaunch -> safe launch of exactly K GPU execution can still overlap the host until you synchronize. ```rust -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; @@ -532,7 +532,7 @@ explains why this can differ from the normal LLVM-to-PTX path. `LaunchConfig` specifies the grid shape: ```rust -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; let config = LaunchConfig { grid_dim: (num_blocks, 1, 1), diff --git a/cuda-oxide/cuda-oxide-book/gpu-programming/virtual-memory-and-peer-access.md b/cuda-oxide/cuda-oxide-book/gpu-programming/virtual-memory-and-peer-access.md index d7bbc9a06f..26ea9c6faf 100644 --- a/cuda-oxide/cuda-oxide-book/gpu-programming/virtual-memory-and-peer-access.md +++ b/cuda-oxide/cuda-oxide-book/gpu-programming/virtual-memory-and-peer-access.md @@ -26,12 +26,12 @@ This chapter covers: :::{seealso} The complete integration test is available in -[`cuda-core/tests/simt_vmm_p2p.rs`](https://github.com/NVIDIA/cuda-rust/blob/main/cuda-core/tests/simt_vmm_p2p.rs). +[`cuda-core/tests/vmm_p2p.rs`](https://github.com/NVIDIA/cuda-rust/blob/main/cuda-core/tests/vmm_p2p.rs). The corresponding implementations are: -* [`cuda-core/src/simt/vmm.rs`](https://github.com/NVIDIA/cuda-rust/blob/main/cuda-core/src/simt/vmm.rs) -* [`cuda-core/src/simt/peer.rs`](https://github.com/NVIDIA/cuda-rust/blob/main/cuda-core/src/simt/peer.rs) +* [`cuda-core/src/vmm.rs`](https://github.com/NVIDIA/cuda-rust/blob/main/cuda-core/src/vmm.rs) +* [`cuda-core/src/peer.rs`](https://github.com/NVIDIA/cuda-rust/blob/main/cuda-core/src/peer.rs) ::: ## VMM and P2P solve different problems @@ -83,7 +83,7 @@ to access each other directly. Always query support before enabling peer access: ```rust -use cuda_core::simt::peer; +use cuda_core::peer; use cuda_core::{CudaContext, DriverError}; fn check_peer_access() -> Result<(), DriverError> { @@ -114,7 +114,7 @@ Use `peer::enable_peer_access` to allow one context to access memory associated with another device: ```rust -use cuda_core::simt::peer; +use cuda_core::peer; use cuda_core::{CudaContext, DriverError}; fn enable_bidirectional_peer_access() -> Result<(), DriverError> { @@ -183,7 +183,7 @@ of the device's allocation granularity. Use `allocation_granularity` and `align_size` before allocating: ```rust -use cuda_core::simt::vmm; +use cuda_core::vmm; use cuda_core::{CudaContext, DriverError}; fn aligned_allocation_size(requested: usize) -> Result { @@ -203,7 +203,7 @@ The following example allocates physical memory, maps it into a virtual address range, writes data through the mapping, and reads it back. ```rust -use cuda_core::simt::vmm; +use cuda_core::vmm; use cuda_core::{CudaContext, DriverError}; fn single_gpu_vmm_roundtrip() -> Result<(), DriverError> { @@ -239,7 +239,7 @@ fn single_gpu_vmm_roundtrip() -> Result<(), DriverError> { let input: Vec = (0..64).map(|i| (i * 3 + 7) as u8).collect(); unsafe { - cuda_core::simt::memory::memcpy_htod_async( + cuda_core::memory::memcpy_htod_async( reservation.base(), input.as_ptr(), input.len(), @@ -252,7 +252,7 @@ fn single_gpu_vmm_roundtrip() -> Result<(), DriverError> { let mut output = vec![0u8; input.len()]; unsafe { - cuda_core::simt::memory::memcpy_dtoh_async( + cuda_core::memory::memcpy_dtoh_async( output.as_mut_ptr(), reservation.base(), output.len(), @@ -292,8 +292,8 @@ The following pattern: * Reads the same physical memory through GPU 1's mapping. ```rust -use cuda_core::simt::{peer, vmm}; use cuda_core::{CudaContext, DriverError, IntoResult}; +use cuda_core::{peer, vmm}; use std::mem::MaybeUninit; fn gpu_count() -> Result { @@ -380,7 +380,7 @@ fn cross_gpu_vmm_roundtrip() -> Result<(), DriverError> { ctx0.bind_to_thread()?; unsafe { - cuda_core::simt::memory::memcpy_htod_async( + cuda_core::memory::memcpy_htod_async( reservation0.base(), input.as_ptr(), byte_length, @@ -395,7 +395,7 @@ fn cross_gpu_vmm_roundtrip() -> Result<(), DriverError> { ctx1.bind_to_thread()?; unsafe { - cuda_core::simt::memory::memcpy_dtoh_async( + cuda_core::memory::memcpy_dtoh_async( output.as_mut_ptr(), reservation1.base(), byte_length, diff --git a/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_ctm/src/main.rs b/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_ctm/src/main.rs index 64ee1118e4..771e74a7eb 100644 --- a/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_ctm/src/main.rs +++ b/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_ctm/src/main.rs @@ -361,10 +361,10 @@ fn main() { let mut times_ms = Vec::with_capacity(samples); for _ in 0..samples { let start = ctx - .new_event(Some(cuda_core::sys::CUevent_flags_enum_CU_EVENT_DEFAULT)) + .new_event_with_flags(Some(cuda_core::sys::CUevent_flags_enum_CU_EVENT_DEFAULT)) .expect("create start event"); let end = ctx - .new_event(Some(cuda_core::sys::CUevent_flags_enum_CU_EVENT_DEFAULT)) + .new_event_with_flags(Some(cuda_core::sys::CUevent_flags_enum_CU_EVENT_DEFAULT)) .expect("create end event"); start.record(&stream).expect("record start event"); diff --git a/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_cute/src/main.rs b/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_cute/src/main.rs index 72b67ce03f..edfc235f7a 100644 --- a/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_cute/src/main.rs +++ b/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_cute/src/main.rs @@ -691,10 +691,10 @@ fn main() { let mut times_ms = Vec::with_capacity(SAMPLES); for _ in 0..SAMPLES { let start = ctx - .new_event(Some(cuda_core::sys::CUevent_flags_enum_CU_EVENT_DEFAULT)) + .new_event_with_flags(Some(cuda_core::sys::CUevent_flags_enum_CU_EVENT_DEFAULT)) .expect("create start event"); let end = ctx - .new_event(Some(cuda_core::sys::CUevent_flags_enum_CU_EVENT_DEFAULT)) + .new_event_with_flags(Some(cuda_core::sys::CUevent_flags_enum_CU_EVENT_DEFAULT)) .expect("create end event"); start.record(&stream).expect("record start event"); diff --git a/cuda-oxide/cute-rs-preview/examples/elementwise_cute/src/main.rs b/cuda-oxide/cute-rs-preview/examples/elementwise_cute/src/main.rs index 967b24730a..8126eab3b8 100644 --- a/cuda-oxide/cute-rs-preview/examples/elementwise_cute/src/main.rs +++ b/cuda-oxide/cute-rs-preview/examples/elementwise_cute/src/main.rs @@ -22,7 +22,7 @@ use ctm_rs::memory::DisjointSlice; use ctm_rs::{cuda_module, kernel, thread}; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; /// Deliberately not divisible by the tiles: f32 tail of 3, f16 tail of 5. diff --git a/cuda-oxide/cute-rs-preview/examples/ffn_cute/src/operator.rs b/cuda-oxide/cute-rs-preview/examples/ffn_cute/src/operator.rs index ba82a145c8..7252cdc432 100644 --- a/cuda-oxide/cute-rs-preview/examples/ffn_cute/src/operator.rs +++ b/cuda-oxide/cute-rs-preview/examples/ffn_cute/src/operator.rs @@ -4,7 +4,7 @@ */ //! Checked composition using existing buffer ownership and typed weight roles. use crate::kernel::kernels; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaStream, DeviceBuffer}; use cute_rs::{ BFloat16, diff --git a/cuda-oxide/cute-rs-preview/examples/fmha_decode_sm120a_cute/src/runner.rs b/cuda-oxide/cute-rs-preview/examples/fmha_decode_sm120a_cute/src/runner.rs index ea8e987fb4..18f105c57b 100644 --- a/cuda-oxide/cute-rs-preview/examples/fmha_decode_sm120a_cute/src/runner.rs +++ b/cuda-oxide/cute-rs-preview/examples/fmha_decode_sm120a_cute/src/runner.rs @@ -160,8 +160,8 @@ fn run_shape(args: Args) -> Result<()> { let events = (0..args.iters) .map(|_| { Ok(( - ctx.new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, - ctx.new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, + ctx.new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, + ctx.new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, )) }) .collect::>>()?; diff --git a/cuda-oxide/cute-rs-preview/examples/fmha_prefill_sm120a_cute/src/runner.rs b/cuda-oxide/cute-rs-preview/examples/fmha_prefill_sm120a_cute/src/runner.rs index e26ac291e9..0d67ce582e 100644 --- a/cuda-oxide/cute-rs-preview/examples/fmha_prefill_sm120a_cute/src/runner.rs +++ b/cuda-oxide/cute-rs-preview/examples/fmha_prefill_sm120a_cute/src/runner.rs @@ -260,8 +260,8 @@ fn run_shape let events = (0..args.iters) .map(|_| { Ok(( - ctx.new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, - ctx.new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, + ctx.new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, + ctx.new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, )) }) .collect::>>()?; diff --git a/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_ctm/src/main.rs b/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_ctm/src/main.rs index 83225e3bad..6b64737d9a 100644 --- a/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_ctm/src/main.rs +++ b/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_ctm/src/main.rs @@ -537,8 +537,8 @@ fn main() -> Result<()> { let events = (0..args.iters) .map(|_| { Ok(( - ctx.new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, - ctx.new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, + ctx.new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, + ctx.new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, )) }) .collect::>>()?; diff --git a/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_cute/src/main.rs b/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_cute/src/main.rs index 011e19e5bf..1713dcccf7 100644 --- a/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_cute/src/main.rs +++ b/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_cute/src/main.rs @@ -615,8 +615,8 @@ fn main() -> Result<()> { let events = (0..args.iters) .map(|_| { Ok(( - ctx.new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, - ctx.new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, + ctx.new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, + ctx.new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?, )) }) .collect::>>()?; diff --git a/cuda-oxide/cute-rs-preview/examples/nvfp4_gemv_cute/src/main.rs b/cuda-oxide/cute-rs-preview/examples/nvfp4_gemv_cute/src/main.rs index f81772a990..d6e0e406fd 100644 --- a/cuda-oxide/cute-rs-preview/examples/nvfp4_gemv_cute/src/main.rs +++ b/cuda-oxide/cute-rs-preview/examples/nvfp4_gemv_cute/src/main.rs @@ -21,7 +21,7 @@ use ctm_rs::memory::DisjointSlice; use ctm_rs::{cuda_module, kernel, thread}; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, DeviceBuffer}; use cute_rs::{ BlockScaledTensor, Float4E2M1FN, Float8E8M0FNU, KMajor, Mkl, Nkl, Sm1xxBlockScaleKMajor, diff --git a/cuda-oxide/cute-rs-preview/examples/rmsnorm_cute/src/main.rs b/cuda-oxide/cute-rs-preview/examples/rmsnorm_cute/src/main.rs index 8c79daf8a6..220a45d7a2 100644 --- a/cuda-oxide/cute-rs-preview/examples/rmsnorm_cute/src/main.rs +++ b/cuda-oxide/cute-rs-preview/examples/rmsnorm_cute/src/main.rs @@ -5,7 +5,7 @@ //! RMSNorm validation with FP32 accumulation and explicit BF16 rounding boundaries. -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use cute_rs::BFloat16; diff --git a/cuda-oxide/cute-rs-preview/examples/rope_cache_cute/src/operator.rs b/cuda-oxide/cute-rs-preview/examples/rope_cache_cute/src/operator.rs index 262371d2e2..7a1bed95ef 100644 --- a/cuda-oxide/cute-rs-preview/examples/rope_cache_cute/src/operator.rs +++ b/cuda-oxide/cute-rs-preview/examples/rope_cache_cute/src/operator.rs @@ -4,7 +4,7 @@ */ //! Synchronous BF16 rotary composition with privately owned key/value caches. use crate::{kernel::kernels, rotary::HostTables}; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaStream, DeviceBuffer}; use cute_rs::{ BFloat16, diff --git a/cuda-oxide/cute-rs-preview/examples/sm100_fmha_prefill_cute/src/main.rs b/cuda-oxide/cute-rs-preview/examples/sm100_fmha_prefill_cute/src/main.rs index 2d3bcb5f46..7596d2f642 100644 --- a/cuda-oxide/cute-rs-preview/examples/sm100_fmha_prefill_cute/src/main.rs +++ b/cuda-oxide/cute-rs-preview/examples/sm100_fmha_prefill_cute/src/main.rs @@ -346,8 +346,8 @@ fn main() -> Result<()> { run()?; } stream.synchronize()?; - let start = ctx.new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; - let end = ctx.new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; + let start = ctx.new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; + let end = ctx.new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; start.record(&stream)?; for _ in 0..args.iters { run()?; diff --git a/cuda-oxide/cute-rs-preview/examples/sm100_mla_decode_fp16_cute/src/main.rs b/cuda-oxide/cute-rs-preview/examples/sm100_mla_decode_fp16_cute/src/main.rs index 43d145b047..744fbdf09a 100644 --- a/cuda-oxide/cute-rs-preview/examples/sm100_mla_decode_fp16_cute/src/main.rs +++ b/cuda-oxide/cute-rs-preview/examples/sm100_mla_decode_fp16_cute/src/main.rs @@ -373,8 +373,10 @@ fn main() -> Result<()> { run()?; } stream.synchronize()?; - let start = context.new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; - let end = context.new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; + let start = + context.new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; + let end = + context.new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; start.record(&stream)?; for _ in 0..args.iterations { run()?; diff --git a/cuda-oxide/cute-rs-preview/examples/swiglu_cute/src/main.rs b/cuda-oxide/cute-rs-preview/examples/swiglu_cute/src/main.rs index eec123d579..d58d5323e5 100644 --- a/cuda-oxide/cute-rs-preview/examples/swiglu_cute/src/main.rs +++ b/cuda-oxide/cute-rs-preview/examples/swiglu_cute/src/main.rs @@ -9,7 +9,7 @@ use ctm_rs::memory::DisjointSlice; use ctm_rs::{cuda_module, kernel, thread}; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; /// Empty inputs, sub-tile inputs, aligned FFN widths, and a partial final tile. diff --git a/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/src/benchmark.rs b/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/src/benchmark.rs index 8a1d5cdb33..c562187423 100644 --- a/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/src/benchmark.rs +++ b/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/src/benchmark.rs @@ -190,8 +190,8 @@ pub struct Qwen3Engine { } /// Make a 128-thread 1D launch configuration for `work` items. -fn grid1d(work: usize) -> cuda_core::simt::LaunchConfig { - cuda_core::simt::LaunchConfig { +fn grid1d(work: usize) -> cuda_core::LaunchConfig { + cuda_core::LaunchConfig { grid_dim: (work.div_ceil(128) as u32, 1, 1), block_dim: (128, 1, 1), shared_mem_bytes: 0, @@ -254,7 +254,7 @@ impl Qwen3Engine { )?; } else { let tiles = rows.div_ceil(16) * outputs.div_ceil(8); - let cfg = cuda_core::simt::LaunchConfig { + let cfg = cuda_core::LaunchConfig { grid_dim: (tiles.div_ceil(4) as u32, 1, 1), block_dim: (128, 1, 1), shared_mem_bytes: 0, diff --git a/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/src/checkpoint.rs b/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/src/checkpoint.rs index 46bf88a5fd..890aa1953b 100644 --- a/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/src/checkpoint.rs +++ b/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/src/checkpoint.rs @@ -17,7 +17,7 @@ use crate::nvfp4::Nvfp4Projection; use crate::operator::Result; use ctm_rs::memory::DisjointSlice; use ctm_rs::{cuda_module, kernel, thread}; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaStream, DeviceBuffer}; use cute_rs::BFloat16; use cute_rs::decoder::{ diff --git a/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/src/nvfp4.rs b/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/src/nvfp4.rs index 6732a501ca..5d2c924ba0 100644 --- a/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/src/nvfp4.rs +++ b/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/src/nvfp4.rs @@ -26,7 +26,7 @@ use crate::model::Batch; use crate::operator::Result; use ctm_rs::memory::DisjointSlice; use ctm_rs::{cuda_module, kernel, thread}; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaStream, DeviceBuffer}; use cute_rs::block_scaled::{ CANONICAL_SCALE_ATOM_BYTES, CANONICAL_SCALE_K_GROUPS, CANONICAL_SCALE_M, diff --git a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/codec_attention.rs b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/codec_attention.rs index bdc203d7cf..0041f61267 100644 --- a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/codec_attention.rs +++ b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/codec_attention.rs @@ -5,7 +5,7 @@ //! Full-sequence D64 causal sliding attention for the waveform decoder. use crate::{model::Batch, operator::Result, rotary::HostTables}; use ctm_rs::{cuda_module, kernel, launch_contract, memory::DisjointSlice, thread}; -use cuda_core::{CudaStream, simt::LaunchConfig}; +use cuda_core::{CudaStream, LaunchConfig}; use cute_rs::{BFloat16, attention::AttentionShape}; use std::sync::Arc; diff --git a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/codec_ops.rs b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/codec_ops.rs index e33fc0aa89..01e1908c5d 100644 --- a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/codec_ops.rs +++ b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/codec_ops.rs @@ -5,7 +5,7 @@ //! Checked time-major waveform decoder operations. use crate::{codec_kernel, model::Batch, operator::Result}; -use cuda_core::{CudaStream, DeviceBuffer, LaunchConfig1D, simt::LaunchConfig}; +use cuda_core::{CudaStream, DeviceBuffer, LaunchConfig, LaunchConfig1D}; use cute_rs::BFloat16; use std::sync::Arc; diff --git a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/decode_linear.rs b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/decode_linear.rs index 1b4ed6c9bf..a03a8f414d 100644 --- a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/decode_linear.rs +++ b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/decode_linear.rs @@ -537,7 +537,7 @@ fn fixture( // Run the original four-warp entry on the same buffers. The new // launch must be bitwise equal, not merely within the oracle's bound. let mut original = Batch::::allocate(&runner.stream, rows)?; - let config = cuda_core::simt::LaunchConfig { + let config = cuda_core::LaunchConfig { grid_dim: (OUT.div_ceil(8).div_ceil(4) as u32, 1, 1), block_dim: (128, 1, 1), shared_mem_bytes: 0, diff --git a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/decode_linear_bench.rs b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/decode_linear_bench.rs index cd3ffaa941..e610d38248 100644 --- a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/decode_linear_bench.rs +++ b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/decode_linear_bench.rs @@ -58,17 +58,17 @@ pub fn run(stream: Arc) -> Result<()> { DeviceBuffer::from_host(&stream, &vec![BFloat16::from_f32(0.0); rows * outputs])?; let config = LaunchConfig1D::new(outputs.div_ceil(8) as u32, 32, 0); let prepared = module.prepare_decode_linear_bf16(config)?; - let original = cuda_core::simt::LaunchConfig { + let original = cuda_core::LaunchConfig { grid_dim: (outputs.div_ceil(32) as u32, 1, 1), block_dim: (128, 1, 1), shared_mem_bytes: 0, }; let start = stream .context() - .new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; + .new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; let end = stream .context() - .new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; + .new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; for variant in ["original", "decode", "reference"] { if variant == "reference" && plan == ReductionPlan::MmaChain { continue; diff --git a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/decoder_execution.rs b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/decoder_execution.rs index 75b7ea9bdf..deb4596a28 100644 --- a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/decoder_execution.rs +++ b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/decoder_execution.rs @@ -11,7 +11,7 @@ use crate::reference_linear_kernel::kernels as reference; use crate::{ attention_kernel, element_kernel, ffn_kernel, kernel, norm_kernel, predictor_graph_kernel, }; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaStream, DeviceBuffer, LaunchConfig1D, PreparedLaunch}; use cute_rs::{ BFloat16, diff --git a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/model.rs b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/model.rs index d2b6ed9bb4..0c27341446 100644 --- a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/model.rs +++ b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/model.rs @@ -15,7 +15,7 @@ //! preserving the general MMA reduction and BF16 rounding boundaries. use crate::operator::Result; use crate::{element_kernel, ffn_kernel, norm_kernel, operator, rotary::HostTables}; -use cuda_core::simt::LaunchConfig; +use cuda_core::LaunchConfig; use cuda_core::{CudaStream, DeviceBuffer}; use cute_rs::{ BFloat16, diff --git a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/norm_specialized.rs b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/norm_specialized.rs index d04fa16817..86b9b72b0e 100644 --- a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/norm_specialized.rs +++ b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/src/norm_specialized.rs @@ -7,7 +7,7 @@ use crate::{ norm_kernel::{ROWS_PER_BLOCK, kernels}, operator::Result, }; -use cuda_core::{CudaStream, DeviceBuffer, simt::LaunchConfig, sys}; +use cuda_core::{CudaStream, DeviceBuffer, LaunchConfig, sys}; use cute_rs::BFloat16; use std::sync::Arc; @@ -330,10 +330,10 @@ pub fn benchmark(stream: Arc) -> Result<()> { }; let start = stream .context() - .new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; + .new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; let end = stream .context() - .new_event(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; + .new_event_with_flags(Some(sys::CUevent_flags_enum_CU_EVENT_DEFAULT))?; stream.context().synchronize()?; let mut submit = |new: bool| -> Result<()> { // SAFETY: ready independent buffers, full warp ownership and diff --git a/cuda-oxide/scripts/check-host-api-paths.sh b/cuda-oxide/scripts/check-host-api-paths.sh index ef5a576890..e697f14636 100755 --- a/cuda-oxide/scripts/check-host-api-paths.sh +++ b/cuda-oxide/scripts/check-host-api-paths.sh @@ -1,34 +1,8 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Reject host-API paths that no longer resolve on the shared host crates. -# -# cuda-bindings, cuda-core, and cuda-async are shared with cutile-rs. Their -# crate roots carry cutile's own Tile API; the cuda-oxide (SIMT) surface lives -# under `cuda_core::simt` and `cuda_async::simt`. cuda-core re-exports most of -# the SIMT items at its root, with four deliberate exceptions that collide with -# Tile-side names or were never re-exported: -# -# cuda_core::LaunchConfig -> cuda_core::simt::LaunchConfig -# (root LaunchConfig is cutile's struct; -# LaunchConfig1D/2D/3D stay at the root) -# cuda_core::{memory,peer,vmm} -> cuda_core::simt::{memory,peer,vmm} -# (root vmm is cutile's fork) -# cuda_core::error::IntoResult -> cuda_core::IntoResult -# cuda_async:::: -> cuda_async::simt:::: -# (root device_operation, device_context, -# device_box, launch, error ... are Tile) -# -# A doc snippet, README, or `cargo oxide new` template that spells the old -# root path either fails to compile or, worse, binds the Tile-side type and -# fails later with a type mismatch. Nothing compiles those snippets, which is -# how some forty stale lines survived the crate switch, so this guard greps -# them. Rust sources under crates/ are covered too, for their doc comments; -# they are compiled, but a `//!` example is not. -# -# Root `cuda_async::zip!` / `unzip!` are fine: they expand to `.zip()` and -# dispatch on whichever `Zippable` is in scope, and they are the one shared -# entry point (the simt copies cannot be re-exported without a root collision). +# Reject paths into the removed cuda-core SIMT namespace. Shared handles and +# typed buffers now live at the crate root. cuda-async is migrated separately. set -euo pipefail export LC_ALL=C cd "$(dirname "$0")/.." @@ -48,14 +22,12 @@ mapfile -t files < <(git -C "${GIT_ROOT}" ls-files \ 'cuda-oxide/crates/cuda-device/src/**/*.rs' 'cuda-oxide/crates/cuda-device/src/*.rs' \ | sort -u) -# Each pattern is an ERE; `cuda_core::simt::...` never matches because the +# Each pattern is an ERE; `cuda_core::...` never matches because the # module name has to follow `cuda_core::` directly. patterns=( - 'cuda_core::LaunchConfig([^0-9A-Za-z_]|$)' - 'cuda_core::(memory|peer|vmm|error|launch|stream|module|context|event|pinned_host_buffer|device_buffer)::' - 'use cuda_core::\{[^}]*(^|[^A-Za-z0-9_])(LaunchConfig|peer|vmm|memory)([^0-9A-Za-z_]|\})' - 'cuda_async::(device_operation|device_context|device_box|device_future|scheduling_policies|reclaim|launch|error)([^0-9A-Za-z_]|$)' - 'cuda_core::simt::simt|cuda_async::simt::simt' + 'cuda_core::simt' + 'cuda_core::error::' + 'cuda_async::simt::simt' ) status=0 @@ -68,7 +40,7 @@ for pattern in "${patterns[@]}"; do done if [ "${status}" -ne 0 ]; then - echo "hint: SIMT items live under cuda_core::simt / cuda_async::simt; see the header of $0." >&2 + echo "hint: cuda-core handles and buffers use the shared crate-root API." >&2 exit 1 fi echo "OK: no stale host-API root paths in $((${#files[@]})) doc and source files." diff --git a/cutile-rs/CHANGELOG.md b/cutile-rs/CHANGELOG.md index 11732161e4..23f4a15330 100644 --- a/cutile-rs/CHANGELOG.md +++ b/cutile-rs/CHANGELOG.md @@ -39,6 +39,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - Advance the lockstep Tile and shared host crates to v0.5.0 for the breaking safety-contract and host-runtime changes. +- Tile and SIMT now share the `cuda-core` context, stream, event, module, launch + configuration, and allocation types. Replace `cuda_core::simt` imports with + crate-root imports; use `Context` for context ownership and `Device` for a + physical GPU identity. Foreign allocations must provide their actual context. +- Typed buffers and tensors can share allocations without copying through + `Tensor::from_buffer`, `Tensor::into_buffer`, and contiguous borrowed buffer + views. Views share context identity, byte offsets, and access exclusion. + Completion leases retain allocations after early drops and forgotten launches; + buffer and stream cleanup no longer waits for GPU work on the dropping thread. +- Tile numeric storage wrappers implement `DeviceCopy` alongside `DType`; the + traits remain independent and packed FP4 keeps its byte storage representation. + - cuTile Rust now lives in the [NVIDIA/cuda-rust](https://github.com/NVIDIA/cuda-rust) repository under `cutile-rs/`, next to cuda-oxide (SIMT) and the shared host crates (`cuda-bindings`, `cuda-core`, `cuda-core-derive`, `cuda-async`) at the diff --git a/cutile-rs/cutile-benchmarks/benches/execution_lock.rs b/cutile-rs/cutile-benchmarks/benches/execution_lock.rs index 8bea76b85c..2769551975 100644 --- a/cutile-rs/cutile-benchmarks/benches/execution_lock.rs +++ b/cutile-rs/cutile-benchmarks/benches/execution_lock.rs @@ -14,7 +14,7 @@ use cuda_async::device_operation::{value, DeviceOp}; use std::time::Duration; fn has_gpu() -> bool { - cuda_core::Device::device_count() + cuda_core::Context::device_count() .map(|n| n > 0) .unwrap_or(false) } @@ -27,7 +27,7 @@ fn bench_execution_lock(c: &mut Criterion) { // Run on a fresh thread to get a clean CUDA context. let (stream,) = std::thread::spawn(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); (stream,) }) @@ -55,7 +55,7 @@ fn bench_execution_lock(c: &mut Criterion) { group.bench_function("async_on_no_lock", |b| { b.iter(|| { unsafe { value(42).async_on(&stream).unwrap() }; - unsafe { stream.synchronize() }.unwrap(); + stream.synchronize().unwrap(); }); }); diff --git a/cutile-rs/cutile-benchmarks/benches/fmha.rs b/cutile-rs/cutile-benchmarks/benches/fmha.rs index 3d4700a319..3e7a9c74b9 100644 --- a/cutile-rs/cutile-benchmarks/benches/fmha.rs +++ b/cutile-rs/cutile-benchmarks/benches/fmha.rs @@ -4,7 +4,7 @@ */ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use cutile::api::{randn_f16, zeros}; use cutile::core::f16; use cutile::tensor::{IntoPartition, Partition, Tensor}; @@ -159,7 +159,7 @@ fn ocean_fmha(c: &mut Criterion) { .measurement_time(Duration::from_millis(2000)); } - let device = Device::new(0).expect("Failed to get device."); + let device = Context::new(0).expect("Failed to get device."); let stream = device.new_stream().expect("Failed to get stream."); // This is what the ocean benchmark uses. @@ -238,7 +238,7 @@ fn ocean_fmha(c: &mut Criterion) { out.grid().expect("Invalid grid."), ((b * h) as u32, (m / bm) as u32, 1) ); - unsafe { stream.synchronize() }.expect("Failed to synchronize."); + stream.synchronize().expect("Failed to synchronize."); let start = Instant::now(); for _i in 0..iters { let (_, _, _, out_local, _, _) = unsafe { @@ -256,7 +256,7 @@ fn ocean_fmha(c: &mut Criterion) { }; out = out_local; } - unsafe { stream.synchronize() }.expect("Failed to synchronize."); + stream.synchronize().expect("Failed to synchronize."); start.elapsed() }); }, diff --git a/cutile-rs/cutile-benchmarks/benches/fmha_causal.rs b/cutile-rs/cutile-benchmarks/benches/fmha_causal.rs index 15e6548d6a..19d7bf8070 100644 --- a/cutile-rs/cutile-benchmarks/benches/fmha_causal.rs +++ b/cutile-rs/cutile-benchmarks/benches/fmha_causal.rs @@ -4,7 +4,7 @@ */ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use cutile::api::{randn_f16, zeros}; use cutile::core::f16; use cutile::tensor::{IntoPartition, Partition, Tensor}; @@ -149,7 +149,7 @@ fn ocean_fmha_causal(c: &mut Criterion) { .measurement_time(Duration::from_millis(2000)); } - let device = Device::new(0).expect("Failed to get device."); + let device = Context::new(0).expect("Failed to get device."); let stream = device.new_stream().expect("Failed to get stream."); // This is what the ocean benchmark uses. @@ -239,7 +239,7 @@ fn ocean_fmha_causal(c: &mut Criterion) { out.grid().expect("Invalid grid."), ((b * h) as u32, (m / bm) as u32, 1) ); - unsafe { stream.synchronize() }.expect("Failed to synchronize."); + stream.synchronize().expect("Failed to synchronize."); let start = Instant::now(); for _i in 0..iters { let (_, _, _, out_local, _, _, _) = unsafe { @@ -258,7 +258,7 @@ fn ocean_fmha_causal(c: &mut Criterion) { }; out = out_local; } - unsafe { stream.synchronize() }.expect("Failed to synchronize."); + stream.synchronize().expect("Failed to synchronize."); start.elapsed() }); }, diff --git a/cutile-rs/cutile-benchmarks/benches/gemm.rs b/cutile-rs/cutile-benchmarks/benches/gemm.rs index a327b73db1..02ed25536c 100644 --- a/cutile-rs/cutile-benchmarks/benches/gemm.rs +++ b/cutile-rs/cutile-benchmarks/benches/gemm.rs @@ -4,7 +4,7 @@ */ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use cuda_async::device_operation::{value, DeviceOp}; -use cuda_core::Device; +use cuda_core::Context; use cutile::api; use cutile::core::f16; use cutile::tile_kernel::{PartitionOp, TileKernel}; @@ -91,7 +91,7 @@ fn ocean_gemm(c: &mut Criterion) { .measurement_time(Duration::from_millis(2000)); } - let device = Device::new(0).expect("Failed to get device."); + let device = Context::new(0).expect("Failed to get device."); let stream = device.new_stream().expect("Failed to get stream."); // This is what the ocean benchmark uses. @@ -135,7 +135,7 @@ fn ocean_gemm(c: &mut Criterion) { .partition([bm, bn]) .sync_on(&stream) .expect("Failed."); - unsafe { stream.synchronize() }.expect("Failed to synchronize."); + stream.synchronize().expect("Failed to synchronize."); let start = Instant::now(); for _i in 0..iters { unsafe { @@ -146,7 +146,7 @@ fn ocean_gemm(c: &mut Criterion) { z = local_z; } } - unsafe { stream.synchronize() }.expect("Failed to synchronize."); + stream.synchronize().expect("Failed to synchronize."); start.elapsed() }); }); diff --git a/cutile-rs/cutile-benchmarks/benches/host_overhead.rs b/cutile-rs/cutile-benchmarks/benches/host_overhead.rs index ceae70d232..8db63ec401 100644 --- a/cutile-rs/cutile-benchmarks/benches/host_overhead.rs +++ b/cutile-rs/cutile-benchmarks/benches/host_overhead.rs @@ -16,7 +16,7 @@ use criterion::{criterion_group, criterion_main, Criterion}; use cuda_async::cuda_graph::CudaGraph; -use cuda_core::Device; +use cuda_core::Context; use cutile::prelude::*; use std::future::IntoFuture; use std::time::{Duration, Instant}; @@ -76,7 +76,7 @@ fn host_overhead(c: &mut Criterion) { .measurement_time(Duration::from_millis(2000)); } - let device = Device::new(0).expect("device"); + let device = Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); let a: Arc> = api::ones::(&[N]).sync_on(&stream).expect("a").into(); @@ -117,7 +117,7 @@ fn host_overhead(c: &mut Criterion) { z = local_z; } elapsed += start.elapsed(); - unsafe { stream.synchronize() }.expect("drain"); + stream.synchronize().expect("drain"); done += batch; } elapsed @@ -150,7 +150,7 @@ fn host_overhead(c: &mut Criterion) { z = local_z; } elapsed += start.elapsed(); - unsafe { stream.synchronize() }.expect("drain"); + stream.synchronize().expect("drain"); done += batch; } elapsed @@ -203,7 +203,7 @@ fn host_overhead(c: &mut Criterion) { unsafe { graph.launch().async_on(&stream) }.expect("replay"); } elapsed += start.elapsed(); - unsafe { stream.synchronize() }.expect("drain"); + stream.synchronize().expect("drain"); done += batch; } elapsed diff --git a/cutile-rs/cutile-benchmarks/benches/launch_overhead.rs b/cutile-rs/cutile-benchmarks/benches/launch_overhead.rs index 20e80b44bd..c353483d9b 100644 --- a/cutile-rs/cutile-benchmarks/benches/launch_overhead.rs +++ b/cutile-rs/cutile-benchmarks/benches/launch_overhead.rs @@ -15,7 +15,7 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use cutile::api; use cutile::core::f16; use cutile::tensor::{IntoPartition, Partition, Tensor}; @@ -63,7 +63,7 @@ fn launch_overhead(c: &mut Criterion) { .measurement_time(Duration::from_millis(2000)); } - let device = Device::new(0).expect("Failed to get device."); + let device = Context::new(0).expect("Failed to get device."); let stream = device.new_stream().expect("Failed to get stream."); let n: usize = 1024; @@ -100,17 +100,17 @@ fn launch_overhead(c: &mut Criterion) { z = local_z; } } - unsafe { stream.synchronize() }.expect("Failed."); + stream.synchronize().expect("Failed."); std::thread::sleep(Duration::from_millis(200)); - // 1. sync_on: execute + unsafe { stream.synchronize() }, no callbacks. + // 1. sync_on: execute + stream.synchronize(), no callbacks. group.bench_function(BenchmarkId::new("mode", "sync_on"), |b| { b.iter_custom(|iters| { let mut z: Partition> = api::zeros::(&[n, n]) .sync_on(&stream) .expect("Failed.") .partition([bm, bn]); - unsafe { stream.synchronize() }.expect("Failed."); + stream.synchronize().expect("Failed."); let start = Instant::now(); for _ in 0..iters { let (local_z, _, _, _) = unsafe { @@ -146,7 +146,7 @@ fn launch_overhead(c: &mut Criterion) { } }); } - unsafe { stream.synchronize() }.expect("Failed."); + stream.synchronize().expect("Failed."); std::thread::sleep(Duration::from_millis(200)); // 2. await: execute on first poll + cuLaunchHostFunc callback. @@ -157,7 +157,7 @@ fn launch_overhead(c: &mut Criterion) { .sync_on(&stream) .expect("Failed.") .partition([bm, bn]); - unsafe { stream.synchronize() }.expect("Failed."); + stream.synchronize().expect("Failed."); let start = Instant::now(); for _ in 0..iters { let (local_z, _, _, _) = unsafe { @@ -188,7 +188,7 @@ fn launch_overhead(c: &mut Criterion) { } } } - unsafe { stream.synchronize() }.expect("Failed."); + stream.synchronize().expect("Failed."); std::thread::sleep(Duration::from_millis(200)); // 3. async_on: execute only, single synchronize at end. @@ -198,7 +198,7 @@ fn launch_overhead(c: &mut Criterion) { .sync_on(&stream) .expect("Failed.") .partition([bm, bn]); - unsafe { stream.synchronize() }.expect("Failed."); + stream.synchronize().expect("Failed."); let start = Instant::now(); for _ in 0..iters { unsafe { @@ -209,7 +209,7 @@ fn launch_overhead(c: &mut Criterion) { z = local_z; } } - unsafe { stream.synchronize() }.expect("Failed."); + stream.synchronize().expect("Failed."); start.elapsed() }); }); diff --git a/cutile-rs/cutile-benchmarks/benches/rmsnorm.rs b/cutile-rs/cutile-benchmarks/benches/rmsnorm.rs index b592eda4fe..88aba6a1c4 100644 --- a/cutile-rs/cutile-benchmarks/benches/rmsnorm.rs +++ b/cutile-rs/cutile-benchmarks/benches/rmsnorm.rs @@ -4,7 +4,7 @@ */ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use cutile::api::{randn_f16, zeros}; use cutile::core::f16; use cutile::tensor::{IntoPartition, Partition, Tensor}; @@ -80,7 +80,7 @@ fn ocean_rmsnorm(c: &mut Criterion) { .measurement_time(Duration::from_millis(2000)); } - let device = Device::new(0).expect("Failed to get device."); + let device = Context::new(0).expect("Failed to get device."); let stream = device.new_stream().expect("Failed to get stream."); let shapes = (0..6) @@ -111,7 +111,7 @@ fn ocean_rmsnorm(c: &mut Criterion) { .sync_on(&stream) .expect("Failed.") .partition([1, n]); - unsafe { stream.synchronize() }.expect("Failed to synchronize."); + stream.synchronize().expect("Failed to synchronize."); let start = Instant::now(); for _i in 0..iters { unsafe { @@ -122,7 +122,7 @@ fn ocean_rmsnorm(c: &mut Criterion) { out = local_out; } } - unsafe { stream.synchronize() }.expect("Failed to synchronize."); + stream.synchronize().expect("Failed to synchronize."); start.elapsed() }); }); diff --git a/cutile-rs/cutile-benchmarks/benches/softmax.rs b/cutile-rs/cutile-benchmarks/benches/softmax.rs index c99075ab64..5a88976fc9 100644 --- a/cutile-rs/cutile-benchmarks/benches/softmax.rs +++ b/cutile-rs/cutile-benchmarks/benches/softmax.rs @@ -4,7 +4,7 @@ */ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use cutile::api::{randn_f16, zeros}; use cutile::core::f16; use cutile::tensor::{IntoPartition, Partition, Tensor}; @@ -49,7 +49,7 @@ fn softmax(c: &mut Criterion) { .measurement_time(Duration::from_millis(2000)); } - let device = Device::new(0).expect("Failed to get device."); + let device = Context::new(0).expect("Failed to get device."); let stream = device.new_stream().expect("Failed to get stream."); let shapes = (0..6) @@ -82,7 +82,7 @@ fn softmax(c: &mut Criterion) { .sync_on(&stream) .expect("Failed.") .partition([bm, bn]); - unsafe { stream.synchronize() }.expect("Failed to synchronize."); + stream.synchronize().expect("Failed to synchronize."); let start = Instant::now(); for _i in 0..iters { unsafe { @@ -93,7 +93,7 @@ fn softmax(c: &mut Criterion) { out = local_out; } } - unsafe { stream.synchronize() }.expect("Failed to synchronize."); + stream.synchronize().expect("Failed to synchronize."); start.elapsed() }); }, diff --git a/cutile-rs/cutile-benchmarks/paper/app_exp1_work_distribution/README.md b/cutile-rs/cutile-benchmarks/paper/app_exp1_work_distribution/README.md index d2c37c899d..1f556d8bbe 100644 --- a/cutile-rs/cutile-benchmarks/paper/app_exp1_work_distribution/README.md +++ b/cutile-rs/cutile-benchmarks/paper/app_exp1_work_distribution/README.md @@ -72,7 +72,7 @@ the workers once and reuses them via command/done channels: `spawn_local`'d worker tasks. **Context binding.** In Mode B each worker calls -`stream.device().bind_to_thread()` on startup, binding the CUDA +`stream.context().bind_to_thread()` on startup, binding the CUDA context to that OS thread. Without this, driver calls serialize against whichever thread currently owns the context. diff --git a/cutile-rs/cutile-benchmarks/paper/app_exp1_work_distribution/bimodal_gemms_rust/src/main.rs b/cutile-rs/cutile-benchmarks/paper/app_exp1_work_distribution/bimodal_gemms_rust/src/main.rs index 688bfd9556..e7aa71828c 100644 --- a/cutile-rs/cutile-benchmarks/paper/app_exp1_work_distribution/bimodal_gemms_rust/src/main.rs +++ b/cutile-rs/cutile-benchmarks/paper/app_exp1_work_distribution/bimodal_gemms_rust/src/main.rs @@ -39,7 +39,7 @@ use cuda_async::device_future::DeviceFuture; use cuda_async::device_operation::{DeviceOp, ExecutionContext}; use cuda_core::sys::CUctx_flags_enum_CU_CTX_SCHED_SPIN; -use cuda_core::{sys, Device, IntoResult, Stream}; +use cuda_core::{sys, Context, IntoResult, Stream}; use cutile::api; use cutile::core::f16; use cutile::tensor::{IntoPartition, Partition, Tensor}; @@ -243,7 +243,7 @@ fn make_workload(n: usize, small_ratio: f64) -> Vec { fn run_serial_samples( workload: &[Size], - device: &Arc, + device: &Arc, samples: usize, warmup: bool, spec: SizeSpec, @@ -251,7 +251,7 @@ fn run_serial_samples( ) -> Vec { let stream = device.new_stream().expect("stream"); let mut worker = Worker::new(stream.clone(), spec); - unsafe { stream.synchronize() }.expect("sync"); + stream.synchronize().expect("sync"); let run_once = |worker: &mut Worker| -> f64 { let t0 = Instant::now(); @@ -295,7 +295,7 @@ enum Cmd { fn run_threaded_samples( workload: &[Size], - device: &Arc, + device: &Arc, stream_count: usize, samples: usize, warmup: bool, @@ -308,7 +308,7 @@ fn run_threaded_samples( .collect(); let workers: Vec = streams.into_iter().map(|s| Worker::new(s, spec)).collect(); for w in &workers { - unsafe { w.stream.synchronize() }.expect("sync"); + w.stream.synchronize().expect("sync"); } let (done_tx, done_rx) = mpsc::channel::(); @@ -320,7 +320,7 @@ fn run_threaded_samples( cmd_senders.push(cmd_tx); let done_tx = done_tx.clone(); let h = std::thread::spawn(move || { - worker.stream.device().bind_to_thread().expect("bind"); + worker.stream.context().bind_to_thread().expect("bind"); while let Ok(cmd) = cmd_rx.recv() { match cmd { Cmd::Quit => break, @@ -391,7 +391,7 @@ fn run_threaded_samples( fn run_async_samples( workload: &[Size], - device: &Arc, + device: &Arc, stream_count: usize, samples: usize, warmup: bool, @@ -436,7 +436,7 @@ fn run_async_samples( let h = tokio::spawn(async move { let stream = device.new_stream().expect("stream"); let mut worker = Worker::new(stream, spec); - unsafe { worker.stream.synchronize() }.expect("sync"); + worker.stream.synchronize().expect("sync"); while let Some(maybe_queue) = cmd_rx.recv().await { let queue = match maybe_queue { Some(q) => q, @@ -604,7 +604,7 @@ fn main() -> Result<(), Box> { } let spec = SizeSpec { small_m, large_m }; - let device = Device::new(0).expect("device"); + let device = Context::new(0).expect("device"); // Pin the context to CU_CTX_SCHED_SPIN for a fair cross-mode // comparison. The CUDA default CU_CTX_SCHED_AUTO adapts sync diff --git a/cutile-rs/cutile-benchmarks/paper/sec3_example1_race_detection/permute_race_rust.rs b/cutile-rs/cutile-benchmarks/paper/sec3_example1_race_detection/permute_race_rust.rs index 339f631d3b..aa2ef635b0 100644 --- a/cutile-rs/cutile-benchmarks/paper/sec3_example1_race_detection/permute_race_rust.rs +++ b/cutile-rs/cutile-benchmarks/paper/sec3_example1_race_detection/permute_race_rust.rs @@ -24,7 +24,7 @@ * cargo run -p cutile-examples --example permute_race_rust */ -use cuda_core::Device; +use cuda_core::Context; use cutile::error::Error; use cutile::prelude::*; use std::sync::Arc; @@ -57,7 +57,7 @@ mod kernels { } fn main() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let b = 2usize; diff --git a/cutile-rs/cutile-benchmarks/paper/sec5_exp1_safety_overhead/diagnostics/side_studies/token_ordering_rust.rs b/cutile-rs/cutile-benchmarks/paper/sec5_exp1_safety_overhead/diagnostics/side_studies/token_ordering_rust.rs index 4fb5947737..2f07ad2dbd 100644 --- a/cutile-rs/cutile-benchmarks/paper/sec5_exp1_safety_overhead/diagnostics/side_studies/token_ordering_rust.rs +++ b/cutile-rs/cutile-benchmarks/paper/sec5_exp1_safety_overhead/diagnostics/side_studies/token_ordering_rust.rs @@ -16,7 +16,7 @@ * Look for print_ir output on stderr. */ -use cuda_core::Device; +use cuda_core::Context; use cutile::error::Error; use cutile::prelude::*; use std::sync::Arc; @@ -72,7 +72,7 @@ mod kernels { } fn main() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let n = 1024usize; diff --git a/cutile-rs/cutile-benchmarks/paper/sec5_exp1_safety_overhead/rust/elemwise/src/main.rs b/cutile-rs/cutile-benchmarks/paper/sec5_exp1_safety_overhead/rust/elemwise/src/main.rs index f0130c6235..86c29d54c3 100644 --- a/cutile-rs/cutile-benchmarks/paper/sec5_exp1_safety_overhead/rust/elemwise/src/main.rs +++ b/cutile-rs/cutile-benchmarks/paper/sec5_exp1_safety_overhead/rust/elemwise/src/main.rs @@ -43,7 +43,7 @@ // Lock clocks for paper-final runs: `sudo nvidia-smi -lgc 2400,2400`. use cuda_async::device_operation::DeviceOp; -use cuda_core::{sys, Device, IntoResult, Stream}; +use cuda_core::{sys, Context, IntoResult, Stream}; use cutile::api; use cutile::core::f16; use cutile::tile_kernel::{CompileOptions, PartitionOp, TileKernel}; @@ -352,10 +352,10 @@ fn output_path(file_name: &str) -> std::path::PathBuf { fn sync_stream(stream: &Arc, context: &str) { stream - .device() + .context() .bind_to_thread() .expect("bind stream device"); - unsafe { stream.synchronize() }.expect(context); + stream.synchronize().expect(context); } struct TimingEvent(sys::CUevent); @@ -363,7 +363,7 @@ struct TimingEvent(sys::CUevent); impl TimingEvent { fn new(stream: &Arc) -> Self { stream - .device() + .context() .bind_to_thread() .expect("bind stream device"); let mut event = std::mem::MaybeUninit::uninit(); @@ -380,7 +380,7 @@ impl TimingEvent { fn record(&self, stream: &Arc) { stream - .device() + .context() .bind_to_thread() .expect("bind stream device"); unsafe { sys::cuEventRecord(self.0, stream.cu_stream()) } @@ -390,7 +390,7 @@ impl TimingEvent { fn synchronize(&self, stream: &Arc) { stream - .device() + .context() .bind_to_thread() .expect("bind stream device"); unsafe { sys::cuEventSynchronize(self.0) } @@ -400,7 +400,7 @@ impl TimingEvent { fn elapsed_us(&self, end: &Self, stream: &Arc) -> f64 { stream - .device() + .context() .bind_to_thread() .expect("bind stream device"); let mut ms: f32 = 0.0; @@ -785,7 +785,7 @@ fn main() -> Result<(), Box> { std::process::exit(2); } - let device = Device::new(0).expect("device"); + let device = Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); // Sweep N from 2^20 (1M, ~2 MB per f16 tensor) to 2^28 (256M, diff --git a/cutile-rs/cutile-benchmarks/paper/sec5_exp1_safety_overhead/rust/gemm/src/main.rs b/cutile-rs/cutile-benchmarks/paper/sec5_exp1_safety_overhead/rust/gemm/src/main.rs index 6b8ef27f02..ada3103bae 100644 --- a/cutile-rs/cutile-benchmarks/paper/sec5_exp1_safety_overhead/rust/gemm/src/main.rs +++ b/cutile-rs/cutile-benchmarks/paper/sec5_exp1_safety_overhead/rust/gemm/src/main.rs @@ -17,7 +17,7 @@ // to a stable reference independent of upstream changes. use cuda_async::device_operation::DeviceOp; -use cuda_core::{Device, IntoResult, Stream}; +use cuda_core::{Context, IntoResult, Stream}; use cutile::api; use cutile::core::f16; use cutile::prelude::*; @@ -756,7 +756,7 @@ fn option_label(value: Option) -> String { .unwrap_or_else(|| "default".to_string()) } -fn device_sm_count(device: &Arc) -> Result { +fn device_sm_count(device: &Arc) -> Result { let mut result = 0i32; unsafe { cuda_core::sys::cuDeviceGetAttribute( @@ -769,7 +769,7 @@ fn device_sm_count(device: &Arc) -> Result Ok(result) } -fn device_sm_arch(device: &Arc) -> Result { +fn device_sm_arch(device: &Arc) -> Result { let mut major = 0i32; let mut minor = 0i32; unsafe { @@ -968,10 +968,10 @@ fn output_path(file_name: &str) -> std::path::PathBuf { fn sync_stream(stream: &Arc, context: &str) { stream - .device() + .context() .bind_to_thread() .expect("bind stream device"); - unsafe { stream.synchronize() }.expect(context); + stream.synchronize().expect(context); } fn check_gemm_output( @@ -1684,7 +1684,7 @@ fn main() -> Result<(), Box> { "optimized" }; - let device = Device::new(0).expect("device"); + let device = Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); let num_sms = device_sm_count(&device).unwrap_or(148); let sm_arch = device_sm_arch(&device).unwrap_or(0); diff --git a/cutile-rs/cutile-benchmarks/paper/sec5_exp2_execution_mode_overhead/launch_overhead_rust/src/bin/async_throughput.rs b/cutile-rs/cutile-benchmarks/paper/sec5_exp2_execution_mode_overhead/launch_overhead_rust/src/bin/async_throughput.rs index 8cd450b644..6f5af3bab1 100644 --- a/cutile-rs/cutile-benchmarks/paper/sec5_exp2_execution_mode_overhead/launch_overhead_rust/src/bin/async_throughput.rs +++ b/cutile-rs/cutile-benchmarks/paper/sec5_exp2_execution_mode_overhead/launch_overhead_rust/src/bin/async_throughput.rs @@ -37,7 +37,7 @@ use cuda_async::cuda_graph::CudaGraph; use cuda_async::device_future::DeviceFuture; use cuda_async::device_operation::{BoxedDeviceOp, DeviceOp, ExecutionContext}; -use cuda_core::{Device, Stream}; +use cuda_core::{Context, Stream}; use cutile::api; use cutile::core::f16; use cutile::tensor::{IntoPartition, Partition, Tensor}; @@ -200,11 +200,11 @@ fn bench_sync( graph.launch().sync_on(stream).expect("graph replay"); spin_us(w_us); } - unsafe { stream.synchronize() }.expect("sync"); + stream.synchronize().expect("sync"); let mut throughput = Vec::with_capacity(n_samples); for _ in 0..n_samples { - unsafe { stream.synchronize() }.expect("sync"); + stream.synchronize().expect("sync"); let t0 = Instant::now(); for _ in 0..iters { graph.launch().sync_on(stream).expect("graph replay"); @@ -252,11 +252,11 @@ fn bench_async( for _ in 0..n_warmup { one_iter!(); } - unsafe { stream_clone.synchronize() }.expect("sync"); + stream_clone.synchronize().expect("sync"); let mut throughput = Vec::with_capacity(n_samples); for _ in 0..n_samples { - unsafe { stream_clone.synchronize() }.expect("sync"); + stream_clone.synchronize().expect("sync"); let t0 = Instant::now(); for _ in 0..iters { one_iter!(); @@ -308,7 +308,7 @@ fn main() -> Result<(), Box> { .unwrap_or("results_part_b.csv") .to_string(); - let device = Device::new(0).expect("device"); + let device = Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); let d = D as usize; diff --git a/cutile-rs/cutile-benchmarks/paper/sec5_exp2_execution_mode_overhead/launch_overhead_rust/src/main.rs b/cutile-rs/cutile-benchmarks/paper/sec5_exp2_execution_mode_overhead/launch_overhead_rust/src/main.rs index 818d2b3660..a8122b2254 100644 --- a/cutile-rs/cutile-benchmarks/paper/sec5_exp2_execution_mode_overhead/launch_overhead_rust/src/main.rs +++ b/cutile-rs/cutile-benchmarks/paper/sec5_exp2_execution_mode_overhead/launch_overhead_rust/src/main.rs @@ -32,7 +32,7 @@ use cuda_async::cuda_graph::CudaGraph; use cuda_async::device_operation::{BoxedDeviceOp, DeviceOp, Unzippable3}; -use cuda_core::{Device, Stream}; +use cuda_core::{Context, Stream}; use cutile::api; use cutile::core::f16; use cutile::tensor::{IntoPartition, Partition, Tensor}; @@ -173,11 +173,11 @@ fn bench_sync_individual( for _ in 0..n_warmup { part = run(part); } - unsafe { stream.synchronize() }.expect("sync"); + stream.synchronize().expect("sync"); let mut samples = Vec::with_capacity(n_samples); for _ in 0..n_samples { - unsafe { stream.synchronize() }.expect("sync"); + stream.synchronize().expect("sync"); let t0 = Instant::now(); for _ in 0..iters { part = run(part); @@ -207,7 +207,7 @@ fn bench_sync_chained( let chain = build_chain(part, n, &x, &g, generics); part = chain.sync().expect("sync chain"); } - unsafe { stream.synchronize() }.expect("sync"); + stream.synchronize().expect("sync"); // Time chain *execution* only, not construction. build_chain boxes one op // per launch; that boxing is an artifact of assembling the chain @@ -216,7 +216,7 @@ fn bench_sync_chained( // the per-launch cost remains in the measured region. let mut samples = Vec::with_capacity(n_samples); for _ in 0..n_samples { - unsafe { stream.synchronize() }.expect("sync"); + stream.synchronize().expect("sync"); let mut elapsed = std::time::Duration::ZERO; for _ in 0..iters { let chain = build_chain(part, n, &x, &g, generics); @@ -285,7 +285,7 @@ fn bench_async( let chain = build_chain(part, n, &x, &g, generics); part = chain.await.expect("chain"); } - unsafe { stream.synchronize() }.expect("sync"); + stream.synchronize().expect("sync"); // Time chain execution (.await) only; build_chain is built outside the // timer for the same reason as the sync-chained mode (its per-launch @@ -293,7 +293,7 @@ fn bench_async( // path), while per-launch cost stays in the execute walk. let mut samples = Vec::with_capacity(n_samples); for _ in 0..n_samples { - unsafe { stream.synchronize() }.expect("sync"); + stream.synchronize().expect("sync"); let mut elapsed = std::time::Duration::ZERO; for _ in 0..iters { let chain = build_chain(part, n, &x, &g, generics); @@ -328,11 +328,11 @@ fn bench_graph( for _ in 0..n_warmup { graph.launch().sync_on(stream).expect("graph warmup"); } - unsafe { stream.synchronize() }.expect("sync"); + stream.synchronize().expect("sync"); let mut samples = Vec::with_capacity(n_samples); for _ in 0..n_samples { - unsafe { stream.synchronize() }.expect("sync"); + stream.synchronize().expect("sync"); let t0 = Instant::now(); for _ in 0..iters { graph.launch().sync_on(stream).expect("graph replay"); @@ -371,7 +371,7 @@ fn main() -> Result<(), Box> { .and_then(|s| s.parse().ok()) .unwrap_or(DEFAULT_N_SAMPLES); - let device = Device::new(0).expect("device"); + let device = Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); let d = D as usize; diff --git a/cutile-rs/cutile-book/guide/host-vs-device.md b/cutile-rs/cutile-book/guide/host-vs-device.md index 0cc781d7d3..44048d6d1b 100644 --- a/cutile-rs/cutile-book/guide/host-vs-device.md +++ b/cutile-rs/cutile-book/guide/host-vs-device.md @@ -7,7 +7,7 @@ use cutile::prelude::*; use kernels::scale; fn main() -> Result<(), cuda_async::error::DeviceError> { - let device = cuda_core::Device::new(0)?; + let device = cuda_core::Context::new(0)?; let stream = device.new_stream()?; let x = api::ones::(&[32, 32]).sync_on(&stream)?; diff --git a/cutile-rs/cutile-book/guide/interoperability.md b/cutile-rs/cutile-book/guide/interoperability.md index c804768401..1d1964029c 100644 --- a/cutile-rs/cutile-book/guide/interoperability.md +++ b/cutile-rs/cutile-book/guide/interoperability.md @@ -77,7 +77,7 @@ launcher.set_launch_config(LaunchConfig { launcher.await?; ``` -Scalar arguments implementing `DType` use the safe `push_arg` method. Device +Scalar arguments implementing `DType` use the safe `push_arg` method. Context pointers require `unsafe { push_device_ptr() }`. The caller must check that allocations are valid on the target GPU, large enough for the kernel, and free of conflicting accesses. Argument order and types must match the kernel's @@ -224,13 +224,13 @@ requirements. ## Borrowing Foreign CUDA Handles -If your application already owns CUDA handles through another Rust binding crate — cudarc, Candle, or a hand-rolled `bindgen` wrapper — wrap them into a cuTile `Device`, `Stream`, `Module`, or `Function` without transferring ownership. +If your application already owns CUDA handles through another Rust binding crate — cudarc, Candle, or a hand-rolled `bindgen` wrapper — wrap them into a cuTile `Context`, `Stream`, `Module`, or `Function` without transferring ownership. The `borrow_raw` constructors take raw C primitives (`*mut c_void` for opaque handles, `c_int` for `CUdevice`) rather than `cuda_bindings` typedefs, so no nominal-type mismatch between binding crates gets in the way: ```rust use core::ffi::{c_int, c_void}; -use cuda_core::{Device, Stream}; +use cuda_core::{Context, Stream}; // `foreign` is a handle bundle from another Rust binding crate (cudarc, // Candle, a bindgen wrapper, …). Its `CUcontext` / `CUstream` typedefs are @@ -240,7 +240,7 @@ use cuda_core::{Device, Stream}; // Safety: the caller guarantees the handles are valid, outlive the returned // wrappers, and are not concurrently destroyed. let device = unsafe { - Device::borrow_raw( + Context::borrow_raw( foreign.cu_ctx as *mut c_void, foreign.cu_device as c_int, foreign.ordinal, @@ -252,7 +252,7 @@ let stream = unsafe { Stream::borrow_raw(foreign.cu_stream as *mut c_void, &devi let result = my_kernel(out, x).sync_on(&stream)?; ``` -Borrowed wrappers skip destruction on drop: `Stream` does not call `cuStreamDestroy`, `Module` does not unload, and `Device` does not release the primary context. The source framework retains full control over handle lifetimes. +Borrowed wrappers skip destruction on drop: `Stream` does not call `cuStreamDestroy`, `Module` does not unload, and `Context` does not release the primary context. The source framework retains full control over handle lifetimes. `Module::borrow_raw` and `Function::borrow_raw` follow the same pattern for pre-compiled modules and already-resolved functions. See the `cudarc_interop` example in `cutile-examples/examples/` for an end-to-end walkthrough. @@ -291,4 +291,4 @@ Both front-ends use the same underlying Tile IR compilation pipeline and generat --- -Continue to [Debugging and Profiling](debugging-and-profiling.md) for troubleshooting. Interop kernels use the `DeviceOp` model described in [Device Operations](device-operations.md). +Continue to [Debugging and Profiling](debugging-and-profiling.md) for troubleshooting. Interop kernels use the `DeviceOp` model described in [Context Operations](device-operations.md). diff --git a/cutile-rs/cutile-book/guide/introduction.md b/cutile-rs/cutile-book/guide/introduction.md index 0850f82ea2..6b4e59bb7f 100644 --- a/cutile-rs/cutile-book/guide/introduction.md +++ b/cutile-rs/cutile-book/guide/introduction.md @@ -31,7 +31,7 @@ mod my_module { } fn main() -> Result<(), cuda_async::error::DeviceError> { - let device = cuda_core::Device::new(0)?; + let device = cuda_core::Context::new(0)?; let stream = device.new_stream()?; let x = api::ones::(&[32, 32]).sync_on(&stream)?; diff --git a/cutile-rs/cutile-book/reference/host-api.md b/cutile-rs/cutile-book/reference/host-api.md index 7b4cfd0d62..c8aa369ee9 100644 --- a/cutile-rs/cutile-book/reference/host-api.md +++ b/cutile-rs/cutile-book/reference/host-api.md @@ -2,9 +2,9 @@ The host API covers tensors, CUDA contexts and streams, kernel launches, and `DeviceOp` execution. For an introduction, see -[Host vs. Device Code](../guide/host-vs-device.md), +[Host vs. Context Code](../guide/host-vs-device.md), [Tensors and Tiles](../guide/tensors-and-tiles.md), and -[Device Operations](../guide/device-operations.md). +[Context Operations](../guide/device-operations.md). --- @@ -156,7 +156,7 @@ existing tensor without allocation or copying. --- -## Host-Device and Device-Device Transfers +## Host-Context and Context-Context Transfers Moving data between CPU and GPU, or between two device tensors, uses APIs that return `DeviceOp`s — the copy is scheduled when the op runs, not constructed: @@ -175,10 +175,10 @@ return `DeviceOp`s — the copy is scheduled when the op runs, not constructed: let data: Arc> = Arc::new(vec![1.0; 1024]); let tensor: Tensor = api::copy_host_vec_to_device(&data).sync_on(&stream)?; -// Device -> host +// Context -> host let result: Vec = tensor.to_host_vec().sync_on(&stream)?; -// Device -> device +// Context -> device let copy = tensor.dup().sync_on(&stream)?; ``` @@ -210,24 +210,24 @@ graph.launch().sync_on(&stream)?; ## Devices and Streams -Every host program starts with a `Device`, plus one or more `Stream`s for scheduling GPU work: +Every host program starts with a `Context`, plus one or more `Stream`s for scheduling GPU work: ```rust -use cuda_core::Device; +use cuda_core::Context; -let device = Device::new(0)?; // Device ordinal 0 +let device = Context::new(0)?; // Primary context for GPU 0 let stream = device.new_stream()?; // A new stream owned by this device ``` | Method | Returns | Description | |---|---|---| -| `Device::new(ordinal: usize)` | `Result, DriverError>` | Create a device handle bound to a GPU ordinal | -| `Device::device_count()` | `Result` | Number of CUDA-capable devices | +| `Context::new(ordinal: usize)` | `Result, DriverError>` | Create a device handle bound to a GPU ordinal | +| `Context::device_count()` | `Result` | Number of CUDA-capable devices | | `device.ordinal()` | `usize` | GPU ordinal this handle represents | -| `device.name()` | `Result` | Device name | +| `device.name()` | `Result` | Context name | | `device.new_stream()` | `Result, DriverError>` | Create a new stream on this device | -| `unsafe Device::borrow_raw(cu_ctx, cu_device, ordinal)` | `Arc` | Borrow an externally owned CUDA context/device for interop | -| `unsafe Device::borrow_with_owner(cu_ctx, cu_device, ordinal, owner)` | `Arc` | Same, holding an `Arc` so the foreign owner outlives the handle | +| `unsafe Context::borrow_raw(cu_ctx, cu_device, ordinal)` | `Arc` | Borrow an externally owned CUDA context/device for interop | +| `unsafe Context::borrow_with_owner(cu_ctx, cu_device, ordinal, owner)` | `Arc` | Same, holding an `Arc` so the foreign owner outlives the handle | | `unsafe Stream::borrow_raw(cu_stream, &device)` | `Arc` | Borrow an externally owned CUDA stream for interop | | `unsafe Stream::borrow_with_owner(cu_stream, &device, owner)` | `Arc` | Same, holding an `Arc` | | `unsafe Module::borrow_raw(...)` / `unsafe Function::borrow_raw(...)` | CUDA module/function wrappers | Borrow externally owned CUDA handles | @@ -712,7 +712,7 @@ error short-circuits to the caller. | Variant | When it occurs | |---|---| | `Driver(DriverError)` | CUDA driver call failed (OOM, invalid argument, etc.) | -| `Context { device_id, message }` | Device context assertion failed | +| `Context { device_id, message }` | Context context assertion failed | | `KernelCache(String)` | Declared, but no current code path constructs it | | `Scheduling(String)` | No stream available or policy misconfigured | | `Launch(String)` | A launcher-generated precondition failed (`kernel_launch_assert`: argument shapes, declared `preconditions`) | @@ -836,14 +836,14 @@ Public host-side API not covered in the sections above: | launcher `.compile()` / `.compile_on(&stream)` | JIT-compile and cache the specialization without launching | | launcher `.specialize()`, `.l1_cache_key()`, `.l2_cache_key()` | Resolve the specialization identity and the in-memory / on-disk cache keys without compiling or launching | | `api::meta::(&shape)` | Placeholder tensor carrying shape and dtype without allocating, for `.compile()` warmups | -| `CompileOptions::{opt_level, device_debug, lineinfo, sanitize_memcheck}` | Device compiler flags; each is part of the cache keys (see [Debugging and Profiling](../guide/debugging-and-profiling.md)) | -| `cutile::bench::{do_bench, do_bench_paired, BenchOptions, Measurement}` | Device-event kernel timing (see [Performance](../guide/performance.md)) | +| `CompileOptions::{opt_level, device_debug, lineinfo, sanitize_memcheck}` | Context compiler flags; each is part of the cache keys (see [Debugging and Profiling](../guide/debugging-and-profiling.md)) | +| `cutile::bench::{do_bench, do_bench_paired, BenchOptions, Measurement}` | Context-event kernel timing (see [Performance](../guide/performance.md)) | | `cuda_async::device_context::{set_default_device, global_policy, with_device_policy}` | Set the thread's default device; read or borrow a device's scheduling policy | --- ## See Also -- [Device Operations](../guide/device-operations.md) — tutorial-style guide to streams, scheduling, and composition patterns +- [Context Operations](../guide/device-operations.md) — tutorial-style guide to streams, scheduling, and composition patterns - [Tutorial 10](../tutorials/10-cuda-graphs.md) — end-to-end CUDA graph example - [Interoperability](../guide/interoperability.md) — integrating custom CUDA C++ kernels into the DeviceOp model diff --git a/cutile-rs/cutile-book/tutorials/01-hello-world.md b/cutile-rs/cutile-book/tutorials/01-hello-world.md index e7b8f34480..791a4e17e4 100644 --- a/cutile-rs/cutile-book/tutorials/01-hello-world.md +++ b/cutile-rs/cutile-book/tutorials/01-hello-world.md @@ -41,7 +41,7 @@ mod hello_world_module { use hello_world_module::hello_world_kernel; fn main() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let launcher = hello_world_kernel(); launcher.grid((2, 2, 1)).sync_on(&stream)?; @@ -90,7 +90,7 @@ The following host-side code will launch the device-side code: ```rust fn main() -> Result<(), Error> { - let device = Device::new(0)?; // Connect to GPU + let device = Context::new(0)?; // Connect to GPU let stream = device.new_stream()?; // Create a work queue let launcher = hello_world_kernel(); // Get the kernel launcher launcher.grid((2, 2, 1)).sync_on(&stream)?; // Launch 2×2×1 = 4 programs diff --git a/cutile-rs/cutile-book/tutorials/02-vector-addition.md b/cutile-rs/cutile-book/tutorials/02-vector-addition.md index c75df9db21..a8b00891f7 100644 --- a/cutile-rs/cutile-book/tutorials/02-vector-addition.md +++ b/cutile-rs/cutile-book/tutorials/02-vector-addition.md @@ -8,7 +8,7 @@ In cutile, tile programs run concurrently and each program knows its grid coordi ```rust use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use std::sync::Arc; use cutile; use cutile::api::{ones, zeros}; @@ -35,7 +35,7 @@ mod my_module { use my_module::add; fn main() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; // Create input tensors: 32×32 matrices filled with 1.0 @@ -84,14 +84,14 @@ Mutable tensors **must** be partitioned on the host side before kernel launch. T On the host side, the generated launcher expects a partition for every `&mut Tensor` parameter — `Partition>` from `tensor.partition(...)`, `Partition<&mut Tensor>` from `(&mut tensor).partition(...)`, or a mapped partition — so you always call `.partition(...)` before passing the tensor. -### Device-Side Partitioning (Available for `&Tensor`) +### Context-Side Partitioning (Available for `&Tensor`) Read-only inputs are passed as `&Tensor`, `Tensor`, `Arc>`, or `&TensorView` on the host side — no host-side partitioning required. Multiple tile blocks can safely read from the same or overlapping regions, so there is no exclusive-access constraint to enforce. Instead, read-only tensors can be partitioned **inside the kernel** using `.partition(shape![M, N])`: ```rust -// Device-side: partition a read-only tensor inside the kernel. +// Context-side: partition a read-only tensor inside the kernel. let part_x = x.partition(shape![BM, BK]); let tile = part_x.load([i, j]); ``` @@ -163,7 +163,7 @@ fn my_kernel(...) { | Concept | What It Means | |---------|---------------| | **Host-side partitioning** | `.partition([M, N])` on the host — required for `&mut Tensor` to ensure exclusive write access and determine the launch grid | -| **Device-side partitioning** | `.partition(shape![M, N])` inside the kernel — available for `&Tensor`, flexible access patterns | +| **Context-side partitioning** | `.partition(shape![M, N])` inside the kernel — available for `&Tensor`, flexible access patterns | | **Static shape `S`** | The tile shape carried in the compiled variant | | **Dynamic shape `[-1, -1]`** | Dynamic tensor shape; does not create a new compiled variant when tensor shape changes | | **load_like** | Convenience for element-wise ops: loads from the input using the same partitioning and mapping as the output (also available as the free function `load_tile_like`). The input must have the same shape as the whole output tensor — the tensor the host partitioned, not the per-program sub-tensor — and the output must be a partitioned mutable tensor | diff --git a/cutile-rs/cutile-book/tutorials/03-saxpy.md b/cutile-rs/cutile-book/tutorials/03-saxpy.md index 641c961631..5f33ed7179 100644 --- a/cutile-rs/cutile-book/tutorials/03-saxpy.md +++ b/cutile-rs/cutile-book/tutorials/03-saxpy.md @@ -23,7 +23,7 @@ The compiler implements broadcasting without allocating memory for the copies. ```rust use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use std::sync::Arc; use cutile; use cutile::api::arange; @@ -50,7 +50,7 @@ mod my_module { use my_module::saxpy; fn main() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let a = 2.0f32; // Our scalar multiplier diff --git a/cutile-rs/cutile-book/tutorials/04-matrix-multiplication.md b/cutile-rs/cutile-book/tutorials/04-matrix-multiplication.md index d3cebe398d..4cf0b613ef 100644 --- a/cutile-rs/cutile-book/tutorials/04-matrix-multiplication.md +++ b/cutile-rs/cutile-book/tutorials/04-matrix-multiplication.md @@ -60,7 +60,7 @@ BM times. This reduces global-memory traffic. ```rust use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use std::sync::Arc; use cutile; use cutile::api; @@ -98,7 +98,7 @@ mod my_module { use my_module::gemm; fn main() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let (bm, bn, bk) = (16usize, 16usize, 8usize); diff --git a/cutile-rs/cutile-book/tutorials/05-fused-softmax.md b/cutile-rs/cutile-book/tutorials/05-fused-softmax.md index a00d6b8544..bf1a8838f8 100644 --- a/cutile-rs/cutile-book/tutorials/05-fused-softmax.md +++ b/cutile-rs/cutile-book/tutorials/05-fused-softmax.md @@ -45,7 +45,7 @@ exp(x_i - max) / Σ exp(x_j - max) ```rust use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use cutile; use cutile::api::arange; use cutile::error::Error; @@ -83,7 +83,7 @@ mod my_module { use my_module::softmax; fn main() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let (m, n) = (4usize, 8usize); diff --git a/cutile-rs/cutile-book/tutorials/06-flash-attention.md b/cutile-rs/cutile-book/tutorials/06-flash-attention.md index c33546a4a6..c8b9a5a68a 100644 --- a/cutile-rs/cutile-book/tutorials/06-flash-attention.md +++ b/cutile-rs/cutile-book/tutorials/06-flash-attention.md @@ -130,7 +130,7 @@ For each Q tile (row block of the output): ```rust use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use std::sync::Arc; use cutile; use cutile::api::{randn, zeros}; @@ -227,7 +227,7 @@ mod fmha_module { use fmha_module::fmha; fn main() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let (batch, heads, seq_len, head_dim) = (2, 4, 128, 64); diff --git a/cutile-rs/cutile-book/tutorials/10-cuda-graphs.md b/cutile-rs/cutile-book/tutorials/10-cuda-graphs.md index c2e0d88159..136f42bc36 100644 --- a/cutile-rs/cutile-book/tutorials/10-cuda-graphs.md +++ b/cutile-rs/cutile-book/tutorials/10-cuda-graphs.md @@ -276,7 +276,7 @@ Each `forward` call: ```rust fn main() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let cfg = Config { d: 2048, n_layers: 22, block: 128, bn: 16, bk: 16, eps: 1e-5 }; diff --git a/cutile-rs/cutile-compiler/src/cuda_tile_runtime_utils.rs b/cutile-rs/cutile-compiler/src/cuda_tile_runtime_utils.rs index 203dbdd5c1..782ffe03c5 100644 --- a/cutile-rs/cutile-compiler/src/cuda_tile_runtime_utils.rs +++ b/cutile-rs/cutile-compiler/src/cuda_tile_runtime_utils.rs @@ -7,7 +7,7 @@ //! Provides GPU detection and bytecode compilation helpers. use crate::error::JITError; -use cuda_core::{get_device_sm_name, Device}; +use cuda_core::{get_device_sm_name, Context}; use cutile_ir::bytecode::{write_bytecode_version, BytecodeVersion}; use std::collections::HashMap; use std::env; @@ -56,7 +56,7 @@ impl ToolkitCapabilities { type Key = (String, Option, String); static CACHE: OnceLock>>> = OnceLock::new(); let tileiras = tileiras_binary(); - let device = Device::raw_device(device_id) + let device = Context::raw_device(device_id) .map_err(|e| JITError::Generic(format!("cannot query CUDA device {device_id}: {e}")))?; // SAFETY: the driver returned this device handle. let architecture = unsafe { get_device_sm_name(device) } @@ -120,7 +120,7 @@ pub fn get_gpu_name(device_id: usize) -> String { [const { OnceLock::new() }; MAX_CACHED_DEVICES]; let query = || -> String { - let dev = Device::raw_device(device_id).unwrap_or_else(|e| { + let dev = Context::raw_device(device_id).unwrap_or_else(|e| { panic!( "failed to get CUDA device {device_id}: {e}\n\ Ensure an NVIDIA GPU is visible to the process and the CUDA driver is installed." diff --git a/cutile-rs/cutile-examples/examples/autotune.rs b/cutile-rs/cutile-examples/examples/autotune.rs index 0792b5bac8..f6807226bd 100644 --- a/cutile-rs/cutile-examples/examples/autotune.rs +++ b/cutile-rs/cutile-examples/examples/autotune.rs @@ -11,7 +11,7 @@ //! ``` use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use cutile::api::{randn, zeros}; use cutile::error::Error; use cutile::tensor::{IntoPartition, Tensor}; @@ -71,7 +71,7 @@ fn main() -> Result<(), Error> { return Ok(()); } - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let (m, n) = (512usize, 1024usize); let eps: f32 = 1e-8; diff --git a/cutile-rs/cutile-examples/examples/batch_matmul.rs b/cutile-rs/cutile-examples/examples/batch_matmul.rs index 3faf0b9241..d69561f08b 100644 --- a/cutile-rs/cutile-examples/examples/batch_matmul.rs +++ b/cutile-rs/cutile-examples/examples/batch_matmul.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use cutile::api::{ones, zeros}; use cutile::error::Error; use cutile::tensor::{IntoPartition, Partition, Tensor, ToHostVec}; @@ -50,7 +50,7 @@ fn main() -> Result<(), Error> { return Ok(()); } - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let batch = 4usize; diff --git a/cutile-rs/cutile-examples/examples/cuda_graphs.rs b/cutile-rs/cutile-examples/examples/cuda_graphs.rs index e775224f4a..c8ff49f8e0 100644 --- a/cutile-rs/cutile-examples/examples/cuda_graphs.rs +++ b/cutile-rs/cutile-examples/examples/cuda_graphs.rs @@ -17,7 +17,7 @@ * cargo run -p cutile-examples --example cuda_graphs */ -use cuda_core::{Device, Stream}; +use cuda_core::{Context, Stream}; use cutile::error::Error; use cutile::prelude::*; use std::time::Instant; @@ -160,7 +160,7 @@ impl GraphModel { let mut buffers: Vec<_> = (0..cfg.n_layers) .map(|_| LayerBuffers::allocate(cfg.d, stream)) .collect::>()?; - unsafe { stream.synchronize() }?; + stream.synchronize()?; // Capture the forward pass as a CUDA graph. Each s.record() // records a graph node, releasing borrows between kernels. @@ -291,7 +291,7 @@ fn main() -> Result<(), Error> { return Ok(()); } - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let cfg = Config { diff --git a/cutile-rs/cutile-examples/examples/cuda_graphs_deviceop.rs b/cutile-rs/cutile-examples/examples/cuda_graphs_deviceop.rs index 799112820b..7aa00f609e 100644 --- a/cutile-rs/cutile-examples/examples/cuda_graphs_deviceop.rs +++ b/cutile-rs/cutile-examples/examples/cuda_graphs_deviceop.rs @@ -14,7 +14,7 @@ * cargo run -p cutile-examples --example cuda_graphs_deviceop */ -use cuda_core::{Device, Stream}; +use cuda_core::{Context, Stream}; use cutile::error::Error; use cutile::prelude::*; use std::time::Instant; @@ -167,7 +167,7 @@ impl GraphModel { let buffers: Vec<_> = (0..cfg.n_layers) .map(|_| LayerBuffers::allocate(cfg.d, stream)) .collect::>()?; - unsafe { stream.synchronize() }?; + stream.synchronize()?; // Create an Arc that shares input's device memory for build_forward. // This requires aliased storage (input + input_arc point to same @@ -351,7 +351,7 @@ fn main() -> Result<(), Error> { return Ok(()); } - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let cfg = Config { diff --git a/cutile-rs/cutile-examples/examples/cudarc_interop.rs b/cutile-rs/cutile-examples/examples/cudarc_interop.rs index f3a57a20ee..6f7e275df9 100644 --- a/cutile-rs/cutile-examples/examples/cudarc_interop.rs +++ b/cutile-rs/cutile-examples/examples/cudarc_interop.rs @@ -20,7 +20,7 @@ use cuda_async::device_buffer::DeviceAllocation; use cuda_core::sys::CUdeviceptr; -use cuda_core::{free_async, malloc_async, memcpy_dtoh_async, memcpy_htod_async, Device, Stream}; +use cuda_core::{free_async, malloc_async, memcpy_dtoh_async, memcpy_htod_async, Context, Stream}; use cutile::error::Error; use cutile::prelude::*; use std::sync::Arc; @@ -43,7 +43,7 @@ mod foreign { Arc::new(Self { dptr, len_bytes, - device_id: stream.device().ordinal(), + device_id: stream.context().ordinal(), stream: stream.clone(), }) } @@ -61,6 +61,9 @@ mod foreign { // SAFETY: `dptr` is a live `len_bytes` device allocation on `device_id` for // this buffer's whole lifetime (it is freed only in `Drop`). unsafe impl DeviceAllocation for Buffer { + fn context(&self) -> std::sync::Arc { + self.stream.context().clone() + } fn device_ptr(&self) -> CUdeviceptr { self.dptr } @@ -97,7 +100,7 @@ fn main() -> Result<(), Error> { const N: usize = 1024; const TILE: usize = 128; - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; // The framework allocates and owns three buffers, and fills the inputs. diff --git a/cutile-rs/cutile-examples/examples/dropout.rs b/cutile-rs/cutile-examples/examples/dropout.rs index a9ebe4ba29..1210f549a6 100644 --- a/cutile-rs/cutile-examples/examples/dropout.rs +++ b/cutile-rs/cutile-examples/examples/dropout.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use cutile::api::{rand, randn, zeros}; use cutile::error::Error; use cutile::tensor::{IntoPartition, Partition, Tensor, ToHostVec}; @@ -40,7 +40,7 @@ fn main() -> Result<(), Error> { return Ok(()); } - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let (m,) = (16,); let bm = 4; diff --git a/cutile-rs/cutile-examples/examples/flash_attention.rs b/cutile-rs/cutile-examples/examples/flash_attention.rs index 89174f048d..7d104833ef 100644 --- a/cutile-rs/cutile-examples/examples/flash_attention.rs +++ b/cutile-rs/cutile-examples/examples/flash_attention.rs @@ -5,7 +5,7 @@ extern crate core; use cuda_async::device_operation::{DeviceOp, Unzippable6}; -use cuda_core::Device; +use cuda_core::Context; use cutile::api::{randn, zeros}; use cutile::error::Error; use cutile::tensor::{IntoPartition, Partition, Tensor, ToHostVec, Unpartition}; @@ -149,8 +149,8 @@ fn fmha( bn: usize, // k, v seq len part size. bbh: usize, // batch * num_heads part size. ) -> Result<(), Error> { - // Create a context. Device 0 is associated with the context. - let device = Device::new(0)?; + // Create the primary context for GPU 0. + let device = Context::new(0)?; // Create a new stream on which we run CUDA operations. let stream = device.new_stream()?; diff --git a/cutile-rs/cutile-examples/examples/flash_attention_causal.rs b/cutile-rs/cutile-examples/examples/flash_attention_causal.rs index 6a532aa284..c6c583bec9 100644 --- a/cutile-rs/cutile-examples/examples/flash_attention_causal.rs +++ b/cutile-rs/cutile-examples/examples/flash_attention_causal.rs @@ -5,7 +5,7 @@ extern crate core; use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use cutile::api::{randn, zeros}; use cutile::error::Error; use cutile::tensor::{IntoPartition, Partition, Tensor, ToHostVec}; @@ -206,7 +206,7 @@ fn fmha_ref_cpu( } fn run_attention_fmha(causal: bool) -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let (batch, heads, heads_kv, seq_len, head_dim) = (2usize, 8usize, 8usize, 64usize, 32usize); diff --git a/cutile-rs/cutile-examples/examples/gemm.rs b/cutile-rs/cutile-examples/examples/gemm.rs index ccd6c595b1..861ce2ff59 100644 --- a/cutile-rs/cutile-examples/examples/gemm.rs +++ b/cutile-rs/cutile-examples/examples/gemm.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ use cuda_async::device_operation::*; -use cuda_core::Device; +use cuda_core::Context; use cutile::api; use cutile::error::Error; use cutile::tensor::*; @@ -37,7 +37,7 @@ mod my_module { } fn gemm() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let scale = 2usize.pow(10); // On the order of megabytes. let (bm, bn, bk) = (16, 16, 8); diff --git a/cutile-rs/cutile-examples/examples/gemm_static.rs b/cutile-rs/cutile-examples/examples/gemm_static.rs index ae112beeac..e93af84123 100644 --- a/cutile-rs/cutile-examples/examples/gemm_static.rs +++ b/cutile-rs/cutile-examples/examples/gemm_static.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ use cuda_async::device_operation::*; -use cuda_core::Device; +use cuda_core::Context; use cutile::api; use cutile::error::Error; use cutile::tensor::*; @@ -47,7 +47,7 @@ mod my_module { } fn gemm() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let scale = 2usize.pow(10); // On the order of megabytes. let (bm, bn, bk) = (16, 16, 8); diff --git a/cutile-rs/cutile-examples/examples/hello_world.rs b/cutile-rs/cutile-examples/examples/hello_world.rs index a28314d02a..0f7862ec90 100644 --- a/cutile-rs/cutile-examples/examples/hello_world.rs +++ b/cutile-rs/cutile-examples/examples/hello_world.rs @@ -5,7 +5,7 @@ #![allow(unused_variables)] use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use cutile::error::Error; use cutile::tile_kernel::TileKernel; @@ -41,7 +41,7 @@ fn main() -> Result<(), Error> { return Ok(()); } - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let launcher = hello_world_kernel(); launcher.grid((1, 1, 1)).sync_on(&stream)?; diff --git a/cutile-rs/cutile-examples/examples/mxfp8.rs b/cutile-rs/cutile-examples/examples/mxfp8.rs index 06df19c6d2..b1ceff59bb 100644 --- a/cutile-rs/cutile-examples/examples/mxfp8.rs +++ b/cutile-rs/cutile-examples/examples/mxfp8.rs @@ -75,7 +75,7 @@ fn main() -> Result<(), Error> { } fn run() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let gpu_name = get_gpu_name(0); println!("Target GPU: {gpu_name}"); diff --git a/cutile-rs/cutile-examples/examples/nvfp4.rs b/cutile-rs/cutile-examples/examples/nvfp4.rs index 2c79c5f65e..084b275fc7 100644 --- a/cutile-rs/cutile-examples/examples/nvfp4.rs +++ b/cutile-rs/cutile-examples/examples/nvfp4.rs @@ -84,7 +84,7 @@ fn main() -> Result<(), Error> { } fn run() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let gpu_name = get_gpu_name(0); println!("Target GPU: {gpu_name}"); diff --git a/cutile-rs/cutile-examples/examples/nvfp4_moe.rs b/cutile-rs/cutile-examples/examples/nvfp4_moe.rs index a739d32cfe..d1820c3f58 100644 --- a/cutile-rs/cutile-examples/examples/nvfp4_moe.rs +++ b/cutile-rs/cutile-examples/examples/nvfp4_moe.rs @@ -23,7 +23,7 @@ use cuda_async::cuda_graph::CudaGraph; use cuda_async::device_operation::{DeviceOp, GraphNode}; -use cuda_core::{Device, Stream}; +use cuda_core::{Context, Stream}; use cutile::api; use cutile::cuda_core::{f4e2m1fnx2, f8e4m3fn}; use cutile::half::{bf16, f16}; @@ -1816,8 +1816,8 @@ fn time_graph( graph: &CudaGraph<()>, stream: &Arc, ) -> Result> { - let start = stream.device().new_event()?; - let stop = stream.device().new_event()?; + let start = stream.context().new_event()?; + let stop = stream.context().new_event()?; let timing = (|| -> Result> { // SAFETY: the graph retains its buffers and remains alive throughout // these same-stream launches. The stream is drained below on success @@ -1848,7 +1848,7 @@ fn main() -> Result<(), Box> { return Ok(()); } - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; println!("NVFP4 MoE batch-size benchmark"); println!("H={HIDDEN}, I={INTERMEDIATE}, experts={NUM_EXPERTS}, top-k={TOP_K}, tile_n={TILE_N}"); diff --git a/cutile-rs/cutile-examples/examples/pdl.rs b/cutile-rs/cutile-examples/examples/pdl.rs index 7899d8389e..d3de4fd69e 100644 --- a/cutile-rs/cutile-examples/examples/pdl.rs +++ b/cutile-rs/cutile-examples/examples/pdl.rs @@ -19,7 +19,7 @@ * cargo run -p cutile-examples --example pdl */ -use cuda_core::Device; +use cuda_core::Context; use cutile::error::Error; use cutile::prelude::*; @@ -58,7 +58,7 @@ fn main() -> Result<(), Error> { return Ok(()); } - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let staging = api::zeros::(&[N]).sync_on(&stream)?; let output = api::zeros::(&[N]).sync_on(&stream)?; diff --git a/cutile-rs/cutile-examples/examples/persistent_gemm.rs b/cutile-rs/cutile-examples/examples/persistent_gemm.rs index 50e27a4cea..3a94240003 100644 --- a/cutile-rs/cutile-examples/examples/persistent_gemm.rs +++ b/cutile-rs/cutile-examples/examples/persistent_gemm.rs @@ -71,7 +71,7 @@ fn main() { } fn run() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; const BM: usize = 16; diff --git a/cutile-rs/cutile-examples/examples/rms_norm.rs b/cutile-rs/cutile-examples/examples/rms_norm.rs index 770f529ce1..a99c53af7d 100644 --- a/cutile-rs/cutile-examples/examples/rms_norm.rs +++ b/cutile-rs/cutile-examples/examples/rms_norm.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ use cuda_async::device_operation::DeviceOp; -use cuda_core::Device; +use cuda_core::Context; use cutile::api::{randn, zeros}; use cutile::error::Error; use cutile::tensor::{IntoPartition, Partition, Tensor, ToHostVec}; @@ -67,8 +67,8 @@ fn main() -> Result<(), Error> { return Ok(()); } - // Create a context. Device 0 is associated with the context. - let device = Device::new(0)?; + // Create the primary context for GPU 0. + let device = Context::new(0)?; // Create a new stream on which we run CUDA operations. let stream = device.new_stream()?; let (m, n) = (4, 8); diff --git a/cutile-rs/cutile-examples/examples/saxpy.rs b/cutile-rs/cutile-examples/examples/saxpy.rs index abac8a0e92..4ab7df784e 100644 --- a/cutile-rs/cutile-examples/examples/saxpy.rs +++ b/cutile-rs/cutile-examples/examples/saxpy.rs @@ -26,7 +26,7 @@ fn main() -> Result<(), Error> { } // Create a context. Device 0 is associated with the context. - let device = Device::new(0)?; + let device = Context::new(0)?; // Create a new stream on which we run CUDA operations. let stream = device.new_stream()?; let a = 2.0; diff --git a/cutile-rs/cutile-examples/examples/softmax.rs b/cutile-rs/cutile-examples/examples/softmax.rs index 8108adda8e..d6c387f64a 100644 --- a/cutile-rs/cutile-examples/examples/softmax.rs +++ b/cutile-rs/cutile-examples/examples/softmax.rs @@ -33,7 +33,7 @@ fn main() -> Result<(), Error> { } // Create a context. Device 0 is associated with the context. - let device = Device::new(0)?; + let device = Context::new(0)?; // Create a new stream on which we run CUDA operations. let stream = device.new_stream()?; let (m, n) = (4, 8); diff --git a/cutile-rs/cutile-examples/src/lib.rs b/cutile-rs/cutile-examples/src/lib.rs index 3ed97a27a1..8650617066 100644 --- a/cutile-rs/cutile-examples/src/lib.rs +++ b/cutile-rs/cutile-examples/src/lib.rs @@ -14,7 +14,7 @@ use candle_nn::ops::softmax; #[cfg(feature = "reference-cpu")] use cuda_core::DType; -use cuda_core::{get_device_clock_rate, Device}; +use cuda_core::{get_device_clock_rate, Context}; /// Convert a host slice to a candle CPU tensor. #[cfg(feature = "reference-cpu")] @@ -134,7 +134,7 @@ pub fn rtx_5090_tensorcore_f16_sol_tflops_at(clock_speed: f64) -> f64 { /// Returns RTX 5090 theoretical peak f16 tensor core TFLOPS using the device's clock rate. pub fn rtx_5090_tensorcore_f16_sol_tflops(device_id: usize) -> f64 { - let device = Device::new(device_id).unwrap(); + let device = Context::new(device_id).unwrap(); let clock_rate = unsafe { get_device_clock_rate(device.cu_device()).unwrap() } as f64; rtx_5090_tensorcore_f16_sol_tflops_at(clock_rate) } @@ -142,11 +142,11 @@ pub fn rtx_5090_tensorcore_f16_sol_tflops(device_id: usize) -> f64 { #[cfg(test)] mod test_peak { use crate::rtx_5090_tensorcore_f16_sol_tflops_at; - use cuda_core::{get_device_clock_rate, Device}; + use cuda_core::{get_device_clock_rate, Context}; #[test] fn test_5090() { - let device = Device::new(0).unwrap(); + let device = Context::new(0).unwrap(); // This appears to be correct / matches architecture documentation. let clock_rate = unsafe { get_device_clock_rate(device.cu_device()).unwrap() } as f64; let clock_mhz = clock_rate * 1e-3; diff --git a/cutile-rs/cutile-macro/src/_module.rs b/cutile-rs/cutile-macro/src/_module.rs index a601b6cda3..bce690e9a3 100644 --- a/cutile-rs/cutile-macro/src/_module.rs +++ b/cutile-rs/cutile-macro/src/_module.rs @@ -403,7 +403,7 @@ fn module_inner( use #tile_rust_crate_root::tile_kernel::{*}; use #tile_rust_crate_root::cuda_async::error::{*}; use #tile_rust_crate_root::cuda_async::scheduling_policies::SchedulingPolicy; - use #tile_rust_crate_root::cuda_core::{Device, Function, Module, Stream, DriverError, LaunchConfig}; + use #tile_rust_crate_root::cuda_core::{Context, Function, Module, Stream, DriverError, LaunchConfig}; // use #tile_rust_crate_root::cutile_compiler::cuda_tile::ModuleOperation; // use #tile_rust_crate_root::cutile_compiler::compiler::{CUDATileModules, CUDATileFunctionCompiler}; // Module asts and generated type data. diff --git a/cutile-rs/cutile/src/api.rs b/cutile-rs/cutile/src/api.rs index 8f01a7845c..5d03d777d5 100644 --- a/cutile-rs/cutile/src/api.rs +++ b/cutile-rs/cutile/src/api.rs @@ -180,7 +180,7 @@ impl DeviceOp for CopyDeviceToDevice { let tensor = Tensor::from_raw_parts( dst, num_bytes, - ctx.get_device_id(), + ctx.get_cuda_stream().clone(), self.shape, self.strides, ); @@ -349,7 +349,7 @@ impl DeviceOp for CopyDeviceToHostVec { }; // `execute` exposes a Vec to host-side combinators immediately. // Prove initialization here, independent of pageable-copy behavior. - if let Err(error) = unsafe { ctx.get_cuda_stream().synchronize() } { + if let Err(error) = ctx.get_cuda_stream().synchronize() { std::mem::forget(host); return Err(error.into()); } @@ -417,7 +417,7 @@ impl DeviceOp for CopyHostVecToDevice { let tensor = Tensor::from_raw_parts( dptr, element_size * num_elements, - ctx.get_device_id(), + ctx.get_cuda_stream().clone(), shape.clone(), strides.clone(), ); @@ -740,7 +740,7 @@ impl DeviceOp for AllocUninitialized { Tensor::from_raw_parts( ptr, num_bytes, - ctx.get_device_id(), + ctx.get_cuda_stream().clone(), vec![self.len as i32], vec![1], ) diff --git a/cutile-rs/cutile/src/bench.rs b/cutile-rs/cutile/src/bench.rs index f6f62781c3..26b1e83b2a 100644 --- a/cutile-rs/cutile/src/bench.rs +++ b/cutile-rs/cutile/src/bench.rs @@ -131,7 +131,7 @@ impl L2Clear { /// Twice the L2 size flushes reliably; clamp keeps pathological /// attribute readings from allocating absurd buffers. fn new(stream: &Arc) -> Result { - let l2 = stream.device().l2_cache_size_bytes().unwrap_or(0); + let l2 = stream.context().l2_cache_size_bytes().unwrap_or(0); let num_bytes = (l2 * 2).clamp(64 << 20, 512 << 20); // Safety: freed in Drop on the same stream; never exposed. let dptr = unsafe { cuda_core::malloc_async(num_bytes, stream) }?; @@ -165,7 +165,7 @@ fn time_one(stream: &Arc, f: &mut F) -> Result where F: FnMut(&Arc) -> Result<(), Error>, { - let device = stream.device(); + let device = stream.context(); let start = device.new_event()?; let end = device.new_event()?; start.record(stream)?; diff --git a/cutile-rs/cutile/src/prelude.rs b/cutile-rs/cutile/src/prelude.rs index bb5c1f3700..ed74a3b408 100644 --- a/cutile-rs/cutile/src/prelude.rs +++ b/cutile-rs/cutile/src/prelude.rs @@ -29,7 +29,7 @@ pub use crate::tensor::{ pub use crate::tile_kernel::{PartitionOp, TileKernel, ToHostVecOp}; // Common types from cuda-core -pub use cuda_core::{DType, Device}; +pub use cuda_core::{Context, DType}; // Common dependencies pub use std::sync::Arc; diff --git a/cutile-rs/cutile/src/tensor.rs b/cutile-rs/cutile/src/tensor.rs index 5060c1f1cf..ec629736ae 100644 --- a/cutile-rs/cutile/src/tensor.rs +++ b/cutile-rs/cutile/src/tensor.rs @@ -576,6 +576,10 @@ impl StorageHandle { fn is_unique(&self) -> bool { Arc::strong_count(&self.owners) == 1 + && match &self.allocation { + Allocation::Device(buffer) => buffer.is_unique(), + Allocation::Meta { .. } => true, + } } } @@ -590,6 +594,13 @@ impl Storage { "tensor and execution stream are on different devices".into(), )); } + if let Allocation::Device(buffer) = &self.allocation { + if buffer.context().as_ref() != ctx.context().as_ref() { + return Err(DeviceError::Internal( + "tensor and execution stream belong to different CUDA contexts".into(), + )); + } + } let id = self.register_access( ctx.get_cuda_stream().cu_stream() as usize, write, @@ -644,8 +655,21 @@ impl Storage { "tensor has a conflicting in-flight access on another stream; await its operation before reusing it".into(), )); } + let allocation_lease = match &self.allocation { + Allocation::Device(buffer) => Some( + buffer + .acquire_on_stream(stream, write, recording) + .map_err(|error| { + DeviceError::Internal(format!( + "allocation has a conflicting in-flight access: {error}" + )) + })?, + ), + Allocation::Meta { .. } => None, + }; let id = NEXT_ACCESS_ID.fetch_add(1, AtomicOrdering::Relaxed); accesses.push(StorageAccess { + _allocation_lease: allocation_lease, id, stream, write, @@ -710,6 +734,7 @@ static NEXT_ACCESS_ID: AtomicU64 = AtomicU64::new(1); #[derive(Debug)] struct StorageAccess { + _allocation_lease: Option, id: u64, stream: usize, write: bool, @@ -1001,18 +1026,92 @@ impl Tensor { } } + /// Consumes a typed host-runtime buffer as a contiguous tensor without copying. + /// Shape must describe exactly the buffer's element count. + pub fn from_buffer(buffer: cuda_core::DeviceBuffer, shape: Vec) -> Self + where + T: cuda_core::DeviceCopy, + { + let len = buffer.len(); + let strides = contiguous_strides(&shape); + let tensor = Self::from_device_buffer(buffer.into_storage(), shape, strides); + assert_eq!( + tensor.size(), + len, + "tensor shape must match the buffer's element count" + ); + tensor + } + + /// Returns a typed buffer view of contiguous tensor storage without copying. + /// Requires unique tensor ownership; otherwise an element cast could mutate + /// bytes still exposed through an alias with a different element type. + pub fn into_buffer(self) -> cuda_core::DeviceBuffer + where + T: cuda_core::DeviceCopy, + { + assert!( + self.storage.is_unique(), + "cannot transfer aliased tensor storage to a typed buffer" + ); + assert!( + self.is_contiguous(), + "a strided tensor cannot become a contiguous buffer" + ); + let Allocation::Device(buffer) = &self.storage.allocation else { + panic!("metadata-only tensors have no device buffer"); + }; + // SAFETY: this tensor holds valid T representations and the same shared + // owner/access tracker protects every outstanding alias and submission. + unsafe { + cuda_core::DeviceBuffer::from_storage_copy( + buffer + .slice_bytes(0, self.num_bytes()) + .expect("tensor extent"), + self.size(), + ) + } + .expect("tensor storage must match its typed extent") + } + + /// Borrows contiguous tensor storage for a typed SIMT kernel argument. + /// The view retains the same allocation and access tracker without copying. + pub fn as_buffer(&self) -> Result, Error> + where + T: cuda_core::DeviceCopy, + { + if !self.is_contiguous() { + return tensor_error_result("strided tensor cannot become a contiguous buffer view"); + } + let Allocation::Device(buffer) = &self.storage.allocation else { + return tensor_error_result("metadata-only tensors have no device buffer"); + }; + // SAFETY: tensor validity and access tracking apply to this borrowed view. + unsafe { cuda_core::DeviceBufferView::borrow_storage(buffer, 0, self.size()) } + .map_err(Into::into) + } + + /// The actual CUDA context owning this tensor's storage. + /// Panics for metadata-only tensors, which have no CUDA resource. + pub fn context(&self) -> &Arc { + match &self.storage.allocation { + Allocation::Device(buffer) => buffer.context(), + Allocation::Meta { .. } => panic!("metadata-only tensors have no CUDA context"), + } + } + /// Rebuilds a tensor from raw device allocation parts and validates the metadata /// against the provided byte length before taking ownership of the pointer. pub unsafe fn from_raw_parts( dptr: CUdeviceptr, len_bytes: usize, - device_id: usize, + stream: Arc, shape: Vec, strides: Vec, ) -> Self { Self::assert_valid_metadata(&shape, &strides, len_bytes); Self::from_device_buffer( - DeviceBuffer::from_raw_parts(dptr, len_bytes, device_id), + DeviceBuffer::from_raw_parts_async(dptr, len_bytes, stream), shape, strides, ) @@ -1065,18 +1164,18 @@ impl Tensor { /// store only `0` or `1`, since host code reads the bytes back as `bool` /// without validation. /// - /// [`Device::borrow_raw`]: cuda_core::Device::borrow_raw + /// [`Context::borrow_raw`]: cuda_core::Context::borrow_raw /// [`Stream::borrow_raw`]: cuda_core::Stream::borrow_raw pub unsafe fn borrow_raw_parts( dptr: CUdeviceptr, - device_id: usize, + context: Arc, shape: Vec, strides: Vec, ) -> Self { let len_bytes = checked_num_bytes_i32::(&shape) .expect("Tensor shape contains invalid dimensions or overflows."); Self::from_device_buffer( - DeviceBuffer::borrowed_from_raw_parts(dptr, len_bytes, device_id), + DeviceBuffer::borrowed_from_raw_parts(dptr, len_bytes, context), shape, strides, ) @@ -1098,7 +1197,7 @@ impl Tensor { /// /// # Safety /// Everything positional is verified at construction: the pointer, length, - /// and device ordinal come from `owner`'s one-time + /// and CUDA context come from `owner`'s one-time /// [`DeviceAllocation`] assertion; liveness is the held refcount; and the /// shape/stride addressable extent is checked (overflow-checked) against /// `owner.len_bytes()` right here. What cannot be verified at any call @@ -1118,6 +1217,10 @@ impl Tensor { /// floating-point `T`, but a `Tensor` over foreign memory requires /// the owner to store only `0` or `1`, because host code reads the bytes /// back as `bool` without validation. + /// - Derived typed buffers may change their element representation. The + /// caller must preserve validity for any external typed aliases after + /// those writes too, even when accesses are separated in time. Retaining + /// an owner proves liveness, not exclusive ownership of its typed views. /// /// Constructing two tensors over the same allocation and using either as /// a mutable output is a data race the type system cannot see: distinct @@ -1486,6 +1589,25 @@ pub struct TensorView<'a, T: DType> { } impl<'a, T: DType> TensorView<'a, T> { + /// Borrows this contiguous slice for a typed SIMT kernel argument, preserving + /// its byte offset and the base allocation's context and access tracker. + pub fn as_buffer(&self) -> Result, Error> + where + T: cuda_core::DeviceCopy, + { + if self.strides != contiguous_strides(&self.shape) { + return tensor_error_result("strided tensor cannot become a contiguous buffer view"); + } + let Allocation::Device(buffer) = &self.base.storage.allocation else { + return tensor_error_result("metadata-only tensors have no device buffer"); + }; + // SAFETY: slicing validated the extent; the parent tensor guarantees T + // validity and this view shares its allocation and access tracker. + unsafe { + cuda_core::DeviceBufferView::borrow_storage(buffer, self.offset_bytes, self.size()) + } + .map_err(Into::into) + } pub fn shape(&self) -> &[i32] { &self.shape } @@ -2417,6 +2539,9 @@ mod tests { len_bytes: usize, } unsafe impl DeviceAllocation for FakeAlloc { + fn context(&self) -> std::sync::Arc { + unsafe { cuda_core::Context::borrow_raw(0x100usize as *mut _, 0, 0) } + } fn device_ptr(&self) -> CUdeviceptr { 16 // 16-aligned sentinel; not dereferenced in from_foreign's spec calc } diff --git a/cutile-rs/cutile/tests/autotune.rs b/cutile-rs/cutile/tests/autotune.rs index 25d1764bbb..1e20be99d7 100644 --- a/cutile-rs/cutile/tests/autotune.rs +++ b/cutile-rs/cutile/tests/autotune.rs @@ -7,7 +7,7 @@ #![cfg(feature = "experimental-tune")] -use cuda_core::Device; +use cuda_core::Context; use cutile::bench::BenchOptions; use cutile::prelude::*; use cutile::tune::{Autotuner, Config, ParamValue, TrialState}; @@ -29,7 +29,7 @@ fn quick_bench() -> BenchOptions { /// a winner produced, JSONL log written and resumable. #[test] fn end_to_end_grid_tune_on_gpu() { - let device = Device::new(0).expect("device 0"); + let device = Context::new(0).expect("device 0"); let stream = device.new_stream().expect("stream"); let configs: Vec = [1usize << 16, 1 << 20, 0] @@ -129,7 +129,7 @@ mod tune_test_module { fn record_end_to_end_with_real_l2_verification() { use cutile::tune::{Record, RecordEntry, Workspace}; - let device = Device::new(0).expect("device 0"); + let device = Context::new(0).expect("device 0"); let stream = device.new_stream().expect("stream"); let n: usize = 1 << 18; diff --git a/cutile-rs/cutile/tests/do_bench.rs b/cutile-rs/cutile/tests/do_bench.rs index 9ff82410bc..634df09747 100644 --- a/cutile-rs/cutile/tests/do_bench.rs +++ b/cutile-rs/cutile/tests/do_bench.rs @@ -5,7 +5,7 @@ //! GPU integration tests for `cutile::bench` (device-event timing). -use cuda_core::Device; +use cuda_core::Context; use cutile::bench::{do_bench, do_bench_paired, BenchOptions}; use cutile::prelude::*; use std::time::Duration; @@ -22,7 +22,7 @@ fn quick_opts() -> BenchOptions { #[test] fn do_bench_times_a_kernel() { - let device = Device::new(0).expect("device 0"); + let device = Context::new(0).expect("device 0"); let stream = device.new_stream().expect("stream"); let result = do_bench(&stream, &quick_opts(), |s| { @@ -45,7 +45,7 @@ fn do_bench_times_a_kernel() { #[test] fn do_bench_paired_runs_equal_reps() { - let device = Device::new(0).expect("device 0"); + let device = Context::new(0).expect("device 0"); let stream = device.new_stream().expect("stream"); let (a, b) = do_bench_paired( @@ -73,7 +73,7 @@ fn do_bench_paired_runs_equal_reps() { #[test] fn closure_errors_propagate() { - let device = Device::new(0).expect("device 0"); + let device = Context::new(0).expect("device 0"); let stream = device.new_stream().expect("stream"); let err = do_bench(&stream, &quick_opts(), |_| { diff --git a/cutile-rs/cutile/tests/gpu/add_refs.rs b/cutile-rs/cutile/tests/gpu/add_refs.rs index 3cd947b34d..b073a72f4f 100644 --- a/cutile-rs/cutile/tests/gpu/add_refs.rs +++ b/cutile-rs/cutile/tests/gpu/add_refs.rs @@ -31,7 +31,7 @@ use my_module::add; #[test] fn smoke_add_refs() { common::with_test_stack(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let x = api::ones::(&[32]).sync_on(&stream).unwrap(); diff --git a/cutile-rs/cutile/tests/gpu/atomic_red_view.rs b/cutile-rs/cutile/tests/gpu/atomic_red_view.rs index dd5e263217..80e6e1caa5 100644 --- a/cutile-rs/cutile/tests/gpu/atomic_red_view.rs +++ b/cutile-rs/cutile/tests/gpu/atomic_red_view.rs @@ -44,7 +44,7 @@ fn atomic_red_view_accumulates_across_blocks() { if !common::supports_tile_ir(&[cutile_ir::requirements::Feature::AtomicRedView]) { return; } - let device = cuda_core::Device::new(0).expect("device"); + let device = cuda_core::Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); const BLOCKS: usize = 32; diff --git a/cutile-rs/cutile/tests/gpu/borrow_raw_parts.rs b/cutile-rs/cutile/tests/gpu/borrow_raw_parts.rs index b10a8b7fae..678862fef1 100644 --- a/cutile-rs/cutile/tests/gpu/borrow_raw_parts.rs +++ b/cutile-rs/cutile/tests/gpu/borrow_raw_parts.rs @@ -22,9 +22,8 @@ use crate::common; fn borrow_raw_parts_does_not_free_foreign_memory() { common::with_test_stack(|| { let _policy = global_policy(0); - let device = cuda_core::Device::new(0).expect("device"); + let device = cuda_core::Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); - let ordinal = device.ordinal(); const N: usize = 256; let bytes = N * std::mem::size_of::(); @@ -42,8 +41,14 @@ fn borrow_raw_parts_does_not_free_foreign_memory() { // tensor would free `mem` on the first drop, making borrows 2 and 3 // use-after-free; a borrowed tensor leaves `mem` untouched. for _ in 0..3 { - let t = - unsafe { Tensor::::borrow_raw_parts(mem, ordinal, vec![N as i32], vec![1]) }; + let t = unsafe { + Tensor::::borrow_raw_parts( + mem, + stream.context().clone(), + vec![N as i32], + vec![1], + ) + }; let out = t .to_host_vec() .sync_on(&stream) @@ -88,6 +93,9 @@ impl Drop for Owner { // SAFETY: `dptr` is a live `len_bytes` allocation on `device_id` until `Owner` drops. unsafe impl DeviceAllocation for Owner { + fn context(&self) -> std::sync::Arc { + self.stream.context().clone() + } fn device_ptr(&self) -> CUdeviceptr { self.dptr } @@ -103,7 +111,7 @@ unsafe impl DeviceAllocation for Owner { fn from_foreign_keep_alive_holds_owner() { common::with_test_stack(|| { let _policy = global_policy(0); - let device = cuda_core::Device::new(0).expect("device"); + let device = cuda_core::Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); const N: usize = 256; @@ -150,6 +158,6 @@ fn from_foreign_keep_alive_holds_owner() { dropped.load(Ordering::SeqCst), "owner not dropped after the last borrowing tensor was dropped" ); - unsafe { stream.synchronize().expect("drain free") }; + stream.synchronize().expect("drain free"); }); } diff --git a/cutile-rs/cutile/tests/gpu/global_counter.rs b/cutile-rs/cutile/tests/gpu/global_counter.rs index da56aaf3e1..b00b2cdf16 100644 --- a/cutile-rs/cutile/tests/gpu/global_counter.rs +++ b/cutile-rs/cutile/tests/gpu/global_counter.rs @@ -45,7 +45,7 @@ mod atomic_counter_kernels { fn atomic_global_increments_are_unique_across_tile_programs() { common::with_test_stack(|| { const N: usize = 1024; - let device = cuda_core::Device::new(0).expect("device"); + let device = cuda_core::Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); let mut out = api::zeros::(&[N]).sync_on(&stream).expect("zeros"); // Same specialization twice: test uniqueness under contention and @@ -66,7 +66,7 @@ use global_kernels::update_counter_ordered; #[test] fn smoke_global_counter_ordered() { common::with_test_stack(|| { - let device = cuda_core::Device::new(0).expect("device"); + let device = cuda_core::Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); let mut first = api::zeros::(&[1]).sync_on(&stream).expect("zeros"); diff --git a/cutile-rs/cutile/tests/gpu/graph_scope_inputs.rs b/cutile-rs/cutile/tests/gpu/graph_scope_inputs.rs index f2a306618f..868ff067cd 100644 --- a/cutile-rs/cutile/tests/gpu/graph_scope_inputs.rs +++ b/cutile-rs/cutile/tests/gpu/graph_scope_inputs.rs @@ -37,7 +37,7 @@ use graph_scope_module::add; #[test] fn pre_allocated_inputs_record_and_replay() { common::with_test_stack(|| { - let device = cuda_core::Device::new(0).expect("device"); + let device = cuda_core::Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); let a = api::arange::(8).sync_on(&stream).expect("a"); diff --git a/cutile-rs/cutile/tests/gpu/inter_module.rs b/cutile-rs/cutile/tests/gpu/inter_module.rs index 2205344835..7e48701f6f 100644 --- a/cutile-rs/cutile/tests/gpu/inter_module.rs +++ b/cutile-rs/cutile/tests/gpu/inter_module.rs @@ -56,7 +56,8 @@ use my_kernels::{apply_relu, apply_relu_square}; #[test] fn smoke_inter_module() { common::with_test_stack(|| { - let device: std::sync::Arc = cuda_core::Device::new(0).expect("device"); + let device: std::sync::Arc = + cuda_core::Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); let block: usize = 64; let n: usize = 128; diff --git a/cutile-rs/cutile/tests/gpu/submission_lifetimes.rs b/cutile-rs/cutile/tests/gpu/submission_lifetimes.rs index 60fb31b5d3..9336d7d624 100644 --- a/cutile-rs/cutile/tests/gpu/submission_lifetimes.rs +++ b/cutile-rs/cutile/tests/gpu/submission_lifetimes.rs @@ -30,6 +30,9 @@ struct Allocation(Tensor); // SAFETY: the owned tensor has a stable, live allocation. The test's rescue // handle is used only to observe refcounts, never to access its GPU bytes. unsafe impl DeviceAllocation for Allocation { + fn context(&self) -> std::sync::Arc { + self.0.context().clone() + } fn device_ptr(&self) -> u64 { self.0.device_pointer().cu_deviceptr() } @@ -81,7 +84,7 @@ impl Gate { let expired_flag = expired.clone(); GATES_ARMED.fetch_add(1, std::sync::atomic::Ordering::SeqCst); *LAST_GATE_ARMED_AT.lock().unwrap() = Some(std::time::Instant::now()); - unsafe { + { stream .launch_host_function(move || { GATES_STARTED.fetch_add(1, std::sync::atomic::Ordering::SeqCst); @@ -138,7 +141,7 @@ fn assert_gated_pending( panic!( "gated submission completed on the first poll while its Gate should still be blocking the stream (newest gate armed {age:?} ago; gates armed: {armed}, callbacks started: {}, finished: {finished}, wait outcomes: {outcomes:?}, stream query now: {:?})", GATES_STARTED.load(std::sync::atomic::Ordering::SeqCst), - unsafe { stream.query() }, + stream.query(), ) } Poll::Ready(Err(e)) => panic!("gated submission failed on the first poll: {e}"), @@ -155,7 +158,7 @@ fn forget_pending(op: O, stream: &Arc, gate: &Ga fn on_gpu(f: impl FnOnce(Arc, Arc) + Send + 'static) { crate::common::with_test_stack(move || { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let other = device.new_stream().unwrap(); let src = api::ones::(&[32]).sync_on(&stream).unwrap(); @@ -176,11 +179,9 @@ fn on_gpu(f: impl FnOnce(Arc, Arc) + Send let _ = api::dup(&src).sync_on(&stream).unwrap(); api::memcpy(&mut dst, &src).sync_on(&stream).unwrap(); // Initialize the completion backend before placing any work behind a gate. - unsafe { - stream - .launch_host_function(|| std::thread::sleep(Duration::from_millis(10))) - .unwrap() - }; + stream + .launch_host_function(|| std::thread::sleep(Duration::from_millis(10))) + .unwrap(); block_on(DeviceFuture::scheduled( value(()), ExecutionContext::new(stream.clone()), @@ -220,7 +221,7 @@ fn forgotten_borrowed_kernel_retains_inputs_outputs_and_excludes_conflicts() { "borrowed output was released after recover" ); drop(gate); - unsafe { stream.synchronize().unwrap() }; + stream.synchronize().unwrap(); }); } @@ -241,7 +242,7 @@ fn forgotten_view_and_mapped_output_keep_their_storage() { assert!(Arc::strong_count(&src_owner) > 1); assert!(Arc::strong_count(&dst_owner) > 1); drop(gate); - unsafe { stream.synchronize().unwrap() }; + stream.synchronize().unwrap(); }); } @@ -256,7 +257,7 @@ fn forgotten_memcpy_keeps_both_allocations() { assert!(Arc::strong_count(&src_owner) > 1); assert!(Arc::strong_count(&dst_owner) > 1); drop(gate); - unsafe { stream.synchronize().unwrap() }; + stream.synchronize().unwrap(); }); } @@ -304,7 +305,7 @@ fn projected_dup_keeps_source_until_submission_completes() { forget_pending(op, &stream, &gate); assert!(Arc::strong_count(&owner) > 1); drop(gate); - unsafe { stream.synchronize().unwrap() }; + stream.synchronize().unwrap(); }); } @@ -394,7 +395,7 @@ fn graph_replay_retains_projected_storage_and_releases_launch_accesses() { assert!(Arc::strong_count(&src_owner) > 1); assert!(Arc::strong_count(&dst_owner) > 1); drop(gate); - unsafe { other.synchronize().unwrap() }; + other.synchronize().unwrap(); }); } diff --git a/cutile-rs/cutile/tests/gpu/tensor.rs b/cutile-rs/cutile/tests/gpu/tensor.rs index 5d369c3f6f..6e783ad5f6 100644 --- a/cutile-rs/cutile/tests/gpu/tensor.rs +++ b/cutile-rs/cutile/tests/gpu/tensor.rs @@ -19,7 +19,7 @@ fn reinterpret_rejects_misaligned_storage() { Tensor::::from_raw_parts( base.device_pointer().cu_deviceptr() + 1, 4, - base.device_id(), + base.context().default_stream(), vec![4], vec![1], ) @@ -42,7 +42,7 @@ fn from_raw_parts_rejects_shape_storage_mismatch() { Tensor::::from_raw_parts( base.device_pointer().cu_deviceptr(), 4, - base.device_id(), + base.context().default_stream(), vec![2], vec![1], ) diff --git a/cutile-rs/cutile/tests/gpu/tensor_slicing.rs b/cutile-rs/cutile/tests/gpu/tensor_slicing.rs index ccaa1df4e4..a591d43c0a 100644 --- a/cutile-rs/cutile/tests/gpu/tensor_slicing.rs +++ b/cutile-rs/cutile/tests/gpu/tensor_slicing.rs @@ -40,7 +40,7 @@ use my_module::{add, scale}; #[test] fn smoke_tensor_slicing() { common::with_test_stack(|| { - let device = cuda_core::Device::new(0).expect("device"); + let device = cuda_core::Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); let block = 128; diff --git a/cutile-rs/cutile/tests/gpu/tile_ir_13_4.rs b/cutile-rs/cutile/tests/gpu/tile_ir_13_4.rs index f847fa81d5..7e500f07e5 100644 --- a/cutile-rs/cutile/tests/gpu/tile_ir_13_4.rs +++ b/cutile-rs/cutile/tests/gpu/tile_ir_13_4.rs @@ -293,7 +293,7 @@ mod coverage { #[test] fn launch_opt_in_checks_capabilities_even_during_warmup() { crate::common::with_test_stack(|| { - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let caps = ToolkitCapabilities::for_device(0).unwrap(); // The empty consumer has no predecessor-dependent accesses. @@ -337,7 +337,7 @@ fn dependent_launch_waits_for_producer_data() { if !supports(&[Feature::ProgrammaticDependentLaunch]) { return; } - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let input = api::zeros::(&[1]).sync_on(&stream).unwrap(); let output = api::zeros::(&[1]).sync_on(&stream).unwrap(); diff --git a/cutile-rs/cutile/tests/shared_host_buffers.rs b/cutile-rs/cutile/tests/shared_host_buffers.rs new file mode 100644 index 0000000000..e4fa400137 --- /dev/null +++ b/cutile-rs/cutile/tests/shared_host_buffers.rs @@ -0,0 +1,81 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use cuda_core::{Context, DeviceBuffer}; +use cutile::prelude::*; +use std::sync::Arc; + +#[cutile::module] +mod kernels { + use cutile::core::*; + #[cutile::entry] + fn double(out: &mut Tensor, input: &Tensor) { + out.store(load_tile_like(input, out) + load_tile_like(input, out)); + } +} + +#[test] +fn typed_buffer_tile_kernel_and_back_preserve_allocation() { + let context = Context::new(0).unwrap(); + let stream = context.new_stream().unwrap(); + let buffer = DeviceBuffer::from_host(&stream, &[3.0f32; 32]).unwrap(); + let pointer = buffer.cu_deviceptr(); + let identity = buffer.storage().allocation_id(); + let tensor = Tensor::from_buffer(buffer, vec![32]); + assert_eq!(tensor.as_buffer().unwrap().cu_deviceptr(), pointer); + assert_eq!( + tensor.as_buffer().unwrap().storage().allocation_id(), + identity + ); + assert_eq!(tensor.context(), &context); + let output = Tensor::from_buffer(DeviceBuffer::::zeroed(&stream, 32).unwrap(), vec![32]); + let output_pointer = output.as_buffer().unwrap().cu_deviceptr(); + let output = kernels::double(output.partition([32]), &tensor) + .sync_on(&stream) + .unwrap(); + let output = output.0.unpartition().into_buffer(); + assert_eq!(output.cu_deviceptr(), output_pointer); + assert_eq!(output.to_host_vec(&stream).unwrap(), vec![6.0; 32]); + let buffer = tensor.into_buffer(); + assert_eq!(buffer.cu_deviceptr(), pointer); + assert_eq!(buffer.storage().allocation_id(), identity); +} + +#[test] +fn tensor_slices_preserve_byte_offset_and_reject_strided_buffer_views() { + let context = Context::new(0).unwrap(); + let stream = context.new_stream().unwrap(); + let tensor = Tensor::from_buffer( + DeviceBuffer::from_host(&stream, &[1u32; 16]).unwrap(), + vec![4, 4], + ); + let slice = tensor.slice(&[1..3]).unwrap(); + let view = slice.as_buffer().unwrap(); + let base = tensor.as_buffer().unwrap(); + assert_eq!(view.cu_deviceptr(), base.cu_deviceptr() + 16); + assert_eq!(view.len(), 8); + assert_eq!(view.context(), base.context()); + assert_eq!( + view.storage().allocation_id(), + base.storage().allocation_id() + ); + assert!(tensor.slice(&[0..4, 1..3]).unwrap().as_buffer().is_err()); +} + +#[test] +fn typed_ownership_transfer_rejects_a_live_boolean_alias() { + let context = Context::new(0).unwrap(); + let stream = context.new_stream().unwrap(); + let tensor = Arc::new(Tensor::from_buffer( + DeviceBuffer::from_host(&stream, &[true; 8]).unwrap(), + vec![8], + )); + let alias = (&tensor).reshape(&[2, 4]).unwrap(); + let tensor = Arc::try_unwrap(tensor).ok().unwrap(); + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| tensor.into_buffer())).is_err() + ); + assert_eq!(alias.as_buffer().unwrap().len(), 8); +} diff --git a/cutile-rs/cutile/tests/slice_non_divisible.rs b/cutile-rs/cutile/tests/slice_non_divisible.rs index dd400320a7..c7397e0f44 100644 --- a/cutile-rs/cutile/tests/slice_non_divisible.rs +++ b/cutile-rs/cutile/tests/slice_non_divisible.rs @@ -472,7 +472,7 @@ fn gemm_non_divisible_m_and_n() { k.to_string(), ]; - let device = cuda_core::Device::new(0).unwrap(); + let device = cuda_core::Context::new(0).unwrap(); let stream = device.new_stream().unwrap(); let z = api::zeros::(&[m, n]) diff --git a/cutile-rs/cutile/tests/tensor_and_matrix_ops.rs b/cutile-rs/cutile/tests/tensor_and_matrix_ops.rs index 794b1fd5e8..2f36572b7b 100644 --- a/cutile-rs/cutile/tests/tensor_and_matrix_ops.rs +++ b/cutile-rs/cutile/tests/tensor_and_matrix_ops.rs @@ -4,7 +4,7 @@ */ use cutile::cuda_core::{f4e2m1fnx2, f8e4m3fn}; use cutile::prelude::{ - api, Arc, Device, DeviceOp, DeviceOpReshape, IntoPartition, Tensor, ToHostVec, + api, Arc, Context, DeviceOp, DeviceOpReshape, IntoPartition, Tensor, ToHostVec, }; use cutile_compiler::compiler::utils::CompileOptions; use cutile_ir::requirements::Feature; @@ -613,7 +613,7 @@ fn execute_raw_mmaf_scaled_nvfp4_e4_scale() { return; } - let device = Device::new(0).expect("device"); + let device = Context::new(0).expect("device"); let stream = device.new_stream().expect("stream"); let fp4_one = f4e2m1fnx2::from_nibbles(0x2, 0x2); let scale_one = f8e4m3fn(0x38); diff --git a/cutile-rs/cutile/tests/ui/graph_scope_rejects_allocating_input.rs b/cutile-rs/cutile/tests/ui/graph_scope_rejects_allocating_input.rs index fd24146b7b..42cd20a44f 100644 --- a/cutile-rs/cutile/tests/ui/graph_scope_rejects_allocating_input.rs +++ b/cutile-rs/cutile/tests/ui/graph_scope_rejects_allocating_input.rs @@ -18,7 +18,7 @@ mod kernels { } fn main() -> Result<(), Error> { - let device = Device::new(0)?; + let device = Context::new(0)?; let stream = device.new_stream()?; let x = api::ones::(&[4]).sync_on(&stream)?; let _graph = CudaGraph::scope(&stream, |s| { diff --git a/cutile-rs/scripts/run_cpu_tests.sh b/cutile-rs/scripts/run_cpu_tests.sh index d6658f7ba9..2653e2324d 100755 --- a/cutile-rs/scripts/run_cpu_tests.sh +++ b/cutile-rs/scripts/run_cpu_tests.sh @@ -130,9 +130,10 @@ run_step \ run_step \ "cuda-core compile-time contract tests" \ host_cargo test -p cuda-core \ - --test simt_device_copy_derive \ - --test simt_device_copy_impls \ - --test simt_launch_contract_types + --test device_copy_derive \ + --test device_copy_impls \ + --test device_buffer_casts \ + --test launch_contract_types # Builds the compile_fail soundness contracts (non-Send launch storage, # brand/rank rejection, buffer aliasing) that no other target covers. diff --git a/cutile-rs/scripts/run_gpu_tests.sh b/cutile-rs/scripts/run_gpu_tests.sh index b40b2254a4..330ed0caa5 100755 --- a/cutile-rs/scripts/run_gpu_tests.sh +++ b/cutile-rs/scripts/run_gpu_tests.sh @@ -83,21 +83,22 @@ run_step \ "cuda-core GPU integration test vmm_multicast" \ host_cargo test -p cuda-core --test vmm_multicast -# The SIMT integration tests carried over from cuda-oxide. simt_vmm_p2p +# The SIMT integration tests carried over from cuda-oxide. vmm_p2p # exercises the real two-GPU P2P path (it self-skips on single-device -# machines or when peer access is unavailable); simt_vmm_multicast +# machines or when peer access is unavailable); vmm_multicast # self-skips without NVLink switch multicast support. for test_target in \ borrow_raw \ - simt_context_limits \ - simt_context_sync_policy \ - simt_device_buffer \ - simt_device_buffer_leaks \ - simt_pinned_host_buffer \ - simt_stream_priority \ - simt_stream_query \ - simt_vmm_multicast \ - simt_vmm_p2p + context_limits \ + context_sync_policy \ + device_buffer \ + device_buffer_leaks \ + pinned_host_buffer \ + stream_priority \ + stream_query \ + nvlink_multicast \ + shared_allocation \ + vmm_p2p do run_step \ "cuda-core GPU integration test ${test_target}" \ @@ -132,3 +133,5 @@ done print_summary_and_exit \ "All GPU tests passed!" \ "Some GPU tests failed. See output above for details." + +run_step "shared Tile and SIMT host buffer interop" cargo test -p cutile --test shared_host_buffers -- --test-threads=1