Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
263 changes: 239 additions & 24 deletions cuda_core/cuda/core/_cpp/resource_handles.cpp

Large diffs are not rendered by default.

6 changes: 5 additions & 1 deletion cuda_core/cuda/core/_cpp/resource_handles.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ void clear_last_error() noexcept;
extern decltype(&cuDevicePrimaryCtxRetain) p_cuDevicePrimaryCtxRetain;
extern decltype(&cuDevicePrimaryCtxRelease) p_cuDevicePrimaryCtxRelease;
extern decltype(&cuCtxGetCurrent) p_cuCtxGetCurrent;
extern decltype(&cuCtxSetCurrent) p_cuCtxSetCurrent;
extern decltype(&cuGreenCtxCreate) p_cuGreenCtxCreate;
extern decltype(&cuGreenCtxDestroy) p_cuGreenCtxDestroy;
extern decltype(&cuCtxFromGreenCtx) p_cuCtxFromGreenCtx;
Expand Down Expand Up @@ -423,7 +424,10 @@ DevicePtrHandle deviceptr_import_ipc(
StreamHandle deallocation_stream(const DevicePtrHandle& h) noexcept;

// Set the deallocation stream for a device pointer handle.
void set_deallocation_stream(const DevicePtrHandle& h, const StreamHandle& h_stream) noexcept;
// Returns CUDA_ERROR_INVALID_CONTEXT when a default-stream token cannot be
// bound because no CUDA context is current.
CUresult set_deallocation_stream(
const DevicePtrHandle& h, const StreamHandle& h_stream) noexcept;

// ============================================================================
// Library handle functions
Expand Down
56 changes: 53 additions & 3 deletions cuda_core/cuda/core/_memory/_buffer.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,15 @@ class Buffer:
...

@classmethod
def _init(cls, ptr: DevicePointerType, size: int, mr: MemoryResource | None=None, ipc_descriptor: IPCBufferDescriptor | None=None, owner: object | None=None) -> Buffer:
def _init(cls, ptr: DevicePointerType, size: int, mr: MemoryResource | None=None, ipc_descriptor: IPCBufferDescriptor | None=None, owner: object | None=None, *, stream: Stream | GraphBuilder | None=None) -> Buffer:
"""Create a Buffer from a raw pointer.

When ``mr`` is provided, the buffer takes ownership: ``mr.deallocate()``
is called when the buffer is closed or garbage collected. When ``owner``
is provided, the owner is kept alive but no deallocation is performed.
When ``mr`` is provided, a deallocation stream is recorded at creation
(``stream`` if given, otherwise ``default_stream()``). Recording a
default-stream token requires a CUDA context to be current.
"""

@staticmethod
Expand All @@ -55,7 +58,7 @@ class Buffer:
...

@staticmethod
def from_handle(ptr: DevicePointerType, size: int, mr: MemoryResource | None=None, owner: object | None=None) -> Buffer:
def from_handle(ptr: DevicePointerType, size: int, mr: MemoryResource | None=None, owner: object | None=None, *, stream: Stream | GraphBuilder | None=None) -> Buffer:
"""Create a new :class:`Buffer` object from a pointer.

Parameters
Expand All @@ -72,6 +75,13 @@ class Buffer:
An object holding external allocation that the ``ptr`` points to.
The reference is kept as long as the buffer is alive.
The ``owner`` and ``mr`` cannot be specified together.
stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder`, optional
Keyword-only. The stream used to order the buffer's deallocation
when ``mr`` owns the pointer. Defaults to ``default_stream()``.
Recording a default-stream token requires a CUDA context to be
current. If the buffer may be freed from a different host thread,
pass a stream other than the per-thread default stream, which
refers to a different stream on each thread.

Note
----
Expand Down Expand Up @@ -117,6 +127,41 @@ class Buffer:
stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder`, optional
The stream object to use for asynchronous deallocation. If None,
the deallocation stream stored in the handle is used.

See Also
--------
set_deallocation_stream
Change the deallocation stream without closing the buffer.
"""

def set_deallocation_stream(self, stream: Stream | GraphBuilder) -> None:
"""Change the stream that orders this buffer's eventual deallocation.

The buffer remains open and usable. A later :meth:`close` without a
stream, garbage collection, or release of the final retained device
pointer handle uses the replacement stream.

This method does not synchronize streams or establish dependencies.
The caller must ensure that allocation and all accesses are ordered
before the deallocation on ``stream``.

Parameters
----------
stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder`
The stream to use for eventual asynchronous deallocation.

Raises
------
RuntimeError
If the buffer is already closed, or if a default-stream token
cannot be bound because no CUDA context is current.
TypeError
If ``stream`` is ``None`` or is not an accepted stream object.

Notes
-----
Synchronizing concurrent mutation and destruction of the same buffer
is the caller's responsibility.
"""

def __enter__(self):
Expand Down Expand Up @@ -264,7 +309,12 @@ class MemoryResource:
stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder`
Keyword-only. The stream on which to perform the allocation
asynchronously. Must be passed explicitly; pass
``device.default_stream`` to use the default stream.
``device.default_stream`` to use the default stream. For subclasses
that support stream-ordered deallocation, this stream also orders
the buffer's eventual deallocation, so if the buffer may be freed
from a different host thread, prefer a stream other than the
per-thread default stream, which refers to a different stream on
each thread.

Returns
-------
Expand Down
138 changes: 116 additions & 22 deletions cuda_core/cuda/core/_memory/_buffer.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,14 @@ from cuda.core._memory cimport _ipc
from cuda.core._resource_handles cimport (
DevicePtrHandle,
StreamHandle,
ContextHandle,
deviceptr_create_with_owner,
deviceptr_create_with_mr,
register_mr_dealloc_callback,
as_intptr,
as_cu,
get_current_context,
get_stream_context,
set_deallocation_stream,
)
from cuda.core.typing import DevicePointerType
Expand Down Expand Up @@ -49,23 +52,21 @@ cdef void _mr_dealloc_callback(
size_t size,
const StreamHandle& h_stream,
) noexcept:
"""Called by the C++ deleter to deallocate via MemoryResource.deallocate.

This is the C++ teardown path: there is no Python caller frame from
which to obtain a stream. If the device-pointer handle was created
without ``set_deallocation_stream`` being called (e.g. buffers minted
via ``Buffer.from_handle(ptr, size, mr=mr)`` from DLPack import,
third-party adapters, or other foreign sources), ``h_stream`` is
empty here. Stream-ordered MR ``deallocate`` overrides reject
``stream=None`` (issue #2001), so without a fallback the destructor
would print a warning and leak the allocation. Fall back to the
legacy/per-thread default stream so the free still happens; this is
the unique exception to the "no implicit default-stream fallback"
policy because the teardown has no other source of truth.
"""
"""Called by the C++ deleter to deallocate via MemoryResource.deallocate."""
cdef Stream stream
try:
stream = Stream._from_handle(Stream, h_stream) if h_stream else default_stream()
if not h_stream:
print(
"Warning: no deallocation stream was recorded; falling back to "
"the default stream for mr.deallocate() during Buffer "
"destruction. This is an internal cuda-core error; please "
"report it with your CUDA driver, CUDA Toolkit, and "
"cuda-python versions.",
file=sys.stderr,
)
stream = default_stream()
else:
stream = Stream._from_handle(Stream, h_stream)
mr.deallocate(int(ptr), size, stream=stream)
except Exception as exc:
print(f"Warning: mr.deallocate() failed during Buffer destruction: {exc}",
Expand All @@ -74,6 +75,21 @@ cdef void _mr_dealloc_callback(
register_mr_dealloc_callback(_mr_dealloc_callback)


cdef inline void _require_deallocation_stream_context(Stream s) except *:
"""Default-stream tokens need a current context to pin into the free recipe."""
cdef ContextHandle h_ctx
if get_stream_context(s._h_stream):
return
h_ctx = get_current_context()
if h_ctx:
return
raise RuntimeError(
"Cannot record a default deallocation stream when no CUDA context is "
"current. Call Device.set_current() first, or pass stream= with a "
"non-default Stream."
)


__all__ = ['Buffer', 'MemoryResource']


Expand Down Expand Up @@ -176,20 +192,33 @@ cdef class Buffer:
def _init(
cls, ptr: DevicePointerType, size_t size, mr: MemoryResource | None = None,
ipc_descriptor: IPCBufferDescriptor | None = None,
owner : object | None = None
owner : object | None = None,
*,
stream: Stream | GraphBuilder | None = None,
) -> Buffer:
"""Create a Buffer from a raw pointer.

When ``mr`` is provided, the buffer takes ownership: ``mr.deallocate()``
is called when the buffer is closed or garbage collected. When ``owner``
is provided, the owner is kept alive but no deallocation is performed.
When ``mr`` is provided, a deallocation stream is recorded at creation
(``stream`` if given, otherwise ``default_stream()``). Recording a
default-stream token requires a CUDA context to be current.
"""
if mr is not None and owner is not None:
raise ValueError("owner and memory resource cannot be both specified together")
if stream is not None and mr is None:
raise ValueError("stream requires a memory resource (mr)")
cdef Buffer self = Buffer.__new__(cls)
cdef uintptr_t c_ptr = <uintptr_t>(int(ptr))
cdef Stream s
if mr is not None:
# Validate before taking ownership so a bad stream does not cause
# construction failure to deallocate the caller's pointer.
s = default_stream() if stream is None else Stream_accept(stream)
_require_deallocation_stream_context(s)
self._h_ptr = deviceptr_create_with_mr(c_ptr, size, mr)
HANDLE_RETURN(set_deallocation_stream(self._h_ptr, s._h_stream))
else:
self._h_ptr = deviceptr_create_with_owner(c_ptr, owner)
self._size = size
Expand All @@ -201,10 +230,17 @@ cdef class Buffer:

@staticmethod
def _reduce_helper(mr, ipc_descriptor):
cdef ContextHandle h_ctx = get_current_context()
cdef int device_id
if not h_ctx:
# Spawned processes unpickle arguments before entering their target,
# so initialize the context needed to bind the default-stream token.
device_id = mr.device_id
(Device(device_id) if device_id >= 0 else Device()).set_current()
# The parent process's stream is not portable across processes, so the
# pickle path cannot thread an explicit stream through. Seed the
# imported buffer's deallocation with the current context's default
# stream; the receiver can override via buffer.close(stream).
# stream; the receiver can override it before or during close.
return Buffer.from_ipc_descriptor(mr, ipc_descriptor, stream=default_stream())

def __reduce__(self) -> tuple[object, ...]:
Expand All @@ -217,6 +253,8 @@ cdef class Buffer:
def from_handle(
ptr: DevicePointerType, size_t size, mr: MemoryResource | None = None,
owner: object | None = None,
*,
stream: Stream | GraphBuilder | None = None,
) -> Buffer:
"""Create a new :class:`Buffer` object from a pointer.

Expand All @@ -234,14 +272,21 @@ cdef class Buffer:
An object holding external allocation that the ``ptr`` points to.
The reference is kept as long as the buffer is alive.
The ``owner`` and ``mr`` cannot be specified together.
stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder`, optional
Keyword-only. The stream used to order the buffer's deallocation
when ``mr`` owns the pointer. Defaults to ``default_stream()``.
Recording a default-stream token requires a CUDA context to be
current. If the buffer may be freed from a different host thread,
pass a stream other than the per-thread default stream, which
refers to a different stream on each thread.

Note
----
When neither ``mr`` nor ``owner`` is specified, this creates a
non-owning reference. The pointer will NOT be freed when the
:class:`Buffer` is closed or garbage collected.
"""
return Buffer._init(ptr, size, mr=mr, owner=owner)
return Buffer._init(ptr, size, mr=mr, owner=owner, stream=stream)

@classmethod
def from_ipc_descriptor(
Expand Down Expand Up @@ -290,9 +335,45 @@ cdef class Buffer:
stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder`, optional
The stream object to use for asynchronous deallocation. If None,
the deallocation stream stored in the handle is used.

See Also
--------
set_deallocation_stream
Change the deallocation stream without closing the buffer.
"""
Buffer_close(self, stream)

def set_deallocation_stream(self, stream: Stream | GraphBuilder) -> None:
"""Change the stream that orders this buffer's eventual deallocation.

The buffer remains open and usable. A later :meth:`close` without a
stream, garbage collection, or release of the final retained device
pointer handle uses the replacement stream.

This method does not synchronize streams or establish dependencies.
The caller must ensure that allocation and all accesses are ordered
before the deallocation on ``stream``.

Parameters
----------
stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder`
The stream to use for eventual asynchronous deallocation.

Raises
------
RuntimeError
If the buffer is already closed, or if a default-stream token
cannot be bound because no CUDA context is current.
TypeError
If ``stream`` is ``None`` or is not an accepted stream object.

Notes
-----
Synchronizing concurrent mutation and destruction of the same buffer
is the caller's responsibility.
"""
Buffer_set_deallocation_stream(self, stream)

def __enter__(self):
return self

Expand Down Expand Up @@ -546,7 +627,12 @@ cdef class MemoryResource:
stream : :obj:`~_stream.Stream` | :obj:`~graph.GraphBuilder`
Keyword-only. The stream on which to perform the allocation
asynchronously. Must be passed explicitly; pass
``device.default_stream`` to use the default stream.
``device.default_stream`` to use the default stream. For subclasses
that support stream-ordered deallocation, this stream also orders
the buffer's eventual deallocation, so if the buffer may be freed
from a different host thread, prefer a stream other than the
per-thread default stream, which refers to a different stream on
each thread.

Returns
-------
Expand Down Expand Up @@ -619,15 +705,23 @@ cdef Buffer Buffer_from_deviceptr_handle(
return buf


cdef inline void Buffer_set_deallocation_stream(Buffer self, object stream):
"""Validate and replace a live buffer's deallocation recipe."""
cdef Stream s
if not self._h_ptr:
raise RuntimeError("Cannot set the deallocation stream on a closed Buffer")
s = Stream_accept(stream)
_require_deallocation_stream_context(s)
HANDLE_RETURN(set_deallocation_stream(self._h_ptr, s._h_stream))


cdef inline void Buffer_close(Buffer self, object stream):
"""Close a buffer, freeing its memory."""
cdef Stream s
if not self._h_ptr:
return
# Update deallocation stream if provided
if stream is not None:
s = Stream_accept(stream)
set_deallocation_stream(self._h_ptr, s._h_stream)
Buffer_set_deallocation_stream(self, stream)
# Reset handle - RAII deleter will free the memory (and release owner ref in C++)
self._h_ptr.reset()
self._size = 0
Expand Down
2 changes: 1 addition & 1 deletion cuda_core/cuda/core/_memory/_graph_memory_resource.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ cdef inline Buffer GMR_allocate(cyGraphMemoryResource self, size_t size, Stream
return Buffer_from_deviceptr_handle(h_ptr, size, self, None)


cdef inline void GMR_deallocate(intptr_t ptr, size_t size, Stream stream) noexcept:
cdef inline void GMR_deallocate(intptr_t ptr, size_t size, Stream stream) except *:
cdef cydriver.CUstream s = as_cu(stream._h_stream)
cdef cydriver.CUdeviceptr devptr = <cydriver.CUdeviceptr>ptr
with nogil:
Expand Down
11 changes: 10 additions & 1 deletion cuda_core/cuda/core/_memory/_managed_buffer.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,8 @@ def from_handle(
size: int,
mr: MemoryResource | None = None,
owner: object | None = None,
*,
stream: Stream | GraphBuilder | None = None,
) -> Buffer:
"""Wrap an existing managed-memory pointer in a :class:`ManagedBuffer`.

Expand All @@ -173,8 +175,15 @@ def from_handle(
owner : object, optional
An object that keeps the underlying allocation alive.
``owner`` and ``mr`` cannot both be specified.
stream : Stream | GraphBuilder, optional
Keyword-only. The stream used to order the buffer's deallocation
when ``mr`` owns the pointer. Defaults to ``default_stream()``.
Recording a default-stream token requires a CUDA context to be
current. If the buffer may be freed from a different host thread,
pass a stream other than the per-thread default stream, which
refers to a different stream on each thread.
"""
return cls._init(ptr, size, mr=mr, owner=owner)
return cls._init(ptr, size, mr=mr, owner=owner, stream=stream)

@property
def read_mostly(self) -> bool:
Expand Down
Loading
Loading