diff --git a/accelerator/cpu_accelerator.py b/accelerator/cpu_accelerator.py index c089ac07fb69..d1f3f89912ad 100644 --- a/accelerator/cpu_accelerator.py +++ b/accelerator/cpu_accelerator.py @@ -316,9 +316,9 @@ def get_op_builder(self, class_name): # is op_builder from deepspeed or a 3p version? this should only succeed if it's deepspeed # if successful this also means we're doing a local install and not JIT compile path from op_builder import __deepspeed__ # noqa: F401 # type: ignore - from op_builder.cpu import AsyncIOBuilder, CCLCommBuilder, ShareMemCommBuilder, FusedAdamBuilder, CPUAdamBuilder, NotImplementedBuilder + from op_builder.cpu import AsyncIOBuilder, CCLCommBuilder, ShareMemCommBuilder, FusedAdamBuilder, CPUAdamBuilder, PinMemoryBuilder, NotImplementedBuilder except ImportError: - from deepspeed.ops.op_builder.cpu import AsyncIOBuilder, CCLCommBuilder, ShareMemCommBuilder, FusedAdamBuilder, CPUAdamBuilder, NotImplementedBuilder + from deepspeed.ops.op_builder.cpu import AsyncIOBuilder, CCLCommBuilder, ShareMemCommBuilder, FusedAdamBuilder, CPUAdamBuilder, PinMemoryBuilder, NotImplementedBuilder if class_name == "CCLCommBuilder": return CCLCommBuilder @@ -330,6 +330,8 @@ def get_op_builder(self, class_name): return CPUAdamBuilder elif class_name == "AsyncIOBuilder": return AsyncIOBuilder + elif class_name == "PinMemoryBuilder": + return PinMemoryBuilder else: # return a NotImplementedBuilder to avoid get NoneType[Name] in unit tests return NotImplementedBuilder diff --git a/accelerator/xpu_accelerator.py b/accelerator/xpu_accelerator.py index a3332df75f06..aecf96161ee3 100644 --- a/accelerator/xpu_accelerator.py +++ b/accelerator/xpu_accelerator.py @@ -233,8 +233,9 @@ def _pin_memory(self, tensor, align_bytes=1): if align_bytes == 1: return tensor.pin_memory(device=self.current_device_name()) elif align_bytes == 0: - from deepspeed.ops.op_builder.xpu import AsyncIOBuilder - self.aio_handle = AsyncIOBuilder().load().aio_handle(128 * 1024, 8, False, False, False) + # Page-locked host buffer without starting AIO worker threads. + from deepspeed.ops.op_builder import PinMemoryBuilder + self.aio_handle = PinMemoryBuilder().load().pin_handle() aligned_t = self.aio_handle.new_cpu_locked_tensor(tensor.numel(), tensor) aligned_t = aligned_t[:tensor.numel()].copy_(tensor) self.aligned_tensors.append([aligned_t.data_ptr(), aligned_t[-1].data_ptr()]) diff --git a/csrc/aio/common/deepspeed_aio_utils.cpp b/csrc/aio/common/deepspeed_aio_utils.cpp index c8e577f299ae..8c85e1c9c116 100644 --- a/csrc/aio/common/deepspeed_aio_utils.cpp +++ b/csrc/aio/common/deepspeed_aio_utils.cpp @@ -116,25 +116,3 @@ int64_t get_fd_file_size(const int fd, int64_t& size) size = st.st_size; return 0; } - -void* ds_page_aligned_alloc(const int64_t size, const bool lock) -{ - void* ptr; - int retval; - - retval = posix_memalign(&ptr, (size_t)sysconf(_SC_PAGESIZE), size); - if (retval) { return nullptr; } - - if (lock == false) { return ptr; } - - auto mlock_ret = mlock(ptr, size); - if (mlock_ret != 0) { - auto mlock_error = errno; - std::cerr << "mlock failed to allocate " << size << " bytes with error no " << mlock_error - << " msg " << strerror(mlock_error) << std::endl; - free(ptr); - return nullptr; - } - - return ptr; -} diff --git a/csrc/aio/common/deepspeed_aio_utils.h b/csrc/aio/common/deepspeed_aio_utils.h index 8742bf5bff54..d951bbd9f158 100644 --- a/csrc/aio/common/deepspeed_aio_utils.h +++ b/csrc/aio/common/deepspeed_aio_utils.h @@ -76,7 +76,5 @@ struct io_prep_generator { int prep_iocbs(const int n_iocbs, std::vector* iocbs); }; -void* ds_page_aligned_alloc(const int64_t size, const bool lock = false); - int64_t get_file_size(const char* filename, int64_t& size); int64_t get_fd_file_size(const int fd, int64_t& size); diff --git a/csrc/aio/py_lib/deepspeed_pin_tensor.h b/csrc/aio/py_lib/deepspeed_pin_tensor.h index 2370121a5f0e..61ce2350ba11 100644 --- a/csrc/aio/py_lib/deepspeed_pin_tensor.h +++ b/csrc/aio/py_lib/deepspeed_pin_tensor.h @@ -1,37 +1,11 @@ -// Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 // DeepSpeed Team /* -Functionality for managing CPU tensors occupying page-locked memory. -TODO: Implement a full-featured manager that -1. Avoid page-locked memory leaks -2. Minimize page-locked memory usage by reducing internal fragmentation -Functionality for managing CPU tensors occupying page-locked memory. +Compatibility include: pin-tensor manager lives under csrc/pin_memory/. */ -#include -#include -#include -#include "deepspeed_py_aio.h" +#pragma once -struct deepspeed_pin_tensor_t { - std::map _locked_tensors; - std::mutex _mutex; - - deepspeed_pin_tensor_t() = default; - - ~deepspeed_pin_tensor_t(); - - // Process-wide shared manager so that pinned-buffer recognition is consistent - // across every io handle (each handle references this single instance). - static std::shared_ptr shared(); - - torch::Tensor alloc(const int64_t num_elem, const at::ScalarType& elem_type); - torch::Tensor alloc(const int64_t num_elem, const torch::TensorOptions& options); - - bool free(torch::Tensor& locked_tensor); - - bool is_managed(const torch::Tensor& buffer); -}; +#include "../../pin_memory/deepspeed_pin_tensor.h" diff --git a/csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp b/csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp new file mode 100644 index 000000000000..fb9cd05aa852 --- /dev/null +++ b/csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 + +// DeepSpeed Team + +/* +Resolve the process-wide pin-tensor manager exported by the pin_memory op. +The manager is compiled only into pin_memory; async_io/gds must load that op +first (see AsyncIOBuilder.load) so this symbol is visible via RTLD_GLOBAL. +*/ + +#include "deepspeed_pin_tensor.h" + +#include +#include + +std::shared_ptr deepspeed_pin_tensor_t::shared() +{ + using holder_fn_t = void* (*)(); + auto* fn = + reinterpret_cast(dlsym(RTLD_DEFAULT, "deepspeed_pin_tensor_mgr_holder")); + if (fn == nullptr) { + throw std::runtime_error( + "DeepSpeed pin_memory op must be loaded before async_io/gds (missing " + "deepspeed_pin_tensor_mgr_holder). Load PinMemoryBuilder first."); + } + auto* holder = static_cast*>(fn()); + return *holder; +} diff --git a/csrc/aio/py_lib/deepspeed_pin_tensor.cpp b/csrc/pin_memory/deepspeed_pin_tensor.cpp similarity index 85% rename from csrc/aio/py_lib/deepspeed_pin_tensor.cpp rename to csrc/pin_memory/deepspeed_pin_tensor.cpp index e9e2337a3273..9c4d1c0ad5cd 100644 --- a/csrc/aio/py_lib/deepspeed_pin_tensor.cpp +++ b/csrc/pin_memory/deepspeed_pin_tensor.cpp @@ -1,4 +1,3 @@ -// Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 // DeepSpeed Team @@ -8,6 +7,11 @@ Functionality for managing CPU tensors occupying page-locked memory. */ #include "deepspeed_pin_tensor.h" +#include "page_alloc.h" + +#include + +#include using namespace std; @@ -26,6 +30,15 @@ std::shared_ptr deepspeed_pin_tensor_t::shared() return mgr; } +extern "C" void* deepspeed_pin_tensor_mgr_holder() +{ + // Heap-allocate the shared_ptr so its control block outlives any transient + // copies made by other extensions that resolve this symbol via dlsym. + static auto* holder = + new std::shared_ptr(deepspeed_pin_tensor_t::shared()); + return static_cast(holder); +} + torch::Tensor deepspeed_pin_tensor_t::alloc(const int64_t num_elem, const torch::TensorOptions& options) { diff --git a/csrc/pin_memory/deepspeed_pin_tensor.h b/csrc/pin_memory/deepspeed_pin_tensor.h new file mode 100644 index 000000000000..97457fb09be0 --- /dev/null +++ b/csrc/pin_memory/deepspeed_pin_tensor.h @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 + +// DeepSpeed Team + +/* +Functionality for managing CPU tensors occupying page-locked memory. +TODO: Implement a full-featured manager that +1. Avoid page-locked memory leaks +2. Minimize page-locked memory usage by reducing internal fragmentation +*/ + +#pragma once + +#include +#include +#include +#include + +struct deepspeed_pin_tensor_t { + std::map _locked_tensors; + std::mutex _mutex; + + deepspeed_pin_tensor_t() = default; + + ~deepspeed_pin_tensor_t(); + + // Process-wide shared manager so that pinned-buffer recognition is consistent + // across every io handle (each handle references this single instance). + // Canonical instance lives in the pin_memory extension; other ops resolve it + // via deepspeed_pin_tensor_mgr_holder() after that extension is loaded. + static std::shared_ptr shared(); + + torch::Tensor alloc(const int64_t num_elem, const at::ScalarType& elem_type); + torch::Tensor alloc(const int64_t num_elem, const torch::TensorOptions& options); + + bool free(torch::Tensor& locked_tensor); + + bool is_managed(const torch::Tensor& buffer); +}; + +// Exported so async_io/gds can resolve the same manager across .so boundaries. +extern "C" void* deepspeed_pin_tensor_mgr_holder(); diff --git a/csrc/pin_memory/page_alloc.cpp b/csrc/pin_memory/page_alloc.cpp new file mode 100644 index 000000000000..ff2f769b0626 --- /dev/null +++ b/csrc/pin_memory/page_alloc.cpp @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 + +// DeepSpeed Team + +#include "page_alloc.h" + +#include +#include +#include +#include +#include +#include + +void* ds_page_aligned_alloc(const int64_t size, const bool lock) +{ + void* ptr; + int retval; + + retval = posix_memalign(&ptr, (size_t)sysconf(_SC_PAGESIZE), size); + if (retval) { return nullptr; } + + if (lock == false) { return ptr; } + + auto mlock_ret = mlock(ptr, size); + if (mlock_ret != 0) { + auto mlock_error = errno; + std::cerr << "mlock failed to allocate " << size << " bytes with error no " << mlock_error + << " msg " << strerror(mlock_error) << std::endl; + free(ptr); + return nullptr; + } + + return ptr; +} diff --git a/csrc/pin_memory/page_alloc.h b/csrc/pin_memory/page_alloc.h new file mode 100644 index 000000000000..9164b446d36c --- /dev/null +++ b/csrc/pin_memory/page_alloc.h @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 + +// DeepSpeed Team + +/* +Page-aligned host allocation with optional mlock, independent of libaio. +*/ + +#pragma once + +#include + +void* ds_page_aligned_alloc(const int64_t size, const bool lock = false); diff --git a/csrc/pin_memory/py_ds_pin_memory.cpp b/csrc/pin_memory/py_ds_pin_memory.cpp new file mode 100644 index 000000000000..52a6e8d9a69e --- /dev/null +++ b/csrc/pin_memory/py_ds_pin_memory.cpp @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 + +// DeepSpeed Team + +#include +#include "deepspeed_pin_tensor.h" + +using namespace pybind11::literals; + +struct deepspeed_pin_handle_t { + std::shared_ptr _pinned_tensor_mgr; + + deepspeed_pin_handle_t() : _pinned_tensor_mgr(deepspeed_pin_tensor_t::shared()) {} + + torch::Tensor new_cpu_locked_tensor(const int64_t num_elem, const torch::Tensor& example_tensor) + { + return _pinned_tensor_mgr->alloc(num_elem, example_tensor.options()); + } + + bool free_cpu_locked_tensor(torch::Tensor& locked_tensor) + { + return _pinned_tensor_mgr->free(locked_tensor); + } + + bool is_pinned(const torch::Tensor& buffer) + { + return buffer.is_pinned() || _pinned_tensor_mgr->is_managed(buffer); + } +}; + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) +{ + py::class_(m, "pin_handle") + .def(py::init<>(), "Pin-memory handle constructor") + .def("new_cpu_locked_tensor", + &deepspeed_pin_handle_t::new_cpu_locked_tensor, + "Allocate a page-locked CPU tensor", + "num_elem"_a, + "example_tensor"_a) + .def("free_cpu_locked_tensor", + &deepspeed_pin_handle_t::free_cpu_locked_tensor, + "Free a page-locked CPU tensor", + "locked_tensor"_a) + .def("is_pinned", + &deepspeed_pin_handle_t::is_pinned, + "Return whether buffer is torch-pinned or managed by DeepSpeed pin_memory", + "buffer"_a); +} diff --git a/deepspeed/ops/pin_memory/__init__.py b/deepspeed/ops/pin_memory/__init__.py new file mode 100644 index 000000000000..16c0611c9105 --- /dev/null +++ b/deepspeed/ops/pin_memory/__init__.py @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +from ..op_builder.pin_memory import PinMemoryBuilder diff --git a/docs/_tutorials/deepnvme.md b/docs/_tutorials/deepnvme.md index a6d4545815dc..a2522469dc8c 100644 --- a/docs/_tutorials/deepnvme.md +++ b/docs/_tutorials/deepnvme.md @@ -146,6 +146,21 @@ True 1073741824 ``` +You can also allocate page-locked host tensors directly through the standalone `pin_memory` operator, which pins CPU memory (`posix_memalign` + `mlock`). This is useful on CPU-only hosts or when you want a pinned buffer without starting an I/O handle. Buffers allocated this way are recognized by DeepNVMe I/O handles (they share one process-wide pin manager), so DeepNVMe can still skip bounce buffers for them. + +```bash +>>> import torch +>>> from deepspeed.ops.op_builder import PinMemoryBuilder +>>> pin = PinMemoryBuilder().load().pin_handle() +>>> t = pin.new_cpu_locked_tensor(1024**3, torch.empty(0, dtype=torch.uint8)) +>>> pin.is_pinned(t) +True +>>> pin.free_cpu_locked_tensor(t) +True +``` + +The `pin_handle` exposes `new_cpu_locked_tensor(num_elem, example_tensor)`, `free_cpu_locked_tensor(tensor)`, and `is_pinned(tensor)`. The same methods remain available on DeepNVMe I/O handles as thin wrappers, so existing code continues to work unchanged. + On the other hand,`gds_handle` provides `new_pinned_device_tensor()` and `pin_device_tensor()` functions for pinning CUDA tensors. The following example illustrates writing a pinned CUDA tensor to a local NVMe file. ```bash diff --git a/docs/code-docs/source/memory.rst b/docs/code-docs/source/memory.rst index 9f05495001c6..769943878a40 100644 --- a/docs/code-docs/source/memory.rst +++ b/docs/code-docs/source/memory.rst @@ -265,7 +265,7 @@ Note about gradients: While gradients are stored in fp16 (2 bytes), during the w **Pinned Memory** -Pinned general RAM is included in normal general RAM allocations (i.e. this is not extra memory allocations but simply shows how much of the general RAM is pinned). Pinning is controlled by the ``pin_memory`` field of ``offload_optimizer`` / ``offload_param`` (both default to ``true``); set to ``false`` on hosts with tight memlock limits (``ulimit -l``). +Pinned general RAM is included in normal general RAM allocations (i.e. this is not extra memory allocations but simply shows how much of the general RAM is pinned). Pinning is controlled by the ``pin_memory`` field of ``offload_optimizer`` / ``offload_param`` (both default to ``true``); set to ``false`` on hosts with tight memlock limits (``ulimit -l``). See :ref:`host-memory-pinning` for how host memory is page-locked (accelerator / torch pinning and the DeepSpeed ``pin_memory`` op). * ZeRO-1/2: controlled by ``offload_optimizer.pin_memory`` @@ -288,3 +288,91 @@ With pinning enabled there are 2 sub-cases: XXX: For Transformers is probably around (2* seq * attn_heads + 16 * hidden_size) * sequence * batch/gpu This needs to be completed. + + +.. _host-memory-pinning: + +Host Memory Pinning +----------------------- + +DeepSpeed page-locks (pins) host memory so DMA engines can move data efficiently +between CPU RAM and accelerators or NVMe. Host tensors can be pinned either with +PyTorch / the accelerator hook, or with the standalone DeepSpeed ``pin_memory`` +operator (``posix_memalign`` + ``mlock``). + +Accelerator API (torch path) +============================ + +.. code-block:: python + + from deepspeed.accelerator import get_accelerator + + pinned = get_accelerator().pin_memory(tensor) + assert get_accelerator().is_pinned(pinned) + +On most accelerators this routes to ``tensor.pin_memory()``. The CPU accelerator +historically treats this as a no-op (torch cannot pin a CPU tensor to a device). + +DeepSpeed ``pin_memory`` op +========================== + +Allocate and free page-locked host tensors through ``PinMemoryBuilder`` / +``pin_handle``: + +.. code-block:: python + + import torch + from deepspeed.ops.op_builder import PinMemoryBuilder + + pin = PinMemoryBuilder().load().pin_handle() + t = pin.new_cpu_locked_tensor(1024**3, torch.empty(0, dtype=torch.uint8)) + assert pin.is_pinned(t) + pin.free_cpu_locked_tensor(t) + +``pin_handle`` exposes: + +* ``new_cpu_locked_tensor(num_elem, example_tensor)`` — allocate a page-locked CPU tensor +* ``free_cpu_locked_tensor(tensor)`` — unlock and free the allocation +* ``is_pinned(tensor)`` — ``True`` for torch-pinned buffers and for tensors whose + storage falls inside a DeepSpeed-managed locked range (slices/views included) + +The same three methods remain available on DeepNVMe I/O handles as thin wrappers. +Buffers allocated via ``pin_handle`` and via an I/O handle share one process-wide +pin manager, so DeepNVMe bounce-buffer skipping stays consistent across both paths. + +.. list-table:: Differences between torch pinning and the DeepSpeed ``pin_memory`` op + :header-rows: 1 + :widths: 22 39 39 + + * - Aspect + - Torch / accelerator ``pin_memory`` + - DeepSpeed ``pin_memory`` op + * - Allocator + - ``torch.Tensor.pin_memory()`` (device-specific accelerator hook) + - ``posix_memalign`` + ``mlock`` via ``PinMemoryBuilder`` / ``pin_handle`` + * - Build dependency + - None beyond the active accelerator / PyTorch + - DeepSpeed **pin_memory** op must build and load + * - Visible to DeepNVMe + - Yes (``is_pinned`` / bounce-buffer skip on I/O handles) + - Yes (same process-wide pin manager) + * - ``is_pinned`` + - Torch pinned status (``tensor.is_pinned()``) + - Torch-pinned **or** DeepSpeed-managed locked range (slices/views included) + * - Explicit free + - Freed when the tensor is garbage-collected (PyTorch) + - ``free_cpu_locked_tensor`` releases the mlocked pages immediately + * - CPU accelerator + - Historical no-op (torch cannot pin CPU tensors) + - Real ``mlock`` pins on CPU-only hosts + +Requirements +============ + +* The DeepSpeed **pin_memory** op must build and load successfully. +* The process must be allowed to lock the requested host memory (see + ``ulimit -l`` / ``memlock`` limits). Large ZeRO CPU-offload or DeepNVMe + footprints can otherwise hit the memlock ceiling. + +See also the DeepNVMe tutorial for end-to-end examples of pinning host and device +tensors for NVMe I/O. diff --git a/op_builder/async_io.py b/op_builder/async_io.py index f59cc6810c6f..196e7a80e68c 100644 --- a/op_builder/async_io.py +++ b/op_builder/async_io.py @@ -27,7 +27,7 @@ def lib_sources(self): 'csrc/aio/common/deepspeed_aio_utils.cpp', 'csrc/aio/common/deepspeed_aio_common.cpp', 'csrc/aio/common/deepspeed_aio_types.cpp', 'csrc/aio/py_lib/deepspeed_cpu_op.cpp', 'csrc/aio/py_lib/deepspeed_aio_op_desc.cpp', 'csrc/aio/py_lib/deepspeed_py_copy.cpp', - 'csrc/aio/py_lib/deepspeed_pin_tensor.cpp' + 'csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp' ] return src_list @@ -46,7 +46,7 @@ def include_paths(self): os.path.join(torch.utils.cpp_extension.ROCM_HOME, "include", "rocrand"), os.path.join(torch.utils.cpp_extension.ROCM_HOME, "include", "hiprand"), ] - return ['csrc/aio/py_lib', 'csrc/aio/common'] + CUDA_INCLUDE + return ['csrc/aio/py_lib', 'csrc/aio/common', 'csrc/pin_memory'] + CUDA_INCLUDE def cxx_args(self): # -O0 for improved debugging, since performance is bound by I/O @@ -93,6 +93,13 @@ def check_for_libaio_pkg(self): break return found + def load(self, verbose=False): + # Pin manager is compiled only into pin_memory; load it first so aio can + # resolve the shared manager (and related symbols) across .so boundaries. + from .pin_memory import PinMemoryBuilder + PinMemoryBuilder().load(verbose=verbose) + return super().load(verbose=verbose) + def is_compatible(self, verbose=False): # Check for the existence of libaio by using distutils # to compile and link a test program that calls io_submit, diff --git a/op_builder/cpu/__init__.py b/op_builder/cpu/__init__.py index 7084db8469f1..5bb6f028a05f 100644 --- a/op_builder/cpu/__init__.py +++ b/op_builder/cpu/__init__.py @@ -9,3 +9,4 @@ from .cpu_adam import CPUAdamBuilder from .no_impl import NotImplementedBuilder from .async_io import AsyncIOBuilder +from .pin_memory import PinMemoryBuilder diff --git a/op_builder/cpu/async_io.py b/op_builder/cpu/async_io.py index dcb9feabcfc3..e6f0afca698d 100644 --- a/op_builder/cpu/async_io.py +++ b/op_builder/cpu/async_io.py @@ -26,7 +26,7 @@ def lib_sources(self): 'csrc/aio/common/deepspeed_aio_utils.cpp', 'csrc/aio/common/deepspeed_aio_common.cpp', 'csrc/aio/common/deepspeed_aio_types.cpp', 'csrc/aio/py_lib/deepspeed_cpu_op.cpp', 'csrc/aio/py_lib/deepspeed_aio_op_desc.cpp', 'csrc/aio/py_lib/deepspeed_py_copy.cpp', - 'csrc/aio/py_lib/deepspeed_pin_tensor.cpp' + 'csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp' ] return src_list @@ -34,7 +34,7 @@ def sources(self): return self.lib_sources() + ['csrc/aio/py_lib/py_ds_aio.cpp'] def include_paths(self): - return ['csrc/aio/py_lib', 'csrc/aio/common'] + return ['csrc/aio/py_lib', 'csrc/aio/common', 'csrc/pin_memory'] def cxx_args(self): # -O0 for improved debugging, since performance is bound by I/O @@ -71,6 +71,13 @@ def check_for_libaio_pkg(self): break return found + def load(self, verbose=False): + # Pin manager is compiled only into pin_memory; load it first so aio can + # resolve the shared manager (and related symbols) across .so boundaries. + from .pin_memory import PinMemoryBuilder + PinMemoryBuilder().load(verbose=verbose) + return super().load(verbose=verbose) + def is_compatible(self, verbose=False): # Check for the existence of libaio by using distutils # to compile and link a test program that calls io_submit, diff --git a/op_builder/cpu/pin_memory.py b/op_builder/cpu/pin_memory.py new file mode 100644 index 000000000000..f59e8dbb1de8 --- /dev/null +++ b/op_builder/cpu/pin_memory.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# Host pin_memory is accelerator-agnostic; reuse the top-level builder. +try: + from op_builder.pin_memory import PinMemoryBuilder +except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + +__all__ = ["PinMemoryBuilder"] diff --git a/op_builder/gds.py b/op_builder/gds.py index d815ccea387a..bf0a5e137bbc 100644 --- a/op_builder/gds.py +++ b/op_builder/gds.py @@ -30,7 +30,7 @@ def cxx_args(self): def include_paths(self): import torch CUDA_INCLUDE = [os.path.join(torch.utils.cpp_extension.CUDA_HOME, "include")] - return ['csrc/aio/py_lib', 'csrc/aio/common'] + CUDA_INCLUDE + return ['csrc/aio/py_lib', 'csrc/aio/common', 'csrc/pin_memory'] + CUDA_INCLUDE def extra_ldflags(self): return super().extra_ldflags() + ['-lcufile'] diff --git a/op_builder/npu/__init__.py b/op_builder/npu/__init__.py index e512e25a2a29..608fa7a4d232 100644 --- a/op_builder/npu/__init__.py +++ b/op_builder/npu/__init__.py @@ -6,6 +6,7 @@ from .fused_adam import FusedAdamBuilder from .async_io import AsyncIOBuilder +from .pin_memory import PinMemoryBuilder from .no_impl import NotImplementedBuilder from .cpu_adam import CPUAdamBuilder from .cpu_adagrad import CPUAdagradBuilder diff --git a/op_builder/npu/async_io.py b/op_builder/npu/async_io.py index b87495758f0b..30bf79bc68c7 100644 --- a/op_builder/npu/async_io.py +++ b/op_builder/npu/async_io.py @@ -25,13 +25,13 @@ def sources(self): 'csrc/aio/py_lib/deepspeed_py_aio.cpp', 'csrc/aio/py_lib/deepspeed_py_aio_handle.cpp', 'csrc/aio/py_lib/deepspeed_aio_thread.cpp', 'csrc/aio/common/deepspeed_aio_utils.cpp', 'csrc/aio/common/deepspeed_aio_common.cpp', 'csrc/aio/common/deepspeed_aio_types.cpp', - 'csrc/aio/py_lib/deepspeed_pin_tensor.cpp', 'csrc/aio/py_lib/deepspeed_py_io_handle.cpp', + 'csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp', 'csrc/aio/py_lib/deepspeed_py_io_handle.cpp', 'csrc/aio/py_lib/deepspeed_aio_op_desc.cpp', 'csrc/aio/py_lib/deepspeed_cpu_op.cpp' ] def include_paths(self): args = super().include_paths() - args += ['csrc/aio/py_lib', 'csrc/aio/common'] + args += ['csrc/aio/py_lib', 'csrc/aio/common', 'csrc/pin_memory'] return args def cxx_args(self): @@ -84,6 +84,16 @@ def check_for_libaio_pkg(self): break return found + def load(self, verbose=False): + # Pin manager is compiled only into pin_memory; load it first so aio can + # resolve the shared manager (and related symbols) across .so boundaries. + try: + from op_builder.pin_memory import PinMemoryBuilder + except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + PinMemoryBuilder().load(verbose=verbose) + return super().load(verbose=verbose) + def is_compatible(self, verbose=False): # Check for the existence of libaio by using distutils # to compile and link a test program that calls io_submit, diff --git a/op_builder/npu/pin_memory.py b/op_builder/npu/pin_memory.py new file mode 100644 index 000000000000..f59e8dbb1de8 --- /dev/null +++ b/op_builder/npu/pin_memory.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# Host pin_memory is accelerator-agnostic; reuse the top-level builder. +try: + from op_builder.pin_memory import PinMemoryBuilder +except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + +__all__ = ["PinMemoryBuilder"] diff --git a/op_builder/pin_memory.py b/op_builder/pin_memory.py new file mode 100644 index 000000000000..339dccffdbee --- /dev/null +++ b/op_builder/pin_memory.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +import sys + +from .builder import OpBuilder +from .pin_memory_load import load_pin_memory_module + + +class PinMemoryBuilder(OpBuilder): + BUILD_VAR = "DS_BUILD_PIN_MEMORY" + NAME = "pin_memory" + + def __init__(self): + super().__init__(name=self.NAME) + + def absolute_name(self): + return f'deepspeed.ops.pin_memory.{self.NAME}_op' + + def is_compatible(self, verbose=False): + # The allocator relies on POSIX mlock/posix_memalign, which are unavailable on Windows. + if sys.platform == "win32": + if verbose: + self.warning(f"{self.NAME} is only supported on POSIX platforms, not Windows.") + return False + return super().is_compatible(verbose) + + def sources(self): + return [ + 'csrc/pin_memory/page_alloc.cpp', + 'csrc/pin_memory/deepspeed_pin_tensor.cpp', + 'csrc/pin_memory/py_ds_pin_memory.cpp', + ] + + def include_paths(self): + return ['csrc/pin_memory'] + + def load(self, verbose=False): + return load_pin_memory_module(self, verbose=verbose) diff --git a/op_builder/pin_memory_load.py b/op_builder/pin_memory_load.py new file mode 100644 index 000000000000..179b4b38ab3e --- /dev/null +++ b/op_builder/pin_memory_load.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +import ctypes +import sys + +# Process-wide cache so pin_memory is loaded once even if OpBuilder is imported +# under both ``op_builder`` and ``deepspeed.ops.op_builder`` (separate _loaded_ops). +_SYS_CACHE_ATTR = "_deepspeed_pin_memory_op_module" + + +def load_pin_memory_module(builder, verbose=False): + module = getattr(sys, _SYS_CACHE_ATTR, None) + if module is None: + # Call OpBuilder.load without going through subclass load() recursion. + module = super(type(builder), builder).load(verbose=verbose) + setattr(sys, _SYS_CACHE_ATTR, module) + so_path = getattr(module, "__file__", None) + if so_path: + ctypes.CDLL(so_path, mode=ctypes.RTLD_GLOBAL) + return module diff --git a/op_builder/supa/__init__.py b/op_builder/supa/__init__.py index 3986207db057..9cf71a1b39a9 100644 --- a/op_builder/supa/__init__.py +++ b/op_builder/supa/__init__.py @@ -8,6 +8,7 @@ from .inference import InferenceBuilder from .quantizer import QuantizerBuilder from .async_io import AsyncIOBuilder +from .pin_memory import PinMemoryBuilder from .no_impl import NotImplementedBuilder from .cpu_adam import CPUAdamBuilder from .cpu_lion import CPULionBuilder diff --git a/op_builder/supa/async_io.py b/op_builder/supa/async_io.py index 054eb16d3af4..ed98b1253cda 100644 --- a/op_builder/supa/async_io.py +++ b/op_builder/supa/async_io.py @@ -27,7 +27,7 @@ def sources(self): 'csrc/aio/common/deepspeed_aio_utils.cpp', 'csrc/aio/common/deepspeed_aio_common.cpp', 'csrc/aio/common/deepspeed_aio_types.cpp', - 'csrc/aio/py_lib/deepspeed_pin_tensor.cpp', + 'csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp', 'csrc/aio/py_lib/deepspeed_py_io_handle.cpp', 'csrc/aio/py_lib/deepspeed_cpu_op.cpp', 'csrc/aio/py_lib/deepspeed_aio_op_desc.cpp', @@ -35,7 +35,7 @@ def sources(self): def include_paths(self): args = super().include_paths() - args += ['csrc/aio/py_lib', 'csrc/aio/common'] + args += ['csrc/aio/py_lib', 'csrc/aio/common', 'csrc/pin_memory'] return args def cxx_args(self): @@ -67,6 +67,16 @@ def check_for_libaio_pkg(self): break return found + def load(self, verbose=False): + # Pin manager is compiled only into pin_memory; load it first so aio can + # resolve the shared manager (and related symbols) across .so boundaries. + try: + from op_builder.pin_memory import PinMemoryBuilder + except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + PinMemoryBuilder().load(verbose=verbose) + return super().load(verbose=verbose) + def is_compatible(self, verbose=False): # Check for the existence of libaio by using distutils # to compile and link a test program that calls io_submit, diff --git a/op_builder/supa/pin_memory.py b/op_builder/supa/pin_memory.py new file mode 100644 index 000000000000..f59e8dbb1de8 --- /dev/null +++ b/op_builder/supa/pin_memory.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# Host pin_memory is accelerator-agnostic; reuse the top-level builder. +try: + from op_builder.pin_memory import PinMemoryBuilder +except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + +__all__ = ["PinMemoryBuilder"] diff --git a/op_builder/xpu/__init__.py b/op_builder/xpu/__init__.py index a91775a3bb13..400c06e23061 100755 --- a/op_builder/xpu/__init__.py +++ b/op_builder/xpu/__init__.py @@ -7,6 +7,7 @@ from .cpu_adagrad import CPUAdagradBuilder from .fused_adam import FusedAdamBuilder from .async_io import AsyncIOBuilder +from .pin_memory import PinMemoryBuilder from .flash_attn import FlashAttentionBuilder from .no_impl import NotImplementedBuilder from .packbits import PackbitsBuilder diff --git a/op_builder/xpu/async_io.py b/op_builder/xpu/async_io.py index 8ec030880368..fd60e1747f28 100644 --- a/op_builder/xpu/async_io.py +++ b/op_builder/xpu/async_io.py @@ -29,14 +29,14 @@ def sources(self): 'csrc/aio/common/deepspeed_aio_utils.cpp', 'csrc/aio/common/deepspeed_aio_common.cpp', 'csrc/aio/common/deepspeed_aio_types.cpp', - 'csrc/aio/py_lib/deepspeed_pin_tensor.cpp', + 'csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp', 'csrc/aio/py_lib/deepspeed_py_io_handle.cpp', 'csrc/aio/py_lib/deepspeed_cpu_op.cpp', 'csrc/aio/py_lib/deepspeed_aio_op_desc.cpp', ] def include_paths(self): - return ['csrc/aio/py_lib', 'csrc/aio/common'] + return ['csrc/aio/py_lib', 'csrc/aio/common', 'csrc/pin_memory'] def cxx_args(self): import torch @@ -86,6 +86,16 @@ def check_for_libaio_pkg(self): break return found + def load(self, verbose=False): + # Pin manager is compiled only into pin_memory; load it first so aio can + # resolve the shared manager (and related symbols) across .so boundaries. + try: + from op_builder.pin_memory import PinMemoryBuilder + except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + PinMemoryBuilder().load(verbose=verbose) + return super().load(verbose=verbose) + def is_compatible(self, verbose=False): # Check for the existence of libaio by using distutils # to compile and link a test program that calls io_submit, diff --git a/op_builder/xpu/pin_memory.py b/op_builder/xpu/pin_memory.py new file mode 100644 index 000000000000..f59e8dbb1de8 --- /dev/null +++ b/op_builder/xpu/pin_memory.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# Host pin_memory is accelerator-agnostic; reuse the top-level builder. +try: + from op_builder.pin_memory import PinMemoryBuilder +except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + +__all__ = ["PinMemoryBuilder"] diff --git a/setup.py b/setup.py index b383e5271381..5a2906a352c5 100755 --- a/setup.py +++ b/setup.py @@ -201,6 +201,21 @@ def op_enabled(op_name): install_ops[op_name] = op_enabled(op_name) ext_modules.append(builder.builder()) +# async_io/gds load the pin_memory op's shared manager at runtime. If they are +# precompiled but pin_memory is not, deployments without a compiler would hit a +# runtime JIT build, so precompile pin_memory alongside them. +pin_memory_dependents = [name for name in ("async_io", "gds") if install_ops.get(name)] +if pin_memory_dependents and not install_ops.get("pin_memory"): + pin_memory_builder = ALL_OPS.get("pin_memory") + if pin_memory_builder is not None and pin_memory_builder.is_compatible(): + assert torch_available, "Unable to pre-compile pin_memory, please first install torch" + install_ops["pin_memory"] = 1 + ext_modules.append(pin_memory_builder.builder()) + else: + raise RuntimeError( + f"Cannot pre-compile {', '.join(pin_memory_dependents)} without pin_memory, which is required at " + "runtime but is not compatible on this platform.") + print(f'Install Ops={install_ops}') # Write out version/git info. diff --git a/tests/unit/v1/nvme/test_pinned_manager.py b/tests/unit/v1/nvme/test_pinned_manager.py index 1067edfccbba..c6ac3c76d38d 100644 --- a/tests/unit/v1/nvme/test_pinned_manager.py +++ b/tests/unit/v1/nvme/test_pinned_manager.py @@ -48,3 +48,36 @@ def test_buffer_is_shared_across_handles(): def test_unmanaged_buffer_is_not_pinned(): handle = _new_handle() assert not handle.is_pinned(torch.empty(NUM_ELEMS, dtype=torch.float)) + + +def test_pin_handle_buffer_recognized_by_aio_handle(): + # Buffers allocated via the standalone pin_memory op must share the same + # process-wide manager as async_io, so aio_handle.is_pinned sees them. + from deepspeed.ops.op_builder import PinMemoryBuilder + if not deepspeed.ops.__compatible_ops__.get(PinMemoryBuilder.NAME, True): + pytest.skip('Skip tests since pin_memory is not compatible') + + pin_handle = PinMemoryBuilder().load().pin_handle() + aio_handle = _new_handle() + buffer = pin_handle.new_cpu_locked_tensor(NUM_ELEMS, torch.empty(0, dtype=torch.float)) + try: + assert pin_handle.is_pinned(buffer) + assert aio_handle.is_pinned(buffer) + assert aio_handle.is_pinned(buffer.narrow(0, BLOCK_SIZE, BLOCK_SIZE)) + finally: + pin_handle.free_cpu_locked_tensor(buffer) + + +def test_aio_handle_buffer_recognized_by_pin_handle(): + from deepspeed.ops.op_builder import PinMemoryBuilder + if not deepspeed.ops.__compatible_ops__.get(PinMemoryBuilder.NAME, True): + pytest.skip('Skip tests since pin_memory is not compatible') + + pin_handle = PinMemoryBuilder().load().pin_handle() + aio_handle = _new_handle() + buffer = aio_handle.new_cpu_locked_tensor(NUM_ELEMS, torch.empty(0, dtype=torch.float)) + try: + assert aio_handle.is_pinned(buffer) + assert pin_handle.is_pinned(buffer) + finally: + aio_handle.free_cpu_locked_tensor(buffer) diff --git a/tests/unit/v1/pin_memory/test_pin_memory_op.py b/tests/unit/v1/pin_memory/test_pin_memory_op.py new file mode 100644 index 000000000000..291d4cb2cb2d --- /dev/null +++ b/tests/unit/v1/pin_memory/test_pin_memory_op.py @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +import pytest +import torch +import deepspeed +from deepspeed.ops.op_builder import PinMemoryBuilder + +if not deepspeed.ops.__compatible_ops__.get(PinMemoryBuilder.NAME, True): + pytest.skip('Skip tests since pin_memory is not compatible', allow_module_level=True) + +NUM_ELEMS = 4096 + + +def test_pin_handle_alloc_free_and_is_pinned(): + # pin_memory must work without libaio / async_io. + handle = PinMemoryBuilder().load().pin_handle() + buffer = handle.new_cpu_locked_tensor(NUM_ELEMS, torch.empty(0, dtype=torch.float)) + try: + assert handle.is_pinned(buffer) + assert handle.is_pinned(buffer.narrow(0, 16, 16)) + assert not handle.is_pinned(torch.empty(NUM_ELEMS, dtype=torch.float)) + finally: + assert handle.free_cpu_locked_tensor(buffer) is True