From 38e2ea6bcff3b386e624db3a76422316926712db Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Sat, 8 Aug 2026 23:08:36 +0000 Subject: [PATCH 1/6] Split native host pinning into standalone pin_memory op Native page-locked allocations (posix_memalign/mlock) no longer require libaio or AIO worker threads. Compile the pin manager only in pin_memory and share it with async_io/gds so bounce-buffer recognition stays process-wide. Signed-off-by: Olatunji Ruwase Co-authored-by: Cursor --- accelerator/cpu_accelerator.py | 6 ++- accelerator/xpu_accelerator.py | 5 +- csrc/aio/common/deepspeed_aio_utils.cpp | 22 --------- csrc/aio/common/deepspeed_aio_utils.h | 2 - csrc/aio/py_lib/deepspeed_pin_tensor.h | 31 ++---------- .../py_lib/deepspeed_pin_tensor_client.cpp | 29 +++++++++++ .../deepspeed_pin_tensor.cpp | 14 ++++++ csrc/pin_memory/deepspeed_pin_tensor.h | 43 ++++++++++++++++ csrc/pin_memory/page_alloc.cpp | 35 +++++++++++++ csrc/pin_memory/page_alloc.h | 14 ++++++ csrc/pin_memory/py_ds_pin_memory.cpp | 49 +++++++++++++++++++ deepspeed/ops/pin_memory/__init__.py | 6 +++ op_builder/async_io.py | 11 ++++- op_builder/cpu/__init__.py | 1 + op_builder/cpu/async_io.py | 11 ++++- op_builder/cpu/pin_memory.py | 34 +++++++++++++ op_builder/gds.py | 2 +- op_builder/npu/__init__.py | 1 + op_builder/npu/async_io.py | 14 +++++- op_builder/npu/pin_memory.py | 12 +++++ op_builder/pin_memory.py | 31 ++++++++++++ op_builder/pin_memory_load.py | 23 +++++++++ op_builder/supa/__init__.py | 1 + op_builder/supa/async_io.py | 14 +++++- op_builder/supa/pin_memory.py | 12 +++++ op_builder/xpu/__init__.py | 1 + op_builder/xpu/async_io.py | 14 +++++- op_builder/xpu/pin_memory.py | 12 +++++ tests/unit/v1/nvme/test_pinned_manager.py | 33 +++++++++++++ .../unit/v1/pin_memory/test_pin_memory_op.py | 26 ++++++++++ 30 files changed, 442 insertions(+), 67 deletions(-) create mode 100644 csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp rename csrc/{aio/py_lib => pin_memory}/deepspeed_pin_tensor.cpp (86%) create mode 100644 csrc/pin_memory/deepspeed_pin_tensor.h create mode 100644 csrc/pin_memory/page_alloc.cpp create mode 100644 csrc/pin_memory/page_alloc.h create mode 100644 csrc/pin_memory/py_ds_pin_memory.cpp create mode 100644 deepspeed/ops/pin_memory/__init__.py create mode 100644 op_builder/cpu/pin_memory.py create mode 100644 op_builder/npu/pin_memory.py create mode 100644 op_builder/pin_memory.py create mode 100644 op_builder/pin_memory_load.py create mode 100644 op_builder/supa/pin_memory.py create mode 100644 op_builder/xpu/pin_memory.py create mode 100644 tests/unit/v1/pin_memory/test_pin_memory_op.py diff --git a/accelerator/cpu_accelerator.py b/accelerator/cpu_accelerator.py index c089ac07fb69..d1f3f89912ad 100644 --- a/accelerator/cpu_accelerator.py +++ b/accelerator/cpu_accelerator.py @@ -316,9 +316,9 @@ def get_op_builder(self, class_name): # is op_builder from deepspeed or a 3p version? this should only succeed if it's deepspeed # if successful this also means we're doing a local install and not JIT compile path from op_builder import __deepspeed__ # noqa: F401 # type: ignore - from op_builder.cpu import AsyncIOBuilder, CCLCommBuilder, ShareMemCommBuilder, FusedAdamBuilder, CPUAdamBuilder, NotImplementedBuilder + from op_builder.cpu import AsyncIOBuilder, CCLCommBuilder, ShareMemCommBuilder, FusedAdamBuilder, CPUAdamBuilder, PinMemoryBuilder, NotImplementedBuilder except ImportError: - from deepspeed.ops.op_builder.cpu import AsyncIOBuilder, CCLCommBuilder, ShareMemCommBuilder, FusedAdamBuilder, CPUAdamBuilder, NotImplementedBuilder + from deepspeed.ops.op_builder.cpu import AsyncIOBuilder, CCLCommBuilder, ShareMemCommBuilder, FusedAdamBuilder, CPUAdamBuilder, PinMemoryBuilder, NotImplementedBuilder if class_name == "CCLCommBuilder": return CCLCommBuilder @@ -330,6 +330,8 @@ def get_op_builder(self, class_name): return CPUAdamBuilder elif class_name == "AsyncIOBuilder": return AsyncIOBuilder + elif class_name == "PinMemoryBuilder": + return PinMemoryBuilder else: # return a NotImplementedBuilder to avoid get NoneType[Name] in unit tests return NotImplementedBuilder diff --git a/accelerator/xpu_accelerator.py b/accelerator/xpu_accelerator.py index a3332df75f06..aecf96161ee3 100644 --- a/accelerator/xpu_accelerator.py +++ b/accelerator/xpu_accelerator.py @@ -233,8 +233,9 @@ def _pin_memory(self, tensor, align_bytes=1): if align_bytes == 1: return tensor.pin_memory(device=self.current_device_name()) elif align_bytes == 0: - from deepspeed.ops.op_builder.xpu import AsyncIOBuilder - self.aio_handle = AsyncIOBuilder().load().aio_handle(128 * 1024, 8, False, False, False) + # Page-locked host buffer without starting AIO worker threads. + from deepspeed.ops.op_builder import PinMemoryBuilder + self.aio_handle = PinMemoryBuilder().load().pin_handle() aligned_t = self.aio_handle.new_cpu_locked_tensor(tensor.numel(), tensor) aligned_t = aligned_t[:tensor.numel()].copy_(tensor) self.aligned_tensors.append([aligned_t.data_ptr(), aligned_t[-1].data_ptr()]) diff --git a/csrc/aio/common/deepspeed_aio_utils.cpp b/csrc/aio/common/deepspeed_aio_utils.cpp index c8e577f299ae..8c85e1c9c116 100644 --- a/csrc/aio/common/deepspeed_aio_utils.cpp +++ b/csrc/aio/common/deepspeed_aio_utils.cpp @@ -116,25 +116,3 @@ int64_t get_fd_file_size(const int fd, int64_t& size) size = st.st_size; return 0; } - -void* ds_page_aligned_alloc(const int64_t size, const bool lock) -{ - void* ptr; - int retval; - - retval = posix_memalign(&ptr, (size_t)sysconf(_SC_PAGESIZE), size); - if (retval) { return nullptr; } - - if (lock == false) { return ptr; } - - auto mlock_ret = mlock(ptr, size); - if (mlock_ret != 0) { - auto mlock_error = errno; - std::cerr << "mlock failed to allocate " << size << " bytes with error no " << mlock_error - << " msg " << strerror(mlock_error) << std::endl; - free(ptr); - return nullptr; - } - - return ptr; -} diff --git a/csrc/aio/common/deepspeed_aio_utils.h b/csrc/aio/common/deepspeed_aio_utils.h index 8742bf5bff54..d951bbd9f158 100644 --- a/csrc/aio/common/deepspeed_aio_utils.h +++ b/csrc/aio/common/deepspeed_aio_utils.h @@ -76,7 +76,5 @@ struct io_prep_generator { int prep_iocbs(const int n_iocbs, std::vector* iocbs); }; -void* ds_page_aligned_alloc(const int64_t size, const bool lock = false); - int64_t get_file_size(const char* filename, int64_t& size); int64_t get_fd_file_size(const int fd, int64_t& size); diff --git a/csrc/aio/py_lib/deepspeed_pin_tensor.h b/csrc/aio/py_lib/deepspeed_pin_tensor.h index 2370121a5f0e..19fc88e1d162 100644 --- a/csrc/aio/py_lib/deepspeed_pin_tensor.h +++ b/csrc/aio/py_lib/deepspeed_pin_tensor.h @@ -4,34 +4,9 @@ // DeepSpeed Team /* -Functionality for managing CPU tensors occupying page-locked memory. -TODO: Implement a full-featured manager that -1. Avoid page-locked memory leaks -2. Minimize page-locked memory usage by reducing internal fragmentation -Functionality for managing CPU tensors occupying page-locked memory. +Compatibility include: pin-tensor manager lives under csrc/pin_memory/. */ -#include -#include -#include -#include "deepspeed_py_aio.h" +#pragma once -struct deepspeed_pin_tensor_t { - std::map _locked_tensors; - std::mutex _mutex; - - deepspeed_pin_tensor_t() = default; - - ~deepspeed_pin_tensor_t(); - - // Process-wide shared manager so that pinned-buffer recognition is consistent - // across every io handle (each handle references this single instance). - static std::shared_ptr shared(); - - torch::Tensor alloc(const int64_t num_elem, const at::ScalarType& elem_type); - torch::Tensor alloc(const int64_t num_elem, const torch::TensorOptions& options); - - bool free(torch::Tensor& locked_tensor); - - bool is_managed(const torch::Tensor& buffer); -}; +#include "../../pin_memory/deepspeed_pin_tensor.h" diff --git a/csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp b/csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp new file mode 100644 index 000000000000..3a24d633daa2 --- /dev/null +++ b/csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// DeepSpeed Team + +/* +Resolve the process-wide pin-tensor manager exported by the pin_memory op. +The manager is compiled only into pin_memory; async_io/gds must load that op +first (see AsyncIOBuilder.load) so this symbol is visible via RTLD_GLOBAL. +*/ + +#include "deepspeed_pin_tensor.h" + +#include +#include + +std::shared_ptr deepspeed_pin_tensor_t::shared() +{ + using holder_fn_t = void* (*)(); + auto* fn = + reinterpret_cast(dlsym(RTLD_DEFAULT, "deepspeed_pin_tensor_mgr_holder")); + if (fn == nullptr) { + throw std::runtime_error( + "DeepSpeed pin_memory op must be loaded before async_io/gds (missing " + "deepspeed_pin_tensor_mgr_holder). Load PinMemoryBuilder first."); + } + auto* holder = static_cast*>(fn()); + return *holder; +} diff --git a/csrc/aio/py_lib/deepspeed_pin_tensor.cpp b/csrc/pin_memory/deepspeed_pin_tensor.cpp similarity index 86% rename from csrc/aio/py_lib/deepspeed_pin_tensor.cpp rename to csrc/pin_memory/deepspeed_pin_tensor.cpp index e9e2337a3273..cb36f0fee220 100644 --- a/csrc/aio/py_lib/deepspeed_pin_tensor.cpp +++ b/csrc/pin_memory/deepspeed_pin_tensor.cpp @@ -8,6 +8,11 @@ Functionality for managing CPU tensors occupying page-locked memory. */ #include "deepspeed_pin_tensor.h" +#include "page_alloc.h" + +#include + +#include using namespace std; @@ -26,6 +31,15 @@ std::shared_ptr deepspeed_pin_tensor_t::shared() return mgr; } +extern "C" void* deepspeed_pin_tensor_mgr_holder() +{ + // Heap-allocate the shared_ptr so its control block outlives any transient + // copies made by other extensions that resolve this symbol via dlsym. + static auto* holder = + new std::shared_ptr(deepspeed_pin_tensor_t::shared()); + return static_cast(holder); +} + torch::Tensor deepspeed_pin_tensor_t::alloc(const int64_t num_elem, const torch::TensorOptions& options) { diff --git a/csrc/pin_memory/deepspeed_pin_tensor.h b/csrc/pin_memory/deepspeed_pin_tensor.h new file mode 100644 index 000000000000..e2a7a11947c7 --- /dev/null +++ b/csrc/pin_memory/deepspeed_pin_tensor.h @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// DeepSpeed Team + +/* +Functionality for managing CPU tensors occupying page-locked memory. +TODO: Implement a full-featured manager that +1. Avoid page-locked memory leaks +2. Minimize page-locked memory usage by reducing internal fragmentation +*/ + +#pragma once + +#include +#include +#include +#include + +struct deepspeed_pin_tensor_t { + std::map _locked_tensors; + std::mutex _mutex; + + deepspeed_pin_tensor_t() = default; + + ~deepspeed_pin_tensor_t(); + + // Process-wide shared manager so that pinned-buffer recognition is consistent + // across every io handle (each handle references this single instance). + // Canonical instance lives in the pin_memory extension; other ops resolve it + // via deepspeed_pin_tensor_mgr_holder() after that extension is loaded. + static std::shared_ptr shared(); + + torch::Tensor alloc(const int64_t num_elem, const at::ScalarType& elem_type); + torch::Tensor alloc(const int64_t num_elem, const torch::TensorOptions& options); + + bool free(torch::Tensor& locked_tensor); + + bool is_managed(const torch::Tensor& buffer); +}; + +// Exported so async_io/gds can resolve the same manager across .so boundaries. +extern "C" void* deepspeed_pin_tensor_mgr_holder(); diff --git a/csrc/pin_memory/page_alloc.cpp b/csrc/pin_memory/page_alloc.cpp new file mode 100644 index 000000000000..97bac5028c02 --- /dev/null +++ b/csrc/pin_memory/page_alloc.cpp @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// DeepSpeed Team + +#include "page_alloc.h" + +#include +#include +#include +#include +#include +#include + +void* ds_page_aligned_alloc(const int64_t size, const bool lock) +{ + void* ptr; + int retval; + + retval = posix_memalign(&ptr, (size_t)sysconf(_SC_PAGESIZE), size); + if (retval) { return nullptr; } + + if (lock == false) { return ptr; } + + auto mlock_ret = mlock(ptr, size); + if (mlock_ret != 0) { + auto mlock_error = errno; + std::cerr << "mlock failed to allocate " << size << " bytes with error no " << mlock_error + << " msg " << strerror(mlock_error) << std::endl; + free(ptr); + return nullptr; + } + + return ptr; +} diff --git a/csrc/pin_memory/page_alloc.h b/csrc/pin_memory/page_alloc.h new file mode 100644 index 000000000000..698bddca19ec --- /dev/null +++ b/csrc/pin_memory/page_alloc.h @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// DeepSpeed Team + +/* +Page-aligned host allocation with optional mlock, independent of libaio. +*/ + +#pragma once + +#include + +void* ds_page_aligned_alloc(const int64_t size, const bool lock = false); diff --git a/csrc/pin_memory/py_ds_pin_memory.cpp b/csrc/pin_memory/py_ds_pin_memory.cpp new file mode 100644 index 000000000000..34403a7ac44d --- /dev/null +++ b/csrc/pin_memory/py_ds_pin_memory.cpp @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// DeepSpeed Team + +#include +#include "deepspeed_pin_tensor.h" + +using namespace pybind11::literals; + +struct deepspeed_pin_handle_t { + std::shared_ptr _pinned_tensor_mgr; + + deepspeed_pin_handle_t() : _pinned_tensor_mgr(deepspeed_pin_tensor_t::shared()) {} + + torch::Tensor new_cpu_locked_tensor(const int64_t num_elem, const torch::Tensor& example_tensor) + { + return _pinned_tensor_mgr->alloc(num_elem, example_tensor.options()); + } + + bool free_cpu_locked_tensor(torch::Tensor& locked_tensor) + { + return _pinned_tensor_mgr->free(locked_tensor); + } + + bool is_pinned(const torch::Tensor& buffer) + { + return buffer.is_pinned() || _pinned_tensor_mgr->is_managed(buffer); + } +}; + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) +{ + py::class_(m, "pin_handle") + .def(py::init<>(), "Pin-memory handle constructor") + .def("new_cpu_locked_tensor", + &deepspeed_pin_handle_t::new_cpu_locked_tensor, + "Allocate a page-locked CPU tensor", + "num_elem"_a, + "example_tensor"_a) + .def("free_cpu_locked_tensor", + &deepspeed_pin_handle_t::free_cpu_locked_tensor, + "Free a page-locked CPU tensor", + "locked_tensor"_a) + .def("is_pinned", + &deepspeed_pin_handle_t::is_pinned, + "Return whether buffer is torch-pinned or managed by DeepSpeed pin_memory", + "buffer"_a); +} diff --git a/deepspeed/ops/pin_memory/__init__.py b/deepspeed/ops/pin_memory/__init__.py new file mode 100644 index 000000000000..79821a828fa2 --- /dev/null +++ b/deepspeed/ops/pin_memory/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +from ..op_builder.pin_memory import PinMemoryBuilder diff --git a/op_builder/async_io.py b/op_builder/async_io.py index f59cc6810c6f..196e7a80e68c 100644 --- a/op_builder/async_io.py +++ b/op_builder/async_io.py @@ -27,7 +27,7 @@ def lib_sources(self): 'csrc/aio/common/deepspeed_aio_utils.cpp', 'csrc/aio/common/deepspeed_aio_common.cpp', 'csrc/aio/common/deepspeed_aio_types.cpp', 'csrc/aio/py_lib/deepspeed_cpu_op.cpp', 'csrc/aio/py_lib/deepspeed_aio_op_desc.cpp', 'csrc/aio/py_lib/deepspeed_py_copy.cpp', - 'csrc/aio/py_lib/deepspeed_pin_tensor.cpp' + 'csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp' ] return src_list @@ -46,7 +46,7 @@ def include_paths(self): os.path.join(torch.utils.cpp_extension.ROCM_HOME, "include", "rocrand"), os.path.join(torch.utils.cpp_extension.ROCM_HOME, "include", "hiprand"), ] - return ['csrc/aio/py_lib', 'csrc/aio/common'] + CUDA_INCLUDE + return ['csrc/aio/py_lib', 'csrc/aio/common', 'csrc/pin_memory'] + CUDA_INCLUDE def cxx_args(self): # -O0 for improved debugging, since performance is bound by I/O @@ -93,6 +93,13 @@ def check_for_libaio_pkg(self): break return found + def load(self, verbose=False): + # Pin manager is compiled only into pin_memory; load it first so aio can + # resolve the shared manager (and related symbols) across .so boundaries. + from .pin_memory import PinMemoryBuilder + PinMemoryBuilder().load(verbose=verbose) + return super().load(verbose=verbose) + def is_compatible(self, verbose=False): # Check for the existence of libaio by using distutils # to compile and link a test program that calls io_submit, diff --git a/op_builder/cpu/__init__.py b/op_builder/cpu/__init__.py index 7084db8469f1..5bb6f028a05f 100644 --- a/op_builder/cpu/__init__.py +++ b/op_builder/cpu/__init__.py @@ -9,3 +9,4 @@ from .cpu_adam import CPUAdamBuilder from .no_impl import NotImplementedBuilder from .async_io import AsyncIOBuilder +from .pin_memory import PinMemoryBuilder diff --git a/op_builder/cpu/async_io.py b/op_builder/cpu/async_io.py index dcb9feabcfc3..e6f0afca698d 100644 --- a/op_builder/cpu/async_io.py +++ b/op_builder/cpu/async_io.py @@ -26,7 +26,7 @@ def lib_sources(self): 'csrc/aio/common/deepspeed_aio_utils.cpp', 'csrc/aio/common/deepspeed_aio_common.cpp', 'csrc/aio/common/deepspeed_aio_types.cpp', 'csrc/aio/py_lib/deepspeed_cpu_op.cpp', 'csrc/aio/py_lib/deepspeed_aio_op_desc.cpp', 'csrc/aio/py_lib/deepspeed_py_copy.cpp', - 'csrc/aio/py_lib/deepspeed_pin_tensor.cpp' + 'csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp' ] return src_list @@ -34,7 +34,7 @@ def sources(self): return self.lib_sources() + ['csrc/aio/py_lib/py_ds_aio.cpp'] def include_paths(self): - return ['csrc/aio/py_lib', 'csrc/aio/common'] + return ['csrc/aio/py_lib', 'csrc/aio/common', 'csrc/pin_memory'] def cxx_args(self): # -O0 for improved debugging, since performance is bound by I/O @@ -71,6 +71,13 @@ def check_for_libaio_pkg(self): break return found + def load(self, verbose=False): + # Pin manager is compiled only into pin_memory; load it first so aio can + # resolve the shared manager (and related symbols) across .so boundaries. + from .pin_memory import PinMemoryBuilder + PinMemoryBuilder().load(verbose=verbose) + return super().load(verbose=verbose) + def is_compatible(self, verbose=False): # Check for the existence of libaio by using distutils # to compile and link a test program that calls io_submit, diff --git a/op_builder/cpu/pin_memory.py b/op_builder/cpu/pin_memory.py new file mode 100644 index 000000000000..e0416f03d649 --- /dev/null +++ b/op_builder/cpu/pin_memory.py @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +from .builder import CPUOpBuilder +try: + from op_builder.pin_memory_load import load_pin_memory_module +except ImportError: + from deepspeed.ops.op_builder.pin_memory_load import load_pin_memory_module + + +class PinMemoryBuilder(CPUOpBuilder): + BUILD_VAR = "DS_BUILD_PIN_MEMORY" + NAME = "pin_memory" + + def __init__(self): + super().__init__(name=self.NAME) + + def absolute_name(self): + return f'deepspeed.ops.pin_memory.{self.NAME}_op' + + def sources(self): + return [ + 'csrc/pin_memory/page_alloc.cpp', + 'csrc/pin_memory/deepspeed_pin_tensor.cpp', + 'csrc/pin_memory/py_ds_pin_memory.cpp', + ] + + def include_paths(self): + return ['csrc/pin_memory'] + + def load(self, verbose=False): + return load_pin_memory_module(self, verbose=verbose) diff --git a/op_builder/gds.py b/op_builder/gds.py index d815ccea387a..bf0a5e137bbc 100644 --- a/op_builder/gds.py +++ b/op_builder/gds.py @@ -30,7 +30,7 @@ def cxx_args(self): def include_paths(self): import torch CUDA_INCLUDE = [os.path.join(torch.utils.cpp_extension.CUDA_HOME, "include")] - return ['csrc/aio/py_lib', 'csrc/aio/common'] + CUDA_INCLUDE + return ['csrc/aio/py_lib', 'csrc/aio/common', 'csrc/pin_memory'] + CUDA_INCLUDE def extra_ldflags(self): return super().extra_ldflags() + ['-lcufile'] diff --git a/op_builder/npu/__init__.py b/op_builder/npu/__init__.py index e512e25a2a29..608fa7a4d232 100644 --- a/op_builder/npu/__init__.py +++ b/op_builder/npu/__init__.py @@ -6,6 +6,7 @@ from .fused_adam import FusedAdamBuilder from .async_io import AsyncIOBuilder +from .pin_memory import PinMemoryBuilder from .no_impl import NotImplementedBuilder from .cpu_adam import CPUAdamBuilder from .cpu_adagrad import CPUAdagradBuilder diff --git a/op_builder/npu/async_io.py b/op_builder/npu/async_io.py index b87495758f0b..30bf79bc68c7 100644 --- a/op_builder/npu/async_io.py +++ b/op_builder/npu/async_io.py @@ -25,13 +25,13 @@ def sources(self): 'csrc/aio/py_lib/deepspeed_py_aio.cpp', 'csrc/aio/py_lib/deepspeed_py_aio_handle.cpp', 'csrc/aio/py_lib/deepspeed_aio_thread.cpp', 'csrc/aio/common/deepspeed_aio_utils.cpp', 'csrc/aio/common/deepspeed_aio_common.cpp', 'csrc/aio/common/deepspeed_aio_types.cpp', - 'csrc/aio/py_lib/deepspeed_pin_tensor.cpp', 'csrc/aio/py_lib/deepspeed_py_io_handle.cpp', + 'csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp', 'csrc/aio/py_lib/deepspeed_py_io_handle.cpp', 'csrc/aio/py_lib/deepspeed_aio_op_desc.cpp', 'csrc/aio/py_lib/deepspeed_cpu_op.cpp' ] def include_paths(self): args = super().include_paths() - args += ['csrc/aio/py_lib', 'csrc/aio/common'] + args += ['csrc/aio/py_lib', 'csrc/aio/common', 'csrc/pin_memory'] return args def cxx_args(self): @@ -84,6 +84,16 @@ def check_for_libaio_pkg(self): break return found + def load(self, verbose=False): + # Pin manager is compiled only into pin_memory; load it first so aio can + # resolve the shared manager (and related symbols) across .so boundaries. + try: + from op_builder.pin_memory import PinMemoryBuilder + except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + PinMemoryBuilder().load(verbose=verbose) + return super().load(verbose=verbose) + def is_compatible(self, verbose=False): # Check for the existence of libaio by using distutils # to compile and link a test program that calls io_submit, diff --git a/op_builder/npu/pin_memory.py b/op_builder/npu/pin_memory.py new file mode 100644 index 000000000000..6ec36dd8181c --- /dev/null +++ b/op_builder/npu/pin_memory.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# Host pin_memory is accelerator-agnostic; reuse the top-level builder. +try: + from op_builder.pin_memory import PinMemoryBuilder +except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + +__all__ = ["PinMemoryBuilder"] diff --git a/op_builder/pin_memory.py b/op_builder/pin_memory.py new file mode 100644 index 000000000000..c87205df63cf --- /dev/null +++ b/op_builder/pin_memory.py @@ -0,0 +1,31 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +from .builder import OpBuilder +from .pin_memory_load import load_pin_memory_module + + +class PinMemoryBuilder(OpBuilder): + BUILD_VAR = "DS_BUILD_PIN_MEMORY" + NAME = "pin_memory" + + def __init__(self): + super().__init__(name=self.NAME) + + def absolute_name(self): + return f'deepspeed.ops.pin_memory.{self.NAME}_op' + + def sources(self): + return [ + 'csrc/pin_memory/page_alloc.cpp', + 'csrc/pin_memory/deepspeed_pin_tensor.cpp', + 'csrc/pin_memory/py_ds_pin_memory.cpp', + ] + + def include_paths(self): + return ['csrc/pin_memory'] + + def load(self, verbose=False): + return load_pin_memory_module(self, verbose=verbose) diff --git a/op_builder/pin_memory_load.py b/op_builder/pin_memory_load.py new file mode 100644 index 000000000000..d3fc4f34abde --- /dev/null +++ b/op_builder/pin_memory_load.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +import ctypes +import sys + +# Process-wide cache so pin_memory is loaded once even if OpBuilder is imported +# under both ``op_builder`` and ``deepspeed.ops.op_builder`` (separate _loaded_ops). +_SYS_CACHE_ATTR = "_deepspeed_pin_memory_op_module" + + +def load_pin_memory_module(builder, verbose=False): + module = getattr(sys, _SYS_CACHE_ATTR, None) + if module is None: + # Call OpBuilder.load without going through subclass load() recursion. + module = super(type(builder), builder).load(verbose=verbose) + setattr(sys, _SYS_CACHE_ATTR, module) + so_path = getattr(module, "__file__", None) + if so_path: + ctypes.CDLL(so_path, mode=ctypes.RTLD_GLOBAL) + return module diff --git a/op_builder/supa/__init__.py b/op_builder/supa/__init__.py index 3986207db057..9cf71a1b39a9 100644 --- a/op_builder/supa/__init__.py +++ b/op_builder/supa/__init__.py @@ -8,6 +8,7 @@ from .inference import InferenceBuilder from .quantizer import QuantizerBuilder from .async_io import AsyncIOBuilder +from .pin_memory import PinMemoryBuilder from .no_impl import NotImplementedBuilder from .cpu_adam import CPUAdamBuilder from .cpu_lion import CPULionBuilder diff --git a/op_builder/supa/async_io.py b/op_builder/supa/async_io.py index 054eb16d3af4..ed98b1253cda 100644 --- a/op_builder/supa/async_io.py +++ b/op_builder/supa/async_io.py @@ -27,7 +27,7 @@ def sources(self): 'csrc/aio/common/deepspeed_aio_utils.cpp', 'csrc/aio/common/deepspeed_aio_common.cpp', 'csrc/aio/common/deepspeed_aio_types.cpp', - 'csrc/aio/py_lib/deepspeed_pin_tensor.cpp', + 'csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp', 'csrc/aio/py_lib/deepspeed_py_io_handle.cpp', 'csrc/aio/py_lib/deepspeed_cpu_op.cpp', 'csrc/aio/py_lib/deepspeed_aio_op_desc.cpp', @@ -35,7 +35,7 @@ def sources(self): def include_paths(self): args = super().include_paths() - args += ['csrc/aio/py_lib', 'csrc/aio/common'] + args += ['csrc/aio/py_lib', 'csrc/aio/common', 'csrc/pin_memory'] return args def cxx_args(self): @@ -67,6 +67,16 @@ def check_for_libaio_pkg(self): break return found + def load(self, verbose=False): + # Pin manager is compiled only into pin_memory; load it first so aio can + # resolve the shared manager (and related symbols) across .so boundaries. + try: + from op_builder.pin_memory import PinMemoryBuilder + except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + PinMemoryBuilder().load(verbose=verbose) + return super().load(verbose=verbose) + def is_compatible(self, verbose=False): # Check for the existence of libaio by using distutils # to compile and link a test program that calls io_submit, diff --git a/op_builder/supa/pin_memory.py b/op_builder/supa/pin_memory.py new file mode 100644 index 000000000000..6ec36dd8181c --- /dev/null +++ b/op_builder/supa/pin_memory.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# Host pin_memory is accelerator-agnostic; reuse the top-level builder. +try: + from op_builder.pin_memory import PinMemoryBuilder +except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + +__all__ = ["PinMemoryBuilder"] diff --git a/op_builder/xpu/__init__.py b/op_builder/xpu/__init__.py index a91775a3bb13..400c06e23061 100755 --- a/op_builder/xpu/__init__.py +++ b/op_builder/xpu/__init__.py @@ -7,6 +7,7 @@ from .cpu_adagrad import CPUAdagradBuilder from .fused_adam import FusedAdamBuilder from .async_io import AsyncIOBuilder +from .pin_memory import PinMemoryBuilder from .flash_attn import FlashAttentionBuilder from .no_impl import NotImplementedBuilder from .packbits import PackbitsBuilder diff --git a/op_builder/xpu/async_io.py b/op_builder/xpu/async_io.py index 8ec030880368..fd60e1747f28 100644 --- a/op_builder/xpu/async_io.py +++ b/op_builder/xpu/async_io.py @@ -29,14 +29,14 @@ def sources(self): 'csrc/aio/common/deepspeed_aio_utils.cpp', 'csrc/aio/common/deepspeed_aio_common.cpp', 'csrc/aio/common/deepspeed_aio_types.cpp', - 'csrc/aio/py_lib/deepspeed_pin_tensor.cpp', + 'csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp', 'csrc/aio/py_lib/deepspeed_py_io_handle.cpp', 'csrc/aio/py_lib/deepspeed_cpu_op.cpp', 'csrc/aio/py_lib/deepspeed_aio_op_desc.cpp', ] def include_paths(self): - return ['csrc/aio/py_lib', 'csrc/aio/common'] + return ['csrc/aio/py_lib', 'csrc/aio/common', 'csrc/pin_memory'] def cxx_args(self): import torch @@ -86,6 +86,16 @@ def check_for_libaio_pkg(self): break return found + def load(self, verbose=False): + # Pin manager is compiled only into pin_memory; load it first so aio can + # resolve the shared manager (and related symbols) across .so boundaries. + try: + from op_builder.pin_memory import PinMemoryBuilder + except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + PinMemoryBuilder().load(verbose=verbose) + return super().load(verbose=verbose) + def is_compatible(self, verbose=False): # Check for the existence of libaio by using distutils # to compile and link a test program that calls io_submit, diff --git a/op_builder/xpu/pin_memory.py b/op_builder/xpu/pin_memory.py new file mode 100644 index 000000000000..6ec36dd8181c --- /dev/null +++ b/op_builder/xpu/pin_memory.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# Host pin_memory is accelerator-agnostic; reuse the top-level builder. +try: + from op_builder.pin_memory import PinMemoryBuilder +except ImportError: + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder + +__all__ = ["PinMemoryBuilder"] diff --git a/tests/unit/v1/nvme/test_pinned_manager.py b/tests/unit/v1/nvme/test_pinned_manager.py index 1067edfccbba..c6ac3c76d38d 100644 --- a/tests/unit/v1/nvme/test_pinned_manager.py +++ b/tests/unit/v1/nvme/test_pinned_manager.py @@ -48,3 +48,36 @@ def test_buffer_is_shared_across_handles(): def test_unmanaged_buffer_is_not_pinned(): handle = _new_handle() assert not handle.is_pinned(torch.empty(NUM_ELEMS, dtype=torch.float)) + + +def test_pin_handle_buffer_recognized_by_aio_handle(): + # Buffers allocated via the standalone pin_memory op must share the same + # process-wide manager as async_io, so aio_handle.is_pinned sees them. + from deepspeed.ops.op_builder import PinMemoryBuilder + if not deepspeed.ops.__compatible_ops__.get(PinMemoryBuilder.NAME, True): + pytest.skip('Skip tests since pin_memory is not compatible') + + pin_handle = PinMemoryBuilder().load().pin_handle() + aio_handle = _new_handle() + buffer = pin_handle.new_cpu_locked_tensor(NUM_ELEMS, torch.empty(0, dtype=torch.float)) + try: + assert pin_handle.is_pinned(buffer) + assert aio_handle.is_pinned(buffer) + assert aio_handle.is_pinned(buffer.narrow(0, BLOCK_SIZE, BLOCK_SIZE)) + finally: + pin_handle.free_cpu_locked_tensor(buffer) + + +def test_aio_handle_buffer_recognized_by_pin_handle(): + from deepspeed.ops.op_builder import PinMemoryBuilder + if not deepspeed.ops.__compatible_ops__.get(PinMemoryBuilder.NAME, True): + pytest.skip('Skip tests since pin_memory is not compatible') + + pin_handle = PinMemoryBuilder().load().pin_handle() + aio_handle = _new_handle() + buffer = aio_handle.new_cpu_locked_tensor(NUM_ELEMS, torch.empty(0, dtype=torch.float)) + try: + assert aio_handle.is_pinned(buffer) + assert pin_handle.is_pinned(buffer) + finally: + aio_handle.free_cpu_locked_tensor(buffer) diff --git a/tests/unit/v1/pin_memory/test_pin_memory_op.py b/tests/unit/v1/pin_memory/test_pin_memory_op.py new file mode 100644 index 000000000000..291d4cb2cb2d --- /dev/null +++ b/tests/unit/v1/pin_memory/test_pin_memory_op.py @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +import pytest +import torch +import deepspeed +from deepspeed.ops.op_builder import PinMemoryBuilder + +if not deepspeed.ops.__compatible_ops__.get(PinMemoryBuilder.NAME, True): + pytest.skip('Skip tests since pin_memory is not compatible', allow_module_level=True) + +NUM_ELEMS = 4096 + + +def test_pin_handle_alloc_free_and_is_pinned(): + # pin_memory must work without libaio / async_io. + handle = PinMemoryBuilder().load().pin_handle() + buffer = handle.new_cpu_locked_tensor(NUM_ELEMS, torch.empty(0, dtype=torch.float)) + try: + assert handle.is_pinned(buffer) + assert handle.is_pinned(buffer.narrow(0, 16, 16)) + assert not handle.is_pinned(torch.empty(NUM_ELEMS, dtype=torch.float)) + finally: + assert handle.free_cpu_locked_tensor(buffer) is True From 0dfc55718e59208aa2b86cf6e9129d14dc9e1dfa Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Sat, 8 Aug 2026 23:27:14 +0000 Subject: [PATCH 2/6] Address review: guard Windows, precompile pin_memory with AIO/GDS, add docs - PinMemoryBuilder.is_compatible() returns False on Windows (POSIX-only mlock/posix_memalign). - setup.py precompiles pin_memory whenever async_io/gds are precompiled so wheels without a runtime compiler don't hit a JIT build for the shared manager. - Document the standalone pin_memory op / pin_handle API in the DeepNVMe tutorial. Signed-off-by: Olatunji Ruwase Co-authored-by: Cursor --- docs/_tutorials/deepnvme.md | 15 +++++++++++++++ op_builder/cpu/pin_memory.py | 10 ++++++++++ op_builder/pin_memory.py | 10 ++++++++++ setup.py | 15 +++++++++++++++ 4 files changed, 50 insertions(+) diff --git a/docs/_tutorials/deepnvme.md b/docs/_tutorials/deepnvme.md index a6d4545815dc..c2f6b0f0f248 100644 --- a/docs/_tutorials/deepnvme.md +++ b/docs/_tutorials/deepnvme.md @@ -146,6 +146,21 @@ True 1073741824 ``` +You can also allocate page-locked host tensors directly through the standalone `pin_memory` operator, which pins CPU memory (`posix_memalign` + `mlock`) without requiring `libaio` or the `async_io` worker threads. This is useful on CPU-only hosts or when you want a pinned buffer without an `aio_handle`. Buffers allocated this way are recognized by `aio_handle`/`gds_handle` (they share one process-wide pin manager), so DeepNVMe can still skip bounce buffers for them. + +```bash +>>> import torch +>>> from deepspeed.ops.op_builder import PinMemoryBuilder +>>> pin = PinMemoryBuilder().load().pin_handle() +>>> t = pin.new_cpu_locked_tensor(1024**3, torch.empty(0, dtype=torch.uint8)) +>>> pin.is_pinned(t) +True +>>> pin.free_cpu_locked_tensor(t) +True +``` + +The `pin_handle` exposes `new_cpu_locked_tensor(num_elem, example_tensor)`, `free_cpu_locked_tensor(tensor)`, and `is_pinned(tensor)`. The same methods remain available on `aio_handle` and `gds_handle` as thin wrappers, so existing code continues to work unchanged. + On the other hand,`gds_handle` provides `new_pinned_device_tensor()` and `pin_device_tensor()` functions for pinning CUDA tensors. The following example illustrates writing a pinned CUDA tensor to a local NVMe file. ```bash diff --git a/op_builder/cpu/pin_memory.py b/op_builder/cpu/pin_memory.py index e0416f03d649..79b31ea36c43 100644 --- a/op_builder/cpu/pin_memory.py +++ b/op_builder/cpu/pin_memory.py @@ -3,6 +3,8 @@ # DeepSpeed Team +import sys + from .builder import CPUOpBuilder try: from op_builder.pin_memory_load import load_pin_memory_module @@ -20,6 +22,14 @@ def __init__(self): def absolute_name(self): return f'deepspeed.ops.pin_memory.{self.NAME}_op' + def is_compatible(self, verbose=False): + # The allocator relies on POSIX mlock/posix_memalign, which are unavailable on Windows. + if sys.platform == "win32": + if verbose: + self.warning(f"{self.NAME} is only supported on POSIX platforms, not Windows.") + return False + return super().is_compatible(verbose) + def sources(self): return [ 'csrc/pin_memory/page_alloc.cpp', diff --git a/op_builder/pin_memory.py b/op_builder/pin_memory.py index c87205df63cf..497fcd025be7 100644 --- a/op_builder/pin_memory.py +++ b/op_builder/pin_memory.py @@ -3,6 +3,8 @@ # DeepSpeed Team +import sys + from .builder import OpBuilder from .pin_memory_load import load_pin_memory_module @@ -17,6 +19,14 @@ def __init__(self): def absolute_name(self): return f'deepspeed.ops.pin_memory.{self.NAME}_op' + def is_compatible(self, verbose=False): + # The allocator relies on POSIX mlock/posix_memalign, which are unavailable on Windows. + if sys.platform == "win32": + if verbose: + self.warning(f"{self.NAME} is only supported on POSIX platforms, not Windows.") + return False + return super().is_compatible(verbose) + def sources(self): return [ 'csrc/pin_memory/page_alloc.cpp', diff --git a/setup.py b/setup.py index b383e5271381..5a2906a352c5 100755 --- a/setup.py +++ b/setup.py @@ -201,6 +201,21 @@ def op_enabled(op_name): install_ops[op_name] = op_enabled(op_name) ext_modules.append(builder.builder()) +# async_io/gds load the pin_memory op's shared manager at runtime. If they are +# precompiled but pin_memory is not, deployments without a compiler would hit a +# runtime JIT build, so precompile pin_memory alongside them. +pin_memory_dependents = [name for name in ("async_io", "gds") if install_ops.get(name)] +if pin_memory_dependents and not install_ops.get("pin_memory"): + pin_memory_builder = ALL_OPS.get("pin_memory") + if pin_memory_builder is not None and pin_memory_builder.is_compatible(): + assert torch_available, "Unable to pre-compile pin_memory, please first install torch" + install_ops["pin_memory"] = 1 + ext_modules.append(pin_memory_builder.builder()) + else: + raise RuntimeError( + f"Cannot pre-compile {', '.join(pin_memory_dependents)} without pin_memory, which is required at " + "runtime but is not compatible on this platform.") + print(f'Install Ops={install_ops}') # Write out version/git info. From 046892faea25d0f4c4ade97bee1262b6513ce900 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Sat, 8 Aug 2026 23:48:26 +0000 Subject: [PATCH 3/6] Document pin_memory op in Host Memory Pinning RTD section Add a Host Memory Pinning section to memory.rst covering torch vs the standalone pin_memory op / pin_handle API, shared manager with AIO/GDS, and build/memlock requirements. Signed-off-by: Olatunji Ruwase Co-authored-by: Cursor --- docs/code-docs/source/memory.rst | 98 +++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/docs/code-docs/source/memory.rst b/docs/code-docs/source/memory.rst index 9f05495001c6..67fe93dbc6fd 100644 --- a/docs/code-docs/source/memory.rst +++ b/docs/code-docs/source/memory.rst @@ -265,7 +265,7 @@ Note about gradients: While gradients are stored in fp16 (2 bytes), during the w **Pinned Memory** -Pinned general RAM is included in normal general RAM allocations (i.e. this is not extra memory allocations but simply shows how much of the general RAM is pinned). Pinning is controlled by the ``pin_memory`` field of ``offload_optimizer`` / ``offload_param`` (both default to ``true``); set to ``false`` on hosts with tight memlock limits (``ulimit -l``). +Pinned general RAM is included in normal general RAM allocations (i.e. this is not extra memory allocations but simply shows how much of the general RAM is pinned). Pinning is controlled by the ``pin_memory`` field of ``offload_optimizer`` / ``offload_param`` (both default to ``true``); set to ``false`` on hosts with tight memlock limits (``ulimit -l``). See :ref:`host-memory-pinning` for how host memory is page-locked (``torch`` vs the DeepSpeed ``pin_memory`` op). * ZeRO-1/2: controlled by ``offload_optimizer.pin_memory`` @@ -288,3 +288,99 @@ With pinning enabled there are 2 sub-cases: XXX: For Transformers is probably around (2* seq * attn_heads + 16 * hidden_size) * sequence * batch/gpu This needs to be completed. + + +.. _host-memory-pinning: + +Host Memory Pinning +----------------------- + +DeepSpeed page-locks (pins) host memory so DMA engines can move data efficiently +between CPU RAM and accelerators or NVMe. Host tensors can be pinned either with +PyTorch / the accelerator hook, or with the standalone DeepSpeed ``pin_memory`` +operator (``posix_memalign`` + ``mlock``), which does not require ``libaio`` or +``async_io`` worker threads. + +Accelerator API (torch path) +============================ + +.. code-block:: python + + from deepspeed.accelerator import get_accelerator + + pinned = get_accelerator().pin_memory(tensor) + assert get_accelerator().is_pinned(pinned) + +On most accelerators this routes to ``tensor.pin_memory()``. The CPU accelerator +historically treats this as a no-op (torch cannot pin a CPU tensor to a device). + +DeepSpeed ``pin_memory`` op +========================== + +Allocate and free page-locked host tensors through ``PinMemoryBuilder`` / +``pin_handle``: + +.. code-block:: python + + import torch + from deepspeed.ops.op_builder import PinMemoryBuilder + + pin = PinMemoryBuilder().load().pin_handle() + t = pin.new_cpu_locked_tensor(1024**3, torch.empty(0, dtype=torch.uint8)) + assert pin.is_pinned(t) + pin.free_cpu_locked_tensor(t) + +``pin_handle`` exposes: + +* ``new_cpu_locked_tensor(num_elem, example_tensor)`` — allocate a page-locked CPU tensor +* ``free_cpu_locked_tensor(tensor)`` — unlock and free the allocation +* ``is_pinned(tensor)`` — ``True`` for torch-pinned buffers and for tensors whose + storage falls inside a DeepSpeed-managed locked range (slices/views included) + +The same three methods remain available on ``aio_handle`` / ``gds_handle`` as thin +wrappers. Buffers allocated via ``pin_handle`` and via an I/O handle share one +process-wide pin manager, so DeepNVMe bounce-buffer skipping stays consistent +across both paths. + +.. list-table:: Differences between torch pinning and the DeepSpeed ``pin_memory`` op + :header-rows: 1 + :widths: 22 39 39 + + * - Aspect + - Torch / accelerator ``pin_memory`` + - DeepSpeed ``pin_memory`` op + * - Allocator + - ``torch.Tensor.pin_memory()`` (device-specific accelerator hook) + - ``posix_memalign`` + ``mlock`` via ``PinMemoryBuilder`` / ``pin_handle`` + * - Build dependency + - None beyond the active accelerator / PyTorch + - DeepSpeed **pin_memory** op must build and load (no ``libaio`` required) + * - AIO / GDS worker threads + - Not involved + - Not required (unlike allocating through ``aio_handle`` alone) + * - Visible to DeepNVMe + - Yes (``aio_handle.is_pinned`` / bounce-buffer skip) + - Yes (same process-wide manager as ``async_io`` / ``gds``) + * - ``is_pinned`` + - Torch pinned status (``tensor.is_pinned()``) + - Torch-pinned **or** DeepSpeed-managed locked range (slices/views included) + * - Explicit free + - Freed when the tensor is garbage-collected (PyTorch) + - ``free_cpu_locked_tensor`` releases the mlocked pages immediately + * - CPU accelerator + - Historical no-op (torch cannot pin CPU tensors) + - Real ``mlock`` pins on CPU-only hosts + +Requirements +============ + +* The DeepSpeed **pin_memory** op must build and load successfully. When + ``async_io`` / ``gds`` are precompiled, ``setup.py`` also precompiles + ``pin_memory`` so deployments without a runtime compiler still get the shared + manager. +* The process must be allowed to lock the requested host memory (see + ``ulimit -l`` / ``memlock`` limits). Large ZeRO CPU-offload or DeepNVMe + footprints can otherwise hit the memlock ceiling. + +See also the DeepNVMe tutorial for end-to-end examples of pinning host and device +tensors for NVMe I/O. From e43399487572672a09e868a3a9222c550b807b90 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Sun, 9 Aug 2026 03:00:50 +0000 Subject: [PATCH 4/6] Docs: drop libaio/async_io from pin_memory op RTD Describe the pin_memory op without framing it against libaio or async_io. Signed-off-by: Olatunji Ruwase Co-authored-by: Cursor --- docs/_tutorials/deepnvme.md | 4 ++-- docs/code-docs/source/memory.rst | 24 ++++++++---------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/docs/_tutorials/deepnvme.md b/docs/_tutorials/deepnvme.md index c2f6b0f0f248..a2522469dc8c 100644 --- a/docs/_tutorials/deepnvme.md +++ b/docs/_tutorials/deepnvme.md @@ -146,7 +146,7 @@ True 1073741824 ``` -You can also allocate page-locked host tensors directly through the standalone `pin_memory` operator, which pins CPU memory (`posix_memalign` + `mlock`) without requiring `libaio` or the `async_io` worker threads. This is useful on CPU-only hosts or when you want a pinned buffer without an `aio_handle`. Buffers allocated this way are recognized by `aio_handle`/`gds_handle` (they share one process-wide pin manager), so DeepNVMe can still skip bounce buffers for them. +You can also allocate page-locked host tensors directly through the standalone `pin_memory` operator, which pins CPU memory (`posix_memalign` + `mlock`). This is useful on CPU-only hosts or when you want a pinned buffer without starting an I/O handle. Buffers allocated this way are recognized by DeepNVMe I/O handles (they share one process-wide pin manager), so DeepNVMe can still skip bounce buffers for them. ```bash >>> import torch @@ -159,7 +159,7 @@ True True ``` -The `pin_handle` exposes `new_cpu_locked_tensor(num_elem, example_tensor)`, `free_cpu_locked_tensor(tensor)`, and `is_pinned(tensor)`. The same methods remain available on `aio_handle` and `gds_handle` as thin wrappers, so existing code continues to work unchanged. +The `pin_handle` exposes `new_cpu_locked_tensor(num_elem, example_tensor)`, `free_cpu_locked_tensor(tensor)`, and `is_pinned(tensor)`. The same methods remain available on DeepNVMe I/O handles as thin wrappers, so existing code continues to work unchanged. On the other hand,`gds_handle` provides `new_pinned_device_tensor()` and `pin_device_tensor()` functions for pinning CUDA tensors. The following example illustrates writing a pinned CUDA tensor to a local NVMe file. diff --git a/docs/code-docs/source/memory.rst b/docs/code-docs/source/memory.rst index 67fe93dbc6fd..9be2fdb91b90 100644 --- a/docs/code-docs/source/memory.rst +++ b/docs/code-docs/source/memory.rst @@ -298,8 +298,7 @@ Host Memory Pinning DeepSpeed page-locks (pins) host memory so DMA engines can move data efficiently between CPU RAM and accelerators or NVMe. Host tensors can be pinned either with PyTorch / the accelerator hook, or with the standalone DeepSpeed ``pin_memory`` -operator (``posix_memalign`` + ``mlock``), which does not require ``libaio`` or -``async_io`` worker threads. +operator (``posix_memalign`` + ``mlock``). Accelerator API (torch path) ============================ @@ -337,10 +336,9 @@ Allocate and free page-locked host tensors through ``PinMemoryBuilder`` / * ``is_pinned(tensor)`` — ``True`` for torch-pinned buffers and for tensors whose storage falls inside a DeepSpeed-managed locked range (slices/views included) -The same three methods remain available on ``aio_handle`` / ``gds_handle`` as thin -wrappers. Buffers allocated via ``pin_handle`` and via an I/O handle share one -process-wide pin manager, so DeepNVMe bounce-buffer skipping stays consistent -across both paths. +The same three methods remain available on DeepNVMe I/O handles as thin wrappers. +Buffers allocated via ``pin_handle`` and via an I/O handle share one process-wide +pin manager, so DeepNVMe bounce-buffer skipping stays consistent across both paths. .. list-table:: Differences between torch pinning and the DeepSpeed ``pin_memory`` op :header-rows: 1 @@ -354,13 +352,10 @@ across both paths. - ``posix_memalign`` + ``mlock`` via ``PinMemoryBuilder`` / ``pin_handle`` * - Build dependency - None beyond the active accelerator / PyTorch - - DeepSpeed **pin_memory** op must build and load (no ``libaio`` required) - * - AIO / GDS worker threads - - Not involved - - Not required (unlike allocating through ``aio_handle`` alone) + - DeepSpeed **pin_memory** op must build and load * - Visible to DeepNVMe - - Yes (``aio_handle.is_pinned`` / bounce-buffer skip) - - Yes (same process-wide manager as ``async_io`` / ``gds``) + - Yes (``is_pinned`` / bounce-buffer skip on I/O handles) + - Yes (same process-wide pin manager) * - ``is_pinned`` - Torch pinned status (``tensor.is_pinned()``) - Torch-pinned **or** DeepSpeed-managed locked range (slices/views included) @@ -374,10 +369,7 @@ across both paths. Requirements ============ -* The DeepSpeed **pin_memory** op must build and load successfully. When - ``async_io`` / ``gds`` are precompiled, ``setup.py`` also precompiles - ``pin_memory`` so deployments without a runtime compiler still get the shared - manager. +* The DeepSpeed **pin_memory** op must build and load successfully. * The process must be allowed to lock the requested host memory (see ``ulimit -l`` / ``memlock`` limits). Large ZeRO CPU-offload or DeepNVMe footprints can otherwise hit the memlock ceiling. From a04329401ca63df6ea14722891e425445a7ec0c1 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Sun, 9 Aug 2026 03:03:58 +0000 Subject: [PATCH 5/6] Docs: clarify Host Memory Pinning cross-ref (no native backend) Align wording with rtd-staging; this PR only documents torch/accelerator pinning and the standalone pin_memory op. Signed-off-by: Olatunji Ruwase Co-authored-by: Cursor --- docs/code-docs/source/memory.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/code-docs/source/memory.rst b/docs/code-docs/source/memory.rst index 9be2fdb91b90..769943878a40 100644 --- a/docs/code-docs/source/memory.rst +++ b/docs/code-docs/source/memory.rst @@ -265,7 +265,7 @@ Note about gradients: While gradients are stored in fp16 (2 bytes), during the w **Pinned Memory** -Pinned general RAM is included in normal general RAM allocations (i.e. this is not extra memory allocations but simply shows how much of the general RAM is pinned). Pinning is controlled by the ``pin_memory`` field of ``offload_optimizer`` / ``offload_param`` (both default to ``true``); set to ``false`` on hosts with tight memlock limits (``ulimit -l``). See :ref:`host-memory-pinning` for how host memory is page-locked (``torch`` vs the DeepSpeed ``pin_memory`` op). +Pinned general RAM is included in normal general RAM allocations (i.e. this is not extra memory allocations but simply shows how much of the general RAM is pinned). Pinning is controlled by the ``pin_memory`` field of ``offload_optimizer`` / ``offload_param`` (both default to ``true``); set to ``false`` on hosts with tight memlock limits (``ulimit -l``). See :ref:`host-memory-pinning` for how host memory is page-locked (accelerator / torch pinning and the DeepSpeed ``pin_memory`` op). * ZeRO-1/2: controlled by ``offload_optimizer.pin_memory`` From a6f6ab6e327567bc25714cfe7619431fe162f8b7 Mon Sep 17 00:00:00 2001 From: Olatunji Ruwase Date: Sun, 9 Aug 2026 12:28:32 +0000 Subject: [PATCH 6/6] Address review: drop Microsoft headers; reuse top-level PinMemoryBuilder on CPU New pin_memory sources use SPDX + DeepSpeed Team only. CPU (and other accelerator) builders re-export op_builder.pin_memory.PinMemoryBuilder instead of duplicating the class. Signed-off-by: Olatunji Ruwase Co-authored-by: Cursor --- csrc/aio/py_lib/deepspeed_pin_tensor.h | 1 - .../py_lib/deepspeed_pin_tensor_client.cpp | 1 - csrc/pin_memory/deepspeed_pin_tensor.cpp | 1 - csrc/pin_memory/deepspeed_pin_tensor.h | 1 - csrc/pin_memory/page_alloc.cpp | 1 - csrc/pin_memory/page_alloc.h | 1 - csrc/pin_memory/py_ds_pin_memory.cpp | 1 - deepspeed/ops/pin_memory/__init__.py | 1 - op_builder/cpu/pin_memory.py | 41 ++----------------- op_builder/npu/pin_memory.py | 1 - op_builder/pin_memory.py | 1 - op_builder/pin_memory_load.py | 1 - op_builder/supa/pin_memory.py | 1 - op_builder/xpu/pin_memory.py | 1 - 14 files changed, 4 insertions(+), 50 deletions(-) diff --git a/csrc/aio/py_lib/deepspeed_pin_tensor.h b/csrc/aio/py_lib/deepspeed_pin_tensor.h index 19fc88e1d162..61ce2350ba11 100644 --- a/csrc/aio/py_lib/deepspeed_pin_tensor.h +++ b/csrc/aio/py_lib/deepspeed_pin_tensor.h @@ -1,4 +1,3 @@ -// Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 // DeepSpeed Team diff --git a/csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp b/csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp index 3a24d633daa2..fb9cd05aa852 100644 --- a/csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp +++ b/csrc/aio/py_lib/deepspeed_pin_tensor_client.cpp @@ -1,4 +1,3 @@ -// Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 // DeepSpeed Team diff --git a/csrc/pin_memory/deepspeed_pin_tensor.cpp b/csrc/pin_memory/deepspeed_pin_tensor.cpp index cb36f0fee220..9c4d1c0ad5cd 100644 --- a/csrc/pin_memory/deepspeed_pin_tensor.cpp +++ b/csrc/pin_memory/deepspeed_pin_tensor.cpp @@ -1,4 +1,3 @@ -// Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 // DeepSpeed Team diff --git a/csrc/pin_memory/deepspeed_pin_tensor.h b/csrc/pin_memory/deepspeed_pin_tensor.h index e2a7a11947c7..97457fb09be0 100644 --- a/csrc/pin_memory/deepspeed_pin_tensor.h +++ b/csrc/pin_memory/deepspeed_pin_tensor.h @@ -1,4 +1,3 @@ -// Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 // DeepSpeed Team diff --git a/csrc/pin_memory/page_alloc.cpp b/csrc/pin_memory/page_alloc.cpp index 97bac5028c02..ff2f769b0626 100644 --- a/csrc/pin_memory/page_alloc.cpp +++ b/csrc/pin_memory/page_alloc.cpp @@ -1,4 +1,3 @@ -// Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 // DeepSpeed Team diff --git a/csrc/pin_memory/page_alloc.h b/csrc/pin_memory/page_alloc.h index 698bddca19ec..9164b446d36c 100644 --- a/csrc/pin_memory/page_alloc.h +++ b/csrc/pin_memory/page_alloc.h @@ -1,4 +1,3 @@ -// Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 // DeepSpeed Team diff --git a/csrc/pin_memory/py_ds_pin_memory.cpp b/csrc/pin_memory/py_ds_pin_memory.cpp index 34403a7ac44d..52a6e8d9a69e 100644 --- a/csrc/pin_memory/py_ds_pin_memory.cpp +++ b/csrc/pin_memory/py_ds_pin_memory.cpp @@ -1,4 +1,3 @@ -// Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 // DeepSpeed Team diff --git a/deepspeed/ops/pin_memory/__init__.py b/deepspeed/ops/pin_memory/__init__.py index 79821a828fa2..16c0611c9105 100644 --- a/deepspeed/ops/pin_memory/__init__.py +++ b/deepspeed/ops/pin_memory/__init__.py @@ -1,4 +1,3 @@ -# Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 # DeepSpeed Team diff --git a/op_builder/cpu/pin_memory.py b/op_builder/cpu/pin_memory.py index 79b31ea36c43..f59e8dbb1de8 100644 --- a/op_builder/cpu/pin_memory.py +++ b/op_builder/cpu/pin_memory.py @@ -1,44 +1,11 @@ -# Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 # DeepSpeed Team -import sys - -from .builder import CPUOpBuilder +# Host pin_memory is accelerator-agnostic; reuse the top-level builder. try: - from op_builder.pin_memory_load import load_pin_memory_module + from op_builder.pin_memory import PinMemoryBuilder except ImportError: - from deepspeed.ops.op_builder.pin_memory_load import load_pin_memory_module - - -class PinMemoryBuilder(CPUOpBuilder): - BUILD_VAR = "DS_BUILD_PIN_MEMORY" - NAME = "pin_memory" - - def __init__(self): - super().__init__(name=self.NAME) - - def absolute_name(self): - return f'deepspeed.ops.pin_memory.{self.NAME}_op' - - def is_compatible(self, verbose=False): - # The allocator relies on POSIX mlock/posix_memalign, which are unavailable on Windows. - if sys.platform == "win32": - if verbose: - self.warning(f"{self.NAME} is only supported on POSIX platforms, not Windows.") - return False - return super().is_compatible(verbose) - - def sources(self): - return [ - 'csrc/pin_memory/page_alloc.cpp', - 'csrc/pin_memory/deepspeed_pin_tensor.cpp', - 'csrc/pin_memory/py_ds_pin_memory.cpp', - ] - - def include_paths(self): - return ['csrc/pin_memory'] + from deepspeed.ops.op_builder.pin_memory import PinMemoryBuilder - def load(self, verbose=False): - return load_pin_memory_module(self, verbose=verbose) +__all__ = ["PinMemoryBuilder"] diff --git a/op_builder/npu/pin_memory.py b/op_builder/npu/pin_memory.py index 6ec36dd8181c..f59e8dbb1de8 100644 --- a/op_builder/npu/pin_memory.py +++ b/op_builder/npu/pin_memory.py @@ -1,4 +1,3 @@ -# Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 # DeepSpeed Team diff --git a/op_builder/pin_memory.py b/op_builder/pin_memory.py index 497fcd025be7..339dccffdbee 100644 --- a/op_builder/pin_memory.py +++ b/op_builder/pin_memory.py @@ -1,4 +1,3 @@ -# Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 # DeepSpeed Team diff --git a/op_builder/pin_memory_load.py b/op_builder/pin_memory_load.py index d3fc4f34abde..179b4b38ab3e 100644 --- a/op_builder/pin_memory_load.py +++ b/op_builder/pin_memory_load.py @@ -1,4 +1,3 @@ -# Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 # DeepSpeed Team diff --git a/op_builder/supa/pin_memory.py b/op_builder/supa/pin_memory.py index 6ec36dd8181c..f59e8dbb1de8 100644 --- a/op_builder/supa/pin_memory.py +++ b/op_builder/supa/pin_memory.py @@ -1,4 +1,3 @@ -# Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 # DeepSpeed Team diff --git a/op_builder/xpu/pin_memory.py b/op_builder/xpu/pin_memory.py index 6ec36dd8181c..f59e8dbb1de8 100644 --- a/op_builder/xpu/pin_memory.py +++ b/op_builder/xpu/pin_memory.py @@ -1,4 +1,3 @@ -# Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 # DeepSpeed Team