Repository navigation
Split native host pinning into standalone pin_memory op #8236
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
38e2ea6
0dfc557
046892f
e433994
a043294
a6f6ab6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,37 +1,11 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // DeepSpeed Team | ||
|
|
||
| /* | ||
| Functionality for managing CPU tensors occupying page-locked memory. | ||
| TODO: Implement a full-featured manager that | ||
| 1. Avoid page-locked memory leaks | ||
| 2. Minimize page-locked memory usage by reducing internal fragmentation | ||
| Functionality for managing CPU tensors occupying page-locked memory. | ||
| Compatibility include: pin-tensor manager lives under csrc/pin_memory/. | ||
| */ | ||
|
|
||
| #include <map> | ||
| #include <memory> | ||
| #include <mutex> | ||
| #include "deepspeed_py_aio.h" | ||
| #pragma once | ||
|
|
||
| struct deepspeed_pin_tensor_t { | ||
| std::map<void*, int64_t> _locked_tensors; | ||
| std::mutex _mutex; | ||
|
|
||
| deepspeed_pin_tensor_t() = default; | ||
|
|
||
| ~deepspeed_pin_tensor_t(); | ||
|
|
||
| // Process-wide shared manager so that pinned-buffer recognition is consistent | ||
| // across every io handle (each handle references this single instance). | ||
| static std::shared_ptr<deepspeed_pin_tensor_t> shared(); | ||
|
|
||
| torch::Tensor alloc(const int64_t num_elem, const at::ScalarType& elem_type); | ||
| torch::Tensor alloc(const int64_t num_elem, const torch::TensorOptions& options); | ||
|
|
||
| bool free(torch::Tensor& locked_tensor); | ||
|
|
||
| bool is_managed(const torch::Tensor& buffer); | ||
| }; | ||
| #include "../../pin_memory/deepspeed_pin_tensor.h" |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // DeepSpeed Team | ||
|
|
||
| /* | ||
| Resolve the process-wide pin-tensor manager exported by the pin_memory op. | ||
| The manager is compiled only into pin_memory; async_io/gds must load that op | ||
| first (see AsyncIOBuilder.load) so this symbol is visible via RTLD_GLOBAL. | ||
| */ | ||
|
|
||
| #include "deepspeed_pin_tensor.h" | ||
|
|
||
| #include <dlfcn.h> | ||
| #include <stdexcept> | ||
|
|
||
| std::shared_ptr<deepspeed_pin_tensor_t> deepspeed_pin_tensor_t::shared() | ||
| { | ||
| using holder_fn_t = void* (*)(); | ||
| auto* fn = | ||
| reinterpret_cast<holder_fn_t>(dlsym(RTLD_DEFAULT, "deepspeed_pin_tensor_mgr_holder")); | ||
| if (fn == nullptr) { | ||
| throw std::runtime_error( | ||
| "DeepSpeed pin_memory op must be loaded before async_io/gds (missing " | ||
| "deepspeed_pin_tensor_mgr_holder). Load PinMemoryBuilder first."); | ||
| } | ||
| auto* holder = static_cast<std::shared_ptr<deepspeed_pin_tensor_t>*>(fn()); | ||
| return *holder; | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // DeepSpeed Team | ||
|
|
||
| /* | ||
| Functionality for managing CPU tensors occupying page-locked memory. | ||
| TODO: Implement a full-featured manager that | ||
| 1. Avoid page-locked memory leaks | ||
| 2. Minimize page-locked memory usage by reducing internal fragmentation | ||
| */ | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <torch/extension.h> | ||
| #include <map> | ||
| #include <memory> | ||
| #include <mutex> | ||
|
|
||
| struct deepspeed_pin_tensor_t { | ||
| std::map<void*, int64_t> _locked_tensors; | ||
| std::mutex _mutex; | ||
|
|
||
| deepspeed_pin_tensor_t() = default; | ||
|
|
||
| ~deepspeed_pin_tensor_t(); | ||
|
|
||
| // Process-wide shared manager so that pinned-buffer recognition is consistent | ||
| // across every io handle (each handle references this single instance). | ||
| // Canonical instance lives in the pin_memory extension; other ops resolve it | ||
| // via deepspeed_pin_tensor_mgr_holder() after that extension is loaded. | ||
| static std::shared_ptr<deepspeed_pin_tensor_t> shared(); | ||
|
|
||
| torch::Tensor alloc(const int64_t num_elem, const at::ScalarType& elem_type); | ||
| torch::Tensor alloc(const int64_t num_elem, const torch::TensorOptions& options); | ||
|
|
||
| bool free(torch::Tensor& locked_tensor); | ||
|
|
||
| bool is_managed(const torch::Tensor& buffer); | ||
| }; | ||
|
|
||
| // Exported so async_io/gds can resolve the same manager across .so boundaries. | ||
| extern "C" void* deepspeed_pin_tensor_mgr_holder(); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // DeepSpeed Team | ||
|
|
||
| #include "page_alloc.h" | ||
|
|
||
| #include <sys/mman.h> | ||
| #include <unistd.h> | ||
| #include <cerrno> | ||
| #include <cstdlib> | ||
| #include <cstring> | ||
| #include <iostream> | ||
|
|
||
| void* ds_page_aligned_alloc(const int64_t size, const bool lock) | ||
| { | ||
| void* ptr; | ||
| int retval; | ||
|
|
||
| retval = posix_memalign(&ptr, (size_t)sysconf(_SC_PAGESIZE), size); | ||
| if (retval) { return nullptr; } | ||
|
|
||
| if (lock == false) { return ptr; } | ||
|
|
||
| auto mlock_ret = mlock(ptr, size); | ||
| if (mlock_ret != 0) { | ||
| auto mlock_error = errno; | ||
| std::cerr << "mlock failed to allocate " << size << " bytes with error no " << mlock_error | ||
| << " msg " << strerror(mlock_error) << std::endl; | ||
| free(ptr); | ||
| return nullptr; | ||
| } | ||
|
|
||
| return ptr; | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // DeepSpeed Team | ||
|
|
||
| /* | ||
| Page-aligned host allocation with optional mlock, independent of libaio. | ||
| */ | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <cstdint> | ||
|
|
||
| void* ds_page_aligned_alloc(const int64_t size, const bool lock = false); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // DeepSpeed Team | ||
|
|
||
| #include <torch/extension.h> | ||
| #include "deepspeed_pin_tensor.h" | ||
|
|
||
| using namespace pybind11::literals; | ||
|
|
||
| struct deepspeed_pin_handle_t { | ||
| std::shared_ptr<deepspeed_pin_tensor_t> _pinned_tensor_mgr; | ||
|
|
||
| deepspeed_pin_handle_t() : _pinned_tensor_mgr(deepspeed_pin_tensor_t::shared()) {} | ||
|
|
||
| torch::Tensor new_cpu_locked_tensor(const int64_t num_elem, const torch::Tensor& example_tensor) | ||
| { | ||
| return _pinned_tensor_mgr->alloc(num_elem, example_tensor.options()); | ||
| } | ||
|
|
||
| bool free_cpu_locked_tensor(torch::Tensor& locked_tensor) | ||
| { | ||
| return _pinned_tensor_mgr->free(locked_tensor); | ||
| } | ||
|
|
||
| bool is_pinned(const torch::Tensor& buffer) | ||
| { | ||
| return buffer.is_pinned() || _pinned_tensor_mgr->is_managed(buffer); | ||
| } | ||
| }; | ||
|
|
||
| PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) | ||
| { | ||
| py::class_<deepspeed_pin_handle_t>(m, "pin_handle") | ||
| .def(py::init<>(), "Pin-memory handle constructor") | ||
| .def("new_cpu_locked_tensor", | ||
| &deepspeed_pin_handle_t::new_cpu_locked_tensor, | ||
| "Allocate a page-locked CPU tensor", | ||
| "num_elem"_a, | ||
| "example_tensor"_a) | ||
| .def("free_cpu_locked_tensor", | ||
| &deepspeed_pin_handle_t::free_cpu_locked_tensor, | ||
| "Free a page-locked CPU tensor", | ||
| "locked_tensor"_a) | ||
| .def("is_pinned", | ||
| &deepspeed_pin_handle_t::is_pinned, | ||
| "Return whether buffer is torch-pinned or managed by DeepSpeed pin_memory", | ||
| "buffer"_a); | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| # DeepSpeed Team | ||
|
|
||
| from ..op_builder.pin_memory import PinMemoryBuilder | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
This commit exposes a new public AGENTS.md reference: AGENTS.md:L26-L26 Useful? React with 👍 / 👎.
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Addressed in 0dfc557 (DeepNVMe tutorial) and follow-ups in |
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When building wheels on Windows, setup.py defaults to prebuilding compatible ops, but this new op inherits the default compatibility and its source uses POSIX-only headers/APIs such as
sys/mman.h,unistd.h,posix_memalign,sysconf, andmlock. A Windows build will now try to compile this source and fail during installation, so add a Windows implementation or makePinMemoryBuilder.is_compatible()return false on Windows.Useful? React with 👍 / 👎.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Addressed in 0dfc557:
PinMemoryBuilder.is_compatible()returnsFalseonwin32because the allocator uses POSIXmlock/posix_memalign.