Skip to content
39 changes: 29 additions & 10 deletions accelerator/abstract_accelerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -260,20 +260,39 @@ def IntTensor(self):
def LongTensor(self):
...

def pin_memory(self, tensor, align_bytes=1):
# Memory pinning. The public methods below dispatch between the native
# backend (``deepspeed.utils.pin_memory``, selected via ``DS_PIN_MEMORY_BACKEND``)
# and the device-specific torch primitives ``_torch_pin_memory``/``_torch_is_pinned``,
# which subclasses override as needed. The native utility is backend-only and
# never calls back here.
def _torch_pin_memory(self, tensor):
return tensor.pin_memory()

def _torch_is_pinned(self, tensor):
return tensor.is_pinned()

def pin_memory(self, tensor, make_copy=True, match_shape=True):
from deepspeed.utils.pin_memory_tracker import track_pinned_memory
track_pinned_memory(tensor.nbytes)
return self._pin_memory(tensor, align_bytes)
from deepspeed.utils.pin_memory import get_active_native_pinned_memory
pins = get_active_native_pinned_memory()
if pins is not None:
return pins.pin(tensor, make_copy=make_copy, match_shape=match_shape)
return self._torch_pin_memory(tensor)

def _pin_memory(self, tensor, align_bytes=1):
"""Device-specific pinning hook. Accelerators that need custom pinning
behavior should override this method rather than ``pin_memory`` so that
the pinned-memory accounting in ``pin_memory`` is preserved."""
return tensor.pin_memory()

@abc.abstractmethod
def is_pinned(self, tensor):
...
from deepspeed.utils.pin_memory import get_active_native_pinned_memory
pins = get_active_native_pinned_memory()
if pins is not None and pins.is_pinned(tensor):
return True
return self._torch_is_pinned(tensor)

def unpin_memory(self, tensor):
from deepspeed.utils.pin_memory import get_active_native_pinned_memory
pins = get_active_native_pinned_memory()
if pins is not None:
return pins.unpin(tensor)
return None

@abc.abstractmethod
def on_accelerator(self, tensor):
Expand Down
18 changes: 12 additions & 6 deletions accelerator/cpu_accelerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,14 +278,20 @@ def IntTensor(self):
def LongTensor(self):
return torch.LongTensor

def pin_memory(self, tensor, align_bytes=1):
# Overrides pin_memory directly (not _pin_memory) to bypass the ABC's
# pinned-memory accounting: this is a no-op, nothing is page-locked, so
# counting would mislead OOM diagnostics. Do not rename to _pin_memory.
def pin_memory(self, tensor, make_copy=True, match_shape=True):
# Torch cannot pin CPU tensors to a device; keep the historical no-op and
# bypass ABC pinned-memory accounting (nothing is page-locked in that path).
from deepspeed.utils.pin_memory import get_active_native_pinned_memory
pins = get_active_native_pinned_memory()
if pins is not None:
from deepspeed.utils.pin_memory_tracker import track_pinned_memory
track_pinned_memory(tensor.nbytes)
return pins.pin(tensor, make_copy=make_copy, match_shape=match_shape)
return tensor

def is_pinned(self, tensor):
return tensor.is_pinned()
def _torch_pin_memory(self, tensor):
# torch cannot pin CPU tensors to a device; keep the historical no-op.
return tensor

def op_builder_dir(self):
try:
Expand Down
3 changes: 0 additions & 3 deletions accelerator/cuda_accelerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -326,9 +326,6 @@ def IntTensor(self):
def LongTensor(self):
return functools.partial(torch.tensor, dtype=torch.long, device='cuda')

def is_pinned(self, tensor):
return tensor.is_pinned()

def on_accelerator(self, tensor):
device_str = str(tensor.device)
if device_str.startswith('cuda:'):
Expand Down
5 changes: 1 addition & 4 deletions accelerator/hpu_accelerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,12 +231,9 @@ def IntTensor(self):
def LongTensor(self):
return functools.partial(torch.tensor, dtype=torch.long, device='hpu')

def _pin_memory(self, tensor, align_bytes=1):
def _torch_pin_memory(self, tensor):
return tensor.pin_memory(self.device())

def is_pinned(self, tensor):
return tensor.is_pinned()

def on_accelerator(self, tensor):
device_str = str(tensor.device)
if device_str.startswith('hpu:'):
Expand Down
3 changes: 0 additions & 3 deletions accelerator/mlu_accelerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,9 +223,6 @@ def IntTensor(self):
def LongTensor(self):
return functools.partial(torch.tensor, dtype=torch.long, device='mlu')

def is_pinned(self, tensor):
return tensor.is_pinned()

def on_accelerator(self, tensor):
device_str = str(tensor.device)
if device_str.startswith('mlu:'):
Expand Down
3 changes: 0 additions & 3 deletions accelerator/mps_accelerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,9 +214,6 @@ def IntTensor(self):
def LongTensor(self):
return

def is_pinned(self, tensor):
return tensor.is_pinned()

def on_accelerator(self, tensor):
device_str = str(tensor.device)
if device_str.startswith("mps"):
Expand Down
3 changes: 0 additions & 3 deletions accelerator/npu_accelerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,9 +222,6 @@ def IntTensor(self):
def LongTensor(self):
return torch.npu.LongTensor

def is_pinned(self, tensor):
return tensor.is_pinned()

def on_accelerator(self, tensor):
device_str = str(tensor.device)
if device_str.startswith('npu:'):
Expand Down
3 changes: 0 additions & 3 deletions accelerator/sdaa_accelerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,9 +251,6 @@ def IntTensor(self):
def LongTensor(self):
return functools.partial(torch.tensor, dtype=torch.long, device='sdaa')

def is_pinned(self, tensor):
return tensor.is_pinned()

def on_accelerator(self, tensor):
device_str = str(tensor.device)
if device_str.startswith('sdaa:'):
Expand Down
3 changes: 0 additions & 3 deletions accelerator/supa_accelerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,9 +214,6 @@ def IntTensor(self):
def LongTensor(self):
return torch.supa.LongTensor

def is_pinned(self, tensor):
return tensor.is_pinned()

def on_accelerator(self, tensor):
device_str = str(tensor.device)
return device_str.startswith('supa:')
Expand Down
26 changes: 5 additions & 21 deletions accelerator/xpu_accelerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ def __init__(self):
# changed to xccl if not using torch-CCL on XPU device
self._communication_backend_name = 'xccl'
self._compile_backend = "inductor"
self.aligned_tensors = []
self.class_dict = None

def is_synchronized_device(self):
Expand Down Expand Up @@ -229,26 +228,11 @@ def IntTensor(self):
def LongTensor(self):
return functools.partial(torch.tensor, dtype=torch.long, device=self._name)

def _pin_memory(self, tensor, align_bytes=1):

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this mean XPU accelerator needs to explicitly set DS_PIN_MEMORY_BACKEND to native with intended page-locked host buffer?

@rogerxfeng8 to confirm wheter XPU still relies on new_cpu_locked_tensor to work in certain scenario.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@delock @rogerxfeng8 yes it will be great to confirm the best support for XPU.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@delock @rogerxfeng8 do we need changes for XPU?

@delock delock Aug 13, 2026 •

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi Tunji, I have validate this on arc770 with pytorch for xpu and met no UT failure. I have approved the PR. @rogerxfeng8 if you think any additional test is needed plz comment this week, thanks!

if align_bytes == 1:
return tensor.pin_memory(device=self.current_device_name())
elif align_bytes == 0:
# Page-locked host buffer without starting AIO worker threads.
from deepspeed.ops.op_builder import PinMemoryBuilder
self.aio_handle = PinMemoryBuilder().load().pin_handle()
aligned_t = self.aio_handle.new_cpu_locked_tensor(tensor.numel(), tensor)
aligned_t = aligned_t[:tensor.numel()].copy_(tensor)
self.aligned_tensors.append([aligned_t.data_ptr(), aligned_t[-1].data_ptr()])
return aligned_t

def is_pinned(self, tensor):
if tensor.is_pinned(device=self.current_device_name()):
return True
else:
for begin, end in self.aligned_tensors:
if begin <= tensor.data_ptr() and tensor.data_ptr() <= end:
return True
return False
def _torch_pin_memory(self, tensor):
return tensor.pin_memory(device=self.current_device_name())

def _torch_is_pinned(self, tensor):
return tensor.is_pinned(device=self.current_device_name())

def op_builder_dir(self):
try:
Expand Down
13 changes: 9 additions & 4 deletions csrc/pin_memory/deepspeed_pin_tensor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,18 @@ torch::Tensor deepspeed_pin_tensor_t::alloc(const int64_t num_elem, const at::Sc
}

bool deepspeed_pin_tensor_t::free(torch::Tensor& locked_tensor)
{
return free(locked_tensor.data_ptr());
}

bool deepspeed_pin_tensor_t::free(void* addr)
{
std::lock_guard<std::mutex> guard(_mutex);
auto addr = locked_tensor.data_ptr();
if (_locked_tensors.find(addr) != _locked_tensors.end()) {
munlock(addr, _locked_tensors[addr]);
auto iter = _locked_tensors.find(addr);
if (iter != _locked_tensors.end()) {
munlock(addr, iter->second);
std::free(addr);
_locked_tensors.erase(addr);
_locked_tensors.erase(iter);
return true;
}

Expand Down
4 changes: 4 additions & 0 deletions csrc/pin_memory/deepspeed_pin_tensor.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ struct deepspeed_pin_tensor_t {

bool free(torch::Tensor& locked_tensor);

// Free by allocation base address. Lets callers release the original locked
// region even when the tensor's storage pointer has since been redirected.
bool free(void* addr);

bool is_managed(const torch::Tensor& buffer);
};

Expand Down
10 changes: 10 additions & 0 deletions csrc/pin_memory/py_ds_pin_memory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// DeepSpeed Team

#include <torch/extension.h>
#include <cstdint>
#include "deepspeed_pin_tensor.h"

using namespace pybind11::literals;
Expand All @@ -22,6 +23,11 @@ struct deepspeed_pin_handle_t {
return _pinned_tensor_mgr->free(locked_tensor);
}

bool free_cpu_locked_tensor_by_ptr(const uintptr_t address)
{
return _pinned_tensor_mgr->free(reinterpret_cast<void*>(address));
}

bool is_pinned(const torch::Tensor& buffer)
{
return buffer.is_pinned() || _pinned_tensor_mgr->is_managed(buffer);
Expand All @@ -41,6 +47,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m)
&deepspeed_pin_handle_t::free_cpu_locked_tensor,
"Free a page-locked CPU tensor",
"locked_tensor"_a)
.def("free_cpu_locked_tensor_by_ptr",
&deepspeed_pin_handle_t::free_cpu_locked_tensor_by_ptr,
"Free a page-locked CPU tensor by its allocation base address",
"address"_a)
.def("is_pinned",
&deepspeed_pin_handle_t::is_pinned,
"Return whether buffer is torch-pinned or managed by DeepSpeed pin_memory",
Expand Down
4 changes: 3 additions & 1 deletion deepspeed/compile/patch_fake_tensor.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

import torch

from deepspeed.accelerator import get_accelerator

try:
from torch._subclasses import FakeTensorMode
from torch._subclasses.fake_tensor import unset_fake_temporarily
Expand All @@ -20,7 +22,7 @@ def wrap_if_ds_param(t):
dtype=t.dtype,
layout=t.layout,
device=t.device,
pin_memory=t.is_pinned(),
pin_memory=get_accelerator().is_pinned(t),
requires_grad=t.requires_grad)
if isinstance(t, torch.nn.Parameter):
t = torch.nn.Parameter(data, requires_grad=t.requires_grad)
Expand Down
2 changes: 1 addition & 1 deletion deepspeed/compile/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ def convert(t):
device=t.device if device is None else device,
layout=t.layout,
requires_grad=t.requires_grad,
pin_memory=t.is_pinned())
pin_memory=get_accelerator().is_pinned(t))
else:
return torch.zeros(t.shape,
dtype=t.dtype,
Expand Down
17 changes: 17 additions & 0 deletions deepspeed/runtime/zenflow/zenflow_stage_1_and_2.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,23 @@ def create(cls, zenflow_config):
else:
return ZenFlowZeroOptimizerSequential

def _unpin_offload_buffers(self):
super()._unpin_offload_buffers()
if not (self.cpu_offload and self.cpu_offload_pin_memory):
return
# ZenFlow additionally pins per-parameter selective-optimizer state and, for the
# parallel variant, per-partition overlap-grad buffers on the host.
accelerator = get_accelerator()
for group in self.bit16_groups:
for param in group:
for attr in ('exp_avg_cpu_data', 'exp_avg_sq_cpu_data'):
buffer = getattr(param, attr, None)
if buffer is not None:
accelerator.unpin_memory(buffer)
for fp32_partition in self.single_partition_of_fp32_groups:
for buffer in getattr(fp32_partition, 'overlap_grad', None) or []:
accelerator.unpin_memory(buffer)

def _configure_zenflow(self, zenflow_config):
"""
Configure ZenFlow optimizer
Expand Down
31 changes: 31 additions & 0 deletions deepspeed/runtime/zero/stage3.py
Original file line number Diff line number Diff line change
Expand Up @@ -531,6 +531,37 @@ def destroy(self):
hook.remove()
print_rank_0("Removed grad acc hooks", force=False)
self.ipg_buckets.clear()
self._unpin_offload_buffers()

def _unpin_offload_buffers(self):
# Release the page-locked host buffers we pinned for parameter/optimizer offload.
# unpin_memory is a no-op for the torch backend and only frees under
# DS_PIN_MEMORY_BACKEND=native, where these would otherwise persist until GC.
accelerator = get_accelerator()
for buffer in getattr(self, 'param_groups_fp16_flat_cpu_memory', []):
accelerator.unpin_memory(buffer)
for attr in ('grad_partitions_flat_buffer', 'lp_param_contiguous_pin_buffer',
'lp_grad_partitions_flat_pin_buffers'):
buffer = getattr(self, attr, None)
if buffer is not None:
accelerator.unpin_memory(buffer)
for buffer in getattr(self, 'hp_params_pin_buffers', []):
accelerator.unpin_memory(buffer)
if self.offload_optimizer_pin_memory:
for fp32_partition in self.fp32_partitioned_groups_flat:
if fp32_partition.grad is not None:
accelerator.unpin_memory(fp32_partition.grad)
if self.zenflow:
# ZenFlow keeps per-parameter selective-optimizer state and per-subgroup
# overlap-grad buffers pinned on the host.
for param in self.module.parameters():
for attr in ('exp_avg_cpu_data', 'exp_avg_sq_cpu_data'):
buffer = getattr(param, attr, None)
if buffer is not None:
accelerator.unpin_memory(buffer)
for fp32_partition in getattr(self, 'fp32_partitioned_groups_flat', []):
for buffer in getattr(fp32_partition, 'overlap_grad', None) or []:
accelerator.unpin_memory(buffer)

def create_zenflow_hooks(self):
from functools import partial
Expand Down
18 changes: 18 additions & 0 deletions deepspeed/runtime/zero/stage_1_and_2.py
Original file line number Diff line number Diff line change
Expand Up @@ -681,6 +681,24 @@ def destroy(self):
for hook in self._grad_acc_hooks:
hook.remove()
self.print_rank_0("Removed grad acc hooks")
self._unpin_offload_buffers()

def _unpin_offload_buffers(self):
# Release the page-locked host buffers we pinned for CPU offload. unpin_memory is a
# no-op for the torch backend and only frees under DS_PIN_MEMORY_BACKEND=native,
# where the mlocked allocation would otherwise persist until garbage collection.
if not (self.cpu_offload and self.cpu_offload_pin_memory):
return
accelerator = get_accelerator()
for fp32_partition in self.single_partition_of_fp32_groups:
accelerator.unpin_memory(fp32_partition)
if fp32_partition.grad is not None:
accelerator.unpin_memory(fp32_partition.grad)
for buffer in self.param_buffer_of_bit16_for_cpu_offload_groups:
accelerator.unpin_memory(buffer)
temp_grad_buffer = getattr(self, 'temp_grad_buffer_for_cpu_offload', None)
if temp_grad_buffer is not None:
accelerator.unpin_memory(temp_grad_buffer)

def _enable_universal_checkpoint(self):
self._universal_checkpoint_info = None
Expand Down
Loading
Loading