Skip to content
7 changes: 7 additions & 0 deletions ggml/src/ggml-cuda/common.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -1443,6 +1443,13 @@ struct ggml_backend_cuda_context {

std::unique_ptr<ggml_cuda_stream_context> concurrent_stream_context;

struct {
const ggml_tensor * mask = nullptr;
int64_t key[7] = {};
std::vector<const ggml_tensor *> inputs;
std::unique_ptr<ggml_cuda_pool_alloc<uint8_t>> buf;
} fattn_cls;

// dedicated buffer for the branches of overlapped concurrent regions (attention QKV, MoE shared
// expert), reused across layers so their scratch never aliases tensors read across the region
ggml_backend_buffer_t concurrent_scratch = nullptr;
Expand Down
Loading
Loading