From 0e084b142e3efa33e0fc2733a51037450067daa2 Mon Sep 17 00:00:00 2001 From: Annie Ren Date: Tue, 17 Nov 2026 23:24:49 -0700 Subject: [PATCH] Reduce vmcnt for fa --- ggml/src/ggml-cuda/fattn-common.cuh | 4 +++- ggml/src/ggml-cuda/fattn-vec.cuh | 31 +++++++++++++++++------------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index beb877941c75..56f4c7fedd2e 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -105,7 +105,9 @@ static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_f16( #pragma unroll for (int k_KQ_1 = 0; k_KQ_1 < cpy_ne; ++k_KQ_1) { #ifdef V_DOT2_F32_F16_AVAILABLE - ggml_cuda_mad(sum, tmp[k_KQ_1] , ((const half2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]); + // Not ggml_cuda_mad: its asm volatile keeps the compiler from issuing the next K loads before this dot product. + const half2 Q_k = ((const half2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]; + asm("v_dot2_f32_f16 %0, %1, %2, %0" : "+v"(sum) : "v"(tmp[k_KQ_1]), "v"(Q_k)); #else ggml_cuda_mad(sum, __half22float2(tmp[k_KQ_1]), ((const float2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]); #endif // V_DOT2_F32_F16_AVAILABLE diff --git a/ggml/src/ggml-cuda/fattn-vec.cuh b/ggml/src/ggml-cuda/fattn-vec.cuh index 9df15128f0ca..a123df02ebab 100644 --- a/ggml/src/ggml-cuda/fattn-vec.cuh +++ b/ggml/src/ggml-cuda/fattn-vec.cuh @@ -260,10 +260,15 @@ static __global__ void flash_attn_ext_vec( // Calculate KQ tile and keep track of new maximum KQ values: float KQ_reg[ncols]; // KQ in registers. - float KQ_max_new[ncols]; + // Each thread keeps one KQ row of the tile, row tid. Load the mask value for that row ahead of the K loads and + // apply softcap, mask and max after the dot products, so there are no branches or mask loads between the K loads: + float KQ_mask[ncols]; #pragma unroll for (int j = 0; j < ncols; ++j) { - KQ_max_new[j] = KQ_max[j]; + KQ_mask[j] = 0.0f; + if (mask && (ncols == 1 || ic0 + j < int(ne01.z))) { + KQ_mask[j] = slope*__half2float(maskh[j*ne11 + tid]); + } } #pragma unroll @@ -275,26 +280,26 @@ static __global__ void flash_attn_ext_vec( float sum = vec_dot_KQ(K + i_KQ*nb11, Q_reg[j], Q_i32[j], Q_ds[j]); sum = warp_reduce_sum(sum); - if (use_logit_softcap) { - sum = logit_softcap*tanhf(sum); - } - - if (mask && (ncols == 1 || ic0 + j < int(ne01.z))) { - sum += slope*__half2float(maskh[j*ne11 + i_KQ]); - } - - KQ_max_new[j] = fmaxf(KQ_max_new[j], sum + FATTN_KQ_MAX_OFFSET); - if ((nthreads_KQ == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_KQ) == uint32_t(i_KQ_0)) { KQ_reg[j] = sum; } } } + float KQ_max_new[ncols]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { + if (use_logit_softcap) { + KQ_reg[j] = logit_softcap*tanhf(KQ_reg[j]); + } + KQ_reg[j] += KQ_mask[j]; + KQ_max_new[j] = fmaxf(KQ_max[j], KQ_reg[j] + FATTN_KQ_MAX_OFFSET); + } + #pragma unroll for (int j = 0; j < ncols; ++j) { #pragma unroll - for (int offset = nthreads_KQ; offset < WARP_SIZE; offset <<= 1) { + for (int offset = 1; offset < WARP_SIZE; offset <<= 1) { KQ_max_new[j] = fmaxf(KQ_max_new[j], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[j], offset, WARP_SIZE)); } const float KQ_max_scale = expf(KQ_max[j] - KQ_max_new[j]);