Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion ggml/src/ggml-cuda/fattn-common.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,9 @@ static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_f16(
#pragma unroll
for (int k_KQ_1 = 0; k_KQ_1 < cpy_ne; ++k_KQ_1) {
#ifdef V_DOT2_F32_F16_AVAILABLE
ggml_cuda_mad(sum, tmp[k_KQ_1] , ((const half2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]);
// Not ggml_cuda_mad: its asm volatile keeps the compiler from issuing the next K loads before this dot product.
const half2 Q_k = ((const half2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1];
asm("v_dot2_f32_f16 %0, %1, %2, %0" : "+v"(sum) : "v"(tmp[k_KQ_1]), "v"(Q_k));
#else
ggml_cuda_mad(sum, __half22float2(tmp[k_KQ_1]), ((const float2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1]);
#endif // V_DOT2_F32_F16_AVAILABLE
Expand Down
31 changes: 18 additions & 13 deletions ggml/src/ggml-cuda/fattn-vec.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -260,10 +260,15 @@ static __global__ void flash_attn_ext_vec(
// Calculate KQ tile and keep track of new maximum KQ values:
float KQ_reg[ncols]; // KQ in registers.

float KQ_max_new[ncols];
// Each thread keeps one KQ row of the tile, row tid. Load the mask value for that row ahead of the K loads and
// apply softcap, mask and max after the dot products, so there are no branches or mask loads between the K loads:
float KQ_mask[ncols];
#pragma unroll
for (int j = 0; j < ncols; ++j) {
KQ_max_new[j] = KQ_max[j];
KQ_mask[j] = 0.0f;
if (mask && (ncols == 1 || ic0 + j < int(ne01.z))) {
KQ_mask[j] = slope*__half2float(maskh[j*ne11 + tid]);
}
}

#pragma unroll
Expand All @@ -275,26 +280,26 @@ static __global__ void flash_attn_ext_vec(
float sum = vec_dot_KQ(K + i_KQ*nb11, Q_reg[j], Q_i32[j], Q_ds[j]);
sum = warp_reduce_sum<nthreads_KQ>(sum);

if (use_logit_softcap) {
sum = logit_softcap*tanhf(sum);
}

if (mask && (ncols == 1 || ic0 + j < int(ne01.z))) {
sum += slope*__half2float(maskh[j*ne11 + i_KQ]);
}

KQ_max_new[j] = fmaxf(KQ_max_new[j], sum + FATTN_KQ_MAX_OFFSET);

if ((nthreads_KQ == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_KQ) == uint32_t(i_KQ_0)) {
KQ_reg[j] = sum;
}
}
}

float KQ_max_new[ncols];
#pragma unroll
for (int j = 0; j < ncols; ++j) {
if (use_logit_softcap) {
KQ_reg[j] = logit_softcap*tanhf(KQ_reg[j]);
}
KQ_reg[j] += KQ_mask[j];
KQ_max_new[j] = fmaxf(KQ_max[j], KQ_reg[j] + FATTN_KQ_MAX_OFFSET);
}

#pragma unroll
for (int j = 0; j < ncols; ++j) {
#pragma unroll
for (int offset = nthreads_KQ; offset < WARP_SIZE; offset <<= 1) {
for (int offset = 1; offset < WARP_SIZE; offset <<= 1) {
KQ_max_new[j] = fmaxf(KQ_max_new[j], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[j], offset, WARP_SIZE));
}
const float KQ_max_scale = expf(KQ_max[j] - KQ_max_new[j]);
Expand Down
Loading