mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-10-29 07:07:59 +01:00
Pranesh Gonegandlaandpraneshgo
5839ba3524
* CUDA: dedup MoE gate/up activation quantization (fp4) For MoE gate/up projections the src1 activation is broadcast across the routed experts (ne11 == 1), so ids_src1 maps every one of a token's n_expert_used slots to the same physical row. The MMQ path therefore re-quantized each token's activation n_expert_used times. For fp4 (NVFP4/MXFP4) src0, quantize each unique token row once instead of once per expert. For NVFP4 a single quantize+scatter kernel (quantize_scatter_mmq_nvfp4) quantizes each token once and writes the resulting block_fp4_mmq straight to all n_expert_used slots, using an inverse token->compact-row map (build_tok2c). MXFP4, and GGML_CUDA_MOE_QUANT_GATHER=1, use a two-kernel variant: quantize unique rows then gather into the expert-sorted layout (gather_mmq_fp4_blocks). Both are bit-identical to the previous gather-then-quantize path (identical source data, deterministic per-block quantization), verified by test-backend-ops MUL_MAT_ID (type_a=nvfp4, broadcast b=1; 790/790 for the default, gather, and per-expert paths) and by coherent end-to-end generation. Set GGML_CUDA_NO_MOE_QUANT_DEDUP=1 to force the original per-expert path. Same-binary A/B on RTX 5090 (sm_120), Qwen3.6-35B-A3B-NVFP4 prefill @8192 (nsys, graphs-off; the unchanged mul_mat_q GEMM confirms stable clocks): activation-quant GPU-busy drops 61% (78.2 -> 30.4 ms) with the fused quantize+scatter, vs 33% (78.2 -> 52.8 ms) for the two-kernel gather. The fused path avoids materializing and re-reading the 8x compact buffer, writing the expert copies directly from registers. * CUDA: bounds-check token ids in build_tok2c_kernel Guard against malformed ids_src1: skip out-of-range token ids (t < 0 or t >= n_tokens) and drop entries beyond n_expert_used per token instead of writing past the token's tok2c region. No behavior change for valid MoE routing data; test-backend-ops MUL_MAT_ID 790/790. * Refactor the code based on review comments - Removed previously added kernels that were not necessary anymore\ - Added an inverse mapping from (token, slot) to compact row. Each token is quantized once and scattered to its compact rows. * Adding q8_1 support for dedup and addressing review comments * Add pragma unrolls * Remove redundant cudaMemsetAsync call * Removing follow up redundancies --------- Co-authored-by: praneshgo <227579474+praneshgo@users.noreply.github.com>
67 lines
3.1 KiB
Plaintext
67 lines
3.1 KiB
Plaintext
#pragma once
|
|
|
|
#include "common.cuh"
|
|
#include "mmq.cuh"
|
|
|
|
#include <cstdint>
|
|
|
|
#define CUDA_QUANTIZE_BLOCK_SIZE 256
|
|
#define CUDA_QUANTIZE_BLOCK_SIZE_MMQ 128
|
|
|
|
static_assert(MATRIX_ROW_PADDING % CUDA_QUANTIZE_BLOCK_SIZE == 0, "Risk of out-of-bounds access.");
|
|
static_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, "Risk of out-of-bounds access.");
|
|
|
|
typedef void (*quantize_cuda_t)(
|
|
const float * x, const int32_t * ids, void * vy,
|
|
ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
|
|
int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
|
|
|
|
void quantize_row_q8_1_cuda(
|
|
const float * x, const int32_t * ids, void * vy,
|
|
ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
|
|
int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
|
|
|
|
void quantize_mmq_q8_1_cuda(
|
|
const float * x, const int32_t * ids, void * vy,
|
|
ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
|
|
int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
|
|
|
|
void quantize_mmq_fp4_cuda(const float * x,
|
|
const int32_t * ids,
|
|
void * vy,
|
|
ggml_type type_src0,
|
|
int64_t ne00,
|
|
int64_t s01,
|
|
int64_t s02,
|
|
int64_t s03,
|
|
int64_t ne0,
|
|
int64_t ne1,
|
|
int64_t ne2,
|
|
int64_t ne3,
|
|
cudaStream_t stream);
|
|
|
|
// quantize each token once and scatter the block to its compact rows (via the inverse map)
|
|
void quantize_scatter_mmq_fp4_cuda(const float * x,
|
|
const int32_t * ids_src1_inv,
|
|
void * vy,
|
|
ggml_type type_src0,
|
|
int64_t ne00,
|
|
int64_t stride_token,
|
|
int64_t ne0,
|
|
int64_t n_tokens,
|
|
int64_t nrows_dst,
|
|
int n_expert_used,
|
|
cudaStream_t stream);
|
|
|
|
void quantize_scatter_mmq_q8_1_cuda(const float * x,
|
|
const int32_t * ids_src1_inv,
|
|
void * vy,
|
|
ggml_type type_src0,
|
|
int64_t ne00,
|
|
int64_t stride_token,
|
|
int64_t ne0,
|
|
int64_t n_tokens,
|
|
int64_t nrows_dst,
|
|
int n_expert_used,
|
|
cudaStream_t stream);
|