mozempk/ggml-org-llama-cpp-mirror
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-11 07:20:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

model : use exact GELU for ModernBERT encoders (#30108)

Browse Source 
* model : use exact GELU for ModernBERT encoders

Assisted-by: Codex

* model : keep tanh GELU aliases on ggml_geglu

Assisted-by: Claude Opus 5.5

* model : map gelu_python to ggml_geglu_erf

Assisted-by: Claude Opus 5.5
This commit is contained in:
bosh
2026-10-09 23:26:56 +08:00
committed by GitHub
parent 8b54361025
commit 6184e92c57
4 changed files with 22 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-graph.cpp
+5
View File
@@ -1963,6 +1963,11 @@ ggml_tensor * llm_graph_context::build_ffn(
cur = ggml_geglu(ctx0, cur);
cb(cur, "ffn_geglu", il);
} break;
case LLM_FFN_GEGLU_ERF:
{
cur = ggml_geglu_erf(ctx0, cur);
cb(cur, "ffn_geglu_erf", il);
} break;
case LLM_FFN_REGLU:
{
cur = ggml_reglu(ctx0, cur);
src/llama-graph.h
+1
View File
@@ -62,6 +62,7 @@ enum llm_ffn_op_type : int {
LLM_FFN_RELU_SQR,
LLM_FFN_SWIGLU,
LLM_FFN_GEGLU,
LLM_FFN_GEGLU_ERF,
LLM_FFN_REGLU,
LLM_FFN_SWIGLU_OAI_MOE,
LLM_FFN_SITU, // kimi-k3
src/llama-model.cpp
+14 -11
View File
@@ -1064,18 +1064,21 @@ static llama_rope_scaling_type llama_rope_scaling_type_from_string(const std::st
return LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;
}
// Maps the GGUF `<arch>.hidden_activation` string to the FFN op type used by the
// graph builders. Only gated activations that map cleanly to llm_ffn_op_type are
// listed; unrecognized values fall back to GeGLU, which matches the historical
// default for ModernBert-style architectures.
// Maps GGUF activation names to the FFN op type used by the graph builders.
static const std::map<std::string, llm_ffn_op_type> LLM_FFN_OP_TYPES_FROM_STRING = {
{ "gelu", LLM_FFN_GEGLU },
{ "geglu", LLM_FFN_GEGLU },
{ "silu", LLM_FFN_SWIGLU },
{ "swish", LLM_FFN_SWIGLU },
{ "swiglu", LLM_FFN_SWIGLU },
{ "relu", LLM_FFN_RELU },
{ "reglu", LLM_FFN_REGLU },
{ "gelu", LLM_FFN_GEGLU_ERF },
{ "gelu_python", LLM_FFN_GEGLU_ERF },
{ "gelu_pytorch_tanh", LLM_FFN_GEGLU },
{ "gelu_new", LLM_FFN_GEGLU },
{ "gelu_fast", LLM_FFN_GEGLU },
{ "gelu_accurate", LLM_FFN_GEGLU },
{ "gelu_python_tanh", LLM_FFN_GEGLU },
{ "geglu", LLM_FFN_GEGLU },
{ "silu", LLM_FFN_SWIGLU },
{ "swish", LLM_FFN_SWIGLU },
{ "swiglu", LLM_FFN_SWIGLU },
{ "relu", LLM_FFN_RELU },
{ "reglu", LLM_FFN_REGLU },
};
// transformers names, "gelu" is the exact (erf) variant
src/models/modern-bert.cpp
+2 -2
View File
@@ -17,10 +17,10 @@ void llama_model_modern_bert::load_arch_hparams(llama_model_loader & ml) {
// Some ModernBert derivatives (e.g. IBM Granite Embedding 97m R2) use
// SiLU/SwiGLU in the FFN instead of the default GELU/GeGLU.
hparams.llm_ffn_op = LLM_FFN_GEGLU;
hparams.llm_ffn_op = LLM_FFN_GEGLU_ERF;
std::string hidden_act;
if (ml.get_key(LLM_KV_HIDDEN_ACT, hidden_act, false)) {
hparams.llm_ffn_op = llm_ffn_op_type_from_string(hidden_act, LLM_FFN_GEGLU);
hparams.llm_ffn_op = llm_ffn_op_type_from_string(hidden_act, LLM_FFN_GEGLU_ERF);
}
// GGUFs without a classifier pooling type use mean (gte-reranker-modernbert-base)