mozempk/ggml-org-llama-cpp-mirror
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-06 14:42:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add RMS norm and use it (#187)

Browse Source 
* add ggml_rms_norm

* update op num
This commit is contained in:
hoangmit
2023-03-16 00:41:38 +02:00
committed by GitHub
parent 27944c4206
commit 6eac39ba95
3 changed files with 134 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.cpp
+3 -3
View File
@@ -588,7 +588,7 @@ bool llama_eval(
// norm
{
cur = ggml_norm(ctx0, inpL);
cur = ggml_rms_norm(ctx0, inpL);
// cur = attention_norm*cur
cur = ggml_mul(ctx0,
@@ -678,7 +678,7 @@ bool llama_eval(
{
// norm
{
cur = ggml_norm(ctx0, inpFF);
cur = ggml_rms_norm(ctx0, inpFF);
// cur = ffn_norm*cur
cur = ggml_mul(ctx0,
@@ -713,7 +713,7 @@ bool llama_eval(
// norm
{
inpL = ggml_norm(ctx0, inpL);
inpL = ggml_rms_norm(ctx0, inpL);
// inpL = norm*inpL
inpL = ggml_mul(ctx0,