mozempk/ggml-org-llama-cpp-mirror
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-13 00:10:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

convert : fix token configuration for PLaMo-3 (#29843)

Browse Source 
* convert : Fix token configuration for PLaMo-3

The reasoning and tool calling tags in PLaMo-3 consist of three tokens
each. For example, for reasoning:

* reasoning start: `<|plamo:begin_`, `think`, `:plamo|>`
* reasoning end: `<|plamo:end_`, `think`, `:plamo|>`

Registering `<|plamo:begin_`, `<|plamo:end_`, and `:plamo|>` as
`USER_DEFINED` so that they are parsed correctly.

Also PLaMo-3 models use <|plamo:tag|> as EOT, while PLaMo-2 models
use <|plamo:op|>.

Take EOT token as a parameter and look it up so that PLaMo-2 and
PLaMo-3 can use their appropriate ones.

* use NORMAL instead of USER_DEFINED
This commit is contained in:
Toki Nasin
2026-10-07 11:34:16 +02:00
committed by GitHub
parent ad21565331
commit 7481354a17
2 changed files with 17 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/base.py
+12 -6
View File
@@ -2499,7 +2499,11 @@ class TextModel(ModelBase):
if template is not None:
self.gguf_writer.add_chat_template(template)
def _set_vocab_plamo(self):
def _set_vocab_plamo(
self,
eot_token: str,
normal_tokens: Iterable[str] = (),
):
# PLaMo models use a custom tokenizer with a .jsonl file
tokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl"
tokenizer_config_path = self.dir_model / "tokenizer_config.json"
@@ -2523,27 +2527,30 @@ class TextModel(ModelBase):
tokens = []
scores = []
toktypes = []
normal_tokens = set(normal_tokens)
with open(tokenizer_jsonl_path, "r", encoding="utf-8") as f:
for line_num, line in enumerate(f):
if line.strip():
token_data = json.loads(line)
# Format: [token, score, type, ?, ?, ?, ?]
token = token_data[0].encode("utf-8")
token_str = token_data[0]
token = token_str.encode("utf-8")
score = float(token_data[1])
token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL"
tokens.append(token)
scores.append(score)
if token_type_str == "UNKNOWN":
if token_str in normal_tokens:
toktypes.append(gguf.TokenType.NORMAL)
elif token_type_str == "UNKNOWN":
toktypes.append(gguf.TokenType.UNKNOWN)
elif token_type_str == "CONTROL":
toktypes.append(gguf.TokenType.CONTROL)
elif token_type_str == "BYTE":
toktypes.append(gguf.TokenType.BYTE)
else:
token_str = token_data[0]
if token_str.startswith("<|plamo:") and token_str.endswith("|>"):
toktypes.append(gguf.TokenType.CONTROL)
else:
@@ -2580,8 +2587,7 @@ class TextModel(ModelBase):
token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8"))
self.gguf_writer.add_unk_token_id(token_id)
# Add <|plamo:op|> as EOT to ensure appropriate end of generation
self.gguf_writer.add_eot_token_id(4)
self.gguf_writer.add_eot_token_id(tokens.index(eot_token.encode("utf-8")))
self.gguf_writer.add_add_space_prefix(False)