mozempk/ggml-org-llama-cpp-mirror
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-11 07:20:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

server: read the GGUF metadata once per model

Browse Source 
Read the decision type and the trained context in a single GGUF open,
accept only a UINT32 context length like the model loader, and reset
n_ctx_train with the other caps so a failed refresh drops it.
This commit is contained in:
Pascal
2026-10-09 14:30:26 +02:00
parent a872e623be
commit 831d7e7427
3 changed files with 32 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/common.cpp
+20 -53
View File
@@ -1195,7 +1195,9 @@ common_decision_type common_get_decision_type(const struct llama_model * model)
return common_decision_type_from_string(buf);
}
common_decision_type common_get_decision_type(const std::string & fname) {
common_gguf_info common_get_gguf_info(const std::string & fname) {
common_gguf_info info;
struct gguf_init_params gguf_params = {
/* .no_alloc = */ true,
/* .ctx = */ nullptr,
@@ -1203,67 +1205,32 @@ common_decision_type common_get_decision_type(const std::string & fname) {
gguf_context_ptr gguf_ctx(gguf_init_from_file(fname.c_str(), gguf_params));
if (!gguf_ctx) {
return COMMON_DECISION_TYPE_UNKNOWN; // missing or unreadable file
}
std::string arch;
const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");
if (arch_id < 0) {
return COMMON_DECISION_TYPE_UNKNOWN; // no architecture in the metadata
}
if (gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
return COMMON_DECISION_TYPE_UNKNOWN; // malformed metadata
}
arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
if (arch.empty()) {
return COMMON_DECISION_TYPE_UNKNOWN;
}
const std::string key = arch + ".decision.type";
const int64_t type_id = gguf_find_key(gguf_ctx.get(), key.c_str());
if (type_id < 0) {
return COMMON_DECISION_TYPE_NONE;
}
if (gguf_get_kv_type(gguf_ctx.get(), type_id) != GGUF_TYPE_STRING) {
return COMMON_DECISION_TYPE_UNKNOWN; // malformed metadata
}
return common_decision_type_from_string(gguf_get_val_str(gguf_ctx.get(), type_id));
}
uint32_t common_get_gguf_n_ctx_train(const std::string & fname) {
struct gguf_init_params gguf_params = {
/* .no_alloc = */ true,
/* .ctx = */ nullptr,
};
gguf_context_ptr gguf_ctx(gguf_init_from_file(fname.c_str(), gguf_params));
if (!gguf_ctx) {
return 0; // missing or unreadable file
return info; // missing or unreadable file
}
const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");
if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
return 0;
return info; // no architecture in the metadata
}
const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
if (arch.empty()) {
return info;
}
const std::string key = std::string(gguf_get_val_str(gguf_ctx.get(), arch_id)) + ".context_length";
const int64_t key_id = gguf_find_key(gguf_ctx.get(), key.c_str());
if (key_id < 0) {
return 0;
const int64_t type_id = gguf_find_key(gguf_ctx.get(), (arch + ".decision.type").c_str());
if (type_id < 0) {
info.decision_type = COMMON_DECISION_TYPE_NONE;
} else if (gguf_get_kv_type(gguf_ctx.get(), type_id) == GGUF_TYPE_STRING) {
info.decision_type = common_decision_type_from_string(gguf_get_val_str(gguf_ctx.get(), type_id));
}
// writers are not strict about the width of the hparams, so accept both
switch (gguf_get_kv_type(gguf_ctx.get(), key_id)) {
case GGUF_TYPE_UINT32:
return gguf_get_val_u32(gguf_ctx.get(), key_id);
case GGUF_TYPE_UINT64:
{
const uint64_t val = gguf_get_val_u64(gguf_ctx.get(), key_id);
return val <= UINT32_MAX ? (uint32_t) val : 0;
}
default:
return 0;
// same key and type as the model loader
const int64_t ctx_id = gguf_find_key(gguf_ctx.get(), (arch + ".context_length").c_str());
if (ctx_id >= 0 && gguf_get_kv_type(gguf_ctx.get(), ctx_id) == GGUF_TYPE_UINT32) {
info.n_ctx_train = gguf_get_val_u32(gguf_ctx.get(), ctx_id);
}
return info;
}
common_init_result::common_init_result(common_params & params, bool model_only) :
common/common.h
+6 -6
View File
@@ -970,13 +970,13 @@ enum common_decision_type {
common_decision_type common_get_decision_type(const struct llama_model * model);
// same as above, but reads a GGUF file; it does not load the model
// returns COMMON_DECISION_TYPE_UNKNOWN if the file is missing, unreadable, or invalid
common_decision_type common_get_decision_type(const std::string & fname);
// metadata of a GGUF file, read without loading the model
struct common_gguf_info {
common_decision_type decision_type = COMMON_DECISION_TYPE_UNKNOWN; // UNKNOWN if the file is missing, unreadable, or invalid
uint32_t n_ctx_train = 0; // 0 if unknown
};
// reads the trained context of a GGUF file from its metadata; it does not load the model
// returns 0 if the file is missing, unreadable, invalid, or reports no context length
uint32_t common_get_gguf_n_ctx_train(const std::string & fname);
common_gguf_info common_get_gguf_info(const std::string & fname);
// note: defines the model, context, samplers, ets. lifetimes
struct common_init_result {
tools/server/server-models.cpp
+6 -8
View File
@@ -540,6 +540,7 @@ void server_model_meta::update_args(common_preset_context & ctx_preset, std::str
void server_model_meta::update_caps(const common_params & base) {
// reset to the default so a failed refresh cannot keep old values
architecture = server_model_architecture_json(false, false, false, {"text"});
n_ctx_train = 0;
// resolve the model file offline; do not download
common_params params;
@@ -564,10 +565,12 @@ void server_model_meta::update_caps(const common_params & base) {
return;
}
// read the output modalities from the GGUF metadata
// read the output modalities and the trained context from the GGUF metadata
std::vector<std::string> output_modalities = {"text"};
if (!params.model.path.empty()) {
output_modalities = server_model_output_modalities(common_get_decision_type(params.model.path));
const common_gguf_info info = common_get_gguf_info(params.model.path);
output_modalities = server_model_output_modalities(info.decision_type);
n_ctx_train = info.n_ctx_train;
}
bool inp_image = false;
@@ -585,11 +588,6 @@ void server_model_meta::update_caps(const common_params & base) {
// offline discovery cannot see video; a loaded model reports it
architecture = server_model_architecture_json(inp_image, inp_audio, false, output_modalities);
// the trained context, read from the GGUF metadata
if (!params.model.path.empty()) {
n_ctx_train = common_get_gguf_n_ctx_train(params.model.path);
}
}
//
@@ -2127,7 +2125,7 @@ void server_models_routes::init_routes() {
{"source", server_model_source_to_string(meta.source)},
{"can_remove", meta.source == SERVER_MODEL_SOURCE_CACHE},
// {"need_download", meta.need_download},
// TODO: add other fields, may require reading GGUF metadata
// TODO: add other fields from the GGUF metadata
};
if (meta.n_ctx_train > 0) {