diff --git a/common/common.cpp b/common/common.cpp index cfa892f61f..28ab9ac6c8 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1195,7 +1195,9 @@ common_decision_type common_get_decision_type(const struct llama_model * model) return common_decision_type_from_string(buf); } -common_decision_type common_get_decision_type(const std::string & fname) { +common_gguf_info common_get_gguf_info(const std::string & fname) { + common_gguf_info info; + struct gguf_init_params gguf_params = { /* .no_alloc = */ true, /* .ctx = */ nullptr, @@ -1203,67 +1205,32 @@ common_decision_type common_get_decision_type(const std::string & fname) { gguf_context_ptr gguf_ctx(gguf_init_from_file(fname.c_str(), gguf_params)); if (!gguf_ctx) { - return COMMON_DECISION_TYPE_UNKNOWN; // missing or unreadable file - } - - std::string arch; - const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture"); - if (arch_id < 0) { - return COMMON_DECISION_TYPE_UNKNOWN; // no architecture in the metadata - } - if (gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) { - return COMMON_DECISION_TYPE_UNKNOWN; // malformed metadata - } - arch = gguf_get_val_str(gguf_ctx.get(), arch_id); - if (arch.empty()) { - return COMMON_DECISION_TYPE_UNKNOWN; - } - - const std::string key = arch + ".decision.type"; - const int64_t type_id = gguf_find_key(gguf_ctx.get(), key.c_str()); - if (type_id < 0) { - return COMMON_DECISION_TYPE_NONE; - } - if (gguf_get_kv_type(gguf_ctx.get(), type_id) != GGUF_TYPE_STRING) { - return COMMON_DECISION_TYPE_UNKNOWN; // malformed metadata - } - return common_decision_type_from_string(gguf_get_val_str(gguf_ctx.get(), type_id)); -} - -uint32_t common_get_gguf_n_ctx_train(const std::string & fname) { - struct gguf_init_params gguf_params = { - /* .no_alloc = */ true, - /* .ctx = */ nullptr, - }; - - gguf_context_ptr gguf_ctx(gguf_init_from_file(fname.c_str(), gguf_params)); - if (!gguf_ctx) { - return 0; // missing or unreadable file + return info; // missing or unreadable file } const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture"); if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) { - return 0; + return info; // no architecture in the metadata + } + const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id); + if (arch.empty()) { + return info; } - const std::string key = std::string(gguf_get_val_str(gguf_ctx.get(), arch_id)) + ".context_length"; - const int64_t key_id = gguf_find_key(gguf_ctx.get(), key.c_str()); - if (key_id < 0) { - return 0; + const int64_t type_id = gguf_find_key(gguf_ctx.get(), (arch + ".decision.type").c_str()); + if (type_id < 0) { + info.decision_type = COMMON_DECISION_TYPE_NONE; + } else if (gguf_get_kv_type(gguf_ctx.get(), type_id) == GGUF_TYPE_STRING) { + info.decision_type = common_decision_type_from_string(gguf_get_val_str(gguf_ctx.get(), type_id)); } - // writers are not strict about the width of the hparams, so accept both - switch (gguf_get_kv_type(gguf_ctx.get(), key_id)) { - case GGUF_TYPE_UINT32: - return gguf_get_val_u32(gguf_ctx.get(), key_id); - case GGUF_TYPE_UINT64: - { - const uint64_t val = gguf_get_val_u64(gguf_ctx.get(), key_id); - return val <= UINT32_MAX ? (uint32_t) val : 0; - } - default: - return 0; + // same key and type as the model loader + const int64_t ctx_id = gguf_find_key(gguf_ctx.get(), (arch + ".context_length").c_str()); + if (ctx_id >= 0 && gguf_get_kv_type(gguf_ctx.get(), ctx_id) == GGUF_TYPE_UINT32) { + info.n_ctx_train = gguf_get_val_u32(gguf_ctx.get(), ctx_id); } + + return info; } common_init_result::common_init_result(common_params & params, bool model_only) : diff --git a/common/common.h b/common/common.h index d8c05e7b7d..d2fe10fd79 100644 --- a/common/common.h +++ b/common/common.h @@ -970,13 +970,13 @@ enum common_decision_type { common_decision_type common_get_decision_type(const struct llama_model * model); -// same as above, but reads a GGUF file; it does not load the model -// returns COMMON_DECISION_TYPE_UNKNOWN if the file is missing, unreadable, or invalid -common_decision_type common_get_decision_type(const std::string & fname); +// metadata of a GGUF file, read without loading the model +struct common_gguf_info { + common_decision_type decision_type = COMMON_DECISION_TYPE_UNKNOWN; // UNKNOWN if the file is missing, unreadable, or invalid + uint32_t n_ctx_train = 0; // 0 if unknown +}; -// reads the trained context of a GGUF file from its metadata; it does not load the model -// returns 0 if the file is missing, unreadable, invalid, or reports no context length -uint32_t common_get_gguf_n_ctx_train(const std::string & fname); +common_gguf_info common_get_gguf_info(const std::string & fname); // note: defines the model, context, samplers, ets. lifetimes struct common_init_result { diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 454522f04f..60353240ac 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -540,6 +540,7 @@ void server_model_meta::update_args(common_preset_context & ctx_preset, std::str void server_model_meta::update_caps(const common_params & base) { // reset to the default so a failed refresh cannot keep old values architecture = server_model_architecture_json(false, false, false, {"text"}); + n_ctx_train = 0; // resolve the model file offline; do not download common_params params; @@ -564,10 +565,12 @@ void server_model_meta::update_caps(const common_params & base) { return; } - // read the output modalities from the GGUF metadata + // read the output modalities and the trained context from the GGUF metadata std::vector output_modalities = {"text"}; if (!params.model.path.empty()) { - output_modalities = server_model_output_modalities(common_get_decision_type(params.model.path)); + const common_gguf_info info = common_get_gguf_info(params.model.path); + output_modalities = server_model_output_modalities(info.decision_type); + n_ctx_train = info.n_ctx_train; } bool inp_image = false; @@ -585,11 +588,6 @@ void server_model_meta::update_caps(const common_params & base) { // offline discovery cannot see video; a loaded model reports it architecture = server_model_architecture_json(inp_image, inp_audio, false, output_modalities); - - // the trained context, read from the GGUF metadata - if (!params.model.path.empty()) { - n_ctx_train = common_get_gguf_n_ctx_train(params.model.path); - } } // @@ -2127,7 +2125,7 @@ void server_models_routes::init_routes() { {"source", server_model_source_to_string(meta.source)}, {"can_remove", meta.source == SERVER_MODEL_SOURCE_CACHE}, // {"need_download", meta.need_download}, - // TODO: add other fields, may require reading GGUF metadata + // TODO: add other fields from the GGUF metadata }; if (meta.n_ctx_train > 0) {