mozempk/ggml-org-llama-cpp-mirror
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-15 12:08:31 +02:00
Code Issues Packages Projects Releases Wiki Activity

common: apply CPU parameters across tools (#27026)

Browse Source
This commit is contained in:
Niklas Wenzel
2026-08-13 20:40:59 +02:00
committed by GitHub
parent bdffafa5df
commit a94d563ed8
3 changed files with 84 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/common.cpp
+60
View File
@@ -1275,6 +1275,8 @@ struct common_init_result::impl {
// note: the order in which model, context, etc. are declared matters because their destructors will be called bottom-to-top
common_threadpools threadpools;
llama_model_ptr model;
llama_context_ptr context;
@@ -1376,6 +1378,10 @@ common_init_result::common_init_result(common_params & params, bool model_only)
}
pimpl->context.reset(lctx);
set_process_priority(params.cpuparams.priority);
pimpl->threadpools.init(lctx, params);
}
llama_model * common_init_result::model() {
@@ -1724,6 +1730,10 @@ struct llama_context_params common_context_params_to_llama(const common_params &
return cparams;
}
//
// Threadpool utils
//
struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params) {
struct ggml_threadpool_params tpp;
@@ -1740,6 +1750,56 @@ struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const commo
return tpp;
}
common_threadpools::~common_threadpools() {
if (!free_fn) {
return;
}
free_fn(threadpool);
free_fn(threadpool_batch);
}
void common_threadpools::init(llama_context * ctx, const common_params & params) {
GGML_ASSERT(!threadpool);
GGML_ASSERT(!threadpool_batch);
COM_INF("llama threadpool init, n_threads = %d\n", (int) params.cpuparams.n_threads);
auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
if (!cpu_dev) {
COM_WRN("%s", "no CPU backend found\n");
return;
}
auto * reg = ggml_backend_dev_backend_reg(cpu_dev);
auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");
free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");
struct ggml_threadpool_params tpp_batch =
ggml_threadpool_params_from_cpu_params(params.cpuparams_batch);
struct ggml_threadpool_params tpp =
ggml_threadpool_params_from_cpu_params(params.cpuparams);
if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) {
threadpool_batch = ggml_threadpool_new_fn(&tpp_batch);
if (!threadpool_batch) {
COM_WRN("batch threadpool create failed : n_threads %d\n", tpp_batch.n_threads);
return;
}
// start the non-batch threadpool in the paused state
tpp.paused = true;
}
threadpool = ggml_threadpool_new_fn(&tpp);
if (!threadpool) {
COM_WRN("threadpool create failed : n_threads %d\n", tpp.n_threads);
free_fn(threadpool_batch);
threadpool_batch = nullptr;
return;
}
llama_attach_threadpool(ctx, threadpool, threadpool_batch);
}
//
// Batch utils
//
common/common.h
+22 -1
View File
@@ -931,7 +931,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
struct llama_model_params common_model_params_to_llama ( common_params & params);
struct llama_context_params common_context_params_to_llama(const common_params & params);
struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params);
// clear LoRA adapters from context, then apply new list of adapters
void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);
@@ -942,6 +941,28 @@ std::string common_get_model_endpoint();
// for testing purposes
char * common_get_model_or_exit(int, char*[]);
//
// Threadpool utils
//
struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params);
struct common_threadpools {
common_threadpools() = default;
~common_threadpools();
common_threadpools(const common_threadpools &) = delete;
common_threadpools & operator=(const common_threadpools &) = delete;
void init(llama_context * ctx, const common_params & params);
private:
ggml_threadpool * threadpool = nullptr;
ggml_threadpool * threadpool_batch = nullptr;
decltype(ggml_threadpool_free) * free_fn = nullptr;
};
//
// Context utils
//
tools/completion/completion.cpp
-44
View File
@@ -160,47 +160,6 @@ int llama_completion(int argc, char ** argv) {
// start measuring performance timings from here
llama_perf_context_reset(ctx);
LOG_INF("%s: llama threadpool init, n_threads = %d\n", __func__, (int) params.cpuparams.n_threads);
auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
if (!cpu_dev) {
LOG_ERR("%s: no CPU backend found\n", __func__);
return 1;
}
auto * reg = ggml_backend_dev_backend_reg(cpu_dev);
auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");
auto * ggml_threadpool_free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");
struct ggml_threadpool_params tpp_batch =
ggml_threadpool_params_from_cpu_params(params.cpuparams_batch);
struct ggml_threadpool_params tpp =
ggml_threadpool_params_from_cpu_params(params.cpuparams);
if (!set_process_priority(params.cpuparams.priority)) {
LOG_ERR("%s: error: failed to set process priority\n", __func__);
return 1;
}
struct ggml_threadpool * threadpool_batch = NULL;
if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) {
threadpool_batch = ggml_threadpool_new_fn(&tpp_batch);
if (!threadpool_batch) {
LOG_ERR("%s: batch threadpool create failed : n_threads %d\n", __func__, tpp_batch.n_threads);
return 1;
}
// start the non-batch threadpool in the paused state
tpp.paused = true;
}
struct ggml_threadpool * threadpool = ggml_threadpool_new_fn(&tpp);
if (!threadpool) {
LOG_ERR("%s: threadpool create failed : n_threads %d\n", __func__, tpp.n_threads);
return 1;
}
llama_attach_threadpool(ctx, threadpool, threadpool_batch);
const int n_ctx_train = llama_model_n_ctx_train(model);
const int n_ctx = llama_n_ctx(ctx);
@@ -993,8 +952,5 @@ int llama_completion(int argc, char ** argv) {
llama_backend_free();
ggml_threadpool_free_fn(threadpool);
ggml_threadpool_free_fn(threadpool_batch);
return 0;
}