diff --git a/common/common.cpp b/common/common.cpp index 5639bdf412..cea6d3f5cc 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1275,6 +1275,8 @@ struct common_init_result::impl { // note: the order in which model, context, etc. are declared matters because their destructors will be called bottom-to-top + common_threadpools threadpools; + llama_model_ptr model; llama_context_ptr context; @@ -1376,6 +1378,10 @@ common_init_result::common_init_result(common_params & params, bool model_only) } pimpl->context.reset(lctx); + + set_process_priority(params.cpuparams.priority); + + pimpl->threadpools.init(lctx, params); } llama_model * common_init_result::model() { @@ -1724,6 +1730,10 @@ struct llama_context_params common_context_params_to_llama(const common_params & return cparams; } +// +// Threadpool utils +// + struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params) { struct ggml_threadpool_params tpp; @@ -1740,6 +1750,56 @@ struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const commo return tpp; } +common_threadpools::~common_threadpools() { + if (!free_fn) { + return; + } + free_fn(threadpool); + free_fn(threadpool_batch); +} + +void common_threadpools::init(llama_context * ctx, const common_params & params) { + GGML_ASSERT(!threadpool); + GGML_ASSERT(!threadpool_batch); + + COM_INF("llama threadpool init, n_threads = %d\n", (int) params.cpuparams.n_threads); + + auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); + if (!cpu_dev) { + COM_WRN("%s", "no CPU backend found\n"); + return; + } + auto * reg = ggml_backend_dev_backend_reg(cpu_dev); + auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new"); + free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free"); + + struct ggml_threadpool_params tpp_batch = + ggml_threadpool_params_from_cpu_params(params.cpuparams_batch); + struct ggml_threadpool_params tpp = + ggml_threadpool_params_from_cpu_params(params.cpuparams); + + if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) { + threadpool_batch = ggml_threadpool_new_fn(&tpp_batch); + if (!threadpool_batch) { + COM_WRN("batch threadpool create failed : n_threads %d\n", tpp_batch.n_threads); + return; + } + + // start the non-batch threadpool in the paused state + tpp.paused = true; + } + + threadpool = ggml_threadpool_new_fn(&tpp); + if (!threadpool) { + COM_WRN("threadpool create failed : n_threads %d\n", tpp.n_threads); + free_fn(threadpool_batch); + threadpool_batch = nullptr; + return; + } + + llama_attach_threadpool(ctx, threadpool, threadpool_batch); +} + // // Batch utils // diff --git a/common/common.h b/common/common.h index b16bd3bb68..d8a16897b8 100644 --- a/common/common.h +++ b/common/common.h @@ -929,9 +929,8 @@ using common_init_result_ptr = std::unique_ptr; common_init_result_ptr common_init_from_params(common_params & params, bool model_only = false); -struct llama_model_params common_model_params_to_llama ( common_params & params); -struct llama_context_params common_context_params_to_llama(const common_params & params); -struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params); +struct llama_model_params common_model_params_to_llama ( common_params & params); +struct llama_context_params common_context_params_to_llama(const common_params & params); // clear LoRA adapters from context, then apply new list of adapters void common_set_adapter_lora(struct llama_context * ctx, std::vector & lora); @@ -942,6 +941,28 @@ std::string common_get_model_endpoint(); // for testing purposes char * common_get_model_or_exit(int, char*[]); +// +// Threadpool utils +// + +struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params); + +struct common_threadpools { + common_threadpools() = default; + ~common_threadpools(); + + common_threadpools(const common_threadpools &) = delete; + common_threadpools & operator=(const common_threadpools &) = delete; + + void init(llama_context * ctx, const common_params & params); + +private: + ggml_threadpool * threadpool = nullptr; + ggml_threadpool * threadpool_batch = nullptr; + + decltype(ggml_threadpool_free) * free_fn = nullptr; +}; + // // Context utils // diff --git a/tools/completion/completion.cpp b/tools/completion/completion.cpp index 6747558fc5..941b7399b2 100644 --- a/tools/completion/completion.cpp +++ b/tools/completion/completion.cpp @@ -160,47 +160,6 @@ int llama_completion(int argc, char ** argv) { // start measuring performance timings from here llama_perf_context_reset(ctx); - LOG_INF("%s: llama threadpool init, n_threads = %d\n", __func__, (int) params.cpuparams.n_threads); - - auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); - if (!cpu_dev) { - LOG_ERR("%s: no CPU backend found\n", __func__); - return 1; - } - auto * reg = ggml_backend_dev_backend_reg(cpu_dev); - auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new"); - auto * ggml_threadpool_free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free"); - - struct ggml_threadpool_params tpp_batch = - ggml_threadpool_params_from_cpu_params(params.cpuparams_batch); - struct ggml_threadpool_params tpp = - ggml_threadpool_params_from_cpu_params(params.cpuparams); - - if (!set_process_priority(params.cpuparams.priority)) { - LOG_ERR("%s: error: failed to set process priority\n", __func__); - return 1; - } - - struct ggml_threadpool * threadpool_batch = NULL; - if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) { - threadpool_batch = ggml_threadpool_new_fn(&tpp_batch); - if (!threadpool_batch) { - LOG_ERR("%s: batch threadpool create failed : n_threads %d\n", __func__, tpp_batch.n_threads); - return 1; - } - - // start the non-batch threadpool in the paused state - tpp.paused = true; - } - - struct ggml_threadpool * threadpool = ggml_threadpool_new_fn(&tpp); - if (!threadpool) { - LOG_ERR("%s: threadpool create failed : n_threads %d\n", __func__, tpp.n_threads); - return 1; - } - - llama_attach_threadpool(ctx, threadpool, threadpool_batch); - const int n_ctx_train = llama_model_n_ctx_train(model); const int n_ctx = llama_n_ctx(ctx); @@ -993,8 +952,5 @@ int llama_completion(int argc, char ** argv) { llama_backend_free(); - ggml_threadpool_free_fn(threadpool); - ggml_threadpool_free_fn(threadpool_batch); - return 0; }