mozempk/ggml-org-llama-cpp-mirror
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-15 20:18:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

metrics_on_prediction

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-12 13:42:03 +02:00
parent 509866bc9b
commit 6849901f4a
2 changed files with 25 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-common.h
-4
View File
@@ -338,8 +338,6 @@ json format_response_rerank(
// stats and metrics
//
struct server_slot;
// shared between server_slot and server_task_result_*
struct server_slot_stats {
uint64_t n_prompt_cached = 0;
@@ -507,8 +505,6 @@ struct server_metrics {
n_prompt_queued = 0;
}
// this is implemented in server-context.cpp
void on_prediction(const server_slot & slot);
};
//
tools/server/server-context.cpp
+25 -26
View File
@@ -780,30 +780,6 @@ struct server_slot {
};
//
// server_metrics
//
void server_metrics::on_prediction(const server_slot & slot) {
const uint64_t t_us = slot.stats.t_gen_us();
const uint64_t n = slot.stats.n_gen;
const uint64_t n_steps = slot.stats.n_gen_steps();
predict .add(n, n_steps, t_us);
predict_bucket.add(n, n_steps, t_us);
n_draft_tokens += slot.stats.n_draft_tokens;
n_draft_accepted += slot.stats.n_draft_accepted;
n_draft_verif_steps += slot.stats.n_draft_verif_steps;
if (n_accepted_per_pos.size() < slot.n_accepted_per_pos.size()) {
n_accepted_per_pos.resize(slot.n_accepted_per_pos.size(), 0);
}
for (size_t i = 0; i < slot.n_accepted_per_pos.size(); i++) {
n_accepted_per_pos[i] += slot.n_accepted_per_pos[i];
}
}
//
// server_context_impl (private implementation)
@@ -3762,7 +3738,7 @@ private:
// release slot because of stop condition
slot.print_timings();
send_final_response(slot);
metrics.on_prediction(slot);
metrics_on_prediction(slot);
slot.release();
return;
@@ -3883,7 +3859,7 @@ private:
if (!process_token(result, slot)) {
slot.print_timings();
send_final_response(slot);
metrics.on_prediction(slot);
metrics_on_prediction(slot);
slot.release();
return;
@@ -3945,6 +3921,29 @@ private:
metrics.flush_prompt();
}
}
void metrics_on_prediction(const server_slot & slot) {
const uint64_t t_us = slot.stats.t_gen_us();
const uint64_t n = slot.stats.n_gen;
const uint64_t n_steps = slot.stats.n_gen_steps();
metrics.predict .add(n, n_steps, t_us);
metrics.predict_bucket.add(n, n_steps, t_us);
metrics.n_draft_tokens += slot.stats.n_draft_tokens;
metrics.n_draft_accepted += slot.stats.n_draft_accepted;
metrics.n_draft_verif_steps += slot.stats.n_draft_verif_steps;
auto & dst = metrics.n_accepted_per_pos;
const auto & src = slot.n_accepted_per_pos;
if (dst.size() < src.size()) {
dst.resize(src.size(), 0);
}
for (size_t i = 0; i < src.size(); i++) {
dst[i] += src[i];
}
}
};
//