From 4e4cb0759fc30ad5bea87ccc2c4843dfd3e892ea Mon Sep 17 00:00:00 2001
From: Ade9 <adenyne@pm.me>
Date: Mon, 28 Apr 2025 13:46:58 +0300
Subject: [PATCH] more metrics added

---
 llmproxymetrics.go | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/llmproxymetrics.go b/llmproxymetrics.go
index b8e3d61..8dbb5e2 100644
--- a/llmproxymetrics.go
+++ b/llmproxymetrics.go
@@ -91,6 +91,7 @@ func createProxy(target *url.URL) func(http.ResponseWriter, *http.Request) {
 		proxy.Director = director
 		if r.URL.Path == "/api/generate" || r.URL.Path == "/api/chat" {
 			proxy.ModifyResponse = modifyResponse
+			concurrent_requests.Inc()
 		}
 
 		proxy.ServeHTTP(w, r)
@@ -114,12 +115,15 @@ func handleJsonLine(line []byte, wasstreamed int) {
 		opsProcessed.Inc()
 		tokens_out.Add(jsonData["eval_count"].(float64))
 		tokens_in.Add(jsonData["prompt_eval_count"].(float64))
+		prompt_length.Observe(jsonData["prompt_eval_count"].(float64))
+		eval_length.Observe(jsonData["eval_count"].(float64))
 		eval_time.Observe(duration)
 		if wasstreamed == 1 {
 			streamed.Inc()
 		} else if wasstreamed == 0 {
 			notStreamed.Inc()
 		}
+		concurrent_requests.Dec()
 	}
 }
 
@@ -146,8 +150,22 @@ var tokens_in = promauto.NewCounter(prometheus.CounterOpts{
 })
 var eval_time = promauto.NewHistogram(prometheus.HistogramOpts{
 	Name:    "llmproxymetrics_eval_time",
-	Help:    "Tokens generated.",
-	Buckets: prometheus.LinearBuckets(0, 2.5, 20),
+	Help:    "Seconds spent evaluating the response.",
+	Buckets: prometheus.LinearBuckets(5, 5, 23),
+})
+var prompt_length = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "llmproxymetrics_prompt_length",
+	Help:    "Prompt length",
+	Buckets: prometheus.ExponentialBucketsRange(100, 32000, 20),
+})
+var eval_length = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "llmproxymetrics_eval_length",
+	Help:    "Eval length",
+	Buckets: prometheus.ExponentialBucketsRange(100, 32000, 20),
+})
+var concurrent_requests = promauto.NewGauge(prometheus.GaugeOpts{
+	Name: "llmproxymetrics_concurrent_requests",
+	Help: "Concurrent requests",
 })
 
 func main() {