few more metrics added

llmproxymetrics.go

@@ -39,6 +39,14 @@ func createProxy(target *url.URL) func(http.ResponseWriter, *http.Request) {
 			req.URL.Scheme = target.Scheme
 			req.URL.Host = target.Host
 		}
+		var jsonData map[string]interface{}
+		json.NewDecoder(r.Body).Decode(&jsonData)
+
+		if jsonData["stream"].(bool) == false {
+			notStreamed.Inc()
+		} else {
+			streamed.Inc()
+		}
 
 		modifyResponse := func(response *http.Response) error {
 			pr, pw := io.Pipe()
@@ -87,10 +95,11 @@ func handleJsonLine(line []byte) {
 	}
 
 	if jsonData["done"].(bool) {
-		duration := jsonData["eval_duration"].(float64)
-		fmt.Printf("Duration: %.2f seconds\n", duration/1000000000.0)
+		duration := jsonData["eval_duration"].(float64) / 1000000000.0
+		fmt.Printf("Duration: %.2f seconds\n", duration)
 		opsProcessed.Inc()
-		tokens_out.Observe(jsonData["prompt_eval_count"].(float64))
+		tokens_out.Add(jsonData["prompt_eval_count"].(float64))
+		eval_time.Observe(duration)
 	}
 }
 
@@ -99,10 +108,23 @@ var opsProcessed = promauto.NewCounter(prometheus.CounterOpts{
 	Help: "The total number of processed events",
 })
 
-var tokens_out = promauto.NewHistogram(prometheus.HistogramOpts{
+var notStreamed = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "llmproxymetrics_non_streamed_requests",
+	Help: "The total number of processed non-streamed events",
+})
+var streamed = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "llmproxymetrics_streamed_requests",
+	Help: "The total number of processed streamed events",
+})
+var tokens_out = promauto.NewCounter(prometheus.CounterOpts{
 	Name: "llmproxymetrics_tokens_out",
 	Help: "Tokens generated.",
-	Buckets: prometheus.LinearBuckets(0, 100, 25),
+})
+
+var eval_time = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "llmproxymetrics_eval_time",
+	Help:    "Tokens generated.",
+	Buckets: prometheus.LinearBuckets(0, 2.5, 20),
 })
 
 func main() {