Ade9/LLM-Proxy-Metrics
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixes

Browse Source
This commit is contained in:
Mikko Tikkanen
2025-04-25 09:06:47 +03:00
parent 4687c67685
commit bb1d8c6e7d
1 changed files with 24 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
llmproxymetrics.go
View File

@@ -2,6 +2,7 @@ package main
import ( import (
"bufio" "bufio"
"bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
@@ -39,16 +40,23 @@ func createProxy(target *url.URL) func(http.ResponseWriter, *http.Request) {
req.URL.Scheme = target.Scheme req.URL.Scheme = target.Scheme
req.URL.Host = target.Host req.URL.Host = target.Host
} }
var jsonData map[string]interface{} data, err := io.ReadAll(r.Body)
json.NewDecoder(r.Body).Decode(&jsonData) if err == nil {
r.Body = io.NopCloser(bytes.NewReader(data))
var jsonData map[string]interface{}
json.NewDecoder(r.Body).Decode(&jsonData)
if jsonData["stream"] != nil { if jsonData["stream"] != nil {
if !jsonData["stream"].(bool) { if !jsonData["stream"].(bool) {
notStreamed.Inc() notStreamed.Inc()
} else { } else {
streamed.Inc() streamed.Inc()
}
} }
} else {
fmt.Println("Error reading body:", err)
} }
r.Body = io.NopCloser(bytes.NewReader(data))
modifyResponse := func(response *http.Response) error { modifyResponse := func(response *http.Response) error {
pr, pw := io.Pipe() pr, pw := io.Pipe()
@@ -89,6 +97,9 @@ func createProxy(target *url.URL) func(http.ResponseWriter, *http.Request) {
} }
func handleJsonLine(line []byte) { func handleJsonLine(line []byte) {
if len(line) == 0 {
return
}
var jsonData map[string]interface{} var jsonData map[string]interface{}
err := json.Unmarshal([]byte(line), &jsonData) err := json.Unmarshal([]byte(line), &jsonData)
if err != nil { if err != nil {
@@ -100,7 +111,8 @@ func handleJsonLine(line []byte) {
duration := jsonData["eval_duration"].(float64) / 1000000000.0 duration := jsonData["eval_duration"].(float64) / 1000000000.0
fmt.Printf("Duration: %.2f seconds\n", duration) fmt.Printf("Duration: %.2f seconds\n", duration)
opsProcessed.Inc() opsProcessed.Inc()
tokens_out.Add(jsonData["prompt_eval_count"].(float64)) tokens_out.Add(jsonData["eval_count"].(float64))
tokens_in.Add(jsonData["prompt_eval_count"].(float64))
eval_time.Observe(duration) eval_time.Observe(duration)
} }
} }
@@ -122,7 +134,10 @@ var tokens_out = promauto.NewCounter(prometheus.CounterOpts{
Name: "llmproxymetrics_tokens_out", Name: "llmproxymetrics_tokens_out",
Help: "Tokens generated.", Help: "Tokens generated.",
}) })
var tokens_in = promauto.NewCounter(prometheus.CounterOpts{
Name: "llmproxymetrics_tokens_in",
Help: "Tokens input.",
})
var eval_time = promauto.NewHistogram(prometheus.HistogramOpts{ var eval_time = promauto.NewHistogram(prometheus.HistogramOpts{
Name: "llmproxymetrics_eval_time", Name: "llmproxymetrics_eval_time",
Help: "Tokens generated.", Help: "Tokens generated.",