Ade9/LLM-Proxy-Metrics
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix stream metrics timing

Browse Source
This commit is contained in:
Mikko Tikkanen
2025-04-25 09:30:34 +03:00
parent bb1d8c6e7d
commit f5df0b5d54
1 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
llmproxymetrics.go
View File

@@ -40,6 +40,7 @@ func createProxy(target *url.URL) func(http.ResponseWriter, *http.Request) {
req.URL.Scheme = target.Scheme req.URL.Scheme = target.Scheme
req.URL.Host = target.Host req.URL.Host = target.Host
} }
wasstreamed := -1
data, err := io.ReadAll(r.Body) data, err := io.ReadAll(r.Body)
if err == nil { if err == nil {
r.Body = io.NopCloser(bytes.NewReader(data)) r.Body = io.NopCloser(bytes.NewReader(data))
@@ -48,9 +49,9 @@ func createProxy(target *url.URL) func(http.ResponseWriter, *http.Request) {
if jsonData["stream"] != nil { if jsonData["stream"] != nil {
if !jsonData["stream"].(bool) { if !jsonData["stream"].(bool) {
notStreamed.Inc() wasstreamed = 0
} else { } else {
streamed.Inc() wasstreamed = 1
} }
} }
} else { } else {
@@ -71,13 +72,13 @@ func createProxy(target *url.URL) func(http.ResponseWriter, *http.Request) {
line, err := reader.ReadBytes('\n') line, err := reader.ReadBytes('\n')
if err != nil { if err != nil {
if err == io.EOF { if err == io.EOF {
handleJsonLine([]byte(string(line))) handleJsonLine([]byte(string(line)), wasstreamed)
pw.Write(line) pw.Write(line)
break break
} }
return return
} }
handleJsonLine(line) handleJsonLine(line, wasstreamed)
pw.Write(line) pw.Write(line)
} }
@@ -96,7 +97,7 @@ func createProxy(target *url.URL) func(http.ResponseWriter, *http.Request) {
} }
} }
func handleJsonLine(line []byte) { func handleJsonLine(line []byte, wasstreamed int) {
if len(line) == 0 { if len(line) == 0 {
return return
} }
@@ -115,6 +116,11 @@ func handleJsonLine(line []byte) {
tokens_in.Add(jsonData["prompt_eval_count"].(float64)) tokens_in.Add(jsonData["prompt_eval_count"].(float64))
eval_time.Observe(duration) eval_time.Observe(duration)
} }
if wasstreamed == 1 {
streamed.Inc()
} else if wasstreamed == 0 {
notStreamed.Inc()
}
} }
var opsProcessed = promauto.NewCounter(prometheus.CounterOpts{ var opsProcessed = promauto.NewCounter(prometheus.CounterOpts{