diff --git a/llmproxymetrics.go b/llmproxymetrics.go index fc11625..0cf0366 100644 --- a/llmproxymetrics.go +++ b/llmproxymetrics.go @@ -40,6 +40,7 @@ func createProxy(target *url.URL) func(http.ResponseWriter, *http.Request) { req.URL.Scheme = target.Scheme req.URL.Host = target.Host } + wasstreamed := -1 data, err := io.ReadAll(r.Body) if err == nil { r.Body = io.NopCloser(bytes.NewReader(data)) @@ -48,9 +49,9 @@ func createProxy(target *url.URL) func(http.ResponseWriter, *http.Request) { if jsonData["stream"] != nil { if !jsonData["stream"].(bool) { - notStreamed.Inc() + wasstreamed = 0 } else { - streamed.Inc() + wasstreamed = 1 } } } else { @@ -71,13 +72,13 @@ func createProxy(target *url.URL) func(http.ResponseWriter, *http.Request) { line, err := reader.ReadBytes('\n') if err != nil { if err == io.EOF { - handleJsonLine([]byte(string(line))) + handleJsonLine([]byte(string(line)), wasstreamed) pw.Write(line) break } return } - handleJsonLine(line) + handleJsonLine(line, wasstreamed) pw.Write(line) } @@ -96,7 +97,7 @@ func createProxy(target *url.URL) func(http.ResponseWriter, *http.Request) { } } -func handleJsonLine(line []byte) { +func handleJsonLine(line []byte, wasstreamed int) { if len(line) == 0 { return } @@ -115,6 +116,11 @@ func handleJsonLine(line []byte) { tokens_in.Add(jsonData["prompt_eval_count"].(float64)) eval_time.Observe(duration) } + if wasstreamed == 1 { + streamed.Inc() + } else if wasstreamed == 0 { + notStreamed.Inc() + } } var opsProcessed = promauto.NewCounter(prometheus.CounterOpts{