201 lines
7.3 KiB
Go
201 lines
7.3 KiB
Go
package metrics
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/longpeng/configcenter/internal/store"
|
|
)
|
|
|
|
type Collector struct {
|
|
mu sync.Mutex
|
|
http map[httpKey]httpValue
|
|
grpc map[grpcKey]httpValue
|
|
|
|
publishAccepted atomic.Uint64
|
|
outboxApplied atomic.Uint64
|
|
outboxFailed atomic.Uint64
|
|
runtimeReads atomic.Uint64
|
|
grayMatches atomic.Uint64
|
|
watchers atomic.Int64
|
|
}
|
|
|
|
type httpKey struct {
|
|
method string
|
|
route string
|
|
status int
|
|
}
|
|
|
|
type grpcKey struct {
|
|
method string
|
|
code string
|
|
}
|
|
|
|
type httpValue struct {
|
|
count uint64
|
|
durationSum float64
|
|
buckets []uint64
|
|
}
|
|
|
|
var httpDurationBuckets = []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5}
|
|
|
|
func New() *Collector {
|
|
return &Collector{http: make(map[httpKey]httpValue), grpc: make(map[grpcKey]httpValue)}
|
|
}
|
|
|
|
func (c *Collector) ObserveHTTP(method, route string, status int, duration time.Duration) {
|
|
if route == "" {
|
|
route = "unmatched"
|
|
}
|
|
key := httpKey{method: method, route: route, status: status}
|
|
c.mu.Lock()
|
|
value := c.http[key]
|
|
if value.buckets == nil {
|
|
value.buckets = make([]uint64, len(httpDurationBuckets))
|
|
}
|
|
value.count++
|
|
seconds := duration.Seconds()
|
|
value.durationSum += seconds
|
|
for index, upperBound := range httpDurationBuckets {
|
|
if seconds <= upperBound {
|
|
value.buckets[index]++
|
|
}
|
|
}
|
|
c.http[key] = value
|
|
c.mu.Unlock()
|
|
}
|
|
|
|
func (c *Collector) ObserveGRPC(method, code string, duration time.Duration) {
|
|
key := grpcKey{method: method, code: code}
|
|
c.mu.Lock()
|
|
value := c.grpc[key]
|
|
if value.buckets == nil {
|
|
value.buckets = make([]uint64, len(httpDurationBuckets))
|
|
}
|
|
value.count++
|
|
seconds := duration.Seconds()
|
|
value.durationSum += seconds
|
|
for index, upperBound := range httpDurationBuckets {
|
|
if seconds <= upperBound {
|
|
value.buckets[index]++
|
|
}
|
|
}
|
|
c.grpc[key] = value
|
|
c.mu.Unlock()
|
|
}
|
|
|
|
func (c *Collector) PublishAccepted() { c.publishAccepted.Add(1) }
|
|
func (c *Collector) OutboxApplied() { c.outboxApplied.Add(1) }
|
|
func (c *Collector) OutboxFailed() { c.outboxFailed.Add(1) }
|
|
func (c *Collector) RuntimeRead() { c.runtimeReads.Add(1) }
|
|
func (c *Collector) GrayMatched(count int) {
|
|
if count > 0 {
|
|
c.grayMatches.Add(uint64(count))
|
|
}
|
|
}
|
|
func (c *Collector) WatcherAdded() { c.watchers.Add(1) }
|
|
func (c *Collector) WatcherRemoved() { c.watchers.Add(-1) }
|
|
|
|
func (c *Collector) Handler(repository store.Store) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
|
stats, err := repository.OutboxStats(r.Context())
|
|
if err != nil {
|
|
http.Error(w, "collect outbox metrics", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
var output strings.Builder
|
|
writeHelp(&output, "configcenter_http_requests_total", "HTTP requests by method, route and status", "counter")
|
|
writeHelp(&output, "configcenter_http_request_duration_seconds", "HTTP request duration by method, route and status", "histogram")
|
|
c.mu.Lock()
|
|
keys := make([]httpKey, 0, len(c.http))
|
|
for key := range c.http {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Slice(keys, func(i, j int) bool {
|
|
if keys[i].route != keys[j].route {
|
|
return keys[i].route < keys[j].route
|
|
}
|
|
if keys[i].method != keys[j].method {
|
|
return keys[i].method < keys[j].method
|
|
}
|
|
return keys[i].status < keys[j].status
|
|
})
|
|
for _, key := range keys {
|
|
value := c.http[key]
|
|
labels := fmt.Sprintf(`method="%s",route="%s",status="%d"`, escape(key.method), escape(key.route), key.status)
|
|
fmt.Fprintf(&output, "configcenter_http_requests_total{%s} %d\n", labels, value.count)
|
|
for index, upperBound := range httpDurationBuckets {
|
|
fmt.Fprintf(&output, "configcenter_http_request_duration_seconds_bucket{%s,le=\"%s\"} %d\n", labels, strconv.FormatFloat(upperBound, 'g', -1, 64), value.buckets[index])
|
|
}
|
|
fmt.Fprintf(&output, "configcenter_http_request_duration_seconds_bucket{%s,le=\"+Inf\"} %d\n", labels, value.count)
|
|
fmt.Fprintf(&output, "configcenter_http_request_duration_seconds_sum{%s} %s\n", labels, strconv.FormatFloat(value.durationSum, 'f', 6, 64))
|
|
fmt.Fprintf(&output, "configcenter_http_request_duration_seconds_count{%s} %d\n", labels, value.count)
|
|
}
|
|
c.mu.Unlock()
|
|
|
|
writeHelp(&output, "configcenter_grpc_requests_total", "gRPC requests by method and status code", "counter")
|
|
writeHelp(&output, "configcenter_grpc_request_duration_seconds", "gRPC request duration by method and status code", "histogram")
|
|
c.mu.Lock()
|
|
grpcKeys := make([]grpcKey, 0, len(c.grpc))
|
|
for key := range c.grpc {
|
|
grpcKeys = append(grpcKeys, key)
|
|
}
|
|
sort.Slice(grpcKeys, func(i, j int) bool {
|
|
if grpcKeys[i].method != grpcKeys[j].method {
|
|
return grpcKeys[i].method < grpcKeys[j].method
|
|
}
|
|
return grpcKeys[i].code < grpcKeys[j].code
|
|
})
|
|
for _, key := range grpcKeys {
|
|
value := c.grpc[key]
|
|
labels := fmt.Sprintf(`method="%s",code="%s"`, escape(key.method), escape(key.code))
|
|
fmt.Fprintf(&output, "configcenter_grpc_requests_total{%s} %d\n", labels, value.count)
|
|
for index, upperBound := range httpDurationBuckets {
|
|
fmt.Fprintf(&output, "configcenter_grpc_request_duration_seconds_bucket{%s,le=\"%s\"} %d\n", labels, strconv.FormatFloat(upperBound, 'g', -1, 64), value.buckets[index])
|
|
}
|
|
fmt.Fprintf(&output, "configcenter_grpc_request_duration_seconds_bucket{%s,le=\"+Inf\"} %d\n", labels, value.count)
|
|
fmt.Fprintf(&output, "configcenter_grpc_request_duration_seconds_sum{%s} %s\n", labels, strconv.FormatFloat(value.durationSum, 'f', 6, 64))
|
|
fmt.Fprintf(&output, "configcenter_grpc_request_duration_seconds_count{%s} %d\n", labels, value.count)
|
|
}
|
|
c.mu.Unlock()
|
|
|
|
writeCounter(&output, "configcenter_publish_accepted_total", "Accepted configuration releases", c.publishAccepted.Load())
|
|
writeCounter(&output, "configcenter_outbox_applied_total", "Successfully applied outbox entries", c.outboxApplied.Load())
|
|
writeCounter(&output, "configcenter_outbox_failed_attempts_total", "Failed outbox apply attempts", c.outboxFailed.Load())
|
|
writeCounter(&output, "configcenter_runtime_reads_total", "Runtime configuration reads", c.runtimeReads.Load())
|
|
writeCounter(&output, "configcenter_gray_rule_matches_total", "Gray rules matched during runtime reads and watches", c.grayMatches.Load())
|
|
writeGauge(&output, "configcenter_watch_subscribers", "Current SSE and gRPC watch subscribers", c.watchers.Load())
|
|
writeGauge(&output, "configcenter_outbox_pending", "Pending outbox entries", stats.Pending)
|
|
writeGauge(&output, "configcenter_outbox_processing", "Processing outbox entries", stats.Processing)
|
|
writeGauge(&output, "configcenter_outbox_failed", "Terminally failed outbox entries", stats.Failed)
|
|
_, _ = w.Write([]byte(output.String()))
|
|
}
|
|
}
|
|
|
|
func writeHelp(output *strings.Builder, name, help, metricType string) {
|
|
fmt.Fprintf(output, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, metricType)
|
|
}
|
|
|
|
func writeCounter(output *strings.Builder, name, help string, value uint64) {
|
|
writeHelp(output, name, help, "counter")
|
|
fmt.Fprintf(output, "%s %d\n", name, value)
|
|
}
|
|
|
|
func writeGauge(output *strings.Builder, name, help string, value int64) {
|
|
writeHelp(output, name, help, "gauge")
|
|
fmt.Fprintf(output, "%s %d\n", name, value)
|
|
}
|
|
|
|
func escape(value string) string {
|
|
value = strings.ReplaceAll(value, `\`, `\\`)
|
|
value = strings.ReplaceAll(value, `"`, `\"`)
|
|
return strings.ReplaceAll(value, "\n", `\n`)
|
|
}
|