tokens-reef/backend/internal/pkg/metrics/metrics.go

// Package metrics provides Prometheus metrics for Sub2API.
package metrics

import (
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

var (
	// HTTPRequestsTotal counts total HTTP requests
	HTTPRequestsTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sub2api_http_requests_total",
			Help: "Total HTTP requests",
		},
		[]string{"method", "path", "status"},
	)

	// HTTPRequestDuration tracks HTTP request duration
	HTTPRequestDuration = promauto.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "sub2api_http_request_duration_seconds",
			Help:    "HTTP request duration in seconds",
			Buckets: prometheus.DefBuckets,
		},
		[]string{"method", "path"},
	)

	// GatewayRequestsTotal counts gateway requests by platform and model
	GatewayRequestsTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sub2api_gateway_requests_total",
			Help: "Total gateway requests",
		},
		[]string{"platform", "model", "status"},
	)

	// GatewayLatency tracks gateway request latency
	GatewayLatency = promauto.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "sub2api_gateway_latency_seconds",
			Help:    "Gateway request latency",
			Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10},
		},
		[]string{"platform", "model"},
	)

	// TTFTSeconds tracks Time To First Token
	TTFTSeconds = promauto.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "sub2api_gateway_ttft_seconds",
			Help:    "Time to first token in seconds",
			Buckets: []float64{.1, .25, .5, 1, 2, 3, 5, 10},
		},
		[]string{"platform", "model"},
	)

	// TokenConsumedTotal tracks token consumption
	TokenConsumedTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sub2api_tokens_consumed_total",
			Help: "Total tokens consumed",
		},
		[]string{"platform", "model", "token_type"},
	)

	// UpstreamRequestsTotal counts upstream requests
	UpstreamRequestsTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sub2api_upstream_requests_total",
			Help: "Total upstream requests",
		},
		[]string{"platform", "account_id", "status"},
	)

	// UpstreamLatency tracks upstream request latency
	UpstreamLatency = promauto.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "sub2api_upstream_latency_seconds",
			Help:    "Upstream request latency",
			Buckets: []float64{.1, .25, .5, 1, 2, 3, 5, 10, 30},
		},
		[]string{"platform", "model"},
	)

	// UpstreamErrorsTotal counts upstream errors
	UpstreamErrorsTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sub2api_upstream_errors_total",
			Help: "Total upstream errors",
		},
		[]string{"platform", "error_type"},
	)

	// AccountSwitchesTotal counts account switches
	AccountSwitchesTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sub2api_account_switches_total",
			Help: "Total account switches",
		},
		[]string{"platform", "reason"},
	)

	// DatabaseConnections tracks database connection pool stats
	DatabaseConnections = promauto.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "sub2api_db_connections",
			Help: "Database connection pool stats",
		},
		[]string{"state"},
	)

	// RedisConnections tracks Redis connection pool stats
	RedisConnections = promauto.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "sub2api_redis_connections",
			Help: "Redis connection pool stats",
		},
		[]string{"state"},
	)

	// ConcurrencyQueueDepth tracks concurrency queue depth
	ConcurrencyQueueDepth = promauto.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "sub2api_concurrency_queue_depth",
			Help: "Concurrency queue depth by account",
		},
		[]string{"account_id"},
	)

	// ActiveRequests tracks active requests
	ActiveRequests = promauto.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "sub2api_active_requests",
			Help: "Number of active requests",
		},
		[]string{"platform"},
	)

	// JobHeartbeatLastSuccess tracks last successful job heartbeat
	JobHeartbeatLastSuccess = promauto.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "sub2api_job_heartbeat_last_success_timestamp",
			Help: "Last successful job heartbeat timestamp",
		},
		[]string{"job_name"},
	)

	// SLOAvailability tracks SLO availability ratio
	SLOAvailability = promauto.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "sub2api_slo_availability_ratio",
			Help: "Current SLO availability ratio",
		},
		[]string{"slo_name", "window"},
	)

	// ErrorBudgetBurnRate tracks error budget burn rate
	ErrorBudgetBurnRate = promauto.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "sub2api_error_budget_burn_rate",
			Help: "Error budget burn rate",
		},
		[]string{"slo_name"},
	)

	// ---- 速率限制指标 ----

	// RateLimitHitsTotal counts rate limit hits
	RateLimitHitsTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sub2api_rate_limit_hits_total",
			Help: "Total rate limit hits by type and key scope",
		},
		[]string{"limit_type", "scope"}, // limit_type: concurrent|rpm|tpm; scope: user|group|global
	)

	// RateLimitWaitDuration tracks time spent waiting for rate limit tokens
	RateLimitWaitDuration = promauto.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "sub2api_rate_limit_wait_duration_seconds",
			Help:    "Time spent waiting for rate limit token acquisition",
			Buckets: []float64{.001, .005, .01, .05, .1, .5, 1, 5},
		},
		[]string{"limit_type"},
	)

	// ---- 缓存指标 ----

	// CacheOperationsTotal counts cache operations (hit/miss/set/del)
	CacheOperationsTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sub2api_cache_operations_total",
			Help: "Total cache operations",
		},
		[]string{"cache_type", "operation", "result"}, // cache_type: redis|local; operation: get|set|del; result: hit|miss|ok|err
	)

	// CacheSize tracks approximate cache size in items
	CacheSize = promauto.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "sub2api_cache_size_items",
			Help: "Approximate number of items in cache",
		},
		[]string{"cache_type"},
	)

	// ---- 计费处理指标 ----

	// BillingRecordsTotal counts billing record processing results
	BillingRecordsTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sub2api_billing_records_total",
			Help: "Total billing records processed",
		},
		[]string{"status"}, // status: success|failed|skipped
	)

	// BillingAmountTotal tracks total billing amount (in micro-units to avoid float precision)
	BillingAmountTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sub2api_billing_amount_total",
			Help: "Total billed amount in micro USD (1e-6 USD per unit)",
		},
		[]string{"platform", "model"},
	)

	// BillingProcessingDuration tracks billing batch processing latency
	BillingProcessingDuration = promauto.NewHistogram(
		prometheus.HistogramOpts{
			Name:    "sub2api_billing_processing_duration_seconds",
			Help:    "Time spent processing a billing batch",
			Buckets: []float64{.01, .05, .1, .5, 1, 5, 10},
		},
	)

	// ---- 并发控制指标 ----

	// ConcurrencySlotWaits counts times a request had to wait for a concurrency slot
	ConcurrencySlotWaits = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "sub2api_concurrency_slot_waits_total",
			Help: "Total times requests waited for a concurrency slot",
		},
		[]string{"platform"},
	)

	// ConcurrencySlotWaitDuration tracks time spent waiting for a concurrency slot
	ConcurrencySlotWaitDuration = promauto.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "sub2api_concurrency_slot_wait_duration_seconds",
			Help:    "Time spent waiting for a concurrency slot",
			Buckets: []float64{.01, .05, .1, .5, 1, 2, 5, 10},
		},
		[]string{"platform"},
	)
)

// RecordHTTPRequest records an HTTP request metric
func RecordHTTPRequest(method, path, status string, duration float64) {
	HTTPRequestsTotal.WithLabelValues(method, path, status).Inc()
	HTTPRequestDuration.WithLabelValues(method, path).Observe(duration)
}

// RecordGatewayRequest records a gateway request metric
func RecordGatewayRequest(platform, model, status string, duration float64) {
	GatewayRequestsTotal.WithLabelValues(platform, model, status).Inc()
	GatewayLatency.WithLabelValues(platform, model).Observe(duration)
}

// RecordTTFT records Time To First Token
func RecordTTFT(platform, model string, seconds float64) {
	TTFTSeconds.WithLabelValues(platform, model).Observe(seconds)
}

// RecordTokens records token consumption
func RecordTokens(platform, model string, inputTokens, outputTokens int64) {
	TokenConsumedTotal.WithLabelValues(platform, model, "input").Add(float64(inputTokens))
	TokenConsumedTotal.WithLabelValues(platform, model, "output").Add(float64(outputTokens))
}

// RecordUpstreamRequest records an upstream request
func RecordUpstreamRequest(platform, accountID, status string, duration float64) {
	UpstreamRequestsTotal.WithLabelValues(platform, accountID, status).Inc()
	UpstreamLatency.WithLabelValues(platform, "").Observe(duration)
}

// RecordUpstreamError records an upstream error
func RecordUpstreamError(platform, errorType string) {
	UpstreamErrorsTotal.WithLabelValues(platform, errorType).Inc()
}

// RecordAccountSwitch records an account switch
func RecordAccountSwitch(platform, reason string) {
	AccountSwitchesTotal.WithLabelValues(platform, reason).Inc()
}

// SetDBConnections sets database connection metrics
func SetDBConnections(active, idle, max int) {
	DatabaseConnections.WithLabelValues("active").Set(float64(active))
	DatabaseConnections.WithLabelValues("idle").Set(float64(idle))
	DatabaseConnections.WithLabelValues("max").Set(float64(max))
}

// SetRedisConnections sets Redis connection metrics
func SetRedisConnections(total, idle int) {
	RedisConnections.WithLabelValues("total").Set(float64(total))
	RedisConnections.WithLabelValues("idle").Set(float64(idle))
}

// SetActiveRequests sets active request count
func SetActiveRequests(platform string, count int) {
	ActiveRequests.WithLabelValues(platform).Set(float64(count))
}

// SetJobHeartbeat sets job heartbeat timestamp
func SetJobHeartbeat(jobName string, timestamp float64) {
	JobHeartbeatLastSuccess.WithLabelValues(jobName).Set(timestamp)
}

// RecordRateLimitHit records a rate limit hit event
func RecordRateLimitHit(limitType, scope string) {
	RateLimitHitsTotal.WithLabelValues(limitType, scope).Inc()
}

// RecordRateLimitWait records time spent waiting for rate limit acquisition
func RecordRateLimitWait(limitType string, seconds float64) {
	RateLimitWaitDuration.WithLabelValues(limitType).Observe(seconds)
}

// RecordCacheHit records a cache hit
func RecordCacheHit(cacheType string) {
	CacheOperationsTotal.WithLabelValues(cacheType, "get", "hit").Inc()
}

// RecordCacheMiss records a cache miss
func RecordCacheMiss(cacheType string) {
	CacheOperationsTotal.WithLabelValues(cacheType, "get", "miss").Inc()
}

// RecordCacheSet records a cache set operation
func RecordCacheSet(cacheType string, err error) {
	result := "ok"
	if err != nil {
		result = "err"
	}
	CacheOperationsTotal.WithLabelValues(cacheType, "set", result).Inc()
}

// SetCacheSize sets the approximate item count in a cache
func SetCacheSize(cacheType string, count int) {
	CacheSize.WithLabelValues(cacheType).Set(float64(count))
}

// RecordBillingRecord records the result of a billing record processing
func RecordBillingRecord(status string) {
	BillingRecordsTotal.WithLabelValues(status).Inc()
}

// RecordBillingAmount records billed amount (amountMicroUSD = amount in 1e-6 USD)
func RecordBillingAmount(platform, model string, amountMicroUSD float64) {
	BillingAmountTotal.WithLabelValues(platform, model).Add(amountMicroUSD)
}

// ObserveBillingBatch records the duration of a billing batch processing
func ObserveBillingBatch(seconds float64) {
	BillingProcessingDuration.Observe(seconds)
}

// RecordConcurrencyWait records a concurrency slot wait event
func RecordConcurrencyWait(platform string, seconds float64) {
	ConcurrencySlotWaits.WithLabelValues(platform).Inc()
	ConcurrencySlotWaitDuration.WithLabelValues(platform).Observe(seconds)
}