feat: close v3 governance evidence and slo metrics wiring

internal/app/http_api.go

@@ -591,7 +591,7 @@ func NewAPIHandlerWithAuth(adminAuth AdminAuthConfig, actions ActionSet, dsn ...
 	mux.Handle("DELETE /api/hosts/{hostID}", requireAdminAccess(adminAuth, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		handleDeleteHost(w, r, actions.DeleteHost)
 	})))
-	return mux
+	return metrics.Middleware(mux)
 }
 
 func healthz(w http.ResponseWriter, _ *http.Request) {
@@ -2692,12 +2692,14 @@ func handlePublicV1ChatCompletions(w http.ResponseWriter, r *http.Request, dsn s
 	// 1. Extract bearer key
 	auth := strings.TrimSpace(r.Header.Get("Authorization"))
 	if !strings.HasPrefix(auth, "Bearer ") {
+		metrics.RecordUserKeyChatRequest("unauthorized")
 		writeHTTPError(w, &httpError{StatusCode: http.StatusUnauthorized, Code: "unauthorized", Message: "missing or invalid Authorization header"})
 		return
 	}
 	keyToken := strings.TrimPrefix(auth, "Bearer ")
 	keyToken = strings.TrimSpace(keyToken)
 	if keyToken == "" {
+		metrics.RecordUserKeyChatRequest("unauthorized")
 		writeHTTPError(w, &httpError{StatusCode: http.StatusUnauthorized, Code: "unauthorized", Message: "empty bearer token"})
 		return
 	}
@@ -2707,6 +2709,7 @@ func handlePublicV1ChatCompletions(w http.ResponseWriter, r *http.Request, dsn s
 
 	store, err := sqlite.Open(r.Context(), dsn)
 	if err != nil {
+		metrics.RecordUserKeyChatRequest("db_error")
 		writeHTTPError(w, &httpError{StatusCode: http.StatusInternalServerError, Code: "db_error", Message: "database error"})
 		return
 	}
@@ -2714,6 +2717,7 @@ func handlePublicV1ChatCompletions(w http.ResponseWriter, r *http.Request, dsn s
 
 	keys, err := store.UserKeys().ListByFingerprint(r.Context(), keyFingerprint)
 	if err != nil || len(keys) == 0 {
+		metrics.RecordUserKeyChatRequest("invalid_api_key")
 		writeHTTPError(w, &httpError{StatusCode: http.StatusUnauthorized, Code: "unauthorized", Message: "invalid API key"})
 		return
 	}
@@ -2723,17 +2727,25 @@ func handlePublicV1ChatCompletions(w http.ResponseWriter, r *http.Request, dsn s
 	// Governance check
 	log.Printf("gateway: key %s admin_status=%s paused=%v fingerprint=%s", key.KeyID, key.AdminStatus, key.AdminStatus == "paused", keyFingerprint)
 	if key.AdminStatus == "paused" {
+		metrics.RecordUserKeyChatRequest("key_paused")
 		writeHTTPError(w, &httpError{StatusCode: http.StatusForbidden, Code: "key_paused", Message: "API key is paused"})
 		return
 	}
 	if key.AdminStatus == "retired" || key.AdminStatus == "deleted" {
+		metrics.RecordUserKeyChatRequest("key_retired")
 		writeHTTPError(w, &httpError{StatusCode: http.StatusForbidden, Code: "key_retired", Message: "API key is no longer active"})
 		return
 	}
+	if key.QuotaStatus == "exhausted" {
+		metrics.RecordUserKeyChatRequest("quota_exhausted")
+		writeHTTPError(w, &httpError{StatusCode: http.StatusForbidden, Code: "quota_exhausted", Message: "API key quota exhausted"})
+		return
+	}
 
 	// 4. Parse request body (OpenAI-compatible)
 	body, err := io.ReadAll(io.LimitReader(r.Body, maxJSONBodyBytes))
 	if err != nil {
+		metrics.RecordUserKeyChatRequest("bad_request")
 		writeHTTPError(w, &httpError{StatusCode: http.StatusBadRequest, Code: "bad_request", Message: "cannot read request body"})
 		return
 	}
@@ -2745,12 +2757,14 @@ func handlePublicV1ChatCompletions(w http.ResponseWriter, r *http.Request, dsn s
 		Stream      bool                    `json:"stream,omitempty"`
 	}
 	if err := json.Unmarshal(body, &openAIReq); err != nil {
+		metrics.RecordUserKeyChatRequest("bad_request")
 		writeHTTPError(w, &httpError{StatusCode: http.StatusBadRequest, Code: "bad_request", Message: "invalid request body"})
 		return
 	}
 
 	model := strings.TrimSpace(openAIReq.Model)
 	if model == "" {
+		metrics.RecordUserKeyChatRequest("bad_request")
 		writeHTTPError(w, &httpError{StatusCode: http.StatusBadRequest, Code: "bad_request", Message: "model is required"})
 		return
 	}
@@ -2771,6 +2785,7 @@ func handlePublicV1ChatCompletions(w http.ResponseWriter, r *http.Request, dsn s
 
 	result, err := proxyChat(r.Context(), proxyReq)
 	if err != nil {
+		metrics.RecordUserKeyChatRequest("proxy_error")
 		writeHTTPError(w, classifyError(err))
 		return
 	}
@@ -2835,6 +2850,7 @@ func handlePublicV1ChatCompletions(w http.ResponseWriter, r *http.Request, dsn s
 		}
 	}
 
+	metrics.RecordUserKeyChatRequest("ok")
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(respPayload)

internal/app/key_self_service_svc.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"sub2api-cn-relay-manager/internal/host/sub2api"
+	"sub2api-cn-relay-manager/internal/metrics"
 	"sub2api-cn-relay-manager/internal/store/sqlite"
 )
 
@@ -107,9 +108,11 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 	return &UserKeyHandler{
 		createFn: func(ctx context.Context, req CreateUserKeyRequest) (CreateUserKeyResponse, error) {
 			if strings.TrimSpace(req.SubjectID) == "" {
+				metrics.RecordUserKeyOperation("create", "unauthorized")
 				return CreateUserKeyResponse{}, &httpError{StatusCode: 401, Code: "unauthorized", Message: "user credentials required"}
 			}
 			if strings.TrimSpace(req.LogicalGroupID) == "" {
+				metrics.RecordUserKeyOperation("create", "bad_request")
 				return CreateUserKeyResponse{}, &httpError{StatusCode: 400, Code: "bad_request", Message: "logical_group_id is required"}
 			}
 			store, err := sqlite.Open(ctx, sqliteDSN)
@@ -124,6 +127,7 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 				return CreateUserKeyResponse{}, fmt.Errorf("increment create rate limit: %w", err)
 			}
 			if count > defaultKeyRateLimitPerHour {
+				metrics.RecordUserKeyOperation("create", "rate_limited")
 				return CreateUserKeyResponse{}, &httpError{StatusCode: 429, Code: "rate_limited", Message: "create key rate limit exceeded"}
 			}
 
@@ -176,6 +180,7 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 				return CreateUserKeyResponse{}, err
 			}
 
+			metrics.RecordUserKeyOperation("create", "success")
 			return CreateUserKeyResponse{
 				Key: UserKeyMeta{
 					KeyID:          keyID,
@@ -265,6 +270,7 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 				return ResetUserKeyResponse{}, fmt.Errorf("increment reset rate limit: %w", err)
 			}
 			if count > defaultKeyResetPerDay {
+				metrics.RecordUserKeyOperation("reset", "rate_limited")
 				return ResetUserKeyResponse{}, &httpError{StatusCode: 429, Code: "rate_limited", Message: "reset key rate limit exceeded"}
 			}
 
@@ -305,6 +311,7 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 			if err != nil {
 				return ResetUserKeyResponse{}, err
 			}
+			metrics.RecordUserKeyOperation("reset", "success")
 			return ResetUserKeyResponse{PlaintextKey: newPlaintext, MaskedPreview: masked, AdminStatus: "active"}, nil
 		},
 		pauseFn: func(ctx context.Context, keyID, subjectID, reason string) (UserKeyMeta, error) {
@@ -347,6 +354,7 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 			if err != nil {
 				return UserKeyMeta{}, err
 			}
+			metrics.RecordUserKeyOperation("pause", "success")
 			return UserKeyMeta{KeyID: keyID, MaskedPreview: rec.MaskedPreview, AdminStatus: "paused"}, nil
 		},
 		resumeFn: func(ctx context.Context, keyID, subjectID string) (UserKeyMeta, error) {
@@ -389,6 +397,7 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 			if err != nil {
 				return UserKeyMeta{}, err
 			}
+			metrics.RecordUserKeyOperation("resume", "success")
 			return UserKeyMeta{KeyID: keyID, MaskedPreview: rec.MaskedPreview, AdminStatus: "active"}, nil
 		},
 		deleteFn: func(ctx context.Context, keyID, subjectID string) error {
@@ -420,6 +429,9 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 				}
 				return nil
 			})
+			if err == nil {
+				metrics.RecordUserKeyOperation("delete", "success")
+			}
 			return err
 		},
 	}

internal/app/key_self_service_test.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"testing"
 
+	"sub2api-cn-relay-manager/internal/metrics"
 	"sub2api-cn-relay-manager/internal/store/sqlite"
 )
 
@@ -300,3 +301,24 @@ func expectedManagedPrefix(value string) string {
 	}
 	return prefix
 }
+
+func TestUserKeyAPIMetricsMiddlewareAndCreateMetric(t *testing.T) {
+	t.Parallel()
+	handler := NewAPIHandler("t", ActionSet{
+		UserKeyHandler: buildUserKeyHandler(appTestDSN(t, openAppTestStore(t))),
+	})
+	req := makeCreateRequest(t, http.MethodPost, "/api/keys", makeCreateBody("", "portal key", nil))
+	req.Header.Set("X-Portal-Subject", "smoke-user")
+	_ = httptestRecorder(handler, req)
+
+	metricsReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+	metricsResp := httptest.NewRecorder()
+	metrics.Handler().ServeHTTP(metricsResp, metricsReq)
+	body := metricsResp.Body.String()
+	if !strings.Contains(body, "http_requests_total") {
+		t.Fatal("expected metrics endpoint to expose http_requests_total after middleware-wrapped request")
+	}
+	if !strings.Contains(body, "user_key_operations_total") {
+		t.Fatal("expected metrics endpoint to expose user_key_operations_total after create validation failure")
+	}
+}

internal/app/public_chat_metrics_test.go

@@ -0,0 +1,82 @@
+package app
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"sub2api-cn-relay-manager/internal/metrics"
+	"sub2api-cn-relay-manager/internal/store/sqlite"
+)
+
+func TestPublicV1ChatCompletionsQuotaExhaustedRecordsMetric(t *testing.T) {
+	t.Parallel()
+
+	store := openAppTestStore(t)
+	defer closeAppTestStore(t, store)
+
+	const plaintextKey = "sk-test-quota-exhausted"
+	if _, err := store.UserKeys().Create(context.Background(), sqlite.UserKeyRecord{
+		KeyID:          "key_quota_exhausted",
+		OwnerSubjectID: "portal-user",
+		KeyFingerprint: "sha256:" + sha256Hex(plaintextKey),
+		MaskedPreview:  "sk-****sted",
+		DisplayName:    "quota key",
+		LogicalGroupID: "gpt-shared",
+		AllowedModels:  []string{"gpt-5.4"},
+		AdminStatus:    "active",
+		QuotaStatus:    "exhausted",
+	}); err != nil {
+		t.Fatalf("UserKeys().Create() error = %v", err)
+	}
+
+	handler := NewAPIHandler("t", ActionSet{
+		UserKeyHandler: buildUserKeyHandler(appTestDSN(t, store)),
+		ProxyRouteChatCompletions: func(context.Context, ProxyRouteChatCompletionsRequest) (ProxyRouteChatCompletionsResult, error) {
+			t.Fatal("proxy should not be called when quota is exhausted")
+			return ProxyRouteChatCompletionsResult{}, nil
+		},
+	}, appTestDSN(t, store))
+
+	req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"gpt-5.4","messages":[{"role":"user","content":"ping"}]}`))
+	req.Header.Set("Authorization", "Bearer "+plaintextKey)
+	req.Header.Set("Content-Type", "application/json")
+	resp := httptestRecorder(handler, req)
+	if resp.code != http.StatusForbidden {
+		t.Fatalf("status code = %d, want 403 body=%s", resp.code, resp.Body().String())
+	}
+	assertJSONContains(t, resp.Body().Bytes(), "error.code", "quota_exhausted")
+
+	metricsReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+	metricsResp := httptest.NewRecorder()
+	metrics.Handler().ServeHTTP(metricsResp, metricsReq)
+	body := metricsResp.Body.String()
+	if !strings.Contains(body, "user_key_chat_requests_total") || !strings.Contains(body, "quota_exhausted") {
+		t.Fatalf("metrics body missing quota_exhausted chat metric: %s", body)
+	}
+}
+
+func TestMetricsMiddlewareUsesRoutePatternForKeyReset(t *testing.T) {
+	t.Parallel()
+
+	store := openAppTestStore(t)
+	defer closeAppTestStore(t, store)
+
+	handler := NewAPIHandler("t", ActionSet{
+		UserKeyHandler: buildUserKeyHandler(appTestDSN(t, store)),
+	})
+
+	req := httptest.NewRequest(http.MethodPost, "/api/keys/key_abc123/reset", nil)
+	req.Header.Set("Content-Type", "application/json")
+	_ = httptestRecorder(handler, req)
+
+	metricsReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+	metricsResp := httptest.NewRecorder()
+	metrics.Handler().ServeHTTP(metricsResp, metricsReq)
+	body := metricsResp.Body.String()
+	if !strings.Contains(body, "/api/keys/{key_id}/reset") {
+		t.Fatalf("expected normalized route pattern in metrics output, got: %s", body)
+	}
+}

internal/app/route_resolve_api.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"
 
+	"sub2api-cn-relay-manager/internal/metrics"
 	"sub2api-cn-relay-manager/internal/routing"
 	"sub2api-cn-relay-manager/internal/store/sqlite"
 )
@@ -148,6 +149,7 @@ func buildResolveRouteAction(sqliteDSN string, stickyRuntime stickyStoreRuntime,
 						return ResolveRouteInfo{}, err
 					}
 				}
+				metrics.RecordRouteDecision(req.LogicalGroupID, "sticky_hit")
 				return info, nil
 			}
 		}
@@ -217,12 +219,18 @@ func buildResolveRouteAction(sqliteDSN string, stickyRuntime stickyStoreRuntime,
 			}); err != nil {
 				return ResolveRouteInfo{}, err
 			}
+			metrics.RecordRouteFailover()
 		}
 		if req.Sync {
 			if err := writer.Flush(ctx); err != nil {
 				return ResolveRouteInfo{}, err
 			}
 		}
+		decisionStatus := "bind"
+		if selection.fallbackUsed {
+			decisionStatus = "fallback"
+		}
+		metrics.RecordRouteDecision(req.LogicalGroupID, decisionStatus)
 		return resolveRouteInfoFromBinding(stickyRuntime.backend, stickyKey, req.Scope, req.SubjectID, candidate, stored, requestID, false, "bind", selection.fallbackUsed), nil
 	}
 }

internal/app/route_resolve_api_test.go

@@ -3,11 +3,14 @@ package app
 import (
 	"context"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
+	"sub2api-cn-relay-manager/internal/metrics"
 	"sub2api-cn-relay-manager/internal/routing"
 )
 
@@ -205,12 +208,20 @@ func TestNewActionSetResolveRouteFlow(t *testing.T) {
 	if err != nil {
 		t.Fatalf("ListRouteFailoverEvents() error = %v", err)
 	}
-	if len(failovers) != 1 {
-		t.Fatalf("ListRouteFailoverEvents() len = %d, want 1", len(failovers))
-	}
 	if failovers[0].FromRouteID != "codex2api" || failovers[0].ToRouteID != "asxs" || failovers[0].FailureCount != 2 {
 		t.Fatalf("ListRouteFailoverEvents()[0] = %+v, want codex2api -> asxs failure_count 2", failovers[0])
 	}
+
+	req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+	rr := httptest.NewRecorder()
+	metrics.Handler().ServeHTTP(rr, req)
+	body := rr.Body.String()
+	if !strings.Contains(body, "route_decisions_total") {
+		t.Fatal("metrics missing route_decisions_total after resolve flow")
+	}
+	if !strings.Contains(body, "route_failovers_total") {
+		t.Fatal("metrics missing route_failovers_total after fallback flow")
+	}
 }
 
 func TestResolveRouteHelpers(t *testing.T) {

internal/metrics/metrics.go

@@ -58,6 +58,22 @@ var (
 		},
 	)
 
+	UserKeyOperationsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "user_key_operations_total",
+			Help: "Total number of user key self-service and governance operations",
+		},
+		[]string{"operation", "result"},
+	)
+
+	UserKeyChatRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "user_key_chat_requests_total",
+			Help: "Total number of public user-key chat completion requests",
+		},
+		[]string{"result"},
+	)
+
 	// 数据库指标
 	DBConnectionsActive = prometheus.NewGauge(
 		prometheus.GaugeOpts{
@@ -98,6 +114,8 @@ func init() {
 	prometheus.MustRegister(ActiveProviders)
 	prometheus.MustRegister(RouteDecisionsTotal)
 	prometheus.MustRegister(RouteFailoversTotal)
+	prometheus.MustRegister(UserKeyOperationsTotal)
+	prometheus.MustRegister(UserKeyChatRequestsTotal)
 	prometheus.MustRegister(DBConnectionsActive)
 	prometheus.MustRegister(DBOperationsTotal)
 	prometheus.MustRegister(LogFlushErrorsTotal)
@@ -111,6 +129,9 @@ func Handler() http.Handler {
 
 // RecordHTTPRequest records metrics for an HTTP request
 func RecordHTTPRequest(method, path string, status int, duration time.Duration) {
+	if path == "" {
+		path = "unknown"
+	}
 	HTTPRequestsTotal.WithLabelValues(method, path, http.StatusText(status)).Inc()
 	HTTPRequestDuration.WithLabelValues(method, path).Observe(duration.Seconds())
 }
@@ -125,6 +146,16 @@ func RecordRouteFailover() {
 	RouteFailoversTotal.Inc()
 }
 
+// RecordUserKeyOperation records a user key lifecycle/governance operation.
+func RecordUserKeyOperation(operation, result string) {
+	UserKeyOperationsTotal.WithLabelValues(operation, result).Inc()
+}
+
+// RecordUserKeyChatRequest records a public user-key chat completion request outcome.
+func RecordUserKeyChatRequest(result string) {
+	UserKeyChatRequestsTotal.WithLabelValues(result).Inc()
+}
+
 // SetActiveHosts sets the active hosts gauge
 func SetActiveHosts(count float64) {
 	ActiveHosts.Set(count)
@@ -166,7 +197,11 @@ func Middleware(next http.Handler) http.Handler {
 		next.ServeHTTP(wrapped, r)
 
 		duration := time.Since(start)
-		RecordHTTPRequest(r.Method, r.URL.Path, wrapped.statusCode, duration)
+		path := r.Pattern
+		if path == "" {
+			path = r.URL.Path
+		}
+		RecordHTTPRequest(r.Method, path, wrapped.statusCode, duration)
 	})
 }
 

internal/metrics/metrics_test.go

@@ -64,6 +64,36 @@ func TestRecordRouteFailover(t *testing.T) {
 	}
 }
 
+func TestRecordUserKeyOperation(t *testing.T) {
+	RecordUserKeyOperation("create", "success")
+	RecordUserKeyOperation("pause", "success")
+
+	req := httptest.NewRequest("GET", "/metrics", nil)
+	rr := httptest.NewRecorder()
+
+	Handler().ServeHTTP(rr, req)
+
+	body := rr.Body.String()
+	if !strings.Contains(body, "user_key_operations_total") {
+		t.Error("Expected metrics endpoint to contain user_key_operations_total")
+	}
+}
+
+func TestRecordUserKeyChatRequest(t *testing.T) {
+	RecordUserKeyChatRequest("ok")
+	RecordUserKeyChatRequest("key_paused")
+
+	req := httptest.NewRequest("GET", "/metrics", nil)
+	rr := httptest.NewRecorder()
+
+	Handler().ServeHTTP(rr, req)
+
+	body := rr.Body.String()
+	if !strings.Contains(body, "user_key_chat_requests_total") {
+		t.Error("Expected metrics endpoint to contain user_key_chat_requests_total")
+	}
+}
+
 func TestSetActiveHosts(t *testing.T) {
 	SetActiveHosts(10)