feat: close v3 slo gates and lifecycle metrics

2026-06-08 14:49:06 +08:00
parent dbbb313a36
commit dd6f332b53
14 changed files with 775 additions and 156 deletions
--- a/internal/app/key_self_service_handlers_test.go
+++ b/internal/app/key_self_service_handlers_test.go
@@ -0,0 +1,275 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestGeneratePlaintextKeyAndExtractSubjectID(t *testing.T) {
+	t.Parallel()
+	plaintext, fingerprint := generatePlaintextKey()
+	if !strings.HasPrefix(plaintext, "sk-") {
+		t.Fatalf("plaintext = %q, want sk- prefix", plaintext)
+	}
+	if !strings.HasPrefix(fingerprint, "sha256:") {
+		t.Fatalf("fingerprint = %q, want sha256 prefix", fingerprint)
+	}
+
+	h := &UserKeyHandler{}
+	req := httptest.NewRequest(http.MethodGet, "/api/keys", nil)
+	req.Header.Set("Authorization", "Bearer abcdefgh12345678")
+	subjectID, httpErr := h.extractSubjectID(req)
+	if httpErr != nil {
+		t.Fatalf("extractSubjectID() unexpected error: %+v", httpErr)
+	}
+	if subjectID != "skeleton_user_abcdefgh" {
+		t.Fatalf("subjectID = %q, want skeleton_user_abcdefgh", subjectID)
+	}
+}
+
+func TestHandleUserKeyListNotImplemented(t *testing.T) {
+	t.Parallel()
+	req := httptest.NewRequest(http.MethodGet, "/api/keys", nil)
+	rr := httptest.NewRecorder()
+	serveWithMetrics(t, req, rr, func(w http.ResponseWriter, r *http.Request) {
+		handleListUserKeys(w, r, nil)
+	})
+	if rr.Code != http.StatusNotImplemented {
+		t.Fatalf("status = %d, want 501 body=%s", rr.Code, rr.Body.String())
+	}
+}
+
+func TestHandleUserKeyListSuccess(t *testing.T) {
+	t.Parallel()
+	h := &UserKeyHandler{
+		listFn: func(ctx context.Context, subjectID string) ([]UserKeyMeta, error) {
+			if subjectID != "portal-user:1" {
+				t.Fatalf("subjectID = %q, want portal-user:1", subjectID)
+			}
+			return []UserKeyMeta{{KeyID: "key_1", AdminStatus: "active"}}, nil
+		},
+	}
+	req := httptest.NewRequest(http.MethodGet, "/api/keys", nil)
+	req.Header.Set("X-Portal-Subject", "portal-user:1")
+	rr := httptest.NewRecorder()
+	serveWithMetrics(t, req, rr, func(w http.ResponseWriter, r *http.Request) {
+		handleListUserKeys(w, r, h)
+	})
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200 body=%s", rr.Code, rr.Body.String())
+	}
+	if !strings.Contains(rr.Body.String(), "key_1") {
+		t.Fatalf("body missing key_1: %s", rr.Body.String())
+	}
+}
+
+func TestHandleGetUserKeyMissingKeyID(t *testing.T) {
+	t.Parallel()
+	h := &UserKeyHandler{getFn: func(context.Context, string, string) (UserKeyMeta, error) {
+		t.Fatal("getFn should not be called when key_id is missing")
+		return UserKeyMeta{}, nil
+	}}
+	req := httptest.NewRequest(http.MethodGet, "/api/keys/", nil)
+	req.Header.Set("X-Portal-Subject", "portal-user:1")
+	rr := httptest.NewRecorder()
+	serveWithMetrics(t, req, rr, func(w http.ResponseWriter, r *http.Request) {
+		handleGetUserKey(w, r, h)
+	})
+	if rr.Code != http.StatusBadRequest {
+		t.Fatalf("status = %d, want 400 body=%s", rr.Code, rr.Body.String())
+	}
+}
+
+func TestHandleUserKeyMutationHandlers(t *testing.T) {
+	t.Parallel()
+	meta := UserKeyMeta{KeyID: "key_1", MaskedPreview: "sk-****1234", AdminStatus: "active"}
+	cases := []struct {
+		name        string
+		method      string
+		path        string
+		handlerFn   func(http.ResponseWriter, *http.Request, *UserKeyHandler)
+		userHandler *UserKeyHandler
+		wantStatus  int
+		wantBody    string
+	}{
+		{
+			name:      "get-success",
+			method:    http.MethodGet,
+			path:      "/api/keys/key_1",
+			handlerFn: handleGetUserKey,
+			userHandler: &UserKeyHandler{getFn: func(ctx context.Context, keyID, subjectID string) (UserKeyMeta, error) {
+				if keyID != "key_1" || subjectID != "portal-user:1" {
+					t.Fatalf("getFn args = (%q,%q)", keyID, subjectID)
+				}
+				return meta, nil
+			}},
+			wantStatus: http.StatusOK,
+			wantBody:   "key_1",
+		},
+		{
+			name:      "reset-success",
+			method:    http.MethodPost,
+			path:      "/api/keys/key_1/reset",
+			handlerFn: handleResetUserKey,
+			userHandler: &UserKeyHandler{resetFn: func(ctx context.Context, keyID, subjectID string) (ResetUserKeyResponse, error) {
+				if keyID != "key_1" || subjectID != "portal-user:1" {
+					t.Fatalf("resetFn args = (%q,%q)", keyID, subjectID)
+				}
+				return ResetUserKeyResponse{PlaintextKey: "sk-new", MaskedPreview: "sk-****new", AdminStatus: "active"}, nil
+			}},
+			wantStatus: http.StatusOK,
+			wantBody:   "sk-new",
+		},
+		{
+			name:      "pause-success",
+			method:    http.MethodPost,
+			path:      "/api/keys/key_1/pause",
+			handlerFn: handlePauseUserKey,
+			userHandler: &UserKeyHandler{pauseFn: func(ctx context.Context, keyID, subjectID, reason string) (UserKeyMeta, error) {
+				if keyID != "key_1" || subjectID != "portal-user:1" || reason != "" {
+					t.Fatalf("pauseFn args = (%q,%q,%q)", keyID, subjectID, reason)
+				}
+				paused := meta
+				paused.AdminStatus = "paused"
+				return paused, nil
+			}},
+			wantStatus: http.StatusOK,
+			wantBody:   "paused",
+		},
+		{
+			name:      "resume-success",
+			method:    http.MethodPost,
+			path:      "/api/keys/key_1/resume",
+			handlerFn: handleResumeUserKey,
+			userHandler: &UserKeyHandler{resumeFn: func(ctx context.Context, keyID, subjectID string) (UserKeyMeta, error) {
+				if keyID != "key_1" || subjectID != "portal-user:1" {
+					t.Fatalf("resumeFn args = (%q,%q)", keyID, subjectID)
+				}
+				return meta, nil
+			}},
+			wantStatus: http.StatusOK,
+			wantBody:   "active",
+		},
+		{
+			name:      "delete-success",
+			method:    http.MethodDelete,
+			path:      "/api/keys/key_1",
+			handlerFn: handleDeleteUserKey,
+			userHandler: &UserKeyHandler{deleteFn: func(ctx context.Context, keyID, subjectID string) error {
+				if keyID != "key_1" || subjectID != "portal-user:1" {
+					t.Fatalf("deleteFn args = (%q,%q)", keyID, subjectID)
+				}
+				return nil
+			}},
+			wantStatus: http.StatusOK,
+			wantBody:   "deleted",
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			req := httptest.NewRequest(tc.method, tc.path, nil)
+			req.Header.Set("X-Portal-Subject", "portal-user:1")
+			req.SetPathValue("key_id", "key_1")
+			rr := httptest.NewRecorder()
+			serveWithMetrics(t, req, rr, func(w http.ResponseWriter, r *http.Request) {
+				tc.handlerFn(w, r, tc.userHandler)
+			})
+			if rr.Code != tc.wantStatus {
+				t.Fatalf("status = %d, want %d body=%s", rr.Code, tc.wantStatus, rr.Body.String())
+			}
+			if !strings.Contains(rr.Body.String(), tc.wantBody) {
+				t.Fatalf("body missing %q: %s", tc.wantBody, rr.Body.String())
+			}
+		})
+	}
+}
+
+func serveWithMetrics(t *testing.T, req *http.Request, rr *httptest.ResponseRecorder, fn func(http.ResponseWriter, *http.Request)) {
+	t.Helper()
+	http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fn(w, r)
+	}).ServeHTTP(rr, req)
+}
+
+func TestHandleListUserKeysResponseShape(t *testing.T) {
+	t.Parallel()
+	h := &UserKeyHandler{listFn: func(context.Context, string) ([]UserKeyMeta, error) {
+		return []UserKeyMeta{{KeyID: "key_json", AdminStatus: "active"}}, nil
+	}}
+	req := httptest.NewRequest(http.MethodGet, "/api/keys", nil)
+	req.Header.Set("X-Portal-Subject", "portal-user:json")
+	rr := httptest.NewRecorder()
+	handleListUserKeys(rr, req, h)
+	var payload struct {
+		Keys []UserKeyMeta `json:"keys"`
+	}
+	if err := json.Unmarshal(rr.Body.Bytes(), &payload); err != nil {
+		t.Fatalf("decode response: %v", err)
+	}
+	if len(payload.Keys) != 1 || payload.Keys[0].KeyID != "key_json" {
+		t.Fatalf("payload = %+v, want one key_json entry", payload)
+	}
+}
+
+func TestHandleUserKeyMutationHandlersErrorPaths(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name        string
+		handlerFn   func(http.ResponseWriter, *http.Request, *UserKeyHandler)
+		userHandler *UserKeyHandler
+		wantStatus  int
+	}{
+		{
+			name:      "reset-not-found",
+			handlerFn: handleResetUserKey,
+			userHandler: &UserKeyHandler{resetFn: func(context.Context, string, string) (ResetUserKeyResponse, error) {
+				return ResetUserKeyResponse{}, fmt.Errorf("key %q not found", "key_1")
+			}},
+			wantStatus: http.StatusNotFound,
+		},
+		{
+			name:      "pause-not-found",
+			handlerFn: handlePauseUserKey,
+			userHandler: &UserKeyHandler{pauseFn: func(context.Context, string, string, string) (UserKeyMeta, error) {
+				return UserKeyMeta{}, fmt.Errorf("key %q not found", "key_1")
+			}},
+			wantStatus: http.StatusNotFound,
+		},
+		{
+			name:      "resume-not-found",
+			handlerFn: handleResumeUserKey,
+			userHandler: &UserKeyHandler{resumeFn: func(context.Context, string, string) (UserKeyMeta, error) {
+				return UserKeyMeta{}, fmt.Errorf("key %q not found", "key_1")
+			}},
+			wantStatus: http.StatusNotFound,
+		},
+		{
+			name:      "delete-not-found",
+			handlerFn: handleDeleteUserKey,
+			userHandler: &UserKeyHandler{deleteFn: func(context.Context, string, string) error {
+				return fmt.Errorf("key %q not found", "key_1")
+			}},
+			wantStatus: http.StatusNotFound,
+		},
+	}
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodPost, "/api/keys/key_1", nil)
+			req.Header.Set("X-Portal-Subject", "portal-user:1")
+			req.SetPathValue("key_id", "key_1")
+			rr := httptest.NewRecorder()
+			tc.handlerFn(rr, req, tc.userHandler)
+			if rr.Code != tc.wantStatus {
+				t.Fatalf("status = %d, want %d body=%s", rr.Code, tc.wantStatus, rr.Body.String())
+			}
+		})
+	}
+}
--- a/internal/app/key_self_service_svc.go
+++ b/internal/app/key_self_service_svc.go
@@ -104,6 +104,11 @@ func ensureSubjectHasAccess(ctx context.Context, client *sub2api.Client, subject
 	return apiKey, nil
 }

+func recordUserKeyFailure(operation, result string, err error) error {
+	metrics.RecordUserKeyOperation(operation, result)
+	return err
+}
+
 func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 	return &UserKeyHandler{
 		createFn: func(ctx context.Context, req CreateUserKeyRequest) (CreateUserKeyResponse, error) {
@@ -117,14 +122,14 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 			}
 			store, err := sqlite.Open(ctx, sqliteDSN)
 			if err != nil {
-				return CreateUserKeyResponse{}, fmt.Errorf("open store: %w", err)
+				return CreateUserKeyResponse{}, recordUserKeyFailure("create", "open_store_error", fmt.Errorf("open store: %w", err))
 			}
 			defer store.Close()

 			windowStart := time.Now().UTC().Format("2006-01-02T15:00:00Z")
 			count, err := store.SubjectRateLimits().IncrementWindow(ctx, req.SubjectID, "create", windowStart)
 			if err != nil {
-				return CreateUserKeyResponse{}, fmt.Errorf("increment create rate limit: %w", err)
+				return CreateUserKeyResponse{}, recordUserKeyFailure("create", "rate_limit_store_error", fmt.Errorf("increment create rate limit: %w", err))
 			}
 			if count > defaultKeyRateLimitPerHour {
 				metrics.RecordUserKeyOperation("create", "rate_limited")
@@ -134,15 +139,15 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 			// Resolve logical group → host → group ID → ensure subscription access
 			_, route, hostRow, client, err := resolveLogicalGroupHost(ctx, store, req.LogicalGroupID)
 			if err != nil {
-				return CreateUserKeyResponse{}, fmt.Errorf("resolve host for %q: %w", req.LogicalGroupID, err)
+				return CreateUserKeyResponse{}, recordUserKeyFailure("create", "resolve_host_error", fmt.Errorf("resolve host for %q: %w", req.LogicalGroupID, err))
 			}
 			hostGroupID, err := resolveShadowHostGroupID(ctx, client, route)
 			if err != nil {
-				return CreateUserKeyResponse{}, fmt.Errorf("resolve shadow group id for %q: %w", route.ShadowGroupID, err)
+				return CreateUserKeyResponse{}, recordUserKeyFailure("create", "resolve_shadow_group_error", fmt.Errorf("resolve shadow group id for %q: %w", route.ShadowGroupID, err))
 			}
 			apiKey, err := ensureSubjectHasAccess(ctx, client, req.SubjectID, hostGroupID)
 			if err != nil {
-				return CreateUserKeyResponse{}, fmt.Errorf("ensure access for %q: %w", req.LogicalGroupID, err)
+				return CreateUserKeyResponse{}, recordUserKeyFailure("create", "ensure_access_error", fmt.Errorf("ensure access for %q: %w", req.LogicalGroupID, err))
 			}

 			fingerprint := "sha256:" + sha256Hex(apiKey)
@@ -177,7 +182,7 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 				return nil
 			})
 			if err != nil {
-				return CreateUserKeyResponse{}, err
+				return CreateUserKeyResponse{}, recordUserKeyFailure("create", "db_tx_error", err)
 			}

 			metrics.RecordUserKeyOperation("create", "success")
@@ -253,21 +258,21 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 		resetFn: func(ctx context.Context, keyID, subjectID string) (ResetUserKeyResponse, error) {
 			store, err := sqlite.Open(ctx, sqliteDSN)
 			if err != nil {
-				return ResetUserKeyResponse{}, fmt.Errorf("open store: %w", err)
+				return ResetUserKeyResponse{}, recordUserKeyFailure("reset", "open_store_error", fmt.Errorf("open store: %w", err))
 			}
 			defer store.Close()

 			rec, err := store.UserKeys().GetByID(ctx, keyID)
 			if err != nil {
-				return ResetUserKeyResponse{}, fmt.Errorf("get key: %w", err)
+				return ResetUserKeyResponse{}, recordUserKeyFailure("reset", "get_key_error", fmt.Errorf("get key: %w", err))
 			}
 			if rec.OwnerSubjectID != subjectID && subjectID != "admin" {
-				return ResetUserKeyResponse{}, fmt.Errorf("key %q not found", keyID)
+				return ResetUserKeyResponse{}, recordUserKeyFailure("reset", "not_found", fmt.Errorf("key %q not found", keyID))
 			}
 			windowStart := time.Now().UTC().Format("2006-01-02T00:00:00Z")
 			count, err := store.SubjectRateLimits().IncrementWindow(ctx, subjectID, "reset", windowStart)
 			if err != nil {
-				return ResetUserKeyResponse{}, fmt.Errorf("increment reset rate limit: %w", err)
+				return ResetUserKeyResponse{}, recordUserKeyFailure("reset", "rate_limit_store_error", fmt.Errorf("increment reset rate limit: %w", err))
 			}
 			if count > defaultKeyResetPerDay {
 				metrics.RecordUserKeyOperation("reset", "rate_limited")
@@ -277,15 +282,15 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 			// Re-resolve host access to get a fresh key
 			_, route, _, client, err := resolveLogicalGroupHost(ctx, store, rec.LogicalGroupID)
 			if err != nil {
-				return ResetUserKeyResponse{}, fmt.Errorf("resolve host for %q: %w", rec.LogicalGroupID, err)
+				return ResetUserKeyResponse{}, recordUserKeyFailure("reset", "resolve_host_error", fmt.Errorf("resolve host for %q: %w", rec.LogicalGroupID, err))
 			}
 			hostGroupID, err := resolveShadowHostGroupID(ctx, client, route)
 			if err != nil {
-				return ResetUserKeyResponse{}, fmt.Errorf("resolve shadow group id for %q: %w", route.ShadowGroupID, err)
+				return ResetUserKeyResponse{}, recordUserKeyFailure("reset", "resolve_shadow_group_error", fmt.Errorf("resolve shadow group id for %q: %w", route.ShadowGroupID, err))
 			}
 			newPlaintext, err := ensureSubjectHasAccess(ctx, client, rec.OwnerSubjectID, hostGroupID)
 			if err != nil {
-				return ResetUserKeyResponse{}, fmt.Errorf("ensure access on reset for %q: %w", rec.LogicalGroupID, err)
+				return ResetUserKeyResponse{}, recordUserKeyFailure("reset", "ensure_access_error", fmt.Errorf("ensure access on reset for %q: %w", rec.LogicalGroupID, err))
 			}

 			hostFingerprint := "sha256:" + sha256Hex(newPlaintext)
@@ -309,7 +314,7 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 				return nil
 			})
 			if err != nil {
-				return ResetUserKeyResponse{}, err
+				return ResetUserKeyResponse{}, recordUserKeyFailure("reset", "db_tx_error", err)
 			}
 			metrics.RecordUserKeyOperation("reset", "success")
 			return ResetUserKeyResponse{PlaintextKey: newPlaintext, MaskedPreview: masked, AdminStatus: "active"}, nil
@@ -317,27 +322,27 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 		pauseFn: func(ctx context.Context, keyID, subjectID, reason string) (UserKeyMeta, error) {
 			store, err := sqlite.Open(ctx, sqliteDSN)
 			if err != nil {
-				return UserKeyMeta{}, fmt.Errorf("open store: %w", err)
+				return UserKeyMeta{}, recordUserKeyFailure("pause", "open_store_error", fmt.Errorf("open store: %w", err))
 			}
 			defer store.Close()

 			rec, err := store.UserKeys().GetByID(ctx, keyID)
 			if err != nil {
-				return UserKeyMeta{}, fmt.Errorf("get key: %w", err)
+				return UserKeyMeta{}, recordUserKeyFailure("pause", "get_key_error", fmt.Errorf("get key: %w", err))
 			}
 			if rec.OwnerSubjectID != subjectID && subjectID != "admin" {
-				return UserKeyMeta{}, fmt.Errorf("key %q not found", keyID)
+				return UserKeyMeta{}, recordUserKeyFailure("pause", "not_found", fmt.Errorf("key %q not found", keyID))
 			}
 			_, route, _, client, err := resolveLogicalGroupHost(ctx, store, rec.LogicalGroupID)
 			if err != nil {
-				return UserKeyMeta{}, fmt.Errorf("resolve host for pause %q: %w", rec.LogicalGroupID, err)
+				return UserKeyMeta{}, recordUserKeyFailure("pause", "resolve_host_error", fmt.Errorf("resolve host for pause %q: %w", rec.LogicalGroupID, err))
 			}
 			hostGroupID, err := resolveShadowHostGroupID(ctx, client, route)
 			if err != nil {
-				return UserKeyMeta{}, fmt.Errorf("resolve shadow group id for pause %q: %w", route.ShadowGroupID, err)
+				return UserKeyMeta{}, recordUserKeyFailure("pause", "resolve_shadow_group_error", fmt.Errorf("resolve shadow group id for pause %q: %w", route.ShadowGroupID, err))
 			}
 			if err := client.PauseManagedSubscriptionAccess(ctx, rec.OwnerSubjectID, hostGroupID); err != nil {
-				return UserKeyMeta{}, fmt.Errorf("pause managed subscription access: %w", err)
+				return UserKeyMeta{}, recordUserKeyFailure("pause", "pause_access_error", fmt.Errorf("pause managed subscription access: %w", err))
 			}
 			err = store.WithTx(ctx, func(q *sqlite.Queries) error {
 				if err := q.UserKeys.UpdateStatus(ctx, keyID, "paused"); err != nil {
@@ -352,7 +357,7 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 				return nil
 			})
 			if err != nil {
-				return UserKeyMeta{}, err
+				return UserKeyMeta{}, recordUserKeyFailure("pause", "db_tx_error", err)
 			}
 			metrics.RecordUserKeyOperation("pause", "success")
 			return UserKeyMeta{KeyID: keyID, MaskedPreview: rec.MaskedPreview, AdminStatus: "paused"}, nil
@@ -360,27 +365,27 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 		resumeFn: func(ctx context.Context, keyID, subjectID string) (UserKeyMeta, error) {
 			store, err := sqlite.Open(ctx, sqliteDSN)
 			if err != nil {
-				return UserKeyMeta{}, fmt.Errorf("open store: %w", err)
+				return UserKeyMeta{}, recordUserKeyFailure("resume", "open_store_error", fmt.Errorf("open store: %w", err))
 			}
 			defer store.Close()

 			rec, err := store.UserKeys().GetByID(ctx, keyID)
 			if err != nil {
-				return UserKeyMeta{}, fmt.Errorf("get key: %w", err)
+				return UserKeyMeta{}, recordUserKeyFailure("resume", "get_key_error", fmt.Errorf("get key: %w", err))
 			}
 			if rec.OwnerSubjectID != subjectID && subjectID != "admin" {
-				return UserKeyMeta{}, fmt.Errorf("key %q not found", keyID)
+				return UserKeyMeta{}, recordUserKeyFailure("resume", "not_found", fmt.Errorf("key %q not found", keyID))
 			}
 			_, route, _, client, err := resolveLogicalGroupHost(ctx, store, rec.LogicalGroupID)
 			if err != nil {
-				return UserKeyMeta{}, fmt.Errorf("resolve host for resume %q: %w", rec.LogicalGroupID, err)
+				return UserKeyMeta{}, recordUserKeyFailure("resume", "resolve_host_error", fmt.Errorf("resolve host for resume %q: %w", rec.LogicalGroupID, err))
 			}
 			hostGroupID, err := resolveShadowHostGroupID(ctx, client, route)
 			if err != nil {
-				return UserKeyMeta{}, fmt.Errorf("resolve shadow group id for resume %q: %w", route.ShadowGroupID, err)
+				return UserKeyMeta{}, recordUserKeyFailure("resume", "resolve_shadow_group_error", fmt.Errorf("resolve shadow group id for resume %q: %w", route.ShadowGroupID, err))
 			}
 			if err := client.ResumeManagedSubscriptionAccess(ctx, rec.OwnerSubjectID, hostGroupID); err != nil {
-				return UserKeyMeta{}, fmt.Errorf("resume managed subscription access: %w", err)
+				return UserKeyMeta{}, recordUserKeyFailure("resume", "resume_access_error", fmt.Errorf("resume managed subscription access: %w", err))
 			}
 			err = store.WithTx(ctx, func(q *sqlite.Queries) error {
 				if err := q.UserKeys.UpdateStatus(ctx, keyID, "active"); err != nil {
@@ -395,7 +400,7 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 				return nil
 			})
 			if err != nil {
-				return UserKeyMeta{}, err
+				return UserKeyMeta{}, recordUserKeyFailure("resume", "db_tx_error", err)
 			}
 			metrics.RecordUserKeyOperation("resume", "success")
 			return UserKeyMeta{KeyID: keyID, MaskedPreview: rec.MaskedPreview, AdminStatus: "active"}, nil
@@ -403,16 +408,16 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 		deleteFn: func(ctx context.Context, keyID, subjectID string) error {
 			store, err := sqlite.Open(ctx, sqliteDSN)
 			if err != nil {
-				return fmt.Errorf("open store: %w", err)
+				return recordUserKeyFailure("delete", "open_store_error", fmt.Errorf("open store: %w", err))
 			}
 			defer store.Close()

 			rec, err := store.UserKeys().GetByID(ctx, keyID)
 			if err != nil {
-				return fmt.Errorf("get key: %w", err)
+				return recordUserKeyFailure("delete", "get_key_error", fmt.Errorf("get key: %w", err))
 			}
 			if rec.OwnerSubjectID != subjectID && subjectID != "admin" {
-				return fmt.Errorf("key %q not found", keyID)
+				return recordUserKeyFailure("delete", "not_found", fmt.Errorf("key %q not found", keyID))
 			}
 			err = store.WithTx(ctx, func(q *sqlite.Queries) error {
 				if err := q.UserKeys.UpdateStatus(ctx, keyID, "retired"); err != nil {
@@ -431,8 +436,9 @@ func buildUserKeyHandler(sqliteDSN string) *UserKeyHandler {
 			})
 			if err == nil {
 				metrics.RecordUserKeyOperation("delete", "success")
+				return nil
 			}
-			return err
+			return recordUserKeyFailure("delete", "db_tx_error", err)
 		},
 	}
 }
--- a/internal/app/key_self_service_test.go
+++ b/internal/app/key_self_service_test.go
@@ -322,3 +322,151 @@ func TestUserKeyAPIMetricsMiddlewareAndCreateMetric(t *testing.T) {
 		t.Fatal("expected metrics endpoint to expose user_key_operations_total after create validation failure")
 	}
 }
+
+func TestUserKeyPauseResumeDeleteLifecycleUpdatesHostAndStore(t *testing.T) {
+	t.Parallel()
+
+	store := openAppTestStore(t)
+	defer closeAppTestStore(t, store)
+
+	const logicalGroupID = "gpt-shared"
+	const hostGroupID = "999"
+	const subjectID = "portal-user:lifecycle"
+	const keyID = "key_lifecycle"
+
+	var allowedGroupsUpdates [][]int64
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == http.MethodGet && strings.HasPrefix(r.URL.RequestURI(), "/api/v1/admin/users?"):
+			w.Write([]byte(`{"data":{"items":[{"id":84,"email":"` + expectedManagedIdentity(subjectID, hostGroupID).Email + `"}]}}`))
+		case r.Method == http.MethodPut && r.URL.Path == "/api/v1/admin/users/84":
+			var payload struct {
+				AllowedGroups []int64 `json:"allowed_groups"`
+			}
+			if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
+				t.Fatalf("decode update payload: %v", err)
+			}
+			allowedGroupsUpdates = append(allowedGroupsUpdates, append([]int64(nil), payload.AllowedGroups...))
+			w.Write([]byte(`{"data":{"id":84}}`))
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer server.Close()
+
+	_, _ = store.Hosts().Create(context.Background(), sqlite.Host{
+		HostID:              "test-host",
+		BaseURL:             server.URL,
+		HostVersion:         "0.0.1",
+		CapabilityProbeJSON: "{}",
+		AuthType:            "apikey",
+		AuthToken:           "test-token",
+	})
+	_, _ = store.LogicalGroups().Create(context.Background(), sqlite.LogicalGroup{
+		LogicalGroupID: logicalGroupID,
+		DisplayName:    "GPT Shared",
+		Status:         "active",
+	})
+	_, _ = store.LogicalGroupRoutes().Create(context.Background(), sqlite.LogicalGroupRoute{
+		RouteID:        "test-route",
+		LogicalGroupID: logicalGroupID,
+		Name:           "Test Route",
+		Status:         "active",
+		ShadowHostID:   "test-host",
+		ShadowGroupID:  hostGroupID,
+	})
+	if _, err := store.UserKeys().Create(context.Background(), sqlite.UserKeyRecord{
+		KeyID:          keyID,
+		OwnerSubjectID: subjectID,
+		KeyFingerprint: "sha256:test",
+		MaskedPreview:  "sk-****test",
+		DisplayName:    "lifecycle key",
+		LogicalGroupID: logicalGroupID,
+		AllowedModels:  []string{"gpt-5.4"},
+		AdminStatus:    "active",
+		QuotaStatus:    "ok",
+	}); err != nil {
+		t.Fatalf("UserKeys().Create() error = %v", err)
+	}
+
+	handler := buildUserKeyHandler(appTestDSN(t, store))
+	paused, err := handler.pauseFn(context.Background(), keyID, subjectID, "")
+	if err != nil {
+		t.Fatalf("pauseFn() error = %v", err)
+	}
+	if paused.AdminStatus != "paused" {
+		t.Fatalf("pauseFn() admin_status = %q, want paused", paused.AdminStatus)
+	}
+	row, err := store.UserKeys().GetByID(context.Background(), keyID)
+	if err != nil {
+		t.Fatalf("UserKeys().GetByID() after pause error = %v", err)
+	}
+	if row.AdminStatus != "paused" {
+		t.Fatalf("stored admin_status after pause = %q, want paused", row.AdminStatus)
+	}
+
+	resumed, err := handler.resumeFn(context.Background(), keyID, subjectID)
+	if err != nil {
+		t.Fatalf("resumeFn() error = %v", err)
+	}
+	if resumed.AdminStatus != "active" {
+		t.Fatalf("resumeFn() admin_status = %q, want active", resumed.AdminStatus)
+	}
+	row, err = store.UserKeys().GetByID(context.Background(), keyID)
+	if err != nil {
+		t.Fatalf("UserKeys().GetByID() after resume error = %v", err)
+	}
+	if row.AdminStatus != "active" {
+		t.Fatalf("stored admin_status after resume = %q, want active", row.AdminStatus)
+	}
+
+	if err := handler.deleteFn(context.Background(), keyID, subjectID); err != nil {
+		t.Fatalf("deleteFn() error = %v", err)
+	}
+	row, err = store.UserKeys().GetByID(context.Background(), keyID)
+	if err != nil {
+		t.Fatalf("UserKeys().GetByID() after delete error = %v", err)
+	}
+	if row.AdminStatus != "retired" {
+		t.Fatalf("stored admin_status after delete = %q, want retired", row.AdminStatus)
+	}
+
+	if len(allowedGroupsUpdates) != 2 {
+		t.Fatalf("allowedGroupsUpdates len = %d, want 2", len(allowedGroupsUpdates))
+	}
+	if len(allowedGroupsUpdates[0]) != 0 {
+		t.Fatalf("pause allowed_groups = %#v, want empty", allowedGroupsUpdates[0])
+	}
+	if len(allowedGroupsUpdates[1]) != 1 || allowedGroupsUpdates[1][0] != 999 {
+		t.Fatalf("resume allowed_groups = %#v, want [999]", allowedGroupsUpdates[1])
+	}
+}
+
+func TestResolveShadowHostGroupIDByName(t *testing.T) {
+	t.Parallel()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/v1/admin/groups":
+			w.Write([]byte(`{"data":{"items":[{"id":321,"name":"group-by-name"}]}}`))
+		case "/api/v1/admin/channels", "/api/v1/admin/payment/plans", "/api/v1/admin/accounts":
+			w.Write([]byte(`{"data":{"items":[]}}`))
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer server.Close()
+
+	client, err := newSub2APIClient(server.URL, CreateHostAuth{Type: "apikey", Token: "test-token"})
+	if err != nil {
+		t.Fatalf("newSub2APIClient() error = %v", err)
+	}
+
+	groupID, err := resolveShadowHostGroupID(context.Background(), client, sqlite.LogicalGroupRoute{ShadowGroupID: "group-by-name"})
+	if err != nil {
+		t.Fatalf("resolveShadowHostGroupID() error = %v", err)
+	}
+	if groupID != "321" {
+		t.Fatalf("groupID = %q, want 321", groupID)
+	}
+}
--- a/internal/app/route_resolve_api.go
+++ b/internal/app/route_resolve_api.go
@@ -227,7 +227,9 @@ func buildResolveRouteAction(sqliteDSN string, stickyRuntime stickyStoreRuntime,
 			}
 		}
 		decisionStatus := "bind"
-		if selection.fallbackUsed {
+		if selection.failoverFrom != nil {
+			decisionStatus = "failover"
+		} else if selection.fallbackUsed {
 			decisionStatus = "fallback"
 		}
 		metrics.RecordRouteDecision(req.LogicalGroupID, decisionStatus)
--- a/internal/app/route_resolve_api_test.go
+++ b/internal/app/route_resolve_api_test.go
@@ -222,6 +222,9 @@ func TestNewActionSetResolveRouteFlow(t *testing.T) {
 	if !strings.Contains(body, "route_failovers_total") {
 		t.Fatal("metrics missing route_failovers_total after fallback flow")
 	}
+	if !strings.Contains(body, `route_decisions_total{logical_group="gpt-shared",status="failover"}`) {
+		t.Fatalf("metrics missing failover decision status after resolve flow: %s", body)
+	}
 }

 func TestResolveRouteHelpers(t *testing.T) {
--- a/internal/app/user_key_operation_metrics_test.go
+++ b/internal/app/user_key_operation_metrics_test.go
@@ -0,0 +1,55 @@
+package app
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"sub2api-cn-relay-manager/internal/metrics"
+)
+
+func TestUserKeyCreateResolveHostErrorRecordsMetric(t *testing.T) {
+	t.Parallel()
+	store := openAppTestStore(t)
+	defer closeAppTestStore(t, store)
+
+	handler := NewAPIHandler("t", ActionSet{
+		UserKeyHandler: buildUserKeyHandler(appTestDSN(t, store)),
+	})
+
+	req := makeCreateRequest(t, http.MethodPost, "/api/keys", makeCreateBody("missing-group", "portal key", []string{"gpt-5.4"}))
+	req.Header.Set("X-Portal-Subject", "portal-user")
+	resp := httptestRecorder(handler, req)
+	if resp.code != http.StatusInternalServerError {
+		t.Fatalf("status code = %d, want 500 body=%s", resp.code, resp.Body().String())
+	}
+
+	metricsReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+	metricsResp := httptest.NewRecorder()
+	metrics.Handler().ServeHTTP(metricsResp, metricsReq)
+	body := metricsResp.Body.String()
+	if !strings.Contains(body, `user_key_operations_total{operation="create",result="resolve_host_error"}`) {
+		t.Fatalf("metrics body missing create resolve_host_error metric: %s", body)
+	}
+}
+
+func TestUserKeyDeleteGetKeyErrorRecordsMetric(t *testing.T) {
+	t.Parallel()
+	store := openAppTestStore(t)
+	defer closeAppTestStore(t, store)
+
+	handler := buildUserKeyHandler(appTestDSN(t, store))
+	if err := handler.deleteFn(context.Background(), "key_missing", "portal-user"); err == nil {
+		t.Fatal("expected deleteFn to fail for missing key")
+	}
+
+	metricsReq := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+	metricsResp := httptest.NewRecorder()
+	metrics.Handler().ServeHTTP(metricsResp, metricsReq)
+	body := metricsResp.Body.String()
+	if !strings.Contains(body, `user_key_operations_total{operation="delete",result="get_key_error"}`) {
+		t.Fatalf("metrics body missing delete get_key_error metric: %s", body)
+	}
+}
--- a/internal/metrics/metrics.go
+++ b/internal/metrics/metrics.go
@@ -3,6 +3,7 @@ package metrics
 import (
 	"context"
 	"net/http"
+	"strconv"
 	"time"

 	"github.com/prometheus/client_golang/prometheus"
@@ -132,7 +133,8 @@ func RecordHTTPRequest(method, path string, status int, duration time.Duration)
 	if path == "" {
 		path = "unknown"
 	}
-	HTTPRequestsTotal.WithLabelValues(method, path, http.StatusText(status)).Inc()
+	statusLabel := strconv.Itoa(status)
+	HTTPRequestsTotal.WithLabelValues(method, path, statusLabel).Inc()
 	HTTPRequestDuration.WithLabelValues(method, path).Observe(duration.Seconds())
 }

--- a/internal/metrics/metrics_test.go
+++ b/internal/metrics/metrics_test.go
@@ -28,6 +28,9 @@ func TestHTTPRequestsTotal(t *testing.T) {
 	if !strings.Contains(body, "http_requests_total") {
 		t.Error("Expected metrics endpoint to contain http_requests_total")
 	}
+	if !strings.Contains(body, `status="200"`) {
+		t.Fatalf("expected numeric HTTP status label, got: %s", body)
+	}
 }

 func TestRecordRouteDecision(t *testing.T) {