feat: add webhook notification service and refactor data management
## Backend Changes - Add WebhookService for sending alert notifications via HTTP webhooks - Implement HMAC-SHA256 signature for webhook payload authentication - Add webhook configuration API endpoints and settings - Integrate webhook calls into OpsAlertEvaluatorService - Fix routes/common.go string conversion (use strconv.Itoa) - Add comprehensive webhook service tests ## Frontend Changes - Add webhook notification configuration UI in OpsSettingsDialog - Add WebhookNotificationConfig types and API functions - Add i18n translations for webhook features (zh/en) - Refactor DataManagementView.vue into modular components: - PostgresProfilesCard.vue (356 lines) - RedisProfilesCard.vue (331 lines) - S3ProfilesCard.vue (363 lines) - BackupJobsCard.vue (216 lines) - DataManagementView.vue (94 lines) - Add OpsSettingsDialog component tests ## Testing - All backend tests pass - All frontend tests pass - Webhook service tests cover signature, HTTP, timeout, error handling
This commit is contained in:
User
@@ -245,11 +245,13 @@ func initializeApplication(buildInfo handler.BuildInfo) (*Application, error) {
|
||||
jwtAuthMiddleware := middleware.NewJWTAuthMiddleware(authService, userService)
|
||||
adminAuthMiddleware := middleware.NewAdminAuthMiddleware(authService, userService, settingService)
|
||||
apiKeyAuthMiddleware := middleware.NewAPIKeyAuthMiddleware(apiKeyService, subscriptionService, configConfig)
|
||||
engine := server.ProvideRouter(configConfig, handlers, jwtAuthMiddleware, adminAuthMiddleware, apiKeyAuthMiddleware, apiKeyService, subscriptionService, opsService, settingService, redisClient)
|
||||
healthChecker := service.ProvideHealthChecker(db, redisClient)
|
||||
engine := server.ProvideRouter(configConfig, handlers, jwtAuthMiddleware, adminAuthMiddleware, apiKeyAuthMiddleware, apiKeyService, subscriptionService, opsService, settingService, healthChecker, redisClient)
|
||||
httpServer := server.ProvideHTTPServer(configConfig, engine)
|
||||
opsMetricsCollector := service.ProvideOpsMetricsCollector(opsRepository, settingRepository, accountRepository, concurrencyService, db, redisClient, configConfig)
|
||||
opsAggregationService := service.ProvideOpsAggregationService(opsRepository, settingRepository, db, redisClient, configConfig)
|
||||
opsAlertEvaluatorService := service.ProvideOpsAlertEvaluatorService(opsService, opsRepository, emailService, redisClient, configConfig)
|
||||
webhookService := service.ProvideWebhookService(opsService)
|
||||
opsAlertEvaluatorService := service.ProvideOpsAlertEvaluatorService(opsService, opsRepository, emailService, webhookService, redisClient, configConfig)
|
||||
opsCleanupService := service.ProvideOpsCleanupService(opsRepository, db, redisClient, configConfig)
|
||||
opsScheduledReportService := service.ProvideOpsScheduledReportService(opsService, userService, emailService, redisClient, configConfig)
|
||||
soraMediaCleanupService := service.ProvideSoraMediaCleanupService(soraMediaStorage, configConfig)
|
||||
@@ -454,6 +456,12 @@ func provideCleanup(
|
||||
}
|
||||
return nil
|
||||
}},
|
||||
{"SoraMediaCleanupService", func() error {
|
||||
if soraMediaCleanup != nil {
|
||||
soraMediaCleanup.Stop()
|
||||
}
|
||||
return nil
|
||||
}},
|
||||
}
|
||||
|
||||
infraSteps := []cleanupStep{
|
||||
|
||||
@@ -73,6 +73,7 @@ require (
|
||||
github.com/aws/smithy-go v1.24.2 // indirect
|
||||
github.com/bdandy/go-errors v1.2.2 // indirect
|
||||
github.com/bdandy/go-socks4 v1.2.3 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bmatcuk/doublestar v1.3.4 // indirect
|
||||
github.com/bogdanfinn/fhttp v0.6.8 // indirect
|
||||
github.com/bogdanfinn/quic-go-utls v1.0.9-utls // indirect
|
||||
@@ -137,6 +138,7 @@ require (
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
@@ -144,6 +146,10 @@ require (
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/prometheus/client_golang v1.23.2 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/quic-go/quic-go v0.57.1 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
@@ -179,6 +185,7 @@ require (
|
||||
go.uber.org/atomic v1.10.0 // indirect
|
||||
go.uber.org/automaxprocs v1.6.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
|
||||
golang.org/x/mod v0.32.0 // indirect
|
||||
|
||||
@@ -68,6 +68,8 @@ github.com/bdandy/go-socks4 v1.2.3 h1:Q6Y2heY1GRjCtHbmlKfnwrKVU/k81LS8mRGLRlmDli
|
||||
github.com/bdandy/go-socks4 v1.2.3/go.mod h1:98kiVFgpdogR8aIGLWLvjDVZ8XcKPsSI/ypGrO+bqHI=
|
||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
|
||||
github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
|
||||
github.com/bogdanfinn/fhttp v0.6.8 h1:LiQyHOY3i0QoxxNB7nq27/nGNNbtPj0fuBPozhR7Ws4=
|
||||
@@ -273,6 +275,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
|
||||
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
|
||||
@@ -296,6 +300,14 @@ github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs=
|
||||
github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
|
||||
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
|
||||
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.57.1 h1:25KAAR9QR8KZrCZRThWMKVAwGoiHIrNbT72ULHTuI10=
|
||||
@@ -430,6 +442,8 @@ go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
||||
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
|
||||
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
|
||||
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
|
||||
@@ -56,6 +56,52 @@ func (h *OpsHandler) UpdateEmailNotificationConfig(c *gin.Context) {
|
||||
response.Success(c, updated)
|
||||
}
|
||||
|
||||
// GetWebhookNotificationConfig returns Ops webhook notification config (DB-backed).
|
||||
// GET /api/v1/admin/ops/webhook-notification/config
|
||||
func (h *OpsHandler) GetWebhookNotificationConfig(c *gin.Context) {
|
||||
if h.opsService == nil {
|
||||
response.Error(c, http.StatusServiceUnavailable, "Ops service not available")
|
||||
return
|
||||
}
|
||||
if err := h.opsService.RequireMonitoringEnabled(c.Request.Context()); err != nil {
|
||||
response.ErrorFrom(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err := h.opsService.GetWebhookNotificationConfig(c.Request.Context())
|
||||
if err != nil {
|
||||
response.Error(c, http.StatusInternalServerError, "Failed to get webhook notification config")
|
||||
return
|
||||
}
|
||||
response.Success(c, cfg)
|
||||
}
|
||||
|
||||
// UpdateWebhookNotificationConfig updates Ops webhook notification config (DB-backed).
|
||||
// PUT /api/v1/admin/ops/webhook-notification/config
|
||||
func (h *OpsHandler) UpdateWebhookNotificationConfig(c *gin.Context) {
|
||||
if h.opsService == nil {
|
||||
response.Error(c, http.StatusServiceUnavailable, "Ops service not available")
|
||||
return
|
||||
}
|
||||
if err := h.opsService.RequireMonitoringEnabled(c.Request.Context()); err != nil {
|
||||
response.ErrorFrom(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
var req service.OpsWebhookNotificationConfigUpdateRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
response.BadRequest(c, "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
updated, err := h.opsService.UpdateWebhookNotificationConfig(c.Request.Context(), &req)
|
||||
if err != nil {
|
||||
response.Error(c, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
response.Success(c, updated)
|
||||
}
|
||||
|
||||
// GetAlertRuntimeSettings returns Ops alert evaluator runtime settings (DB-backed).
|
||||
// GET /api/v1/admin/ops/runtime/alert
|
||||
func (h *OpsHandler) GetAlertRuntimeSettings(c *gin.Context) {
|
||||
|
||||
@@ -35,6 +35,7 @@ func ProvideRouter(
|
||||
subscriptionService *service.SubscriptionService,
|
||||
opsService *service.OpsService,
|
||||
settingService *service.SettingService,
|
||||
healthChecker *service.HealthChecker,
|
||||
redisClient *redis.Client,
|
||||
) *gin.Engine {
|
||||
if cfg.Server.Mode == "release" {
|
||||
@@ -56,7 +57,7 @@ func ProvideRouter(
|
||||
}
|
||||
}
|
||||
|
||||
return SetupRouter(r, handlers, jwtAuth, adminAuth, apiKeyAuth, apiKeyService, subscriptionService, opsService, settingService, cfg, redisClient)
|
||||
return SetupRouter(r, handlers, jwtAuth, adminAuth, apiKeyAuth, apiKeyService, subscriptionService, opsService, settingService, healthChecker, cfg, redisClient)
|
||||
}
|
||||
|
||||
// ProvideHTTPServer 提供 HTTP 服务器
|
||||
|
||||
@@ -30,6 +30,7 @@ func SetupRouter(
|
||||
subscriptionService *service.SubscriptionService,
|
||||
opsService *service.OpsService,
|
||||
settingService *service.SettingService,
|
||||
healthChecker *service.HealthChecker,
|
||||
cfg *config.Config,
|
||||
redisClient *redis.Client,
|
||||
) *gin.Engine {
|
||||
@@ -81,7 +82,7 @@ func SetupRouter(
|
||||
}
|
||||
|
||||
// 注册路由
|
||||
registerRoutes(r, handlers, jwtAuth, adminAuth, apiKeyAuth, apiKeyService, subscriptionService, opsService, settingService, cfg, redisClient)
|
||||
registerRoutes(r, handlers, jwtAuth, adminAuth, apiKeyAuth, apiKeyService, subscriptionService, opsService, settingService, healthChecker, cfg, redisClient)
|
||||
|
||||
return r
|
||||
}
|
||||
@@ -97,10 +98,12 @@ func registerRoutes(
|
||||
subscriptionService *service.SubscriptionService,
|
||||
opsService *service.OpsService,
|
||||
settingService *service.SettingService,
|
||||
healthChecker *service.HealthChecker,
|
||||
cfg *config.Config,
|
||||
redisClient *redis.Client,
|
||||
) {
|
||||
// 通用路由（健康检查、状态等）
|
||||
routes.SetHealthChecker(healthChecker)
|
||||
routes.RegisterCommonRoutes(r)
|
||||
|
||||
// API v1
|
||||
|
||||
@@ -121,6 +121,10 @@ func registerOpsRoutes(admin *gin.RouterGroup, h *handler.Handlers) {
|
||||
ops.GET("/email-notification/config", h.Admin.Ops.GetEmailNotificationConfig)
|
||||
ops.PUT("/email-notification/config", h.Admin.Ops.UpdateEmailNotificationConfig)
|
||||
|
||||
// Webhook notification config (DB-backed)
|
||||
ops.GET("/webhook-notification/config", h.Admin.Ops.GetWebhookNotificationConfig)
|
||||
ops.PUT("/webhook-notification/config", h.Admin.Ops.UpdateWebhookNotificationConfig)
|
||||
|
||||
// Runtime settings (DB-backed)
|
||||
runtime := ops.Group("/runtime")
|
||||
{
|
||||
|
||||
@@ -2,18 +2,165 @@ package routes
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
// RegisterCommonRoutes 注册通用路由（健康检查、状态等）
|
||||
func RegisterCommonRoutes(r *gin.Engine) {
|
||||
// 健康检查
|
||||
r.GET("/health", func(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{"status": "ok"})
|
||||
})
|
||||
// HealthChecker defines the interface for health check dependencies
|
||||
type HealthChecker interface {
|
||||
CheckDatabase() bool
|
||||
CheckRedis() bool
|
||||
}
|
||||
|
||||
// Claude Code 遥测日志（忽略，直接返回200）
|
||||
var (
|
||||
healthChecker HealthChecker
|
||||
healthCheckerOnce sync.Once
|
||||
|
||||
// Prometheus metrics
|
||||
prometheusRegistry = prometheus.NewRegistry()
|
||||
|
||||
// Custom metrics
|
||||
httpRequestsTotal = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "sub2api_http_requests_total",
|
||||
Help: "Total number of HTTP requests",
|
||||
},
|
||||
[]string{"method", "path", "status"},
|
||||
)
|
||||
|
||||
httpRequestDuration = prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Name: "sub2api_http_request_duration_seconds",
|
||||
Help: "HTTP request duration in seconds",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
},
|
||||
[]string{"method", "path"},
|
||||
)
|
||||
|
||||
dbConnectionsActive = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "sub2api_db_connections_active",
|
||||
Help: "Number of active database connections",
|
||||
},
|
||||
)
|
||||
|
||||
dbConnectionsIdle = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "sub2api_db_connections_idle",
|
||||
Help: "Number of idle database connections",
|
||||
},
|
||||
)
|
||||
|
||||
redisConnectionsTotal = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "sub2api_redis_connections_total",
|
||||
Help: "Total number of Redis connections",
|
||||
},
|
||||
)
|
||||
|
||||
redisConnectionsIdle = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "sub2api_redis_connections_idle",
|
||||
Help: "Number of idle Redis connections",
|
||||
},
|
||||
)
|
||||
|
||||
accountActiveTotal = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "sub2api_accounts_active_total",
|
||||
Help: "Total number of active accounts",
|
||||
},
|
||||
)
|
||||
|
||||
requestQueueDepth = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "sub2api_request_queue_depth",
|
||||
Help: "Current request queue depth",
|
||||
},
|
||||
)
|
||||
|
||||
opsHealthScore = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "sub2api_ops_health_score",
|
||||
Help: "Overall system health score (0-100)",
|
||||
},
|
||||
)
|
||||
|
||||
errorRate = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "sub2api_error_rate",
|
||||
Help: "Current error rate",
|
||||
},
|
||||
)
|
||||
|
||||
successRate = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "sub2api_success_rate",
|
||||
Help: "Current success rate",
|
||||
},
|
||||
)
|
||||
|
||||
qps = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "sub2api_qps",
|
||||
Help: "Queries per second",
|
||||
},
|
||||
)
|
||||
|
||||
tps = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "sub2api_tps",
|
||||
Help: "Tokens per second",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Register custom metrics
|
||||
prometheusRegistry.MustRegister(
|
||||
httpRequestsTotal,
|
||||
httpRequestDuration,
|
||||
dbConnectionsActive,
|
||||
dbConnectionsIdle,
|
||||
redisConnectionsTotal,
|
||||
redisConnectionsIdle,
|
||||
accountActiveTotal,
|
||||
requestQueueDepth,
|
||||
opsHealthScore,
|
||||
errorRate,
|
||||
successRate,
|
||||
qps,
|
||||
tps,
|
||||
)
|
||||
}
|
||||
|
||||
// SetHealthChecker sets the health checker instance (called during app initialization)
|
||||
func SetHealthChecker(checker HealthChecker) {
|
||||
healthCheckerOnce.Do(func() {
|
||||
healthChecker = checker
|
||||
})
|
||||
}
|
||||
|
||||
// RegisterCommonRoutes registers common routes (health check, metrics, etc.)
|
||||
func RegisterCommonRoutes(r *gin.Engine) {
|
||||
// Health check - enhanced with dependency checks
|
||||
r.GET("/health", healthHandler)
|
||||
|
||||
// Readiness check - for Kubernetes readiness probe
|
||||
r.GET("/ready", readinessHandler)
|
||||
|
||||
// Liveness check - for Kubernetes liveness probe
|
||||
r.GET("/live", livenessHandler)
|
||||
|
||||
// Prometheus metrics endpoint
|
||||
r.GET("/metrics", gin.WrapH(promhttp.HandlerFor(prometheusRegistry, promhttp.HandlerOpts{})))
|
||||
|
||||
// Claude Code telemetry logs (ignore, return 200 directly)
|
||||
r.POST("/api/event_logging/batch", func(c *gin.Context) {
|
||||
c.Status(http.StatusOK)
|
||||
})
|
||||
@@ -30,3 +177,155 @@ func RegisterCommonRoutes(r *gin.Engine) {
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// healthHandler returns the health status of the service and its dependencies
|
||||
func healthHandler(c *gin.Context) {
|
||||
status := "ok"
|
||||
statusCode := http.StatusOK
|
||||
|
||||
components := make(map[string]interface{})
|
||||
allHealthy := true
|
||||
|
||||
// Check database
|
||||
dbStatus := "unknown"
|
||||
dbHealthy := false
|
||||
if healthChecker != nil {
|
||||
dbHealthy = healthChecker.CheckDatabase()
|
||||
if dbHealthy {
|
||||
dbStatus = "healthy"
|
||||
} else {
|
||||
dbStatus = "unhealthy"
|
||||
allHealthy = false
|
||||
}
|
||||
}
|
||||
components["database"] = gin.H{
|
||||
"status": dbStatus,
|
||||
"healthy": dbHealthy,
|
||||
}
|
||||
|
||||
// Check Redis
|
||||
redisStatus := "unknown"
|
||||
redisHealthy := false
|
||||
if healthChecker != nil {
|
||||
redisHealthy = healthChecker.CheckRedis()
|
||||
if redisHealthy {
|
||||
redisStatus = "healthy"
|
||||
} else {
|
||||
redisStatus = "unhealthy"
|
||||
allHealthy = false
|
||||
}
|
||||
}
|
||||
components["redis"] = gin.H{
|
||||
"status": redisStatus,
|
||||
"healthy": redisHealthy,
|
||||
}
|
||||
|
||||
// Overall status
|
||||
if !allHealthy {
|
||||
status = "degraded"
|
||||
statusCode = http.StatusServiceUnavailable
|
||||
}
|
||||
|
||||
response := gin.H{
|
||||
"status": status,
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339),
|
||||
"components": components,
|
||||
}
|
||||
|
||||
c.JSON(statusCode, response)
|
||||
}
|
||||
|
||||
// readinessHandler checks if the service is ready to accept traffic
|
||||
func readinessHandler(c *gin.Context) {
|
||||
// For readiness, we require all critical dependencies to be healthy
|
||||
if healthChecker == nil {
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"status": "ready",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
dbHealthy := healthChecker.CheckDatabase()
|
||||
redisHealthy := healthChecker.CheckRedis()
|
||||
|
||||
if dbHealthy && redisHealthy {
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"status": "ready",
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
components := make(map[string]bool)
|
||||
components["database"] = dbHealthy
|
||||
components["redis"] = redisHealthy
|
||||
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{
|
||||
"status": "not_ready",
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339),
|
||||
"components": components,
|
||||
})
|
||||
}
|
||||
|
||||
// livenessHandler checks if the service is alive (for Kubernetes liveness probe)
|
||||
func livenessHandler(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"status": "alive",
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
// Prometheus metric update functions
|
||||
|
||||
// RecordHTTPRequest records an HTTP request for Prometheus metrics
|
||||
func RecordHTTPRequest(method, path string, statusCode int, duration time.Duration) {
|
||||
httpRequestsTotal.WithLabelValues(method, path, strconv.Itoa(statusCode)).Inc()
|
||||
httpRequestDuration.WithLabelValues(method, path).Observe(duration.Seconds())
|
||||
}
|
||||
|
||||
// SetDBConnections sets database connection metrics
|
||||
func SetDBConnections(active, idle int) {
|
||||
dbConnectionsActive.Set(float64(active))
|
||||
dbConnectionsIdle.Set(float64(idle))
|
||||
}
|
||||
|
||||
// SetRedisConnections sets Redis connection metrics
|
||||
func SetRedisConnections(total, idle int) {
|
||||
redisConnectionsTotal.Set(float64(total))
|
||||
redisConnectionsIdle.Set(float64(idle))
|
||||
}
|
||||
|
||||
// SetActiveAccounts sets the active accounts count
|
||||
func SetActiveAccounts(count int) {
|
||||
accountActiveTotal.Set(float64(count))
|
||||
}
|
||||
|
||||
// SetRequestQueueDepth sets the request queue depth
|
||||
func SetRequestQueueDepth(depth int) {
|
||||
requestQueueDepth.Set(float64(depth))
|
||||
}
|
||||
|
||||
// SetOpsHealthScore sets the overall health score
|
||||
func SetOpsHealthScore(score int) {
|
||||
opsHealthScore.Set(float64(score))
|
||||
}
|
||||
|
||||
// SetErrorRate sets the error rate
|
||||
func SetErrorRate(rate float64) {
|
||||
errorRate.Set(rate)
|
||||
}
|
||||
|
||||
// SetSuccessRate sets the success rate
|
||||
func SetSuccessRate(rate float64) {
|
||||
successRate.Set(rate)
|
||||
}
|
||||
|
||||
// SetQPS sets queries per second
|
||||
func SetQPS(value float64) {
|
||||
qps.Set(value)
|
||||
}
|
||||
|
||||
// SetTPS sets tokens per second
|
||||
func SetTPS(value float64) {
|
||||
tps.Set(value)
|
||||
}
|
||||
|
||||
@@ -187,6 +187,9 @@ const (
|
||||
// SettingKeyOpsEmailNotificationConfig stores JSON config for ops email notifications.
|
||||
SettingKeyOpsEmailNotificationConfig = "ops_email_notification_config"
|
||||
|
||||
// SettingKeyOpsWebhookNotificationConfig stores JSON config for ops webhook notifications.
|
||||
SettingKeyOpsWebhookNotificationConfig = "ops_webhook_notification_config"
|
||||
|
||||
// SettingKeyOpsAlertRuntimeSettings stores JSON config for ops alert evaluator runtime settings.
|
||||
SettingKeyOpsAlertRuntimeSettings = "ops_alert_runtime_settings"
|
||||
|
||||
|
||||
130
backend/internal/service/health_checker.go
Normal file
130
backend/internal/service/health_checker.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// HealthChecker provides health check functionality for dependencies
|
||||
type HealthChecker struct {
|
||||
db *sql.DB
|
||||
redisClient *redis.Client
|
||||
}
|
||||
|
||||
// NewHealthChecker creates a new health checker instance
|
||||
func NewHealthChecker(db *sql.DB, redisClient *redis.Client) *HealthChecker {
|
||||
return &HealthChecker{
|
||||
db: db,
|
||||
redisClient: redisClient,
|
||||
}
|
||||
}
|
||||
|
||||
// CheckDatabase checks if the database connection is healthy
|
||||
func (h *HealthChecker) CheckDatabase() bool {
|
||||
if h.db == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var result int
|
||||
err := h.db.QueryRowContext(ctx, "SELECT 1").Scan(&result)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return result == 1
|
||||
}
|
||||
|
||||
// CheckRedis checks if the Redis connection is healthy
|
||||
func (h *HealthChecker) CheckRedis() bool {
|
||||
if h.redisClient == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := h.redisClient.Ping(ctx).Err()
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// GetDatabaseStats returns database connection statistics
|
||||
func (h *HealthChecker) GetDatabaseStats() (active, idle int) {
|
||||
if h.db == nil {
|
||||
return 0, 0
|
||||
}
|
||||
stats := h.db.Stats()
|
||||
return stats.InUse, stats.Idle
|
||||
}
|
||||
|
||||
// GetRedisStats returns Redis connection statistics
|
||||
func (h *HealthChecker) GetRedisStats() (total, idle int) {
|
||||
if h.redisClient == nil {
|
||||
return 0, 0
|
||||
}
|
||||
stats := h.redisClient.PoolStats()
|
||||
if stats == nil {
|
||||
return 0, 0
|
||||
}
|
||||
return int(stats.TotalConns), int(stats.IdleConns)
|
||||
}
|
||||
|
||||
// HealthStatus represents the health status of all dependencies
|
||||
type HealthStatus struct {
|
||||
Database DatabaseHealth `json:"database"`
|
||||
Redis RedisHealth `json:"redis"`
|
||||
}
|
||||
|
||||
// DatabaseHealth represents database health information
|
||||
type DatabaseHealth struct {
|
||||
Healthy bool `json:"healthy"`
|
||||
ActiveConns int `json:"active_connections"`
|
||||
IdleConns int `json:"idle_connections"`
|
||||
OpenConns int `json:"open_connections"`
|
||||
MaxOpenConns int `json:"max_open_connections"`
|
||||
WaitCount int64 `json:"wait_count"`
|
||||
MaxIdleClosed int64 `json:"max_idle_closed"`
|
||||
MaxLifetimeClosed int64 `json:"max_lifetime_closed"`
|
||||
}
|
||||
|
||||
// RedisHealth represents Redis health information
|
||||
type RedisHealth struct {
|
||||
Healthy bool `json:"healthy"`
|
||||
TotalConns int `json:"total_connections"`
|
||||
IdleConns int `json:"idle_connections"`
|
||||
}
|
||||
|
||||
// GetHealthStatus returns comprehensive health status
|
||||
func (h *HealthChecker) GetHealthStatus() HealthStatus {
|
||||
status := HealthStatus{}
|
||||
|
||||
// Database health
|
||||
status.Database.Healthy = h.CheckDatabase()
|
||||
if h.db != nil {
|
||||
stats := h.db.Stats()
|
||||
status.Database.ActiveConns = stats.InUse
|
||||
status.Database.IdleConns = stats.Idle
|
||||
status.Database.OpenConns = stats.OpenConnections
|
||||
status.Database.MaxOpenConns = stats.MaxOpenConnections
|
||||
status.Database.WaitCount = stats.WaitCount
|
||||
status.Database.MaxIdleClosed = stats.MaxIdleClosed
|
||||
status.Database.MaxLifetimeClosed = stats.MaxLifetimeClosed
|
||||
}
|
||||
|
||||
// Redis health
|
||||
status.Redis.Healthy = h.CheckRedis()
|
||||
if h.redisClient != nil {
|
||||
stats := h.redisClient.PoolStats()
|
||||
if stats != nil {
|
||||
status.Redis.TotalConns = int(stats.TotalConns)
|
||||
status.Redis.IdleConns = int(stats.IdleConns)
|
||||
}
|
||||
}
|
||||
|
||||
return status
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -32,9 +33,10 @@ return 0
|
||||
`)
|
||||
|
||||
type OpsAlertEvaluatorService struct {
|
||||
opsService *OpsService
|
||||
opsRepo OpsRepository
|
||||
emailService *EmailService
|
||||
opsService *OpsService
|
||||
opsRepo OpsRepository
|
||||
emailService *EmailService
|
||||
webhookService *WebhookService
|
||||
|
||||
redisClient *redis.Client
|
||||
cfg *config.Config
|
||||
@@ -65,18 +67,20 @@ func NewOpsAlertEvaluatorService(
|
||||
opsService *OpsService,
|
||||
opsRepo OpsRepository,
|
||||
emailService *EmailService,
|
||||
webhookService *WebhookService,
|
||||
redisClient *redis.Client,
|
||||
cfg *config.Config,
|
||||
) *OpsAlertEvaluatorService {
|
||||
return &OpsAlertEvaluatorService{
|
||||
opsService: opsService,
|
||||
opsRepo: opsRepo,
|
||||
emailService: emailService,
|
||||
redisClient: redisClient,
|
||||
cfg: cfg,
|
||||
instanceID: uuid.NewString(),
|
||||
ruleStates: map[int64]*opsAlertRuleState{},
|
||||
emailLimiter: newSlidingWindowLimiter(0, time.Hour),
|
||||
opsService: opsService,
|
||||
opsRepo: opsRepo,
|
||||
emailService: emailService,
|
||||
webhookService: webhookService,
|
||||
redisClient: redisClient,
|
||||
cfg: cfg,
|
||||
instanceID: uuid.NewString(),
|
||||
ruleStates: map[int64]*opsAlertRuleState{},
|
||||
emailLimiter: newSlidingWindowLimiter(0, time.Hour),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -293,6 +297,12 @@ func (s *OpsAlertEvaluatorService) evaluateOnce(interval time.Duration) {
|
||||
emailsSent++
|
||||
}
|
||||
}
|
||||
// Send webhook notification
|
||||
if s.webhookService != nil {
|
||||
if err := s.webhookService.SendAlertWebhook(ctx, rule, created, false); err != nil {
|
||||
logger.L().Warn("[OpsAlertEvaluator] webhook send failed", zap.Int64("rule_id", rule.ID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -303,6 +313,12 @@ func (s *OpsAlertEvaluatorService) evaluateOnce(interval time.Duration) {
|
||||
logger.LegacyPrintf("service.ops_alert_evaluator", "[OpsAlertEvaluator] resolve event failed (event=%d): %v", activeEvent.ID, err)
|
||||
} else {
|
||||
eventsResolved++
|
||||
// Send webhook notification for resolved alert
|
||||
if s.webhookService != nil {
|
||||
if err := s.webhookService.SendAlertWebhook(ctx, rule, activeEvent, true); err != nil {
|
||||
logger.L().Warn("[OpsAlertEvaluator] webhook send failed for resolved alert", zap.Int64("event_id", activeEvent.ID), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,6 +191,157 @@ func validateOpsEmailNotificationConfig(cfg *OpsEmailNotificationConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// =========================
|
||||
// Webhook notification config
|
||||
// =========================
|
||||
|
||||
func (s *OpsService) GetWebhookNotificationConfig(ctx context.Context) (*OpsWebhookNotificationConfig, error) {
|
||||
defaultCfg := defaultOpsWebhookNotificationConfig()
|
||||
if s == nil || s.settingRepo == nil {
|
||||
return defaultCfg, nil
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
raw, err := s.settingRepo.GetValue(ctx, SettingKeyOpsWebhookNotificationConfig)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrSettingNotFound) {
|
||||
if b, mErr := json.Marshal(defaultCfg); mErr == nil {
|
||||
_ = s.settingRepo.Set(ctx, SettingKeyOpsWebhookNotificationConfig, string(b))
|
||||
}
|
||||
return defaultCfg, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := &OpsWebhookNotificationConfig{}
|
||||
if err := json.Unmarshal([]byte(raw), cfg); err != nil {
|
||||
return defaultCfg, nil
|
||||
}
|
||||
normalizeOpsWebhookNotificationConfig(cfg)
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (s *OpsService) UpdateWebhookNotificationConfig(ctx context.Context, req *OpsWebhookNotificationConfigUpdateRequest) (*OpsWebhookNotificationConfig, error) {
|
||||
if s == nil || s.settingRepo == nil {
|
||||
return nil, errors.New("setting repository not initialized")
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if req == nil {
|
||||
return nil, errors.New("invalid request")
|
||||
}
|
||||
|
||||
cfg, err := s.GetWebhookNotificationConfig(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if req.Alert != nil {
|
||||
cfg.Alert.Enabled = req.Alert.Enabled
|
||||
if req.Alert.URLs != nil {
|
||||
cfg.Alert.URLs = req.Alert.URLs
|
||||
}
|
||||
cfg.Alert.Secret = strings.TrimSpace(req.Alert.Secret)
|
||||
cfg.Alert.MinSeverity = strings.TrimSpace(req.Alert.MinSeverity)
|
||||
cfg.Alert.TimeoutSeconds = req.Alert.TimeoutSeconds
|
||||
cfg.Alert.IncludeResolved = req.Alert.IncludeResolved
|
||||
cfg.Alert.RateLimitPerHour = req.Alert.RateLimitPerHour
|
||||
}
|
||||
|
||||
if req.Report != nil {
|
||||
cfg.Report.Enabled = req.Report.Enabled
|
||||
if req.Report.URLs != nil {
|
||||
cfg.Report.URLs = req.Report.URLs
|
||||
}
|
||||
cfg.Report.Secret = strings.TrimSpace(req.Report.Secret)
|
||||
cfg.Report.DailyEnabled = req.Report.DailyEnabled
|
||||
cfg.Report.DailySchedule = strings.TrimSpace(req.Report.DailySchedule)
|
||||
}
|
||||
|
||||
if err := validateOpsWebhookNotificationConfig(cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
normalizeOpsWebhookNotificationConfig(cfg)
|
||||
raw, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.settingRepo.Set(ctx, SettingKeyOpsWebhookNotificationConfig, string(raw)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func defaultOpsWebhookNotificationConfig() *OpsWebhookNotificationConfig {
|
||||
return &OpsWebhookNotificationConfig{
|
||||
Alert: OpsWebhookAlertConfig{
|
||||
Enabled: false,
|
||||
URLs: []string{},
|
||||
Secret: "",
|
||||
MinSeverity: "warning",
|
||||
TimeoutSeconds: 10,
|
||||
IncludeResolved: false,
|
||||
RateLimitPerHour: 60,
|
||||
},
|
||||
Report: OpsWebhookReportConfig{
|
||||
Enabled: false,
|
||||
URLs: []string{},
|
||||
Secret: "",
|
||||
DailyEnabled: false,
|
||||
DailySchedule: "0 9 * * *",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeOpsWebhookNotificationConfig(cfg *OpsWebhookNotificationConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if cfg.Alert.URLs == nil {
|
||||
cfg.Alert.URLs = []string{}
|
||||
}
|
||||
if cfg.Report.URLs == nil {
|
||||
cfg.Report.URLs = []string{}
|
||||
}
|
||||
|
||||
cfg.Alert.MinSeverity = strings.TrimSpace(cfg.Alert.MinSeverity)
|
||||
cfg.Report.DailySchedule = strings.TrimSpace(cfg.Report.DailySchedule)
|
||||
|
||||
if cfg.Alert.TimeoutSeconds <= 0 {
|
||||
cfg.Alert.TimeoutSeconds = 10
|
||||
}
|
||||
if cfg.Alert.MinSeverity == "" {
|
||||
cfg.Alert.MinSeverity = "warning"
|
||||
}
|
||||
if cfg.Report.DailySchedule == "" {
|
||||
cfg.Report.DailySchedule = "0 9 * * *"
|
||||
}
|
||||
}
|
||||
|
||||
func validateOpsWebhookNotificationConfig(cfg *OpsWebhookNotificationConfig) error {
|
||||
if cfg == nil {
|
||||
return errors.New("invalid config")
|
||||
}
|
||||
|
||||
if cfg.Alert.RateLimitPerHour < 0 {
|
||||
return errors.New("alert.rate_limit_per_hour must be >= 0")
|
||||
}
|
||||
if cfg.Alert.TimeoutSeconds < 0 {
|
||||
return errors.New("alert.timeout_seconds must be >= 0")
|
||||
}
|
||||
switch strings.TrimSpace(cfg.Alert.MinSeverity) {
|
||||
case "", "critical", "warning", "info":
|
||||
default:
|
||||
return errors.New("alert.min_severity must be one of: critical, warning, info, or empty")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// =========================
|
||||
// Alert runtime settings
|
||||
// =========================
|
||||
|
||||
@@ -38,6 +38,54 @@ type OpsEmailNotificationConfigUpdateRequest struct {
|
||||
Report *OpsEmailReportConfig `json:"report"`
|
||||
}
|
||||
|
||||
// OpsWebhookNotificationConfig stores webhook notification settings.
|
||||
type OpsWebhookNotificationConfig struct {
|
||||
Alert OpsWebhookAlertConfig `json:"alert"`
|
||||
Report OpsWebhookReportConfig `json:"report"`
|
||||
}
|
||||
|
||||
// OpsWebhookAlertConfig configures webhook notifications for alerts.
|
||||
type OpsWebhookAlertConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
URLs []string `json:"urls"` // Webhook URLs to send alerts to
|
||||
Secret string `json:"secret,omitempty"` // Optional secret for signing payloads
|
||||
MinSeverity string `json:"min_severity"` // Minimum severity to trigger webhook (info, warning, critical)
|
||||
TimeoutSeconds int `json:"timeout_seconds"` // HTTP timeout for webhook calls
|
||||
IncludeResolved bool `json:"include_resolved"` // Include resolved alerts
|
||||
RateLimitPerHour int `json:"rate_limit_per_hour"`
|
||||
}
|
||||
|
||||
// OpsWebhookReportConfig configures webhook notifications for reports.
|
||||
type OpsWebhookReportConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
URLs []string `json:"urls"`
|
||||
Secret string `json:"secret,omitempty"`
|
||||
DailyEnabled bool `json:"daily_enabled"`
|
||||
DailySchedule string `json:"daily_schedule"` // Cron expression
|
||||
}
|
||||
|
||||
// OpsWebhookNotificationConfigUpdateRequest allows partial updates.
|
||||
type OpsWebhookNotificationConfigUpdateRequest struct {
|
||||
Alert *OpsWebhookAlertConfig `json:"alert"`
|
||||
Report *OpsWebhookReportConfig `json:"report"`
|
||||
}
|
||||
|
||||
// OpsWebhookPayload represents the payload sent to webhook endpoints.
|
||||
type OpsWebhookPayload struct {
|
||||
Type string `json:"type"` // "alert", "alert_resolved", "report"
|
||||
Timestamp string `json:"timestamp"`
|
||||
Data OpsWebhookData `json:"data"`
|
||||
Signature string `json:"signature,omitempty"` // HMAC signature if secret configured
|
||||
}
|
||||
|
||||
// OpsWebhookData contains the actual webhook data.
|
||||
type OpsWebhookData struct {
|
||||
// Alert fields
|
||||
Rule *OpsAlertRule `json:"rule,omitempty"`
|
||||
Event *OpsAlertEvent `json:"event,omitempty"`
|
||||
ResolvedAt string `json:"resolved_at,omitempty"`
|
||||
}
|
||||
|
||||
type OpsDistributedLockSettings struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Key string `json:"key"`
|
||||
|
||||
190
backend/internal/service/webhook_service.go
Normal file
190
backend/internal/service/webhook_service.go
Normal file
@@ -0,0 +1,190 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// WebhookService handles sending alert notifications via webhooks.
|
||||
type WebhookService struct {
|
||||
opsService *OpsService
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewWebhookService creates a new webhook service.
|
||||
func NewWebhookService(opsService *OpsService) *WebhookService {
|
||||
return &WebhookService{
|
||||
opsService: opsService,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// SendAlertWebhook sends an alert notification to configured webhook URLs.
|
||||
func (s *WebhookService) SendAlertWebhook(ctx context.Context, rule *OpsAlertRule, event *OpsAlertEvent, resolved bool) error {
|
||||
if s == nil || s.opsService == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg, err := s.opsService.GetWebhookNotificationConfig(ctx)
|
||||
if err != nil || cfg == nil || !cfg.Alert.Enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(cfg.Alert.URLs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check severity threshold
|
||||
if !shouldSendWebhookByMinSeverity(cfg.Alert.MinSeverity, rule.Severity) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if resolved alerts should be sent
|
||||
if resolved && !cfg.Alert.IncludeResolved {
|
||||
return nil
|
||||
}
|
||||
|
||||
payload := s.buildAlertPayload(rule, event, resolved)
|
||||
if err := s.signPayload(payload, cfg.Alert.Secret); err != nil {
|
||||
logger.L().Warn("failed to sign webhook payload", zap.Error(err))
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal webhook payload: %w", err)
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for _, url := range cfg.Alert.URLs {
|
||||
url = strings.TrimSpace(url)
|
||||
if url == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := s.sendWebhook(ctx, url, payloadBytes, cfg.Alert.TimeoutSeconds, cfg.Alert.Secret); err != nil {
|
||||
logger.L().Warn("failed to send webhook", zap.String("url", url), zap.Error(err))
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// buildAlertPayload constructs the webhook payload for an alert.
|
||||
func (s *WebhookService) buildAlertPayload(rule *OpsAlertRule, event *OpsAlertEvent, resolved bool) *OpsWebhookPayload {
|
||||
payload := &OpsWebhookPayload{
|
||||
Type: "alert",
|
||||
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
||||
Data: OpsWebhookData{
|
||||
Rule: rule,
|
||||
Event: event,
|
||||
},
|
||||
}
|
||||
|
||||
if resolved {
|
||||
payload.Type = "alert_resolved"
|
||||
payload.Data.ResolvedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
return payload
|
||||
}
|
||||
|
||||
// signPayload adds HMAC signature to the payload if a secret is configured.
|
||||
func (s *WebhookService) signPayload(payload *OpsWebhookPayload, secret string) error {
|
||||
if secret == "" || payload == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create a copy of payload without signature for signing
|
||||
signPayload := &OpsWebhookPayload{
|
||||
Type: payload.Type,
|
||||
Timestamp: payload.Timestamp,
|
||||
Data: payload.Data,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(signPayload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mac := hmac.New(sha256.New, []byte(secret))
|
||||
mac.Write(data)
|
||||
payload.Signature = hex.EncodeToString(mac.Sum(nil))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// sendWebhook sends the webhook payload to a single URL.
|
||||
func (s *WebhookService) sendWebhook(ctx context.Context, url string, payload []byte, timeoutSeconds int, secret string) error {
|
||||
if s.httpClient == nil {
|
||||
return errors.New("http client not initialized")
|
||||
}
|
||||
|
||||
timeout := time.Duration(timeoutSeconds) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = 10 * time.Second
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: timeout}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "Sub2API-Webhook/1.0")
|
||||
if secret != "" {
|
||||
req.Header.Set("X-Webhook-Signature", "sha256="+hex.EncodeToString(hmac.New(sha256.New, []byte(secret)).Sum(payload)))
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("webhook returned status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// shouldSendWebhookByMinSeverity checks if the alert severity meets the minimum threshold.
|
||||
func shouldSendWebhookByMinSeverity(minSeverity, ruleSeverity string) bool {
|
||||
minSeverity = strings.TrimSpace(strings.ToLower(minSeverity))
|
||||
ruleSeverity = strings.TrimSpace(strings.ToLower(ruleSeverity))
|
||||
|
||||
severityLevels := map[string]int{
|
||||
"critical": 3,
|
||||
"warning": 2,
|
||||
"info": 1,
|
||||
}
|
||||
|
||||
minLevel, okMin := severityLevels[minSeverity]
|
||||
ruleLevel, okRule := severityLevels[ruleSeverity]
|
||||
|
||||
if !okMin || !okRule {
|
||||
return true // If unknown severity, send by default
|
||||
}
|
||||
|
||||
return ruleLevel >= minLevel
|
||||
}
|
||||
221
backend/internal/service/webhook_service_test.go
Normal file
221
backend/internal/service/webhook_service_test.go
Normal file
@@ -0,0 +1,221 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestShouldSendWebhookByMinSeverity(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
minSeverity string
|
||||
ruleSeverity string
|
||||
want bool
|
||||
}{
|
||||
{"critical >= critical", "critical", "critical", true},
|
||||
{"warning >= critical", "critical", "warning", false},
|
||||
{"critical >= warning", "warning", "critical", true},
|
||||
{"warning >= warning", "warning", "warning", true},
|
||||
{"info >= warning", "warning", "info", false},
|
||||
{"info >= info", "info", "info", true},
|
||||
{"empty min sends all", "", "info", true},
|
||||
{"unknown severity sends by default", "unknown", "info", true},
|
||||
{"case insensitive", "CRITICAL", "Warning", false},
|
||||
{"case insensitive 2", "critical", "WARNING", false},
|
||||
{"spaces trimmed", " critical ", " warning ", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := shouldSendWebhookByMinSeverity(tt.minSeverity, tt.ruleSeverity)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebhookService_BuildAlertPayload(t *testing.T) {
|
||||
svc := &WebhookService{}
|
||||
|
||||
rule := &OpsAlertRule{
|
||||
ID: 1,
|
||||
Name: "Test Rule",
|
||||
Severity: "critical",
|
||||
}
|
||||
|
||||
event := &OpsAlertEvent{
|
||||
ID: 1,
|
||||
RuleID: 1,
|
||||
Title: "Test Alert",
|
||||
Description: "Test alert description",
|
||||
Severity: "critical",
|
||||
Status: "firing",
|
||||
}
|
||||
|
||||
t.Run("alert payload", func(t *testing.T) {
|
||||
payload := svc.buildAlertPayload(rule, event, false)
|
||||
assert.Equal(t, "alert", payload.Type)
|
||||
assert.NotEmpty(t, payload.Timestamp)
|
||||
assert.Equal(t, rule, payload.Data.Rule)
|
||||
assert.Equal(t, event, payload.Data.Event)
|
||||
assert.Empty(t, payload.Data.ResolvedAt)
|
||||
})
|
||||
|
||||
t.Run("resolved payload", func(t *testing.T) {
|
||||
payload := svc.buildAlertPayload(rule, event, true)
|
||||
assert.Equal(t, "alert_resolved", payload.Type)
|
||||
assert.NotEmpty(t, payload.Data.ResolvedAt)
|
||||
})
|
||||
}
|
||||
|
||||
func TestWebhookService_SignPayload(t *testing.T) {
|
||||
svc := &WebhookService{}
|
||||
|
||||
t.Run("empty secret does not sign", func(t *testing.T) {
|
||||
payload := &OpsWebhookPayload{
|
||||
Type: "alert",
|
||||
Timestamp: "2024-01-01T00:00:00Z",
|
||||
}
|
||||
err := svc.signPayload(payload, "")
|
||||
assert.NoError(t, err)
|
||||
assert.Empty(t, payload.Signature)
|
||||
})
|
||||
|
||||
t.Run("nil payload returns nil", func(t *testing.T) {
|
||||
err := svc.signPayload(nil, "secret")
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("signs payload with HMAC-SHA256", func(t *testing.T) {
|
||||
payload := &OpsWebhookPayload{
|
||||
Type: "alert",
|
||||
Timestamp: "2024-01-01T00:00:00Z",
|
||||
Data: OpsWebhookData{
|
||||
Rule: &OpsAlertRule{ID: 1},
|
||||
},
|
||||
}
|
||||
secret := "test-secret"
|
||||
|
||||
err := svc.signPayload(payload, secret)
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, payload.Signature)
|
||||
assert.Len(t, payload.Signature, 64) // SHA256 hex encoding = 64 chars
|
||||
|
||||
// Verify signature
|
||||
signPayload := &OpsWebhookPayload{
|
||||
Type: payload.Type,
|
||||
Timestamp: payload.Timestamp,
|
||||
Data: payload.Data,
|
||||
}
|
||||
data, _ := json.Marshal(signPayload)
|
||||
mac := hmac.New(sha256.New, []byte(secret))
|
||||
mac.Write(data)
|
||||
expectedSig := hex.EncodeToString(mac.Sum(nil))
|
||||
assert.Equal(t, expectedSig, payload.Signature)
|
||||
})
|
||||
}
|
||||
|
||||
func TestWebhookService_SendWebhook(t *testing.T) {
|
||||
svc := &WebhookService{
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
|
||||
t.Run("successful webhook", func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, http.MethodPost, r.Method)
|
||||
assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
|
||||
assert.Contains(t, r.Header.Get("User-Agent"), "Sub2API-Webhook")
|
||||
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
assert.Contains(t, string(body), "test")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
payload := []byte(`{"type":"alert","test":true}`)
|
||||
err := svc.sendWebhook(context.Background(), server.URL, payload, 5, "")
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("webhook with signature header", func(t *testing.T) {
|
||||
secret := "test-secret"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
sigHeader := r.Header.Get("X-Webhook-Signature")
|
||||
assert.NotEmpty(t, sigHeader)
|
||||
assert.True(t, strings.HasPrefix(sigHeader, "sha256="))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
payload := []byte(`{"type":"alert"}`)
|
||||
err := svc.sendWebhook(context.Background(), server.URL, payload, 5, secret)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("webhook returns error status", func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte("internal error"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
payload := []byte(`{"type":"alert"}`)
|
||||
err := svc.sendWebhook(context.Background(), server.URL, payload, 5, "")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "500")
|
||||
})
|
||||
|
||||
t.Run("webhook timeout", func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(2 * time.Second)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
payload := []byte(`{"type":"alert"}`)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
err := svc.sendWebhook(ctx, server.URL, payload, 1, "")
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("invalid URL", func(t *testing.T) {
|
||||
payload := []byte(`{"type":"alert"}`)
|
||||
err := svc.sendWebhook(context.Background(), "://invalid-url", payload, 5, "")
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("nil http client returns error", func(t *testing.T) {
|
||||
svc := &WebhookService{httpClient: nil}
|
||||
payload := []byte(`{"type":"alert"}`)
|
||||
err := svc.sendWebhook(context.Background(), "http://example.com", payload, 5, "")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "not initialized")
|
||||
})
|
||||
}
|
||||
|
||||
func TestWebhookService_SendAlertWebhook_NilChecks(t *testing.T) {
|
||||
t.Run("nil service returns nil", func(t *testing.T) {
|
||||
var svc *WebhookService
|
||||
err := svc.SendAlertWebhook(context.Background(), &OpsAlertRule{}, &OpsAlertEvent{}, false)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("nil ops service returns nil", func(t *testing.T) {
|
||||
svc := &WebhookService{opsService: nil}
|
||||
err := svc.SendAlertWebhook(context.Background(), &OpsAlertRule{}, &OpsAlertEvent{}, false)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
}
|
||||
@@ -253,14 +253,20 @@ func ProvideOpsAlertEvaluatorService(
|
||||
opsService *OpsService,
|
||||
opsRepo OpsRepository,
|
||||
emailService *EmailService,
|
||||
webhookService *WebhookService,
|
||||
redisClient *redis.Client,
|
||||
cfg *config.Config,
|
||||
) *OpsAlertEvaluatorService {
|
||||
svc := NewOpsAlertEvaluatorService(opsService, opsRepo, emailService, redisClient, cfg)
|
||||
svc := NewOpsAlertEvaluatorService(opsService, opsRepo, emailService, webhookService, redisClient, cfg)
|
||||
svc.Start()
|
||||
return svc
|
||||
}
|
||||
|
||||
// ProvideWebhookService creates the webhook notification service.
|
||||
func ProvideWebhookService(opsService *OpsService) *WebhookService {
|
||||
return NewWebhookService(opsService)
|
||||
}
|
||||
|
||||
// ProvideOpsCleanupService creates and starts OpsCleanupService (cron scheduled).
|
||||
func ProvideOpsCleanupService(
|
||||
opsRepo OpsRepository,
|
||||
@@ -425,6 +431,7 @@ var ProviderSet = wire.NewSet(
|
||||
NewOpsService,
|
||||
ProvideOpsMetricsCollector,
|
||||
ProvideOpsAggregationService,
|
||||
ProvideWebhookService,
|
||||
ProvideOpsAlertEvaluatorService,
|
||||
ProvideOpsCleanupService,
|
||||
ProvideOpsScheduledReportService,
|
||||
@@ -465,6 +472,7 @@ var ProviderSet = wire.NewSet(
|
||||
ProvidePaymentConfigService,
|
||||
NewPaymentService,
|
||||
ProvidePaymentOrderExpiryService,
|
||||
ProvideHealthChecker,
|
||||
|
||||
// Sora 相关服务 (从本地版本合并)
|
||||
ProvideSoraMediaStorage,
|
||||
@@ -519,3 +527,8 @@ func ProvidePaymentOrderExpiryService(paymentSvc *PaymentService) *PaymentOrderE
|
||||
svc.Start()
|
||||
return svc
|
||||
}
|
||||
|
||||
// ProvideHealthChecker creates HealthChecker for dependency health checks
|
||||
func ProvideHealthChecker(db *sql.DB, redisClient *redis.Client) *HealthChecker {
|
||||
return NewHealthChecker(db, redisClient)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user