249 lines
6.6 KiB
Go
249 lines
6.6 KiB
Go
package service
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"time"
|
|
|
|
"github.com/company/ai-ops/internal/domain/model"
|
|
"github.com/company/ai-ops/internal/domain/repository"
|
|
)
|
|
|
|
// NotificationTask 是通知任务
|
|
type NotificationTask struct {
|
|
Event *model.AlertEvent
|
|
ChannelIDs []string
|
|
Priority string // P0, P1, P2, P3
|
|
}
|
|
|
|
// NotificationService 是通知服务
|
|
type NotificationService struct {
|
|
channelRepo repository.ChannelRepository
|
|
logRepo repository.NotificationLogRepository
|
|
client *http.Client
|
|
queue chan NotificationTask
|
|
stopCh chan struct{}
|
|
}
|
|
|
|
// NewNotificationService 创建通知服务
|
|
func NewNotificationService(cr repository.ChannelRepository, logRepos ...repository.NotificationLogRepository) *NotificationService {
|
|
var logRepo repository.NotificationLogRepository
|
|
if len(logRepos) > 0 {
|
|
logRepo = logRepos[0]
|
|
}
|
|
ns := &NotificationService{
|
|
channelRepo: cr,
|
|
logRepo: logRepo,
|
|
client: &http.Client{Timeout: 10 * time.Second},
|
|
queue: make(chan NotificationTask, 1000),
|
|
stopCh: make(chan struct{}),
|
|
}
|
|
go ns.worker()
|
|
return ns
|
|
}
|
|
|
|
// Stop 停止通知服务
|
|
func (s *NotificationService) Stop() {
|
|
close(s.stopCh)
|
|
}
|
|
|
|
// Enqueue 将通知任务入队列
|
|
func (s *NotificationService) Enqueue(event *model.AlertEvent, channelIDs []string) {
|
|
task := NotificationTask{
|
|
Event: event,
|
|
ChannelIDs: channelIDs,
|
|
Priority: event.Level,
|
|
}
|
|
select {
|
|
case s.queue <- task:
|
|
slog.Info("notification_enqueued", "event_id", event.ID, "priority", event.Level)
|
|
default:
|
|
slog.Warn("notification_queue_full", "event_id", event.ID)
|
|
}
|
|
}
|
|
|
|
func (s *NotificationService) worker() {
|
|
for {
|
|
select {
|
|
case task := <-s.queue:
|
|
s.processTask(context.Background(), task)
|
|
case <-s.stopCh:
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *NotificationService) processTask(ctx context.Context, task NotificationTask) {
|
|
// 根据优先级设置发送超时
|
|
timeout := 120 * time.Second
|
|
if task.Priority == "P0" || task.Priority == "P1" {
|
|
timeout = 30 * time.Second
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
|
|
channels, err := s.channelRepo.List(ctx)
|
|
if err != nil {
|
|
slog.Error("list_channels_failed", "error", err, "event_id", task.Event.ID)
|
|
return
|
|
}
|
|
|
|
// 按优先级排序渠道
|
|
ordered := s.filterAndOrderChannels(channels, task.ChannelIDs)
|
|
|
|
// 发送通知，失败时自动切换备用渠道
|
|
sent := false
|
|
for _, ch := range ordered {
|
|
logID := s.createSendLog(ctx, task.Event, ch)
|
|
if err := s.sendToChannel(ctx, task.Event, ch); err != nil {
|
|
s.markSendFailed(ctx, logID, 1, err)
|
|
slog.Error("notify_channel_failed",
|
|
"event_id", task.Event.ID,
|
|
"channel_id", ch.ID,
|
|
"channel_type", ch.ChannelType,
|
|
"error", err,
|
|
)
|
|
continue
|
|
}
|
|
s.markSendSent(ctx, logID)
|
|
sent = true
|
|
slog.Info("notify_sent",
|
|
"event_id", task.Event.ID,
|
|
"channel_id", ch.ID,
|
|
"channel_type", ch.ChannelType,
|
|
)
|
|
break
|
|
}
|
|
|
|
if !sent {
|
|
slog.Error("notify_all_channels_failed", "event_id", task.Event.ID)
|
|
}
|
|
}
|
|
|
|
func (s *NotificationService) createSendLog(ctx context.Context, event *model.AlertEvent, ch *model.NotificationChannel) string {
|
|
if s.logRepo == nil {
|
|
return ""
|
|
}
|
|
log := &model.NotificationLog{
|
|
EventID: event.ID,
|
|
ChannelID: ch.ID,
|
|
ChannelType: ch.ChannelType,
|
|
Status: "pending",
|
|
}
|
|
if err := s.logRepo.CreateLog(ctx, log); err != nil {
|
|
slog.Error("create_notification_log_failed", "event_id", event.ID, "channel_id", ch.ID, "error", err)
|
|
return ""
|
|
}
|
|
return log.ID
|
|
}
|
|
|
|
func (s *NotificationService) markSendSent(ctx context.Context, logID string) {
|
|
if s.logRepo == nil || logID == "" {
|
|
return
|
|
}
|
|
if err := s.logRepo.MarkSent(ctx, logID); err != nil {
|
|
slog.Error("mark_notification_sent_failed", "log_id", logID, "error", err)
|
|
}
|
|
}
|
|
|
|
func (s *NotificationService) markSendFailed(ctx context.Context, logID string, retryCount int, err error) {
|
|
if s.logRepo == nil || logID == "" {
|
|
return
|
|
}
|
|
if markErr := s.logRepo.MarkFailed(ctx, logID, retryCount, err.Error()); markErr != nil {
|
|
slog.Error("mark_notification_failed_failed", "log_id", logID, "error", markErr)
|
|
}
|
|
}
|
|
|
|
func (s *NotificationService) filterAndOrderChannels(all []model.NotificationChannel, ids []string) []*model.NotificationChannel {
|
|
idSet := make(map[string]bool)
|
|
for _, id := range ids {
|
|
idSet[id] = true
|
|
}
|
|
|
|
var filtered []*model.NotificationChannel
|
|
for i := range all {
|
|
if idSet[all[i].ID] {
|
|
filtered = append(filtered, &all[i])
|
|
}
|
|
}
|
|
|
|
// 按优先级排序（高优先级在前）
|
|
for i := 0; i < len(filtered)-1; i++ {
|
|
for j := i + 1; j < len(filtered); j++ {
|
|
if filtered[j].Priority > filtered[i].Priority {
|
|
filtered[i], filtered[j] = filtered[j], filtered[i]
|
|
}
|
|
}
|
|
}
|
|
return filtered
|
|
}
|
|
|
|
func (s *NotificationService) sendToChannel(ctx context.Context, event *model.AlertEvent, ch *model.NotificationChannel) error {
|
|
switch ch.ChannelType {
|
|
case "webhook":
|
|
return s.sendWebhook(ctx, event, ch)
|
|
case "email":
|
|
return s.sendEmail(ctx, event, ch)
|
|
case "feishu":
|
|
return s.sendFeishu(ctx, event, ch)
|
|
case "wechat":
|
|
return s.sendWechat(ctx, event, ch)
|
|
default:
|
|
return fmt.Errorf("unsupported channel type: %s", ch.ChannelType)
|
|
}
|
|
}
|
|
|
|
func (s *NotificationService) sendWebhook(ctx context.Context, event *model.AlertEvent, ch *model.NotificationChannel) error {
|
|
url, ok := ch.Config["webhook_url"].(string)
|
|
if !ok || url == "" {
|
|
return fmt.Errorf("webhook_url not configured")
|
|
}
|
|
|
|
payload := map[string]any{
|
|
"alert_id": event.ID,
|
|
"rule_id": event.RuleID,
|
|
"level": event.Level,
|
|
"status": event.Status,
|
|
"resource": event.ResourceID,
|
|
"value": event.CurrentValue,
|
|
"threshold": event.ThresholdValue,
|
|
"timestamp": time.Now().Format(time.RFC3339),
|
|
}
|
|
body, _ := json.Marshal(payload)
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(body))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, err := s.client.Do(req)
|
|
if err != nil {
|
|
return fmt.Errorf("webhook request failed: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode >= 400 {
|
|
return fmt.Errorf("webhook returned status %d", resp.StatusCode)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *NotificationService) sendEmail(ctx context.Context, event *model.AlertEvent, ch *model.NotificationChannel) error {
|
|
return fmt.Errorf("email channel not yet implemented")
|
|
}
|
|
|
|
func (s *NotificationService) sendFeishu(ctx context.Context, event *model.AlertEvent, ch *model.NotificationChannel) error {
|
|
return fmt.Errorf("feishu channel not yet implemented")
|
|
}
|
|
|
|
func (s *NotificationService) sendWechat(ctx context.Context, event *model.AlertEvent, ch *model.NotificationChannel) error {
|
|
return fmt.Errorf("wechat channel not yet implemented")
|
|
}
|