feat(ops): add admin ops alert audit and watchdog alert sink

2026-06-15 18:17:30 +08:00
parent f4aacc1795
commit bf7cce90f9
6 changed files with 252 additions and 6 deletions
--- a/admin/routes/notifications.py
+++ b/admin/routes/notifications.py
@@ -2,6 +2,7 @@ from __future__ import annotations

 import json
 from html import escape
+from pathlib import Path
 from typing import Any, Optional

 from fastapi import APIRouter, Depends, Query
@@ -14,7 +15,7 @@ from admin.db import AdminUser


 router = APIRouter(prefix="/api/admin/notifications", tags=["notifications"])
-
+page_router = APIRouter(tags=["notifications-ui"])

 class NotificationEventResponse(BaseModel):
    order_id: str
@@ -36,6 +37,78 @@ class NotificationListResponse(BaseModel):
    items: list[NotificationEventResponse]


+class OpsAlertEventResponse(BaseModel):
+    created_at: str
+    alert_type: str
+    title: str
+    body: str
+    details: dict[str, Any]
+
+
+class OpsAlertListResponse(BaseModel):
+    total: int
+    items: list[OpsAlertEventResponse]
+
+
+@router.get("/ops-alerts", response_model=OpsAlertListResponse, summary="运维告警列表")
+def list_ops_alerts(
+    limit: int = Query(50, ge=1, le=200),
+    _: AdminUser = Depends(get_current_user),
+    settings: Settings = Depends(get_settings_dep),
+) -> dict[str, Any]:
+    path = Path(settings.ops_alert_log_path)
+    items: list[dict[str, Any]] = []
+    if path.exists():
+        for line in path.read_text(encoding="utf-8").splitlines()[-limit:]:
+            if not line.strip():
+                continue
+            try:
+                item = json.loads(line)
+            except Exception:
+                continue
+            if isinstance(item, dict):
+                items.append(item)
+    return {"total": len(items), "items": items}
+
+
+@page_router.get("/admin/ops-alerts", include_in_schema=False)
+def ops_alert_audit_page(
+    _: AdminUser = Depends(get_current_user),
+    settings: Settings = Depends(get_settings_dep),
+) -> HTMLResponse:
+    payload = list_ops_alerts(limit=200, _=_, settings=settings)
+    rows = []
+    for item in payload["items"]:
+        details = escape(json.dumps(item.get("details") or {}, ensure_ascii=False, indent=2))
+        rows.append(
+            "<tr>"
+            f"<td>{escape(str(item.get('created_at') or '-'))}</td>"
+            f"<td>{escape(str(item.get('alert_type') or '-'))}</td>"
+            f"<td>{escape(str(item.get('title') or '-'))}</td>"
+            f"<td>{escape(str(item.get('body') or '-'))}</td>"
+            f"<td><pre style='white-space:pre-wrap;max-width:520px'>{details}</pre></td>"
+            "</tr>"
+        )
+    rows_html = "".join(rows) or "<tr><td colspan='5'>暂无运维告警</td></tr>"
+    html = f"""<!doctype html>
+<html lang='zh-CN'><head><meta charset='utf-8' /><title>运维告警审计</title></head>
+<body style='font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;background:#f4f7fb;margin:0;padding:32px 20px;color:#172033;'>
+<main style='max-width:1280px;margin:0 auto;display:grid;gap:16px;'>
+<section style='background:#fff;border:1px solid #dbe3f0;border-radius:18px;padding:24px;'>
+  <h1>运维告警审计</h1>
+  <p>日志路径：{escape(settings.ops_alert_log_path)}</p>
+</section>
+<section style='background:#fff;border:1px solid #dbe3f0;border-radius:18px;padding:24px;overflow:auto;'>
+  <table style='width:100%;border-collapse:collapse;'>
+    <thead><tr><th>时间</th><th>类型</th><th>标题</th><th>摘要</th><th>details</th></tr></thead>
+    <tbody>{rows_html}</tbody>
+  </table>
+</section>
+</main>
+</body></html>"""
+    return HTMLResponse(html)
+
+
@router.get("", response_model=NotificationListResponse, summary="通知审计列表")
 def list_notifications(
    limit: int = Query(50, ge=1, le=200),
@@ -103,9 +176,6 @@ def list_notifications(
    }


-page_router = APIRouter(tags=["notifications-ui"])
-
-
@page_router.get("/admin/notifications", include_in_schema=False)
 def notification_audit_admin_page(
    limit: int = Query(50, ge=1, le=200),
--- a/admin/tests/test_ops_alerts_admin.py
+++ b/admin/tests/test_ops_alerts_admin.py
@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from data.notifications.ops_alerts import OpsAlertSink
+
+
+def test_admin_ops_alert_list_and_page(client, auth_headers, settings, tmp_path: Path):
+    sink = OpsAlertSink(log_path=settings.ops_alert_log_path)
+    sink.emit(
+        alert_type="delivery_watchdog_failed",
+        title="Gaokao delivery watchdog detected failed events",
+        body="station failed once",
+        details={"channel": "station", "failed": 1},
+    )
+
+    api = client.get("/api/admin/notifications/ops-alerts", headers=auth_headers)
+    assert api.status_code == 200, api.text
+    body = api.json()
+    assert body["total"] == 1
+    assert body["items"][0]["alert_type"] == "delivery_watchdog_failed"
+    assert body["items"][0]["details"]["channel"] == "station"
+
+    page = client.get("/admin/ops-alerts", headers=auth_headers)
+    assert page.status_code == 200, page.text
+    assert "运维告警审计" in page.text
+    assert "delivery_watchdog_failed" in page.text
+    assert "station failed once" in page.text
--- a/data/notifications/ops_alerts.py
+++ b/data/notifications/ops_alerts.py
@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from data.notifications.smtp_sender import SMTPDeliverySender, SMTPSettings
+from data.orders.models import utc_now_iso
+
+
+@dataclass
+class OpsAlertResult:
+    log_written: bool
+    emails_sent: int = 0
+
+
+class OpsAlertSink:
+    def __init__(
+        self,
+        *,
+        log_path: str,
+        recipients: list[str] | None = None,
+        email_sender: Any | None = None,
+    ) -> None:
+        self._log_path = Path(log_path)
+        self._recipients = recipients or []
+        self._email_sender = email_sender
+
+    def emit(self, *, alert_type: str, title: str, body: str, details: dict[str, Any]) -> OpsAlertResult:
+        self._log_path.parent.mkdir(parents=True, exist_ok=True)
+        payload = {
+            "created_at": utc_now_iso(),
+            "alert_type": alert_type,
+            "title": title,
+            "body": body,
+            "details": details,
+        }
+        with self._log_path.open("a", encoding="utf-8") as fh:
+            fh.write(json.dumps(payload, ensure_ascii=False) + "\n")
+
+        emails_sent = 0
+        if self._email_sender is not None and self._recipients:
+            for recipient in self._recipients:
+                self._email_sender.send_report_ready(
+                    recipient=recipient,
+                    order_id=str(details.get("order_id") or "ops-alert"),
+                    subject=title,
+                    body=body,
+                )
+                emails_sent += 1
+        return OpsAlertResult(log_written=True, emails_sent=emails_sent)
+
+
+def build_alert_sink_from_settings(settings) -> OpsAlertSink:
+    sender = None
+    if settings.smtp_host and settings.smtp_sender:
+        sender = SMTPDeliverySender(
+            SMTPSettings(
+                host=settings.smtp_host,
+                port=settings.smtp_port,
+                sender=settings.smtp_sender,
+                username=settings.smtp_username,
+                password=settings.smtp_password,
+                use_tls=settings.smtp_use_tls,
+                use_ssl=settings.smtp_use_ssl,
+            )
+        )
+    return OpsAlertSink(
+        log_path=settings.ops_alert_log_path,
+        recipients=settings.alert_recipients,
+        email_sender=sender,
+    )
--- a/docs/CURRENT_STATE.md
+++ b/docs/CURRENT_STATE.md
@@ -112,7 +112,7 @@
 ### P0 / P1 真实缺口

 - 用户端 Web 自助支付闭环缺失
- 生产通知链已有底层 validated/delivered 状态机、portal 通知审计页、后台独立通知审计页与站内提示；仍缺告警推送与生产级监控
+- 生产通知链已有底层 validated/delivered 状态机、portal 通知审计页、后台独立通知审计页、运维告警审计页与站内提示；仍缺真正的告警推送集成与生产级监控
 - 前台入口已支持基础资料填写、附件上传与 5 步向导（基础信息 / 偏好与目标 / 已有方案与附件 / 协议确认 / 提交确认）；仍缺更强的多文件策略与更细粒度字段校验
 - 业务数据备份 / 恢复 / 密钥托管已有本地基线与验证，但异机备份和生产接入仍不足
 - 隐私政策 / 服务协议 / 监护人同意 / 数据保留与删除流程仍缺前台/客服自助工单与正式法务版本
--- a/scripts/gaokao-delivery-watchdog.py
+++ b/scripts/gaokao-delivery-watchdog.py
@@ -13,6 +13,7 @@ if str(ROOT) not in sys.path:
 def main() -> int:
    from admin.config import load_settings
    from data.notifications.dispatcher import DeliveryDispatcher
+    from data.notifications.ops_alerts import build_alert_sink_from_settings

    parser = argparse.ArgumentParser(description="Watch delivery dispatch health")
    parser.add_argument("--channel", default="station")
@@ -29,7 +30,25 @@ def main() -> int:
        dispatcher.close()

    print(json.dumps(result.__dict__, ensure_ascii=False, indent=2))
-    return 2 if result.failed > 0 else 0
+    if result.failed > 0:
+        sink = build_alert_sink_from_settings(settings)
+        sink.emit(
+            alert_type="delivery_watchdog_failed",
+            title="Gaokao delivery watchdog detected failed events",
+            body=(
+                f"channel={args.channel} processed={result.processed} "
+                f"validated={getattr(result, 'validated', 0)} delivered={getattr(result, 'delivered', 0)} failed={result.failed}"
+            ),
+            details={
+                "channel": args.channel,
+                "processed": result.processed,
+                "validated": getattr(result, 'validated', 0),
+                "delivered": getattr(result, 'delivered', 0),
+                "failed": result.failed,
+            },
+        )
+        return 2
+    return 0


 if __name__ == "__main__":
--- a/tests/test_ops_alerts.py
+++ b/tests/test_ops_alerts.py
@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from data.notifications.ops_alerts import OpsAlertSink
+
+
+def test_ops_alert_sink_appends_jsonl(tmp_path: Path):
+    log_path = tmp_path / "alerts.jsonl"
+    sink = OpsAlertSink(log_path=str(log_path), recipients=[])
+    result = sink.emit(
+        alert_type="test_alert",
+        title="Test Alert",
+        body="body",
+        details={"order_id": "O-1", "failed": 2},
+    )
+
+    assert result.log_written is True
+    assert result.emails_sent == 0
+    content = log_path.read_text(encoding="utf-8").strip().splitlines()
+    assert len(content) == 1
+    payload = json.loads(content[0])
+    assert payload["alert_type"] == "test_alert"
+    assert payload["details"]["order_id"] == "O-1"
+
+
+def test_ops_alert_sink_can_use_fake_email_sender(tmp_path: Path):
+    log_path = tmp_path / "alerts.jsonl"
+
+    class FakeSender:
+        def __init__(self) -> None:
+            self.sent: list[dict[str, str]] = []
+
+        def send_report_ready(self, *, recipient: str, order_id: str, subject: str, body: str) -> dict[str, str]:
+            payload = {
+                "recipient": recipient,
+                "order_id": order_id,
+                "subject": subject,
+                "body": body,
+            }
+            self.sent.append(payload)
+            return payload
+
+    sender = FakeSender()
+    sink = OpsAlertSink(log_path=str(log_path), recipients=["ops@example.com"], email_sender=sender)
+    result = sink.emit(
+        alert_type="watchdog_failed",
+        title="watchdog failed",
+        body="something failed",
+        details={"order_id": "O-2"},
+    )
+
+    assert result.emails_sent == 1
+    assert len(sender.sent) == 1
+    assert sender.sent[0]["recipient"] == "ops@example.com"