369 lines
12 KiB
Python
369 lines
12 KiB
Python
"""渠道兜底巡检 (T8.4).
|
||
|
||
目标：给 ops 一个可定时执行的健康检查入口，基于本地 SQLite 中已经存在的
|
||
webhook_audit / poller_state / poller_run 事实数据，判断渠道同步链路是否进入
|
||
降级态，并给出人工兜底建议。
|
||
|
||
不做：
|
||
- 不直接调用外部渠道 API（避免引入额外依赖 / 凭证）
|
||
- 不自动创建订单（人工兜底仍由 order-manager CLI 明确执行）
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import sqlite3
|
||
from datetime import datetime, timedelta, timezone
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
DEFAULT_DB_PATH = Path("data/orders.db")
|
||
_STATUS_EXIT_CODE = {"ok": 0, "warn": 1, "critical": 2}
|
||
|
||
|
||
def _utcnow() -> datetime:
|
||
return datetime.now(timezone.utc).replace(microsecond=0)
|
||
|
||
|
||
def _parse_iso(value: str | None) -> datetime | None:
|
||
if not value:
|
||
return None
|
||
text = str(value).strip()
|
||
if not text:
|
||
return None
|
||
if text.endswith("Z"):
|
||
text = text[:-1] + "+00:00"
|
||
try:
|
||
dt = datetime.fromisoformat(text)
|
||
except ValueError:
|
||
return None
|
||
if dt.tzinfo is None:
|
||
return dt.replace(tzinfo=timezone.utc)
|
||
return dt.astimezone(timezone.utc)
|
||
|
||
|
||
def _minutes_ago(now: datetime, ts: str | None) -> float | None:
|
||
dt = _parse_iso(ts)
|
||
if dt is None:
|
||
return None
|
||
return round((now - dt).total_seconds() / 60.0, 2)
|
||
|
||
|
||
def _open_db(db_path: str) -> sqlite3.Connection:
|
||
conn = sqlite3.connect(db_path)
|
||
conn.row_factory = sqlite3.Row
|
||
return conn
|
||
|
||
|
||
def _table_exists(conn: sqlite3.Connection, name: str) -> bool:
|
||
row = conn.execute(
|
||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?", (name,)
|
||
).fetchone()
|
||
return row is not None
|
||
|
||
|
||
def _fetch_one(
|
||
conn: sqlite3.Connection, sql: str, params: tuple[Any, ...]
|
||
) -> dict[str, Any] | None:
|
||
row = conn.execute(sql, params).fetchone()
|
||
return dict(row) if row is not None else None
|
||
|
||
|
||
def _manual_actions(db_path: str, source: str) -> list[str]:
|
||
return [
|
||
(
|
||
"先执行巡检命令确认状态："
|
||
f"python3 scripts/gaokao-channel-fallback --db {db_path} check --source {source} --human"
|
||
),
|
||
(
|
||
"若确认 webhook/poller 不可用，使用人工补录 CLI 新建订单："
|
||
f"python3 scripts/gaokao-order-manager --db {db_path} create --source {source} "
|
||
"--service-version basic --amount-cents 0 --customer-name <姓名> --customer-phone <手机号>"
|
||
),
|
||
(
|
||
"人工补录后按业务推进状态："
|
||
f"python3 scripts/gaokao-order-manager --db {db_path} pay <ORDER_ID>"
|
||
" / deliver <ORDER_ID>"
|
||
),
|
||
]
|
||
|
||
|
||
def summarize_channel_health(
|
||
db_path: str,
|
||
*,
|
||
source: str = "xianyu",
|
||
now: datetime | None = None,
|
||
poller_stale_minutes: int = 15,
|
||
webhook_stale_minutes: int = 30,
|
||
recent_window_minutes: int = 60,
|
||
reject_warn_threshold: int = 5,
|
||
poller_error_warn_threshold: int = 3,
|
||
) -> dict[str, Any]:
|
||
now = now or _utcnow()
|
||
conn = _open_db(db_path)
|
||
try:
|
||
has_audit = _table_exists(conn, "webhook_audit")
|
||
has_state = _table_exists(conn, "poller_state")
|
||
has_run = _table_exists(conn, "poller_run")
|
||
|
||
latest_accepted = None
|
||
latest_recent = None
|
||
recent_counts = {
|
||
"accepted": 0,
|
||
"duplicate": 0,
|
||
"parse_error": 0,
|
||
"rejected": 0,
|
||
}
|
||
if has_audit:
|
||
latest_accepted = _fetch_one(
|
||
conn,
|
||
"""
|
||
SELECT received_at, decision, event_id, reject_reason
|
||
FROM webhook_audit
|
||
WHERE channel=? AND decision='accepted'
|
||
ORDER BY id DESC LIMIT 1
|
||
""",
|
||
(source,),
|
||
)
|
||
latest_recent = _fetch_one(
|
||
conn,
|
||
"""
|
||
SELECT received_at, decision, event_id, reject_reason
|
||
FROM webhook_audit
|
||
WHERE channel=?
|
||
ORDER BY id DESC LIMIT 1
|
||
""",
|
||
(source,),
|
||
)
|
||
since_iso = (now - timedelta(minutes=recent_window_minutes)).isoformat()
|
||
for row in conn.execute(
|
||
"""
|
||
SELECT decision, COUNT(*) AS n
|
||
FROM webhook_audit
|
||
WHERE channel=? AND received_at>=?
|
||
GROUP BY decision
|
||
""",
|
||
(source, since_iso),
|
||
).fetchall():
|
||
decision = str(row["decision"])
|
||
if decision in recent_counts:
|
||
recent_counts[decision] = int(row["n"] or 0)
|
||
|
||
state = None
|
||
if has_state:
|
||
state = _fetch_one(
|
||
conn,
|
||
"""
|
||
SELECT source, last_cursor, last_run_at, last_error, run_count, error_count
|
||
FROM poller_state WHERE source=?
|
||
""",
|
||
(source,),
|
||
)
|
||
|
||
last_run = None
|
||
if has_run:
|
||
last_run = _fetch_one(
|
||
conn,
|
||
"""
|
||
SELECT started_at, finished_at, fetched, inserted, updated, unchanged,
|
||
rejected, error_message
|
||
FROM poller_run
|
||
WHERE source=?
|
||
ORDER BY id DESC LIMIT 1
|
||
""",
|
||
(source,),
|
||
)
|
||
|
||
findings: list[str] = []
|
||
actions: list[str] = []
|
||
status = "ok"
|
||
|
||
def raise_status(next_status: str) -> None:
|
||
nonlocal status
|
||
order = {"ok": 0, "warn": 1, "critical": 2}
|
||
if order[next_status] > order[status]:
|
||
status = next_status
|
||
|
||
if not has_audit:
|
||
raise_status("warn")
|
||
findings.append("缺少 webhook_audit 表，无法判断 webhook 接收健康度")
|
||
if not has_state:
|
||
raise_status("warn")
|
||
findings.append("缺少 poller_state 表，无法判断兜底轮询状态")
|
||
if not has_run:
|
||
raise_status("warn")
|
||
findings.append("缺少 poller_run 表，无法回看最近轮询结果")
|
||
|
||
accepted_age = _minutes_ago(
|
||
now, latest_accepted["received_at"] if latest_accepted else None
|
||
)
|
||
recent_age = _minutes_ago(
|
||
now, latest_recent["received_at"] if latest_recent else None
|
||
)
|
||
poller_age = _minutes_ago(now, state["last_run_at"] if state else None)
|
||
|
||
if state and state.get("last_error"):
|
||
error_count = int(state.get("error_count") or 0)
|
||
if error_count >= poller_error_warn_threshold:
|
||
raise_status("critical")
|
||
findings.append(
|
||
f"poller 连续错误计数={error_count}，last_error={state['last_error']}"
|
||
)
|
||
else:
|
||
raise_status("warn")
|
||
findings.append(f"poller 最近一次报错：{state['last_error']}")
|
||
|
||
if last_run and last_run.get("error_message"):
|
||
raise_status("critical")
|
||
findings.append(f"最近一次 poller_run 失败：{last_run['error_message']}")
|
||
|
||
rejected_total = recent_counts["rejected"] + recent_counts["parse_error"]
|
||
if rejected_total >= reject_warn_threshold:
|
||
raise_status("warn")
|
||
findings.append(
|
||
f"最近 {recent_window_minutes} 分钟 webhook 拒绝/解析失败 {rejected_total} 次"
|
||
)
|
||
|
||
if poller_age is not None and poller_age > poller_stale_minutes:
|
||
raise_status("warn")
|
||
findings.append(
|
||
f"poller 最近运行距今 {poller_age} 分钟，超过阈值 {poller_stale_minutes} 分钟"
|
||
)
|
||
|
||
if (
|
||
accepted_age is not None
|
||
and accepted_age > webhook_stale_minutes
|
||
and recent_counts["rejected"] + recent_counts["parse_error"] > 0
|
||
):
|
||
raise_status("warn")
|
||
findings.append(
|
||
f"最近成功 webhook 距今 {accepted_age} 分钟，且窗口内存在失败事件"
|
||
)
|
||
|
||
if latest_accepted is None and state is None and last_run is None:
|
||
raise_status("warn")
|
||
findings.append(
|
||
"尚无 webhook accepted / poller_state / poller_run 证据，链路可能未启动"
|
||
)
|
||
|
||
if status != "ok":
|
||
actions = _manual_actions(db_path, source)
|
||
elif not findings:
|
||
findings.append("最近 webhook/poller 未发现明显异常")
|
||
|
||
payload = {
|
||
"status": status,
|
||
"checked_at": now.isoformat(),
|
||
"source": source,
|
||
"db_path": db_path,
|
||
"tables": {
|
||
"webhook_audit": has_audit,
|
||
"poller_state": has_state,
|
||
"poller_run": has_run,
|
||
},
|
||
"webhook": {
|
||
"latest_accepted": latest_accepted,
|
||
"latest_event": latest_recent,
|
||
"latest_accepted_age_minutes": accepted_age,
|
||
"latest_event_age_minutes": recent_age,
|
||
"recent_counts": recent_counts,
|
||
},
|
||
"poller": {
|
||
"state": state,
|
||
"last_run": last_run,
|
||
"last_run_age_minutes": poller_age,
|
||
},
|
||
"findings": findings,
|
||
"recommended_actions": actions,
|
||
}
|
||
return payload
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _emit(payload: dict[str, Any], *, human: bool) -> None:
|
||
if not human:
|
||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||
return
|
||
print(f"status: {payload['status']}")
|
||
print(f"source: {payload['source']}")
|
||
print(f"checked_at: {payload['checked_at']}")
|
||
print(f"db_path: {payload['db_path']}")
|
||
print("findings:")
|
||
for item in payload.get("findings", []):
|
||
print(f"- {item}")
|
||
actions = payload.get("recommended_actions", [])
|
||
if actions:
|
||
print("recommended_actions:")
|
||
for item in actions:
|
||
print(f"- {item}")
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
parser = argparse.ArgumentParser(
|
||
prog="gaokao-channel-fallback",
|
||
description="渠道兜底巡检 CLI（T8.4）",
|
||
)
|
||
parser.add_argument(
|
||
"--db",
|
||
default=str(DEFAULT_DB_PATH),
|
||
help="订单 SQLite 路径（默认 data/orders.db）",
|
||
)
|
||
parser.add_argument(
|
||
"--human",
|
||
action="store_true",
|
||
help="终端友好输出（默认 JSON）",
|
||
)
|
||
|
||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||
|
||
check = subparsers.add_parser("check", help="检查 webhook/poller 健康度")
|
||
check.add_argument("--source", default="xianyu", help="渠道 source，默认 xianyu")
|
||
check.add_argument("--human", action="store_true", help=argparse.SUPPRESS)
|
||
check.add_argument("--poller-stale-minutes", type=int, default=15)
|
||
check.add_argument("--webhook-stale-minutes", type=int, default=30)
|
||
check.add_argument("--recent-window-minutes", type=int, default=60)
|
||
check.add_argument("--reject-warn-threshold", type=int, default=5)
|
||
check.add_argument("--poller-error-warn-threshold", type=int, default=3)
|
||
|
||
manual = subparsers.add_parser("manual-template", help="输出人工兜底建议命令")
|
||
manual.add_argument("--source", default="xianyu", help="渠道 source，默认 xianyu")
|
||
manual.add_argument("--human", action="store_true", help=argparse.SUPPRESS)
|
||
return parser
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
parser = build_parser()
|
||
args = parser.parse_args(argv)
|
||
|
||
human = bool(getattr(args, "human", False))
|
||
|
||
if args.command == "manual-template":
|
||
payload = {
|
||
"status": "warn",
|
||
"checked_at": _utcnow().isoformat(),
|
||
"source": args.source,
|
||
"db_path": args.db,
|
||
"findings": ["人工兜底模板"],
|
||
"recommended_actions": _manual_actions(args.db, args.source),
|
||
}
|
||
_emit(payload, human=human)
|
||
return 0
|
||
|
||
payload = summarize_channel_health(
|
||
args.db,
|
||
source=args.source,
|
||
poller_stale_minutes=args.poller_stale_minutes,
|
||
webhook_stale_minutes=args.webhook_stale_minutes,
|
||
recent_window_minutes=args.recent_window_minutes,
|
||
reject_warn_threshold=args.reject_warn_threshold,
|
||
poller_error_warn_threshold=args.poller_error_warn_threshold,
|
||
)
|
||
_emit(payload, human=human)
|
||
return _STATUS_EXIT_CODE[payload["status"]]
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|