"""渠道兜底巡检 (T8.4). 目标:给 ops 一个可定时执行的健康检查入口,基于本地 SQLite 中已经存在的 webhook_audit / poller_state / poller_run 事实数据,判断渠道同步链路是否进入 降级态,并给出人工兜底建议。 不做: - 不直接调用外部渠道 API(避免引入额外依赖 / 凭证) - 不自动创建订单(人工兜底仍由 order-manager CLI 明确执行) """ from __future__ import annotations import argparse import json import sqlite3 from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any DEFAULT_DB_PATH = Path("data/orders.db") _STATUS_EXIT_CODE = {"ok": 0, "warn": 1, "critical": 2} def _utcnow() -> datetime: return datetime.now(timezone.utc).replace(microsecond=0) def _parse_iso(value: str | None) -> datetime | None: if not value: return None text = str(value).strip() if not text: return None if text.endswith("Z"): text = text[:-1] + "+00:00" try: dt = datetime.fromisoformat(text) except ValueError: return None if dt.tzinfo is None: return dt.replace(tzinfo=timezone.utc) return dt.astimezone(timezone.utc) def _minutes_ago(now: datetime, ts: str | None) -> float | None: dt = _parse_iso(ts) if dt is None: return None return round((now - dt).total_seconds() / 60.0, 2) def _open_db(db_path: str) -> sqlite3.Connection: conn = sqlite3.connect(db_path) conn.row_factory = sqlite3.Row return conn def _table_exists(conn: sqlite3.Connection, name: str) -> bool: row = conn.execute( "SELECT name FROM sqlite_master WHERE type='table' AND name=?", (name,) ).fetchone() return row is not None def _fetch_one( conn: sqlite3.Connection, sql: str, params: tuple[Any, ...] ) -> dict[str, Any] | None: row = conn.execute(sql, params).fetchone() return dict(row) if row is not None else None def _manual_actions(db_path: str, source: str) -> list[str]: return [ ( "先执行巡检命令确认状态:" f"python3 scripts/gaokao-channel-fallback --db {db_path} check --source {source} --human" ), ( "若确认 webhook/poller 不可用,使用人工补录 CLI 新建订单:" f"python3 scripts/gaokao-order-manager --db {db_path} create --source {source} " "--service-version basic --amount-cents 0 --customer-name <姓名> --customer-phone <手机号>" ), ( "人工补录后按业务推进状态:" f"python3 scripts/gaokao-order-manager --db {db_path} pay " " / deliver " ), ] def summarize_channel_health( db_path: str, *, source: str = "xianyu", now: datetime | None = None, poller_stale_minutes: int = 15, webhook_stale_minutes: int = 30, recent_window_minutes: int = 60, reject_warn_threshold: int = 5, poller_error_warn_threshold: int = 3, ) -> dict[str, Any]: now = now or _utcnow() conn = _open_db(db_path) try: has_audit = _table_exists(conn, "webhook_audit") has_state = _table_exists(conn, "poller_state") has_run = _table_exists(conn, "poller_run") latest_accepted = None latest_recent = None recent_counts = { "accepted": 0, "duplicate": 0, "parse_error": 0, "rejected": 0, } if has_audit: latest_accepted = _fetch_one( conn, """ SELECT received_at, decision, event_id, reject_reason FROM webhook_audit WHERE channel=? AND decision='accepted' ORDER BY id DESC LIMIT 1 """, (source,), ) latest_recent = _fetch_one( conn, """ SELECT received_at, decision, event_id, reject_reason FROM webhook_audit WHERE channel=? ORDER BY id DESC LIMIT 1 """, (source,), ) since_iso = (now - timedelta(minutes=recent_window_minutes)).isoformat() for row in conn.execute( """ SELECT decision, COUNT(*) AS n FROM webhook_audit WHERE channel=? AND received_at>=? GROUP BY decision """, (source, since_iso), ).fetchall(): decision = str(row["decision"]) if decision in recent_counts: recent_counts[decision] = int(row["n"] or 0) state = None if has_state: state = _fetch_one( conn, """ SELECT source, last_cursor, last_run_at, last_error, run_count, error_count FROM poller_state WHERE source=? """, (source,), ) last_run = None if has_run: last_run = _fetch_one( conn, """ SELECT started_at, finished_at, fetched, inserted, updated, unchanged, rejected, error_message FROM poller_run WHERE source=? ORDER BY id DESC LIMIT 1 """, (source,), ) findings: list[str] = [] actions: list[str] = [] status = "ok" def raise_status(next_status: str) -> None: nonlocal status order = {"ok": 0, "warn": 1, "critical": 2} if order[next_status] > order[status]: status = next_status if not has_audit: raise_status("warn") findings.append("缺少 webhook_audit 表,无法判断 webhook 接收健康度") if not has_state: raise_status("warn") findings.append("缺少 poller_state 表,无法判断兜底轮询状态") if not has_run: raise_status("warn") findings.append("缺少 poller_run 表,无法回看最近轮询结果") accepted_age = _minutes_ago( now, latest_accepted["received_at"] if latest_accepted else None ) recent_age = _minutes_ago( now, latest_recent["received_at"] if latest_recent else None ) poller_age = _minutes_ago(now, state["last_run_at"] if state else None) if state and state.get("last_error"): error_count = int(state.get("error_count") or 0) if error_count >= poller_error_warn_threshold: raise_status("critical") findings.append( f"poller 连续错误计数={error_count},last_error={state['last_error']}" ) else: raise_status("warn") findings.append(f"poller 最近一次报错:{state['last_error']}") if last_run and last_run.get("error_message"): raise_status("critical") findings.append(f"最近一次 poller_run 失败:{last_run['error_message']}") rejected_total = recent_counts["rejected"] + recent_counts["parse_error"] if rejected_total >= reject_warn_threshold: raise_status("warn") findings.append( f"最近 {recent_window_minutes} 分钟 webhook 拒绝/解析失败 {rejected_total} 次" ) if poller_age is not None and poller_age > poller_stale_minutes: raise_status("warn") findings.append( f"poller 最近运行距今 {poller_age} 分钟,超过阈值 {poller_stale_minutes} 分钟" ) if ( accepted_age is not None and accepted_age > webhook_stale_minutes and recent_counts["rejected"] + recent_counts["parse_error"] > 0 ): raise_status("warn") findings.append( f"最近成功 webhook 距今 {accepted_age} 分钟,且窗口内存在失败事件" ) if latest_accepted is None and state is None and last_run is None: raise_status("warn") findings.append( "尚无 webhook accepted / poller_state / poller_run 证据,链路可能未启动" ) if status != "ok": actions = _manual_actions(db_path, source) elif not findings: findings.append("最近 webhook/poller 未发现明显异常") payload = { "status": status, "checked_at": now.isoformat(), "source": source, "db_path": db_path, "tables": { "webhook_audit": has_audit, "poller_state": has_state, "poller_run": has_run, }, "webhook": { "latest_accepted": latest_accepted, "latest_event": latest_recent, "latest_accepted_age_minutes": accepted_age, "latest_event_age_minutes": recent_age, "recent_counts": recent_counts, }, "poller": { "state": state, "last_run": last_run, "last_run_age_minutes": poller_age, }, "findings": findings, "recommended_actions": actions, } return payload finally: conn.close() def _emit(payload: dict[str, Any], *, human: bool) -> None: if not human: print(json.dumps(payload, ensure_ascii=False, indent=2)) return print(f"status: {payload['status']}") print(f"source: {payload['source']}") print(f"checked_at: {payload['checked_at']}") print(f"db_path: {payload['db_path']}") print("findings:") for item in payload.get("findings", []): print(f"- {item}") actions = payload.get("recommended_actions", []) if actions: print("recommended_actions:") for item in actions: print(f"- {item}") def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="gaokao-channel-fallback", description="渠道兜底巡检 CLI(T8.4)", ) parser.add_argument( "--db", default=str(DEFAULT_DB_PATH), help="订单 SQLite 路径(默认 data/orders.db)", ) parser.add_argument( "--human", action="store_true", help="终端友好输出(默认 JSON)", ) subparsers = parser.add_subparsers(dest="command", required=True) check = subparsers.add_parser("check", help="检查 webhook/poller 健康度") check.add_argument("--source", default="xianyu", help="渠道 source,默认 xianyu") check.add_argument("--human", action="store_true", help=argparse.SUPPRESS) check.add_argument("--poller-stale-minutes", type=int, default=15) check.add_argument("--webhook-stale-minutes", type=int, default=30) check.add_argument("--recent-window-minutes", type=int, default=60) check.add_argument("--reject-warn-threshold", type=int, default=5) check.add_argument("--poller-error-warn-threshold", type=int, default=3) manual = subparsers.add_parser("manual-template", help="输出人工兜底建议命令") manual.add_argument("--source", default="xianyu", help="渠道 source,默认 xianyu") manual.add_argument("--human", action="store_true", help=argparse.SUPPRESS) return parser def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) human = bool(getattr(args, "human", False)) if args.command == "manual-template": payload = { "status": "warn", "checked_at": _utcnow().isoformat(), "source": args.source, "db_path": args.db, "findings": ["人工兜底模板"], "recommended_actions": _manual_actions(args.db, args.source), } _emit(payload, human=human) return 0 payload = summarize_channel_health( args.db, source=args.source, poller_stale_minutes=args.poller_stale_minutes, webhook_stale_minutes=args.webhook_stale_minutes, recent_window_minutes=args.recent_window_minutes, reject_warn_threshold=args.reject_warn_threshold, poller_error_warn_threshold=args.poller_error_warn_threshold, ) _emit(payload, human=human) return _STATUS_EXIT_CODE[payload["status"]] if __name__ == "__main__": raise SystemExit(main())