335 lines
10 KiB
Python
335 lines
10 KiB
Python
"""闲鱼兜底轮询器 (T8.1 §3.2)
|
||
|
||
当 Webhook 不可用（平台维护、网络抖动）时，由 ``poller`` 从外部 API
|
||
拉取未确认订单并写入数据库。Webhook 与 poller 共享 ``upsert_by_external_id``
|
||
幂等机制，因此两端都收到同一条订单不会重复入库。
|
||
|
||
本模块只暴露可注入的 :class:`XianyuOpenAPIClient` 协议与 :func:`poll_once`
|
||
函数；CLI/定时器启动由 ``scripts/`` 下的入口负责（见 T8.4 兜底 SOP）。
|
||
|
||
数据契约:
|
||
- 输入: ``XianyuOpenAPIClient.list_orders(since: int) -> Iterable[dict]``
|
||
- 输出: 每条订单都经过 :func:`data.channel_sync.xianyu_adapter.to_order` →
|
||
:func:`data.channel_sync.dao_extension.upsert_by_external_id`
|
||
- cursor: ``max(updated_at)`` 写入 ``poller_state`` 表（新建）
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import sqlite3
|
||
from dataclasses import dataclass, field
|
||
from typing import Iterable, Optional, Protocol
|
||
|
||
from data.orders.models import utc_now_iso
|
||
|
||
from .dao_extension import UpsertResult, upsert_by_external_id
|
||
from .xianyu_adapter import XianyuEventError, parse_event, to_order
|
||
|
||
|
||
POLLER_STATE_SCHEMA: str = """
|
||
CREATE TABLE IF NOT EXISTS poller_state (
|
||
source TEXT PRIMARY KEY,
|
||
last_cursor TEXT,
|
||
last_run_at TEXT,
|
||
last_error TEXT,
|
||
run_count INTEGER NOT NULL DEFAULT 0,
|
||
error_count INTEGER NOT NULL DEFAULT 0
|
||
);
|
||
"""
|
||
|
||
POLLER_RUN_SCHEMA: str = """
|
||
CREATE TABLE IF NOT EXISTS poller_run (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
source TEXT NOT NULL,
|
||
started_at TEXT NOT NULL,
|
||
finished_at TEXT,
|
||
fetched INTEGER NOT NULL DEFAULT 0,
|
||
inserted INTEGER NOT NULL DEFAULT 0,
|
||
updated INTEGER NOT NULL DEFAULT 0,
|
||
unchanged INTEGER NOT NULL DEFAULT 0,
|
||
rejected INTEGER NOT NULL DEFAULT 0,
|
||
error_message TEXT
|
||
);
|
||
"""
|
||
|
||
|
||
class XianyuOpenAPIClient(Protocol):
|
||
"""外部 API 客户端协议，便于注入假实现做单测。"""
|
||
|
||
def list_orders(self, since: Optional[str]) -> list[dict]: ...
|
||
|
||
|
||
class _DefaultClient:
|
||
"""默认空实现；真实接入留给 T8.2 之后的真接入阶段。"""
|
||
|
||
def list_orders(self, since: Optional[str]) -> list[dict]:
|
||
return []
|
||
|
||
|
||
def apply_poller_schema(conn: sqlite3.Connection) -> None:
|
||
conn.executescript(POLLER_STATE_SCHEMA + POLLER_RUN_SCHEMA)
|
||
conn.commit()
|
||
|
||
|
||
@dataclass
|
||
class PollReport:
|
||
source: str
|
||
fetched: int = 0
|
||
inserted: int = 0
|
||
updated: int = 0
|
||
unchanged: int = 0
|
||
rejected: int = 0
|
||
error: Optional[str] = None
|
||
last_cursor: Optional[str] = None
|
||
upsert_results: list[UpsertResult] = field(default_factory=list)
|
||
|
||
|
||
def _ensure_schema(conn: sqlite3.Connection) -> None:
|
||
apply_poller_schema(conn)
|
||
|
||
|
||
def get_cursor(conn: sqlite3.Connection, source: str) -> Optional[str]:
|
||
row = conn.execute(
|
||
"SELECT last_cursor FROM poller_state WHERE source=?", (source,)
|
||
).fetchone()
|
||
return row[0] if row else None
|
||
|
||
|
||
def _set_cursor(
|
||
conn: sqlite3.Connection,
|
||
source: str,
|
||
cursor: Optional[str],
|
||
error: Optional[str] = None,
|
||
run_count_delta: int = 1,
|
||
error_count_delta: int = 0,
|
||
last_run_at: Optional[str] = None,
|
||
) -> None:
|
||
if last_run_at is None:
|
||
last_run_at = utc_now_iso()
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO poller_state(
|
||
source, last_cursor, last_run_at, last_error,
|
||
run_count, error_count
|
||
) VALUES (?, ?, ?, ?, ?, ?)
|
||
ON CONFLICT(source) DO UPDATE SET
|
||
last_cursor = excluded.last_cursor,
|
||
last_run_at = excluded.last_run_at,
|
||
last_error = excluded.last_error,
|
||
run_count = run_count + excluded.run_count,
|
||
error_count = error_count + excluded.error_count
|
||
""",
|
||
(source, cursor, last_run_at, error, run_count_delta, error_count_delta),
|
||
)
|
||
|
||
|
||
def _record_run(
|
||
conn: sqlite3.Connection,
|
||
*,
|
||
source: str,
|
||
started_at: str,
|
||
finished_at: str,
|
||
fetched: int,
|
||
inserted: int,
|
||
updated: int,
|
||
unchanged: int,
|
||
rejected: int,
|
||
error_message: Optional[str] = None,
|
||
) -> int:
|
||
cur = conn.execute(
|
||
"""
|
||
INSERT INTO poller_run(
|
||
source, started_at, finished_at, fetched, inserted,
|
||
updated, unchanged, rejected, error_message
|
||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
""",
|
||
(
|
||
source,
|
||
started_at,
|
||
finished_at,
|
||
fetched,
|
||
inserted,
|
||
updated,
|
||
unchanged,
|
||
rejected,
|
||
error_message,
|
||
),
|
||
)
|
||
return int(cur.lastrowid or 0)
|
||
|
||
|
||
def _parse_event_dicts(raw_orders: Iterable[dict]) -> list[dict]:
|
||
"""把 list_orders() 返回的 dict 列表规整化：必须含 raw_body (JSON str)。
|
||
|
||
兜底模式：poller 拿到的 order 字典与 Webhook body 同构；这里不重新解析，
|
||
留给 :func:`_process_one` 走完整 parse_event 路径以复用校验。
|
||
"""
|
||
out: list[dict] = []
|
||
for raw in raw_orders:
|
||
if "raw_body" not in raw:
|
||
# 上游直接给了字段时，按 Webhook body 形式重新组装
|
||
import json as _json
|
||
|
||
raw = dict(raw)
|
||
raw.setdefault("raw_body", _json.dumps(raw, ensure_ascii=False))
|
||
out.append(raw)
|
||
return out
|
||
|
||
|
||
def _process_one(
|
||
conn: sqlite3.Connection,
|
||
raw: dict,
|
||
*,
|
||
actor: str = "xianyu_poller",
|
||
) -> UpsertResult:
|
||
"""单条订单: parse_event → to_order → upsert_by_external_id。"""
|
||
event = parse_event(raw["raw_body"])
|
||
order = to_order(event)
|
||
return upsert_by_external_id(
|
||
conn,
|
||
order,
|
||
actor=actor,
|
||
reason=f"poller_event_{event.event_id}",
|
||
)
|
||
|
||
|
||
def poll_once(
|
||
conn: sqlite3.Connection,
|
||
*,
|
||
source: str = "xianyu",
|
||
client: XianyuOpenAPIClient | None = None,
|
||
actor: str = "xianyu_poller",
|
||
) -> PollReport:
|
||
"""执行一次轮询；返回 :class:`PollReport`。
|
||
|
||
- 读取上次 cursor
|
||
- 调 ``client.list_orders(since=cursor)``
|
||
- 对每条订单走 parse + upsert
|
||
- 写 ``poller_run`` / 更新 ``poller_state.cursor``
|
||
|
||
失败不会抛出，返回 ``report.error`` 携带异常信息。
|
||
"""
|
||
_ensure_schema(conn)
|
||
if client is None:
|
||
client = _DefaultClient()
|
||
|
||
started_at = utc_now_iso()
|
||
report = PollReport(source=source)
|
||
|
||
try:
|
||
cursor = get_cursor(conn, source)
|
||
raw_orders = client.list_orders(cursor)
|
||
normalized = _parse_event_dicts(raw_orders)
|
||
report.fetched = len(normalized)
|
||
for raw in normalized:
|
||
try:
|
||
res = _process_one(conn, raw, actor=actor)
|
||
except XianyuEventError as e:
|
||
report.rejected += 1
|
||
report.upsert_results.append(
|
||
UpsertResult(
|
||
order_id="",
|
||
action="rejected",
|
||
error=str(e),
|
||
)
|
||
)
|
||
continue
|
||
except Exception as e: # 解析 / 落库异常 → 单条失败不阻塞整体
|
||
report.rejected += 1
|
||
report.upsert_results.append(
|
||
UpsertResult(
|
||
order_id="",
|
||
action="rejected",
|
||
error=f"unexpected: {e!r}",
|
||
)
|
||
)
|
||
continue
|
||
report.upsert_results.append(res)
|
||
if res.action == "inserted":
|
||
report.inserted += 1
|
||
elif res.action == "updated":
|
||
report.updated += 1
|
||
elif res.action == "unchanged":
|
||
report.unchanged += 1
|
||
elif res.action == "illegal_transition":
|
||
report.rejected += 1
|
||
# cursor 推进：取最大外部 updated_at；空列表时不更新;
|
||
# fetched>0 但缺时间戳时回退到 started_at(详见 _compute_new_cursor)
|
||
report.last_cursor = _compute_new_cursor(
|
||
conn, source, raw_orders, now_iso=started_at
|
||
)
|
||
_set_cursor(
|
||
conn,
|
||
source,
|
||
cursor=report.last_cursor,
|
||
error=None,
|
||
run_count_delta=1,
|
||
error_count_delta=0,
|
||
)
|
||
_record_run(
|
||
conn,
|
||
source=source,
|
||
started_at=started_at,
|
||
finished_at=utc_now_iso(),
|
||
fetched=report.fetched,
|
||
inserted=report.inserted,
|
||
updated=report.updated,
|
||
unchanged=report.unchanged,
|
||
rejected=report.rejected,
|
||
)
|
||
conn.commit()
|
||
except Exception as e:
|
||
report.error = f"{type(e).__name__}: {e}"
|
||
_set_cursor(
|
||
conn,
|
||
source,
|
||
cursor=get_cursor(conn, source),
|
||
error=report.error,
|
||
run_count_delta=1,
|
||
error_count_delta=1,
|
||
)
|
||
_record_run(
|
||
conn,
|
||
source=source,
|
||
started_at=started_at,
|
||
finished_at=utc_now_iso(),
|
||
fetched=report.fetched,
|
||
inserted=report.inserted,
|
||
updated=report.updated,
|
||
unchanged=report.unchanged,
|
||
rejected=report.rejected,
|
||
error_message=report.error,
|
||
)
|
||
conn.commit()
|
||
return report
|
||
|
||
|
||
def _compute_new_cursor(
|
||
conn: sqlite3.Connection,
|
||
source: str,
|
||
raw_orders: list[dict],
|
||
*,
|
||
now_iso: Optional[str] = None,
|
||
) -> Optional[str]:
|
||
"""根据本批订单的最大 updated_at 推进 cursor。
|
||
|
||
异常兜底: 当 fetched>0 但本批订单全部缺 updated_at/paid_at/created_at
|
||
时,不允许 cursor 保持 None 而让 poller 永久卡在同一窗口。
|
||
推进策略: 回退到 ``utc_now_iso()``,用 "本次拉取的时间" 作为新的
|
||
cursor 下界。已知限制: 若上游 API 长期返回无时间戳的订单,可能会
|
||
跳过部分窗口;但这是 fail-forward 策略,优于无限循环。
|
||
"""
|
||
candidates: list[str] = []
|
||
for raw in raw_orders:
|
||
ts = raw.get("updated_at") or raw.get("paid_at") or raw.get("created_at")
|
||
if ts:
|
||
candidates.append(str(ts))
|
||
if candidates:
|
||
# ISO8601 字符串可直接字典序比较
|
||
return max(candidates)
|
||
if raw_orders:
|
||
# fetched>0 但本批无任何时间戳,使用当前时间作为推进值
|
||
return now_iso if now_iso is not None else utc_now_iso()
|
||
# 真正空批:沿用旧 cursor(由调用方决定是否更新)
|
||
return get_cursor(conn, source)
|