feat: harden T12 backup and portal workflows

2026-06-14 23:12:18 +08:00
parent 9bb066f69c
commit 7fe4ddcbc4
33 changed files with 1486 additions and 87 deletions
--- a/README.md
+++ b/README.md
@@ -162,19 +162,33 @@ GAOKAO_SKIP_INSTALL=1 bash scripts/dev-verify.sh
 - 执行器：`data.notifications.dispatcher.DeliveryDispatcher`
 - CLI：`python3 scripts/gaokao-delivery-dispatch.py --channel station`
 - watchdog：`python3 scripts/gaokao-delivery-watchdog.py --channel station`
+- runbook：`docs/DELIVERY_RETENTION_OPS_RUNBOOK.md`
+- systemd/cron 样例：`deploy/systemd/gaokao-delivery-*.{service,timer}` / `deploy/cron/gaokao-jobs.crontab`

 最小语义：

 - 交付物齐全（HTML/PDF 存在）→ `ready -> sent`
 - 交付物缺失 → `failed`，并写入 `failure_reason`
 - watchdog 遇到失败事件返回 exit code `2`
+- 当前 `sent` 仍表示“站内交付物校验通过”，不是外部渠道真实发送成功

 ### 数据保留期清理

 - dry-run：`python3 scripts/gaokao-retention-cleanup.py --cutoff <ISO8601> --dry-run`
 - apply：`python3 scripts/gaokao-retention-cleanup.py --cutoff <ISO8601>`
+- 定时模式：`python3 scripts/gaokao-retention-cleanup.py --retention-days 180`
+- 兼容别名：`python3 scripts/gaokao_retention_cleanup.py --retention-days 180 --dry-run`
+- runbook：`docs/DELIVERY_RETENTION_OPS_RUNBOOK.md`
 - 当前语义：对超过保留期的 `completed/refunded` 订单执行匿名化清理

+### 备份 / 恢复最小入口
+
+- 生成快照：`bash scripts/backup_snapshot.sh /tmp/gaokao-backups`
+- 校验快照：`bash scripts/backup_verify.sh --from-backup /tmp/gaokao-backups/backup-<UTC_TIMESTAMP>`
+- 直接对当前 live 数据做一次恢复演练：`bash scripts/backup_verify.sh`
+- runbook：`docs/BACKUP_AND_RECOVERY_PLAN.md`
+- 定时接入口径：`ops/cron/gaokao-backup.crontab.example`、`ops/systemd/gaokao-backup*.service|timer`
+
 ### crowd_db 质量汇总

 - `python3 -m data.crowd_db.quality_summary --human`
--- a/admin/routes/web_public.py
+++ b/admin/routes/web_public.py
@@ -104,11 +104,26 @@ def create_public_order_endpoint(
    payload: PublicOrderCreate,
    settings: Settings = Depends(get_settings_dep),
 ) -> PublicOrderCreated:
+    try:
+        payment_service = _payment_service(settings)
+    except PaymentError as exc:
+        raise HTTPException(
+            status_code=503,
+            detail=f"payment provider unavailable: {exc}",
+        ) from exc
+
    with OrdersDAO.connect(settings.orders_db_path) as dao:
        order = create_public_order(dao, payload)
    portal_token = issue_portal_token(order.id, settings.jwt_secret)
-    payment_service = _payment_service(settings)
-    checkout = payment_service.create_checkout(order.id, portal_token=portal_token)
+    try:
+        checkout = payment_service.create_checkout(order.id, portal_token=portal_token)
+    except PaymentError as exc:
+        with OrdersDAO.connect(settings.orders_db_path) as dao:
+            dao.delete(order.id)
+        raise HTTPException(
+            status_code=503,
+            detail=f"payment checkout unavailable: {exc}",
+        ) from exc
    return PublicOrderCreated(
        order_id=order.id,
        source=order.source,
@@ -743,11 +758,12 @@ def _render_info_page(
 def _render_status_page(token: str, context: dict[str, Any]) -> str:
    order = context["order"]
    report_links = ""
-    if context["report_html_ready"]:
+    can_access_delivery = context["stage"] in {"report_ready", "completed"}
+    if can_access_delivery and context["report_html_ready"]:
        report_links += (
            f'<li><a href="/portal/{escape(token)}/report">查看在线报告</a></li>'
        )
-    if context["report_pdf_ready"]:
+    if can_access_delivery and context["report_pdf_ready"]:
        report_links += (
            f'<li><a href="/portal/{escape(token)}/report.pdf">下载 PDF</a></li>'
        )
--- a/admin/tests/test_order_status_page.py
+++ b/admin/tests/test_order_status_page.py
@@ -110,3 +110,40 @@ def test_delivered_without_artifacts_stays_processing(client, settings):

    report_page = client.get(f"/portal/{token}/report")
    assert report_page.status_code == 409
+
+
+def test_partial_artifacts_do_not_expose_delivery_links_before_report_ready(
+    client, settings, tmp_path: Path
+):
+    order = _seed_order(settings.orders_db_path, order_id="GKO-20260614-STATUS-PARTIAL")
+    _mark_paid(settings, order)
+    IntakeStore.for_db(settings.orders_db_path).save(
+        order_id=order.id,
+        payload={"candidate_score": 578, "candidate_subjects": ["物理"]},
+        submit=True,
+    )
+
+    report_path = tmp_path / "partial-report.html"
+    report_path.write_text("<h1>partial</h1>", encoding="utf-8")
+    with OrdersDAO.connect(settings.orders_db_path) as dao:
+        dao.update(
+            order.id,
+            {"audit_report": str(report_path)},
+            actor="test",
+            reason="attach_partial_report",
+        )
+        dao.transition_status(order.id, "serving", actor="test", reason="processing")
+        dao.transition_status(
+            order.id, "delivered", actor="test", reason="report_ready_without_pdf"
+        )
+
+    token = issue_portal_token(order.id, settings.jwt_secret)
+    status_page = client.get(f"/portal/{token}/status")
+    assert status_page.status_code == 200, status_page.text
+    assert "处理中" in status_page.text
+    assert "查看在线报告" not in status_page.text
+    assert "下载 PDF" not in status_page.text
+    assert "报告生成中" in status_page.text
+
+    report_page = client.get(f"/portal/{token}/report")
+    assert report_page.status_code == 409
--- a/admin/tests/test_web_public.py
+++ b/admin/tests/test_web_public.py
@@ -2,6 +2,10 @@

 from __future__ import annotations

+from fastapi.testclient import TestClient
+
+from data.orders.dao import OrdersDAO
+

 def test_public_landing_page_served(client):
    resp = client.get("/")
@@ -129,3 +133,106 @@ def test_payment_return_redirects_to_portal_status(client):
    resp = client.get(f"/portal/payment-return?token={token}", follow_redirects=False)
    assert resp.status_code == 303, resp.text
    assert resp.headers["location"] == f"/portal/{token}/status"
+
+
+def test_public_create_order_returns_503_without_creating_orphan_order_when_provider_unavailable(
+    tmp_path, monkeypatch
+):
+    admin_db = tmp_path / "admin.db"
+    orders_db = tmp_path / "orders.db"
+    share_db = tmp_path / "share.db"
+    share_reports = tmp_path / "share_reports"
+    share_reports.mkdir()
+
+    monkeypatch.setenv("GAOKAO_ENV", "dev")
+    monkeypatch.setenv("GAOKAO_DB_PATH", str(admin_db))
+    monkeypatch.setenv("GAOKAO_ORDERS_DB_PATH", str(orders_db))
+    monkeypatch.setenv("GAOKAO_SHARE_DB_PATH", str(share_db))
+    monkeypatch.setenv("GAOKAO_SHARE_REPORT_DIR", str(share_reports))
+    monkeypatch.setenv("GAOKAO_ORDERS_FERNET_KEY", "test-secret-for-web-self-service")
+    monkeypatch.setenv("GAOKAO_JWT_SECRET", "x" * 64)
+    monkeypatch.setenv("GAOKAO_JWT_EXP_MIN", "5")
+    monkeypatch.setenv("GAOKAO_ADMIN_USER", "admin")
+    monkeypatch.setenv("GAOKAO_ADMIN_PASS", "test-pass-123")
+    monkeypatch.setenv("GAOKAO_PAYMENT_PROVIDER", "alipay")
+    monkeypatch.setenv("GAOKAO_PAYMENT_BASE_URL", "http://testserver")
+    monkeypatch.setenv("GAOKAO_PAYMENT_WEBHOOK_SECRET", "missing-real-provider-env")
+
+    from admin.app import create_app
+    from admin.config import load_settings
+
+    settings = load_settings()
+    app = create_app(settings)
+
+    with TestClient(app) as client:
+        resp = client.post(
+            "/api/public/orders",
+            json={
+                "service_version": "standard",
+                "amount_cents": 9900,
+                "customer_name": "张家长",
+                "customer_phone": "13800138000",
+                "candidate_province": "湖南",
+            },
+        )
+
+    assert resp.status_code == 503, resp.text
+    assert "payment provider unavailable" in resp.text
+    with OrdersDAO.connect(settings.orders_db_path) as dao:
+        assert dao.count() == 0
+
+
+def test_public_create_order_returns_503_without_creating_orphan_order_when_checkout_fails(
+    tmp_path, monkeypatch
+):
+    admin_db = tmp_path / "admin.db"
+    orders_db = tmp_path / "orders.db"
+    share_db = tmp_path / "share.db"
+    share_reports = tmp_path / "share_reports"
+    share_reports.mkdir()
+
+    monkeypatch.setenv("GAOKAO_ENV", "dev")
+    monkeypatch.setenv("GAOKAO_DB_PATH", str(admin_db))
+    monkeypatch.setenv("GAOKAO_ORDERS_DB_PATH", str(orders_db))
+    monkeypatch.setenv("GAOKAO_SHARE_DB_PATH", str(share_db))
+    monkeypatch.setenv("GAOKAO_SHARE_REPORT_DIR", str(share_reports))
+    monkeypatch.setenv("GAOKAO_ORDERS_FERNET_KEY", "test-secret-for-web-self-service")
+    monkeypatch.setenv("GAOKAO_JWT_SECRET", "x" * 64)
+    monkeypatch.setenv("GAOKAO_JWT_EXP_MIN", "5")
+    monkeypatch.setenv("GAOKAO_ADMIN_USER", "admin")
+    monkeypatch.setenv("GAOKAO_ADMIN_PASS", "test-pass-123")
+    monkeypatch.setenv("GAOKAO_PAYMENT_PROVIDER", "mock")
+    monkeypatch.setenv("GAOKAO_PAYMENT_BASE_URL", "http://testserver")
+    monkeypatch.setenv("GAOKAO_PAYMENT_WEBHOOK_SECRET", "test-payment-secret")
+
+    from admin.app import create_app
+    from admin.config import load_settings
+    from data.payments.service import PaymentError, PaymentService
+
+    original = PaymentService.create_checkout
+
+    def _boom(self, order_id: str, *, portal_token: str):
+        raise PaymentError("checkout transport failed")
+
+    monkeypatch.setattr(PaymentService, "create_checkout", _boom)
+
+    settings = load_settings()
+    app = create_app(settings)
+
+    with TestClient(app) as client:
+        resp = client.post(
+            "/api/public/orders",
+            json={
+                "service_version": "standard",
+                "amount_cents": 9900,
+                "customer_name": "张家长",
+                "customer_phone": "13800138000",
+                "candidate_province": "湖南",
+            },
+        )
+
+    monkeypatch.setattr(PaymentService, "create_checkout", original)
+    assert resp.status_code == 503, resp.text
+    assert "payment checkout unavailable" in resp.text
+    with OrdersDAO.connect(settings.orders_db_path) as dao:
+        assert dao.count() == 0
--- a/data/orders/retention_cleanup.py
+++ b/data/orders/retention_cleanup.py
@@ -3,7 +3,7 @@ from __future__ import annotations
 import argparse
 import json
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timedelta, timezone

 from data.orders.dao import OrdersDAO
 from data.orders.deletion_service import OrderDeletionService
@@ -11,6 +11,8 @@ from data.orders.deletion_service import OrderDeletionService

@dataclass
 class RetentionCleanupResult:
+    cutoff_iso: str
+    dry_run: bool
    scanned: int = 0
    candidates: int = 0
    anonymized: int = 0
@@ -53,11 +55,28 @@ def _iter_candidates(db_path: str, cutoff_iso: str) -> tuple[int, list[str]]:
    return scanned, candidates


+def resolve_cutoff_iso(
+    *, cutoff_iso: str | None = None, retention_days: int | None = None
+) -> str:
+    if cutoff_iso is not None:
+        datetime.fromisoformat(cutoff_iso)
+        return cutoff_iso
+    if retention_days is None or retention_days <= 0:
+        raise ValueError("retention_days must be > 0 when cutoff is not provided")
+    cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
+    return cutoff.replace(microsecond=0).isoformat()
+
+
 def run_cleanup(
    db_path: str, *, cutoff_iso: str, apply: bool
 ) -> RetentionCleanupResult:
    scanned, candidate_ids = _iter_candidates(db_path, cutoff_iso)
-    result = RetentionCleanupResult(scanned=scanned, candidates=len(candidate_ids))
+    result = RetentionCleanupResult(
+        cutoff_iso=cutoff_iso,
+        dry_run=not apply,
+        scanned=scanned,
+        candidates=len(candidate_ids),
+    )
    if not apply:
        return result
    service = OrderDeletionService.for_db(db_path)
@@ -76,11 +95,16 @@ def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        description="Run retention cleanup for old completed/refunded orders"
    )
-    parser.add_argument(
+    cutoff_group = parser.add_mutually_exclusive_group(required=True)
+    cutoff_group.add_argument(
        "--cutoff",
-        required=True,
        help="ISO8601 cutoff; orders on or before this anchor are candidates",
    )
+    cutoff_group.add_argument(
+        "--retention-days",
+        type=int,
+        help="Retention window in days; cutoff is computed as now - retention_days",
+    )
    parser.add_argument("--dry-run", action="store_true")
    return parser

@@ -92,7 +116,14 @@ def main(argv: list[str] | None = None, *, db_path: str | None = None) -> int:
        db_path = load_settings().orders_db_path
    parser = build_parser()
    args = parser.parse_args(argv)
-    result = run_cleanup(db_path, cutoff_iso=args.cutoff, apply=not args.dry_run)
+    try:
+        cutoff_iso = resolve_cutoff_iso(
+            cutoff_iso=args.cutoff,
+            retention_days=args.retention_days,
+        )
+    except ValueError as exc:
+        parser.error(str(exc))
+    result = run_cleanup(db_path, cutoff_iso=cutoff_iso, apply=not args.dry_run)
    print(json.dumps(result.__dict__, ensure_ascii=False, indent=2))
    return 0

--- a/deploy/cron/gaokao-jobs.crontab
+++ b/deploy/cron/gaokao-jobs.crontab
@@ -0,0 +1,13 @@
+SHELL=/bin/bash
+PATH=/usr/local/bin:/usr/bin:/bin
+
+GAOKAO_ENV=prod
+GAOKAO_ORDERS_DB_PATH=/home/long/project/gaokao-volunteer-system/data/orders.db
+GAOKAO_PYTHON_BIN=/home/long/project/gaokao-volunteer-system/.venv/bin/python
+GAOKAO_DELIVERY_CHANNEL=station
+GAOKAO_DELIVERY_LIMIT=100
+GAOKAO_RETENTION_DAYS=180
+
+*/5 * * * * cd /home/long/project/gaokao-volunteer-system && "$GAOKAO_PYTHON_BIN" scripts/gaokao-delivery-dispatch.py --channel "$GAOKAO_DELIVERY_CHANNEL" --limit "$GAOKAO_DELIVERY_LIMIT" >> /var/log/gaokao-volunteer-system/delivery-dispatch.log 2>&1
+*/10 * * * * cd /home/long/project/gaokao-volunteer-system && "$GAOKAO_PYTHON_BIN" scripts/gaokao-delivery-watchdog.py --channel "$GAOKAO_DELIVERY_CHANNEL" --limit "$GAOKAO_DELIVERY_LIMIT" >> /var/log/gaokao-volunteer-system/delivery-watchdog.log 2>&1
+30 3 * * 0 cd /home/long/project/gaokao-volunteer-system && "$GAOKAO_PYTHON_BIN" scripts/gaokao-retention-cleanup.py --retention-days "$GAOKAO_RETENTION_DAYS" >> /var/log/gaokao-volunteer-system/retention-cleanup.log 2>&1
--- a/deploy/systemd/gaokao-delivery-dispatch.service
+++ b/deploy/systemd/gaokao-delivery-dispatch.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Gaokao delivery dispatcher (report_ready -> sent/failed)
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/long/project/gaokao-volunteer-system
+Environment=GAOKAO_PYTHON_BIN=/usr/bin/python3
+Environment=GAOKAO_DELIVERY_CHANNEL=station
+Environment=GAOKAO_DELIVERY_LIMIT=100
+EnvironmentFile=-/home/long/project/gaokao-volunteer-system/deploy/systemd/gaokao-jobs.env
+ExecStart=/bin/sh -lc 'exec "$GAOKAO_PYTHON_BIN" scripts/gaokao-delivery-dispatch.py --channel "$GAOKAO_DELIVERY_CHANNEL" --limit "$GAOKAO_DELIVERY_LIMIT"'
--- a/deploy/systemd/gaokao-delivery-dispatch.timer
+++ b/deploy/systemd/gaokao-delivery-dispatch.timer
@@ -0,0 +1,11 @@
+[Unit]
+Description=Run gaokao delivery dispatcher every 5 minutes
+
+[Timer]
+OnCalendar=*:0/5
+Persistent=true
+RandomizedDelaySec=30
+Unit=gaokao-delivery-dispatch.service
+
+[Install]
+WantedBy=timers.target
--- a/deploy/systemd/gaokao-delivery-watchdog.service
+++ b/deploy/systemd/gaokao-delivery-watchdog.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Gaokao delivery watchdog (non-zero when dispatch sees failures)
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/long/project/gaokao-volunteer-system
+Environment=GAOKAO_PYTHON_BIN=/usr/bin/python3
+Environment=GAOKAO_DELIVERY_CHANNEL=station
+Environment=GAOKAO_DELIVERY_LIMIT=100
+EnvironmentFile=-/home/long/project/gaokao-volunteer-system/deploy/systemd/gaokao-jobs.env
+ExecStart=/bin/sh -lc 'exec "$GAOKAO_PYTHON_BIN" scripts/gaokao-delivery-watchdog.py --channel "$GAOKAO_DELIVERY_CHANNEL" --limit "$GAOKAO_DELIVERY_LIMIT"'
--- a/deploy/systemd/gaokao-delivery-watchdog.timer
+++ b/deploy/systemd/gaokao-delivery-watchdog.timer
@@ -0,0 +1,11 @@
+[Unit]
+Description=Run gaokao delivery watchdog every 10 minutes
+
+[Timer]
+OnCalendar=*:0/10
+Persistent=true
+RandomizedDelaySec=60
+Unit=gaokao-delivery-watchdog.service
+
+[Install]
+WantedBy=timers.target
--- a/deploy/systemd/gaokao-jobs.env.example
+++ b/deploy/systemd/gaokao-jobs.env.example
@@ -0,0 +1,10 @@
+# 复制为 deploy/systemd/gaokao-jobs.env 后再按实际环境覆盖。
+# 目标：给 dispatcher / watchdog / retention cleanup 三个定时作业提供统一环境。
+
+GAOKAO_ENV=prod
+GAOKAO_ORDERS_DB_PATH=/home/long/project/gaokao-volunteer-system/data/orders.db
+GAOKAO_PYTHON_BIN=/home/long/project/gaokao-volunteer-system/.venv/bin/python
+GAOKAO_DELIVERY_CHANNEL=station
+GAOKAO_DELIVERY_LIMIT=100
+GAOKAO_RETENTION_DAYS=180
+PYTHONUNBUFFERED=1
--- a/deploy/systemd/gaokao-retention-cleanup.service
+++ b/deploy/systemd/gaokao-retention-cleanup.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Gaokao retention cleanup (anonymize expired completed/refunded orders)
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/long/project/gaokao-volunteer-system
+Environment=GAOKAO_PYTHON_BIN=/usr/bin/python3
+Environment=GAOKAO_RETENTION_DAYS=180
+EnvironmentFile=-/home/long/project/gaokao-volunteer-system/deploy/systemd/gaokao-jobs.env
+ExecStart=/bin/sh -lc 'exec "$GAOKAO_PYTHON_BIN" scripts/gaokao-retention-cleanup.py --retention-days "$GAOKAO_RETENTION_DAYS"'
--- a/deploy/systemd/gaokao-retention-cleanup.timer
+++ b/deploy/systemd/gaokao-retention-cleanup.timer
@@ -0,0 +1,11 @@
+[Unit]
+Description=Run gaokao retention cleanup every Sunday at 03:30 UTC
+
+[Timer]
+OnCalendar=Sun 03:30
+Persistent=true
+RandomizedDelaySec=300
+Unit=gaokao-retention-cleanup.service
+
+[Install]
+WantedBy=timers.target
--- a/docs/ACTIVE_REMEDIATION_2026-06-13.md
+++ b/docs/ACTIVE_REMEDIATION_2026-06-13.md
@@ -34,26 +34,34 @@
 #### A-1 用户端 Web 自助闭环未完成

 严重度: P0（面向 Web 自助产品时）
-当前状态: 未解决
+当前状态: 部分解决，仍未整体收口
 范围归属: T12（后续阶段）

+已完成：
+
+- 前台公开入口 `/` / `/pricing` / `/checkout/{service_version}`
+- 公开下单与 portal token 链路
+- 支付后资料填写草稿/提交
+- 站内状态页 / 报告查看 / PDF 下载最小闭环
+- 后台订单列表/详情可见资料提交状态
+
 仍缺：

- 前台套餐页完整交互
- 真实支付接入与回调验签
- 前台资料填写闭环
- 站内交付 / 订单状态页
- 自助购买后自动处理主链路
+- 前台套餐页更完整交互与产品化打磨
+- 真实支付接入与回调验签的线上 acceptance
+- `info_submitted -> serving` 自动处理主链
+- 完整站内通知/交付产品化
+- 自助购买后自动处理主链路的生产化闭环

 说明：

- 这是“当前仍然有效的问题”，但**不应再作为对 v2.1 已完成范围的否定证据**。
- 正确表述应为：`v2.1 完成了人工服务运营闭环，T12 承接 Web 自助闭环`。
+- 当前不是“完全没做 T12”，而是“本地 MVP 主链大部分已落地，但整体仍未到线上可验收完成”。
+- 正确表述应为：`v2.1 完成了人工服务运营闭环，T12 已形成本地 MVP 主链，但线上商业闭环仍未完成`。

 建议动作：

 - 以 T12 为唯一主线推进，不在旧报告中分散描述
- 为 T12 单独维护实施计划、验收标准和完成报告
+- 后续优先收 `自动处理主链` 与 `真实支付 acceptance`

 #### A-2 支付、退款、对账闭环未完成

@@ -100,11 +108,12 @@
 - 站内查看 + PDF 下载最小交付闭环
 - `DeliveryDispatcher` + `scripts/gaokao-delivery-dispatch.py`
 - `ready -> sent` / 缺文件 `-> failed` 的最小执行链
+- dispatcher/watchdog runbook + systemd/cron 样例（`docs/DELIVERY_RETENTION_OPS_RUNBOOK.md` / `deploy/systemd` / `deploy/cron`）

 仍缺：

 - 真实邮件/站内通知发送执行器
- 自动重试调度 / 告警链
+- 自动重试调度 / 告警链（watchdog/systemd/cron 样例已补，生产安装未完成）
 - 面向用户的独立通知审计页
 - 对账/退款与交付失败补偿联动

@@ -137,12 +146,12 @@

 - 正式法务审定版本
 - 前台/客服删除工单流程
- 数据保留期自动清理定时任务（脚本已具备，调度未接入）
+- 目标生产主机上的自动清理实际安装/留痕（仓库已提供 runbook + systemd/cron 样例）
 - 合规文本上线前最终校对

 说明：

- 当前不再是“完全没有合规基线”，而是“文档基线 + 后台最小执行入口 + retention cleanup 脚本已建，前台流程与自动调度仍待完成”。
+- 当前不再是“完全没有合规基线”，而是“文档基线 + 后台最小执行入口 + retention cleanup 脚本与调度样例已建，前台流程与生产安装留痕仍待完成”。

 #### A-5 业务数据备份/恢复/密钥托管已形成基线，但生产化仍未完结

@@ -154,17 +163,22 @@

 - `docs/BACKUP_AND_RECOVERY_PLAN.md`
 - `docs/KEY_MANAGEMENT_BASELINE.md`
- `scripts/backup_verify.sh` 本地恢复验证入口
+- `docs/plans/P1-8-backup-restore-drill.md`
+- `scripts/backup_snapshot.sh` 最小目录快照入口
+- `scripts/backup_verify.sh` 本地恢复校验入口
+- `scripts/backup_restore_smoke.py` 恢复副本最小服务 smoke
+- `ops/cron/gaokao-backup.crontab.example`
+- `ops/systemd/gaokao-backup*.service|timer` 定时接入口径示例

 仍缺：

 - 异机/异地备份落地
- 定时备份与恢复演练自动化
+- 目标主机上的 cron/systemd 实际安装、日志与告警验收
 - 密钥轮换的真实执行记录

 说明：

- 当前不再是“没有备份恢复方案”，而是“已有本地方案与验证脚本，但生产化运维链未闭环”。
+- 当前不再是“只有文档没有验证”，而是“已有本地快照、完整性校验、restore smoke 和定时接入口径样例，但生产化运维链未闭环”。

 ---

--- a/docs/BACKUP_AND_RECOVERY_PLAN.md
+++ b/docs/BACKUP_AND_RECOVERY_PLAN.md
@@ -1,7 +1,7 @@
 # BACKUP_AND_RECOVERY_PLAN

 最后更新: 2026-06-14
-状态: 最小恢复基线（非生产级高可用）
+状态: 最小恢复基线（已具备本地快照 + 完整性校验 + restore smoke）

 ## 1. 目标

@@ -11,77 +11,150 @@
 - 报告 HTML/PDF 文件
 - 上传/分享产物
 - 运行所需关键配置与密钥映射关系
+- 本地可执行的备份、校验、恢复演练入口

 ## 2. RPO / RTO 建议

 - RPO: 24 小时内
 - RTO: 4 小时内

-说明：当前是轻量单机 MVP，不承诺秒级恢复。
+说明：当前是轻量单机 MVP，不承诺秒级恢复，也不等同于生产级多机容灾。

 ## 3. 需要备份的对象

 1. 数据库
-   - 管理库
-   - 订单库
-   - 分享库
+   - 管理库 `data/orders/admin.db`
+   - 订单库 `data/orders.db`
+   - 分享库 `data/share/short_links.db`
 2. 文件目录
-   - 报告输出目录
-   - 分享报告目录
-   - 未来上传文件目录
+   - 分享/公开报告目录
+   - 报告 HTML/PDF 产物
+   - `data/examples` 中的样例交付物（用于恢复 smoke）
 3. 配置与运维资料
   - `.env` 实际部署值（不入 Git）
-   - systemd/docker-compose 部署配置
+   - systemd / docker-compose 部署配置
 4. 密钥资料
   - JWT secret
   - Fernet key
   - 支付相关密钥/证书

-## 4. 最小备份策略
+## 4. 当前已落地产物
+
+- `scripts/backup_snapshot.sh`
+  - 生成时间戳快照目录
+  - 复制 DB / 文件 / 可选 config / 可选 secrets
+  - 生成 `manifest.json`
+  - 按 `GAOKAO_BACKUP_KEEP` 保留最近 N 份
+- `scripts/backup_verify.sh`
+  - 可校验 live staging 或已有快照
+  - 校验 SQLite 可读性
+  - 校验 `manifest.json` 哈希完整性
+  - 调用 restore smoke
+- `scripts/backup_restore_smoke.py`
+  - 在恢复副本上启动 FastAPI `TestClient`
+  - 走 `/health`、portal 状态页、报告 HTML、PDF 下载最小链路
+  - 只改临时恢复副本，不改原始快照
+- 定时接入口径示例
+  - `ops/cron/gaokao-backup.crontab.example`
+  - `ops/systemd/gaokao-backup.service`
+  - `ops/systemd/gaokao-backup.timer`
+  - `ops/systemd/gaokao-backup-verify.service`
+  - `ops/systemd/gaokao-backup-verify.timer`
+
+## 5. 最小备份策略

 ### 数据库

- 每日一次离线或冷备
- 备份文件命名包含时间戳
- 保留最近 7 份日备
+- 每日一次目录快照
+- 文件名包含 UTC 时间戳
+- 保留最近 7 份日备（可由 `GAOKAO_BACKUP_KEEP` 调整）

 ### 文件目录

- 每日一次目录归档
- 保留最近 7 份日备
+- 与数据库同批次快照
+- 报告/分享目录和样例报告分开保存，避免后续排障时混淆

-### 密钥与配置
+### 配置与密钥

- 单独存放，不与业务数据库混包
+- `.env`、部署配置、密钥目录不与业务数据库混写到同一路径
+- 通过环境变量显式声明：
+  - `GAOKAO_BACKUP_ENV_FILE`
+  - `GAOKAO_BACKUP_CONFIG_DIR`
+  - `GAOKAO_BACKUP_SECRETS_DIR`
 - 只允许受控人员访问

-## 5. 恢复步骤（最小版）
+## 6. 最小 runbook（本地可执行）

-1. 准备干净工作目录
-2. 恢复 SQLite 数据库文件
-3. 恢复报告/分享文件目录
-4. 恢复环境变量与密钥
-5. 启动服务
-6. 验证：
-   - 后台能读订单
-   - portal 状态页可打开
-   - 已交付报告可下载
+### 6.1 生成快照

-## 6. 当前缺口
+```bash
+bash scripts/backup_snapshot.sh /tmp/gaokao-backups
+```

- 尚无自动化备份脚本
- 尚无定时任务
- 尚无备份完整性校验
+可选环境变量：

-## 7. MVP 上线前最低要求
+```bash
+export GAOKAO_BACKUP_KEEP=7
+export GAOKAO_BACKUP_ENV_FILE=/etc/gaokao/.env
+export GAOKAO_BACKUP_CONFIG_DIR=/etc/gaokao
+export GAOKAO_BACKUP_SECRETS_DIR=/var/lib/gaokao/secrets
+```

-1. 至少完成一次手工备份演练
-2. 至少在临时目录恢复一次可读数据库
+### 6.2 校验最近一份快照
+
+```bash
+bash scripts/backup_verify.sh --from-backup /tmp/gaokao-backups/backup-<UTC_TIMESTAMP>
+```
+
+校验内容：
+
+1. `manifest.json` 哈希一致
+2. SQLite 文件可打开、表可枚举
+3. 恢复副本可跑最小服务 smoke：
+   - `/health`
+   - `/portal/{token}/status`
+   - `/portal/{token}/report`
+   - `/portal/{token}/report.pdf`
+
+### 6.3 从 live 数据直接做一次恢复演练
+
+```bash
+bash scripts/backup_verify.sh
+```
+
+该命令会把当前 live DB / 文件复制到临时目录后执行同样的校验链。
+
+## 7. 定时接入口径
+
+### cron 示例
+
+见：`ops/cron/gaokao-backup.crontab.example`
+
+- 每天 02:30 生成快照
+- 每周一 03:00 对最近一份快照执行 restore smoke
+
+### systemd timer 示例
+
+见：
+
+- `ops/systemd/gaokao-backup.service`
+- `ops/systemd/gaokao-backup.timer`
+- `ops/systemd/gaokao-backup-verify.service`
+- `ops/systemd/gaokao-backup-verify.timer`
+
+说明：这些文件只是仓库内口径样例，不代表目标主机已经安装启用。
+
+## 8. MVP 上线前最低要求
+
+1. 至少完成一次真实快照生成
+2. 至少完成一次 `backup_verify.sh --from-backup ...` 演练
 3. 明确密钥存放位置和负责人
 4. 不得再使用“Git 备份即可恢复系统”作为对外或内部结论

-## 8. 当前已落地产物
+## 9. 当前仍未闭环的外部/后续缺口

- `scripts/backup_verify.sh`
-  - 可复制当前 SQLite 数据库与示例报告到临时目录
-  - 可验证复制后的数据库可读、表可枚举、报告文件可见
+- 异机 / 异地备份尚未落地
+- 目标主机上的 cron / systemd timer 尚未实际安装验收
+- 备份失败告警链（邮件 / IM / 监控）未接入
+- 密钥轮换与应急恢复仍缺真实执行记录
+- 当前 restore smoke 证明的是“最小服务链可用”，不是完整生产级全链恢复
--- a/docs/CURRENT_STATE.md
+++ b/docs/CURRENT_STATE.md
@@ -68,14 +68,14 @@

 未完成主链路包括：

- 用户端落地页 / 套餐页完整交互
- 真实支付接入与回调验签
- 前台资料填写闭环
- 站内交付 / 前台订单状态页完整闭环
+- 真实支付接入与回调验签的线上 acceptance
+- `info_submitted -> serving` 自动处理主链
+- 完整交付通知/发送器与调度
+- 前台删除工单流程与更完整的产品化交互

 因此：

- Web 自助购买场景 = 未完成
+- Web 自助购买场景 = **本地 MVP 主链已大部分落地，但线上闭环未完成**
 - 完整教育 SaaS 化 = 未完成

 ---
@@ -107,7 +107,7 @@
 - 用户端 Web 自助支付闭环缺失
 - 邮件 / 站内自动交付尚未形成当前主系统标准能力
 - 上传入口仍以后台 / CLI / 内部入口为主，用户前台入口不足
- 业务数据备份 / 恢复 / 密钥托管方案仍不足
+- 业务数据备份 / 恢复 / 密钥托管已有本地基线与验证，但异机备份和生产接入仍不足
 - 隐私政策 / 服务协议 / 监护人同意 / 数据保留与删除流程未形成正式产品闭环

 ### P2 工程改进
--- a/docs/DATA_RETENTION_AND_DELETION.md
+++ b/docs/DATA_RETENTION_AND_DELETION.md
@@ -44,12 +44,16 @@
 - 删除时自动清理 HTML/PDF 报告文件
 - `order_deletion_audits` 最小审计表
 - `scripts/gaokao_retention_cleanup.py` / `scripts/gaokao-retention-cleanup.py` 保留期清理入口
+- `scripts/gaokao-retention-cleanup.py --retention-days 180` 最小定时调度入口
+- `docs/DELIVERY_RETENTION_OPS_RUNBOOK.md`
+- `deploy/systemd/gaokao-retention-cleanup.{service,timer}`
+- `deploy/cron/gaokao-jobs.crontab`

 尚缺：

 - 前台/客服删除工单流程
- 数据保留期自动清理定时任务
 - 更细粒度的匿名化策略（如案例长期保留脱敏版）
+- 目标生产主机上的实际安装/值班留痕（仓库内只有调度样例，不代表已部署）

 ## 5. MVP 最低执行要求

@@ -59,3 +63,14 @@
 2. 内部 runbook 明确谁来删、删哪些表和文件
 3. 删除后要复核文件路径已不存在
 4. 不能把“仅删数据库”误报成“数据已全部删除”
+
+## 6. 最小调度与 runbook
+
+仓库当前给出的最小执行口径：
+
+- 人工指定 cutoff：`python3 scripts/gaokao-retention-cleanup.py --cutoff <ISO8601> --dry-run`
+- 定时调度：`python3 scripts/gaokao-retention-cleanup.py --retention-days 180`
+- runbook：`docs/DELIVERY_RETENTION_OPS_RUNBOOK.md`
+- systemd/cron 样例：`deploy/systemd/gaokao-retention-cleanup.{service,timer}` / `deploy/cron/gaokao-jobs.crontab`
+
+真相边界：当前保留期作业执行的是后台匿名化，不应误报成“用户删除请求流程已经完整上线”。
--- a/docs/DELIVERY_RETENTION_OPS_RUNBOOK.md
+++ b/docs/DELIVERY_RETENTION_OPS_RUNBOOK.md
@@ -0,0 +1,130 @@
+# DELIVERY_RETENTION_OPS_RUNBOOK
+
+最后更新: 2026-06-14
+
+## 1. 当前真相
+
+### delivery dispatcher / watchdog
+
+当前已落仓并可接定时任务的事实：
+
+- 执行脚本：`scripts/gaokao-delivery-dispatch.py`
+- 巡检脚本：`scripts/gaokao-delivery-watchdog.py`
+- systemd 样例：`deploy/systemd/gaokao-delivery-dispatch.{service,timer}` / `deploy/systemd/gaokao-delivery-watchdog.{service,timer}`
+- cron 样例：`deploy/cron/gaokao-jobs.crontab`
+
+注意边界：
+
+- 当前 `sent` 仍表示“站内交付物校验通过并推进事件状态”，**不是** 邮件/微信等外部渠道真实发送成功。
+- dispatcher 每次会处理 `ready` 与 `failed` 事件；如果缺少 HTML/PDF，会把事件记为 `failed`，并递增 `attempt_count`。
+- watchdog 复用同一 dispatch 路径，但只要本轮出现失败事件就返回 exit code `2`，适合接 systemd `OnFailure=` 或外部告警。
+
+### retention cleanup
+
+当前已落仓并可接定时任务的事实：
+
+- 手工入口：`scripts/gaokao-retention-cleanup.py --cutoff <ISO8601> [--dry-run]`
+- 定时入口：`scripts/gaokao-retention-cleanup.py --retention-days 180 [--dry-run]`
+- 兼容别名：`scripts/gaokao_retention_cleanup.py`
+- systemd 样例：`deploy/systemd/gaokao-retention-cleanup.{service,timer}`
+- cron 样例：`deploy/cron/gaokao-jobs.crontab`
+
+注意边界：
+
+- 当前清理动作是 **匿名化**（`anonymize_order`），不是物理删除。
+- 只处理 `completed/refunded` 且锚点时间早于 cutoff 的订单。
+- 它会清理订单侧敏感字段并写审计，但不等于“前台删除工单流程已上线”。
+
+## 2. 统一环境配置
+
+建议先复制：
+
+```bash
+cp deploy/systemd/gaokao-jobs.env.example deploy/systemd/gaokao-jobs.env
+```
+
+最少确认以下变量：
+
+- `GAOKAO_ORDERS_DB_PATH`：生产订单 DB 路径
+- `GAOKAO_PYTHON_BIN`：建议指向项目 `.venv/bin/python`
+- `GAOKAO_DELIVERY_CHANNEL`：当前默认 `station`
+- `GAOKAO_DELIVERY_LIMIT`：单次 dispatch/watchdog 扫描上限
+- `GAOKAO_RETENTION_DAYS`：默认 `180`
+
+## 3. 手工执行口径
+
+```bash
+# 主动推进 ready/failed 事件
+python3 scripts/gaokao-delivery-dispatch.py --channel station --limit 100
+
+# 巡检：有失败时返回 2
+python3 scripts/gaokao-delivery-watchdog.py --channel station --limit 100
+
+# retention dry-run（人工指定 cutoff）
+python3 scripts/gaokao-retention-cleanup.py --cutoff 2025-12-31T00:00:00+00:00 --dry-run
+
+# retention 定时模式（自动计算 cutoff）
+python3 scripts/gaokao-retention-cleanup.py --retention-days 180
+```
+
+退出码：
+
+- dispatcher：`0` = 脚本执行完成（结果看 JSON）
+- watchdog：`0` = 本轮无失败；`2` = 本轮发现失败事件
+- retention cleanup：`0` = 脚本执行完成；异常参数或运行错误会非 0 退出
+
+## 4. systemd 安装口径
+
+```bash
+sudo cp deploy/systemd/gaokao-*.service /etc/systemd/system/
+sudo cp deploy/systemd/gaokao-*.timer /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now gaokao-delivery-dispatch.timer
+sudo systemctl enable --now gaokao-delivery-watchdog.timer
+sudo systemctl enable --now gaokao-retention-cleanup.timer
+```
+
+验证：
+
+```bash
+systemctl list-timers 'gaokao-*'
+systemctl start gaokao-delivery-dispatch.service
+systemctl status gaokao-delivery-watchdog.service --no-pager
+journalctl -u gaokao-retention-cleanup.service -n 50 --no-pager
+```
+
+说明：
+
+- watchdog 的 exit code `2` 会让对应 service 进入 failed，便于接 `OnFailure=` 或宿主机监控。
+- 如果部署路径不是当前仓库路径，先修改 unit 里的 `WorkingDirectory` 与 `EnvironmentFile`。
+
+## 5. cron 备用口径
+
+`deploy/cron/gaokao-jobs.crontab` 给出最小样例，导入前先确保：
+
+1. `/var/log/gaokao-volunteer-system/` 已创建且可写；
+2. `GAOKAO_PYTHON_BIN` 指向安装完依赖的解释器；
+3. crontab 中仓库路径与真实部署一致。
+
+导入示例：
+
+```bash
+crontab deploy/cron/gaokao-jobs.crontab
+crontab -l
+```
+
+## 6. 推荐节奏
+
+- dispatcher：每 5 分钟
+- watchdog：每 10 分钟
+- retention cleanup：每周一次（周日 03:30 UTC）
+
+## 7. 仍未收口的缺口
+
+这些仍然没有被本次 runbook 伪装成“已完成”：
+
+1. `sent` 语义仍偏向“站内交付校验完成”，不是外部渠道真实投递成功。
+2. 尚无邮件/微信真实发送执行器。
+3. watchdog 只有失败退出码，还没有仓库内置的告警推送集成。
+4. retention cleanup 只是后台匿名化作业，前台/客服删除工单流程仍未上线。
+5. 这些 unit/timer/cron 样例已落仓，但是否真正安装到目标生产主机，需要部署时另行执行并留存记录。
--- a/docs/DELIVERY_SERVICE_DESIGN.md
+++ b/docs/DELIVERY_SERVICE_DESIGN.md
@@ -54,13 +54,17 @@
 - `delivered` 但无交付物时不再误报 `report_ready`
 - `data.notifications.dispatcher.DeliveryDispatcher`
 - `scripts/gaokao-delivery-dispatch.py`
+- `scripts/gaokao-delivery-watchdog.py`（失败返回 exit code `2`）
 - `ready -> sent` / `缺文件 -> failed` / `attempt_count` 递增
+- `docs/DELIVERY_RETENTION_OPS_RUNBOOK.md`
+- `deploy/systemd/gaokao-delivery-{dispatch,watchdog}.{service,timer}`
+- `deploy/cron/gaokao-jobs.crontab`

 未完成：

 - 独立 `delivery_job` / `delivery_attempt` 模型
 - 邮件/渠道真实发送执行器
- 自动重试调度与告警链
+- 告警推送集成（当前只有 watchdog 非 0 退出码）
 - 多通道统一投递状态页

 ## 6. 当前最短闭环
@@ -72,10 +76,21 @@
 5. `report_ready` 事件落库且幂等
 6. `gaokao-delivery-dispatch.py --channel station` 可把 ready 事件推进到 sent
 7. 缺失交付物时事件转 failed，并记录 `failure_reason`
+8. 可通过 systemd timer / cron 周期执行 dispatcher，并用 watchdog 暴露失败退出码

 ## 7. 下一步实施建议

-1. 把 dispatcher 接入定时任务或 systemd timer
-2. 增加邮件或站内通知真实发送器（二选一先闭环）
-3. 增加失败重试阈值与告警
+1. 增加邮件或站内通知真实发送器（二选一先闭环）
+2. 增加失败重试阈值与告警推送
+3. 把 `sent` 语义拆成“可投递校验通过”与“真实渠道已发送”
 4. 再决定是否扩到多通道
+
+## 8. 生产化接入口径
+
+仓库内已提供最小 runbook 与调度样例：
+
+- runbook：`docs/DELIVERY_RETENTION_OPS_RUNBOOK.md`
+- systemd：`deploy/systemd/gaokao-delivery-dispatch.{service,timer}` / `deploy/systemd/gaokao-delivery-watchdog.{service,timer}`
+- cron：`deploy/cron/gaokao-jobs.crontab`
+
+注意：当前 watchdog 仍是“带失败退出码的 dispatch 巡检”，并不等于独立通知告警系统。
--- a/docs/T12_ACCEPTANCE_CRITERIA.md
+++ b/docs/T12_ACCEPTANCE_CRITERIA.md
@@ -66,7 +66,7 @@ T12 的验收对象不是后台运营链路，而是用户端 Web 自助主链
 1. 真实支付 provider 未完成且目标口径要求真实支付
 2. 交付事件仅停留在路由级补丁，未形成稳定主链能力
 3. 前台仅有文案，无同意字段落库
-4. 备份恢复只有文档，没有最小验证
+4. 备份恢复仍没有最小快照+restore smoke，或虽有脚本但未形成可验证基线

 ---

--- a/docs/plans/P1-8-backup-restore-drill.md
+++ b/docs/plans/P1-8-backup-restore-drill.md
@@ -0,0 +1,61 @@
+# P1-8 backup restore drill
+
+最后更新: 2026-06-14
+状态: 本地最小演练方案已落地，生产主机安装/告警仍待外部收口
+
+## 1. 目标
+
+把“只能证明文件可复制”的状态，提升到“可对快照执行 restore smoke，验证最小服务链可用”。
+
+## 2. 本次落地范围
+
+- 新增 `scripts/backup_snapshot.sh`
+- 扩展 `scripts/backup_verify.sh`
+- 新增 `scripts/backup_restore_smoke.py`
+- 补 `ops/cron` 与 `ops/systemd` 示例接入口径
+- 补 pytest 验证用例 `tests/test_backup_workflow.py`
+
+## 3. 演练步骤
+
+### 3.1 生成快照
+
+```bash
+bash scripts/backup_snapshot.sh /tmp/gaokao-backups
+```
+
+### 3.2 对快照执行恢复校验
+
+```bash
+bash scripts/backup_verify.sh --from-backup /tmp/gaokao-backups/backup-<UTC_TIMESTAMP>
+```
+
+### 3.3 校验成功标准
+
+- `manifest_ok` 输出存在
+- SQLite 文件输出 `sqlite_ok`
+- restore smoke JSON 至少包含：
+  - `health_status = 200`
+  - `portal_status = 200`
+  - `portal_report = 200`
+  - `portal_pdf = 200`
+
+## 4. 设计约束
+
+1. restore smoke 只在临时恢复副本上写入测试订单，不改原始快照
+2. 若仓库内不存在现成交付物，smoke 允许在恢复副本下生成极小 HTML/PDF 占位物
+3. 本方案不宣称“完整生产恢复完成”，只证明最小服务链可被恢复副本支撑
+
+## 5. 非目标
+
+- 异地备份
+- 主机级定时器安装
+- 失败告警链
+- 密钥轮换真实执行记录
+- 完整支付/退款/交付全链恢复
+
+## 6. 后续外部动作
+
+- 在目标主机安装 cron 或 systemd timer
+- 将备份日志接入监控/通知
+- 形成季度恢复演练记录
+- 补密钥轮换 runbook 与演练记录
--- a/ops/cron/gaokao-backup.crontab.example
+++ b/ops/cron/gaokao-backup.crontab.example
@@ -0,0 +1,19 @@
+# 安装前请替换仓库路径、日志路径和备份根目录。
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+GAOKAO_BACKUP_ROOT=/var/backups/gaokao
+GAOKAO_BACKUP_KEEP=7
+GAOKAO_DB_PATH=/var/lib/gaokao/admin.db
+GAOKAO_ORDERS_DB_PATH=/var/lib/gaokao/orders.db
+GAOKAO_SHARE_DB_PATH=/var/lib/gaokao/share/short_links.db
+GAOKAO_SHARE_REPORT_DIR=/var/lib/gaokao/share/reports
+GAOKAO_BACKUP_ENV_FILE=/etc/gaokao/.env
+GAOKAO_BACKUP_CONFIG_DIR=/etc/gaokao
+GAOKAO_BACKUP_SECRETS_DIR=/var/lib/gaokao/secrets
+
+# 每天 02:30 生成一份目录快照
+30 2 * * * cd /srv/gaokao-volunteer-system && bash scripts/backup_snapshot.sh >> /var/log/gaokao-backup.log 2>&1
+
+# 每周一 03:00 对最近一份快照执行 restore smoke（在临时目录，不改原快照）
+0 3 * * 1 cd /srv/gaokao-volunteer-system && latest=$(find "$GAOKAO_BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d -name 'backup-*' | sort | tail -n 1) && test -n "$latest" && bash scripts/backup_verify.sh --from-backup "$latest" >> /var/log/gaokao-backup-verify.log 2>&1
--- a/ops/systemd/gaokao-backup-verify.service
+++ b/ops/systemd/gaokao-backup-verify.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=Gaokao volunteer system backup restore smoke
+Wants=network-online.target
+After=network-online.target gaokao-backup.service
+
+[Service]
+Type=oneshot
+WorkingDirectory=/srv/gaokao-volunteer-system
+Environment=GAOKAO_BACKUP_ROOT=/var/backups/gaokao
+ExecStart=/bin/bash -lc 'latest=$(find "$GAOKAO_BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d -name "backup-*" | sort | tail -n 1); test -n "$latest"; bash scripts/backup_verify.sh --from-backup "$latest"'
--- a/ops/systemd/gaokao-backup-verify.timer
+++ b/ops/systemd/gaokao-backup-verify.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Run gaokao backup restore smoke weekly
+
+[Timer]
+OnCalendar=Mon 03:00:00
+Persistent=true
+Unit=gaokao-backup-verify.service
+
+[Install]
+WantedBy=timers.target
--- a/ops/systemd/gaokao-backup.service
+++ b/ops/systemd/gaokao-backup.service
@@ -0,0 +1,18 @@
+[Unit]
+Description=Gaokao volunteer system backup snapshot
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/srv/gaokao-volunteer-system
+Environment=GAOKAO_BACKUP_ROOT=/var/backups/gaokao
+Environment=GAOKAO_BACKUP_KEEP=7
+Environment=GAOKAO_DB_PATH=/var/lib/gaokao/admin.db
+Environment=GAOKAO_ORDERS_DB_PATH=/var/lib/gaokao/orders.db
+Environment=GAOKAO_SHARE_DB_PATH=/var/lib/gaokao/share/short_links.db
+Environment=GAOKAO_SHARE_REPORT_DIR=/var/lib/gaokao/share/reports
+Environment=GAOKAO_BACKUP_ENV_FILE=/etc/gaokao/.env
+Environment=GAOKAO_BACKUP_CONFIG_DIR=/etc/gaokao
+Environment=GAOKAO_BACKUP_SECRETS_DIR=/var/lib/gaokao/secrets
+ExecStart=/bin/bash scripts/backup_snapshot.sh
--- a/ops/systemd/gaokao-backup.timer
+++ b/ops/systemd/gaokao-backup.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Run gaokao backup snapshot daily
+
+[Timer]
+OnCalendar=*-*-* 02:30:00
+Persistent=true
+Unit=gaokao-backup.service
+
+[Install]
+WantedBy=timers.target
--- a/scripts/backup_restore_smoke.py
+++ b/scripts/backup_restore_smoke.py
@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import uuid
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Iterator
+
+ROOT_DIR = Path(__file__).resolve().parents[1]
+if str(ROOT_DIR) not in sys.path:
+    sys.path.insert(0, str(ROOT_DIR))
+
+DEFAULT_JWT_SECRET = "backup-restore-smoke-secret-0123456789abcdef0123456789abcdef"
+DEFAULT_FERNET_SECRET = "backup-restore-smoke-fernet-secret"
+DEFAULT_ADMIN_PASSWORD = "backup-restore-pass-123"
+
+
+def _choose_restore_file(backup_dir: Path, suffix: str) -> Path | None:
+    candidates = sorted(
+        path
+        for path in (backup_dir / "files").rglob(f"*{suffix}")
+        if path.is_file() and "__pycache__" not in path.parts
+    )
+    return candidates[0] if candidates else None
+
+
+def _ensure_smoke_artifacts(backup_dir: Path) -> tuple[Path, Path]:
+    smoke_dir = backup_dir / "files" / "_restore_smoke"
+    smoke_dir.mkdir(parents=True, exist_ok=True)
+
+    html = _choose_restore_file(backup_dir, ".html")
+    pdf = _choose_restore_file(backup_dir, ".pdf")
+
+    if html is None:
+        html = smoke_dir / "restore-smoke-report.html"
+        html.write_text(
+            "<h1>备份恢复演练报告</h1><p>restore smoke generated</p>",
+            encoding="utf-8",
+        )
+    if pdf is None:
+        pdf = smoke_dir / "restore-smoke-report.pdf"
+        pdf.write_bytes(b"%PDF-1.4\nrestore-smoke\n")
+    return html, pdf
+
+
+@contextmanager
+def _patched_env(overrides: dict[str, str]) -> Iterator[None]:
+    original = {key: os.environ.get(key) for key in overrides}
+    try:
+        for key, value in overrides.items():
+            os.environ[key] = value
+        yield
+    finally:
+        for key, old_value in original.items():
+            if old_value is None:
+                os.environ.pop(key, None)
+            else:
+                os.environ[key] = old_value
+
+
+def run_restore_smoke(backup_dir: str | Path) -> dict[str, object]:
+    from fastapi.testclient import TestClient
+
+    from admin.app import create_app
+    from admin.config import load_settings
+    from data.customer_portal.token import issue_portal_token
+    from data.orders.dao import OrdersDAO
+    from data.orders.models import Order
+    from data.payments.service import PaymentService
+
+    root = Path(backup_dir).resolve()
+    if not root.is_dir():
+        raise FileNotFoundError(f"backup dir not found: {root}")
+
+    db_dir = root / "db"
+    admin_db = db_dir / "admin.db"
+    orders_db = db_dir / "orders.db"
+    share_db = db_dir / "short_links.db"
+    share_report_dir = root / "files" / "reports"
+    if not share_report_dir.exists():
+        share_report_dir = root / "files"
+
+    html_path, pdf_path = _ensure_smoke_artifacts(root)
+    order_id = f"GKO-BACKUP-SMOKE-{uuid.uuid4().hex[:8].upper()}"
+
+    overrides = {
+        "GAOKAO_ENV": os.environ.get("GAOKAO_ENV", "dev"),
+        "GAOKAO_DB_PATH": str(admin_db),
+        "GAOKAO_ORDERS_DB_PATH": str(orders_db),
+        "GAOKAO_SHARE_DB_PATH": str(share_db),
+        "GAOKAO_SHARE_REPORT_DIR": str(share_report_dir),
+        "GAOKAO_ORDERS_FERNET_KEY": os.environ.get(
+            "GAOKAO_ORDERS_FERNET_KEY", DEFAULT_FERNET_SECRET
+        ),
+        "GAOKAO_JWT_SECRET": os.environ.get("GAOKAO_JWT_SECRET", DEFAULT_JWT_SECRET),
+        "GAOKAO_ADMIN_USER": os.environ.get("GAOKAO_ADMIN_USER", "admin"),
+        "GAOKAO_ADMIN_PASS": os.environ.get(
+            "GAOKAO_ADMIN_PASS", DEFAULT_ADMIN_PASSWORD
+        ),
+        "GAOKAO_PAYMENT_PROVIDER": "mock",
+        "GAOKAO_PAYMENT_BASE_URL": "http://testserver",
+        "GAOKAO_PAYMENT_WEBHOOK_SECRET": os.environ.get(
+            "GAOKAO_PAYMENT_WEBHOOK_SECRET", "backup-restore-smoke-payment-secret"
+        ),
+    }
+
+    with _patched_env(overrides):
+        settings = load_settings()
+        app = create_app(settings)
+        with TestClient(app) as client:
+            health = client.get("/health")
+            if health.status_code != 200:
+                raise RuntimeError(f"health failed: {health.status_code} {health.text}")
+
+            order = Order(
+                id=order_id,
+                source="web",
+                service_version="standard",
+                amount_cents=9900,
+                status="pending",
+                customer_name="备份演练家长",
+                customer_phone="13800138000",
+                candidate_name="恢复演练考生",
+                candidate_province="湖南",
+            )
+            with OrdersDAO.connect(settings.orders_db_path) as dao:
+                dao.create(
+                    order, actor="backup_restore_smoke", reason="seed_pending_order"
+                )
+
+            payment_service = PaymentService.for_db(
+                settings.orders_db_path,
+                base_url=settings.payment_base_url,
+                webhook_secret=settings.payment_webhook_secret,
+            )
+            checkout = payment_service.create_checkout(
+                order.id, portal_token="backup-smoke-token"
+            )
+            payload, headers = payment_service.provider.build_webhook_request(
+                payment_id=checkout.payment_id,
+                amount_cents=order.amount_cents,
+                provider_trade_no=f"SMOKE-{order.id}",
+            )
+            payment_service.handle_webhook(payload, headers.get("X-Mock-Signature", ""))
+
+            with OrdersDAO.connect(settings.orders_db_path) as dao:
+                dao.update(
+                    order.id,
+                    {"audit_report": str(html_path), "pdf_path": str(pdf_path)},
+                    actor="backup_restore_smoke",
+                    reason="attach_restore_artifacts",
+                )
+                dao.transition_status(
+                    order.id,
+                    "serving",
+                    actor="backup_restore_smoke",
+                    reason="processing",
+                )
+                dao.transition_status(
+                    order.id,
+                    "delivered",
+                    actor="backup_restore_smoke",
+                    reason="report_ready",
+                )
+
+            token = issue_portal_token(order.id, settings.jwt_secret)
+            status_page = client.get(f"/portal/{token}/status")
+            if status_page.status_code != 200 or "报告已就绪" not in status_page.text:
+                raise RuntimeError(
+                    f"portal status failed: {status_page.status_code} {status_page.text}"
+                )
+
+            report_page = client.get(f"/portal/{token}/report")
+            if report_page.status_code != 200:
+                raise RuntimeError(
+                    f"portal report failed: {report_page.status_code} {report_page.text}"
+                )
+
+            pdf_resp = client.get(f"/portal/{token}/report.pdf")
+            if pdf_resp.status_code != 200:
+                raise RuntimeError(
+                    f"portal pdf failed: {pdf_resp.status_code} {pdf_resp.text}"
+                )
+            if not pdf_resp.headers.get("content-type", "").startswith(
+                "application/pdf"
+            ):
+                raise RuntimeError(
+                    f"portal pdf content-type invalid: {pdf_resp.headers.get('content-type')}"
+                )
+            if not pdf_resp.content.startswith(b"%PDF-"):
+                raise RuntimeError("portal pdf payload is not a PDF header")
+
+    result = {
+        "backup_dir": str(root),
+        "admin_db": str(admin_db),
+        "orders_db": str(orders_db),
+        "report_html": str(html_path),
+        "report_pdf": str(pdf_path),
+        "smoke_order_id": order_id,
+        "health_status": 200,
+        "portal_status": 200,
+        "portal_report": 200,
+        "portal_pdf": 200,
+        "pdf_bytes": len(pdf_resp.content),
+    }
+    return result
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Run restore smoke test against a staged backup directory"
+    )
+    parser.add_argument(
+        "--backup-dir",
+        required=True,
+        help="staged backup directory containing db/ and files/",
+    )
+    args = parser.parse_args(argv)
+
+    result = run_restore_smoke(args.backup_dir)
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/scripts/backup_snapshot.sh
+++ b/scripts/backup_snapshot.sh
@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+BACKUP_ROOT="${1:-${GAOKAO_BACKUP_ROOT:-${ROOT_DIR}/var/backups}}"
+TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+SNAPSHOT_DIR="${BACKUP_ROOT}/backup-${TIMESTAMP}"
+KEEP_COUNT="${GAOKAO_BACKUP_KEEP:-7}"
+ADMIN_DB="${GAOKAO_DB_PATH:-${ROOT_DIR}/data/orders/admin.db}"
+ORDERS_DB="${GAOKAO_ORDERS_DB_PATH:-${ROOT_DIR}/data/orders.db}"
+SHARE_DB="${GAOKAO_SHARE_DB_PATH:-${ROOT_DIR}/data/share/short_links.db}"
+SHARE_REPORT_DIR="${GAOKAO_SHARE_REPORT_DIR:-${ROOT_DIR}/data/share/reports}"
+EXAMPLE_REPORT_DIR="${ROOT_DIR}/data/examples"
+ENV_FILE="${GAOKAO_BACKUP_ENV_FILE:-}"
+CONFIG_DIR="${GAOKAO_BACKUP_CONFIG_DIR:-}"
+SECRETS_DIR="${GAOKAO_BACKUP_SECRETS_DIR:-}"
+
+log() {
+  printf '[backup-snapshot] %s\n' "$1"
+}
+
+copy_file_to() {
+  local src="$1"
+  local dest="$2"
+  if [[ -n "$src" && -f "$src" ]]; then
+    mkdir -p "$(dirname "$dest")"
+    cp "$src" "$dest"
+    log "copied file: $src -> $dest"
+  else
+    log "skip missing file: ${src:-<empty>}"
+  fi
+}
+
+copy_dir_to() {
+  local src="$1"
+  local dest="$2"
+  if [[ -n "$src" && -d "$src" ]]; then
+    mkdir -p "$dest"
+    cp -R "$src/." "$dest/"
+    log "copied directory: $src -> $dest"
+  else
+    log "skip missing directory: ${src:-<empty>}"
+  fi
+}
+
+write_manifest() {
+  ADMIN_DB="$ADMIN_DB" \
+  ORDERS_DB="$ORDERS_DB" \
+  SHARE_DB="$SHARE_DB" \
+  SHARE_REPORT_DIR="$SHARE_REPORT_DIR" \
+  EXAMPLE_REPORT_DIR="$EXAMPLE_REPORT_DIR" \
+  ENV_FILE="$ENV_FILE" \
+  CONFIG_DIR="$CONFIG_DIR" \
+  SECRETS_DIR="$SECRETS_DIR" \
+  python3 - "$SNAPSHOT_DIR" <<'PY'
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import sys
+from pathlib import Path
+
+root = Path(sys.argv[1]).resolve()
+manifest_path = root / "manifest.json"
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+manifest = {
+    "generated_at": root.name.removeprefix("backup-"),
+    "snapshot_dir": str(root),
+    "source_paths": {
+        "admin_db": os.environ.get("ADMIN_DB", ""),
+        "orders_db": os.environ.get("ORDERS_DB", ""),
+        "share_db": os.environ.get("SHARE_DB", ""),
+        "share_report_dir": os.environ.get("SHARE_REPORT_DIR", ""),
+        "example_report_dir": os.environ.get("EXAMPLE_REPORT_DIR", ""),
+        "env_file": os.environ.get("ENV_FILE", ""),
+        "config_dir": os.environ.get("CONFIG_DIR", ""),
+        "secrets_dir": os.environ.get("SECRETS_DIR", ""),
+    },
+    "files": [],
+}
+
+for path in sorted(root.rglob("*")):
+    if not path.is_file() or path == manifest_path:
+        continue
+    manifest["files"].append(
+        {
+            "path": str(path.relative_to(root)),
+            "size_bytes": path.stat().st_size,
+            "sha256": sha256_file(path),
+        }
+    )
+
+manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
+print(manifest_path)
+PY
+}
+
+prune_old_snapshots() {
+  local backup_root="$1"
+  local keep_count="$2"
+  if ! [[ "$keep_count" =~ ^[0-9]+$ ]] || (( keep_count < 1 )); then
+    log "invalid GAOKAO_BACKUP_KEEP=$keep_count, skip prune"
+    return
+  fi
+
+  mapfile -t snapshots < <(find "$backup_root" -mindepth 1 -maxdepth 1 -type d -name 'backup-*' | sort)
+  local total="${#snapshots[@]}"
+  if (( total <= keep_count )); then
+    return
+  fi
+
+  local prune_count=$((total - keep_count))
+  local i
+  for ((i = 0; i < prune_count; i++)); do
+    rm -rf "${snapshots[$i]}"
+    log "pruned old snapshot: ${snapshots[$i]}"
+  done
+}
+
+main() {
+  mkdir -p "$SNAPSHOT_DIR"
+  log "snapshot dir: $SNAPSHOT_DIR"
+
+  copy_file_to "$ADMIN_DB" "$SNAPSHOT_DIR/db/$(basename "$ADMIN_DB")"
+  copy_file_to "$ORDERS_DB" "$SNAPSHOT_DIR/db/$(basename "$ORDERS_DB")"
+  copy_file_to "$SHARE_DB" "$SNAPSHOT_DIR/db/$(basename "$SHARE_DB")"
+  copy_dir_to "$SHARE_REPORT_DIR" "$SNAPSHOT_DIR/files/reports"
+  copy_dir_to "$EXAMPLE_REPORT_DIR" "$SNAPSHOT_DIR/files/examples"
+  copy_file_to "$ENV_FILE" "$SNAPSHOT_DIR/config/.env"
+  copy_dir_to "$CONFIG_DIR" "$SNAPSHOT_DIR/config/deploy"
+  copy_dir_to "$SECRETS_DIR" "$SNAPSHOT_DIR/secrets"
+
+  local manifest_path
+  manifest_path="$(write_manifest)"
+  log "manifest written: $manifest_path"
+
+  prune_old_snapshots "$BACKUP_ROOT" "$KEEP_COUNT"
+  log "backup snapshot finished"
+  printf '%s\n' "$SNAPSHOT_DIR"
+}
+
+main "$@"
--- a/scripts/backup_verify.sh
+++ b/scripts/backup_verify.sh
@@ -2,13 +2,28 @@
 set -euo pipefail

 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-VERIFY_DIR="${1:-$(mktemp -d /tmp/gaokao-backup-verify-XXXXXX)}"
+SOURCE_BACKUP_DIR=""
+VERIFY_DIR=""
+SKIP_SMOKE="0"
 ADMIN_DB="${GAOKAO_DB_PATH:-${ROOT_DIR}/data/orders/admin.db}"
 ORDERS_DB="${GAOKAO_ORDERS_DB_PATH:-${ROOT_DIR}/data/orders.db}"
 SHARE_DB="${GAOKAO_SHARE_DB_PATH:-${ROOT_DIR}/data/share/short_links.db}"
 SHARE_REPORT_DIR="${GAOKAO_SHARE_REPORT_DIR:-${ROOT_DIR}/data/share/reports}"
 EXAMPLE_REPORT_DIR="${ROOT_DIR}/data/examples"

+usage() {
+  cat <<'EOF'
+Usage:
+  bash scripts/backup_verify.sh [verify_dir]
+  bash scripts/backup_verify.sh --from-backup /path/to/backup-dir [--skip-smoke]
+
+Default mode copies live SQLite/files into a temporary verify dir, then validates:
+- SQLite files are readable
+- manifest checksums (if manifest.json exists)
+- restore smoke via FastAPI TestClient (unless --skip-smoke)
+EOF
+}
+
 log() {
  printf '[backup-verify] %s\n' "$1"
 }
@@ -30,13 +45,69 @@ copy_dir_if_exists() {
  local dest_dir="$2"
  if [[ -d "$src" ]]; then
    mkdir -p "$dest_dir"
-    cp -R "$src" "$dest_dir/"
+    cp -R "$src/." "$dest_dir/"
    log "copied directory: $src"
  else
    log "skip missing directory: $src"
  fi
 }

+stage_live_sources() {
+  log "verify dir: $VERIFY_DIR"
+  copy_if_exists "$ADMIN_DB" "$VERIFY_DIR/db"
+  copy_if_exists "$ORDERS_DB" "$VERIFY_DIR/db"
+  copy_if_exists "$SHARE_DB" "$VERIFY_DIR/db"
+  copy_dir_if_exists "$SHARE_REPORT_DIR" "$VERIFY_DIR/files/reports"
+  copy_dir_if_exists "$EXAMPLE_REPORT_DIR" "$VERIFY_DIR/files/examples"
+}
+
+stage_existing_backup() {
+  if [[ ! -d "$SOURCE_BACKUP_DIR" ]]; then
+    printf 'backup dir not found: %s\n' "$SOURCE_BACKUP_DIR" >&2
+    exit 1
+  fi
+  log "staging backup dir: $SOURCE_BACKUP_DIR -> $VERIFY_DIR"
+  mkdir -p "$VERIFY_DIR"
+  cp -R "$SOURCE_BACKUP_DIR/." "$VERIFY_DIR/"
+}
+
+verify_manifest_if_present() {
+  local manifest_path="$1"
+  if [[ ! -f "$manifest_path" ]]; then
+    log "skip missing manifest: $manifest_path"
+    return
+  fi
+
+  log "verifying manifest checksums"
+  python3 - "$manifest_path" <<'PY'
+from __future__ import annotations
+
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+manifest_path = Path(sys.argv[1]).resolve()
+root = manifest_path.parent
+manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+errors: list[str] = []
+for item in manifest.get("files", []):
+    rel = item["path"]
+    target = root / rel
+    if not target.is_file():
+        errors.append(f"missing file: {rel}")
+        continue
+    digest = hashlib.sha256(target.read_bytes()).hexdigest()
+    if digest != item["sha256"]:
+        errors.append(f"sha256 mismatch: {rel}")
+if errors:
+    for err in errors:
+        print(err)
+    raise SystemExit(1)
+print(f"manifest_ok files={len(manifest.get('files', []))}")
+PY
+}
+
 verify_sqlite_file() {
  local db_path="$1"
  if [[ ! -f "$db_path" ]]; then
@@ -46,6 +117,7 @@ verify_sqlite_file() {
 import sqlite3
 import sys
 from pathlib import Path
+
 p = Path(sys.argv[1])
 conn = sqlite3.connect(p)
 try:
@@ -57,16 +129,7 @@ finally:
 PY
 }

-main() {
-  mkdir -p "$VERIFY_DIR"
-  log "verify dir: $VERIFY_DIR"
-
-  copy_if_exists "$ADMIN_DB" "$VERIFY_DIR/db"
-  copy_if_exists "$ORDERS_DB" "$VERIFY_DIR/db"
-  copy_if_exists "$SHARE_DB" "$VERIFY_DIR/db"
-  copy_dir_if_exists "$SHARE_REPORT_DIR" "$VERIFY_DIR/files"
-  copy_dir_if_exists "$EXAMPLE_REPORT_DIR" "$VERIFY_DIR/files"
-
+verify_artifacts() {
  log "verifying copied sqlite files"
  if [[ -d "$VERIFY_DIR/db" ]]; then
    while IFS= read -r -d '' db_file; do
@@ -76,9 +139,61 @@ main() {

  log "verifying copied report artifacts"
  if [[ -d "$VERIFY_DIR/files" ]]; then
-    find "$VERIFY_DIR/files" -type f | sort | sed 's#^#[backup-verify] artifact: #' 
+    find "$VERIFY_DIR/files" -type f | sort | sed 's#^#[backup-verify] artifact: #'
+  fi
+}
+
+run_restore_smoke() {
+  if [[ "$SKIP_SMOKE" == "1" ]]; then
+    log "skip restore smoke"
+    return
  fi

+  log "running restore smoke"
+  python3 "$ROOT_DIR/scripts/backup_restore_smoke.py" --backup-dir "$VERIFY_DIR"
+}
+
+parse_args() {
+  while (( $# > 0 )); do
+    case "$1" in
+      --from-backup)
+        SOURCE_BACKUP_DIR="$2"
+        shift 2
+        ;;
+      --skip-smoke)
+        SKIP_SMOKE="1"
+        shift
+        ;;
+      -h|--help)
+        usage
+        exit 0
+        ;;
+      *)
+        if [[ -n "$VERIFY_DIR" ]]; then
+          printf 'unexpected argument: %s\n' "$1" >&2
+          exit 1
+        fi
+        VERIFY_DIR="$1"
+        shift
+        ;;
+    esac
+  done
+}
+
+main() {
+  parse_args "$@"
+  VERIFY_DIR="${VERIFY_DIR:-$(mktemp -d /tmp/gaokao-backup-verify-XXXXXX)}"
+
+  if [[ -n "$SOURCE_BACKUP_DIR" ]]; then
+    stage_existing_backup
+  else
+    mkdir -p "$VERIFY_DIR"
+    stage_live_sources
+  fi
+
+  verify_manifest_if_present "$VERIFY_DIR/manifest.json"
+  verify_artifacts
+  run_restore_smoke
  log "backup verification finished"
 }

--- a/scripts/gaokao_retention_cleanup.py
+++ b/scripts/gaokao_retention_cleanup.py
@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+
+if __name__ == "__main__":
+    from data.orders.retention_cleanup import main
+
+    raise SystemExit(main())
--- a/tests/test_backup_workflow.py
+++ b/tests/test_backup_workflow.py
@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import subprocess
+from pathlib import Path
+
+from admin.db import ensure_schema
+from data.orders.schema import apply_schema
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+
+
+def _bootstrap_share_db(db_path: str) -> None:
+    conn = sqlite3.connect(db_path)
+    try:
+        conn.execute("CREATE TABLE IF NOT EXISTS short_links (id INTEGER PRIMARY KEY)")
+        conn.commit()
+    finally:
+        conn.close()
+
+
+def _prepare_backup_sources(settings, tmp_path: Path) -> dict[str, str]:
+    ensure_schema(settings.db_path)
+    apply_schema(settings.orders_db_path).close()
+    _bootstrap_share_db(settings.share_db_path)
+
+    share_report_dir = Path(settings.share_report_dir)
+    share_report_dir.mkdir(parents=True, exist_ok=True)
+    (share_report_dir / "sample-report.json").write_text("{}", encoding="utf-8")
+
+    env_file = tmp_path / ".env.runtime"
+    env_file.write_text("GAOKAO_ENV=dev\n", encoding="utf-8")
+
+    config_dir = tmp_path / "deploy-config"
+    config_dir.mkdir(parents=True, exist_ok=True)
+    (config_dir / "docker-compose.yml").write_text("services: {}\n", encoding="utf-8")
+
+    secrets_dir = tmp_path / "secrets"
+    secrets_dir.mkdir(parents=True, exist_ok=True)
+    (secrets_dir / "jwt.secret").write_text("secret", encoding="utf-8")
+
+    return {
+        "GAOKAO_DB_PATH": settings.db_path,
+        "GAOKAO_ORDERS_DB_PATH": settings.orders_db_path,
+        "GAOKAO_SHARE_DB_PATH": settings.share_db_path,
+        "GAOKAO_SHARE_REPORT_DIR": settings.share_report_dir,
+        "GAOKAO_BACKUP_ENV_FILE": str(env_file),
+        "GAOKAO_BACKUP_CONFIG_DIR": str(config_dir),
+        "GAOKAO_BACKUP_SECRETS_DIR": str(secrets_dir),
+    }
+
+
+def _run_snapshot(env: dict[str, str], backup_root: Path) -> Path:
+    proc = subprocess.run(
+        ["bash", "scripts/backup_snapshot.sh", str(backup_root)],
+        cwd=PROJECT_ROOT,
+        text=True,
+        capture_output=True,
+        env=env,
+        check=False,
+    )
+    assert proc.returncode == 0, proc.stderr
+    return Path(proc.stdout.strip().splitlines()[-1])
+
+
+def test_backup_snapshot_creates_manifest_and_prunes_old_backups(settings, tmp_path):
+    env = {
+        **os.environ,
+        **_prepare_backup_sources(settings, tmp_path),
+        "GAOKAO_BACKUP_KEEP": "2",
+    }
+    backup_root = tmp_path / "backups"
+    (backup_root / "backup-20240101T000000Z").mkdir(parents=True, exist_ok=True)
+    (backup_root / "backup-20240102T000000Z").mkdir(parents=True, exist_ok=True)
+
+    snapshot_dir = _run_snapshot(env, backup_root)
+
+    assert snapshot_dir.is_dir()
+    manifest_path = snapshot_dir / "manifest.json"
+    assert manifest_path.is_file()
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    manifest_paths = {item["path"] for item in manifest["files"]}
+    assert "db/admin.db" in manifest_paths
+    assert "db/orders.db" in manifest_paths
+    assert "db/short_links.db" in manifest_paths
+    assert "config/.env" in manifest_paths
+    assert "config/deploy/docker-compose.yml" in manifest_paths
+    assert "secrets/jwt.secret" in manifest_paths
+    assert any(path.startswith("files/examples/") for path in manifest_paths)
+
+    remaining = sorted(path.name for path in backup_root.iterdir() if path.is_dir())
+    assert len(remaining) == 2
+    assert "backup-20240101T000000Z" not in remaining
+    assert snapshot_dir.name in remaining
+
+
+def test_backup_verify_runs_restore_smoke_on_snapshot(settings, tmp_path):
+    env = {
+        **os.environ,
+        **_prepare_backup_sources(settings, tmp_path),
+        "GAOKAO_PAYMENT_PROVIDER": "alipay",
+    }
+    backup_root = tmp_path / "backups"
+    snapshot_dir = _run_snapshot(env, backup_root)
+
+    proc = subprocess.run(
+        ["bash", "scripts/backup_verify.sh", "--from-backup", str(snapshot_dir)],
+        cwd=PROJECT_ROOT,
+        text=True,
+        capture_output=True,
+        env=env,
+        check=False,
+    )
+
+    assert proc.returncode == 0, proc.stderr
+    assert "manifest_ok files=" in proc.stdout
+    assert '"portal_status": 200' in proc.stdout
+    assert '"portal_report": 200' in proc.stdout
+    assert '"portal_pdf": 200' in proc.stdout
+    assert "backup verification finished" in proc.stdout
--- a/tests/test_delivery_retention_ops_artifacts.py
+++ b/tests/test_delivery_retention_ops_artifacts.py
@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_delivery_retention_ops_artifacts_exist_and_reference_scripts():
+    runbook = PROJECT_ROOT / "docs/DELIVERY_RETENTION_OPS_RUNBOOK.md"
+    dispatch_service = PROJECT_ROOT / "deploy/systemd/gaokao-delivery-dispatch.service"
+    dispatch_timer = PROJECT_ROOT / "deploy/systemd/gaokao-delivery-dispatch.timer"
+    watchdog_service = PROJECT_ROOT / "deploy/systemd/gaokao-delivery-watchdog.service"
+    watchdog_timer = PROJECT_ROOT / "deploy/systemd/gaokao-delivery-watchdog.timer"
+    retention_service = PROJECT_ROOT / "deploy/systemd/gaokao-retention-cleanup.service"
+    retention_timer = PROJECT_ROOT / "deploy/systemd/gaokao-retention-cleanup.timer"
+    cron_file = PROJECT_ROOT / "deploy/cron/gaokao-jobs.crontab"
+    env_example = PROJECT_ROOT / "deploy/systemd/gaokao-jobs.env.example"
+
+    for path in (
+        runbook,
+        dispatch_service,
+        dispatch_timer,
+        watchdog_service,
+        watchdog_timer,
+        retention_service,
+        retention_timer,
+        cron_file,
+        env_example,
+    ):
+        assert path.is_file(), path
+
+    assert "gaokao-delivery-dispatch.py" in dispatch_service.read_text(encoding="utf-8")
+    assert "gaokao-delivery-watchdog.py" in watchdog_service.read_text(encoding="utf-8")
+    assert "--retention-days" in retention_service.read_text(encoding="utf-8")
+
+    cron_text = cron_file.read_text(encoding="utf-8")
+    assert "gaokao-delivery-dispatch.py" in cron_text
+    assert "gaokao-delivery-watchdog.py" in cron_text
+    assert "gaokao-retention-cleanup.py --retention-days" in cron_text
+
+    runbook_text = runbook.read_text(encoding="utf-8")
+    assert "systemd" in runbook_text
+    assert "cron" in runbook_text
+    assert "gaokao-retention-cleanup.py --retention-days 180" in runbook_text
--- a/tests/test_retention_cleanup.py
+++ b/tests/test_retention_cleanup.py
@@ -89,4 +89,54 @@ def test_retention_cleanup_script_prints_summary(settings):
        check=False,
    )
    assert proc.returncode == 0, proc.stderr
+    assert '"cutoff_iso": "2025-06-30T00:00:00+00:00"' in proc.stdout
+    assert '"dry_run": true' in proc.stdout
+    assert '"candidates": 1' in proc.stdout
+
+
+def test_retention_cleanup_script_supports_retention_days(settings):
+    _seed_old_completed_order(
+        settings.orders_db_path, order_id="GKO-20250101-RETENTION-DAYS"
+    )
+    env = {**os.environ, "GAOKAO_ORDERS_DB_PATH": settings.orders_db_path}
+    proc = subprocess.run(
+        [
+            "python3",
+            "scripts/gaokao-retention-cleanup.py",
+            "--retention-days",
+            "180",
+            "--dry-run",
+        ],
+        cwd=PROJECT_ROOT,
+        text=True,
+        capture_output=True,
+        env=env,
+        check=False,
+    )
+    assert proc.returncode == 0, proc.stderr
+    assert '"cutoff_iso":' in proc.stdout
+    assert '"dry_run": true' in proc.stdout
+    assert '"candidates": 1' in proc.stdout
+
+
+def test_retention_cleanup_underscore_script_alias_works(settings):
+    _seed_old_completed_order(
+        settings.orders_db_path, order_id="GKO-20250101-RETENTION-ALIAS"
+    )
+    env = {**os.environ, "GAOKAO_ORDERS_DB_PATH": settings.orders_db_path}
+    proc = subprocess.run(
+        [
+            "python3",
+            "scripts/gaokao_retention_cleanup.py",
+            "--cutoff",
+            "2025-06-30T00:00:00+00:00",
+            "--dry-run",
+        ],
+        cwd=PROJECT_ROOT,
+        text=True,
+        capture_output=True,
+        env=env,
+        check=False,
+    )
+    assert proc.returncode == 0, proc.stderr
    assert '"candidates": 1' in proc.stdout