From 7fe4ddcbc4a8f39d6898ef3fbcf6303371e5072f Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sun, 14 Jun 2026 23:12:18 +0800 Subject: [PATCH] feat: harden T12 backup and portal workflows --- README.md | 14 ++ admin/routes/web_public.py | 24 +- admin/tests/test_order_status_page.py | 37 +++ admin/tests/test_web_public.py | 107 ++++++++ data/orders/retention_cleanup.py | 41 +++- deploy/cron/gaokao-jobs.crontab | 13 + .../systemd/gaokao-delivery-dispatch.service | 12 + deploy/systemd/gaokao-delivery-dispatch.timer | 11 + .../systemd/gaokao-delivery-watchdog.service | 12 + deploy/systemd/gaokao-delivery-watchdog.timer | 11 + deploy/systemd/gaokao-jobs.env.example | 10 + .../systemd/gaokao-retention-cleanup.service | 11 + deploy/systemd/gaokao-retention-cleanup.timer | 11 + docs/ACTIVE_REMEDIATION_2026-06-13.md | 44 ++-- docs/BACKUP_AND_RECOVERY_PLAN.md | 149 +++++++++--- docs/CURRENT_STATE.md | 12 +- docs/DATA_RETENTION_AND_DELETION.md | 17 +- docs/DELIVERY_RETENTION_OPS_RUNBOOK.md | 130 ++++++++++ docs/DELIVERY_SERVICE_DESIGN.md | 23 +- docs/T12_ACCEPTANCE_CRITERIA.md | 2 +- docs/plans/P1-8-backup-restore-drill.md | 61 +++++ ops/cron/gaokao-backup.crontab.example | 19 ++ ops/systemd/gaokao-backup-verify.service | 10 + ops/systemd/gaokao-backup-verify.timer | 10 + ops/systemd/gaokao-backup.service | 18 ++ ops/systemd/gaokao-backup.timer | 10 + scripts/backup_restore_smoke.py | 230 ++++++++++++++++++ scripts/backup_snapshot.sh | 152 ++++++++++++ scripts/backup_verify.sh | 141 ++++++++++- scripts/gaokao_retention_cleanup.py | 14 ++ tests/test_backup_workflow.py | 122 ++++++++++ .../test_delivery_retention_ops_artifacts.py | 45 ++++ tests/test_retention_cleanup.py | 50 ++++ 33 files changed, 1486 insertions(+), 87 deletions(-) create mode 100644 deploy/cron/gaokao-jobs.crontab create mode 100644 deploy/systemd/gaokao-delivery-dispatch.service create mode 100644 deploy/systemd/gaokao-delivery-dispatch.timer create mode 100644 deploy/systemd/gaokao-delivery-watchdog.service create mode 100644 deploy/systemd/gaokao-delivery-watchdog.timer create mode 100644 deploy/systemd/gaokao-jobs.env.example create mode 100644 deploy/systemd/gaokao-retention-cleanup.service create mode 100644 deploy/systemd/gaokao-retention-cleanup.timer create mode 100644 docs/DELIVERY_RETENTION_OPS_RUNBOOK.md create mode 100644 docs/plans/P1-8-backup-restore-drill.md create mode 100644 ops/cron/gaokao-backup.crontab.example create mode 100644 ops/systemd/gaokao-backup-verify.service create mode 100644 ops/systemd/gaokao-backup-verify.timer create mode 100644 ops/systemd/gaokao-backup.service create mode 100644 ops/systemd/gaokao-backup.timer create mode 100644 scripts/backup_restore_smoke.py create mode 100644 scripts/backup_snapshot.sh create mode 100644 scripts/gaokao_retention_cleanup.py create mode 100644 tests/test_backup_workflow.py create mode 100644 tests/test_delivery_retention_ops_artifacts.py diff --git a/README.md b/README.md index 372577e..a213427 100644 --- a/README.md +++ b/README.md @@ -162,19 +162,33 @@ GAOKAO_SKIP_INSTALL=1 bash scripts/dev-verify.sh - 执行器：`data.notifications.dispatcher.DeliveryDispatcher` - CLI：`python3 scripts/gaokao-delivery-dispatch.py --channel station` - watchdog：`python3 scripts/gaokao-delivery-watchdog.py --channel station` +- runbook：`docs/DELIVERY_RETENTION_OPS_RUNBOOK.md` +- systemd/cron 样例：`deploy/systemd/gaokao-delivery-*.{service,timer}` / `deploy/cron/gaokao-jobs.crontab` 最小语义： - 交付物齐全（HTML/PDF 存在）→ `ready -> sent` - 交付物缺失 → `failed`，并写入 `failure_reason` - watchdog 遇到失败事件返回 exit code `2` +- 当前 `sent` 仍表示“站内交付物校验通过”，不是外部渠道真实发送成功 ### 数据保留期清理 - dry-run：`python3 scripts/gaokao-retention-cleanup.py --cutoff --dry-run` - apply：`python3 scripts/gaokao-retention-cleanup.py --cutoff ` +- 定时模式：`python3 scripts/gaokao-retention-cleanup.py --retention-days 180` +- 兼容别名：`python3 scripts/gaokao_retention_cleanup.py --retention-days 180 --dry-run` +- runbook：`docs/DELIVERY_RETENTION_OPS_RUNBOOK.md` - 当前语义：对超过保留期的 `completed/refunded` 订单执行匿名化清理 +### 备份 / 恢复最小入口 + +- 生成快照：`bash scripts/backup_snapshot.sh /tmp/gaokao-backups` +- 校验快照：`bash scripts/backup_verify.sh --from-backup /tmp/gaokao-backups/backup-` +- 直接对当前 live 数据做一次恢复演练：`bash scripts/backup_verify.sh` +- runbook：`docs/BACKUP_AND_RECOVERY_PLAN.md` +- 定时接入口径：`ops/cron/gaokao-backup.crontab.example`、`ops/systemd/gaokao-backup*.service|timer` + ### crowd_db 质量汇总 - `python3 -m data.crowd_db.quality_summary --human` diff --git a/admin/routes/web_public.py b/admin/routes/web_public.py index d4ba2a2..b8ae89b 100644 --- a/admin/routes/web_public.py +++ b/admin/routes/web_public.py @@ -104,11 +104,26 @@ def create_public_order_endpoint( payload: PublicOrderCreate, settings: Settings = Depends(get_settings_dep), ) -> PublicOrderCreated: + try: + payment_service = _payment_service(settings) + except PaymentError as exc: + raise HTTPException( + status_code=503, + detail=f"payment provider unavailable: {exc}", + ) from exc + with OrdersDAO.connect(settings.orders_db_path) as dao: order = create_public_order(dao, payload) portal_token = issue_portal_token(order.id, settings.jwt_secret) - payment_service = _payment_service(settings) - checkout = payment_service.create_checkout(order.id, portal_token=portal_token) + try: + checkout = payment_service.create_checkout(order.id, portal_token=portal_token) + except PaymentError as exc: + with OrdersDAO.connect(settings.orders_db_path) as dao: + dao.delete(order.id) + raise HTTPException( + status_code=503, + detail=f"payment checkout unavailable: {exc}", + ) from exc return PublicOrderCreated( order_id=order.id, source=order.source, @@ -743,11 +758,12 @@ def _render_info_page( def _render_status_page(token: str, context: dict[str, Any]) -> str: order = context["order"] report_links = "" - if context["report_html_ready"]: + can_access_delivery = context["stage"] in {"report_ready", "completed"} + if can_access_delivery and context["report_html_ready"]: report_links += ( f'
  • 查看在线报告
    • ' ) - if context["report_pdf_ready"]: + if can_access_delivery and context["report_pdf_ready"]: report_links += ( f'
  • 下载 PDF
    • ' ) diff --git a/admin/tests/test_order_status_page.py b/admin/tests/test_order_status_page.py index b25cca6..3e55c7e 100644 --- a/admin/tests/test_order_status_page.py +++ b/admin/tests/test_order_status_page.py @@ -110,3 +110,40 @@ def test_delivered_without_artifacts_stays_processing(client, settings): report_page = client.get(f"/portal/{token}/report") assert report_page.status_code == 409 + + +def test_partial_artifacts_do_not_expose_delivery_links_before_report_ready( + client, settings, tmp_path: Path +): + order = _seed_order(settings.orders_db_path, order_id="GKO-20260614-STATUS-PARTIAL") + _mark_paid(settings, order) + IntakeStore.for_db(settings.orders_db_path).save( + order_id=order.id, + payload={"candidate_score": 578, "candidate_subjects": ["物理"]}, + submit=True, + ) + + report_path = tmp_path / "partial-report.html" + report_path.write_text("

    partial

    ", encoding="utf-8") + with OrdersDAO.connect(settings.orders_db_path) as dao: + dao.update( + order.id, + {"audit_report": str(report_path)}, + actor="test", + reason="attach_partial_report", + ) + dao.transition_status(order.id, "serving", actor="test", reason="processing") + dao.transition_status( + order.id, "delivered", actor="test", reason="report_ready_without_pdf" + ) + + token = issue_portal_token(order.id, settings.jwt_secret) + status_page = client.get(f"/portal/{token}/status") + assert status_page.status_code == 200, status_page.text + assert "处理中" in status_page.text + assert "查看在线报告" not in status_page.text + assert "下载 PDF" not in status_page.text + assert "报告生成中" in status_page.text + + report_page = client.get(f"/portal/{token}/report") + assert report_page.status_code == 409 diff --git a/admin/tests/test_web_public.py b/admin/tests/test_web_public.py index e5b682c..9b20c86 100644 --- a/admin/tests/test_web_public.py +++ b/admin/tests/test_web_public.py @@ -2,6 +2,10 @@ from __future__ import annotations +from fastapi.testclient import TestClient + +from data.orders.dao import OrdersDAO + def test_public_landing_page_served(client): resp = client.get("/") @@ -129,3 +133,106 @@ def test_payment_return_redirects_to_portal_status(client): resp = client.get(f"/portal/payment-return?token={token}", follow_redirects=False) assert resp.status_code == 303, resp.text assert resp.headers["location"] == f"/portal/{token}/status" + + +def test_public_create_order_returns_503_without_creating_orphan_order_when_provider_unavailable( + tmp_path, monkeypatch +): + admin_db = tmp_path / "admin.db" + orders_db = tmp_path / "orders.db" + share_db = tmp_path / "share.db" + share_reports = tmp_path / "share_reports" + share_reports.mkdir() + + monkeypatch.setenv("GAOKAO_ENV", "dev") + monkeypatch.setenv("GAOKAO_DB_PATH", str(admin_db)) + monkeypatch.setenv("GAOKAO_ORDERS_DB_PATH", str(orders_db)) + monkeypatch.setenv("GAOKAO_SHARE_DB_PATH", str(share_db)) + monkeypatch.setenv("GAOKAO_SHARE_REPORT_DIR", str(share_reports)) + monkeypatch.setenv("GAOKAO_ORDERS_FERNET_KEY", "test-secret-for-web-self-service") + monkeypatch.setenv("GAOKAO_JWT_SECRET", "x" * 64) + monkeypatch.setenv("GAOKAO_JWT_EXP_MIN", "5") + monkeypatch.setenv("GAOKAO_ADMIN_USER", "admin") + monkeypatch.setenv("GAOKAO_ADMIN_PASS", "test-pass-123") + monkeypatch.setenv("GAOKAO_PAYMENT_PROVIDER", "alipay") + monkeypatch.setenv("GAOKAO_PAYMENT_BASE_URL", "http://testserver") + monkeypatch.setenv("GAOKAO_PAYMENT_WEBHOOK_SECRET", "missing-real-provider-env") + + from admin.app import create_app + from admin.config import load_settings + + settings = load_settings() + app = create_app(settings) + + with TestClient(app) as client: + resp = client.post( + "/api/public/orders", + json={ + "service_version": "standard", + "amount_cents": 9900, + "customer_name": "张家长", + "customer_phone": "13800138000", + "candidate_province": "湖南", + }, + ) + + assert resp.status_code == 503, resp.text + assert "payment provider unavailable" in resp.text + with OrdersDAO.connect(settings.orders_db_path) as dao: + assert dao.count() == 0 + + +def test_public_create_order_returns_503_without_creating_orphan_order_when_checkout_fails( + tmp_path, monkeypatch +): + admin_db = tmp_path / "admin.db" + orders_db = tmp_path / "orders.db" + share_db = tmp_path / "share.db" + share_reports = tmp_path / "share_reports" + share_reports.mkdir() + + monkeypatch.setenv("GAOKAO_ENV", "dev") + monkeypatch.setenv("GAOKAO_DB_PATH", str(admin_db)) + monkeypatch.setenv("GAOKAO_ORDERS_DB_PATH", str(orders_db)) + monkeypatch.setenv("GAOKAO_SHARE_DB_PATH", str(share_db)) + monkeypatch.setenv("GAOKAO_SHARE_REPORT_DIR", str(share_reports)) + monkeypatch.setenv("GAOKAO_ORDERS_FERNET_KEY", "test-secret-for-web-self-service") + monkeypatch.setenv("GAOKAO_JWT_SECRET", "x" * 64) + monkeypatch.setenv("GAOKAO_JWT_EXP_MIN", "5") + monkeypatch.setenv("GAOKAO_ADMIN_USER", "admin") + monkeypatch.setenv("GAOKAO_ADMIN_PASS", "test-pass-123") + monkeypatch.setenv("GAOKAO_PAYMENT_PROVIDER", "mock") + monkeypatch.setenv("GAOKAO_PAYMENT_BASE_URL", "http://testserver") + monkeypatch.setenv("GAOKAO_PAYMENT_WEBHOOK_SECRET", "test-payment-secret") + + from admin.app import create_app + from admin.config import load_settings + from data.payments.service import PaymentError, PaymentService + + original = PaymentService.create_checkout + + def _boom(self, order_id: str, *, portal_token: str): + raise PaymentError("checkout transport failed") + + monkeypatch.setattr(PaymentService, "create_checkout", _boom) + + settings = load_settings() + app = create_app(settings) + + with TestClient(app) as client: + resp = client.post( + "/api/public/orders", + json={ + "service_version": "standard", + "amount_cents": 9900, + "customer_name": "张家长", + "customer_phone": "13800138000", + "candidate_province": "湖南", + }, + ) + + monkeypatch.setattr(PaymentService, "create_checkout", original) + assert resp.status_code == 503, resp.text + assert "payment checkout unavailable" in resp.text + with OrdersDAO.connect(settings.orders_db_path) as dao: + assert dao.count() == 0 diff --git a/data/orders/retention_cleanup.py b/data/orders/retention_cleanup.py index 1636753..5a18648 100644 --- a/data/orders/retention_cleanup.py +++ b/data/orders/retention_cleanup.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse import json from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta, timezone from data.orders.dao import OrdersDAO from data.orders.deletion_service import OrderDeletionService @@ -11,6 +11,8 @@ from data.orders.deletion_service import OrderDeletionService @dataclass class RetentionCleanupResult: + cutoff_iso: str + dry_run: bool scanned: int = 0 candidates: int = 0 anonymized: int = 0 @@ -53,11 +55,28 @@ def _iter_candidates(db_path: str, cutoff_iso: str) -> tuple[int, list[str]]: return scanned, candidates +def resolve_cutoff_iso( + *, cutoff_iso: str | None = None, retention_days: int | None = None +) -> str: + if cutoff_iso is not None: + datetime.fromisoformat(cutoff_iso) + return cutoff_iso + if retention_days is None or retention_days <= 0: + raise ValueError("retention_days must be > 0 when cutoff is not provided") + cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days) + return cutoff.replace(microsecond=0).isoformat() + + def run_cleanup( db_path: str, *, cutoff_iso: str, apply: bool ) -> RetentionCleanupResult: scanned, candidate_ids = _iter_candidates(db_path, cutoff_iso) - result = RetentionCleanupResult(scanned=scanned, candidates=len(candidate_ids)) + result = RetentionCleanupResult( + cutoff_iso=cutoff_iso, + dry_run=not apply, + scanned=scanned, + candidates=len(candidate_ids), + ) if not apply: return result service = OrderDeletionService.for_db(db_path) @@ -76,11 +95,16 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Run retention cleanup for old completed/refunded orders" ) - parser.add_argument( + cutoff_group = parser.add_mutually_exclusive_group(required=True) + cutoff_group.add_argument( "--cutoff", - required=True, help="ISO8601 cutoff; orders on or before this anchor are candidates", ) + cutoff_group.add_argument( + "--retention-days", + type=int, + help="Retention window in days; cutoff is computed as now - retention_days", + ) parser.add_argument("--dry-run", action="store_true") return parser @@ -92,7 +116,14 @@ def main(argv: list[str] | None = None, *, db_path: str | None = None) -> int: db_path = load_settings().orders_db_path parser = build_parser() args = parser.parse_args(argv) - result = run_cleanup(db_path, cutoff_iso=args.cutoff, apply=not args.dry_run) + try: + cutoff_iso = resolve_cutoff_iso( + cutoff_iso=args.cutoff, + retention_days=args.retention_days, + ) + except ValueError as exc: + parser.error(str(exc)) + result = run_cleanup(db_path, cutoff_iso=cutoff_iso, apply=not args.dry_run) print(json.dumps(result.__dict__, ensure_ascii=False, indent=2)) return 0 diff --git a/deploy/cron/gaokao-jobs.crontab b/deploy/cron/gaokao-jobs.crontab new file mode 100644 index 0000000..b9adc58 --- /dev/null +++ b/deploy/cron/gaokao-jobs.crontab @@ -0,0 +1,13 @@ +SHELL=/bin/bash +PATH=/usr/local/bin:/usr/bin:/bin + +GAOKAO_ENV=prod +GAOKAO_ORDERS_DB_PATH=/home/long/project/gaokao-volunteer-system/data/orders.db +GAOKAO_PYTHON_BIN=/home/long/project/gaokao-volunteer-system/.venv/bin/python +GAOKAO_DELIVERY_CHANNEL=station +GAOKAO_DELIVERY_LIMIT=100 +GAOKAO_RETENTION_DAYS=180 + +*/5 * * * * cd /home/long/project/gaokao-volunteer-system && "$GAOKAO_PYTHON_BIN" scripts/gaokao-delivery-dispatch.py --channel "$GAOKAO_DELIVERY_CHANNEL" --limit "$GAOKAO_DELIVERY_LIMIT" >> /var/log/gaokao-volunteer-system/delivery-dispatch.log 2>&1 +*/10 * * * * cd /home/long/project/gaokao-volunteer-system && "$GAOKAO_PYTHON_BIN" scripts/gaokao-delivery-watchdog.py --channel "$GAOKAO_DELIVERY_CHANNEL" --limit "$GAOKAO_DELIVERY_LIMIT" >> /var/log/gaokao-volunteer-system/delivery-watchdog.log 2>&1 +30 3 * * 0 cd /home/long/project/gaokao-volunteer-system && "$GAOKAO_PYTHON_BIN" scripts/gaokao-retention-cleanup.py --retention-days "$GAOKAO_RETENTION_DAYS" >> /var/log/gaokao-volunteer-system/retention-cleanup.log 2>&1 diff --git a/deploy/systemd/gaokao-delivery-dispatch.service b/deploy/systemd/gaokao-delivery-dispatch.service new file mode 100644 index 0000000..5a085c2 --- /dev/null +++ b/deploy/systemd/gaokao-delivery-dispatch.service @@ -0,0 +1,12 @@ +[Unit] +Description=Gaokao delivery dispatcher (report_ready -> sent/failed) +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/long/project/gaokao-volunteer-system +Environment=GAOKAO_PYTHON_BIN=/usr/bin/python3 +Environment=GAOKAO_DELIVERY_CHANNEL=station +Environment=GAOKAO_DELIVERY_LIMIT=100 +EnvironmentFile=-/home/long/project/gaokao-volunteer-system/deploy/systemd/gaokao-jobs.env +ExecStart=/bin/sh -lc 'exec "$GAOKAO_PYTHON_BIN" scripts/gaokao-delivery-dispatch.py --channel "$GAOKAO_DELIVERY_CHANNEL" --limit "$GAOKAO_DELIVERY_LIMIT"' diff --git a/deploy/systemd/gaokao-delivery-dispatch.timer b/deploy/systemd/gaokao-delivery-dispatch.timer new file mode 100644 index 0000000..c868062 --- /dev/null +++ b/deploy/systemd/gaokao-delivery-dispatch.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run gaokao delivery dispatcher every 5 minutes + +[Timer] +OnCalendar=*:0/5 +Persistent=true +RandomizedDelaySec=30 +Unit=gaokao-delivery-dispatch.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/gaokao-delivery-watchdog.service b/deploy/systemd/gaokao-delivery-watchdog.service new file mode 100644 index 0000000..2ff611f --- /dev/null +++ b/deploy/systemd/gaokao-delivery-watchdog.service @@ -0,0 +1,12 @@ +[Unit] +Description=Gaokao delivery watchdog (non-zero when dispatch sees failures) +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/long/project/gaokao-volunteer-system +Environment=GAOKAO_PYTHON_BIN=/usr/bin/python3 +Environment=GAOKAO_DELIVERY_CHANNEL=station +Environment=GAOKAO_DELIVERY_LIMIT=100 +EnvironmentFile=-/home/long/project/gaokao-volunteer-system/deploy/systemd/gaokao-jobs.env +ExecStart=/bin/sh -lc 'exec "$GAOKAO_PYTHON_BIN" scripts/gaokao-delivery-watchdog.py --channel "$GAOKAO_DELIVERY_CHANNEL" --limit "$GAOKAO_DELIVERY_LIMIT"' diff --git a/deploy/systemd/gaokao-delivery-watchdog.timer b/deploy/systemd/gaokao-delivery-watchdog.timer new file mode 100644 index 0000000..57175ea --- /dev/null +++ b/deploy/systemd/gaokao-delivery-watchdog.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run gaokao delivery watchdog every 10 minutes + +[Timer] +OnCalendar=*:0/10 +Persistent=true +RandomizedDelaySec=60 +Unit=gaokao-delivery-watchdog.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/gaokao-jobs.env.example b/deploy/systemd/gaokao-jobs.env.example new file mode 100644 index 0000000..57f96a6 --- /dev/null +++ b/deploy/systemd/gaokao-jobs.env.example @@ -0,0 +1,10 @@ +# 复制为 deploy/systemd/gaokao-jobs.env 后再按实际环境覆盖。 +# 目标：给 dispatcher / watchdog / retention cleanup 三个定时作业提供统一环境。 + +GAOKAO_ENV=prod +GAOKAO_ORDERS_DB_PATH=/home/long/project/gaokao-volunteer-system/data/orders.db +GAOKAO_PYTHON_BIN=/home/long/project/gaokao-volunteer-system/.venv/bin/python +GAOKAO_DELIVERY_CHANNEL=station +GAOKAO_DELIVERY_LIMIT=100 +GAOKAO_RETENTION_DAYS=180 +PYTHONUNBUFFERED=1 diff --git a/deploy/systemd/gaokao-retention-cleanup.service b/deploy/systemd/gaokao-retention-cleanup.service new file mode 100644 index 0000000..226e4e7 --- /dev/null +++ b/deploy/systemd/gaokao-retention-cleanup.service @@ -0,0 +1,11 @@ +[Unit] +Description=Gaokao retention cleanup (anonymize expired completed/refunded orders) +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/long/project/gaokao-volunteer-system +Environment=GAOKAO_PYTHON_BIN=/usr/bin/python3 +Environment=GAOKAO_RETENTION_DAYS=180 +EnvironmentFile=-/home/long/project/gaokao-volunteer-system/deploy/systemd/gaokao-jobs.env +ExecStart=/bin/sh -lc 'exec "$GAOKAO_PYTHON_BIN" scripts/gaokao-retention-cleanup.py --retention-days "$GAOKAO_RETENTION_DAYS"' diff --git a/deploy/systemd/gaokao-retention-cleanup.timer b/deploy/systemd/gaokao-retention-cleanup.timer new file mode 100644 index 0000000..67d656a --- /dev/null +++ b/deploy/systemd/gaokao-retention-cleanup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run gaokao retention cleanup every Sunday at 03:30 UTC + +[Timer] +OnCalendar=Sun 03:30 +Persistent=true +RandomizedDelaySec=300 +Unit=gaokao-retention-cleanup.service + +[Install] +WantedBy=timers.target diff --git a/docs/ACTIVE_REMEDIATION_2026-06-13.md b/docs/ACTIVE_REMEDIATION_2026-06-13.md index 8f8e4d7..90c1944 100644 --- a/docs/ACTIVE_REMEDIATION_2026-06-13.md +++ b/docs/ACTIVE_REMEDIATION_2026-06-13.md @@ -34,26 +34,34 @@ #### A-1 用户端 Web 自助闭环未完成 严重度: P0（面向 Web 自助产品时） -当前状态: 未解决 +当前状态: 部分解决，仍未整体收口 范围归属: T12（后续阶段） +已完成： + +- 前台公开入口 `/` / `/pricing` / `/checkout/{service_version}` +- 公开下单与 portal token 链路 +- 支付后资料填写草稿/提交 +- 站内状态页 / 报告查看 / PDF 下载最小闭环 +- 后台订单列表/详情可见资料提交状态 + 仍缺： -- 前台套餐页完整交互 -- 真实支付接入与回调验签 -- 前台资料填写闭环 -- 站内交付 / 订单状态页 -- 自助购买后自动处理主链路 +- 前台套餐页更完整交互与产品化打磨 +- 真实支付接入与回调验签的线上 acceptance +- `info_submitted -> serving` 自动处理主链 +- 完整站内通知/交付产品化 +- 自助购买后自动处理主链路的生产化闭环 说明： -- 这是“当前仍然有效的问题”，但**不应再作为对 v2.1 已完成范围的否定证据**。 -- 正确表述应为：`v2.1 完成了人工服务运营闭环，T12 承接 Web 自助闭环`。 +- 当前不是“完全没做 T12”，而是“本地 MVP 主链大部分已落地，但整体仍未到线上可验收完成”。 +- 正确表述应为：`v2.1 完成了人工服务运营闭环，T12 已形成本地 MVP 主链，但线上商业闭环仍未完成`。 建议动作： - 以 T12 为唯一主线推进，不在旧报告中分散描述 -- 为 T12 单独维护实施计划、验收标准和完成报告 +- 后续优先收 `自动处理主链` 与 `真实支付 acceptance` #### A-2 支付、退款、对账闭环未完成 @@ -100,11 +108,12 @@ - 站内查看 + PDF 下载最小交付闭环 - `DeliveryDispatcher` + `scripts/gaokao-delivery-dispatch.py` - `ready -> sent` / 缺文件 `-> failed` 的最小执行链 +- dispatcher/watchdog runbook + systemd/cron 样例（`docs/DELIVERY_RETENTION_OPS_RUNBOOK.md` / `deploy/systemd` / `deploy/cron`） 仍缺： - 真实邮件/站内通知发送执行器 -- 自动重试调度 / 告警链 +- 自动重试调度 / 告警链（watchdog/systemd/cron 样例已补，生产安装未完成） - 面向用户的独立通知审计页 - 对账/退款与交付失败补偿联动 @@ -137,12 +146,12 @@ - 正式法务审定版本 - 前台/客服删除工单流程 -- 数据保留期自动清理定时任务（脚本已具备，调度未接入） +- 目标生产主机上的自动清理实际安装/留痕（仓库已提供 runbook + systemd/cron 样例） - 合规文本上线前最终校对 说明： -- 当前不再是“完全没有合规基线”，而是“文档基线 + 后台最小执行入口 + retention cleanup 脚本已建，前台流程与自动调度仍待完成”。 +- 当前不再是“完全没有合规基线”，而是“文档基线 + 后台最小执行入口 + retention cleanup 脚本与调度样例已建，前台流程与生产安装留痕仍待完成”。 #### A-5 业务数据备份/恢复/密钥托管已形成基线，但生产化仍未完结 @@ -154,17 +163,22 @@ - `docs/BACKUP_AND_RECOVERY_PLAN.md` - `docs/KEY_MANAGEMENT_BASELINE.md` -- `scripts/backup_verify.sh` 本地恢复验证入口 +- `docs/plans/P1-8-backup-restore-drill.md` +- `scripts/backup_snapshot.sh` 最小目录快照入口 +- `scripts/backup_verify.sh` 本地恢复校验入口 +- `scripts/backup_restore_smoke.py` 恢复副本最小服务 smoke +- `ops/cron/gaokao-backup.crontab.example` +- `ops/systemd/gaokao-backup*.service|timer` 定时接入口径示例 仍缺： - 异机/异地备份落地 -- 定时备份与恢复演练自动化 +- 目标主机上的 cron/systemd 实际安装、日志与告警验收 - 密钥轮换的真实执行记录 说明： -- 当前不再是“没有备份恢复方案”，而是“已有本地方案与验证脚本，但生产化运维链未闭环”。 +- 当前不再是“只有文档没有验证”，而是“已有本地快照、完整性校验、restore smoke 和定时接入口径样例，但生产化运维链未闭环”。 --- diff --git a/docs/BACKUP_AND_RECOVERY_PLAN.md b/docs/BACKUP_AND_RECOVERY_PLAN.md index eb77529..fad3460 100644 --- a/docs/BACKUP_AND_RECOVERY_PLAN.md +++ b/docs/BACKUP_AND_RECOVERY_PLAN.md @@ -1,7 +1,7 @@ # BACKUP_AND_RECOVERY_PLAN 最后更新: 2026-06-14 -状态: 最小恢复基线（非生产级高可用） +状态: 最小恢复基线（已具备本地快照 + 完整性校验 + restore smoke） ## 1. 目标 @@ -11,77 +11,150 @@ - 报告 HTML/PDF 文件 - 上传/分享产物 - 运行所需关键配置与密钥映射关系 +- 本地可执行的备份、校验、恢复演练入口 ## 2. RPO / RTO 建议 - RPO: 24 小时内 - RTO: 4 小时内 -说明：当前是轻量单机 MVP，不承诺秒级恢复。 +说明：当前是轻量单机 MVP，不承诺秒级恢复，也不等同于生产级多机容灾。 ## 3. 需要备份的对象 1. 数据库 - - 管理库 - - 订单库 - - 分享库 + - 管理库 `data/orders/admin.db` + - 订单库 `data/orders.db` + - 分享库 `data/share/short_links.db` 2. 文件目录 - - 报告输出目录 - - 分享报告目录 - - 未来上传文件目录 + - 分享/公开报告目录 + - 报告 HTML/PDF 产物 + - `data/examples` 中的样例交付物（用于恢复 smoke） 3. 配置与运维资料 - `.env` 实际部署值（不入 Git） - - systemd/docker-compose 部署配置 + - systemd / docker-compose 部署配置 4. 密钥资料 - JWT secret - Fernet key - 支付相关密钥/证书 -## 4. 最小备份策略 +## 4. 当前已落地产物 + +- `scripts/backup_snapshot.sh` + - 生成时间戳快照目录 + - 复制 DB / 文件 / 可选 config / 可选 secrets + - 生成 `manifest.json` + - 按 `GAOKAO_BACKUP_KEEP` 保留最近 N 份 +- `scripts/backup_verify.sh` + - 可校验 live staging 或已有快照 + - 校验 SQLite 可读性 + - 校验 `manifest.json` 哈希完整性 + - 调用 restore smoke +- `scripts/backup_restore_smoke.py` + - 在恢复副本上启动 FastAPI `TestClient` + - 走 `/health`、portal 状态页、报告 HTML、PDF 下载最小链路 + - 只改临时恢复副本，不改原始快照 +- 定时接入口径示例 + - `ops/cron/gaokao-backup.crontab.example` + - `ops/systemd/gaokao-backup.service` + - `ops/systemd/gaokao-backup.timer` + - `ops/systemd/gaokao-backup-verify.service` + - `ops/systemd/gaokao-backup-verify.timer` + +## 5. 最小备份策略 ### 数据库 -- 每日一次离线或冷备 -- 备份文件命名包含时间戳 -- 保留最近 7 份日备 +- 每日一次目录快照 +- 文件名包含 UTC 时间戳 +- 保留最近 7 份日备（可由 `GAOKAO_BACKUP_KEEP` 调整） ### 文件目录 -- 每日一次目录归档 -- 保留最近 7 份日备 +- 与数据库同批次快照 +- 报告/分享目录和样例报告分开保存，避免后续排障时混淆 -### 密钥与配置 +### 配置与密钥 -- 单独存放，不与业务数据库混包 +- `.env`、部署配置、密钥目录不与业务数据库混写到同一路径 +- 通过环境变量显式声明： + - `GAOKAO_BACKUP_ENV_FILE` + - `GAOKAO_BACKUP_CONFIG_DIR` + - `GAOKAO_BACKUP_SECRETS_DIR` - 只允许受控人员访问 -## 5. 恢复步骤（最小版） +## 6. 最小 runbook（本地可执行） -1. 准备干净工作目录 -2. 恢复 SQLite 数据库文件 -3. 恢复报告/分享文件目录 -4. 恢复环境变量与密钥 -5. 启动服务 -6. 验证： - - 后台能读订单 - - portal 状态页可打开 - - 已交付报告可下载 +### 6.1 生成快照 -## 6. 当前缺口 +```bash +bash scripts/backup_snapshot.sh /tmp/gaokao-backups +``` -- 尚无自动化备份脚本 -- 尚无定时任务 -- 尚无备份完整性校验 +可选环境变量： -## 7. MVP 上线前最低要求 +```bash +export GAOKAO_BACKUP_KEEP=7 +export GAOKAO_BACKUP_ENV_FILE=/etc/gaokao/.env +export GAOKAO_BACKUP_CONFIG_DIR=/etc/gaokao +export GAOKAO_BACKUP_SECRETS_DIR=/var/lib/gaokao/secrets +``` -1. 至少完成一次手工备份演练 -2. 至少在临时目录恢复一次可读数据库 +### 6.2 校验最近一份快照 + +```bash +bash scripts/backup_verify.sh --from-backup /tmp/gaokao-backups/backup- +``` + +校验内容： + +1. `manifest.json` 哈希一致 +2. SQLite 文件可打开、表可枚举 +3. 恢复副本可跑最小服务 smoke： + - `/health` + - `/portal/{token}/status` + - `/portal/{token}/report` + - `/portal/{token}/report.pdf` + +### 6.3 从 live 数据直接做一次恢复演练 + +```bash +bash scripts/backup_verify.sh +``` + +该命令会把当前 live DB / 文件复制到临时目录后执行同样的校验链。 + +## 7. 定时接入口径 + +### cron 示例 + +见：`ops/cron/gaokao-backup.crontab.example` + +- 每天 02:30 生成快照 +- 每周一 03:00 对最近一份快照执行 restore smoke + +### systemd timer 示例 + +见： + +- `ops/systemd/gaokao-backup.service` +- `ops/systemd/gaokao-backup.timer` +- `ops/systemd/gaokao-backup-verify.service` +- `ops/systemd/gaokao-backup-verify.timer` + +说明：这些文件只是仓库内口径样例，不代表目标主机已经安装启用。 + +## 8. MVP 上线前最低要求 + +1. 至少完成一次真实快照生成 +2. 至少完成一次 `backup_verify.sh --from-backup ...` 演练 3. 明确密钥存放位置和负责人 4. 不得再使用“Git 备份即可恢复系统”作为对外或内部结论 -## 8. 当前已落地产物 +## 9. 当前仍未闭环的外部/后续缺口 -- `scripts/backup_verify.sh` - - 可复制当前 SQLite 数据库与示例报告到临时目录 - - 可验证复制后的数据库可读、表可枚举、报告文件可见 +- 异机 / 异地备份尚未落地 +- 目标主机上的 cron / systemd timer 尚未实际安装验收 +- 备份失败告警链（邮件 / IM / 监控）未接入 +- 密钥轮换与应急恢复仍缺真实执行记录 +- 当前 restore smoke 证明的是“最小服务链可用”，不是完整生产级全链恢复 diff --git a/docs/CURRENT_STATE.md b/docs/CURRENT_STATE.md index d18b81e..9279570 100644 --- a/docs/CURRENT_STATE.md +++ b/docs/CURRENT_STATE.md @@ -68,14 +68,14 @@ 未完成主链路包括： -- 用户端落地页 / 套餐页完整交互 -- 真实支付接入与回调验签 -- 前台资料填写闭环 -- 站内交付 / 前台订单状态页完整闭环 +- 真实支付接入与回调验签的线上 acceptance +- `info_submitted -> serving` 自动处理主链 +- 完整交付通知/发送器与调度 +- 前台删除工单流程与更完整的产品化交互 因此： -- Web 自助购买场景 = 未完成 +- Web 自助购买场景 = **本地 MVP 主链已大部分落地，但线上闭环未完成** - 完整教育 SaaS 化 = 未完成 --- @@ -107,7 +107,7 @@ - 用户端 Web 自助支付闭环缺失 - 邮件 / 站内自动交付尚未形成当前主系统标准能力 - 上传入口仍以后台 / CLI / 内部入口为主，用户前台入口不足 -- 业务数据备份 / 恢复 / 密钥托管方案仍不足 +- 业务数据备份 / 恢复 / 密钥托管已有本地基线与验证，但异机备份和生产接入仍不足 - 隐私政策 / 服务协议 / 监护人同意 / 数据保留与删除流程未形成正式产品闭环 ### P2 工程改进 diff --git a/docs/DATA_RETENTION_AND_DELETION.md b/docs/DATA_RETENTION_AND_DELETION.md index 95d64f8..c428548 100644 --- a/docs/DATA_RETENTION_AND_DELETION.md +++ b/docs/DATA_RETENTION_AND_DELETION.md @@ -44,12 +44,16 @@ - 删除时自动清理 HTML/PDF 报告文件 - `order_deletion_audits` 最小审计表 - `scripts/gaokao_retention_cleanup.py` / `scripts/gaokao-retention-cleanup.py` 保留期清理入口 +- `scripts/gaokao-retention-cleanup.py --retention-days 180` 最小定时调度入口 +- `docs/DELIVERY_RETENTION_OPS_RUNBOOK.md` +- `deploy/systemd/gaokao-retention-cleanup.{service,timer}` +- `deploy/cron/gaokao-jobs.crontab` 尚缺： - 前台/客服删除工单流程 -- 数据保留期自动清理定时任务 - 更细粒度的匿名化策略（如案例长期保留脱敏版） +- 目标生产主机上的实际安装/值班留痕（仓库内只有调度样例，不代表已部署） ## 5. MVP 最低执行要求 @@ -59,3 +63,14 @@ 2. 内部 runbook 明确谁来删、删哪些表和文件 3. 删除后要复核文件路径已不存在 4. 不能把“仅删数据库”误报成“数据已全部删除” + +## 6. 最小调度与 runbook + +仓库当前给出的最小执行口径： + +- 人工指定 cutoff：`python3 scripts/gaokao-retention-cleanup.py --cutoff --dry-run` +- 定时调度：`python3 scripts/gaokao-retention-cleanup.py --retention-days 180` +- runbook：`docs/DELIVERY_RETENTION_OPS_RUNBOOK.md` +- systemd/cron 样例：`deploy/systemd/gaokao-retention-cleanup.{service,timer}` / `deploy/cron/gaokao-jobs.crontab` + +真相边界：当前保留期作业执行的是后台匿名化，不应误报成“用户删除请求流程已经完整上线”。 diff --git a/docs/DELIVERY_RETENTION_OPS_RUNBOOK.md b/docs/DELIVERY_RETENTION_OPS_RUNBOOK.md new file mode 100644 index 0000000..fc64494 --- /dev/null +++ b/docs/DELIVERY_RETENTION_OPS_RUNBOOK.md @@ -0,0 +1,130 @@ +# DELIVERY_RETENTION_OPS_RUNBOOK + +最后更新: 2026-06-14 + +## 1. 当前真相 + +### delivery dispatcher / watchdog + +当前已落仓并可接定时任务的事实： + +- 执行脚本：`scripts/gaokao-delivery-dispatch.py` +- 巡检脚本：`scripts/gaokao-delivery-watchdog.py` +- systemd 样例：`deploy/systemd/gaokao-delivery-dispatch.{service,timer}` / `deploy/systemd/gaokao-delivery-watchdog.{service,timer}` +- cron 样例：`deploy/cron/gaokao-jobs.crontab` + +注意边界： + +- 当前 `sent` 仍表示“站内交付物校验通过并推进事件状态”，**不是** 邮件/微信等外部渠道真实发送成功。 +- dispatcher 每次会处理 `ready` 与 `failed` 事件；如果缺少 HTML/PDF，会把事件记为 `failed`，并递增 `attempt_count`。 +- watchdog 复用同一 dispatch 路径，但只要本轮出现失败事件就返回 exit code `2`，适合接 systemd `OnFailure=` 或外部告警。 + +### retention cleanup + +当前已落仓并可接定时任务的事实： + +- 手工入口：`scripts/gaokao-retention-cleanup.py --cutoff [--dry-run]` +- 定时入口：`scripts/gaokao-retention-cleanup.py --retention-days 180 [--dry-run]` +- 兼容别名：`scripts/gaokao_retention_cleanup.py` +- systemd 样例：`deploy/systemd/gaokao-retention-cleanup.{service,timer}` +- cron 样例：`deploy/cron/gaokao-jobs.crontab` + +注意边界： + +- 当前清理动作是 **匿名化**（`anonymize_order`），不是物理删除。 +- 只处理 `completed/refunded` 且锚点时间早于 cutoff 的订单。 +- 它会清理订单侧敏感字段并写审计，但不等于“前台删除工单流程已上线”。 + +## 2. 统一环境配置 + +建议先复制： + +```bash +cp deploy/systemd/gaokao-jobs.env.example deploy/systemd/gaokao-jobs.env +``` + +最少确认以下变量： + +- `GAOKAO_ORDERS_DB_PATH`：生产订单 DB 路径 +- `GAOKAO_PYTHON_BIN`：建议指向项目 `.venv/bin/python` +- `GAOKAO_DELIVERY_CHANNEL`：当前默认 `station` +- `GAOKAO_DELIVERY_LIMIT`：单次 dispatch/watchdog 扫描上限 +- `GAOKAO_RETENTION_DAYS`：默认 `180` + +## 3. 手工执行口径 + +```bash +# 主动推进 ready/failed 事件 +python3 scripts/gaokao-delivery-dispatch.py --channel station --limit 100 + +# 巡检：有失败时返回 2 +python3 scripts/gaokao-delivery-watchdog.py --channel station --limit 100 + +# retention dry-run（人工指定 cutoff） +python3 scripts/gaokao-retention-cleanup.py --cutoff 2025-12-31T00:00:00+00:00 --dry-run + +# retention 定时模式（自动计算 cutoff） +python3 scripts/gaokao-retention-cleanup.py --retention-days 180 +``` + +退出码： + +- dispatcher：`0` = 脚本执行完成（结果看 JSON） +- watchdog：`0` = 本轮无失败；`2` = 本轮发现失败事件 +- retention cleanup：`0` = 脚本执行完成；异常参数或运行错误会非 0 退出 + +## 4. systemd 安装口径 + +```bash +sudo cp deploy/systemd/gaokao-*.service /etc/systemd/system/ +sudo cp deploy/systemd/gaokao-*.timer /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now gaokao-delivery-dispatch.timer +sudo systemctl enable --now gaokao-delivery-watchdog.timer +sudo systemctl enable --now gaokao-retention-cleanup.timer +``` + +验证： + +```bash +systemctl list-timers 'gaokao-*' +systemctl start gaokao-delivery-dispatch.service +systemctl status gaokao-delivery-watchdog.service --no-pager +journalctl -u gaokao-retention-cleanup.service -n 50 --no-pager +``` + +说明： + +- watchdog 的 exit code `2` 会让对应 service 进入 failed，便于接 `OnFailure=` 或宿主机监控。 +- 如果部署路径不是当前仓库路径，先修改 unit 里的 `WorkingDirectory` 与 `EnvironmentFile`。 + +## 5. cron 备用口径 + +`deploy/cron/gaokao-jobs.crontab` 给出最小样例，导入前先确保： + +1. `/var/log/gaokao-volunteer-system/` 已创建且可写； +2. `GAOKAO_PYTHON_BIN` 指向安装完依赖的解释器； +3. crontab 中仓库路径与真实部署一致。 + +导入示例： + +```bash +crontab deploy/cron/gaokao-jobs.crontab +crontab -l +``` + +## 6. 推荐节奏 + +- dispatcher：每 5 分钟 +- watchdog：每 10 分钟 +- retention cleanup：每周一次（周日 03:30 UTC） + +## 7. 仍未收口的缺口 + +这些仍然没有被本次 runbook 伪装成“已完成”： + +1. `sent` 语义仍偏向“站内交付校验完成”，不是外部渠道真实投递成功。 +2. 尚无邮件/微信真实发送执行器。 +3. watchdog 只有失败退出码，还没有仓库内置的告警推送集成。 +4. retention cleanup 只是后台匿名化作业，前台/客服删除工单流程仍未上线。 +5. 这些 unit/timer/cron 样例已落仓，但是否真正安装到目标生产主机，需要部署时另行执行并留存记录。 diff --git a/docs/DELIVERY_SERVICE_DESIGN.md b/docs/DELIVERY_SERVICE_DESIGN.md index 12b86c0..ab133b3 100644 --- a/docs/DELIVERY_SERVICE_DESIGN.md +++ b/docs/DELIVERY_SERVICE_DESIGN.md @@ -54,13 +54,17 @@ - `delivered` 但无交付物时不再误报 `report_ready` - `data.notifications.dispatcher.DeliveryDispatcher` - `scripts/gaokao-delivery-dispatch.py` +- `scripts/gaokao-delivery-watchdog.py`（失败返回 exit code `2`） - `ready -> sent` / `缺文件 -> failed` / `attempt_count` 递增 +- `docs/DELIVERY_RETENTION_OPS_RUNBOOK.md` +- `deploy/systemd/gaokao-delivery-{dispatch,watchdog}.{service,timer}` +- `deploy/cron/gaokao-jobs.crontab` 未完成： - 独立 `delivery_job` / `delivery_attempt` 模型 - 邮件/渠道真实发送执行器 -- 自动重试调度与告警链 +- 告警推送集成（当前只有 watchdog 非 0 退出码） - 多通道统一投递状态页 ## 6. 当前最短闭环 @@ -72,10 +76,21 @@ 5. `report_ready` 事件落库且幂等 6. `gaokao-delivery-dispatch.py --channel station` 可把 ready 事件推进到 sent 7. 缺失交付物时事件转 failed，并记录 `failure_reason` +8. 可通过 systemd timer / cron 周期执行 dispatcher，并用 watchdog 暴露失败退出码 ## 7. 下一步实施建议 -1. 把 dispatcher 接入定时任务或 systemd timer -2. 增加邮件或站内通知真实发送器（二选一先闭环） -3. 增加失败重试阈值与告警 +1. 增加邮件或站内通知真实发送器（二选一先闭环） +2. 增加失败重试阈值与告警推送 +3. 把 `sent` 语义拆成“可投递校验通过”与“真实渠道已发送” 4. 再决定是否扩到多通道 + +## 8. 生产化接入口径 + +仓库内已提供最小 runbook 与调度样例： + +- runbook：`docs/DELIVERY_RETENTION_OPS_RUNBOOK.md` +- systemd：`deploy/systemd/gaokao-delivery-dispatch.{service,timer}` / `deploy/systemd/gaokao-delivery-watchdog.{service,timer}` +- cron：`deploy/cron/gaokao-jobs.crontab` + +注意：当前 watchdog 仍是“带失败退出码的 dispatch 巡检”，并不等于独立通知告警系统。 diff --git a/docs/T12_ACCEPTANCE_CRITERIA.md b/docs/T12_ACCEPTANCE_CRITERIA.md index 7bb2dbc..49e4f5f 100644 --- a/docs/T12_ACCEPTANCE_CRITERIA.md +++ b/docs/T12_ACCEPTANCE_CRITERIA.md @@ -66,7 +66,7 @@ T12 的验收对象不是后台运营链路，而是用户端 Web 自助主链 1. 真实支付 provider 未完成且目标口径要求真实支付 2. 交付事件仅停留在路由级补丁，未形成稳定主链能力 3. 前台仅有文案，无同意字段落库 -4. 备份恢复只有文档，没有最小验证 +4. 备份恢复仍没有最小快照+restore smoke，或虽有脚本但未形成可验证基线 --- diff --git a/docs/plans/P1-8-backup-restore-drill.md b/docs/plans/P1-8-backup-restore-drill.md new file mode 100644 index 0000000..eb40389 --- /dev/null +++ b/docs/plans/P1-8-backup-restore-drill.md @@ -0,0 +1,61 @@ +# P1-8 backup restore drill + +最后更新: 2026-06-14 +状态: 本地最小演练方案已落地，生产主机安装/告警仍待外部收口 + +## 1. 目标 + +把“只能证明文件可复制”的状态，提升到“可对快照执行 restore smoke，验证最小服务链可用”。 + +## 2. 本次落地范围 + +- 新增 `scripts/backup_snapshot.sh` +- 扩展 `scripts/backup_verify.sh` +- 新增 `scripts/backup_restore_smoke.py` +- 补 `ops/cron` 与 `ops/systemd` 示例接入口径 +- 补 pytest 验证用例 `tests/test_backup_workflow.py` + +## 3. 演练步骤 + +### 3.1 生成快照 + +```bash +bash scripts/backup_snapshot.sh /tmp/gaokao-backups +``` + +### 3.2 对快照执行恢复校验 + +```bash +bash scripts/backup_verify.sh --from-backup /tmp/gaokao-backups/backup- +``` + +### 3.3 校验成功标准 + +- `manifest_ok` 输出存在 +- SQLite 文件输出 `sqlite_ok` +- restore smoke JSON 至少包含： + - `health_status = 200` + - `portal_status = 200` + - `portal_report = 200` + - `portal_pdf = 200` + +## 4. 设计约束 + +1. restore smoke 只在临时恢复副本上写入测试订单，不改原始快照 +2. 若仓库内不存在现成交付物，smoke 允许在恢复副本下生成极小 HTML/PDF 占位物 +3. 本方案不宣称“完整生产恢复完成”，只证明最小服务链可被恢复副本支撑 + +## 5. 非目标 + +- 异地备份 +- 主机级定时器安装 +- 失败告警链 +- 密钥轮换真实执行记录 +- 完整支付/退款/交付全链恢复 + +## 6. 后续外部动作 + +- 在目标主机安装 cron 或 systemd timer +- 将备份日志接入监控/通知 +- 形成季度恢复演练记录 +- 补密钥轮换 runbook 与演练记录 diff --git a/ops/cron/gaokao-backup.crontab.example b/ops/cron/gaokao-backup.crontab.example new file mode 100644 index 0000000..01c0975 --- /dev/null +++ b/ops/cron/gaokao-backup.crontab.example @@ -0,0 +1,19 @@ +# 安装前请替换仓库路径、日志路径和备份根目录。 +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +GAOKAO_BACKUP_ROOT=/var/backups/gaokao +GAOKAO_BACKUP_KEEP=7 +GAOKAO_DB_PATH=/var/lib/gaokao/admin.db +GAOKAO_ORDERS_DB_PATH=/var/lib/gaokao/orders.db +GAOKAO_SHARE_DB_PATH=/var/lib/gaokao/share/short_links.db +GAOKAO_SHARE_REPORT_DIR=/var/lib/gaokao/share/reports +GAOKAO_BACKUP_ENV_FILE=/etc/gaokao/.env +GAOKAO_BACKUP_CONFIG_DIR=/etc/gaokao +GAOKAO_BACKUP_SECRETS_DIR=/var/lib/gaokao/secrets + +# 每天 02:30 生成一份目录快照 +30 2 * * * cd /srv/gaokao-volunteer-system && bash scripts/backup_snapshot.sh >> /var/log/gaokao-backup.log 2>&1 + +# 每周一 03:00 对最近一份快照执行 restore smoke（在临时目录，不改原快照） +0 3 * * 1 cd /srv/gaokao-volunteer-system && latest=$(find "$GAOKAO_BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d -name 'backup-*' | sort | tail -n 1) && test -n "$latest" && bash scripts/backup_verify.sh --from-backup "$latest" >> /var/log/gaokao-backup-verify.log 2>&1 diff --git a/ops/systemd/gaokao-backup-verify.service b/ops/systemd/gaokao-backup-verify.service new file mode 100644 index 0000000..780f760 --- /dev/null +++ b/ops/systemd/gaokao-backup-verify.service @@ -0,0 +1,10 @@ +[Unit] +Description=Gaokao volunteer system backup restore smoke +Wants=network-online.target +After=network-online.target gaokao-backup.service + +[Service] +Type=oneshot +WorkingDirectory=/srv/gaokao-volunteer-system +Environment=GAOKAO_BACKUP_ROOT=/var/backups/gaokao +ExecStart=/bin/bash -lc 'latest=$(find "$GAOKAO_BACKUP_ROOT" -mindepth 1 -maxdepth 1 -type d -name "backup-*" | sort | tail -n 1); test -n "$latest"; bash scripts/backup_verify.sh --from-backup "$latest"' diff --git a/ops/systemd/gaokao-backup-verify.timer b/ops/systemd/gaokao-backup-verify.timer new file mode 100644 index 0000000..d0e6c87 --- /dev/null +++ b/ops/systemd/gaokao-backup-verify.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run gaokao backup restore smoke weekly + +[Timer] +OnCalendar=Mon 03:00:00 +Persistent=true +Unit=gaokao-backup-verify.service + +[Install] +WantedBy=timers.target diff --git a/ops/systemd/gaokao-backup.service b/ops/systemd/gaokao-backup.service new file mode 100644 index 0000000..536725f --- /dev/null +++ b/ops/systemd/gaokao-backup.service @@ -0,0 +1,18 @@ +[Unit] +Description=Gaokao volunteer system backup snapshot +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/srv/gaokao-volunteer-system +Environment=GAOKAO_BACKUP_ROOT=/var/backups/gaokao +Environment=GAOKAO_BACKUP_KEEP=7 +Environment=GAOKAO_DB_PATH=/var/lib/gaokao/admin.db +Environment=GAOKAO_ORDERS_DB_PATH=/var/lib/gaokao/orders.db +Environment=GAOKAO_SHARE_DB_PATH=/var/lib/gaokao/share/short_links.db +Environment=GAOKAO_SHARE_REPORT_DIR=/var/lib/gaokao/share/reports +Environment=GAOKAO_BACKUP_ENV_FILE=/etc/gaokao/.env +Environment=GAOKAO_BACKUP_CONFIG_DIR=/etc/gaokao +Environment=GAOKAO_BACKUP_SECRETS_DIR=/var/lib/gaokao/secrets +ExecStart=/bin/bash scripts/backup_snapshot.sh diff --git a/ops/systemd/gaokao-backup.timer b/ops/systemd/gaokao-backup.timer new file mode 100644 index 0000000..166f13d --- /dev/null +++ b/ops/systemd/gaokao-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run gaokao backup snapshot daily + +[Timer] +OnCalendar=*-*-* 02:30:00 +Persistent=true +Unit=gaokao-backup.service + +[Install] +WantedBy=timers.target diff --git a/scripts/backup_restore_smoke.py b/scripts/backup_restore_smoke.py new file mode 100644 index 0000000..8247d2b --- /dev/null +++ b/scripts/backup_restore_smoke.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import sys +import uuid +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator + +ROOT_DIR = Path(__file__).resolve().parents[1] +if str(ROOT_DIR) not in sys.path: + sys.path.insert(0, str(ROOT_DIR)) + +DEFAULT_JWT_SECRET = "backup-restore-smoke-secret-0123456789abcdef0123456789abcdef" +DEFAULT_FERNET_SECRET = "backup-restore-smoke-fernet-secret" +DEFAULT_ADMIN_PASSWORD = "backup-restore-pass-123" + + +def _choose_restore_file(backup_dir: Path, suffix: str) -> Path | None: + candidates = sorted( + path + for path in (backup_dir / "files").rglob(f"*{suffix}") + if path.is_file() and "__pycache__" not in path.parts + ) + return candidates[0] if candidates else None + + +def _ensure_smoke_artifacts(backup_dir: Path) -> tuple[Path, Path]: + smoke_dir = backup_dir / "files" / "_restore_smoke" + smoke_dir.mkdir(parents=True, exist_ok=True) + + html = _choose_restore_file(backup_dir, ".html") + pdf = _choose_restore_file(backup_dir, ".pdf") + + if html is None: + html = smoke_dir / "restore-smoke-report.html" + html.write_text( + "

    备份恢复演练报告

    restore smoke generated

    ", + encoding="utf-8", + ) + if pdf is None: + pdf = smoke_dir / "restore-smoke-report.pdf" + pdf.write_bytes(b"%PDF-1.4\nrestore-smoke\n") + return html, pdf + + +@contextmanager +def _patched_env(overrides: dict[str, str]) -> Iterator[None]: + original = {key: os.environ.get(key) for key in overrides} + try: + for key, value in overrides.items(): + os.environ[key] = value + yield + finally: + for key, old_value in original.items(): + if old_value is None: + os.environ.pop(key, None) + else: + os.environ[key] = old_value + + +def run_restore_smoke(backup_dir: str | Path) -> dict[str, object]: + from fastapi.testclient import TestClient + + from admin.app import create_app + from admin.config import load_settings + from data.customer_portal.token import issue_portal_token + from data.orders.dao import OrdersDAO + from data.orders.models import Order + from data.payments.service import PaymentService + + root = Path(backup_dir).resolve() + if not root.is_dir(): + raise FileNotFoundError(f"backup dir not found: {root}") + + db_dir = root / "db" + admin_db = db_dir / "admin.db" + orders_db = db_dir / "orders.db" + share_db = db_dir / "short_links.db" + share_report_dir = root / "files" / "reports" + if not share_report_dir.exists(): + share_report_dir = root / "files" + + html_path, pdf_path = _ensure_smoke_artifacts(root) + order_id = f"GKO-BACKUP-SMOKE-{uuid.uuid4().hex[:8].upper()}" + + overrides = { + "GAOKAO_ENV": os.environ.get("GAOKAO_ENV", "dev"), + "GAOKAO_DB_PATH": str(admin_db), + "GAOKAO_ORDERS_DB_PATH": str(orders_db), + "GAOKAO_SHARE_DB_PATH": str(share_db), + "GAOKAO_SHARE_REPORT_DIR": str(share_report_dir), + "GAOKAO_ORDERS_FERNET_KEY": os.environ.get( + "GAOKAO_ORDERS_FERNET_KEY", DEFAULT_FERNET_SECRET + ), + "GAOKAO_JWT_SECRET": os.environ.get("GAOKAO_JWT_SECRET", DEFAULT_JWT_SECRET), + "GAOKAO_ADMIN_USER": os.environ.get("GAOKAO_ADMIN_USER", "admin"), + "GAOKAO_ADMIN_PASS": os.environ.get( + "GAOKAO_ADMIN_PASS", DEFAULT_ADMIN_PASSWORD + ), + "GAOKAO_PAYMENT_PROVIDER": "mock", + "GAOKAO_PAYMENT_BASE_URL": "http://testserver", + "GAOKAO_PAYMENT_WEBHOOK_SECRET": os.environ.get( + "GAOKAO_PAYMENT_WEBHOOK_SECRET", "backup-restore-smoke-payment-secret" + ), + } + + with _patched_env(overrides): + settings = load_settings() + app = create_app(settings) + with TestClient(app) as client: + health = client.get("/health") + if health.status_code != 200: + raise RuntimeError(f"health failed: {health.status_code} {health.text}") + + order = Order( + id=order_id, + source="web", + service_version="standard", + amount_cents=9900, + status="pending", + customer_name="备份演练家长", + customer_phone="13800138000", + candidate_name="恢复演练考生", + candidate_province="湖南", + ) + with OrdersDAO.connect(settings.orders_db_path) as dao: + dao.create( + order, actor="backup_restore_smoke", reason="seed_pending_order" + ) + + payment_service = PaymentService.for_db( + settings.orders_db_path, + base_url=settings.payment_base_url, + webhook_secret=settings.payment_webhook_secret, + ) + checkout = payment_service.create_checkout( + order.id, portal_token="backup-smoke-token" + ) + payload, headers = payment_service.provider.build_webhook_request( + payment_id=checkout.payment_id, + amount_cents=order.amount_cents, + provider_trade_no=f"SMOKE-{order.id}", + ) + payment_service.handle_webhook(payload, headers.get("X-Mock-Signature", "")) + + with OrdersDAO.connect(settings.orders_db_path) as dao: + dao.update( + order.id, + {"audit_report": str(html_path), "pdf_path": str(pdf_path)}, + actor="backup_restore_smoke", + reason="attach_restore_artifacts", + ) + dao.transition_status( + order.id, + "serving", + actor="backup_restore_smoke", + reason="processing", + ) + dao.transition_status( + order.id, + "delivered", + actor="backup_restore_smoke", + reason="report_ready", + ) + + token = issue_portal_token(order.id, settings.jwt_secret) + status_page = client.get(f"/portal/{token}/status") + if status_page.status_code != 200 or "报告已就绪" not in status_page.text: + raise RuntimeError( + f"portal status failed: {status_page.status_code} {status_page.text}" + ) + + report_page = client.get(f"/portal/{token}/report") + if report_page.status_code != 200: + raise RuntimeError( + f"portal report failed: {report_page.status_code} {report_page.text}" + ) + + pdf_resp = client.get(f"/portal/{token}/report.pdf") + if pdf_resp.status_code != 200: + raise RuntimeError( + f"portal pdf failed: {pdf_resp.status_code} {pdf_resp.text}" + ) + if not pdf_resp.headers.get("content-type", "").startswith( + "application/pdf" + ): + raise RuntimeError( + f"portal pdf content-type invalid: {pdf_resp.headers.get('content-type')}" + ) + if not pdf_resp.content.startswith(b"%PDF-"): + raise RuntimeError("portal pdf payload is not a PDF header") + + result = { + "backup_dir": str(root), + "admin_db": str(admin_db), + "orders_db": str(orders_db), + "report_html": str(html_path), + "report_pdf": str(pdf_path), + "smoke_order_id": order_id, + "health_status": 200, + "portal_status": 200, + "portal_report": 200, + "portal_pdf": 200, + "pdf_bytes": len(pdf_resp.content), + } + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Run restore smoke test against a staged backup directory" + ) + parser.add_argument( + "--backup-dir", + required=True, + help="staged backup directory containing db/ and files/", + ) + args = parser.parse_args(argv) + + result = run_restore_smoke(args.backup_dir) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/backup_snapshot.sh b/scripts/backup_snapshot.sh new file mode 100644 index 0000000..9af4e89 --- /dev/null +++ b/scripts/backup_snapshot.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BACKUP_ROOT="${1:-${GAOKAO_BACKUP_ROOT:-${ROOT_DIR}/var/backups}}" +TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)" +SNAPSHOT_DIR="${BACKUP_ROOT}/backup-${TIMESTAMP}" +KEEP_COUNT="${GAOKAO_BACKUP_KEEP:-7}" +ADMIN_DB="${GAOKAO_DB_PATH:-${ROOT_DIR}/data/orders/admin.db}" +ORDERS_DB="${GAOKAO_ORDERS_DB_PATH:-${ROOT_DIR}/data/orders.db}" +SHARE_DB="${GAOKAO_SHARE_DB_PATH:-${ROOT_DIR}/data/share/short_links.db}" +SHARE_REPORT_DIR="${GAOKAO_SHARE_REPORT_DIR:-${ROOT_DIR}/data/share/reports}" +EXAMPLE_REPORT_DIR="${ROOT_DIR}/data/examples" +ENV_FILE="${GAOKAO_BACKUP_ENV_FILE:-}" +CONFIG_DIR="${GAOKAO_BACKUP_CONFIG_DIR:-}" +SECRETS_DIR="${GAOKAO_BACKUP_SECRETS_DIR:-}" + +log() { + printf '[backup-snapshot] %s\n' "$1" +} + +copy_file_to() { + local src="$1" + local dest="$2" + if [[ -n "$src" && -f "$src" ]]; then + mkdir -p "$(dirname "$dest")" + cp "$src" "$dest" + log "copied file: $src -> $dest" + else + log "skip missing file: ${src:-}" + fi +} + +copy_dir_to() { + local src="$1" + local dest="$2" + if [[ -n "$src" && -d "$src" ]]; then + mkdir -p "$dest" + cp -R "$src/." "$dest/" + log "copied directory: $src -> $dest" + else + log "skip missing directory: ${src:-}" + fi +} + +write_manifest() { + ADMIN_DB="$ADMIN_DB" \ + ORDERS_DB="$ORDERS_DB" \ + SHARE_DB="$SHARE_DB" \ + SHARE_REPORT_DIR="$SHARE_REPORT_DIR" \ + EXAMPLE_REPORT_DIR="$EXAMPLE_REPORT_DIR" \ + ENV_FILE="$ENV_FILE" \ + CONFIG_DIR="$CONFIG_DIR" \ + SECRETS_DIR="$SECRETS_DIR" \ + python3 - "$SNAPSHOT_DIR" <<'PY' +from __future__ import annotations + +import hashlib +import json +import os +import sys +from pathlib import Path + +root = Path(sys.argv[1]).resolve() +manifest_path = root / "manifest.json" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +manifest = { + "generated_at": root.name.removeprefix("backup-"), + "snapshot_dir": str(root), + "source_paths": { + "admin_db": os.environ.get("ADMIN_DB", ""), + "orders_db": os.environ.get("ORDERS_DB", ""), + "share_db": os.environ.get("SHARE_DB", ""), + "share_report_dir": os.environ.get("SHARE_REPORT_DIR", ""), + "example_report_dir": os.environ.get("EXAMPLE_REPORT_DIR", ""), + "env_file": os.environ.get("ENV_FILE", ""), + "config_dir": os.environ.get("CONFIG_DIR", ""), + "secrets_dir": os.environ.get("SECRETS_DIR", ""), + }, + "files": [], +} + +for path in sorted(root.rglob("*")): + if not path.is_file() or path == manifest_path: + continue + manifest["files"].append( + { + "path": str(path.relative_to(root)), + "size_bytes": path.stat().st_size, + "sha256": sha256_file(path), + } + ) + +manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") +print(manifest_path) +PY +} + +prune_old_snapshots() { + local backup_root="$1" + local keep_count="$2" + if ! [[ "$keep_count" =~ ^[0-9]+$ ]] || (( keep_count < 1 )); then + log "invalid GAOKAO_BACKUP_KEEP=$keep_count, skip prune" + return + fi + + mapfile -t snapshots < <(find "$backup_root" -mindepth 1 -maxdepth 1 -type d -name 'backup-*' | sort) + local total="${#snapshots[@]}" + if (( total <= keep_count )); then + return + fi + + local prune_count=$((total - keep_count)) + local i + for ((i = 0; i < prune_count; i++)); do + rm -rf "${snapshots[$i]}" + log "pruned old snapshot: ${snapshots[$i]}" + done +} + +main() { + mkdir -p "$SNAPSHOT_DIR" + log "snapshot dir: $SNAPSHOT_DIR" + + copy_file_to "$ADMIN_DB" "$SNAPSHOT_DIR/db/$(basename "$ADMIN_DB")" + copy_file_to "$ORDERS_DB" "$SNAPSHOT_DIR/db/$(basename "$ORDERS_DB")" + copy_file_to "$SHARE_DB" "$SNAPSHOT_DIR/db/$(basename "$SHARE_DB")" + copy_dir_to "$SHARE_REPORT_DIR" "$SNAPSHOT_DIR/files/reports" + copy_dir_to "$EXAMPLE_REPORT_DIR" "$SNAPSHOT_DIR/files/examples" + copy_file_to "$ENV_FILE" "$SNAPSHOT_DIR/config/.env" + copy_dir_to "$CONFIG_DIR" "$SNAPSHOT_DIR/config/deploy" + copy_dir_to "$SECRETS_DIR" "$SNAPSHOT_DIR/secrets" + + local manifest_path + manifest_path="$(write_manifest)" + log "manifest written: $manifest_path" + + prune_old_snapshots "$BACKUP_ROOT" "$KEEP_COUNT" + log "backup snapshot finished" + printf '%s\n' "$SNAPSHOT_DIR" +} + +main "$@" diff --git a/scripts/backup_verify.sh b/scripts/backup_verify.sh index 4d8f42d..b5df2fd 100644 --- a/scripts/backup_verify.sh +++ b/scripts/backup_verify.sh @@ -2,13 +2,28 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -VERIFY_DIR="${1:-$(mktemp -d /tmp/gaokao-backup-verify-XXXXXX)}" +SOURCE_BACKUP_DIR="" +VERIFY_DIR="" +SKIP_SMOKE="0" ADMIN_DB="${GAOKAO_DB_PATH:-${ROOT_DIR}/data/orders/admin.db}" ORDERS_DB="${GAOKAO_ORDERS_DB_PATH:-${ROOT_DIR}/data/orders.db}" SHARE_DB="${GAOKAO_SHARE_DB_PATH:-${ROOT_DIR}/data/share/short_links.db}" SHARE_REPORT_DIR="${GAOKAO_SHARE_REPORT_DIR:-${ROOT_DIR}/data/share/reports}" EXAMPLE_REPORT_DIR="${ROOT_DIR}/data/examples" +usage() { + cat <<'EOF' +Usage: + bash scripts/backup_verify.sh [verify_dir] + bash scripts/backup_verify.sh --from-backup /path/to/backup-dir [--skip-smoke] + +Default mode copies live SQLite/files into a temporary verify dir, then validates: +- SQLite files are readable +- manifest checksums (if manifest.json exists) +- restore smoke via FastAPI TestClient (unless --skip-smoke) +EOF +} + log() { printf '[backup-verify] %s\n' "$1" } @@ -30,13 +45,69 @@ copy_dir_if_exists() { local dest_dir="$2" if [[ -d "$src" ]]; then mkdir -p "$dest_dir" - cp -R "$src" "$dest_dir/" + cp -R "$src/." "$dest_dir/" log "copied directory: $src" else log "skip missing directory: $src" fi } +stage_live_sources() { + log "verify dir: $VERIFY_DIR" + copy_if_exists "$ADMIN_DB" "$VERIFY_DIR/db" + copy_if_exists "$ORDERS_DB" "$VERIFY_DIR/db" + copy_if_exists "$SHARE_DB" "$VERIFY_DIR/db" + copy_dir_if_exists "$SHARE_REPORT_DIR" "$VERIFY_DIR/files/reports" + copy_dir_if_exists "$EXAMPLE_REPORT_DIR" "$VERIFY_DIR/files/examples" +} + +stage_existing_backup() { + if [[ ! -d "$SOURCE_BACKUP_DIR" ]]; then + printf 'backup dir not found: %s\n' "$SOURCE_BACKUP_DIR" >&2 + exit 1 + fi + log "staging backup dir: $SOURCE_BACKUP_DIR -> $VERIFY_DIR" + mkdir -p "$VERIFY_DIR" + cp -R "$SOURCE_BACKUP_DIR/." "$VERIFY_DIR/" +} + +verify_manifest_if_present() { + local manifest_path="$1" + if [[ ! -f "$manifest_path" ]]; then + log "skip missing manifest: $manifest_path" + return + fi + + log "verifying manifest checksums" + python3 - "$manifest_path" <<'PY' +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +manifest_path = Path(sys.argv[1]).resolve() +root = manifest_path.parent +manifest = json.loads(manifest_path.read_text(encoding="utf-8")) +errors: list[str] = [] +for item in manifest.get("files", []): + rel = item["path"] + target = root / rel + if not target.is_file(): + errors.append(f"missing file: {rel}") + continue + digest = hashlib.sha256(target.read_bytes()).hexdigest() + if digest != item["sha256"]: + errors.append(f"sha256 mismatch: {rel}") +if errors: + for err in errors: + print(err) + raise SystemExit(1) +print(f"manifest_ok files={len(manifest.get('files', []))}") +PY +} + verify_sqlite_file() { local db_path="$1" if [[ ! -f "$db_path" ]]; then @@ -46,6 +117,7 @@ verify_sqlite_file() { import sqlite3 import sys from pathlib import Path + p = Path(sys.argv[1]) conn = sqlite3.connect(p) try: @@ -57,16 +129,7 @@ finally: PY } -main() { - mkdir -p "$VERIFY_DIR" - log "verify dir: $VERIFY_DIR" - - copy_if_exists "$ADMIN_DB" "$VERIFY_DIR/db" - copy_if_exists "$ORDERS_DB" "$VERIFY_DIR/db" - copy_if_exists "$SHARE_DB" "$VERIFY_DIR/db" - copy_dir_if_exists "$SHARE_REPORT_DIR" "$VERIFY_DIR/files" - copy_dir_if_exists "$EXAMPLE_REPORT_DIR" "$VERIFY_DIR/files" - +verify_artifacts() { log "verifying copied sqlite files" if [[ -d "$VERIFY_DIR/db" ]]; then while IFS= read -r -d '' db_file; do @@ -76,9 +139,61 @@ main() { log "verifying copied report artifacts" if [[ -d "$VERIFY_DIR/files" ]]; then - find "$VERIFY_DIR/files" -type f | sort | sed 's#^#[backup-verify] artifact: #' + find "$VERIFY_DIR/files" -type f | sort | sed 's#^#[backup-verify] artifact: #' + fi +} + +run_restore_smoke() { + if [[ "$SKIP_SMOKE" == "1" ]]; then + log "skip restore smoke" + return fi + log "running restore smoke" + python3 "$ROOT_DIR/scripts/backup_restore_smoke.py" --backup-dir "$VERIFY_DIR" +} + +parse_args() { + while (( $# > 0 )); do + case "$1" in + --from-backup) + SOURCE_BACKUP_DIR="$2" + shift 2 + ;; + --skip-smoke) + SKIP_SMOKE="1" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + if [[ -n "$VERIFY_DIR" ]]; then + printf 'unexpected argument: %s\n' "$1" >&2 + exit 1 + fi + VERIFY_DIR="$1" + shift + ;; + esac + done +} + +main() { + parse_args "$@" + VERIFY_DIR="${VERIFY_DIR:-$(mktemp -d /tmp/gaokao-backup-verify-XXXXXX)}" + + if [[ -n "$SOURCE_BACKUP_DIR" ]]; then + stage_existing_backup + else + mkdir -p "$VERIFY_DIR" + stage_live_sources + fi + + verify_manifest_if_present "$VERIFY_DIR/manifest.json" + verify_artifacts + run_restore_smoke log "backup verification finished" } diff --git a/scripts/gaokao_retention_cleanup.py b/scripts/gaokao_retention_cleanup.py new file mode 100644 index 0000000..30741df --- /dev/null +++ b/scripts/gaokao_retention_cleanup.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +if __name__ == "__main__": + from data.orders.retention_cleanup import main + + raise SystemExit(main()) diff --git a/tests/test_backup_workflow.py b/tests/test_backup_workflow.py new file mode 100644 index 0000000..cea10bd --- /dev/null +++ b/tests/test_backup_workflow.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import json +import os +import sqlite3 +import subprocess +from pathlib import Path + +from admin.db import ensure_schema +from data.orders.schema import apply_schema + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +def _bootstrap_share_db(db_path: str) -> None: + conn = sqlite3.connect(db_path) + try: + conn.execute("CREATE TABLE IF NOT EXISTS short_links (id INTEGER PRIMARY KEY)") + conn.commit() + finally: + conn.close() + + +def _prepare_backup_sources(settings, tmp_path: Path) -> dict[str, str]: + ensure_schema(settings.db_path) + apply_schema(settings.orders_db_path).close() + _bootstrap_share_db(settings.share_db_path) + + share_report_dir = Path(settings.share_report_dir) + share_report_dir.mkdir(parents=True, exist_ok=True) + (share_report_dir / "sample-report.json").write_text("{}", encoding="utf-8") + + env_file = tmp_path / ".env.runtime" + env_file.write_text("GAOKAO_ENV=dev\n", encoding="utf-8") + + config_dir = tmp_path / "deploy-config" + config_dir.mkdir(parents=True, exist_ok=True) + (config_dir / "docker-compose.yml").write_text("services: {}\n", encoding="utf-8") + + secrets_dir = tmp_path / "secrets" + secrets_dir.mkdir(parents=True, exist_ok=True) + (secrets_dir / "jwt.secret").write_text("secret", encoding="utf-8") + + return { + "GAOKAO_DB_PATH": settings.db_path, + "GAOKAO_ORDERS_DB_PATH": settings.orders_db_path, + "GAOKAO_SHARE_DB_PATH": settings.share_db_path, + "GAOKAO_SHARE_REPORT_DIR": settings.share_report_dir, + "GAOKAO_BACKUP_ENV_FILE": str(env_file), + "GAOKAO_BACKUP_CONFIG_DIR": str(config_dir), + "GAOKAO_BACKUP_SECRETS_DIR": str(secrets_dir), + } + + +def _run_snapshot(env: dict[str, str], backup_root: Path) -> Path: + proc = subprocess.run( + ["bash", "scripts/backup_snapshot.sh", str(backup_root)], + cwd=PROJECT_ROOT, + text=True, + capture_output=True, + env=env, + check=False, + ) + assert proc.returncode == 0, proc.stderr + return Path(proc.stdout.strip().splitlines()[-1]) + + +def test_backup_snapshot_creates_manifest_and_prunes_old_backups(settings, tmp_path): + env = { + **os.environ, + **_prepare_backup_sources(settings, tmp_path), + "GAOKAO_BACKUP_KEEP": "2", + } + backup_root = tmp_path / "backups" + (backup_root / "backup-20240101T000000Z").mkdir(parents=True, exist_ok=True) + (backup_root / "backup-20240102T000000Z").mkdir(parents=True, exist_ok=True) + + snapshot_dir = _run_snapshot(env, backup_root) + + assert snapshot_dir.is_dir() + manifest_path = snapshot_dir / "manifest.json" + assert manifest_path.is_file() + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest_paths = {item["path"] for item in manifest["files"]} + assert "db/admin.db" in manifest_paths + assert "db/orders.db" in manifest_paths + assert "db/short_links.db" in manifest_paths + assert "config/.env" in manifest_paths + assert "config/deploy/docker-compose.yml" in manifest_paths + assert "secrets/jwt.secret" in manifest_paths + assert any(path.startswith("files/examples/") for path in manifest_paths) + + remaining = sorted(path.name for path in backup_root.iterdir() if path.is_dir()) + assert len(remaining) == 2 + assert "backup-20240101T000000Z" not in remaining + assert snapshot_dir.name in remaining + + +def test_backup_verify_runs_restore_smoke_on_snapshot(settings, tmp_path): + env = { + **os.environ, + **_prepare_backup_sources(settings, tmp_path), + "GAOKAO_PAYMENT_PROVIDER": "alipay", + } + backup_root = tmp_path / "backups" + snapshot_dir = _run_snapshot(env, backup_root) + + proc = subprocess.run( + ["bash", "scripts/backup_verify.sh", "--from-backup", str(snapshot_dir)], + cwd=PROJECT_ROOT, + text=True, + capture_output=True, + env=env, + check=False, + ) + + assert proc.returncode == 0, proc.stderr + assert "manifest_ok files=" in proc.stdout + assert '"portal_status": 200' in proc.stdout + assert '"portal_report": 200' in proc.stdout + assert '"portal_pdf": 200' in proc.stdout + assert "backup verification finished" in proc.stdout diff --git a/tests/test_delivery_retention_ops_artifacts.py b/tests/test_delivery_retention_ops_artifacts.py new file mode 100644 index 0000000..7455dc2 --- /dev/null +++ b/tests/test_delivery_retention_ops_artifacts.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +def test_delivery_retention_ops_artifacts_exist_and_reference_scripts(): + runbook = PROJECT_ROOT / "docs/DELIVERY_RETENTION_OPS_RUNBOOK.md" + dispatch_service = PROJECT_ROOT / "deploy/systemd/gaokao-delivery-dispatch.service" + dispatch_timer = PROJECT_ROOT / "deploy/systemd/gaokao-delivery-dispatch.timer" + watchdog_service = PROJECT_ROOT / "deploy/systemd/gaokao-delivery-watchdog.service" + watchdog_timer = PROJECT_ROOT / "deploy/systemd/gaokao-delivery-watchdog.timer" + retention_service = PROJECT_ROOT / "deploy/systemd/gaokao-retention-cleanup.service" + retention_timer = PROJECT_ROOT / "deploy/systemd/gaokao-retention-cleanup.timer" + cron_file = PROJECT_ROOT / "deploy/cron/gaokao-jobs.crontab" + env_example = PROJECT_ROOT / "deploy/systemd/gaokao-jobs.env.example" + + for path in ( + runbook, + dispatch_service, + dispatch_timer, + watchdog_service, + watchdog_timer, + retention_service, + retention_timer, + cron_file, + env_example, + ): + assert path.is_file(), path + + assert "gaokao-delivery-dispatch.py" in dispatch_service.read_text(encoding="utf-8") + assert "gaokao-delivery-watchdog.py" in watchdog_service.read_text(encoding="utf-8") + assert "--retention-days" in retention_service.read_text(encoding="utf-8") + + cron_text = cron_file.read_text(encoding="utf-8") + assert "gaokao-delivery-dispatch.py" in cron_text + assert "gaokao-delivery-watchdog.py" in cron_text + assert "gaokao-retention-cleanup.py --retention-days" in cron_text + + runbook_text = runbook.read_text(encoding="utf-8") + assert "systemd" in runbook_text + assert "cron" in runbook_text + assert "gaokao-retention-cleanup.py --retention-days 180" in runbook_text diff --git a/tests/test_retention_cleanup.py b/tests/test_retention_cleanup.py index aabbb39..ac4f806 100644 --- a/tests/test_retention_cleanup.py +++ b/tests/test_retention_cleanup.py @@ -89,4 +89,54 @@ def test_retention_cleanup_script_prints_summary(settings): check=False, ) assert proc.returncode == 0, proc.stderr + assert '"cutoff_iso": "2025-06-30T00:00:00+00:00"' in proc.stdout + assert '"dry_run": true' in proc.stdout + assert '"candidates": 1' in proc.stdout + + +def test_retention_cleanup_script_supports_retention_days(settings): + _seed_old_completed_order( + settings.orders_db_path, order_id="GKO-20250101-RETENTION-DAYS" + ) + env = {**os.environ, "GAOKAO_ORDERS_DB_PATH": settings.orders_db_path} + proc = subprocess.run( + [ + "python3", + "scripts/gaokao-retention-cleanup.py", + "--retention-days", + "180", + "--dry-run", + ], + cwd=PROJECT_ROOT, + text=True, + capture_output=True, + env=env, + check=False, + ) + assert proc.returncode == 0, proc.stderr + assert '"cutoff_iso":' in proc.stdout + assert '"dry_run": true' in proc.stdout + assert '"candidates": 1' in proc.stdout + + +def test_retention_cleanup_underscore_script_alias_works(settings): + _seed_old_completed_order( + settings.orders_db_path, order_id="GKO-20250101-RETENTION-ALIAS" + ) + env = {**os.environ, "GAOKAO_ORDERS_DB_PATH": settings.orders_db_path} + proc = subprocess.run( + [ + "python3", + "scripts/gaokao_retention_cleanup.py", + "--cutoff", + "2025-06-30T00:00:00+00:00", + "--dry-run", + ], + cwd=PROJECT_ROOT, + text=True, + capture_output=True, + env=env, + check=False, + ) + assert proc.returncode == 0, proc.stderr assert '"candidates": 1' in proc.stdout