From a82c5b3ccab2a3e20d86078f7fa16dde23efdcdb Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Thu, 18 Jun 2026 15:20:16 +0800 Subject: [PATCH] fix: align retention cleanup across notifications logs and share telemetry --- admin/tests/test_order_deletion.py | 38 +++++++++++ admin/tests/test_order_info_form.py | 42 ++++++++++++ data/orders/deletion_service.py | 5 ++ data/orders/retention_cleanup.py | 92 ++++++++++++++++++++++++-- docs/DATA_RETENTION_AND_DELETION.md | 29 ++++---- docs/DELIVERY_RETENTION_OPS_RUNBOOK.md | 10 ++- tests/test_delivery_dispatcher.py | 89 +++++++++++++++++++++++++ 7 files changed, 287 insertions(+), 18 deletions(-) diff --git a/admin/tests/test_order_deletion.py b/admin/tests/test_order_deletion.py index 099bea5..5c32336 100644 --- a/admin/tests/test_order_deletion.py +++ b/admin/tests/test_order_deletion.py @@ -220,3 +220,41 @@ def test_admin_anonymize_order_removes_portal_attachments( intake_store.close() assert intake is not None assert intake.payload == {} + + +def test_admin_anonymize_order_scrubs_delivery_notifications_payload( + client, auth_headers, settings +): + order = _seed_order(settings.orders_db_path, order_id="GKO-20260618-ANON-NOTIFY") + _mark_paid(settings, order) + + notification_service = DeliveryNotificationService.for_db(settings.orders_db_path) + try: + notification_service.notify_event( + order.id, + event_type="report_ready", + channel="station", + payload_json='{"customer_email":"parent@example.com","report_path":"/tmp/report.pdf"}', + ) + notification_service.mark_delivered( + order.id, + channel="station", + payload_json='{"customer_email":"parent@example.com","report_path":"/tmp/report.pdf"}', + ) + finally: + notification_service.close() + + resp = client.delete( + f"/api/orders/{order.id}?mode=anonymize&reason=retention_expired", + headers=auth_headers, + ) + assert resp.status_code == 200, resp.text + + notification_service = DeliveryNotificationService.for_db(settings.orders_db_path) + try: + events = notification_service.list_events(order.id) + finally: + notification_service.close() + + assert len(events) == 1 + assert events[0].payload_json == "{}" diff --git a/admin/tests/test_order_info_form.py b/admin/tests/test_order_info_form.py index 86024f2..96c32bc 100644 --- a/admin/tests/test_order_info_form.py +++ b/admin/tests/test_order_info_form.py @@ -262,6 +262,48 @@ def test_portal_deletion_request_is_logged_and_visible_in_admin(client, auth_hea assert "parent@example.com" in admin_page.text +def test_deletion_request_log_is_purged_by_retention_cleanup(settings): + from pathlib import Path + + from data.orders.retention_cleanup import run_cleanup + + order = Order( + id="GKO-20250101-DELETE-LOG", + source="web", + service_version="standard", + amount_cents=9900, + status="completed", + customer_name="张家长", + customer_phone="13800138000", + candidate_name="张三", + candidate_province="湖南", + created_at="2025-01-01T00:00:00+00:00", + completed_at="2025-01-02T00:00:00+00:00", + status_updated_at="2025-01-02T00:00:00+00:00", + ) + with OrdersDAO.connect(settings.orders_db_path) as dao: + dao.create(order, actor="test", reason="seed_old_completed") + + log_path = Path(settings.deletion_request_log_path) + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text( + '{"created_at":"2025-01-02T00:00:00Z","order_id":"GKO-20250101-DELETE-LOG","requester_contact":"parent@example.com"}\n' + '{"created_at":"2026-01-02T00:00:00Z","order_id":"GKO-KEEP-LOG","requester_contact":"keep@example.com"}\n', + encoding="utf-8", + ) + + result = run_cleanup( + settings.orders_db_path, + cutoff_iso="2025-06-30T00:00:00+00:00", + apply=True, + ) + + assert result.anonymized == 1 + remaining = log_path.read_text(encoding="utf-8") + assert "GKO-20250101-DELETE-LOG" not in remaining + assert "GKO-KEEP-LOG" in remaining + + def test_order_info_page_uses_checkbox_checked_state_for_submit(client, settings): order = _seed_order(settings.orders_db_path, order_id="GKO-20260615-INFO-CHECKED") _mark_paid(settings, order) diff --git a/data/orders/deletion_service.py b/data/orders/deletion_service.py index 6ab4b11..14821e5 100644 --- a/data/orders/deletion_service.py +++ b/data/orders/deletion_service.py @@ -125,6 +125,11 @@ class OrderDeletionService: "UPDATE order_intakes SET payload_json='{}', updated_at=? WHERE order_id=?", (now, order_id), ) + if self._table_exists("delivery_notifications"): + self._conn.execute( + "UPDATE delivery_notifications SET payload_json='{}' WHERE order_id=?", + (order_id,), + ) self._insert_audit( order_id=order_id, action="anonymize", diff --git a/data/orders/retention_cleanup.py b/data/orders/retention_cleanup.py index 5a18648..64a19aa 100644 --- a/data/orders/retention_cleanup.py +++ b/data/orders/retention_cleanup.py @@ -4,6 +4,7 @@ import argparse import json from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from pathlib import Path from data.orders.dao import OrdersDAO from data.orders.deletion_service import OrderDeletionService @@ -16,6 +17,8 @@ class RetentionCleanupResult: scanned: int = 0 candidates: int = 0 anonymized: int = 0 + deletion_logs_pruned: int = 0 + share_events_pruned: int = 0 _TERMINAL_STATUSES = {"completed", "refunded"} @@ -55,6 +58,54 @@ def _iter_candidates(db_path: str, cutoff_iso: str) -> tuple[int, list[str]]: return scanned, candidates +def _prune_deletion_request_log(log_path: str, candidate_ids: list[str]) -> int: + path = Path(log_path) + if not path.exists() or not candidate_ids: + return 0 + keep: list[str] = [] + pruned = 0 + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + item = json.loads(line) + except json.JSONDecodeError: + keep.append(line) + continue + if str(item.get("order_id") or "") in candidate_ids: + pruned += 1 + continue + keep.append(json.dumps(item, ensure_ascii=False)) + path.write_text("\n".join(keep) + ("\n" if keep else ""), encoding="utf-8") + return pruned + + +def _prune_share_access_events(share_db_path: str, candidate_ids: list[str]) -> int: + if not candidate_ids: + return 0 + from data.share.short_link import ShortLinkService + + service = ShortLinkService(db_path=share_db_path) + with service._connect() as conn: + rows = conn.execute( + "SELECT code FROM share_links WHERE report_id IN ({})".format( + ",".join("?" for _ in candidate_ids) + ), + tuple(candidate_ids), + ).fetchall() + codes = [str(row[0]) for row in rows] + if not codes: + return 0 + conn.execute( + "DELETE FROM share_link_access_events WHERE code IN ({})".format( + ",".join("?" for _ in codes) + ), + tuple(codes), + ) + conn.commit() + return int(conn.total_changes) + + def resolve_cutoff_iso( *, cutoff_iso: str | None = None, retention_days: int | None = None ) -> str: @@ -68,8 +119,24 @@ def resolve_cutoff_iso( def run_cleanup( - db_path: str, *, cutoff_iso: str, apply: bool + db_path: str, + *, + cutoff_iso: str, + apply: bool, + deletion_request_log_path: str | None = None, + share_db_path: str | None = None, ) -> RetentionCleanupResult: + if deletion_request_log_path is None or share_db_path is None: + try: + from admin.config import load_settings + + settings = load_settings() + except Exception: + settings = None + if deletion_request_log_path is None and settings is not None: + deletion_request_log_path = settings.deletion_request_log_path + if share_db_path is None and settings is not None: + share_db_path = settings.share_db_path scanned, candidate_ids = _iter_candidates(db_path, cutoff_iso) result = RetentionCleanupResult( cutoff_iso=cutoff_iso, @@ -88,6 +155,14 @@ def run_cleanup( result.anonymized += 1 finally: service.close() + if deletion_request_log_path: + result.deletion_logs_pruned = _prune_deletion_request_log( + deletion_request_log_path, candidate_ids + ) + if share_db_path: + result.share_events_pruned = _prune_share_access_events( + share_db_path, candidate_ids + ) return result @@ -110,10 +185,11 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None, *, db_path: str | None = None) -> int: - if db_path is None: - from admin.config import load_settings + from admin.config import load_settings - db_path = load_settings().orders_db_path + settings = load_settings() + if db_path is None: + db_path = settings.orders_db_path parser = build_parser() args = parser.parse_args(argv) try: @@ -123,7 +199,13 @@ def main(argv: list[str] | None = None, *, db_path: str | None = None) -> int: ) except ValueError as exc: parser.error(str(exc)) - result = run_cleanup(db_path, cutoff_iso=cutoff_iso, apply=not args.dry_run) + result = run_cleanup( + db_path, + cutoff_iso=cutoff_iso, + apply=not args.dry_run, + deletion_request_log_path=getattr(settings, "deletion_request_log_path", None), + share_db_path=getattr(settings, "share_db_path", None), + ) print(json.dumps(result.__dict__, ensure_ascii=False, indent=2)) return 0 diff --git a/docs/DATA_RETENTION_AND_DELETION.md b/docs/DATA_RETENTION_AND_DELETION.md index c428548..e7ee49e 100644 --- a/docs/DATA_RETENTION_AND_DELETION.md +++ b/docs/DATA_RETENTION_AND_DELETION.md @@ -8,15 +8,17 @@ ## 2. 数据分类与建议保留期 -| 数据类型 | 示例 | 建议保留期 | 删除方式 | -| ------------ | --------------------------- | ----------------------- | -------------------------- | -| 订单主数据 | orders 表、状态、金额、套餐 | 服务完成后 180 天 | 逻辑受理 + 物理删除/匿名化 | -| 资料提交数据 | `order_intakes` payload | 服务完成后 180 天 | 与订单一起删除 | -| 报告交付文件 | HTML/PDF 报告 | 服务完成后 180 天 | 删除文件并清理路径 | -| 通知审计 | `delivery_notifications` | 服务完成后 180 天 | 随订单删除或单独清理 | -| 支付审计 | 支付单/回调记录 | 至少 180 天，争议中延长 | 审计期结束后清理 | -| 系统日志 | 服务日志/错误日志 | 30~90 天 | 日志轮转清理 | -| 脱敏案例数据 | 仅用于内部复盘 | 长期，但必须脱敏 | 仅保留脱敏版 | +| 数据类型 | 示例 | 建议保留期 | 删除方式 | +| -------------- | ------------------------------------ | ----------------------- | -------------------------------------- | +| 订单主数据 | orders 表、状态、金额、套餐 | 服务完成后 180 天 | 逻辑受理 + 物理删除/匿名化 | +| 资料提交数据 | `order_intakes` payload | 服务完成后 180 天 | 与订单一起删除 | +| 报告交付文件 | HTML/PDF 报告 | 服务完成后 180 天 | 删除文件并清理路径 | +| 通知审计 | `delivery_notifications` | 服务完成后 180 天 | 删除时物理清除；匿名化时 payload 置空 | +| 支付审计 | 支付单/回调记录 | 至少 180 天，争议中延长 | 审计期结束后清理 | +| 删除申请日志 | `deletion-requests.jsonl` | 随订单 retention 同步 | retention 作业按候选订单移除日志项 | +| 分享访问遥测 | `share_link_access_events` | 随订单 retention 同步 | retention 作业删除对应 report_id 事件 | +| 系统日志 | 服务日志/错误日志 | 30~90 天 | 日志轮转清理 | +| 脱敏案例数据 | 仅用于内部复盘 | 长期，但必须脱敏 | 仅保留脱敏版 | ## 3. 删除请求处理原则 @@ -32,6 +34,7 @@ - 删除资料提交记录 - 删除报告 HTML/PDF 文件 - 删除关联通知记录 + - 删除申请日志 / 分享访问遥测中的对应订单旁路记录 4. 在审计记录中留下“已删除/已匿名化”的最小操作痕迹 ## 4. 当前系统能力与缺口 @@ -41,9 +44,11 @@ - `OrdersDAO.delete()` 删除订单能力 - ON DELETE CASCADE 清理部分关联表 - 后台 `DELETE /api/orders/{id}?mode=delete|anonymize&reason=...` 最小执行入口 -- 删除时自动清理 HTML/PDF 报告文件 +- 删除时自动清理 HTML/PDF 报告文件与 portal 上传附件 +- 匿名化时同步清空 `delivery_notifications.payload_json` - `order_deletion_audits` 最小审计表 - `scripts/gaokao_retention_cleanup.py` / `scripts/gaokao-retention-cleanup.py` 保留期清理入口 +- retention 作业会同步修剪 `deletion-requests.jsonl` 与 `share_link_access_events` - `scripts/gaokao-retention-cleanup.py --retention-days 180` 最小定时调度入口 - `docs/DELIVERY_RETENTION_OPS_RUNBOOK.md` - `deploy/systemd/gaokao-retention-cleanup.{service,timer}` @@ -51,7 +56,7 @@ 尚缺： -- 前台/客服删除工单流程 +- 前台/客服删除工单流程仍是 portal 提交 + 后台处理，不是全自动闭环 - 更细粒度的匿名化策略（如案例长期保留脱敏版） - 目标生产主机上的实际安装/值班留痕（仓库内只有调度样例，不代表已部署） @@ -73,4 +78,4 @@ - runbook：`docs/DELIVERY_RETENTION_OPS_RUNBOOK.md` - systemd/cron 样例：`deploy/systemd/gaokao-retention-cleanup.{service,timer}` / `deploy/cron/gaokao-jobs.crontab` -真相边界：当前保留期作业执行的是后台匿名化，不应误报成“用户删除请求流程已经完整上线”。 +真相边界：当前保留期作业执行的是后台匿名化，并同步修剪删除申请日志/分享访问遥测；这仍不应误报成“用户删除请求流程已经完整上线”。 diff --git a/docs/DELIVERY_RETENTION_OPS_RUNBOOK.md b/docs/DELIVERY_RETENTION_OPS_RUNBOOK.md index 5ef7ca8..7976ab2 100644 --- a/docs/DELIVERY_RETENTION_OPS_RUNBOOK.md +++ b/docs/DELIVERY_RETENTION_OPS_RUNBOOK.md @@ -42,7 +42,8 @@ - 当前清理动作是 **匿名化**（`anonymize_order`），不是物理删除。 - 只处理 `completed/refunded` 且锚点时间早于 cutoff 的订单。 -- 它会清理订单侧敏感字段并写审计，但不等于“前台删除工单流程已上线”。 +- 它会清理订单侧敏感字段、清空 `delivery_notifications.payload_json`，并修剪 `deletion-requests.jsonl` / `share_link_access_events` 中对应订单的旁路记录。 +- 它不等于“前台删除工单流程已上线”。 ## 2. 统一环境配置 @@ -76,6 +77,13 @@ python3 scripts/gaokao-retention-cleanup.py --cutoff 2025-12-31T00:00:00+00:00 - python3 scripts/gaokao-retention-cleanup.py --retention-days 180 ``` +执行后至少抽查： + +- 候选订单已匿名化 +- 对应 `delivery_notifications.payload_json` 已变成 `{}` +- `deletion-requests.jsonl` 中对应订单记录已被修剪 +- `share_link_access_events` 中对应 report_id 的访问事件已被修剪 + 退出码： - dispatcher：`0` = 脚本执行完成（结果看 JSON） diff --git a/tests/test_delivery_dispatcher.py b/tests/test_delivery_dispatcher.py index b40017b..26eedbc 100644 --- a/tests/test_delivery_dispatcher.py +++ b/tests/test_delivery_dispatcher.py @@ -329,3 +329,92 @@ def test_delivery_watchdog_exits_nonzero_when_failures_detected(settings, tmp_pa assert proc.returncode == 2, proc.stdout + proc.stderr assert '"failed": 1' in proc.stdout + + +def test_retention_cleanup_scrubs_notification_payload_for_expired_orders(settings): + from data.orders.retention_cleanup import run_cleanup + + order = Order( + id="GKO-20250101-RETENTION-NOTIFY", + source="web", + service_version="standard", + amount_cents=9900, + status="completed", + customer_name="张家长", + customer_phone="13800138000", + candidate_name="张三", + candidate_province="湖南", + created_at="2025-01-01T00:00:00+00:00", + completed_at="2025-01-02T00:00:00+00:00", + status_updated_at="2025-01-02T00:00:00+00:00", + ) + with OrdersDAO.connect(settings.orders_db_path) as dao: + dao.create(order, actor="test", reason="seed_old_completed") + + service = DeliveryNotificationService.for_db(settings.orders_db_path) + try: + service.notify_event( + order.id, + event_type="report_ready", + channel="email", + payload_json='{"customer_email":"parent@example.com","detail":"keep"}', + ) + finally: + service.close() + + result = run_cleanup( + settings.orders_db_path, + cutoff_iso="2025-06-30T00:00:00+00:00", + apply=True, + ) + + assert result.anonymized == 1 + service = DeliveryNotificationService.for_db(settings.orders_db_path) + try: + events = service.list_events(order.id) + finally: + service.close() + assert len(events) == 1 + assert events[0].payload_json == "{}" + + +def test_share_access_events_have_retention_cleanup(settings): + from data.orders.retention_cleanup import run_cleanup + from data.share.short_link import ShortLinkService + + order = Order( + id="GKO-20250101-RETENTION-SHARE", + source="web", + service_version="standard", + amount_cents=9900, + status="completed", + customer_name="张家长", + customer_phone="13800138000", + candidate_name="张三", + candidate_province="湖南", + created_at="2025-01-01T00:00:00+00:00", + completed_at="2025-01-02T00:00:00+00:00", + status_updated_at="2025-01-02T00:00:00+00:00", + ) + with OrdersDAO.connect(settings.orders_db_path) as dao: + dao.create(order, actor="test", reason="seed_old_completed") + + share = ShortLinkService(db_path=settings.share_db_path) + link = share.create(report_id=order.id) + share.resolve(link.code, visitor_token="visitor-1") + + result = run_cleanup( + settings.orders_db_path, + cutoff_iso="2025-06-30T00:00:00+00:00", + apply=True, + share_db_path=settings.share_db_path, + ) + + assert result.anonymized == 1 + with share._connect() as conn: + row = conn.execute( + "SELECT COUNT(*) FROM share_link_access_events WHERE code=?", + (link.code,), + ).fetchone() + assert row is not None + assert int(row[0]) == 0