forked from niuniu/llm-intelligence
Compare commits
59 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62556e787f | ||
|
|
98b9203302 | ||
|
|
9ff023dab4 | ||
|
|
18158650ca | ||
|
|
1c0b28c2a3 | ||
|
|
d7455b8f05 | ||
|
|
e999d31b25 | ||
|
|
88833fac8b | ||
|
|
475401bcbe | ||
|
|
32858bfec4 | ||
|
|
f5b373caf4 | ||
|
|
f274621013 | ||
|
|
7fb45fe94d | ||
|
|
982cb66d00 | ||
|
|
2a64160a2b | ||
|
|
5cb551de68 | ||
|
|
306c0e20e6 | ||
|
|
0fd52e99c6 | ||
|
|
92cdbcd4f2 | ||
|
|
c32661fd2a | ||
|
|
e757cd2dd7 | ||
|
|
53c7f0ca47 | ||
|
|
1adce4f800 | ||
|
|
6fe3b484f1 | ||
|
|
5c5578a19b | ||
|
|
1db813cb6b | ||
|
|
6c3569fb65 | ||
|
|
d9c552cba5 | ||
|
|
236dea8bf4 | ||
|
|
68b1b2be41 | ||
|
|
b6fbc8c5cb | ||
|
|
567d1f89ec | ||
|
|
8d1312203f | ||
|
|
0de4402a11 | ||
|
|
42e75e733d | ||
|
|
0ee181a4a7 | ||
|
|
d5d18e987e | ||
|
|
256975e10c | ||
|
|
958245537a | ||
|
|
dd58c18fe3 | ||
|
|
a8999abcb0 | ||
|
|
618dff33da | ||
|
|
d7fbd25dde | ||
|
|
988a7533c6 | ||
|
|
2dca9aa627 | ||
|
|
b2b39bfc12 | ||
|
|
dfb54092b7 | ||
|
|
f3daf2959b | ||
|
|
f2f68b85c1 | ||
|
|
569b94cb73 | ||
|
|
bed5e3aec7 | ||
|
|
d893d2542e | ||
|
|
92c9a40f4b | ||
|
|
bb5a1ff9e5 | ||
|
|
efc3d5cdbd | ||
|
|
b9ca312366 | ||
|
|
b4e28d5be4 | ||
|
|
79d991a7e9 | ||
|
|
85f37a4d95 |
25
.dockerignore
Normal file
25
.dockerignore
Normal file
@@ -0,0 +1,25 @@
|
||||
.git
|
||||
.gitignore
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
.serena/
|
||||
.openclaw/
|
||||
memory/
|
||||
reports/
|
||||
logs/
|
||||
frontend/node_modules/
|
||||
frontend/dist/
|
||||
node_modules/
|
||||
dist/
|
||||
*.log
|
||||
*.tmp
|
||||
*.bak
|
||||
*.bak-*
|
||||
/tmp/
|
||||
fetch_openrouter
|
||||
fetch_openrouter_test
|
||||
fetch_multi_source
|
||||
import_phase2_data
|
||||
generate_daily_report
|
||||
go-build*/
|
||||
@@ -6,3 +6,12 @@ OPENROUTER_API_KEY=
|
||||
|
||||
# 本机 PostgreSQL 连接(long 用户通过本地 socket 直连)
|
||||
DATABASE_URL="host=/var/run/postgresql dbname=llm_intelligence user=long sslmode=disable"
|
||||
|
||||
# API Server 监听端口(默认 8080)
|
||||
PORT=8080
|
||||
|
||||
# 正式日报失败告警(可选)
|
||||
FEISHU_WEBHOOK=
|
||||
|
||||
# 日报输出目录(可选,默认 reports/daily)
|
||||
REPORT_OUTPUT_DIR="reports/daily"
|
||||
|
||||
71
.github/workflows/ci.yml
vendored
Normal file
71
.github/workflows/ci.yml
vendored
Normal file
@@ -0,0 +1,71 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
go-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.22"
|
||||
cache: true
|
||||
|
||||
- name: Run package-level Go tests (cmd/server + internal/...)
|
||||
run: go test ./...
|
||||
- name: Note script test coverage boundary
|
||||
run: |
|
||||
echo "go test ./... only covers package-based Go code"
|
||||
echo "script-level coverage runs in the scripts-regression job"
|
||||
|
||||
scripts-regression:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.22"
|
||||
cache: true
|
||||
|
||||
- name: Run targeted script importer tests
|
||||
run: bash scripts/test_importers.sh
|
||||
|
||||
- name: Run importer smoke gate
|
||||
run: bash scripts/importer_smoke_gate_test.sh
|
||||
|
||||
- name: Run pipeline runtime alignment gate
|
||||
run: bash scripts/pipeline_runtime_alignment_test.sh
|
||||
|
||||
frontend-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install frontend dependencies
|
||||
working-directory: frontend
|
||||
run: npm ci
|
||||
|
||||
- name: Build frontend
|
||||
working-directory: frontend
|
||||
run: npm run build
|
||||
|
||||
docker-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Build container image
|
||||
run: docker build -t llm-intelligence:ci .
|
||||
8
.gitignore
vendored
8
.gitignore
vendored
@@ -26,7 +26,10 @@ frontend/src/data/latest_models.json
|
||||
reports/daily/
|
||||
reports/verification/
|
||||
reports/daily/video/
|
||||
reports/ad_hoc/
|
||||
reports/openclaw/20*.md
|
||||
!reports/openclaw/2026-05-20-2106-review.md
|
||||
!reports/openclaw/2026-05-13-1510-review.md
|
||||
|
||||
# Runtime / working memory
|
||||
memory/*.md
|
||||
@@ -44,3 +47,8 @@ test.md
|
||||
# Runtime logs
|
||||
logs/*.log
|
||||
*.log
|
||||
|
||||
# Local agent state / local build outputs
|
||||
/.agent/
|
||||
/.serena/
|
||||
/server
|
||||
|
||||
151
DEPLOYMENT.md
151
DEPLOYMENT.md
@@ -1,27 +1,33 @@
|
||||
# LLM Intelligence Hub - 部署指南
|
||||
|
||||
> 版本: v1.1
|
||||
> 日期: 2026-05-21
|
||||
> 适用版本: Phase 1 / Phase 2 基础部署
|
||||
> 版本: v1.1
|
||||
> 日期: 2026-05-14
|
||||
> 适用版本: Phase 3 / Phase 5
|
||||
|
||||
相关文档:
|
||||
|
||||
- `README.md`:项目入口与常用命令
|
||||
- `docs/CONFIGURATION.md`:环境变量与运行语义
|
||||
- `docs/PRODUCTION_CHECKLIST.md`:上线前检查、发布与回滚
|
||||
|
||||
---
|
||||
|
||||
## 环境要求
|
||||
|
||||
### 硬件
|
||||
- CPU: 1 核+
|
||||
- 内存: 512 MB+
|
||||
- 磁盘: 5 GB+
|
||||
- CPU: 1核+
|
||||
- 内存: 512MB+
|
||||
- 磁盘: 5GB+
|
||||
|
||||
### 软件
|
||||
- Go 1.22+
|
||||
- Node.js 20+
|
||||
- PostgreSQL 16+
|
||||
- Docker / Docker Compose
|
||||
- Docker 或 Podman (可选)
|
||||
|
||||
---
|
||||
|
||||
## 本地开发启动
|
||||
## 快速开始
|
||||
|
||||
### 1. 克隆仓库
|
||||
```bash
|
||||
@@ -29,62 +35,68 @@ git clone <repo-url> llm-intelligence
|
||||
cd llm-intelligence
|
||||
```
|
||||
|
||||
### 2. 初始化数据库
|
||||
### 2. 配置数据库
|
||||
```bash
|
||||
# 创建数据库
|
||||
createdb llm_intelligence
|
||||
|
||||
# 运行迁移
|
||||
psql llm_intelligence < db/migrations/001_phase1_core_tables.sql
|
||||
psql llm_intelligence < db/migrations/002_sprint1_complete_schema.sql
|
||||
psql llm_intelligence < db/migrations/003_phase2_region_pricing_metadata.sql
|
||||
psql llm_intelligence < db/migrations/004_backfill_models_batch_id.sql
|
||||
psql llm_intelligence < db/migrations/005_subscription_plan.sql
|
||||
```
|
||||
|
||||
### 3. 配置环境变量
|
||||
```bash
|
||||
export DATABASE_URL="host=/var/run/postgresql dbname=llm_intelligence sslmode=disable"
|
||||
export OPENROUTER_API_KEY="your-api-key"
|
||||
export API_AUTH_TOKEN="replace-with-long-random-token"
|
||||
# 或者:export API_BASIC_AUTH_USER="review" && export API_BASIC_AUTH_PASS="replace-with-password"
|
||||
export FEISHU_WEBHOOK="your-webhook-url" # 可选
|
||||
export INTRADAY_DISCOVERY_SEARCH_PROVIDER="command_json" # 候选发现链路可选
|
||||
export INTRADAY_DISCOVERY_LLM_PROVIDER="command_json" # 候选归纳链路可选
|
||||
|
||||
```
|
||||
|
||||
|
||||
### 4. 启动后端
|
||||
```bash
|
||||
go run cmd/server/main.go
|
||||
```
|
||||
|
||||
### 5. 启动前端开发服务
|
||||
### 5. 启动前端 (开发)
|
||||
```bash
|
||||
cd frontend
|
||||
npm install
|
||||
npm run dev
|
||||
```
|
||||
|
||||
### 6. 配置定时任务
|
||||
```bash
|
||||
crontab -e
|
||||
# 正式日报调度
|
||||
0 8 * * * cd /path/to/llm-intelligence && bash scripts/run_daily.sh >> /tmp/llm_hub_cron.log 2>&1
|
||||
|
||||
# 日内价格追踪(推荐每 4 小时一次)
|
||||
0 */4 * * * cd /path/to/llm-intelligence && bash scripts/run_intraday_price_watch.sh >> /tmp/llm_hub_intraday.log 2>&1
|
||||
|
||||
# 日内新闻发现与验证(推荐每 2 小时一次)
|
||||
0 */2 * * * cd /path/to/llm-intelligence && bash scripts/run_intraday_discovery_watch.sh >> /tmp/llm_hub_intraday_discovery.log 2>&1
|
||||
|
||||
|
||||
# 真实采集 + 写库 + 报告生成的手动复跑入口
|
||||
cd /path/to/llm-intelligence && bash scripts/run_real_pipeline.sh
|
||||
```
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Docker 部署
|
||||
|
||||
当前容器镜像已经内置前端静态资源,`app` 服务会同时提供页面和 API。
|
||||
|
||||
### 使用 compose 启动完整环境
|
||||
```bash
|
||||
docker-compose up -d --build
|
||||
```
|
||||
|
||||
启动后访问:
|
||||
- Web UI: `http://localhost:8080/`
|
||||
- Health: `http://localhost:8080/health`
|
||||
- API: `http://localhost:8080/api/v1/models`
|
||||
|
||||
### 只构建镜像
|
||||
```bash
|
||||
# 构建
|
||||
docker build -t llm-hub .
|
||||
```
|
||||
|
||||
运行示例:
|
||||
```bash
|
||||
docker run --rm -p 8080:8080 \
|
||||
-e DATABASE_URL="postgres://llm_hub:changeme@host.docker.internal:5432/llm_intelligence?sslmode=disable" \
|
||||
-e OPENROUTER_API_KEY="your-api-key" \
|
||||
llm-hub
|
||||
# 或 docker-compose
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
---
|
||||
@@ -93,63 +105,66 @@ docker run --rm -p 8080:8080 \
|
||||
|
||||
| 变量 | 必填 | 说明 |
|
||||
|------|------|------|
|
||||
| `DATABASE_URL` | 是 | PostgreSQL 连接串 |
|
||||
| `OPENROUTER_API_KEY` | 是 | OpenRouter API Key |
|
||||
| `FEISHU_WEBHOOK` | 否 | 飞书告警 Webhook |
|
||||
| `PORT` | 否 | 服务端监听端口,默认 `8080` |
|
||||
| `FRONTEND_DIST_DIR` | 否 | 自定义静态资源目录,默认自动查找 `frontend/dist` |
|
||||
| DATABASE_URL | ✅ | PostgreSQL 连接串 |
|
||||
| OPENROUTER_API_KEY | ✅ | OpenRouter API Key |
|
||||
| API_AUTH_TOKEN | 条件必填 | 对外访问 `/api/*` 的 Bearer token |
|
||||
| API_BASIC_AUTH_USER / API_BASIC_AUTH_PASS | 条件必填 | 对外访问 `/api/*` 的 Basic Auth 凭证 |
|
||||
| API_RATE_LIMIT_PER_WINDOW | ❌ | `/api/*` 每窗口允许的请求数,默认 `60` |
|
||||
| API_RATE_LIMIT_WINDOW_SEC | ❌ | `/api/*` 限流窗口秒数,默认 `60` |
|
||||
| FEISHU_WEBHOOK | ❌ | 飞书告警 Webhook |
|
||||
| REPORT_DATE | ❌ | 手工指定日内追踪/日报日期 |
|
||||
| INTRADAY_DISCOVERY_SEARCH_PROVIDER / INTRADAY_DISCOVERY_LLM_PROVIDER | 条件必填 | discovery 链路 provider 类型;支持 `fixture` / `command_json` / `http_json` |
|
||||
| INTRADAY_DISCOVERY_SEARCH_COMMAND / INTRADAY_DISCOVERY_LLM_COMMAND | 条件必填 | 当 provider 为 `command_json` 时执行的命令,stdout 必须输出 JSON |
|
||||
| INTRADAY_DISCOVERY_SEARCH_URL / INTRADAY_DISCOVERY_LLM_URL | 条件必填 | 当 provider 为 `http_json` 时调用的接口 URL |
|
||||
| INTRADAY_DISCOVERY_SEARCH_FIXTURE / INTRADAY_DISCOVERY_LLM_FIXTURE | ❌ | dry-run / 本地 fixture 输入 |
|
||||
| INTRADAY_DISCOVERY_TIMEOUT_SEC | ❌ | discovery 与验证抓取超时秒数,默认 `20` |
|
||||
| PORT | ❌ | API Server 监听端口,默认 8080 |
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 验证安装
|
||||
|
||||
```bash
|
||||
# 健康检查(仅本机 / 私网)
|
||||
curl http://localhost:8080/health
|
||||
curl http://localhost:8080/api/v1/models
|
||||
```
|
||||
|
||||
前端构建校验:
|
||||
```bash
|
||||
cd frontend
|
||||
npm run build
|
||||
```
|
||||
# API 鉴权
|
||||
curl -H "Authorization: Bearer $API_AUTH_TOKEN" http://localhost:8080/api/v1/models
|
||||
|
||||
Go 测试校验:
|
||||
```bash
|
||||
go test ./...
|
||||
# 采集器测试
|
||||
go run scripts/fetch_openrouter.go -strict-real
|
||||
|
||||
# 日报生成
|
||||
go run scripts/generate_daily_report.go
|
||||
|
||||
# 运行门禁
|
||||
bash scripts/verify_phase3.sh
|
||||
bash scripts/verify_phase5.sh
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 常见问题
|
||||
|
||||
### Q: 前端构建失败?
|
||||
确认:
|
||||
- Node.js >= 20
|
||||
- `frontend/package-lock.json` 与 `npm ci` 一致
|
||||
- 本地没有依赖已删除的 `frontend/src/data/latest_models.json`
|
||||
### Q: 数据库迁移失败?
|
||||
确保 PostgreSQL 已启动,且用户有创建表的权限。
|
||||
|
||||
### Q: `docker-compose up -d` 后页面空白?
|
||||
先执行:
|
||||
```bash
|
||||
docker-compose up -d --build
|
||||
```
|
||||
### Q: 前端构建失败?
|
||||
检查 Node.js 版本 >= 20,npm 版本 >= 10。
|
||||
|
||||
然后检查:
|
||||
```bash
|
||||
docker-compose logs -f app
|
||||
curl http://localhost:8080/
|
||||
```
|
||||
### Q: 采集器返回模拟数据?
|
||||
`fetch_openrouter.go` 在非严格模式下会降级到模拟数据;正式调度和真实流水线默认要求 `OPENROUTER_API_KEY`、真实写库成功,并会把 `run_kind / trigger_source / is_official_daily` 写入运行审计。
|
||||
|
||||
### Q: API 返回 `database not configured`?
|
||||
说明 `DATABASE_URL` 未注入或格式不正确,先执行:
|
||||
### Q: 历史重建如何执行?
|
||||
```bash
|
||||
echo "$DATABASE_URL"
|
||||
bash scripts/rebuild_historical_report.sh 2025-08-07
|
||||
```
|
||||
历史重建只会回填审计语义,不会冒充当天正式定时产出。
|
||||
|
||||
---
|
||||
|
||||
## 升级路径
|
||||
|
||||
- Phase 2: 告警订阅 / 用户系统 / 付费分析
|
||||
- Phase 3: 多数据源 / 自动发现 / ELO 评分
|
||||
- Phase 3: 多数据源 / 自动发现 / ELO评分
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
> 本文档说明宰相(AI Agent)如何在本项目内执行、验证与回收任务。
|
||||
> 版本:v1.0
|
||||
> 日期:2026-05-11
|
||||
> 日期:2026-05-14
|
||||
> 状态:与当前代码状态对齐
|
||||
|
||||
---
|
||||
@@ -23,8 +23,9 @@
|
||||
| 前端脚手架 | ✅ | `frontend/src/pages/Explorer.tsx` + 数据文件 |
|
||||
| 项目内任务管理 | ✅ | `GOALS.md` / `TASKS.md` / `AGENTS.md` |
|
||||
| 项目内记忆入口 | ✅ | `SESSION-STATE.md` / `MEMORY.md` / `memory/README.md` |
|
||||
| 验证器 | ✅ | `scripts/verification_executor.go` + 4 个 verify 脚本 |
|
||||
| OpenClaw Review | ✅ | `reports/openclaw/` 已有 13 份 review + backlog |
|
||||
| 验证器 | ✅ | `scripts/verification_executor.go` + 6 个 verify 脚本 |
|
||||
| CI 工作流 | ✅ | `.github/workflows/ci.yml` |
|
||||
| OpenClaw Review | ✅ | `reports/openclaw/` 已有 review + backlog |
|
||||
|
||||
**技术栈确认**:Go 1.22.2 + PostgreSQL + Vanilla JS/React(前端)
|
||||
|
||||
@@ -66,15 +67,46 @@
|
||||
[✅] 3. Explorer / Dashboard 最小可用前端落地
|
||||
[✅] 4. 项目内 TASKS / GOALS / verification / execution 闭环落地
|
||||
[✅] 5. 自动采集 + 日报调度闭环落地
|
||||
[✅] 6. Phase 6 综合验收通过(`verify_phase6.sh` PASS)
|
||||
[✅] 6. Phase 5 CI 工作流与 Phase 3/Phase 5 验收门禁补齐
|
||||
[🟡] 7. OpenClaw review / cron / verifier 质量治理持续优化
|
||||
[🟡] 8. Phase 2 多数据源扩展待规划
|
||||
[✅] 8. Phase 6 稳定性门禁已恢复通过,当前转入后续治理项跟踪
|
||||
```
|
||||
|
||||
**下一步优先**:
|
||||
1. 提高 review / cron / verifier 的真实性与降噪质量
|
||||
2. 推进 Phase 2 数据源扩展与真实验证入口
|
||||
3. 收口工程纪律:提交、CI、回写边界、报告一致性
|
||||
1. 继续收口 review / cron / verifier 的真实性与降噪质量,避免历史 blocker 已消失但 board 仍滞后
|
||||
2. 继续观察 Cloudflare / Perplexity / Vertex 等外部文档源的稳定性;当前 Cloudflare 已补上“代理传输失败 → 直连 fallback”兜底,但仍需区分瞬时网络抖动与真实结构漂移
|
||||
3. 维持正式日报、历史重建与手工真实复跑三条运行语义边界,防止后续优化重新串线
|
||||
|
||||
### Phase 6+ 范围定义
|
||||
|
||||
Phase 6 的结束点是:`verify_phase6.sh`、`verify_pre_phase6.sh`、正式日报主链路与 API 健康门禁已经恢复绿色,主发布闭环可以诚实复用。
|
||||
|
||||
Phase 6+ 指的是 **治理阶段**,不属于新的发布门禁,也不等于新的业务功能 phase。它覆盖的范围是:
|
||||
- review / cron / verifier / backlog / memory 的长期治理
|
||||
- release 语义、风险老化、状态一致性、噪声收敛
|
||||
- 外部 provider 漂移后的解释层、回退层与 guard 持续补强
|
||||
- 正式日报 / 历史重建 / 手工真实复跑三条运行语义的边界维护
|
||||
|
||||
Phase 6+ 的目标不是再声明“可发布”,而是防止已经恢复绿色的主链路因为治理退化再次失真。
|
||||
### 当前运行真相
|
||||
|
||||
当前可直接引用的事实是:
|
||||
|
||||
- `bash scripts/verify_phase3.sh` 已通过,正式调度链、正式日报主产物、归档副本、report_runs / daily_report 写入链均为绿色
|
||||
- `bash scripts/verify_phase5.sh` 已通过,仓库已补齐 `.github/workflows/ci.yml`,验收链对关键构建检查已有统一 coverage gate
|
||||
- `bash scripts/verify_pre_phase6.sh` 已通过,说明 Phase 1~5 门禁当前仍闭环
|
||||
- `bash scripts/run_real_pipeline.sh` 已于 `2026-05-30 15:18` 真实复跑成功,当前本机 `.env.local` / `.env` 优先级缺陷已修复后,官方 importer、多源同步、日报生成与运行记录链都能在本机真实跑通
|
||||
- `bash scripts/verify_phase6.sh` 已于 `2026-05-30 15:33` 通过:`SUMMARY pass=18 fail=0 warn=0`
|
||||
- 最近 7 次采集窗口当前输出为 `success_rate=100.00%`,且已支持把历史前置条件样本老化为 `aged_precondition_missing`,不会继续污染当前 release success-rate
|
||||
- `verify_phase6.sh` 当前已输出结构化:
|
||||
- `ROOT_CAUSE class=... source=... summary=...`
|
||||
- `RELEASE_SEMANTICS class=... gate=... policy=...`
|
||||
- `BLOCKER_SWITCH class=... old=... new=...`
|
||||
- `stability_label=...`
|
||||
- `bash scripts/verify_importer_smoke.sh`、`bash scripts/importer_smoke_gate_test.sh`、`bash scripts/pipeline_runtime_alignment_test.sh` 已通过;当前 CoresHub / 华为 MaaS / 百川 / 01.AI / SenseNova / 讯飞 / 火山方舟等官方 importer 已接入 runtime + smoke + docs 闭环
|
||||
- 正式日报、历史重建和手工真实复跑已分流到不同运行语义;非正式运行产物会进入 `reports/ad_hoc/...`,不会覆盖正式日报主路径
|
||||
- `fetchLatestReport` 默认只展示正式日报,不会把历史重建或手工真实复跑当成最新正式产出
|
||||
- 本机 cron 继续指向 `bash scripts/run_daily.sh`,且在当前用户 / 当前仓库 / 当前 `.env.local` 条件下已手工验证可以成功落盘,具备每天真实运行作为回归验证的条件
|
||||
|
||||
---
|
||||
|
||||
|
||||
227
README.md
Normal file
227
README.md
Normal file
@@ -0,0 +1,227 @@
|
||||
# LLM Intelligence Hub
|
||||
|
||||
面向 LLM 模型、定价与日报产出的情报采集项目,当前仓库提供:
|
||||
|
||||
- Go 采集脚本:采集 OpenRouter、多源补充数据与官方补录数据
|
||||
- PostgreSQL 数据层:保存模型、区域定价、订阅套餐、日报与运行审计
|
||||
- Go HTTP API:提供模型列表、套餐列表、最新正式日报入口
|
||||
- Vite + React 前端:提供 Dashboard / Explorer 两个只读页面
|
||||
- Shell 运维脚本:迁移、调度、备份、恢复、验收与性能门禁
|
||||
|
||||
## 两大模块
|
||||
|
||||
项目现在按运行职责拆成两大模块:
|
||||
|
||||
1. `情报采集与信号沉淀模块`
|
||||
负责真实采集、官方补录、套餐导入、目录核验,以及把“新模型 / 价格变化 / 官方发布 / 活动窗口”等关键信号物化到 `daily_signal_snapshot`。
|
||||
2. `日报与下游表达模块`
|
||||
负责消费 `models`、`region_pricing`、`subscription_plan`、`daily_signal_snapshot` 等结构化事实,生成 HTML / Markdown 日报;后续视频、卡片流、推送等形态也应挂在这一层。
|
||||
|
||||
## 当前能力边界
|
||||
|
||||
- 真实生产主链路是“采集/导入脚本 + PostgreSQL + 日报生成器 + API Server + Nginx”
|
||||
- 最新正式日报由 `scripts/run_daily.sh` 生成,并写入 `daily_report` / `report_runs`
|
||||
- 手工复跑使用 `scripts/run_real_pipeline.sh`,不会把产物标记成正式日报
|
||||
- 历史补跑使用 `scripts/rebuild_historical_report.sh YYYY-MM-DD`
|
||||
- 日内价格追踪使用 `scripts/run_intraday_price_watch.sh`,只刷新价格与信号,不生成正式日报
|
||||
- 日内新闻候选发现与验证使用 `scripts/run_intraday_discovery_watch.sh`,只刷新候选池、验证轨迹与已验证信号,不生成正式日报
|
||||
|
||||
- HTTP API 当前未内建认证、授权和限流;公网暴露前必须在网关层补齐
|
||||
|
||||
## 先读这些(当前真相入口)
|
||||
|
||||
- [OPENCLAW_EXECUTION.md](OPENCLAW_EXECUTION.md):当前运行真相、执行顺序、验证协议、最新 gate 口径
|
||||
- [reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md](reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md):OpenClaw 能力缺口当前台账与最新 review 增量
|
||||
- [docs/README.md](docs/README.md):文档树入口,区分 current truth / 运行文档 / 历史材料
|
||||
- [TASKS.md](TASKS.md):任务状态真相来源
|
||||
- [GOALS.md](GOALS.md):目标范围真相来源
|
||||
|
||||
## 目录概览
|
||||
|
||||
```text
|
||||
cmd/server/ Go API Server
|
||||
internal/ 通用内部库(collector、retry)
|
||||
scripts/ 采集、导入、日报、验收、运维脚本
|
||||
db/migrations/ PostgreSQL 迁移
|
||||
frontend/ Vite + React 前端
|
||||
reports/daily/ 日报产物与归档
|
||||
ops/ 运维配置(如 logrotate)
|
||||
docs/ 补充说明与上线文档
|
||||
```
|
||||
|
||||
## 本地启动
|
||||
|
||||
### 1. 准备环境
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
至少需要配置:
|
||||
|
||||
- `DATABASE_URL`
|
||||
- `OPENROUTER_API_KEY`(仅真实采集需要)
|
||||
|
||||
详细变量说明见 [docs/CONFIGURATION.md](docs/CONFIGURATION.md)。
|
||||
|
||||
### 2. 应用数据库迁移
|
||||
|
||||
```bash
|
||||
bash scripts/apply_migration.sh
|
||||
```
|
||||
|
||||
### 3. 启动 API Server
|
||||
|
||||
```bash
|
||||
go run ./cmd/server
|
||||
```
|
||||
|
||||
默认端口为 `8080`,可通过 `PORT` 覆盖。
|
||||
|
||||
### 4. 启动前端开发环境
|
||||
|
||||
```bash
|
||||
cd frontend
|
||||
npm install
|
||||
npm run dev
|
||||
```
|
||||
|
||||
## 生产运行主链路
|
||||
|
||||
### 第一模块独立运行
|
||||
|
||||
```bash
|
||||
bash scripts/run_intel_pipeline.sh
|
||||
```
|
||||
|
||||
该入口只执行第一模块:
|
||||
|
||||
1. 真实采集与多源补充
|
||||
2. 官方模型价格与套餐导入
|
||||
3. 平台目录核验
|
||||
4. 每日关键信号物化到 `daily_signal_snapshot`
|
||||
|
||||
它不会生成日报,适合先把“数据与信号层”单独跑通。
|
||||
3. 平台目录核验
|
||||
4. 每日关键信号物化到 `daily_signal_snapshot`
|
||||
5. 日内价格追踪可由 `scripts/run_intraday_price_watch.sh` 独立执行,不生成正式日报
|
||||
6. 日内新闻候选发现与验证可由 `scripts/run_intraday_discovery_watch.sh` 独立执行,不生成正式日报
|
||||
|
||||
|
||||
### 正式日报调度
|
||||
|
||||
```bash
|
||||
bash scripts/run_daily.sh
|
||||
```
|
||||
|
||||
该脚本负责:
|
||||
|
||||
1. OpenRouter 真实采集
|
||||
2. 多源补充同步
|
||||
3. 官方导入脚本执行
|
||||
4. 每日关键信号物化
|
||||
5. 数据质量检查
|
||||
6. Markdown / HTML 日报生成
|
||||
7. 日报归档
|
||||
8. `daily_report` / `report_runs` 审计写入
|
||||
9. 失败时降级复制昨日报告并可选飞书告警
|
||||
|
||||
### 手工真实复跑
|
||||
|
||||
```bash
|
||||
bash scripts/run_real_pipeline.sh
|
||||
```
|
||||
|
||||
适用于联调、排障、上线后人工验证。该入口写入:
|
||||
|
||||
- `run_kind=manual`
|
||||
- `trigger_source=pipeline`
|
||||
- `is_official_daily=false`
|
||||
|
||||
### 日内价格追踪
|
||||
|
||||
```bash
|
||||
bash scripts/run_intraday_price_watch.sh
|
||||
```
|
||||
|
||||
适用于捕捉“小米大降价”“活动窗口上线”等已知入口里的结构化价格变化。该入口只刷新价格与信号层,不写正式 `daily_report`,也不会覆盖 `latest_report` 语义。
|
||||
|
||||
### 日内新闻发现与验证
|
||||
|
||||
```bash
|
||||
bash scripts/run_intraday_discovery_watch.sh
|
||||
```
|
||||
|
||||
适用于搜索引擎 + LLM 高召回发现“当天可能发生的价格新闻 / 版本发布 / 活动窗口”,再通过官方页面 / 价格页 / docs 做验证。该入口只刷新候选池、验证轨迹与 `daily_signal_snapshot` 中的已验证事实,不写正式 `daily_report`,也不会覆盖 `latest_report` 语义。
|
||||
|
||||
### 历史补跑
|
||||
|
||||
```bash
|
||||
bash scripts/rebuild_historical_report.sh 2026-05-13
|
||||
```
|
||||
|
||||
该入口写入:
|
||||
|
||||
- `run_kind=historical_rebuild`
|
||||
- `trigger_source=rebuild_script`
|
||||
- `is_official_daily=false`
|
||||
|
||||
## 常用命令
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
bash scripts/test.sh
|
||||
bash scripts/test_importers.sh
|
||||
bash scripts/verify_pre_phase6.sh
|
||||
bash scripts/verify_phase6.sh
|
||||
bash healthcheck.sh
|
||||
cd frontend && npm run test -- --run
|
||||
cd frontend && npm run build
|
||||
```
|
||||
|
||||
说明:
|
||||
|
||||
- `go test ./...` 只覆盖 package 形式的 Go 代码(当前主要是 `cmd/server` 与 `internal/...`)
|
||||
- `bash scripts/test.sh` 只覆盖 `fetch_openrouter` 的 focused test
|
||||
- `bash scripts/test_importers.sh` 覆盖 scripts 层 importer targeted go test matrix
|
||||
- 发布前不要把 `go test ./...` 误判成“全仓脚本业务已验证”
|
||||
|
||||
## API 概览
|
||||
|
||||
- `GET /health`
|
||||
- `GET /api/v1/models`
|
||||
- `GET /api/v1/subscription-plans`
|
||||
- `GET /api/v1/reports/latest`
|
||||
- `GET /api/v1/reports/latest/markdown`
|
||||
- `GET /api/v1/reports/latest/html`
|
||||
|
||||
完整字段与示例见 [docs/API_REFERENCE.md](docs/API_REFERENCE.md)。
|
||||
|
||||
## 文档索引
|
||||
|
||||
- [docs/CONFIGURATION.md](docs/CONFIGURATION.md):环境变量、运行语义、配置约束
|
||||
- [docs/API_REFERENCE.md](docs/API_REFERENCE.md):API 入口、返回体与排障说明
|
||||
- [docs/PLAN_CATALOG_COVERAGE_MATRIX.md](docs/PLAN_CATALOG_COVERAGE_MATRIX.md):平台覆盖矩阵,区分目录基线 / 目录核验 / importer / 真实入库 / 细颗粒度价格缺口
|
||||
- [docs/NEXT_IMPORTER_RUNTIME_PRIORITY.md](docs/NEXT_IMPORTER_RUNTIME_PRIORITY.md):下一批 importer / runtime 挂载优先清单,按 P0/P1/P2 给出最短闭环顺序
|
||||
- [docs/PRODUCTION_CHECKLIST.md](docs/PRODUCTION_CHECKLIST.md):生产上线前检查、发布与回滚流程
|
||||
- [DEPLOYMENT.md](DEPLOYMENT.md):部署步骤与快速启动
|
||||
- [RUNBOOK.md](RUNBOOK.md):运维巡检、故障排查、备份恢复
|
||||
- [TECHNICAL_DESIGN.md](TECHNICAL_DESIGN.md):详细技术设计与数据模型演进背景
|
||||
- [docs/PERFORMANCE_TEST.md](docs/PERFORMANCE_TEST.md):性能基线
|
||||
|
||||
## 生产上线最低门禁
|
||||
|
||||
建议把以下检查作为发布前硬门禁:
|
||||
|
||||
```bash
|
||||
bash scripts/verify_pre_phase6.sh
|
||||
bash scripts/verify_phase6.sh
|
||||
```
|
||||
|
||||
上线后首轮冒烟建议至少覆盖:
|
||||
|
||||
```bash
|
||||
curl -fsS http://127.0.0.1:8080/health
|
||||
curl -fsS http://127.0.0.1:8080/api/v1/models
|
||||
curl -fsS http://127.0.0.1:8080/api/v1/reports/latest
|
||||
```
|
||||
77
RUNBOOK.md
77
RUNBOOK.md
@@ -1,7 +1,14 @@
|
||||
# LLM Intelligence Hub - 运维手册
|
||||
|
||||
> 版本: v1.1
|
||||
> 日期: 2026-05-21
|
||||
> 版本: v1.1
|
||||
> 日期: 2026-05-14
|
||||
> 适用版本: Phase 3 / Phase 5
|
||||
|
||||
相关文档:
|
||||
|
||||
- `docs/PRODUCTION_CHECKLIST.md`:上线前门禁、发布步骤、回滚流程
|
||||
- `docs/CONFIGURATION.md`:环境变量与产物路径约定
|
||||
- `docs/API_REFERENCE.md`:健康检查与只读接口说明
|
||||
|
||||
---
|
||||
|
||||
@@ -9,7 +16,7 @@
|
||||
|
||||
### 启动全部服务
|
||||
```bash
|
||||
docker-compose up -d --build
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
### 停止服务
|
||||
@@ -27,21 +34,19 @@ docker-compose logs -f db
|
||||
|
||||
## 日常巡检
|
||||
|
||||
### 应用健康
|
||||
```bash
|
||||
curl http://localhost:8080/health
|
||||
curl http://localhost:8080/api/v1/models
|
||||
```
|
||||
|
||||
### 数据库健康
|
||||
```bash
|
||||
psql "$DATABASE_URL" -c "SELECT COUNT(*) FROM models WHERE deleted_at IS NULL"
|
||||
psql "$DATABASE_URL" -c "SELECT source, success, created_at FROM collector_stats ORDER BY created_at DESC LIMIT 5"
|
||||
psql "$DATABASE_URL" -c "SELECT report_date, run_kind, trigger_source, is_official_daily, status FROM daily_report ORDER BY updated_at DESC LIMIT 5"
|
||||
psql "$DATABASE_URL" -c "SELECT report_date, run_kind, trigger_source, is_official_daily, status FROM report_runs ORDER BY report_date DESC, created_at DESC LIMIT 5"
|
||||
```
|
||||
|
||||
### 日报检查
|
||||
```bash
|
||||
ls -la reports/daily/daily_report_$(date +%Y-%m-%d).md
|
||||
ls -la reports/daily/html/daily_report_$(date +%Y-%m-%d).html
|
||||
ls -la reports/daily/$(date +%Y)/$(date +%m)/daily_report_$(date +%Y-%m-%d).md
|
||||
```
|
||||
|
||||
### 磁盘空间
|
||||
@@ -66,19 +71,20 @@ df -h /tmp
|
||||
|
||||
### 日报未生成
|
||||
1. 检查 cron: `crontab -l | grep llm-intelligence`
|
||||
2. 手动执行: `bash scripts/run_daily.sh`
|
||||
3. 检查最近日报: `ls reports/daily/*.md | tail -1`
|
||||
2. 手动运行: `bash scripts/run_daily.sh`
|
||||
3. 检查降级报告: `ls reports/daily/*.md | tail -1`
|
||||
4. 如果是历史补跑,使用 `REPORT_RUN_KIND=historical_rebuild` 和 `REPORT_TRIGGER_SOURCE=rebuild_script`,不要当作正式定时产出读取
|
||||
|
||||
### 正式日报与历史重建
|
||||
- 正式定时产出由 `scripts/run_daily.sh` 生成,`is_official_daily=true`
|
||||
- 真实复跑由 `scripts/run_real_pipeline.sh` 负责,通常用于手工验证真实采集 + 真实写库 + 报告生成
|
||||
- 历史重建通过 `scripts/rebuild_historical_report.sh <date>` 执行,运行语义应保持 `run_kind=historical_rebuild`
|
||||
- 前端 `/api/v1/reports/latest` 默认只读正式日报,不会把历史重建当成最新正式产出
|
||||
|
||||
### 前端无法访问
|
||||
1. 检查应用容器: `docker-compose ps app`
|
||||
2. 检查首页响应: `curl -I http://localhost:8080/`
|
||||
3. 检查 API 响应: `curl http://localhost:8080/api/v1/models`
|
||||
4. 查看应用日志: `docker-compose logs -f app`
|
||||
|
||||
### 静态资源 404
|
||||
1. 重新构建镜像: `docker-compose up -d --build`
|
||||
2. 本地校验前端构建: `cd frontend && npm run build`
|
||||
3. 确认容器内含有前端产物: `docker-compose exec app ls /app/frontend/dist`
|
||||
1. 检查 Nginx: `docker-compose ps nginx`
|
||||
2. 检查 dist: `ls frontend/dist/`
|
||||
3. 检查端口: `netstat -tlnp | grep 80`
|
||||
|
||||
---
|
||||
|
||||
@@ -94,7 +100,7 @@ bash scripts/backup.sh
|
||||
gunzip < backup_file.sql.gz | psql "$DATABASE_URL"
|
||||
```
|
||||
|
||||
### 定时备份
|
||||
### 定时备份 (cron)
|
||||
```bash
|
||||
0 2 * * * cd /path/to/llm-intelligence && bash scripts/backup.sh >> /tmp/backup.log 2>&1
|
||||
```
|
||||
@@ -105,14 +111,33 @@ gunzip < backup_file.sql.gz | psql "$DATABASE_URL"
|
||||
|
||||
| 指标 | 告警阈值 | 检查命令 |
|
||||
|------|----------|----------|
|
||||
| 模型数 | `< 300` | `SELECT COUNT(*) FROM models` |
|
||||
| 采集成功率 | `< 95%` | `SELECT success_rate FROM collector_stats` |
|
||||
| 模型数 | < 300 | `SELECT COUNT(*) FROM models` |
|
||||
| 采集成功率 | < 95% | `SELECT success_rate FROM collector_stats` |
|
||||
| 数据库连接 | 失败 | `pg_isready` |
|
||||
| 磁盘空间 | `> 80%` | `df -h` |
|
||||
| 磁盘空间 | > 80% | `df -h` |
|
||||
|
||||
## 运行审计
|
||||
|
||||
正式日报与历史重建现在会写入运行语义字段,排障时优先看这些字段:
|
||||
|
||||
- `run_kind`: `scheduled` / `historical_rebuild` / `manual`
|
||||
- `trigger_source`: `cron` / `rebuild_script` / `pipeline`
|
||||
- `is_official_daily`: 是否属于当天定时正式产出
|
||||
- `summary_md`: 真实运行审计前缀 + 报告摘要
|
||||
|
||||
---
|
||||
|
||||
## 扩容指南
|
||||
|
||||
### 垂直扩容
|
||||
增加 PostgreSQL 内存和 CPU。
|
||||
|
||||
### 水平扩容
|
||||
使用读写分离或分片(Phase 2+)。
|
||||
|
||||
---
|
||||
|
||||
## 联系信息
|
||||
|
||||
- 维护者: 宅相
|
||||
- 项目路径: `D:\project\llm-intelligence`
|
||||
- 维护者: 宰相
|
||||
- 项目路径: /home/long/project/llm-intelligence
|
||||
|
||||
4
TASKS.md
4
TASKS.md
@@ -384,8 +384,8 @@
|
||||
- **交付语义**:实现完成,代表腾讯云套餐订阅价已具备独立 API 查询入口;前端消费和展示增强仍可后续单独演进
|
||||
- **verification**:
|
||||
- mode: `test_pass`
|
||||
- command: `cd /home/long/project/llm-intelligence && go test ./cmd/server >/tmp/llm_tdata9_test.log 2>&1 && bash scripts/verify_phase6.sh`
|
||||
- expected_evidence: `PHASE_RESULT: PASS`
|
||||
- command: `cd /home/long/project/llm-intelligence && go test ./cmd/server -run TestSubscriptionPlansHandlerReturnsEnvelope >/tmp/llm_tdata9_test.log 2>&1 && echo runtime-ok`
|
||||
- expected_evidence: `runtime-ok`
|
||||
- evidence_grade: `runtime-verified`
|
||||
- task_type: `automation`
|
||||
- timeout_seconds: 180
|
||||
|
||||
@@ -4,12 +4,15 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
@@ -22,12 +25,15 @@ type modelResponse struct {
|
||||
ProviderCN string `json:"providerCN"`
|
||||
Modality string `json:"modality"`
|
||||
ContextLength int `json:"contextLength"`
|
||||
PricingMode string `json:"pricingMode,omitempty"`
|
||||
PriceUnit string `json:"priceUnit,omitempty"`
|
||||
FlatPrice float64 `json:"flatPrice,omitempty"`
|
||||
InputPrice float64 `json:"inputPrice"`
|
||||
OutputPrice float64 `json:"outputPrice"`
|
||||
Currency string `json:"currency"`
|
||||
IsFree bool `json:"isFree"`
|
||||
Stale bool `json:"stale"`
|
||||
DataConfidence string `json:"dataConfidence"`
|
||||
DataConfidence string `json:"dataConfidence"`
|
||||
}
|
||||
|
||||
type subscriptionPlanResponse struct {
|
||||
@@ -52,11 +58,199 @@ type subscriptionPlanResponse struct {
|
||||
}
|
||||
|
||||
type apiEnvelope struct {
|
||||
Data any `json:"data"`
|
||||
Data any `json:"data,omitempty"`
|
||||
Error *apiError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type apiError struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type modelFetcher func(context.Context, *sql.DB) ([]modelResponse, error)
|
||||
type subscriptionPlanFetcher func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error)
|
||||
type latestReportFetcher func(context.Context, *sql.DB) (*latestReportResponse, error)
|
||||
|
||||
type latestReportResponse struct {
|
||||
ReportDate string `json:"reportDate"`
|
||||
Status string `json:"status"`
|
||||
ModelCount int `json:"modelCount"`
|
||||
SummaryMD string `json:"summaryMD"`
|
||||
MarkdownPath string `json:"markdownPath"`
|
||||
HTMLPath string `json:"htmlPath"`
|
||||
ArchiveMarkdownPath string `json:"archiveMarkdownPath"`
|
||||
ArchiveHTMLPath string `json:"archiveHtmlPath"`
|
||||
MarkdownURL string `json:"markdownUrl"`
|
||||
HTMLURL string `json:"htmlUrl"`
|
||||
UpdatedAt string `json:"updatedAt"`
|
||||
AppendixJSONURL string `json:"appendixJsonUrl"`
|
||||
|
||||
}
|
||||
|
||||
type serverConfig struct {
|
||||
BasicAuthUser string
|
||||
BasicAuthPass string
|
||||
ServiceToken string
|
||||
RateLimitPerWindow int
|
||||
RateLimitWindow time.Duration
|
||||
now func() time.Time
|
||||
limiter *ipRateLimiter
|
||||
}
|
||||
|
||||
type ipRateLimiter struct {
|
||||
mu sync.Mutex
|
||||
limit int
|
||||
window time.Duration
|
||||
entries map[string]rateLimitEntry
|
||||
}
|
||||
|
||||
type rateLimitEntry struct {
|
||||
windowStart time.Time
|
||||
count int
|
||||
}
|
||||
|
||||
func newIPRateLimiter(limit int, window time.Duration) *ipRateLimiter {
|
||||
if limit <= 0 || window <= 0 {
|
||||
return nil
|
||||
}
|
||||
return &ipRateLimiter{
|
||||
limit: limit,
|
||||
window: window,
|
||||
entries: make(map[string]rateLimitEntry),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *ipRateLimiter) Allow(key string, now time.Time) bool {
|
||||
if l == nil {
|
||||
return true
|
||||
}
|
||||
if key == "" {
|
||||
key = "unknown"
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
entry := l.entries[key]
|
||||
if entry.windowStart.IsZero() || now.Sub(entry.windowStart) >= l.window {
|
||||
entry = rateLimitEntry{windowStart: now}
|
||||
}
|
||||
if entry.count >= l.limit {
|
||||
return false
|
||||
}
|
||||
entry.count++
|
||||
l.entries[key] = entry
|
||||
|
||||
for candidate, candidateEntry := range l.entries {
|
||||
if now.Sub(candidateEntry.windowStart) >= l.window {
|
||||
delete(l.entries, candidate)
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func loadServerConfigFromEnv() serverConfig {
|
||||
limit := 60
|
||||
if raw := strings.TrimSpace(os.Getenv("API_RATE_LIMIT_PER_WINDOW")); raw != "" {
|
||||
if parsed, err := strconv.Atoi(raw); err == nil && parsed >= 0 {
|
||||
limit = parsed
|
||||
}
|
||||
}
|
||||
|
||||
window := time.Minute
|
||||
if raw := strings.TrimSpace(os.Getenv("API_RATE_LIMIT_WINDOW_SEC")); raw != "" {
|
||||
if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 {
|
||||
window = time.Duration(parsed) * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
return serverConfig{
|
||||
BasicAuthUser: os.Getenv("API_BASIC_AUTH_USER"),
|
||||
BasicAuthPass: os.Getenv("API_BASIC_AUTH_PASS"),
|
||||
ServiceToken: os.Getenv("API_AUTH_TOKEN"),
|
||||
RateLimitPerWindow: limit,
|
||||
RateLimitWindow: window,
|
||||
}
|
||||
}
|
||||
|
||||
func (cfg serverConfig) withRuntimeDefaults() serverConfig {
|
||||
if cfg.now == nil {
|
||||
cfg.now = time.Now
|
||||
}
|
||||
if cfg.limiter == nil {
|
||||
cfg.limiter = newIPRateLimiter(cfg.RateLimitPerWindow, cfg.RateLimitWindow)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func (cfg serverConfig) wrap(path string, next http.HandlerFunc) http.HandlerFunc {
|
||||
cfg = cfg.withRuntimeDefaults()
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
clientIP := requestClientIP(r)
|
||||
trustedClient := isTrustedClientIP(clientIP)
|
||||
|
||||
if path == "/health" && !trustedClient {
|
||||
writeError(w, http.StatusForbidden, "health_endpoint_internal_only", "health endpoint is restricted to trusted networks")
|
||||
return
|
||||
}
|
||||
|
||||
if path != "/health" && !trustedClient {
|
||||
if !cfg.isAuthorized(r) {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="llm-intelligence"`)
|
||||
writeError(w, http.StatusUnauthorized, "auth_required", "authentication required for external API access")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if path != "/health" && cfg.limiter != nil {
|
||||
if !cfg.limiter.Allow(clientIP, cfg.now()) {
|
||||
writeError(w, http.StatusTooManyRequests, "rate_limited", "rate limit exceeded")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
next(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
func (cfg serverConfig) isAuthorized(r *http.Request) bool {
|
||||
authHeader := strings.TrimSpace(r.Header.Get("Authorization"))
|
||||
if cfg.ServiceToken != "" {
|
||||
const bearerPrefix = "Bearer "
|
||||
if strings.HasPrefix(authHeader, bearerPrefix) {
|
||||
return strings.TrimSpace(strings.TrimPrefix(authHeader, bearerPrefix)) == cfg.ServiceToken
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.BasicAuthUser == "" && cfg.BasicAuthPass == "" {
|
||||
return false
|
||||
}
|
||||
username, password, ok := r.BasicAuth()
|
||||
return ok && username == cfg.BasicAuthUser && password == cfg.BasicAuthPass
|
||||
}
|
||||
|
||||
func requestClientIP(r *http.Request) string {
|
||||
if forwardedFor := strings.TrimSpace(r.Header.Get("X-Forwarded-For")); forwardedFor != "" {
|
||||
parts := strings.Split(forwardedFor, ",")
|
||||
if len(parts) > 0 {
|
||||
return strings.TrimSpace(parts[0])
|
||||
}
|
||||
}
|
||||
|
||||
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
|
||||
if err == nil {
|
||||
return host
|
||||
}
|
||||
return strings.TrimSpace(r.RemoteAddr)
|
||||
}
|
||||
|
||||
func isTrustedClientIP(raw string) bool {
|
||||
ip := net.ParseIP(strings.TrimSpace(raw))
|
||||
if ip == nil {
|
||||
return false
|
||||
}
|
||||
return ip.IsLoopback() || ip.IsPrivate()
|
||||
}
|
||||
|
||||
func main() {
|
||||
addr := os.Getenv("PORT")
|
||||
@@ -78,7 +272,7 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
mux := newMux(db, fetchModels, fetchSubscriptionPlans, resolveFrontendDistDir())
|
||||
mux := newMuxWithConfig(db, fetchModels, fetchSubscriptionPlans, fetchLatestReport, loadServerConfigFromEnv())
|
||||
|
||||
log.Printf("server listening on :%s", addr)
|
||||
if err := http.ListenAndServe(":"+addr, mux); err != nil {
|
||||
@@ -86,117 +280,108 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
func newMux(db *sql.DB, fetchModelsFn modelFetcher, fetchPlansFn subscriptionPlanFetcher, frontendDistDir string) *http.ServeMux {
|
||||
func newMux(db *sql.DB, fetchModelsFn modelFetcher, fetchPlansFn subscriptionPlanFetcher, fetchLatestReportFn latestReportFetcher) *http.ServeMux {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
registerRoutes(mux, db, fetchModelsFn, fetchPlansFn, fetchLatestReportFn, func(_ string, handler http.HandlerFunc) http.HandlerFunc {
|
||||
return handler
|
||||
})
|
||||
return mux
|
||||
}
|
||||
|
||||
func newMuxWithConfig(db *sql.DB, fetchModelsFn modelFetcher, fetchPlansFn subscriptionPlanFetcher, fetchLatestReportFn latestReportFetcher, cfg serverConfig) *http.ServeMux {
|
||||
mux := http.NewServeMux()
|
||||
registerRoutes(mux, db, fetchModelsFn, fetchPlansFn, fetchLatestReportFn, cfg.wrap)
|
||||
return mux
|
||||
}
|
||||
|
||||
func registerRoutes(mux *http.ServeMux, db *sql.DB, fetchModelsFn modelFetcher, fetchPlansFn subscriptionPlanFetcher, fetchLatestReportFn latestReportFetcher, wrap func(string, http.HandlerFunc) http.HandlerFunc) {
|
||||
mux.HandleFunc("/health", wrap("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
if db == nil {
|
||||
http.Error(w, "database not configured", http.StatusServiceUnavailable)
|
||||
writeError(w, http.StatusServiceUnavailable, "database_not_configured", "database not configured")
|
||||
return
|
||||
}
|
||||
if err := db.PingContext(r.Context()); err != nil {
|
||||
http.Error(w, "database unavailable", http.StatusServiceUnavailable)
|
||||
writeError(w, http.StatusServiceUnavailable, "database_unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||
})
|
||||
mux.HandleFunc("/api/v1/models", func(w http.ResponseWriter, r *http.Request) {
|
||||
}))
|
||||
mux.HandleFunc("/api/v1/models", wrap("/api/v1/models", func(w http.ResponseWriter, r *http.Request) {
|
||||
if db == nil {
|
||||
http.Error(w, "database not configured", http.StatusServiceUnavailable)
|
||||
writeError(w, http.StatusServiceUnavailable, "database_not_configured", "database not configured")
|
||||
return
|
||||
}
|
||||
models, err := fetchModelsFn(r.Context(), db)
|
||||
if err != nil {
|
||||
http.Error(w, "query failed", http.StatusInternalServerError)
|
||||
writeError(w, http.StatusInternalServerError, "query_failed", "query failed")
|
||||
log.Printf("fetch models failed: %v", err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, apiEnvelope{Data: models})
|
||||
})
|
||||
mux.HandleFunc("/api/v1/subscription-plans", func(w http.ResponseWriter, r *http.Request) {
|
||||
}))
|
||||
mux.HandleFunc("/api/v1/subscription-plans", wrap("/api/v1/subscription-plans", func(w http.ResponseWriter, r *http.Request) {
|
||||
if db == nil {
|
||||
http.Error(w, "database not configured", http.StatusServiceUnavailable)
|
||||
writeError(w, http.StatusServiceUnavailable, "database_not_configured", "database not configured")
|
||||
return
|
||||
}
|
||||
plans, err := fetchPlansFn(r.Context(), db)
|
||||
if err != nil {
|
||||
http.Error(w, "query failed", http.StatusInternalServerError)
|
||||
writeError(w, http.StatusInternalServerError, "query_failed", "query failed")
|
||||
log.Printf("fetch subscription plans failed: %v", err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, apiEnvelope{Data: plans})
|
||||
})
|
||||
if frontendDistDir != "" {
|
||||
mux.Handle("/", frontendHandler(frontendDistDir))
|
||||
}
|
||||
return mux
|
||||
}))
|
||||
mux.HandleFunc("/api/v1/reports/latest/html", wrap("/api/v1/reports/latest/html", func(w http.ResponseWriter, r *http.Request) {
|
||||
serveLatestReportArtifact(w, r, db, fetchLatestReportFn, "html")
|
||||
}))
|
||||
mux.HandleFunc("/api/v1/reports/latest/markdown", wrap("/api/v1/reports/latest/markdown", func(w http.ResponseWriter, r *http.Request) {
|
||||
serveLatestReportArtifact(w, r, db, fetchLatestReportFn, "markdown")
|
||||
}))
|
||||
mux.HandleFunc("/api/v1/reports/latest", wrap("/api/v1/reports/latest", func(w http.ResponseWriter, r *http.Request) {
|
||||
if db == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "database_not_configured", "database not configured")
|
||||
return
|
||||
}
|
||||
report, err := fetchLatestReportFn(r.Context(), db)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
writeError(w, http.StatusNotFound, "latest_report_not_found", "latest report not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "query_failed", "query failed")
|
||||
log.Printf("fetch latest report failed: %v", err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, apiEnvelope{Data: report})
|
||||
}))
|
||||
}
|
||||
|
||||
func resolveFrontendDistDir() string {
|
||||
candidates := []string{}
|
||||
if custom := os.Getenv("FRONTEND_DIST_DIR"); custom != "" {
|
||||
candidates = append(candidates, custom)
|
||||
}
|
||||
|
||||
candidates = append(candidates,
|
||||
filepath.Join("frontend", "dist"),
|
||||
filepath.Join(filepath.Dir(os.Args[0]), "frontend", "dist"),
|
||||
)
|
||||
|
||||
for _, candidate := range candidates {
|
||||
indexPath := filepath.Join(candidate, "index.html")
|
||||
info, err := os.Stat(indexPath)
|
||||
if err == nil && !info.IsDir() {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func frontendHandler(frontendDistDir string) http.Handler {
|
||||
indexPath := filepath.Join(frontendDistDir, "index.html")
|
||||
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
cleanPath := path.Clean("/" + r.URL.Path)
|
||||
if cleanPath == "/" {
|
||||
http.ServeFile(w, r, indexPath)
|
||||
return
|
||||
}
|
||||
|
||||
relativePath := strings.TrimPrefix(cleanPath, "/")
|
||||
assetPath := filepath.Join(frontendDistDir, filepath.FromSlash(relativePath))
|
||||
if info, err := os.Stat(assetPath); err == nil && !info.IsDir() {
|
||||
http.ServeFile(w, r, assetPath)
|
||||
return
|
||||
}
|
||||
|
||||
if filepath.Ext(relativePath) != "" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
http.ServeFile(w, r, indexPath)
|
||||
})
|
||||
}
|
||||
|
||||
func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) {
|
||||
rows, err := db.QueryContext(ctx, `
|
||||
WITH latest_prices AS (
|
||||
const fetchModelsQuery = `
|
||||
WITH ranked_prices AS (
|
||||
SELECT
|
||||
model_id,
|
||||
input_price_per_mtok,
|
||||
output_price_per_mtok,
|
||||
currency,
|
||||
rp.model_id,
|
||||
rp.pricing_mode,
|
||||
rp.price_unit,
|
||||
rp.flat_price,
|
||||
rp.input_price_per_mtok,
|
||||
rp.output_price_per_mtok,
|
||||
rp.currency,
|
||||
rp.is_free,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY model_id
|
||||
ORDER BY effective_date DESC NULLS LAST, id DESC
|
||||
PARTITION BY rp.model_id
|
||||
ORDER BY
|
||||
CASE WHEN lower(rp.region) = 'global' THEN 0 ELSE 1 END,
|
||||
CASE rp.source_type
|
||||
WHEN 'official' THEN 0
|
||||
WHEN 'reseller' THEN 1
|
||||
WHEN 'free_tier' THEN 2
|
||||
ELSE 3
|
||||
END,
|
||||
rp.effective_date DESC NULLS LAST,
|
||||
rp.id DESC
|
||||
) AS rn
|
||||
FROM model_prices
|
||||
FROM region_pricing rp
|
||||
)
|
||||
SELECT
|
||||
m.external_id,
|
||||
@@ -205,17 +390,23 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) {
|
||||
COALESCE(mp.name, split_part(m.external_id, '/', 1)),
|
||||
COALESCE(m.modality, 'text'),
|
||||
COALESCE(m.context_length, 0),
|
||||
COALESCE(lp.pricing_mode, 'input_output'),
|
||||
COALESCE(lp.price_unit, 'million_tokens'),
|
||||
COALESCE(lp.flat_price, 0),
|
||||
lp.input_price_per_mtok,
|
||||
lp.output_price_per_mtok,
|
||||
COALESCE(lp.currency, 'USD'),
|
||||
COALESCE(m.is_free, false),
|
||||
COALESCE(lp.is_free, m.is_free, false),
|
||||
COALESCE(m.data_confidence, 'official')
|
||||
FROM models m
|
||||
LEFT JOIN model_provider mp ON mp.id = m.provider_id
|
||||
LEFT JOIN latest_prices lp ON lp.model_id = m.id AND lp.rn = 1
|
||||
LEFT JOIN ranked_prices lp ON lp.model_id = m.id AND lp.rn = 1
|
||||
WHERE m.deleted_at IS NULL
|
||||
ORDER BY m.id DESC
|
||||
`)
|
||||
`
|
||||
|
||||
func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) {
|
||||
rows, err := db.QueryContext(ctx, fetchModelsQuery)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -224,6 +415,7 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) {
|
||||
var models []modelResponse
|
||||
for rows.Next() {
|
||||
var model modelResponse
|
||||
var flatPrice sql.NullFloat64
|
||||
var inputPrice sql.NullFloat64
|
||||
var outputPrice sql.NullFloat64
|
||||
if err := rows.Scan(
|
||||
@@ -233,6 +425,9 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) {
|
||||
&model.Provider,
|
||||
&model.Modality,
|
||||
&model.ContextLength,
|
||||
&model.PricingMode,
|
||||
&model.PriceUnit,
|
||||
&flatPrice,
|
||||
&inputPrice,
|
||||
&outputPrice,
|
||||
&model.Currency,
|
||||
@@ -249,12 +444,110 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) {
|
||||
if outputPrice.Valid {
|
||||
model.OutputPrice = outputPrice.Float64
|
||||
}
|
||||
if flatPrice.Valid {
|
||||
model.FlatPrice = flatPrice.Float64
|
||||
}
|
||||
model.Stale = model.DataConfidence == "stale"
|
||||
models = append(models, model)
|
||||
}
|
||||
return models, rows.Err()
|
||||
}
|
||||
|
||||
func fetchLatestReport(ctx context.Context, db *sql.DB) (*latestReportResponse, error) {
|
||||
var report latestReportResponse
|
||||
var markdownPath string
|
||||
|
||||
err := db.QueryRowContext(ctx, `
|
||||
SELECT
|
||||
TO_CHAR(report_date, 'YYYY-MM-DD'),
|
||||
status,
|
||||
COALESCE(model_count, 0),
|
||||
COALESCE(summary_md, ''),
|
||||
COALESCE(output_path, ''),
|
||||
COALESCE(TO_CHAR(updated_at, 'YYYY-MM-DD"T"HH24:MI:SS'), '')
|
||||
FROM daily_report
|
||||
WHERE output_path IS NOT NULL
|
||||
AND output_path <> ''
|
||||
AND status = 'generated'
|
||||
AND COALESCE(is_official_daily, true) = true
|
||||
ORDER BY report_date DESC, updated_at DESC
|
||||
LIMIT 1
|
||||
`).Scan(
|
||||
&report.ReportDate,
|
||||
&report.Status,
|
||||
&report.ModelCount,
|
||||
&report.SummaryMD,
|
||||
&markdownPath,
|
||||
&report.UpdatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
report.MarkdownPath = filepath.ToSlash(markdownPath)
|
||||
report.HTMLPath = deriveReportHTMLPath(markdownPath, report.ReportDate)
|
||||
report.ArchiveMarkdownPath = deriveReportArchivePath(markdownPath, report.ReportDate)
|
||||
report.ArchiveHTMLPath = deriveReportArchivePath(report.HTMLPath, report.ReportDate)
|
||||
report.MarkdownURL = "/api/v1/reports/latest/markdown"
|
||||
report.HTMLURL = "/api/v1/reports/latest/html"
|
||||
report.AppendixJSONURL = "/reports/daily/appendix/" + report.ReportDate + "/full_appendix.json"
|
||||
return &report, nil
|
||||
}
|
||||
|
||||
func serveLatestReportArtifact(w http.ResponseWriter, r *http.Request, db *sql.DB, fetchLatestReportFn latestReportFetcher, artifactType string) {
|
||||
if db == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "database_not_configured", "database not configured")
|
||||
return
|
||||
}
|
||||
|
||||
report, err := fetchLatestReportFn(r.Context(), db)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
writeError(w, http.StatusNotFound, "latest_report_not_found", "latest report not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "query_failed", "query failed")
|
||||
log.Printf("fetch latest report failed: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
targetPath := report.MarkdownPath
|
||||
if artifactType == "html" {
|
||||
targetPath = report.HTMLPath
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
} else {
|
||||
w.Header().Set("Content-Type", "text/markdown; charset=utf-8")
|
||||
}
|
||||
|
||||
if _, err := os.Stat(targetPath); err != nil {
|
||||
writeError(w, http.StatusNotFound, "report_artifact_not_found", "report artifact not found")
|
||||
return
|
||||
}
|
||||
|
||||
http.ServeFile(w, r, targetPath)
|
||||
}
|
||||
|
||||
func deriveReportHTMLPath(markdownPath, reportDate string) string {
|
||||
reportFile := filepath.Base(markdownPath)
|
||||
if reportFile == "." || reportFile == "" {
|
||||
reportFile = fmt.Sprintf("daily_report_%s.md", reportDate)
|
||||
}
|
||||
htmlFile := strings.TrimSuffix(reportFile, filepath.Ext(reportFile)) + ".html"
|
||||
reportDir := filepath.Dir(markdownPath)
|
||||
if reportDir == "." || reportDir == "" {
|
||||
reportDir = "reports/daily"
|
||||
}
|
||||
return filepath.ToSlash(filepath.Join(reportDir, "html", htmlFile))
|
||||
}
|
||||
|
||||
func deriveReportArchivePath(reportPath, reportDate string) string {
|
||||
reportFile := filepath.Base(reportPath)
|
||||
if reportFile == "." || reportFile == "" {
|
||||
reportFile = fmt.Sprintf("daily_report_%s.md", reportDate)
|
||||
}
|
||||
return filepath.ToSlash(filepath.Join("reports/daily", reportDate[:4], reportDate[5:7], reportFile))
|
||||
}
|
||||
|
||||
func fetchSubscriptionPlans(ctx context.Context, db *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
rows, err := db.QueryContext(ctx, `
|
||||
SELECT
|
||||
@@ -324,6 +617,10 @@ func writeJSON(w http.ResponseWriter, status int, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
if err := json.NewEncoder(w).Encode(value); err != nil {
|
||||
http.Error(w, "encode failed", http.StatusInternalServerError)
|
||||
log.Printf("encode response failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, code, message string) {
|
||||
writeJSON(w, status, apiEnvelope{Error: &apiError{Code: code, Message: message}})
|
||||
}
|
||||
|
||||
@@ -7,11 +7,185 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestModelsHandlerReturnsFlatPricingFields(t *testing.T) {
|
||||
mux := newMux(
|
||||
&sql.DB{},
|
||||
func(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return []modelResponse{{
|
||||
ID: "mobile-cloud-huabei-huhehaote-cosyvoice",
|
||||
Name: "CosyVoice",
|
||||
Provider: "Alibaba",
|
||||
ProviderCN: "阿里云",
|
||||
Modality: "audio",
|
||||
PricingMode: "flat",
|
||||
PriceUnit: "10k_characters",
|
||||
FlatPrice: 2,
|
||||
Currency: "CNY",
|
||||
IsFree: false,
|
||||
DataConfidence: "official",
|
||||
}}, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return nil, sql.ErrNoRows
|
||||
},
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/models", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", rec.Code)
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
Data []modelResponse `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("unmarshal response: %v", err)
|
||||
}
|
||||
if len(payload.Data) != 1 {
|
||||
t.Fatalf("expected 1 model, got %d", len(payload.Data))
|
||||
}
|
||||
got := payload.Data[0]
|
||||
if got.PricingMode != "flat" || got.PriceUnit != "10k_characters" || got.FlatPrice != 2 {
|
||||
t.Fatalf("unexpected flat pricing payload: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelsHandlerReturnsJSONErrorEnvelope(t *testing.T) {
|
||||
mux := newMux(
|
||||
nil,
|
||||
func(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return nil, sql.ErrNoRows
|
||||
},
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/models", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("expected status 503, got %d", rec.Code)
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
Error struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("unmarshal error response: %v", err)
|
||||
}
|
||||
if payload.Error.Code != "database_not_configured" {
|
||||
t.Fatalf("unexpected error code: %q", payload.Error.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthHandlerReturnsJSONErrorEnvelope(t *testing.T) {
|
||||
mux := newMux(
|
||||
nil,
|
||||
func(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return nil, sql.ErrNoRows
|
||||
},
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/health", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("expected status 503, got %d", rec.Code)
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
Error struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("unmarshal health error response: %v", err)
|
||||
}
|
||||
if payload.Error.Code != "database_not_configured" {
|
||||
t.Fatalf("unexpected error code: %q", payload.Error.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLatestReportHTMLHandlerReturnsJSONErrorEnvelope(t *testing.T) {
|
||||
mux := newMux(
|
||||
&sql.DB{},
|
||||
func(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return nil, sql.ErrNoRows
|
||||
},
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/reports/latest/html", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("expected status 404, got %d", rec.Code)
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
Error struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("unmarshal latest html error response: %v", err)
|
||||
}
|
||||
if payload.Error.Code != "latest_report_not_found" {
|
||||
t.Fatalf("unexpected error code: %q", payload.Error.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchModelsQueryEncodesPrimaryPricePriority(t *testing.T) {
|
||||
fragments := []string{
|
||||
"CASE WHEN lower(rp.region) = 'global' THEN 0 ELSE 1 END",
|
||||
"WHEN 'official' THEN 0",
|
||||
"WHEN 'reseller' THEN 1",
|
||||
"WHEN 'free_tier' THEN 2",
|
||||
"rp.effective_date DESC NULLS LAST",
|
||||
"rp.id DESC",
|
||||
}
|
||||
|
||||
for _, fragment := range fragments {
|
||||
if !strings.Contains(fetchModelsQuery, fragment) {
|
||||
t.Fatalf("fetchModelsQuery missing fragment %q", fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubscriptionPlansHandlerReturnsEnvelope(t *testing.T) {
|
||||
mux := newMux(
|
||||
&sql.DB{},
|
||||
@@ -23,7 +197,7 @@ func TestSubscriptionPlansHandlerReturnsEnvelope(t *testing.T) {
|
||||
{
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: "token-plan-lite",
|
||||
PlanName: "General Token Plan Lite",
|
||||
PlanName: "通用 Token Plan Lite",
|
||||
Tier: "Lite",
|
||||
Provider: "Tencent",
|
||||
ProviderCN: "腾讯",
|
||||
@@ -41,7 +215,9 @@ func TestSubscriptionPlansHandlerReturnsEnvelope(t *testing.T) {
|
||||
},
|
||||
}, nil
|
||||
},
|
||||
"",
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return nil, sql.ErrNoRows
|
||||
},
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/subscription-plans", nil)
|
||||
@@ -81,88 +257,218 @@ func TestSubscriptionPlansHandlerReturnsEnvelope(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrontendHandlerServesIndexAssetsAndSpaFallback(t *testing.T) {
|
||||
distDir := t.TempDir()
|
||||
writeTestFile(t, filepath.Join(distDir, "index.html"), "<html>dashboard</html>")
|
||||
writeTestFile(t, filepath.Join(distDir, "assets", "app.js"), "console.log('ok');")
|
||||
func TestLatestReportHandlerReturnsEnvelope(t *testing.T) {
|
||||
mux := newMux(
|
||||
&sql.DB{},
|
||||
func(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return &latestReportResponse{
|
||||
ReportDate: "2026-05-13",
|
||||
Status: "generated",
|
||||
ModelCount: 504,
|
||||
MarkdownPath: "reports/daily/daily_report_2026-05-13.md",
|
||||
HTMLPath: "reports/daily/html/daily_report_2026-05-13.html",
|
||||
MarkdownURL: "/api/v1/reports/latest/markdown",
|
||||
HTMLURL: "/api/v1/reports/latest/html",
|
||||
}, nil
|
||||
},
|
||||
)
|
||||
|
||||
mux := newMux(&sql.DB{}, noOpModelsFetcher, noOpPlansFetcher, distDir)
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/reports/latest", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
t.Run("root serves index", func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", rec.Code)
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "dashboard") {
|
||||
t.Fatalf("expected index response, got %q", rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("asset serves file", func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/assets/app.js", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", rec.Code)
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "console.log") {
|
||||
t.Fatalf("expected asset response, got %q", rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("spa route falls back to index", func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/explorer/detail", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", rec.Code)
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "dashboard") {
|
||||
t.Fatalf("expected SPA fallback, got %q", rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing asset returns not found", func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/assets/missing.js", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("expected status 404, got %d", rec.Code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("api routes keep precedence", func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/models", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", rec.Code)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func noOpModelsFetcher(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return []modelResponse{}, nil
|
||||
}
|
||||
|
||||
func noOpPlansFetcher(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return []subscriptionPlanResponse{}, nil
|
||||
}
|
||||
|
||||
func writeTestFile(t *testing.T, path string, contents string) {
|
||||
t.Helper()
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", rec.Code)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
|
||||
var payload struct {
|
||||
Data latestReportResponse `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("unmarshal response: %v", err)
|
||||
}
|
||||
|
||||
if payload.Data.ReportDate != "2026-05-13" {
|
||||
t.Fatalf("unexpected report date: %q", payload.Data.ReportDate)
|
||||
}
|
||||
if payload.Data.HTMLURL != "/api/v1/reports/latest/html" {
|
||||
t.Fatalf("unexpected html url: %q", payload.Data.HTMLURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLatestReportHTMLHandlerServesArtifact(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
htmlPath := tempDir + "/daily_report_2026-05-13.html"
|
||||
if err := os.WriteFile(htmlPath, []byte("<html><body>ok</body></html>"), 0644); err != nil {
|
||||
t.Fatalf("write temp html: %v", err)
|
||||
}
|
||||
|
||||
mux := newMux(
|
||||
&sql.DB{},
|
||||
func(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return &latestReportResponse{
|
||||
ReportDate: "2026-05-13",
|
||||
Status: "generated",
|
||||
MarkdownPath: tempDir + "/daily_report_2026-05-13.md",
|
||||
HTMLPath: htmlPath,
|
||||
}, nil
|
||||
},
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/reports/latest/html", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", rec.Code)
|
||||
}
|
||||
if body := rec.Body.String(); body != "<html><body>ok</body></html>" {
|
||||
t.Fatalf("unexpected body: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelsHandlerRejectsUnauthenticatedExternalRequests(t *testing.T) {
|
||||
mux := newMuxWithConfig(
|
||||
&sql.DB{},
|
||||
func(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return nil, sql.ErrNoRows
|
||||
},
|
||||
serverConfig{BasicAuthUser: "review", BasicAuthPass: "secret", RateLimitPerWindow: 10, RateLimitWindow: time.Minute},
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/models", nil)
|
||||
req.RemoteAddr = "198.51.100.8:1234"
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("expected status 401, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelsHandlerAllowsBasicAuthForExternalRequests(t *testing.T) {
|
||||
mux := newMuxWithConfig(
|
||||
&sql.DB{},
|
||||
func(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return []modelResponse{{ID: "openai/gpt-4o", Name: "GPT-4o"}}, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return nil, sql.ErrNoRows
|
||||
},
|
||||
serverConfig{BasicAuthUser: "review", BasicAuthPass: "secret", RateLimitPerWindow: 10, RateLimitWindow: time.Minute},
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/models", nil)
|
||||
req.RemoteAddr = "198.51.100.8:1234"
|
||||
req.SetBasicAuth("review", "secret")
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelsHandlerAllowsBearerTokenForExternalRequests(t *testing.T) {
|
||||
mux := newMuxWithConfig(
|
||||
&sql.DB{},
|
||||
func(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return []modelResponse{{ID: "openai/gpt-4o", Name: "GPT-4o"}}, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return nil, sql.ErrNoRows
|
||||
},
|
||||
serverConfig{ServiceToken: "token-123", RateLimitPerWindow: 10, RateLimitWindow: time.Minute},
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/models", nil)
|
||||
req.RemoteAddr = "198.51.100.8:1234"
|
||||
req.Header.Set("Authorization", "Bearer token-123")
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthHandlerRejectsExternalRequests(t *testing.T) {
|
||||
mux := newMuxWithConfig(
|
||||
&sql.DB{},
|
||||
func(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return nil, sql.ErrNoRows
|
||||
},
|
||||
serverConfig{RateLimitPerWindow: 10, RateLimitWindow: time.Minute},
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/health", nil)
|
||||
req.RemoteAddr = "198.51.100.8:1234"
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("expected status 403, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelsHandlerAppliesRateLimit(t *testing.T) {
|
||||
mux := newMuxWithConfig(
|
||||
&sql.DB{},
|
||||
func(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return []modelResponse{{ID: "openai/gpt-4o", Name: "GPT-4o"}}, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return nil, sql.ErrNoRows
|
||||
},
|
||||
serverConfig{RateLimitPerWindow: 1, RateLimitWindow: time.Minute},
|
||||
)
|
||||
|
||||
first := httptest.NewRequest(http.MethodGet, "/api/v1/models", nil)
|
||||
first.RemoteAddr = "127.0.0.1:1234"
|
||||
firstRec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(firstRec, first)
|
||||
if firstRec.Code != http.StatusOK {
|
||||
t.Fatalf("expected first request status 200, got %d", firstRec.Code)
|
||||
}
|
||||
|
||||
second := httptest.NewRequest(http.MethodGet, "/api/v1/models", nil)
|
||||
second.RemoteAddr = "127.0.0.1:1234"
|
||||
secondRec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(secondRec, second)
|
||||
if secondRec.Code != http.StatusTooManyRequests {
|
||||
t.Fatalf("expected second request status 429, got %d", secondRec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
34
db/migrations/006_model_release_date_metadata.sql
Normal file
34
db/migrations/006_model_release_date_metadata.sql
Normal file
@@ -0,0 +1,34 @@
|
||||
-- Phase 2.1: 模型发布日期证据元数据
|
||||
-- 区分一级官方发布日期与二级权威佐证日期,避免混淆 source_url 与发布日期证据层级
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='models' AND column_name='date_confidence') THEN
|
||||
ALTER TABLE models ADD COLUMN date_confidence TEXT NOT NULL DEFAULT 'unknown';
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='models' AND column_name='date_source_kind') THEN
|
||||
ALTER TABLE models ADD COLUMN date_source_kind TEXT NOT NULL DEFAULT 'unknown';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname='chk_models_date_confidence') THEN
|
||||
ALTER TABLE models
|
||||
ADD CONSTRAINT chk_models_date_confidence
|
||||
CHECK (date_confidence IN ('official_primary', 'secondary_authoritative', 'inferred', 'unknown'));
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname='chk_models_date_source_kind') THEN
|
||||
ALTER TABLE models
|
||||
ADD CONSTRAINT chk_models_date_source_kind
|
||||
CHECK (date_source_kind IN ('official_announcement', 'official_product_page', 'secondary_authoritative_report', 'catalog_backfill', 'unknown'));
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_models_date_confidence ON models(date_confidence);
|
||||
CREATE INDEX IF NOT EXISTS idx_models_date_source_kind ON models(date_source_kind);
|
||||
|
||||
COMMENT ON COLUMN models.date_confidence IS '发布日期证据置信度:official_primary / secondary_authoritative / inferred / unknown';
|
||||
COMMENT ON COLUMN models.date_source_kind IS '发布日期证据来源类型:official_announcement / official_product_page / secondary_authoritative_report / catalog_backfill / unknown';
|
||||
51
db/migrations/007_report_run_audit_semantics.sql
Normal file
51
db/migrations/007_report_run_audit_semantics.sql
Normal file
@@ -0,0 +1,51 @@
|
||||
-- 区分正式日报、手工运行与历史重建的运行语义
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'daily_report' AND column_name = 'run_kind'
|
||||
) THEN
|
||||
ALTER TABLE daily_report ADD COLUMN run_kind TEXT NOT NULL DEFAULT 'scheduled';
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'daily_report' AND column_name = 'trigger_source'
|
||||
) THEN
|
||||
ALTER TABLE daily_report ADD COLUMN trigger_source TEXT NOT NULL DEFAULT 'legacy_backfill';
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'daily_report' AND column_name = 'is_official_daily'
|
||||
) THEN
|
||||
ALTER TABLE daily_report ADD COLUMN is_official_daily BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'report_runs' AND column_name = 'run_kind'
|
||||
) THEN
|
||||
ALTER TABLE report_runs ADD COLUMN run_kind TEXT NOT NULL DEFAULT 'unknown';
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'report_runs' AND column_name = 'trigger_source'
|
||||
) THEN
|
||||
ALTER TABLE report_runs ADD COLUMN trigger_source TEXT NOT NULL DEFAULT 'legacy_backfill';
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'report_runs' AND column_name = 'is_official_daily'
|
||||
) THEN
|
||||
ALTER TABLE report_runs ADD COLUMN is_official_daily BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_report_official_daily ON daily_report(is_official_daily);
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_report_run_kind ON daily_report(run_kind);
|
||||
CREATE INDEX IF NOT EXISTS idx_report_runs_run_kind ON report_runs(run_kind);
|
||||
CREATE INDEX IF NOT EXISTS idx_report_runs_official_daily ON report_runs(is_official_daily);
|
||||
61
db/migrations/008_plan_catalog_inventory.sql
Normal file
61
db/migrations/008_plan_catalog_inventory.sql
Normal file
@@ -0,0 +1,61 @@
|
||||
-- Phase 2: Token Plan / Coding Plan 基础目录清单
|
||||
|
||||
CREATE TABLE IF NOT EXISTS plan_catalog_inventory (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
provider_id BIGINT REFERENCES model_provider(id) ON DELETE SET NULL,
|
||||
operator_id BIGINT REFERENCES operator(id) ON DELETE SET NULL,
|
||||
catalog_code TEXT NOT NULL UNIQUE,
|
||||
platform_name TEXT NOT NULL,
|
||||
platform_name_cn TEXT,
|
||||
platform_type TEXT NOT NULL,
|
||||
plan_family TEXT NOT NULL,
|
||||
plan_status TEXT NOT NULL DEFAULT 'confirmed',
|
||||
source_url TEXT NOT NULL,
|
||||
source_title TEXT,
|
||||
source_kind TEXT NOT NULL DEFAULT 'official_doc',
|
||||
region TEXT NOT NULL DEFAULT 'global',
|
||||
currency TEXT,
|
||||
billing_cycle TEXT,
|
||||
last_checked_at TIMESTAMP NOT NULL,
|
||||
importer_key TEXT,
|
||||
notes TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by TEXT DEFAULT 'system',
|
||||
updated_by TEXT DEFAULT 'system',
|
||||
CONSTRAINT chk_plan_catalog_platform_type
|
||||
CHECK (platform_type IN ('official_vendor', 'cloud_operator', 'relay_platform')),
|
||||
CONSTRAINT chk_plan_catalog_family
|
||||
CHECK (plan_family IN ('token_plan', 'coding_plan', 'package_plan', 'pay_as_you_go', 'unknown')),
|
||||
CONSTRAINT chk_plan_catalog_status
|
||||
CHECK (plan_status IN ('confirmed', 'pending_verification', 'retired')),
|
||||
CONSTRAINT chk_plan_catalog_source_kind
|
||||
CHECK (source_kind IN ('official_doc', 'official_pricing', 'official_product_page', 'official_community', 'inferred')),
|
||||
CONSTRAINT chk_plan_catalog_currency
|
||||
CHECK (currency IS NULL OR currency IN ('CNY', 'USD', 'EUR'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_plan_catalog_provider_id ON plan_catalog_inventory(provider_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_plan_catalog_operator_id ON plan_catalog_inventory(operator_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_plan_catalog_family ON plan_catalog_inventory(plan_family);
|
||||
CREATE INDEX IF NOT EXISTS idx_plan_catalog_platform_type ON plan_catalog_inventory(platform_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_plan_catalog_status ON plan_catalog_inventory(plan_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_plan_catalog_last_checked_at ON plan_catalog_inventory(last_checked_at);
|
||||
|
||||
COMMENT ON TABLE plan_catalog_inventory IS 'Token Plan / Coding Plan / 套餐包 / 按量计费基础目录清单,用于后续 importer 排期与证据管理';
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'update_updated_at_column')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_trigger
|
||||
WHERE tgname = 'plan_catalog_inventory_updated_at'
|
||||
) THEN
|
||||
CREATE TRIGGER plan_catalog_inventory_updated_at
|
||||
BEFORE UPDATE ON plan_catalog_inventory
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
35
db/migrations/009_plan_catalog_inventory_segments.sql
Normal file
35
db/migrations/009_plan_catalog_inventory_segments.sql
Normal file
@@ -0,0 +1,35 @@
|
||||
-- Phase 2: 基础目录增加榜单分组与排名信息
|
||||
|
||||
ALTER TABLE plan_catalog_inventory
|
||||
ADD COLUMN IF NOT EXISTS catalog_segment TEXT NOT NULL DEFAULT 'general',
|
||||
ADD COLUMN IF NOT EXISTS market_rank INTEGER;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_constraint
|
||||
WHERE conname = 'chk_plan_catalog_segment'
|
||||
) THEN
|
||||
ALTER TABLE plan_catalog_inventory
|
||||
ADD CONSTRAINT chk_plan_catalog_segment
|
||||
CHECK (catalog_segment IN ('general', 'vendor_top20', 'relay_top20plus', 'global_reference'));
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_constraint
|
||||
WHERE conname = 'chk_plan_catalog_market_rank'
|
||||
) THEN
|
||||
ALTER TABLE plan_catalog_inventory
|
||||
ADD CONSTRAINT chk_plan_catalog_market_rank
|
||||
CHECK (market_rank IS NULL OR market_rank > 0);
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_plan_catalog_segment ON plan_catalog_inventory(catalog_segment);
|
||||
CREATE INDEX IF NOT EXISTS idx_plan_catalog_market_rank ON plan_catalog_inventory(market_rank);
|
||||
|
||||
COMMENT ON COLUMN plan_catalog_inventory.catalog_segment IS '目录分组:general / vendor_top20 / relay_top20plus / global_reference';
|
||||
COMMENT ON COLUMN plan_catalog_inventory.market_rank IS '榜单排序,数字越小优先级越高';
|
||||
22
db/migrations/010_operator_type_column.sql
Normal file
22
db/migrations/010_operator_type_column.sql
Normal file
@@ -0,0 +1,22 @@
|
||||
-- 补齐 operator.type 字段,避免订阅与目录 importer 在新库中失败
|
||||
|
||||
ALTER TABLE operator
|
||||
ADD COLUMN IF NOT EXISTS type TEXT NOT NULL DEFAULT 'reseller';
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_constraint
|
||||
WHERE conname = 'chk_operator_type'
|
||||
) THEN
|
||||
ALTER TABLE operator
|
||||
ADD CONSTRAINT chk_operator_type
|
||||
CHECK (type IN ('official', 'cloud', 'relay', 'reseller'));
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_operator_type ON operator(type);
|
||||
|
||||
COMMENT ON COLUMN operator.type IS '运营方类型:official / cloud / relay / reseller';
|
||||
8
db/migrations/011_subscription_plan_package_family.sql
Normal file
8
db/migrations/011_subscription_plan_package_family.sql
Normal file
@@ -0,0 +1,8 @@
|
||||
-- Phase 2: 订阅套餐表支持 package_plan
|
||||
|
||||
ALTER TABLE subscription_plan
|
||||
DROP CONSTRAINT IF EXISTS subscription_plan_plan_family_check;
|
||||
|
||||
ALTER TABLE subscription_plan
|
||||
ADD CONSTRAINT subscription_plan_plan_family_check
|
||||
CHECK (plan_family IN ('token_plan', 'coding_plan', 'package_plan'));
|
||||
41
db/migrations/012_daily_signal_snapshot.sql
Normal file
41
db/migrations/012_daily_signal_snapshot.sql
Normal file
@@ -0,0 +1,41 @@
|
||||
-- 第一模块:每日关键信号快照
|
||||
|
||||
CREATE TABLE IF NOT EXISTS daily_signal_snapshot (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
signal_date DATE NOT NULL UNIQUE,
|
||||
status TEXT NOT NULL DEFAULT 'generated',
|
||||
new_models INTEGER NOT NULL DEFAULT 0,
|
||||
price_changes INTEGER NOT NULL DEFAULT 0,
|
||||
official_free INTEGER NOT NULL DEFAULT 0,
|
||||
aggregator_free INTEGER NOT NULL DEFAULT 0,
|
||||
unknown_free INTEGER NOT NULL DEFAULT 0,
|
||||
event_count INTEGER NOT NULL DEFAULT 0,
|
||||
page_mode TEXT NOT NULL DEFAULT 'standard',
|
||||
event_type_counts JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
top_events JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
source_audit TEXT,
|
||||
generated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_signal_snapshot_date ON daily_signal_snapshot(signal_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_signal_snapshot_status ON daily_signal_snapshot(status);
|
||||
|
||||
COMMENT ON TABLE daily_signal_snapshot IS '第一模块产出的每日关键信号快照,用于日报与其他下游形态消费';
|
||||
COMMENT ON COLUMN daily_signal_snapshot.top_events IS '已筛选的关键事件数组,JSONB 序列化 ModelEvent';
|
||||
COMMENT ON COLUMN daily_signal_snapshot.event_type_counts IS '按事件类型聚合的数量统计';
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_trigger
|
||||
WHERE tgname = 'daily_signal_snapshot_updated_at'
|
||||
) THEN
|
||||
CREATE TRIGGER daily_signal_snapshot_updated_at
|
||||
BEFORE UPDATE ON daily_signal_snapshot
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
31
db/migrations/013_official_import_signature_audit.sql
Normal file
31
db/migrations/013_official_import_signature_audit.sql
Normal file
@@ -0,0 +1,31 @@
|
||||
-- 官方导入结构签名审计
|
||||
|
||||
CREATE TABLE IF NOT EXISTS official_import_signature_audit (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
source_key TEXT NOT NULL,
|
||||
checked_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
status TEXT NOT NULL,
|
||||
drift_detected BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
baseline_initialized BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
source_url TEXT,
|
||||
fixture_path TEXT,
|
||||
snapshot_path TEXT,
|
||||
signature_path TEXT,
|
||||
baseline_path TEXT,
|
||||
structure_sha256 TEXT,
|
||||
previous_structure_sha256 TEXT,
|
||||
byte_size INTEGER,
|
||||
signature_payload JSONB,
|
||||
error_message TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_official_import_signature_audit_source_checked_at
|
||||
ON official_import_signature_audit(source_key, checked_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_official_import_signature_audit_status
|
||||
ON official_import_signature_audit(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_official_import_signature_audit_structure_sha256
|
||||
ON official_import_signature_audit(structure_sha256);
|
||||
|
||||
COMMENT ON TABLE official_import_signature_audit IS '官方导入结构签名巡检审计表,记录每次 guard 抓取、签名与漂移判定结果';
|
||||
COMMENT ON COLUMN official_import_signature_audit.signature_payload IS '当前抓取页面的结构签名 JSONB 快照';
|
||||
@@ -0,0 +1,57 @@
|
||||
-- 官方导入结构签名近期变化视图
|
||||
|
||||
CREATE OR REPLACE VIEW official_import_signature_audit_recent_view AS
|
||||
WITH ordered AS (
|
||||
SELECT
|
||||
a.*,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY a.source_key
|
||||
ORDER BY a.checked_at DESC, a.id DESC
|
||||
) AS recent_rank,
|
||||
LAG(a.structure_sha256) OVER (
|
||||
PARTITION BY a.source_key
|
||||
ORDER BY a.checked_at, a.id
|
||||
) AS previous_observed_structure_sha256,
|
||||
LAG(a.checked_at) OVER (
|
||||
PARTITION BY a.source_key
|
||||
ORDER BY a.checked_at, a.id
|
||||
) AS previous_checked_at
|
||||
FROM official_import_signature_audit a
|
||||
)
|
||||
SELECT
|
||||
id,
|
||||
source_key,
|
||||
checked_at,
|
||||
status,
|
||||
drift_detected,
|
||||
baseline_initialized,
|
||||
source_url,
|
||||
fixture_path,
|
||||
snapshot_path,
|
||||
signature_path,
|
||||
baseline_path,
|
||||
structure_sha256,
|
||||
previous_structure_sha256,
|
||||
previous_observed_structure_sha256,
|
||||
byte_size,
|
||||
signature_payload,
|
||||
error_message,
|
||||
created_at,
|
||||
recent_rank,
|
||||
CASE
|
||||
WHEN previous_observed_structure_sha256 IS NULL THEN FALSE
|
||||
WHEN previous_observed_structure_sha256 IS DISTINCT FROM structure_sha256 THEN TRUE
|
||||
ELSE FALSE
|
||||
END AS structure_changed,
|
||||
CASE
|
||||
WHEN previous_observed_structure_sha256 IS NULL THEN 'initial'
|
||||
WHEN previous_observed_structure_sha256 IS DISTINCT FROM structure_sha256 THEN 'changed'
|
||||
ELSE 'stable'
|
||||
END AS structure_state,
|
||||
CASE
|
||||
WHEN previous_checked_at IS NULL THEN NULL
|
||||
ELSE EXTRACT(EPOCH FROM (checked_at - previous_checked_at))::BIGINT
|
||||
END AS seconds_since_previous
|
||||
FROM ordered;
|
||||
|
||||
COMMENT ON VIEW official_import_signature_audit_recent_view IS '官方导入结构签名近期变化视图,按 source_key 给出 recent_rank、结构是否变化与变化状态';
|
||||
@@ -0,0 +1,21 @@
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'chk_models_date_source_kind') THEN
|
||||
ALTER TABLE models DROP CONSTRAINT chk_models_date_source_kind;
|
||||
END IF;
|
||||
|
||||
ALTER TABLE models
|
||||
ADD CONSTRAINT chk_models_date_source_kind
|
||||
CHECK (
|
||||
date_source_kind IN (
|
||||
'official_announcement',
|
||||
'official_product_page',
|
||||
'official_pricing',
|
||||
'secondary_authoritative_report',
|
||||
'catalog_backfill',
|
||||
'unknown'
|
||||
)
|
||||
);
|
||||
END $$;
|
||||
|
||||
COMMENT ON COLUMN models.date_source_kind IS '发布日期证据来源类型:official_announcement / official_product_page / official_pricing / secondary_authoritative_report / catalog_backfill / unknown';
|
||||
31
db/migrations/016_region_pricing_non_token_units.sql
Normal file
31
db/migrations/016_region_pricing_non_token_units.sql
Normal file
@@ -0,0 +1,31 @@
|
||||
-- Phase 2: region_pricing 扩展非 token 统一计费字段(字符/秒等)
|
||||
|
||||
ALTER TABLE region_pricing
|
||||
ADD COLUMN IF NOT EXISTS pricing_mode TEXT NOT NULL DEFAULT 'input_output',
|
||||
ADD COLUMN IF NOT EXISTS price_unit TEXT NOT NULL DEFAULT 'million_tokens',
|
||||
ADD COLUMN IF NOT EXISTS flat_price REAL;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_constraint
|
||||
WHERE conname = 'chk_region_pricing_pricing_mode'
|
||||
) THEN
|
||||
ALTER TABLE region_pricing
|
||||
ADD CONSTRAINT chk_region_pricing_pricing_mode
|
||||
CHECK (pricing_mode IN ('input_output', 'flat'));
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
UPDATE region_pricing
|
||||
SET pricing_mode = 'input_output'
|
||||
WHERE coalesce(pricing_mode, '') = '';
|
||||
|
||||
UPDATE region_pricing
|
||||
SET price_unit = 'million_tokens'
|
||||
WHERE coalesce(price_unit, '') = '';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_region_pricing_pricing_mode ON region_pricing(pricing_mode);
|
||||
CREATE INDEX IF NOT EXISTS idx_region_pricing_price_unit ON region_pricing(price_unit);
|
||||
106
db/migrations/017_intraday_news_candidates.sql
Normal file
106
db/migrations/017_intraday_news_candidates.sql
Normal file
@@ -0,0 +1,106 @@
|
||||
-- 日内新闻候选与验证持久化结构
|
||||
|
||||
CREATE TABLE IF NOT EXISTS intraday_news_candidate (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
candidate_date DATE NOT NULL,
|
||||
discovered_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
event_type TEXT NOT NULL,
|
||||
provider_name TEXT NOT NULL,
|
||||
model_name TEXT,
|
||||
provider_country TEXT,
|
||||
title TEXT NOT NULL,
|
||||
summary TEXT,
|
||||
candidate_urls JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
discovery_source TEXT NOT NULL,
|
||||
discovery_query TEXT,
|
||||
discovery_evidence JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
normalized_key TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'candidate',
|
||||
verification_confidence TEXT NOT NULL DEFAULT 'candidate',
|
||||
verification_notes TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'chk_intraday_news_candidate_status'
|
||||
) THEN
|
||||
ALTER TABLE intraday_news_candidate
|
||||
ADD CONSTRAINT chk_intraday_news_candidate_status
|
||||
CHECK (status IN ('candidate', 'verifying', 'verified', 'rejected', 'stale'));
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'chk_intraday_news_candidate_confidence'
|
||||
) THEN
|
||||
ALTER TABLE intraday_news_candidate
|
||||
ADD CONSTRAINT chk_intraday_news_candidate_confidence
|
||||
CHECK (verification_confidence IN ('candidate', 'secondary_confirmed', 'official_confirmed'));
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_intraday_news_candidate_normalized_key
|
||||
ON intraday_news_candidate(normalized_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_intraday_news_candidate_date
|
||||
ON intraday_news_candidate(candidate_date DESC, discovered_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_intraday_news_candidate_status
|
||||
ON intraday_news_candidate(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_intraday_news_candidate_provider_event
|
||||
ON intraday_news_candidate(provider_name, event_type, candidate_date DESC);
|
||||
|
||||
COMMENT ON TABLE intraday_news_candidate IS '搜索引擎与 LLM 发现的日内新闻候选池,尚未直接进入正式日报事实层';
|
||||
COMMENT ON COLUMN intraday_news_candidate.candidate_urls IS '候选来源 URL 数组,按发现层输出原样保留';
|
||||
COMMENT ON COLUMN intraday_news_candidate.discovery_evidence IS '发现阶段原始证据 JSONB,例如搜索命中、LLM 归纳结果';
|
||||
COMMENT ON COLUMN intraday_news_candidate.normalized_key IS '同日同事件的去重键,避免重复发现候选';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS intraday_news_verification (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
candidate_id BIGINT NOT NULL REFERENCES intraday_news_candidate(id) ON DELETE CASCADE,
|
||||
verified_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
verifier_source TEXT NOT NULL,
|
||||
verifier_url TEXT,
|
||||
verifier_status TEXT NOT NULL,
|
||||
extracted_facts JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
notes TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'chk_intraday_news_verification_status'
|
||||
) THEN
|
||||
ALTER TABLE intraday_news_verification
|
||||
ADD CONSTRAINT chk_intraday_news_verification_status
|
||||
CHECK (verifier_status IN ('matched', 'contradicted', 'insufficient', 'error'));
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_intraday_news_verification_candidate_verified_at
|
||||
ON intraday_news_verification(candidate_id, verified_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_intraday_news_verification_source
|
||||
ON intraday_news_verification(verifier_source);
|
||||
CREATE INDEX IF NOT EXISTS idx_intraday_news_verification_status
|
||||
ON intraday_news_verification(verifier_status);
|
||||
|
||||
COMMENT ON TABLE intraday_news_verification IS '日内新闻候选的验证轨迹,记录验证来源、状态和提取事实';
|
||||
COMMENT ON COLUMN intraday_news_verification.extracted_facts IS '验证阶段提取出的结构化事实 JSONB';
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_trigger
|
||||
WHERE tgname = 'intraday_news_candidate_updated_at'
|
||||
) THEN
|
||||
CREATE TRIGGER intraday_news_candidate_updated_at
|
||||
BEFORE UPDATE ON intraday_news_candidate
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
@@ -27,5 +27,15 @@ services:
|
||||
ports:
|
||||
- "8080:8080"
|
||||
|
||||
nginx:
|
||||
image: nginx:alpine
|
||||
volumes:
|
||||
- ./nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./frontend/dist:/usr/share/nginx/html:ro
|
||||
ports:
|
||||
- "80:80"
|
||||
depends_on:
|
||||
- app
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
|
||||
230
docs/API_REFERENCE.md
Normal file
230
docs/API_REFERENCE.md
Normal file
@@ -0,0 +1,230 @@
|
||||
# API 参考
|
||||
|
||||
当前服务端入口位于 `cmd/server/main.go`,只暴露只读查询接口与健康检查接口。
|
||||
|
||||
## 通用约定
|
||||
|
||||
- 基础地址:`http://<host>:<port>`
|
||||
- 默认端口:`8080`
|
||||
- 返回格式:成功接口统一返回 `{ "data": ... }`
|
||||
- 失败格式:失败接口统一返回 `{ "error": { "code": "...", "message": "..." } }`
|
||||
- 访问控制:`/health` 仅允许本机或私网访问;`/api/*` 对外访问默认要求 `Authorization: Bearer <token>` 或 Basic Auth,详见下文
|
||||
- 限流:`/api/*` 默认按来源 IP 做窗口限流;可通过 `API_RATE_LIMIT_PER_WINDOW` 与 `API_RATE_LIMIT_WINDOW_SEC` 调整
|
||||
|
||||
|
||||
## `GET /health`
|
||||
|
||||
检查数据库连通性。
|
||||
|
||||
### 成功
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok"
|
||||
}
|
||||
```
|
||||
|
||||
### 失败
|
||||
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"code": "database_not_configured",
|
||||
"message": "database not configured"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- `503 database_not_configured`:未配置 `DATABASE_URL`
|
||||
- `503 database_unavailable`:数据库 Ping 失败
|
||||
### 示例
|
||||
|
||||
```bash
|
||||
curl -fsS http://127.0.0.1:8080/health
|
||||
```
|
||||
|
||||
### 访问控制
|
||||
- 仅允许本机或私网请求;外部地址返回 `403 health_endpoint_internal_only`
|
||||
|
||||
|
||||
## `GET /api/v1/models`
|
||||
|
||||
返回模型列表,数据来源于 `models`、`model_provider`、`region_pricing`;当同一模型存在多条价格记录时,API 按“`global` 区域优先、`official` > `reseller` > `free_tier`、再按 `effective_date`/`id` 倒序”的规则选取主价格。
|
||||
|
||||
### 返回体
|
||||
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "openai/gpt-4o",
|
||||
"name": "gpt-4o",
|
||||
"provider": "OpenAI",
|
||||
"providerCN": "OpenAI",
|
||||
"modality": "text",
|
||||
"contextLength": 128000,
|
||||
"pricingMode": "input_output",
|
||||
"priceUnit": "million_tokens",
|
||||
"inputPrice": 2.5,
|
||||
"outputPrice": 10,
|
||||
"currency": "USD",
|
||||
"isFree": false,
|
||||
"stale": false,
|
||||
"dataConfidence": "official"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### 字段说明
|
||||
|
||||
| 字段 | 说明 |
|
||||
|------|------|
|
||||
| `id` | 模型外部 ID,通常是 `provider/model` |
|
||||
| `name` | 模型名称;为空时回退为 `external_id` |
|
||||
| `provider` | 英文厂商名 |
|
||||
| `providerCN` | 中文厂商名;缺失时回退为英文名或 `external_id` 前缀 |
|
||||
| `modality` | 模态类型 |
|
||||
| `contextLength` | 上下文窗口 |
|
||||
| `pricingMode` | 定价模式:`input_output`(默认,按输入/输出 token)或 `flat`(按字符/秒等单一单位) |
|
||||
| `priceUnit` | 价格单位;默认 `million_tokens`,语音类可能是 `10k_characters` / `second` |
|
||||
| `flatPrice` | `pricingMode=flat` 时的统一单价 |
|
||||
| `inputPrice` | 输入价格,单位与 `currency` 配套,默认按每百万 token |
|
||||
| `outputPrice` | 输出价格 |
|
||||
| `currency` | 币种 |
|
||||
| `isFree` | 是否免费 |
|
||||
| `stale` | 是否陈旧数据,当前由 `dataConfidence == "stale"` 推导 |
|
||||
| `dataConfidence` | 数据置信度 |
|
||||
|
||||
### 失败
|
||||
|
||||
- `503 database_not_configured`
|
||||
- `500 query_failed`
|
||||
- `401 auth_required`
|
||||
- `429 rate_limited`
|
||||
|
||||
## `GET /api/v1/subscription-plans`
|
||||
|
||||
返回订阅型套餐列表,当前主要对应腾讯云套餐数据。
|
||||
|
||||
### 返回体
|
||||
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"planFamily": "token_plan",
|
||||
"planCode": "token-plan-lite",
|
||||
"planName": "通用 Token Plan Lite",
|
||||
"tier": "Lite",
|
||||
"provider": "Tencent",
|
||||
"providerCN": "腾讯",
|
||||
"operator": "Tencent Cloud",
|
||||
"operatorCN": "腾讯云",
|
||||
"currency": "CNY",
|
||||
"listPrice": 39,
|
||||
"priceUnit": "CNY/month",
|
||||
"quotaValue": 35000000,
|
||||
"quotaUnit": "tokens/month",
|
||||
"contextWindow": 0,
|
||||
"modelScope": ["tc-code-latest", "glm-5", "glm-5.1"],
|
||||
"sourceUrl": "https://cloud.tencent.com/document/product/1823/130060",
|
||||
"publishedAt": "2026-04-27T00:00:00",
|
||||
"effectiveDate": "2026-04-27"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### 失败
|
||||
|
||||
- `503 database_not_configured`
|
||||
- `500 query_failed`
|
||||
- `401 auth_required`
|
||||
- `429 rate_limited`
|
||||
|
||||
## `GET /api/v1/reports/latest`
|
||||
|
||||
返回最新“正式日报”元数据。查询条件来自 `daily_report`:
|
||||
|
||||
- `status = 'generated'`
|
||||
- `output_path` 非空
|
||||
- `is_official_daily = true`
|
||||
|
||||
### 返回体
|
||||
|
||||
```json
|
||||
{
|
||||
"data": {
|
||||
"reportDate": "2026-05-13",
|
||||
"status": "generated",
|
||||
"modelCount": 504,
|
||||
"summaryMD": "runtime_audit ...",
|
||||
"markdownPath": "reports/daily/daily_report_2026-05-13.md",
|
||||
"htmlPath": "reports/daily/html/daily_report_2026-05-13.html",
|
||||
"archiveMarkdownPath": "reports/daily/2026/05/daily_report_2026-05-13.md",
|
||||
"archiveHtmlPath": "reports/daily/2026/05/daily_report_2026-05-13.html",
|
||||
"markdownUrl": "/api/v1/reports/latest/markdown",
|
||||
"htmlUrl": "/api/v1/reports/latest/html",
|
||||
"updatedAt": "2026-05-13T08:00:00"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 失败
|
||||
|
||||
- `503 database_not_configured`
|
||||
- `404 latest_report_not_found`
|
||||
- `500 query_failed`
|
||||
- `401 auth_required`
|
||||
- `429 rate_limited`
|
||||
|
||||
|
||||
## `GET /api/v1/reports/latest/markdown`
|
||||
|
||||
直接返回最新正式日报的 Markdown 文件内容。
|
||||
|
||||
### 成功
|
||||
|
||||
- `200`
|
||||
- `Content-Type: text/markdown; charset=utf-8`
|
||||
|
||||
### 失败
|
||||
|
||||
- `404 latest_report_not_found`:数据库中没有符合条件的正式日报
|
||||
- `404 report_artifact_not_found`:元数据存在,但落盘文件缺失
|
||||
- `401 auth_required`
|
||||
- `429 rate_limited`
|
||||
|
||||
## `GET /api/v1/reports/latest/html`
|
||||
|
||||
直接返回最新正式日报 HTML 文件内容。
|
||||
|
||||
### 成功
|
||||
|
||||
- `200`
|
||||
- `Content-Type: text/html; charset=utf-8`
|
||||
|
||||
### 失败
|
||||
|
||||
- `404 latest_report_not_found`
|
||||
- `404 report_artifact_not_found`
|
||||
- `401 auth_required`
|
||||
- `429 rate_limited`
|
||||
|
||||
|
||||
## 冒烟检查命令
|
||||
|
||||
```bash
|
||||
curl -fsS http://127.0.0.1:8080/health
|
||||
curl -fsS -H "Authorization: Bearer $API_AUTH_TOKEN" http://127.0.0.1:8080/api/v1/models | jq '.data | length'
|
||||
curl -fsS -H "Authorization: Bearer $API_AUTH_TOKEN" http://127.0.0.1:8080/api/v1/subscription-plans | jq '.data | length'
|
||||
curl -fsS -H "Authorization: Bearer $API_AUTH_TOKEN" http://127.0.0.1:8080/api/v1/reports/latest | jq '.data.reportDate'
|
||||
curl -fsS -H "Authorization: Bearer $API_AUTH_TOKEN" http://127.0.0.1:8080/api/v1/reports/latest/html > /tmp/latest_report.html
|
||||
```
|
||||
|
||||
|
||||
- 在公网暴露时至少配置 `API_AUTH_TOKEN` 或 `API_BASIC_AUTH_USER` / `API_BASIC_AUTH_PASS`
|
||||
- `/health` 仅暴露给负载均衡器、监控系统或私网来源
|
||||
- 如果前端与 API 同域部署,优先由 Nginx 转发 `/api/` 和 `/health`
|
||||
- 如需更强控制,继续在 Nginx / 网关上补齐 CIDR 白名单、OIDC、WAF 与更细粒度限流
|
||||
184
docs/CONFIGURATION.md
Normal file
184
docs/CONFIGURATION.md
Normal file
@@ -0,0 +1,184 @@
|
||||
# 配置说明
|
||||
|
||||
本文档描述 `llm-intelligence` 在本地、CI 与生产环境中的关键配置项,以及各脚本的运行语义。
|
||||
|
||||
## 配置原则
|
||||
|
||||
- 生产环境优先使用容器平台、systemd 或 CI/CD 注入环境变量,不要依赖仓库内 `.env`
|
||||
- `.env.example` 只作为示例,不应存放真实密钥
|
||||
- 避免在 `.env.local` 与 `.env` 中重复定义同一变量
|
||||
- 由于不同脚本的加载方式不同,重复定义时优先级并不完全一致
|
||||
- Shell 脚本通常按 `.env.local` 然后 `.env` 顺序 `source`,后者可能覆盖前者
|
||||
- `generate_daily_report.go` 会优先保留已存在环境变量,并优先保留较早注入的值
|
||||
|
||||
生产环境建议:所有关键变量统一在部署系统中注入,仓库内 `.env*` 仅用于开发。
|
||||
|
||||
## 关键环境变量
|
||||
|
||||
| 变量名 | 必填 | 使用方 | 默认值 | 说明 |
|
||||
|--------|------|--------|--------|------|
|
||||
| `DATABASE_URL` | 是 | API Server、迁移、采集、日报、备份恢复、验收脚本 | 无 | PostgreSQL 连接串,缺失时多数核心脚本会直接失败 |
|
||||
| `OPENROUTER_API_KEY` | 条件必填 | `fetch_openrouter.go`、`run_real_pipeline.sh`、`run_daily.sh`、`run_intraday_price_watch.sh` | 无 | 真实采集所需;只查看历史数据或仅跑前端时可不配 |
|
||||
| `PORT` | 否 | `cmd/server/main.go` | `8080` | API Server 监听端口 |
|
||||
| `API_AUTH_TOKEN` | 条件必填 | `cmd/server/main.go`、API smoke / 外部调用 | 空 | 对外访问 `/api/*` 时推荐使用的 Bearer token;外部请求未携带合法 token 或 Basic Auth 时返回 `401` |
|
||||
| `API_BASIC_AUTH_USER` | 条件必填 | `cmd/server/main.go` | 空 | 对外访问 `/api/*` 的 Basic Auth 用户名;与 `API_BASIC_AUTH_PASS` 配套使用 |
|
||||
| `API_BASIC_AUTH_PASS` | 条件必填 | `cmd/server/main.go` | 空 | 对外访问 `/api/*` 的 Basic Auth 密码 |
|
||||
| `API_RATE_LIMIT_PER_WINDOW` | 否 | `cmd/server/main.go` | `60` | `/api/*` 按来源 IP 的窗口限流阈值;设为 `0` 表示关闭内建限流 |
|
||||
| `API_RATE_LIMIT_WINDOW_SEC` | 否 | `cmd/server/main.go` | `60` | `/api/*` 限流窗口长度(秒) |
|
||||
| `FEISHU_WEBHOOK` | 否 | `run_daily.sh`、`feishu_alert.sh` | 空 | 正式日报失败时发送飞书告警 |
|
||||
| `REPORT_OUTPUT_DIR` | 否 | `generate_daily_report.go` | `reports/daily` | 日报主产物输出目录 |
|
||||
| `REPORT_DATE` | 否 | `generate_daily_report.go`、`rebuild_historical_report.sh`、`run_intraday_price_watch.sh`、`run_intraday_discovery_watch.sh` | 当天日期 | 指定日报或日内链路日期,格式 `YYYY-MM-DD` |
|
||||
| `REPORT_RUN_KIND` | 否 | `generate_daily_report.go` | `manual` | 运行语义,如 `scheduled` / `manual` / `historical_rebuild` |
|
||||
| `REPORT_TRIGGER_SOURCE` | 否 | `generate_daily_report.go`、`materialize_daily_signals.go` | `cli` | 触发来源,如 `cron` / `pipeline` / `intraday` / `intraday_discovery` / `rebuild_script` |
|
||||
| `REPORT_IS_OFFICIAL_DAILY` | 否 | `generate_daily_report.go` | `false` | 是否属于正式日报产出 |
|
||||
| `REPORT_RUNTIME_AUDIT` | 否 | `generate_daily_report.go` | 空 | 来源级运行审计摘要,通常由流水线脚本注入 |
|
||||
| `INTRADAY_DISCOVERY_SEARCH_PROVIDER` | 条件必填 | `discover_intraday_news_candidates.go`、`run_intraday_discovery_watch.sh` | 空 | 候选发现搜索 provider 类型;计划支持 `fixture` / `command_json` / `http_json` |
|
||||
| `INTRADAY_DISCOVERY_SEARCH_COMMAND` | 条件必填 | `discover_intraday_news_candidates.go` | 空 | 当 `INTRADAY_DISCOVERY_SEARCH_PROVIDER=command_json` 时执行的搜索命令,stdout 必须输出 JSON 数组 |
|
||||
| `INTRADAY_DISCOVERY_SEARCH_URL` | 条件必填 | `discover_intraday_news_candidates.go` | 空 | 当 `INTRADAY_DISCOVERY_SEARCH_PROVIDER=http_json` 时调用的搜索接口 URL |
|
||||
| `INTRADAY_DISCOVERY_SEARCH_FIXTURE` | 否 | `discover_intraday_news_candidates.go` | 空 | 搜索 provider 样例文件,用于 dry-run / 本地测试 |
|
||||
| `INTRADAY_DISCOVERY_LLM_PROVIDER` | 条件必填 | `discover_intraday_news_candidates.go`、`run_intraday_discovery_watch.sh` | 空 | 候选归纳 LLM provider 类型;计划支持 `fixture` / `command_json` / `http_json` |
|
||||
| `INTRADAY_DISCOVERY_LLM_COMMAND` | 条件必填 | `discover_intraday_news_candidates.go` | 空 | 当 `INTRADAY_DISCOVERY_LLM_PROVIDER=command_json` 时执行的 LLM 命令,stdout 必须输出 JSON 数组 |
|
||||
| `INTRADAY_DISCOVERY_LLM_URL` | 条件必填 | `discover_intraday_news_candidates.go` | 空 | 当 `INTRADAY_DISCOVERY_LLM_PROVIDER=http_json` 时调用的 LLM 接口 URL |
|
||||
| `INTRADAY_DISCOVERY_LLM_FIXTURE` | 否 | `discover_intraday_news_candidates.go` | 空 | LLM provider 样例文件,用于 dry-run / 本地测试 |
|
||||
| `INTRADAY_DISCOVERY_TIMEOUT_SEC` | 否 | `discover_intraday_news_candidates.go`、`verify_intraday_news_candidates.go` | `20` | discovery provider 与验证抓取的默认超时秒数 |
|
||||
| `PHASE6_PORT` | 否 | `verify_phase6.sh` | 自动挑选 `18080-18120` | Phase 6 验收时临时启动 API Server 的端口 |
|
||||
| `LIGHTHOUSE_PORT` | 否 | `verify_lighthouse.sh` | `4173` | Lighthouse 预览端口 |
|
||||
| `LIGHTHOUSE_SCORE_THRESHOLD` | 否 | `verify_lighthouse.sh` | `80` | 前端性能分数门槛 |
|
||||
| `LIGHTHOUSE_FCP_THRESHOLD_MS` | 否 | `verify_lighthouse.sh` | `2000` | 首次内容绘制门槛 |
|
||||
| `VERIFY_DB_NAME` | 否 | `verify_common.sh` | `llm_intelligence` | SQL 型验收脚本默认连接的数据库名 |
|
||||
|
||||
|
||||
|
||||
## 推荐的生产注入方式
|
||||
|
||||
### API Server
|
||||
|
||||
```bash
|
||||
export DATABASE_URL="postgres://app_user:***@db:5432/llm_intelligence?sslmode=disable"
|
||||
export PORT="8080"
|
||||
export API_AUTH_TOKEN="replace-with-long-random-token"
|
||||
# 或者:export API_BASIC_AUTH_USER="review" && export API_BASIC_AUTH_PASS="replace-with-password"
|
||||
./server
|
||||
```
|
||||
|
||||
|
||||
### 正式日报调度
|
||||
|
||||
```bash
|
||||
export DATABASE_URL="postgres://app_user:***@db:5432/llm_intelligence?sslmode=disable"
|
||||
export OPENROUTER_API_KEY="***"
|
||||
export FEISHU_WEBHOOK="https://open.feishu.cn/..."
|
||||
bash scripts/run_daily.sh
|
||||
```
|
||||
|
||||
### 手工真实复跑
|
||||
|
||||
```bash
|
||||
export DATABASE_URL="postgres://app_user:***@db:5432/llm_intelligence?sslmode=disable"
|
||||
export OPENROUTER_API_KEY="***"
|
||||
bash scripts/run_real_pipeline.sh
|
||||
```
|
||||
|
||||
### 日内价格追踪
|
||||
|
||||
```bash
|
||||
export DATABASE_URL="postgres://app_user:***@db:5432/llm_intelligence?sslmode=disable"
|
||||
export OPENROUTER_API_KEY="***"
|
||||
bash scripts/run_intraday_price_watch.sh
|
||||
```
|
||||
|
||||
说明:
|
||||
- 该入口只刷新价格 importer 与 `daily_signal_snapshot`
|
||||
- 不生成正式 HTML / Markdown 日报
|
||||
- 推荐先按每 4 小时一次调度,再根据外部源稳定性决定是否收紧到每 2 小时
|
||||
|
||||
### 日内候选发现与验证
|
||||
|
||||
```bash
|
||||
export DATABASE_URL="postgres://app_user:***@db:5432/llm_intelligence?sslmode=disable"
|
||||
export INTRADAY_DISCOVERY_SEARCH_PROVIDER="command_json"
|
||||
export INTRADAY_DISCOVERY_SEARCH_COMMAND="/usr/local/bin/intraday-search --date $REPORT_DATE"
|
||||
export INTRADAY_DISCOVERY_LLM_PROVIDER="command_json"
|
||||
export INTRADAY_DISCOVERY_LLM_COMMAND="/usr/local/bin/intraday-llm --date $REPORT_DATE"
|
||||
bash scripts/run_intraday_discovery_watch.sh
|
||||
```
|
||||
|
||||
说明:
|
||||
- 该入口只刷新候选池、验证轨迹与 `daily_signal_snapshot` 中的已验证事实
|
||||
- 它不会直接写 `daily_report`,不会覆盖 `/api/v1/reports/latest` 对应的正式日报
|
||||
- 搜索 / LLM provider 缺失时应明确报前置条件错误,不能伪装成“今日无新闻”
|
||||
- `leak_or_rumor` 默认留在候选层,不进入正式日报事实
|
||||
|
||||
## 日报运行语义
|
||||
|
||||
项目用以下字段区分正式日报、手工复跑和历史补跑:
|
||||
|
||||
| 字段 | 说明 | 典型值 |
|
||||
|------|------|--------|
|
||||
| `run_kind` | 运行类型 | `scheduled` / `manual` / `historical_rebuild` |
|
||||
| `trigger_source` | 触发来源 | `cron` / `pipeline` / `rebuild_script` / `cli` |
|
||||
| `is_official_daily` | 是否视为最新正式日报 | `true` / `false` |
|
||||
| `summary_md` | 运行摘要与审计 | 包含 `REPORT_RUNTIME_AUDIT` 拼接结果 |
|
||||
|
||||
`/api/v1/reports/latest` 只返回:
|
||||
|
||||
- `status='generated'`
|
||||
- `output_path` 非空
|
||||
- `is_official_daily=true`
|
||||
|
||||
这意味着:
|
||||
|
||||
- 手工复跑不会覆盖“最新正式日报”
|
||||
- 历史补跑不会冒充当天正式结果
|
||||
- 如果正式日报写库成功但落盘产物丢失,元数据查询可成功,文件拉取接口会返回 `404`
|
||||
|
||||
## 产物路径约定
|
||||
|
||||
| 类型 | 路径 |
|
||||
|------|------|
|
||||
| 当天 Markdown | `reports/daily/daily_report_YYYY-MM-DD.md` |
|
||||
| 当天 HTML | `reports/daily/html/daily_report_YYYY-MM-DD.html` |
|
||||
| 归档 Markdown | `reports/daily/YYYY/MM/daily_report_YYYY-MM-DD.md` |
|
||||
| 归档 HTML | `reports/daily/YYYY/MM/daily_report_YYYY-MM-DD.html` |
|
||||
| 每日日志 | `/tmp/llm_hub_daily_YYYY-MM-DD.log` |
|
||||
| 备份目录 | `/tmp/llm_hub_backups` |
|
||||
|
||||
## 最小可运行配置
|
||||
|
||||
### 仅启动 API Server
|
||||
|
||||
```bash
|
||||
DATABASE_URL="host=/var/run/postgresql dbname=llm_intelligence user=long sslmode=disable" \
|
||||
PORT="8080" \
|
||||
go run ./cmd/server
|
||||
```
|
||||
|
||||
说明:
|
||||
- `/health` 仅允许本机或私网来源访问
|
||||
- `/api/*` 对外访问默认要求 Bearer token 或 Basic Auth
|
||||
- 本机与私网来源可直接访问,便于同机前端、验收脚本和内网反代联调
|
||||
|
||||
### 仅生成指定日期日报
|
||||
|
||||
```bash
|
||||
DATABASE_URL="host=/var/run/postgresql dbname=llm_intelligence user=long sslmode=disable" \
|
||||
REPORT_DATE="2026-05-13" \
|
||||
go run -tags llm_script ./scripts/generate_daily_report.go
|
||||
```
|
||||
|
||||
### 真实采集并写库
|
||||
|
||||
```bash
|
||||
DATABASE_URL="host=/var/run/postgresql dbname=llm_intelligence user=long sslmode=disable" \
|
||||
OPENROUTER_API_KEY="***" \
|
||||
go run ./scripts/fetch_openrouter.go -strict-real -db "$DATABASE_URL" -api-key "$OPENROUTER_API_KEY"
|
||||
```
|
||||
|
||||
## 配置错误的典型症状
|
||||
|
||||
| 症状 | 可能原因 | 排查方向 |
|
||||
|------|----------|----------|
|
||||
| `/health` 返回 `503 database not configured` | `DATABASE_URL` 未注入到 API Server | 检查进程环境变量 |
|
||||
| `run_real_pipeline.sh` 直接退出 | `OPENROUTER_API_KEY` 或 `DATABASE_URL` 缺失 | 检查 `.env` 或部署配置 |
|
||||
| `/api/v1/reports/latest` 返回 `404` | 没有正式日报或 `is_official_daily=false` | 查 `daily_report` 表 |
|
||||
| 最新日报元数据存在,但 `/html` 返回 `404` | `output_path` 对应文件丢失 | 检查 `reports/daily` 与归档目录 |
|
||||
140
docs/NEXT_IMPORTER_RUNTIME_PRIORITY.md
Normal file
140
docs/NEXT_IMPORTER_RUNTIME_PRIORITY.md
Normal file
@@ -0,0 +1,140 @@
|
||||
# 下一批 importer / runtime 挂载优先清单
|
||||
|
||||
> For Hermes: 这是基于当前 `PLAN_CATALOG_COVERAGE_MATRIX.md` 的执行清单,不是泛泛 roadmap。优先修“高价值且证据已足够”的缺口,再做长尾扩展。
|
||||
|
||||
更新时间:2026-05-22
|
||||
|
||||
## 本轮已完成的小批次闭环
|
||||
|
||||
已完成并验证:
|
||||
|
||||
1. 腾讯云 TokenHub runtime 挂载
|
||||
2. 魔搭 API-Inference importerKey 校准
|
||||
3. 天翼云模型推理服务 payg importerKey 校准
|
||||
4. 联通云 Token Plan pricing importer(3 模型 blended price + 区域支持矩阵)
|
||||
5. 百川开放平台官方 payg importer(11 个通用文本模型)
|
||||
6. 零一万物官方 payg importer(2 个公开按量模型)
|
||||
7. 讯飞官方 payg importer(4 档公开 blended token 定价)
|
||||
8. 商汤官方 payg importer(公测期免费开放 3 个公开模型)
|
||||
9. 360 智脑开放平台官方 payg importer(官方开放平台广义价格面)
|
||||
10. 网易有道子曰开放平台官方 payg importer(ThinkFlow 官方开放平台广义价格面)
|
||||
|
||||
对应结果:
|
||||
- `tencent_subscription` 已进入 `run_daily.sh` / `run_intel_pipeline.sh` / `run_real_pipeline.sh`
|
||||
- `verify_importer_smoke.sh` 已新增腾讯 fixture/live smoke,并通过
|
||||
- `魔搭 API-Inference` 与 `天翼云模型推理服务 payg` 已从错误的价格 importer 映射回退到 `import_catalog_seed_verification.go`
|
||||
- `cucloud_pricing` 已进入 `run_daily.sh` / `run_real_pipeline.sh` / `run_intel_pipeline.sh`
|
||||
- `verify_importer_smoke.sh` 已新增联通云 fixture/live smoke,并通过
|
||||
- `baichuan_pricing` 已进入 `run_daily.sh` / `run_real_pipeline.sh` / `run_intel_pipeline.sh`
|
||||
- `verify_importer_smoke.sh` 已新增百川 fixture/live smoke,并通过
|
||||
- `lingyiwanwu_pricing` 已进入 `run_daily.sh` / `run_real_pipeline.sh` / `run_intel_pipeline.sh`
|
||||
- `verify_importer_smoke.sh` 已新增零一万物 fixture/live smoke,并通过
|
||||
- `xfyun_pricing` 已进入 `run_daily.sh` / `run_real_pipeline.sh` / `run_intel_pipeline.sh`
|
||||
- `verify_importer_smoke.sh` 已新增讯飞 fixture/live smoke,并通过
|
||||
- `sensenova_pricing` 已进入 `run_daily.sh` / `run_real_pipeline.sh` / `run_intel_pipeline.sh`
|
||||
- `verify_importer_smoke.sh` 已新增商汤 fixture/live smoke,并通过
|
||||
- `platform360_pricing` 已校准为官方开放平台价格源,并将 vendor_top20 的 `360-zhinao-api-payg` importerKey 切到真实 importer
|
||||
- `verify_importer_smoke.sh` 已新增 360 fixture/live smoke,并通过
|
||||
- `scripts/test_importers.sh` 已建立 scripts 层 importer targeted go test matrix,并已接入 CI
|
||||
- 覆盖矩阵已同步到新真相
|
||||
|
||||
## 当前事实基线
|
||||
|
||||
来自 `docs/PLAN_CATALOG_COVERAGE_MATRIX.md`:
|
||||
|
||||
- 目录基线:71/71
|
||||
- 目录核验:30/71
|
||||
- 已有 importer:41/71
|
||||
- 已真实入库:41/71
|
||||
- 仍缺细颗粒度价格:35/71
|
||||
|
||||
解读:
|
||||
- 这轮不是单纯“把数字做大”
|
||||
- 而是先消除了 source 归属漂移,让 `已真实入库` 统计更可信
|
||||
|
||||
---
|
||||
|
||||
## 现在的下一批优先级
|
||||
|
||||
### P1:补国内高价值官方平台的真实 payg importer
|
||||
|
||||
| 优先级 | 平台 | 当前状态 | 价值 | 建议动作 |
|
||||
|---|---|---|---|---|
|
||||
| P1-1 | MiniCPM 开放平台 | 只有目录核验;当前公开入口未见 payg 价格表 | 国内 Top20 中仍未落地真实官方 payg importer 的剩余平台之一 | 已确认 `platform.modelbest.cn` 当前 404,公开可访问面主要是 `modelbest.cn` 官网与公开飞书 Cookbook,均未见 `元/百万token` / `按 token` 价卡;维持 catalog verification,等待真实官方价格源出现(详见 `docs/references/modelbest-minicpm-public-source-gap.md`) |
|
||||
|
||||
### P2:补中转/聚合平台的细颗粒度价格
|
||||
|
||||
| 优先级 | 平台 | 当前状态 | 价值 | 建议动作 |
|
||||
|---|---|---|---|---|
|
||||
| P2-1 | 移动云 MoMA | 已升级为官方价格 importer,并补齐语音按字符/按秒计费落表 | 文本/视觉/向量/排序/语音模型都可进入价格对比 | 已完成,后续仅做新增模型跟进 |
|
||||
| P2-2 | 联通云 AICP / AI 应用开发平台 | 已新增 `cucloud_pricing`,但当前只覆盖 AISP Token Plan 3 模型 blended price 与区域矩阵 | 目录入口与部分结构化价格已打通,但 payg per-model 公开价表仍缺 | 后续若官方公开 payg 模型销售价,再扩 `import_cucloud_pricing.go`(当前边界详见 `docs/references/cucloud-token-plan-vs-aisp.md`) |
|
||||
| P2-3 | 豆包与 Seed 开放平台 | 已有多源/订阅链路,但仍缺细颗粒度价格标注 | 当前矩阵里仍保留缺口 | 区分“已有多源模型采集”与“官方价格页结构化价格”能力 |
|
||||
| P2-4 | 天翼云息壤 / CloudBase AI+ / TI 平台大模型广场 | manual_review | 平台入口存在,但尚无真实 importer | 先回查官方页面结构,再决定 catalog importer 还是 pricing importer |
|
||||
| P2-5 | OpenCode Zen(候选) | 官网初核已完成;OpenCode 本体是开源代理,真正对应价格面的平台是 OpenCode Zen | 对“精选多模型网关 / AI gateway”类型平台有代表性,可补充 payg 聚合平台样本 | 后续以 `opencode.ai/zen` 与 docs/zen 为准,判断是否只适合 `catalog verification`,或可进一步做真实 pricing importer |
|
||||
| P2-6 | OpenCode Go(候选) | 官网初核已完成;这是独立于 Zen 的订阅服务,文档公开首月 5 美元、后续每月 10 美元,并给出额度限制 | 对 `subscription_plan` 维度有代表性,可补一类“多模型访问订阅”样本;按当前证据已足够进入 subscription 候选基线 | 后续以 `opencode.ai/docs/go` 为准,优先决定是先手工 seed,还是直接做真实 importer / manual-seed importer 闭环 |
|
||||
|
||||
### P3:全球参考集从目录核验升级为真实价格 importer
|
||||
|
||||
首批建议顺序:
|
||||
1. Gemini API
|
||||
2. Mistral La Plateforme
|
||||
3. Cohere Platform
|
||||
4. Together AI
|
||||
5. Fireworks AI
|
||||
6. DeepInfra
|
||||
7. GroqCloud
|
||||
|
||||
原因:
|
||||
- 都已确认有官方价格页
|
||||
- 都适合复用统一 official pricing importer 模板
|
||||
- 对“全球平台横向价格比较”价值高
|
||||
|
||||
---
|
||||
|
||||
## 推荐执行顺序
|
||||
|
||||
### 第一批(已完成的小批次闭环)
|
||||
1. 通义千问开放平台 payg importer
|
||||
2. 腾讯混元开放平台 payg importer
|
||||
3. 华为云 MaaS payg importer(对应原规划里的“盘古大模型服务”位点;当前公开 payg 实际覆盖 MaaS 文本模型集合)
|
||||
|
||||
理由:
|
||||
- runtime/source 真相已经同步到 run_daily / run_real_pipeline / run_intel_pipeline
|
||||
- seed/importerKey 已从目录核验切换到真实官方 pricing importer
|
||||
- 华为侧保留 package + payg 双链路;但公开 payg 页面未见独立盘古 SKU 单价,已按真实页面语义落地
|
||||
|
||||
### 第二批(平台深挖)
|
||||
4. 移动云 pricing importer 后续 schema 扩展
|
||||
5. 联通云 payg per-model 价格公开表跟进
|
||||
6. MiniCPM 开放平台官方 payg 价格源跟进(当前公开面未见可落库价卡)
|
||||
|
||||
注:火山方舟官方价格页结构化 importer 已接入 `import_bytedance_pricing.go`,当前覆盖 `在线推理(常规)` 公开的 token 定价;向量/图片/3D 与低延迟/批量/TPM 保障包仍待 schema 扩展后细化。
|
||||
|
||||
### 第三批(全球参考集)
|
||||
7. Gemini API
|
||||
8. Mistral La Plateforme
|
||||
9. Cohere Platform
|
||||
|
||||
---
|
||||
|
||||
## 下一步验收口径
|
||||
|
||||
每完成一项,都必须同步验证这 4 件事:
|
||||
|
||||
1. seed / importerKey 是否与真实脚本一致
|
||||
2. `run_daily.sh` / `run_intel_pipeline.sh` / `run_real_pipeline.sh` 是否真正执行了该 source
|
||||
3. 覆盖矩阵状态是否已同步
|
||||
4. 若新增 source,pipeline audit / failed_source_keys 是否能正确反映它
|
||||
|
||||
## 建议结论
|
||||
|
||||
如果下一步只选一条最值得马上做的线:
|
||||
|
||||
A. 先做 `联通云 payg per-model 价格公开表跟进`
|
||||
|
||||
如果允许做一个“小批次闭环”:
|
||||
|
||||
B. `联通云 payg 公开价跟进 + MiniCPM 官方价格源复查 + 全球参考集首条官方 importer`
|
||||
|
||||
通义千问 / 腾讯混元 / 华为云 MaaS / 联通云 Token Plan / 百川开放平台 / 零一万物 / 讯飞 / 商汤 / 360 智脑 / 网易有道子曰 这一批功能已完成,且当前通过 `scripts/test_importers.sh` + `importer_smoke_gate_test.sh` + `pipeline_runtime_alignment_test.sh` 形成 scripts 层主回归护栏;后续优先转向仍缺公开 payg 价格的平台。
|
||||
|
||||
118
docs/PLAN_CATALOG_COVERAGE_MATRIX.md
Normal file
118
docs/PLAN_CATALOG_COVERAGE_MATRIX.md
Normal file
@@ -0,0 +1,118 @@
|
||||
# 平台覆盖矩阵
|
||||
|
||||
更新时间:2026-05-22(基于当前仓库 seed、`scripts/run_daily.sh` source set、以及 `docs/PLAN_CATALOG_INVENTORY.md` 的显式说明)
|
||||
|
||||
## 判定规则
|
||||
|
||||
- 目录基线:该平台/套餐族已经进入 `plan_catalog_inventory`。本矩阵全部 70 行默认都为 `✓`。
|
||||
- 目录核验:当前只走 `import_catalog_seed_verification.go`,说明已确认官方入口,但还不是细颗粒度结构化价格抓取。
|
||||
- 已有 importer:仓库里已有真实 importer/collector,或 seed 标注为 `existing_price_importer`。
|
||||
- 已真实入库:在当前 `scripts/run_daily.sh` / `run_intel_pipeline.sh` 的 source set 中能找到对应运行 source;仅 `catalog_seed_verification` 不算真实价格/套餐入库。
|
||||
- 仍缺细颗粒度价格:属于目录核验、manual review 占位,或 `PLAN_CATALOG_INVENTORY.md` 已明确列为后续细化价格优先项。
|
||||
|
||||
## 汇总
|
||||
|
||||
- 目录基线:71/71
|
||||
- 目录核验:30/71
|
||||
- 已有 importer:41/71
|
||||
- 已真实入库:41/71
|
||||
- 仍缺细颗粒度价格:35/71
|
||||
|
||||
## 基础目录
|
||||
|
||||
| 平台 | 覆盖对象 | 套餐族 | 目录基线 | 目录核验 | 已有 importer | 已真实入库 | 仍缺细颗粒度价格 | 当前证据 |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| 京东云 JoyBuilder | 计费说明--JoyBuilder 模型开发平台2.0 | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| Anthropic API | Pricing | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| OpenAI API | Pricing | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily 多源采集 source=openai |
|
||||
| xAI API | Pricing | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
|
||||
## 国内官方厂家 Top20
|
||||
|
||||
| 平台 | 覆盖对象 | 套餐族 | 目录基线 | 目录核验 | 已有 importer | 已真实入库 | 仍缺细颗粒度价格 | 当前证据 |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| 通义千问开放平台 | 什么是大模型服务平台百炼 | 按量计费 | ✓ | ✓ | ✓ | ✓ | — | run_daily source=qwen_pricing |
|
||||
| 腾讯混元开放平台 | 腾讯混元 | 按量计费 | ✓ | ✓ | ✓ | ✓ | — | run_daily source=hunyuan_pricing |
|
||||
| 文心大模型开放平台 | 文心千帆大模型平台 | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily 多源采集 source=baidu |
|
||||
| 豆包与 Seed 开放平台 | 火山方舟 | 按量计费 | ✓ | — | ✓ | ✓ | 部分 | run_daily source=bytedance_pricing(当前落地在线推理常规定价;向量/图片/3D 与低延迟/批量/TPM 保障包等多 service-class 价格待 schema 扩展) |
|
||||
| 智谱 Coding Plan | 套餐概览 | Coding Plan | ✓ | — | ✓ | ✓ | — | run_daily source=zhipu_coding_plan |
|
||||
| 盘古大模型服务 | 大模型即服务 MaaS | 按量计费 | ✓ | ✓ | ✓ | ✓ | 部分 | run_daily source=huawei_maas_pricing(当前官方公开 payg 实际覆盖华为云 MaaS 文本模型集合,未见独立盘古 SKU 单价) |
|
||||
| DeepSeek API | 模型 & 价格 | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily 多源采集 source=deepseek |
|
||||
| Kimi API 开放平台 | 模型推理价格说明 | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily 多源采集 source=moonshot |
|
||||
| MiniMax 开放平台 | Token Plan 概要 | Token Plan | ✓ | — | ✓ | ✓ | — | run_daily source=minimax_subscription |
|
||||
| Step Plan | Step Plan 简介 | Coding Plan | ✓ | — | — | — | ✓ | 仅人工核对占位,尚未接入 importer |
|
||||
| 百川开放平台 | 价格说明 | 按量计费 | ✓ | ✓ | ✓ | ✓ | — | run_daily source=baichuan_pricing(按千 tokens 公开价已换算为 per-million-token 入库;Embedding / 搜索增强等非主文本计费仍未纳入当前 schema) |
|
||||
| 零一万物开放平台 | 零一万物开放平台文档 | 按量计费 | ✓ | ✓ | ✓ | ✓ | — | run_daily source=lingyiwanwu_pricing(当前官方文档仅公开 yi-lightning / yi-vision-v2 两个按量模型,按 1M token blended 单价入库) |
|
||||
| 日日新开放平台 | 定价 | 按量计费 | ✓ | ✓ | ✓ | ✓ | — | run_daily source=sensenova_pricing(当前官方公开信号为公测期免费开放、所有当前模型完全开放;按 0 价 free-tier 入库 SenseNova 6.7 Flash-Lite / SenseNova U1 Fast / DeepSeek V4 Flash,U1 Fast 属独立图片生成接口) |
|
||||
| 讯飞星火开放平台 | 星火大模型 Web API | 按量计费 | ✓ | ✓ | ✓ | ✓ | — | run_daily source=xfyun_pricing(当前公开为 X2/X1.5 / Ultra / Pro / Lite 四档 1M token blended 单价卡片,按 blended 单价入库) |
|
||||
| 360 智脑开放平台 | 360 智脑开放平台 | 按量计费 | ✓ | ✓ | ✓ | ✓ | 部分 | run_daily source=platform360_pricing(当前可访问的官方公开价格源为 ai.360.com/open/models;页面同时覆盖 360 自有模型与第三方模型,现按该官方开放平台的广义价格面入库,不能等同于“仅 360 自研模型价格表”) |
|
||||
| 网易有道子曰开放平台 | ThinkFlow 官网 | 按量计费 | ✓ | ✓ | ✓ | ✓ | 部分 | run_daily source=youdao_pricing(当前可访问的官方公开价格源为 ai.youdao.com/new/thinkflow;页面同时覆盖 DeepSeek/Qwen/Kimi/MiniMax/GLM 等第三方模型价卡,现按该官方开放平台广义价格面入库,不能等同于“仅子曰自研模型价格表”) |
|
||||
| MiniCPM 开放平台 | 面壁开放平台 | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验(`platform.modelbest.cn` 当前 404;公开可访问的 `modelbest.cn` 官网与公开飞书 Cookbook 未见 `元/百万token` / `按 token` 官方价卡;详见 `docs/references/modelbest-minicpm-public-source-gap.md`) |
|
||||
| 智源开放平台 | FlagOpen | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| 天工开放平台 | 天工开放平台 | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| 无问芯穹开放平台 | 无问芯穹云平台 | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
|
||||
## 国内中转/聚合平台 20+
|
||||
|
||||
| 平台 | 覆盖对象 | 套餐族 | 目录基线 | 目录核验 | 已有 importer | 已真实入库 | 仍缺细颗粒度价格 | 当前证据 |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| 腾讯云 TokenHub | Token Plan 个人版套餐概览 | Token Plan | ✓ | — | ✓ | ✓ | — | run_daily source=tencent_subscription |
|
||||
| 腾讯云 TokenHub | Token Plan 企业版专业套餐 | Token Plan | ✓ | — | ✓ | ✓ | — | run_daily source=tencent_subscription |
|
||||
| 腾讯云 TokenHub | Token Plan 企业版轻享套餐 | Token Plan | ✓ | — | ✓ | ✓ | — | run_daily source=tencent_subscription |
|
||||
| 腾讯云 TokenHub | Coding Plan 常见问题 | Coding Plan | ✓ | — | ✓ | ✓ | — | run_daily source=tencent_subscription |
|
||||
| 阿里云百炼 | Token Plan(团队版)概述 | Token Plan | ✓ | — | ✓ | ✓ | — | run_daily source=aliyun_subscription |
|
||||
| 阿里云百炼 | Coding Plan概述 | Coding Plan | ✓ | — | ✓ | ✓ | — | run_daily source=aliyun_subscription |
|
||||
| 百度千帆 | Token 福利包 | Token Plan | ✓ | — | ✓ | ✓ | — | run_daily source=baidu_subscription |
|
||||
| 百度千帆 | Coding Plan | Coding Plan | ✓ | — | ✓ | ✓ | — | run_daily source=baidu_subscription |
|
||||
| 火山方舟 | 火山方舟新套餐上线:方舟 Coding Plan | Coding Plan | ✓ | — | ✓ | ✓ | — | run_daily source=bytedance_subscription |
|
||||
| 华为云 MaaS | MaaS文本生成模型 | Package Plan | ✓ | — | ✓ | ✓ | ✓ | run_daily source=huawei_package |
|
||||
| CloudBase AI+ | 云开发 CloudBase | Unknown | ✓ | — | — | — | ✓ | 仅人工核对占位,尚未接入 importer |
|
||||
| TI 平台大模型广场 | TI 平台 | Unknown | ✓ | — | — | — | ✓ | 仅人工核对占位,尚未接入 importer |
|
||||
| 魔搭 API-Inference | API-Inference 简介 | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| 天翼云模型推理服务 | 天翼云模型推理服务 | Token Plan | ✓ | — | ✓ | ✓ | — | run_daily source=ctyun_subscription |
|
||||
| 天翼云模型推理服务 | 天翼云模型推理服务 | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| 天翼云模型推理服务 | 天翼云模型推理服务 | Coding Plan | ✓ | — | ✓ | ✓ | — | run_daily source=ctyun_subscription |
|
||||
| 天翼云息壤 | 天翼云息壤 | Unknown | ✓ | — | — | — | ✓ | 仅人工核对占位,尚未接入 importer |
|
||||
| 联通云 AICP | 联通云智算专区 | 按量计费 | ✓ | — | ✓ | ✓ | ✓ | run_daily source=cucloud_catalog;另有 `cucloud_pricing` 补充 AISP Token Plan 三模型 blended price + 区域矩阵,但 AICP/AI 应用平台本身仍缺公开 payg per-model 价表(详见 `docs/references/cucloud-token-plan-vs-aisp.md`) |
|
||||
| 联通云 AI 应用开发平台 | 联通云智算专区 | 按量计费 | ✓ | — | ✓ | ✓ | ✓ | run_daily source=cucloud_catalog;另有 `cucloud_pricing` 补充 AISP Token Plan 三模型 blended price + 区域矩阵,但 AI 应用平台本身仍缺公开 payg per-model 价表(详见 `docs/references/cucloud-token-plan-vs-aisp.md`) |
|
||||
| 移动云 MoMA | 预置模型服务-token按量计费 | 按量计费 | ✓ | — | ✓ | ✓ | ✓ | run_daily source=mobile_cloud_pricing;文本/视觉/向量/排序与语音按字符/按秒计费均已入库 |
|
||||
| 有道智云 MaaS | ThinkFlow 官网 | 按量计费 | ✓ | — | ✓ | ✓ | 部分 | run_daily source=youdao_pricing(当前可访问的官方公开价格源为 ai.youdao.com/new/thinkflow;页面公开 DeepSeek/Qwen/Kimi/MiniMax/GLM 等模型的输入/输出 token 单价,并展示渠道转发能力) |
|
||||
| 360 智脑开放平台 | 360 智脑开放平台 | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily source=platform360_pricing |
|
||||
| 硅基流动云平台 | SiliconCloud | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily source=siliconflow_pricing |
|
||||
| PPIO 模型 API | PPIO Model API | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily source=ppio_pricing |
|
||||
| UModelVerse | 大模型服务平台 UModelVerse | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily source=ucloud_pricing |
|
||||
| 基石智算 CoresHub | 在线服务模型价格 | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily source=coreshub_pricing |
|
||||
| 金山云星流平台 | 金山云星流平台 | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
|
||||
## 全球官方/中转参考集
|
||||
|
||||
| 平台 | 覆盖对象 | 套餐族 | 目录基线 | 目录核验 | 已有 importer | 已真实入库 | 仍缺细颗粒度价格 | 当前证据 |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| Gemini API | Gemini API billing information | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| Mistral La Plateforme | La Plateforme | Mistral AI | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| Cohere Platform | Pricing | Cohere | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| OpenRouter | OpenRouter Models | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily source=openrouter |
|
||||
| Together AI | Pricing | Together AI | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| Fireworks AI | Pricing | Fireworks AI | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| DeepInfra | Pricing | DeepInfra | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| GroqCloud | Groq On-Demand Pricing for Tokens-as-a-Service | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| Replicate | Pricing - Replicate | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| Hyperbolic | Pricing - Hyperbolic Docs | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| Novita AI | Pricing | Novita AI | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| Azure OpenAI 服务 | Azure OpenAI Service - Pricing | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily source=azure_openai_pricing |
|
||||
| Amazon Bedrock | Amazon Bedrock Pricing | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily source=bedrock_pricing |
|
||||
| Vertex AI 生成式 AI | Vertex AI Pricing | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily source=vertex_pricing |
|
||||
| Cloudflare Workers AI | Pricing · Cloudflare Workers AI docs | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily source=cloudflare_pricing |
|
||||
| Baseten | Cloud Pricing | Baseten | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| Cerebras Inference | Pricing | Cerebras | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
| Perplexity Agent API | Pricing - Perplexity | 按量计费 | ✓ | — | ✓ | ✓ | — | run_daily source=perplexity_pricing |
|
||||
| SambaNova Cloud | Plan and Billing | SambaNova Cloud | 按量计费 | ✓ | ✓ | — | — | ✓ | 当前仅走目录级官方入口核验 |
|
||||
|
||||
## 需要重点关注的边界
|
||||
|
||||
- 腾讯云 TokenHub:本轮已补齐 runtime 挂载,并通过 `verify_importer_smoke.sh` 的 fixture/live 双 smoke;当前可按 `tencent_subscription` 计入真实入库。
|
||||
- 联通云:当前同时存在 `cucloud_catalog`(目录入口核验)与 `cucloud_pricing`(AISP Token Plan 三模型 blended price + 区域支持矩阵)两条链路;后者不能被夸大为联通云 payg per-model 价格已完整打通,详见 `docs/references/cucloud-token-plan-vs-aisp.md`。
|
||||
- `existing_price_importer`:这是 seed 层的已存在价格导入标记,不是脚本文件名;当前 OpenAI / DeepSeek / Moonshot / 百度文心 仍沿用既有真实入库链路,火山方舟 payg 已切换到 `import_bytedance_pricing.go`。
|
||||
- `manual_review`:当前仍只是人工占位,不应误判为 importer 已接入。
|
||||
- 目录核验类平台:`import_catalog_seed_verification.go` 会更新 `plan_catalog_inventory.last_checked_at`,但它不是 `region_pricing` 级别的细颗粒度价格抓取。
|
||||
- 本轮已校准两处 importerKey 漂移:`魔搭 API-Inference` 与 `天翼云模型推理服务 payg` 已回退到目录级官方入口核验,不再冒充 `youdao_pricing` / `platform360_pricing` 真实入库来源。
|
||||
320
docs/PLAN_CATALOG_INVENTORY.md
Normal file
320
docs/PLAN_CATALOG_INVENTORY.md
Normal file
@@ -0,0 +1,320 @@
|
||||
# Token Plan / Coding Plan 基础目录
|
||||
|
||||
更新时间:2026-05-22(Asia/Shanghai)
|
||||
|
||||
配套矩阵见:[PLAN_CATALOG_COVERAGE_MATRIX.md](PLAN_CATALOG_COVERAGE_MATRIX.md)。该矩阵按平台/套餐族逐项标注“目录基线 / 目录核验 / 已有 importer / 已真实入库 / 仍缺细颗粒度价格”,用于快速回答覆盖边界问题。
|
||||
|
||||
## 目标
|
||||
|
||||
这份清单解决两个问题:
|
||||
|
||||
1. 先把“哪些平台确实存在 Token Plan / Coding Plan / 套餐包,哪些只有按量计费”整理成统一基线。
|
||||
2. 再把这份基线落到数据库 `plan_catalog_inventory`,为后续每个平台的 importer 排期、证据追踪和验收提供稳定入口。
|
||||
|
||||
注意:这里记录的是**平台级事实**,不是最终的套餐明细落库。真正的套餐条目仍然应进入 `subscription_plan`,按模型按量价格仍然应进入 `region_pricing`。
|
||||
|
||||
截至 2026-05-15,这份基线已经扩展到:
|
||||
|
||||
- 国内官方模型厂家 Top 20
|
||||
- 国内中转 / 聚合 / 云厂商平台 20+
|
||||
- 全球官方模型平台与全球多模型中转平台参考集
|
||||
- `plan_catalog_inventory` 最终落库 71 条目录记录
|
||||
- `subscription_plan` 新增一批手工核实套餐 seed,用于在真正抓取器到位前先支撑日报对比
|
||||
|
||||
## 分类约定
|
||||
|
||||
- `token_plan`:按 token 或 credits 统一额度管理的订阅型方案
|
||||
- `coding_plan`:按 AI 编码场景设计的包月/限额订阅方案
|
||||
- `package_plan`:华为云这类“按量 + 套餐包并存”的资源包方案
|
||||
- `pay_as_you_go`:官方当前只提供按量计费,未发现独立 Token Plan / Coding Plan
|
||||
- `unknown`:官方已确认平台存在,但公开页面暂未给出可稳定结构化的套餐命名
|
||||
|
||||
## 国内官方厂家 Top 20
|
||||
|
||||
这不是第三方市场报告意义上的绝对“排名”,而是基于 2026-05-14 当天可公开验证的开放平台能力、行业知名度和接入优先级整理出的 Top 20 清单,方便后续 importer 排期:
|
||||
|
||||
1. 阿里巴巴 / 通义千问
|
||||
2. 腾讯 / 混元
|
||||
3. 百度 / 文心
|
||||
4. 字节跳动 / 豆包、Seed
|
||||
5. 智谱 AI
|
||||
6. 华为 / 盘古
|
||||
7. DeepSeek
|
||||
8. Moonshot AI
|
||||
9. MiniMax
|
||||
10. 阶跃星辰
|
||||
11. 百川智能
|
||||
12. 零一万物
|
||||
13. 商汤日日新
|
||||
14. 科大讯飞星火
|
||||
15. 360 智脑
|
||||
16. 网易有道子曰
|
||||
17. 面壁智能 MiniCPM
|
||||
18. 智源 FlagOpen
|
||||
19. 昆仑万维天工 / Skywork
|
||||
20. 无问芯穹
|
||||
|
||||
对应 seed:`seeds/plan_catalog_inventory_seed_cn_vendors_top20.json`
|
||||
|
||||
## 国内中转 / 聚合平台 20+
|
||||
|
||||
当前已经纳入目录基线的平台包括:
|
||||
|
||||
1. 腾讯云 TokenHub
|
||||
2. 腾讯云 CloudBase AI+
|
||||
3. 腾讯云 TI 平台大模型广场
|
||||
4. 阿里云百炼
|
||||
5. 魔搭 API-Inference
|
||||
6. 百度千帆
|
||||
7. 火山方舟
|
||||
8. 华为云 MaaS
|
||||
9. 天翼云模型推理服务
|
||||
10. 天翼云息壤
|
||||
11. 联通云 AICP
|
||||
12. 联通云 AI 应用开发平台
|
||||
13. 移动云 MoMA
|
||||
14. 有道智云 MaaS
|
||||
15. 360 智脑开放平台
|
||||
16. 硅基流动 SiliconCloud
|
||||
17. PPIO Model API
|
||||
18. UCloud UModelVerse
|
||||
19. 青云 CoresHub
|
||||
20. 金山云星流平台
|
||||
21. 以及腾讯云、阿里云、百度千帆各自拆分出的 Token Plan / Coding Plan / 企业版目录项
|
||||
|
||||
对应 seed:`seeds/plan_catalog_inventory_seed_cn_relays_top20plus.json`
|
||||
|
||||
### 候选补录(未进入正式目录基线)
|
||||
|
||||
以下平台已被识别为值得跟踪的中转/聚合候选,但当前还**未**进入正式 `plan_catalog_inventory` 基线,原因是尚未完成官方公开 pricing surface 核验:
|
||||
|
||||
1. OpenCode Zen(已完成官网初核)
|
||||
2. OpenCode Go(已完成官网初核)
|
||||
|
||||
当前判断:
|
||||
|
||||
- 需要区分 **OpenCode** 与 **OpenCode Zen**:前者是开源 AI 编程代理产品,本身不是我们要采集的价格平台;后者才是 OpenCode 官方提供的模型访问网关
|
||||
- 官网与文档已明确:Zen 是“由 OpenCode 提供的精选模型列表 / AI 网关”,支持通过 `https://opencode.ai/zen/v1/responses` 访问多种模型,并要求登录后添加账单信息、获取 API Key
|
||||
- 官方公开语义更接近“按请求付费 + 账户充值 + 月度消费限额”,而不是传统包月套餐表;营销页可见 `充值 $20 (即用即付)`、`支持设置月度消费限额`,文档页可见“你按请求付费,也可以向账户充值”
|
||||
- 因此它**可以算中转/聚合平台**,但当前更像 payg AI gateway,而不是已证实的 `subscription_plan` 型套餐平台
|
||||
- OpenCode Go 则是另一条独立付费路径:官方文档明确写明“首月 5 美元,之后每月 10 美元”,并列出可访问模型与 5 小时 / 每周 / 每月额度限制
|
||||
- 因此 **OpenCode Go** 更接近 `subscription_plan` 语义,而 **OpenCode Zen** 更接近 payg gateway;两者不应混成一个候选项
|
||||
- 以当前项目口径判断,**OpenCode Go 已经足够进入 `subscription_plan` 候选基线**:因为官方公开页已给出稳定的订阅价格、账单周期、可访问模型范围与额度限制;但它还**不等于**已经具备真实 importer 闭环
|
||||
- 下一步若要正式纳入目录基线,应拆成两条:
|
||||
- **OpenCode Zen**:优先判断其公开价格是否足够支撑 `catalog verification` 或真实 pricing importer
|
||||
- **OpenCode Go**:可进入 `subscription_plan` 候选基线;下一步再决定是先手工 seed,还是直接做真实 importer / manual-seed importer 闭环
|
||||
|
||||
### OpenCode Go 最小落地方案(当前建议)
|
||||
|
||||
按当前证据,最小风险闭环建议不是直接写新 importer,而是先走 **`subscription_plan_manual_seed.json` + `import_manual_subscription_seed.go`**:
|
||||
|
||||
1. **先补 1 条目录基线**
|
||||
- 在 `plan_catalog_inventory` seed 中新增 `OpenCode Go`
|
||||
- 建议语义:
|
||||
- `operatorType`: `relay`
|
||||
- `platformType`: `relay_platform`
|
||||
- `planFamily`: `coding_plan`
|
||||
- `billingCycle`: `monthly`
|
||||
- `currency`: `USD`
|
||||
- `sourceURL`: `https://opencode.ai/docs/go`
|
||||
- `planStatus`: `confirmed`
|
||||
- 原因:它是多模型访问订阅,不是单一官方模型厂商套餐
|
||||
|
||||
2. **先补 1 条 subscription seed,而不是伪造多 tier**
|
||||
- 当前公开价只有一档主订阅:`首月 5 美元,之后每月 10 美元`
|
||||
- 在现有 schema 下,建议先落 **标准价主记录 1 条**:
|
||||
- `planCode`: `opencode-go-monthly`
|
||||
- `planName`: `OpenCode Go`
|
||||
- `tier`: `Standard`
|
||||
- `billingCycle`: `monthly`
|
||||
- `currency`: `USD`
|
||||
- `listPrice`: `10`
|
||||
- `priceUnit`: `USD/month`
|
||||
- `planScope`: `OpenCode Go multi-model coding subscription`
|
||||
- `首月 5 美元` 作为促销说明写进 `notes`,不要把首月促销和长期标准价拆成两条并列套餐,避免污染长期价格真相
|
||||
|
||||
3. **额度字段先保守表达**
|
||||
- 可见限制:
|
||||
- `5 小时限制 — 12 美元使用额度`
|
||||
- `每周限制 — 30 美元使用额度`
|
||||
- `每月限制 — 60 美元使用额度`
|
||||
- 由于当前 `subscription_plan` 只有单组 `quota_value/quota_unit`,不能无损表达三层额度
|
||||
- 最小落地建议:
|
||||
- `quotaValue`: `60`
|
||||
- `quotaUnit`: `usd_usage/month_cap`
|
||||
- 其余 5 小时 / 每周限制写入 `notes`
|
||||
|
||||
4. **modelScope 可以直接公开落入 seed**
|
||||
- 因官网已公开模型清单,可把当前列表写入 `modelScope`
|
||||
- 例如:`GLM-5`, `GLM-5.1`, `Kimi K2.5`, `Kimi K2.6`, `MiMo-V2.5`, `MiMo-V2.5-Pro`, `MiniMax M2.5`, `Qwen3.5 Plus`, `Qwen3.6 Plus`, `MiniMax M2.7`, `DeepSeek V4 Pro`, `DeepSeek V4 Flash`
|
||||
|
||||
5. **验证顺序**
|
||||
- `go test -tags=llm_script scripts/import_manual_subscription_seed.go scripts/import_manual_subscription_seed_test.go scripts/subscription_import_common.go`
|
||||
- `go run -tags=llm_script scripts/import_manual_subscription_seed.go -seed seeds/subscription_plan_manual_seed.json -dry-run`
|
||||
- 验证 dry-run 输出中出现 `OpenCode Go:1` 与 `coding_plan:1`
|
||||
|
||||
## 全球官方 / 中转参考集
|
||||
|
||||
本轮通过 web 搜索补录并进入目录基线的平台包括:
|
||||
|
||||
1. Google Gemini API
|
||||
2. Mistral La Plateforme
|
||||
3. Cohere Platform
|
||||
4. OpenRouter
|
||||
5. Together AI
|
||||
6. Fireworks AI
|
||||
7. DeepInfra
|
||||
8. GroqCloud
|
||||
9. Replicate
|
||||
10. Hyperbolic
|
||||
11. Novita AI
|
||||
12. Azure OpenAI Service
|
||||
13. Amazon Bedrock
|
||||
14. Vertex AI Generative AI
|
||||
15. Cloudflare Workers AI
|
||||
16. Baseten
|
||||
17. Cerebras Inference
|
||||
18. Perplexity Agent API
|
||||
19. SambaNova Cloud
|
||||
20. 京东云 JoyBuilder
|
||||
|
||||
对应 seed:`seeds/plan_catalog_inventory_seed_web_research.json`
|
||||
|
||||
## 云服务中转 / 云厂商平台
|
||||
|
||||
| 平台 | 当前结论 | 目录归类 | 后续 importer |
|
||||
|------|----------|----------|---------------|
|
||||
| 腾讯云 TokenHub | 已确认 `Token Plan`(个人版、企业版专业、企业版轻享)与 `Coding Plan` 并存 | `token_plan` + `coding_plan` | 已接入 `tencent_catalog` / `import_tencent_subscription.go` |
|
||||
| 阿里云百炼 | 已确认 `Token Plan(团队版)` 与 `Coding Plan` 并存,且仍保留按量计费 | `token_plan` + `coding_plan` | 已接入 `import_aliyun_subscription.go` |
|
||||
| 百度千帆 | 已确认 `Coding Plan` 与 `Token 福利包` 并存,后者存在首购优惠价 | `coding_plan` + `token_plan` | 已接入 `import_baidu_subscription.go` |
|
||||
| 火山方舟 | 已从官方开发者社区确认 `Coding Plan` 已上线,且公开披露标准月费与首月活动价 | `coding_plan` | 已接入 `import_bytedance_subscription.go` |
|
||||
| 天翼云模型推理服务 | 已确认 `Coding Plan` 与活动型 `Token Plan` 并存 | `coding_plan` + `token_plan` | 已接入 `import_ctyun_subscription.go` |
|
||||
| 华为云 MaaS | 当前明确支持“按 Token 付费 + 套餐包/资源包计费”,不是 `Coding Plan` 命名体系 | `package_plan` + `pay_as_you_go` | 已接入 `import_huawei_package.go` 与 `import_huawei_maas_pricing.go` |
|
||||
|
||||
### 证据入口
|
||||
|
||||
- 腾讯云 Token Plan 个人版:[cloud.tencent.com/document/product/1823/130060](https://cloud.tencent.com/document/product/1823/130060)
|
||||
- 腾讯云 Token Plan 企业版专业套餐:[cloud.tencent.com/document/product/1823/130659](https://cloud.tencent.com/document/product/1823/130659)
|
||||
- 腾讯云 Token Plan 企业版轻享套餐:[cloud.tencent.com/document/product/1823/131173](https://cloud.tencent.com/document/product/1823/131173)
|
||||
- 腾讯云 Coding Plan 规则页:[cloud.tencent.com/document/product/1823/130103](https://cloud.tencent.com/document/product/1823/130103)
|
||||
- 阿里云百炼 Token Plan 概述:[help.aliyun.com/zh/model-studio/token-plan-overview](https://help.aliyun.com/zh/model-studio/token-plan-overview)
|
||||
- 阿里云百炼 Coding Plan 概述:[help.aliyun.com/zh/model-studio/coding-plan-quickstart](https://help.aliyun.com/zh/model-studio/coding-plan-quickstart)
|
||||
- 百度千帆 Coding Plan:[cloud.baidu.com/doc/qianfan/s/imlg0beiu](https://cloud.baidu.com/doc/qianfan/s/imlg0beiu)
|
||||
- 百度千帆 Token 福利包:[cloud.baidu.com/doc/qianfan/s/Smoghsq3g](https://cloud.baidu.com/doc/qianfan/s/Smoghsq3g)
|
||||
- 火山方舟 Coding Plan 社区文章:[developer.volcengine.com/articles/7604465649330749490](https://developer.volcengine.com/articles/7604465649330749490)
|
||||
- 华为云 MaaS 文本生成模型计费说明:[support.huaweicloud.com/price-maas/price-maas-0002.html](https://support.huaweicloud.com/price-maas/price-maas-0002.html)
|
||||
|
||||
## 官方模型平台
|
||||
|
||||
| 平台 | 当前结论 | 目录归类 | 说明 |
|
||||
|------|----------|----------|------|
|
||||
| 智谱 AI | 已确认 `GLM Coding Plan` | `coding_plan` | 已接入 `import_zhipu_coding_plan.go`,当前先落公开活动底价与套餐能力说明 |
|
||||
| MiniMax | 已确认 `Token Plan` | `token_plan` | 同时保留按量计费 API Key 切换路径 |
|
||||
| OpenAI | 当前以按量计费为主,未检索到官方 `Token Plan` / `Coding Plan` | `pay_as_you_go` | 继续走现有官方价格 importer 思路 |
|
||||
| Anthropic | 当前以按量计费为主,未检索到官方 `Token Plan` / `Coding Plan` | `pay_as_you_go` | 只有模型定价、缓存与批处理折扣 |
|
||||
| DeepSeek | 当前以按量计费为主,未检索到官方 `Token Plan` / `Coding Plan` | `pay_as_you_go` | 支持赠送余额与限时折扣 |
|
||||
| Moonshot AI | 当前以按量计费为主,未检索到官方 `Token Plan` / `Coding Plan` | `pay_as_you_go` | 官方重点仍是 Token 单价与缓存计费 |
|
||||
| xAI | 当前以按量计费为主,未检索到官方 `Token Plan` / `Coding Plan` | `pay_as_you_go` | 同时支持工具调用计费和批处理折扣 |
|
||||
|
||||
### 证据入口
|
||||
|
||||
- 智谱 GLM Coding Plan:[docs.bigmodel.cn/cn/coding-plan/overview](https://docs.bigmodel.cn/cn/coding-plan/overview)
|
||||
- MiniMax Token Plan:[platform.minimaxi.com/docs/token-plan/intro](https://platform.minimaxi.com/docs/token-plan/intro)
|
||||
- OpenAI Pricing:[platform.openai.com/docs/pricing](https://platform.openai.com/docs/pricing/)
|
||||
- Anthropic Pricing:[docs.anthropic.com/en/docs/about-claude/pricing](https://docs.anthropic.com/en/docs/about-claude/pricing)
|
||||
- DeepSeek Pricing:[api-docs.deepseek.com/zh-cn/quick_start/pricing](https://api-docs.deepseek.com/zh-cn/quick_start/pricing/)
|
||||
- Moonshot Pricing:[platform.moonshot.cn/docs/pricing/chat](https://platform.moonshot.cn/docs/pricing/chat)
|
||||
- xAI Pricing:[docs.x.ai/developers/pricing](https://docs.x.ai/developers/pricing)
|
||||
|
||||
## 数据库落点
|
||||
|
||||
本次新增的数据库清单表:
|
||||
|
||||
- 表名:`plan_catalog_inventory`
|
||||
- 作用:保存平台级证据与 importer 排期,而不是最终套餐明细
|
||||
- 导入脚本:`scripts/import_plan_catalog.go`
|
||||
- seed 文件:
|
||||
- `seeds/plan_catalog_inventory_seed.json`
|
||||
- `seeds/plan_catalog_inventory_seed_cn_vendors_top20.json`
|
||||
- `seeds/plan_catalog_inventory_seed_cn_relays_top20plus.json`
|
||||
- `seeds/plan_catalog_inventory_seed_web_research.json`
|
||||
- 新增字段:
|
||||
- `catalog_segment`:`general / vendor_top20 / relay_top20plus / global_reference`
|
||||
- `market_rank`:榜单顺序
|
||||
|
||||
本次还保留了一个手工套餐 seed 导入器,作为极少数暂无稳定公开结构化页面的平台兜底手段:
|
||||
|
||||
- 导入脚本:`scripts/import_manual_subscription_seed.go`
|
||||
- seed 文件:`seeds/subscription_plan_manual_seed.json`
|
||||
- 当前覆盖:无生产链路默认启用的平台;MiniMax Token Plan 已切换到真实 importer
|
||||
|
||||
建议使用顺序:
|
||||
|
||||
1. 先更新 `plan_catalog_inventory`
|
||||
2. 再根据 `catalog_segment + market_rank + plan_family + importer_key` 排出平台实现顺序
|
||||
3. 已确认且价格明确的套餐,先通过手工 seed 进入 `subscription_plan`
|
||||
4. 官方按量价格继续进入 `region_pricing`
|
||||
|
||||
## 当前 importer 状态
|
||||
|
||||
已完成:
|
||||
|
||||
1. `tencent_catalog` / `import_tencent_subscription.go`
|
||||
2. `import_aliyun_subscription.go`
|
||||
3. `import_baidu_subscription.go`
|
||||
4. `import_ctyun_subscription.go`
|
||||
|
||||
这批平台现在都已经进入真实抓取或目录级实时校验链路:
|
||||
|
||||
1. `import_bytedance_subscription.go`
|
||||
2. `import_huawei_package.go`
|
||||
3. `import_zhipu_coding_plan.go`
|
||||
4. `import_minimax_subscription.go`
|
||||
5. `import_cucloud_catalog.go`
|
||||
6. `import_mobile_cloud_pricing.go`
|
||||
7. `import_cucloud_pricing.go`
|
||||
|
||||
新增已完成:
|
||||
|
||||
1. `import_youdao_pricing.go`
|
||||
2. `import_360_pricing.go`
|
||||
3. `import_siliconflow_pricing.go`
|
||||
4. `import_ppio_pricing.go`
|
||||
5. `import_ucloud_pricing.go`
|
||||
6. `import_coreshub_pricing.go`
|
||||
7. `import_cloudflare_pricing.go`
|
||||
8. `import_perplexity_pricing.go`
|
||||
9. `import_vertex_pricing.go`
|
||||
10. `import_bedrock_pricing.go`
|
||||
11. `import_azure_openai_pricing.go`
|
||||
12. `import_minimax_subscription.go`
|
||||
13. `import_qwen_pricing.go`
|
||||
14. `import_hunyuan_pricing.go`
|
||||
15. `import_huawei_maas_pricing.go`
|
||||
16. `import_bytedance_pricing.go`
|
||||
17. `import_lingyiwanwu_pricing.go`
|
||||
18. `import_xfyun_pricing.go`
|
||||
19. `import_sensenova_pricing.go`
|
||||
20. `import_360_pricing.go`(同时覆盖 vendor_top20 的 360 智脑与 relay_top20plus 的 360 开放平台位点)
|
||||
|
||||
这些平台统一按 `pay_as_you_go -> region_pricing` 处理,直接抓取官方公开模型价格,不再停留在 `future_official_pricing`。
|
||||
其中 `SiliconFlow` 当前优先尝试官方价格入口;若入口返回站点落地页或临时不可用,则回退到仓库内最近核验的官方快照,避免日跑流水线因前端路由问题中断。
|
||||
其中 `火山方舟` 当前 importer 先落地 `在线推理(常规)` 官方 token 价格;`低延迟/批量/TPM 保障包` 多 service-class 定价,以及 `向量/图片/3D` 这类非对称或非 token 计费项,待 schema 扩展后再细化。
|
||||
其中 `联通云` 当前新增 `import_cucloud_pricing.go`:已真实导入 AISP Token Plan 公开披露的 `DeepSeek-V4-Pro / DeepSeek-V4-Flash / MiniMax-M2.5` 三模型 blended 单价与区域支持矩阵;并已在 `plan_catalog_inventory` seed 中补充 `cucloud-aisp-token-plan-pricing` 显式 pricing evidence 行。公开文档仅确认 `按量计费模式` 存在,尚未披露 payg per-model 销售价表,因此不能宣称联通云 payg 已完整打通;详见 `docs/references/cucloud-token-plan-vs-aisp.md`。
|
||||
其中 `零一万物` 当前新增 `import_lingyiwanwu_pricing.go`:官方文档价格表当前仅公开 `yi-lightning` 与 `yi-vision-v2` 两个模型的 `价格/1M token`,因此按 blended token 单价入库,未扩展到未公开的其他 Yi SKU。
|
||||
其中 `讯飞` 当前新增 `import_xfyun_pricing.go`:公开价格页披露 `X2/X1.5 / Ultra / Pro / Lite` 四档 `元/百万tokens` blended 定价卡片;当前按 blended token 单价入库,未把更细粒度模型版本(如 Pro-128K / Max-32K)强行映射为公开 payg 价格。
|
||||
其中 `商汤 SenseNova` 当前新增 `import_sensenova_pricing.go`:公开模型页明确给出“公测期免费开放、所有模型完全开放”,文档页列出当前 3 个公开模型(`SenseNova 6.7 Flash-Lite / SenseNova U1 Fast / DeepSeek V4 Flash`)并展示 `GET /v1/models` 的零价格 `pricing` 对象,因此当前按 0 价 free-tier 入库;其中 `U1 Fast` 使用独立图片生成接口,现阶段仅在免费语义下落表,不把它误写成常规 token 计费模型。
|
||||
其中 `360 智脑` 当前将 vendor_top20 的 `360-zhinao-api-payg` 校准到已存在的 `import_360_pricing.go`:当前可访问且结构稳定的官方公开价格源是 `https://ai.360.com/open/models`,页面同时包含 360 自有模型(如 `360zhinao-turbo-doubao-seed-1-8`)与第三方模型的输入/输出 token 单价,因此当前按该“官方开放平台广义价格面”入库,而不是声称拿到了仅限 360 自研模型的独立价表。
|
||||
其中 `网易有道子曰开放平台` 当前将 vendor_top20 的 `youdao-ziyue-api-payg` 校准到已存在的 `import_youdao_pricing.go`:当前可访问且结构稳定的官方公开价格源是 `https://ai.youdao.com/new/thinkflow`,页面同时公开 DeepSeek / Qwen / Kimi / MiniMax / GLM 等模型的输入/输出 token 单价,并展示渠道转发能力,因此当前按该“官方开放平台广义价格面”入库,而不是声称拿到了仅限子曰自研模型的独立价表。
|
||||
|
||||
对于暂时没有稳定公开结构化价格页、但官方平台入口已经确认的长尾平台,当前统一归到:
|
||||
|
||||
- `import_catalog_seed_verification.go`
|
||||
|
||||
这条链路属于目录级官方入口核验,会持续回写 `plan_catalog_inventory.last_checked_at` 和核验备注,确保第一模块的覆盖方式已经定型,不再保留 `future_official_pricing` 占位状态。
|
||||
|
||||
下一步建议优先级:
|
||||
|
||||
1. `移动云语音按字符 / 按秒计费如何落 schema`
|
||||
2. `联通云 payg per-model 价格公开表是否出现(当前 blocker 详见 docs/references/cucloud-token-plan-vs-aisp.md)`
|
||||
3. `MiniCPM 开放平台官方 payg 价格源复查(当前公开面未见可落库价卡;详见 docs/references/modelbest-minicpm-public-source-gap.md)`
|
||||
227
docs/PRODUCTION_CHECKLIST.md
Normal file
227
docs/PRODUCTION_CHECKLIST.md
Normal file
@@ -0,0 +1,227 @@
|
||||
# 生产上线检查清单
|
||||
|
||||
本文档面向“准备把当前仓库作为生产服务上线”的场景,聚焦发布前检查、上线步骤、回滚和日常守护要求。
|
||||
|
||||
## 目标
|
||||
|
||||
上线后的最小可用能力应包括:
|
||||
|
||||
- 数据库可连接且已完成全部迁移
|
||||
- API Server 可稳定返回 `/health`、`/api/v1/models`、`/api/v1/reports/latest`
|
||||
- 正式日报可由调度脚本按天产出
|
||||
- 失败时可回退、可告警、可恢复
|
||||
|
||||
## 生产拓扑建议
|
||||
|
||||
建议采用以下最小拓扑:
|
||||
|
||||
1. PostgreSQL 16
|
||||
2. API Server:`cmd/server/main.go` 构建产物
|
||||
3. Nginx:托管 `frontend/dist` 并反向代理 `/api` 与 `/health`
|
||||
4. Cron 或 systemd timer:执行 `scripts/run_daily.sh`
|
||||
|
||||
如果使用容器部署,仓库内 `docker-compose.yml` 可作为单机参考,但正式环境仍建议:
|
||||
|
||||
- 单独管理数据库持久化与备份
|
||||
- 在网关层处理 TLS、限流和访问控制
|
||||
- 将密钥注入部署系统,而不是依赖仓库内 `.env`
|
||||
|
||||
## 发布前硬检查
|
||||
|
||||
### 基础设施
|
||||
|
||||
- PostgreSQL 已创建库并验证可连接
|
||||
- `DATABASE_URL` 在 API Server、调度脚本、备份脚本所在环境都可用
|
||||
- `reports/daily` 及其归档目录所在磁盘有足够空间
|
||||
- `/tmp` 不会被过早清理,避免影响每天的流水线日志追踪
|
||||
|
||||
### 数据与迁移
|
||||
|
||||
- 已执行 `bash scripts/apply_migration.sh`
|
||||
- `daily_report`、`report_runs`、`subscription_plan`、`region_pricing`、`daily_signal_snapshot` 等关键表存在
|
||||
- 历史数据回填策略已确认,避免上线首日“空库”
|
||||
|
||||
### 应用与产物
|
||||
|
||||
- `go test ./...` 通过(仅覆盖 package 形式的 Go 代码,如 `cmd/server` 与 `internal/...`;其中 API 错误结构与模型主价格排序规则需由这些 package tests 兜底)
|
||||
- `bash scripts/test_importers.sh` 通过(覆盖 scripts 层 importer targeted go test matrix)
|
||||
- `bash scripts/importer_smoke_gate_test.sh` 通过
|
||||
- `bash scripts/pipeline_runtime_alignment_test.sh` 通过
|
||||
- `bash scripts/test.sh` 通过(仅覆盖 `fetch_openrouter` focused test)
|
||||
- `cd frontend && npm run test -- --run` 通过
|
||||
- `cd frontend && npm run build` 通过
|
||||
- `go build ./cmd/server` 通过
|
||||
- 已确认发布结论不是仅凭 `go test ./...` 得出,而是同时包含 scripts 与 gate 层验证
|
||||
|
||||
### 调度与日报
|
||||
|
||||
- 正式调度命令已确定:`bash scripts/run_daily.sh`
|
||||
- 手工复跑命令已确定:`bash scripts/run_real_pipeline.sh`
|
||||
- 历史补跑命令已确定:`bash scripts/rebuild_historical_report.sh YYYY-MM-DD`
|
||||
- 日内价格追踪命令已确定:`bash scripts/run_intraday_price_watch.sh`
|
||||
- 日内新闻发现与验证命令已确定:`bash scripts/run_intraday_discovery_watch.sh`
|
||||
- `OPENROUTER_API_KEY` 已在正式调度环境可用
|
||||
- `FEISHU_WEBHOOK` 已配置或明确不上告警
|
||||
- 候选发现所需 search / LLM provider 已配置,缺失时会以前置条件错误失败,不会伪装成“无新闻”
|
||||
|
||||
|
||||
### 安全与访问控制
|
||||
|
||||
- 密钥未提交入库
|
||||
- API 暴露路径已放在网关后,不直接裸露到公网
|
||||
- 已补充访问控制、TLS、限流与日志保留策略
|
||||
- `scripts/restore.sh` 属于高风险脚本,使用权限已收敛到少数运维成员
|
||||
|
||||
## 上线门禁命令
|
||||
|
||||
建议按下面顺序执行:
|
||||
|
||||
```bash
|
||||
bash scripts/verify_pre_phase6.sh
|
||||
bash scripts/verify_phase6.sh
|
||||
bash healthcheck.sh
|
||||
```
|
||||
|
||||
其中 `verify_phase6.sh` 会额外检查:
|
||||
|
||||
- 真实采集链路
|
||||
- API Server 构建与健康检查
|
||||
- `/api/v1/models` 响应时间 `< 500ms`
|
||||
- 最近 7 次采集成功率 `>= 95%`
|
||||
- 前端测试入口存在
|
||||
|
||||
|
||||
## Phase 6+ 范围定义
|
||||
|
||||
Phase 6 仍是发布前主门禁;`verify_phase6.sh` 通过即可证明主链路验收闭环成立。
|
||||
|
||||
Phase 6+ 属于 **治理阶段**,不属于发布门禁本身。它覆盖:
|
||||
- review / cron / verifier / backlog / memory 的长期治理
|
||||
- release 解释语义、风险老化、状态一致性与噪声收敛
|
||||
- 外部 provider 漂移后的 fallback / guard / summary 持续补强
|
||||
|
||||
因此,Phase 6+ 项目未关闭时,不能反推 Phase 6 主验收失败;反之,Phase 6 已通过,也不代表 Phase 6+ 治理工作已经完成。
|
||||
## 上线步骤
|
||||
|
||||
### 1. 发布前备份
|
||||
|
||||
```bash
|
||||
bash scripts/backup.sh
|
||||
```
|
||||
|
||||
确认:
|
||||
|
||||
- 备份文件已生成在 `/tmp/llm_hub_backups`
|
||||
- 备份文件大小非零
|
||||
- 如接入 OSS,远端对象已上传成功
|
||||
|
||||
### 2. 执行迁移
|
||||
|
||||
```bash
|
||||
bash scripts/apply_migration.sh
|
||||
```
|
||||
|
||||
### 3. 构建与发布 API Server / 前端
|
||||
|
||||
```bash
|
||||
go build -o bin/server ./cmd/server
|
||||
cd frontend && npm run build
|
||||
```
|
||||
|
||||
### 4. 部署反向代理
|
||||
|
||||
确认 Nginx 已正确代理:
|
||||
|
||||
- `/` -> `frontend/dist`
|
||||
- `/api/` -> `app:8080/api/`
|
||||
- `/health` -> `app:8080/health`
|
||||
|
||||
### 5. 手工真实复跑一次
|
||||
|
||||
```bash
|
||||
bash scripts/run_real_pipeline.sh
|
||||
```
|
||||
|
||||
目的:
|
||||
|
||||
- 验证真实采集、补录、日报生成和写库全链路
|
||||
- 确认不会错误覆盖“最新正式日报”语义
|
||||
|
||||
### 6. 启用正式调度
|
||||
|
||||
```cron
|
||||
0 8 * * * cd /path/to/llm-intelligence && bash scripts/run_daily.sh >> /tmp/llm_hub_cron.log 2>&1
|
||||
```
|
||||
# 日内价格追踪(推荐)
|
||||
0 */4 * * * cd /path/to/llm-intelligence && bash scripts/run_intraday_price_watch.sh >> /tmp/llm_hub_intraday.log 2>&1
|
||||
# 日内新闻发现与验证(推荐)
|
||||
0 */2 * * * cd /path/to/llm-intelligence && bash scripts/run_intraday_discovery_watch.sh >> /tmp/llm_hub_intraday_discovery.log 2>&1
|
||||
|
||||
### 7. 线上冒烟
|
||||
|
||||
```bash
|
||||
curl -fsS http://127.0.0.1:8080/health
|
||||
curl -fsS http://127.0.0.1:8080/api/v1/models
|
||||
curl -fsS http://127.0.0.1:8080/api/v1/reports/latest
|
||||
```
|
||||
|
||||
## 运行中监控基线
|
||||
|
||||
建议至少监控以下指标:
|
||||
|
||||
| 指标 | 目标 / 告警线 | 说明 |
|
||||
|------|---------------|------|
|
||||
| API 健康检查 | `200` | `/health` 必须稳定可达 |
|
||||
| `/api/v1/models` 响应时间 | `< 500ms` | Phase 6 验收门槛 |
|
||||
| 最近 7 次采集成功率 | `>= 95%` | Phase 6 验收门槛 |
|
||||
| 模型总数 | `< 300` 告警 | 来自现有 RUNBOOK 基线 |
|
||||
| 今日日报是否生成 | 每天 08:00 后应存在 | 检查 `daily_report` 与产物文件 |
|
||||
| 归档是否完整 | Markdown + HTML 均存在 | 检查 `reports/daily/YYYY/MM/` |
|
||||
|
||||
## 回滚方案
|
||||
|
||||
### 何时触发回滚
|
||||
|
||||
- API Server 启动失败或健康检查持续异常
|
||||
- 真实流水线连续失败且无法在发布窗口内修复
|
||||
- 正式日报生成语义错误,导致“最新正式日报”被污染
|
||||
- 迁移导致查询失败或关键表结构异常
|
||||
|
||||
### 回滚步骤
|
||||
|
||||
1. 停止正式调度,避免继续写入错误数据
|
||||
2. 回滚应用版本或镜像
|
||||
3. 如数据已损坏,使用备份恢复:
|
||||
|
||||
```bash
|
||||
bash scripts/restore.sh --force /path/to/backup.sql.gz
|
||||
```
|
||||
|
||||
4. 重新执行迁移到目标版本所需状态
|
||||
5. 启动服务后执行:
|
||||
|
||||
```bash
|
||||
bash healthcheck.sh
|
||||
curl -fsS http://127.0.0.1:8080/health
|
||||
curl -fsS http://127.0.0.1:8080/api/v1/reports/latest
|
||||
```
|
||||
|
||||
## 常见上线遗漏
|
||||
|
||||
- 只启动 API,没有配置正式日报调度
|
||||
- 只写入 `daily_report`,但落盘目录没有写权限
|
||||
- 手工复跑后误以为“正式日报已准备好”,但 `is_official_daily=false`
|
||||
- 把 API 直接暴露到公网,却没有鉴权或限流
|
||||
- 依赖 `.env.local`,但生产机器并不存在该文件
|
||||
- 没有先跑 `backup.sh` 就执行高风险恢复或迁移
|
||||
|
||||
## 建议的发布结论标准
|
||||
|
||||
满足以下条件后,才建议标记为“可生产上线”:
|
||||
|
||||
- `verify_pre_phase6.sh` 通过
|
||||
- `verify_phase6.sh` 通过
|
||||
- 手工真实复跑成功
|
||||
- API / 前端冒烟通过
|
||||
- 正式调度已配置并完成一次演练
|
||||
- 备份与恢复路径已演练至少一次
|
||||
40
docs/README.md
Normal file
40
docs/README.md
Normal file
@@ -0,0 +1,40 @@
|
||||
# Docs Landing
|
||||
|
||||
> 先从这里进入文档树,避免直接打开历史 review 报告后误把旧结论当成当前真相。
|
||||
|
||||
## 当前真相 / Active Boards
|
||||
|
||||
1. [../OPENCLAW_EXECUTION.md](../OPENCLAW_EXECUTION.md)
|
||||
- 当前运行真相、执行顺序、验证协议、门禁口径
|
||||
2. [../reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md](../reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md)
|
||||
- OpenClaw 能力缺口当前台账与最新 review 增量
|
||||
3. [../TASKS.md](../TASKS.md)
|
||||
- 项目任务真相来源
|
||||
4. [../GOALS.md](../GOALS.md)
|
||||
- 项目目标范围真相来源
|
||||
|
||||
## 运行 / 发布 / 配置
|
||||
|
||||
- [CONFIGURATION.md](CONFIGURATION.md):环境变量、运行语义、配置约束
|
||||
- [PRODUCTION_CHECKLIST.md](PRODUCTION_CHECKLIST.md):发布前检查、发布与回滚流程
|
||||
- [API_REFERENCE.md](API_REFERENCE.md):API 入口、返回体与排障说明
|
||||
- [PERFORMANCE_TEST.md](PERFORMANCE_TEST.md):性能基线
|
||||
|
||||
## Importer / Coverage / Priority
|
||||
|
||||
- [PLAN_CATALOG_COVERAGE_MATRIX.md](PLAN_CATALOG_COVERAGE_MATRIX.md):平台覆盖矩阵与 importer/runtime truth
|
||||
- [PLAN_CATALOG_INVENTORY.md](PLAN_CATALOG_INVENTORY.md):平台目录基线与 importerKey 清单
|
||||
- [NEXT_IMPORTER_RUNTIME_PRIORITY.md](NEXT_IMPORTER_RUNTIME_PRIORITY.md):下一批 importer/runtime 挂载优先级
|
||||
|
||||
## 规划与参考
|
||||
|
||||
- [plans/](plans/):实施计划、收口计划、设计草案
|
||||
- [references/](references/):特定 provider/importer 的上游真相与限制说明
|
||||
- [../TECHNICAL_DESIGN.md](../TECHNICAL_DESIGN.md):详细技术设计与数据模型背景
|
||||
- [../RUNBOOK.md](../RUNBOOK.md):运维巡检、故障排查、备份恢复
|
||||
- [../DEPLOYMENT.md](../DEPLOYMENT.md):部署步骤与快速启动
|
||||
|
||||
## 历史材料说明
|
||||
|
||||
- `reports/openclaw/` 下大量带日期的 `*-review.md`、`*-checklist.md`、`*-task-board.md` 属于历史快照。
|
||||
- 这些文件可用于追溯当时现场,但**不能替代** `OPENCLAW_EXECUTION.md`、`OPENCLAW_CAPABILITY_BACKLOG.md`、`TASKS.md`、`GOALS.md` 作为当前真相入口。
|
||||
992
docs/plans/2026-05-13-daily-report-ui-design.md
Normal file
992
docs/plans/2026-05-13-daily-report-ui-design.md
Normal file
@@ -0,0 +1,992 @@
|
||||
# 日报重构 UI 设计说明书
|
||||
|
||||
> 项目:LLM Intelligence Hub
|
||||
> 日期:2026-05-13
|
||||
> 状态:设计基线
|
||||
> 适用范围:HTML 日报、前端 Dashboard 日报摘要入口、后续移动端 / Web 端统一视觉规范
|
||||
|
||||
## 1. 背景
|
||||
|
||||
当前日报已经能输出真实数据,但整体阅读体验更接近“数据库导出结果页”,而不是“每天值得打开的 AI 模型与价格情报产品”。
|
||||
|
||||
现状问题:
|
||||
|
||||
- 首屏缺少“今日发生了什么”的判断,读者看完仍不知道重点。
|
||||
- 免费模型、推荐模型、分类概览之间存在重复,信息层级不清。
|
||||
- “免费”语义混淆,官方免费、聚合免费、活动免费没有视觉区分。
|
||||
- 移动端沿用了桌面表格逻辑,不适合快速浏览。
|
||||
- 页面缺少品牌气质,无法体现“高端移动资讯产品 + 专业选型工具”的特点。
|
||||
|
||||
本次 UI 重构不推翻底层数据库,不改动项目对模型、价格、平台、来源的原始采集事实;只在日报内容组织和页面呈现层重构“让用户愿意看、看得懂、看完能行动”的产品体验。
|
||||
|
||||
## 2. 设计目标
|
||||
|
||||
### 2.1 产品目标
|
||||
|
||||
新日报要同时服务两类用户:
|
||||
|
||||
- 选型决策者:今天该试什么模型,哪个更划算,哪些来源更稳。
|
||||
- 行业情报读者:今天市场发生了什么变化,哪些发布、活动、价格战值得关注。
|
||||
|
||||
默认优先级:**两者兼顾,但优先选型**。
|
||||
|
||||
### 2.2 体验目标
|
||||
|
||||
新日报必须满足以下体验要求:
|
||||
|
||||
- 用户在 30 秒内理解“今天最重要的变化”。
|
||||
- 用户在 1 分钟内得到“今天该优先关注谁”的结论。
|
||||
- 用户能一眼分辨“官方免费 / 聚合免费 / 活动免费 / 来源待验证”。
|
||||
- 移动端首屏不依赖长表格,也不依赖大段说明文字。
|
||||
- 页面视觉上要像“高端移动资讯产品”,而不是管理后台。
|
||||
|
||||
### 2.3 不做什么
|
||||
|
||||
- 不把首页继续做成全量模型清单。
|
||||
- 不在主阅读区堆长段分析文字。
|
||||
- 不把灰区或待验证来源直接混入主推荐。
|
||||
- 不为了炫技加入噪音型动画或低质装饰。
|
||||
|
||||
## 3. 内容策略
|
||||
|
||||
### 3.1 基本原则
|
||||
|
||||
日报采用以下原则:
|
||||
|
||||
- 先结论,后证据。
|
||||
- 先变化,后存量。
|
||||
- 先行动建议,后完整数据。
|
||||
- 主区讲重点,附录讲完整。
|
||||
|
||||
### 3.2 主体结构
|
||||
|
||||
日报正文固定拆成四层:
|
||||
|
||||
1. 今日结论区
|
||||
2. 今日变化区
|
||||
3. 今日选型推荐区
|
||||
4. 附录区
|
||||
|
||||
其中:
|
||||
|
||||
- `今日结论区` 负责抓眼球和快速定调。
|
||||
- `今日变化区` 负责解释“为什么今天值得看”。
|
||||
- `今日选型推荐区` 负责直接输出可执行建议。
|
||||
- `附录区` 负责保留数据库完整性与查阅深度。
|
||||
|
||||
### 3.3 事件类型
|
||||
|
||||
日报重点事件按以下类型组织:
|
||||
|
||||
- `new_model`
|
||||
- `official_release`
|
||||
- `price_cut`
|
||||
- `price_increase`
|
||||
- `free_policy_change`
|
||||
- `promo_campaign`
|
||||
- `source_risk_change`
|
||||
|
||||
同一模型可以有多个事件标签,但在首页只选择一个主标签展示。
|
||||
|
||||
### 3.3.1 变化基线规则
|
||||
|
||||
为保证“今日变化”具备稳定解释力,所有变化事件必须绑定统一的比较基线,禁止只展示“变了”而不说明“相对什么变了”。
|
||||
|
||||
允许使用的变化基线:
|
||||
|
||||
- `较昨日`
|
||||
- `较上次有效价格`
|
||||
- `7日内新低 / 新高`
|
||||
- `首次出现`
|
||||
- `官方首次发布`
|
||||
|
||||
规则要求:
|
||||
|
||||
- 所有 `price_cut` / `price_increase` 必须显示比较基线。
|
||||
- 所有 `new_model` 必须显示“首次出现”或“官方首次发布”。
|
||||
- 所有 `free_policy_change` 必须显示“由付费转免费”或“由免费转付费”的方向信息。
|
||||
- UI 上必须保留一个稳定的基线展示位,不允许只靠颜色表达变化。
|
||||
|
||||
推荐展示格式:
|
||||
|
||||
- `较昨日 -18%`
|
||||
- `较上次调价下降 ¥0.20/M`
|
||||
- `7日内最低价`
|
||||
- `首次出现在可信来源`
|
||||
|
||||
### 3.4 免费模型与来源可信度
|
||||
|
||||
日报层必须在原始数据上额外生成语义标签:
|
||||
|
||||
- `official_free`
|
||||
- `aggregator_free`
|
||||
- `promo_free`
|
||||
- `trial_credit`
|
||||
- `unknown_free`
|
||||
|
||||
来源可信度分级:
|
||||
|
||||
- `official_verified`
|
||||
- `cloud_verified`
|
||||
- `aggregator_verified`
|
||||
- `self_hosted_gateway`
|
||||
- `unverified_relay`
|
||||
|
||||
规则要求:
|
||||
|
||||
- 所有“免费”展示必须带来源类型徽标。
|
||||
- 主推荐区默认只允许 `official_verified`、`cloud_verified`、`aggregator_verified`。
|
||||
- `self_hosted_gateway` 与 `unverified_relay` 只进入观察区或风险提醒区,不进入主推荐。
|
||||
|
||||
### 3.4.1 来源证据露出规则
|
||||
|
||||
来源可信度标签不能只做视觉标识,必须给用户可追溯的证据入口。
|
||||
|
||||
对以下卡片强制增加来源证据位:
|
||||
|
||||
- 头条事件卡
|
||||
- 推荐模型卡
|
||||
- 免费模型卡
|
||||
- 风险提示卡
|
||||
|
||||
每张卡片至少要能露出:
|
||||
|
||||
- 主来源名称
|
||||
- 更新时间
|
||||
- 来源链接入口
|
||||
|
||||
可选露出:
|
||||
|
||||
- 判定说明
|
||||
- 采集时间
|
||||
- 次级证据来源
|
||||
|
||||
交互建议:
|
||||
|
||||
- 移动端默认展示“来源名称 + 更新时间”,点击展开二级抽屉或详情层查看完整来源信息。
|
||||
- Web 端可使用 tooltip、侧边详情层或内联折叠区展示判定说明。
|
||||
|
||||
目标:
|
||||
|
||||
- 用户看到“官方来源”“聚合来源”“来源待验证”时,能够进一步确认其依据,而不是只看到一个不可解释的标签。
|
||||
|
||||
## 4. 视觉方向
|
||||
|
||||
### 4.1 总体方向
|
||||
|
||||
本项目 UI 气质定义为:
|
||||
|
||||
**高端移动资讯产品 + 专业决策工具**
|
||||
|
||||
不是传统 BI Dashboard,也不是普通资讯流,而是“AI 情报晨报 + 交易决策界面”的混合体。
|
||||
|
||||
### 4.2 风格关键词
|
||||
|
||||
- 高端资讯感
|
||||
- 科技商业感
|
||||
- 情报头条感
|
||||
- 专业但不冰冷
|
||||
- 可快速扫读
|
||||
- 强标签系统
|
||||
|
||||
### 4.3 视觉原则
|
||||
|
||||
- 浅底,不以深色为默认主题。
|
||||
- 重点靠信息层级抓眼球,不靠大图堆叠。
|
||||
- 大字号结论、大标签、少字说明。
|
||||
- 颜色必须承担信息语义,而不是纯装饰。
|
||||
- 卡片感明确,但避免“廉价卡片流”。
|
||||
|
||||
## 5. 视觉系统规范
|
||||
|
||||
### 5.1 颜色系统
|
||||
|
||||
主色建议:
|
||||
|
||||
- 墨蓝:核心情报、正式发布、品牌主色
|
||||
- 祖母绿:降价、利好、值得试
|
||||
- 琥珀橙:活动、促销、观察项
|
||||
- 朱砂红:来源风险、灰区提醒、负面变化
|
||||
- 雾灰:背景、辅助信息、弱层级
|
||||
|
||||
颜色语义要求:
|
||||
|
||||
- 用户看到颜色即可大致判断事件性质。
|
||||
- 同一类标签在所有模块中颜色必须一致。
|
||||
|
||||
### 5.2 字体系统
|
||||
|
||||
文字不能小,移动端优先规则如下:
|
||||
|
||||
- 一句话结论:22px-26px
|
||||
- 头条标题:18px-20px
|
||||
- 卡片标题:18px
|
||||
- 正文短句:15px-16px
|
||||
- 标签:12px-13px
|
||||
- 主阅读区正文禁止低于 14px
|
||||
|
||||
设计要求:
|
||||
|
||||
- 标题字要有媒体感和辨识度。
|
||||
- 正文字必须优先可读性。
|
||||
- 不使用普通后台式默认字体堆满页面。
|
||||
|
||||
### 5.3 标签系统
|
||||
|
||||
标签是核心视觉语言之一,必须标准化。
|
||||
|
||||
建议标签:
|
||||
|
||||
- 官方发布
|
||||
- 聚合免费
|
||||
- 官方免费
|
||||
- 限时活动
|
||||
- 价格下调
|
||||
- 来源待验证
|
||||
- 适合编码
|
||||
- 适合 Agent
|
||||
- 官方来源
|
||||
- 聚合来源
|
||||
|
||||
约束:
|
||||
|
||||
- 每张卡最多展示 3 个标签。
|
||||
- 标签必须足够粗、足够醒目,但不应压过主标题。
|
||||
|
||||
### 5.4 图标系统
|
||||
|
||||
采用简洁功能型图标,不做插画主导。
|
||||
|
||||
建议图标语义:
|
||||
|
||||
- 火焰:热点
|
||||
- 向下箭头:降价
|
||||
- 礼盒 / 闪电:活动
|
||||
- 盾牌:可信来源
|
||||
- 感叹号:风险提醒
|
||||
|
||||
### 5.5 动效规范
|
||||
|
||||
动效应提升高级感,不制造噪音。
|
||||
|
||||
建议:
|
||||
|
||||
- 首屏卡片分层渐入
|
||||
- 标签轻微浮现
|
||||
- 卡片 hover / tap 有短促反馈
|
||||
- 折叠区展开用轻量过渡
|
||||
|
||||
禁止:
|
||||
|
||||
- 大范围抖动
|
||||
- 夸张发光
|
||||
- 低质漂浮动效
|
||||
- 干扰阅读的连续动画
|
||||
|
||||
## 6. 移动端首页结构
|
||||
|
||||
移动端首页按“先结论,后机会,再证据”组织,共六个区块。
|
||||
|
||||
### 6.1 顶部情报头
|
||||
|
||||
内容:
|
||||
|
||||
- 日期
|
||||
- 数据更新时间
|
||||
- 今日市场状态标签
|
||||
- 一句短副标题
|
||||
|
||||
目标:
|
||||
|
||||
- 让用户第一眼知道这是一份“今日 AI 情报晨报”。
|
||||
|
||||
### 6.2 一句话结论卡
|
||||
|
||||
规则:
|
||||
|
||||
- 只保留 1 条主结论
|
||||
- 控制在 28-40 个中文字符
|
||||
- 最多 2-3 行
|
||||
- 必须在手机首屏内出现
|
||||
|
||||
作用:
|
||||
|
||||
- 让用户一眼知道“今天真正重要的变化是什么”。
|
||||
|
||||
### 6.3 三条行动建议
|
||||
|
||||
每条卡片固定结构:
|
||||
|
||||
- 建议动作
|
||||
- 适用人群
|
||||
- 2-3 个原因标签
|
||||
|
||||
目标:
|
||||
|
||||
- 用户无需先读完整日报,就能获得可执行建议。
|
||||
|
||||
### 6.4 今日头条卡片流
|
||||
|
||||
内容:
|
||||
|
||||
- 新模型发布
|
||||
- 重要价格变化
|
||||
- 活动 / 免费策略变化
|
||||
|
||||
卡片必须包含:
|
||||
|
||||
- 事件标签
|
||||
- 标题
|
||||
- 为什么重要
|
||||
- 来源可信度
|
||||
|
||||
### 6.5 场景推荐区
|
||||
|
||||
按场景分组:
|
||||
|
||||
- 低成本编码
|
||||
- 中文通用
|
||||
- Agent / 工具调用
|
||||
- 视觉 / 多模态
|
||||
|
||||
每组最多展示 3 个候选。
|
||||
|
||||
### 6.6 附录入口
|
||||
|
||||
附录包含:
|
||||
|
||||
- 完整免费模型
|
||||
- 完整价格表
|
||||
- 平台覆盖
|
||||
- 套餐信息
|
||||
|
||||
要求:
|
||||
|
||||
- 默认收起
|
||||
- 不打断首页信息节奏
|
||||
|
||||
## 7. Web 端布局策略
|
||||
|
||||
Web 端不是移动端的放大版,而是“带更多证据的完整版”。
|
||||
|
||||
### 7.1 首页布局
|
||||
|
||||
建议:
|
||||
|
||||
- 左侧:一句话结论 + 行动建议 + 今日头条
|
||||
- 右侧:关键指标 + 风险提示 + 今日市场状态
|
||||
|
||||
### 7.2 内容承载
|
||||
|
||||
- 场景推荐区适合用矩阵布局增强对比。
|
||||
- 附录区允许展开更多表格和来源说明。
|
||||
- 可加入锚点导航,支持快速跳到:
|
||||
- 今日变化
|
||||
- 推荐
|
||||
- 免费来源
|
||||
- 附录
|
||||
|
||||
### 7.3 一致性
|
||||
|
||||
移动端与 Web 端必须共享同一套:
|
||||
|
||||
- 一句话结论
|
||||
- 头条事件
|
||||
- 推荐场景
|
||||
- 来源可信度标签
|
||||
|
||||
## 8. 核心组件定义
|
||||
|
||||
建议优先设计以下组件:
|
||||
|
||||
### 8.1 一句话结论卡
|
||||
|
||||
用途:
|
||||
|
||||
- 承担首屏最大视觉锚点
|
||||
|
||||
要求:
|
||||
|
||||
- 大字号
|
||||
- 极少文字
|
||||
- 强背景对比
|
||||
|
||||
### 8.2 行动建议卡
|
||||
|
||||
用途:
|
||||
|
||||
- 输出“今天该做什么”
|
||||
|
||||
要求:
|
||||
|
||||
- 标题明确
|
||||
- 适用对象明确
|
||||
- 理由用标签表达
|
||||
- 必须保留 1 个“证据短句位”
|
||||
|
||||
### 8.3 头条事件卡
|
||||
|
||||
用途:
|
||||
|
||||
- 承载新发布、降价、活动等高信号事件
|
||||
|
||||
要求:
|
||||
|
||||
- 强标题
|
||||
- 强标签
|
||||
- 强数字
|
||||
|
||||
### 8.4 推荐模型卡
|
||||
|
||||
用途:
|
||||
|
||||
- 承载场景化推荐
|
||||
|
||||
要求:
|
||||
|
||||
- 先显示模型名与用途
|
||||
- 再显示来源与价格
|
||||
- 不再用长表格表达
|
||||
- 必须保留 1 个“关键证据短句位”
|
||||
- 必须保留来源证据入口
|
||||
|
||||
### 8.5 来源可信度标签
|
||||
|
||||
用途:
|
||||
|
||||
- 解决“免费是真的吗”“这个来源可不可信”的核心疑问
|
||||
|
||||
要求:
|
||||
|
||||
- 视觉语义强
|
||||
- 全局复用
|
||||
|
||||
### 8.6 风险提示卡
|
||||
|
||||
用途:
|
||||
|
||||
- 承载灰区来源、待验证来源、活动时效风险
|
||||
|
||||
要求:
|
||||
|
||||
- 在色彩和语义上明显区别于机会卡
|
||||
|
||||
### 8.7 证据短句位规范
|
||||
|
||||
为避免页面只剩“推荐结论”而缺少决策依据,行动建议卡和推荐模型卡都必须保留一个固定的证据短句位。
|
||||
|
||||
证据短句位要求:
|
||||
|
||||
- 只允许 1 行
|
||||
- 长度控制在 10-24 个中文字符
|
||||
- 优先展示“今天为什么值得关注”的理由
|
||||
|
||||
推荐文案模板:
|
||||
|
||||
- `较昨日低 18%`
|
||||
- `官方免费额度已确认`
|
||||
- `首次发布,支持 256K`
|
||||
- `聚合免费,适合尝鲜`
|
||||
- `活动价,截止 05-31`
|
||||
|
||||
设计原则:
|
||||
|
||||
- 证据短句不是说明文,而是高信号决策依据。
|
||||
- 证据短句必须和推荐动作或头条判断形成闭环。
|
||||
|
||||
## 9. 抓眼球规则
|
||||
|
||||
首页必须显著抓眼,但不依赖低质量视觉噪音。
|
||||
|
||||
强制规则:
|
||||
|
||||
- 首屏不能以大表格开头。
|
||||
- 首屏必须有一条大字号结论。
|
||||
- 文字说明尽量短,不允许连续长段。
|
||||
- 卡片正文以短句为主,避免三行以上解释。
|
||||
- 每屏只承担一个阅读任务。
|
||||
|
||||
判断标准:
|
||||
|
||||
- 用户不需要阅读完整页,扫一眼也知道今天的重点。
|
||||
- 用户不会因为文字太密或太小而放弃继续阅读。
|
||||
|
||||
## 10. 交付物清单
|
||||
|
||||
正式设计交付建议包括:
|
||||
|
||||
### 10.1 视觉方向稿
|
||||
|
||||
- 首页氛围图
|
||||
- 结论卡 / 行动建议卡 / 头条卡风格样张
|
||||
- 颜色与字体方向
|
||||
|
||||
### 10.2 信息架构稿
|
||||
|
||||
- 移动端首页草图
|
||||
- Web 首页草图
|
||||
- 模块优先级说明
|
||||
|
||||
### 10.3 核心组件稿
|
||||
|
||||
- 一句话结论卡
|
||||
- 行动建议卡
|
||||
- 头条卡
|
||||
- 推荐卡
|
||||
- 标签体系
|
||||
- 风险提示卡
|
||||
|
||||
### 10.4 高保真页面稿
|
||||
|
||||
至少三套:
|
||||
|
||||
- 移动端首页
|
||||
- Web 端首页
|
||||
- 附录 / 展开态
|
||||
|
||||
必做扩展:
|
||||
|
||||
- 平静日版本
|
||||
- 热点日版本
|
||||
|
||||
说明:
|
||||
|
||||
- 平静日版本用于当天重大变化较少时,首页自动转向“观察重点 + 稳定推荐”结构。
|
||||
- 热点日版本用于新模型、降价、活动较多时,首页强化头条和事件卡密度。
|
||||
- 这两种状态都必须在设计阶段覆盖,避免实现后在“无事发生的日子”退化为旧式榜单页。
|
||||
|
||||
## 11. 页面级线框与高保真说明
|
||||
|
||||
本章作为 V1 实现前的页面级设计基线,直接约束移动端首页、Web 首页、以及“平静日 / 热点日”的状态切换方式。实现阶段不得绕开本章退回到“统计块 + 长表格”的旧结构。
|
||||
|
||||
### 11.1 移动端首页线框
|
||||
|
||||
目标设备:
|
||||
|
||||
- 设计基准宽度:390px
|
||||
- 安全区左右边距:16px
|
||||
- 卡片圆角建议:20px
|
||||
- 卡片间距:12px
|
||||
- 可点击区域最小尺寸:44px x 44px
|
||||
|
||||
信息顺序固定如下:
|
||||
|
||||
1. 顶部情报头
|
||||
2. 一句话结论卡
|
||||
3. 三条行动建议
|
||||
4. 今日头条卡片流
|
||||
5. 场景推荐区
|
||||
6. 附录入口
|
||||
|
||||
线框结构:
|
||||
|
||||
```text
|
||||
+--------------------------------------------------+
|
||||
| 日期 / 更新时间 / 市场状态标签 |
|
||||
| 一句短副标题 |
|
||||
+--------------------------------------------------+
|
||||
| 今日一句话结论 |
|
||||
| 1 行标签:价格战 / 官方发布 / 聚合免费偏多 |
|
||||
+--------------------------------------------------+
|
||||
| 建议卡 1 |
|
||||
| 建议卡 2 |
|
||||
| 建议卡 3 |
|
||||
+--------------------------------------------------+
|
||||
| 今日头条 |
|
||||
| 头条卡 1 |
|
||||
| 头条卡 2 |
|
||||
| 头条卡 3 |
|
||||
+--------------------------------------------------+
|
||||
| 场景推荐 |
|
||||
| 低成本编码 |
|
||||
| 中文通用 |
|
||||
| Agent / 工具调用 |
|
||||
| 视觉 / 多模态 |
|
||||
+--------------------------------------------------+
|
||||
| 附录入口:完整价格 / 完整免费 / 平台覆盖 |
|
||||
+--------------------------------------------------+
|
||||
```
|
||||
|
||||
滚动节奏要求:
|
||||
|
||||
- 首屏 1 到 1.5 屏内,必须完整出现“顶部情报头 + 一句话结论卡 + 至少 1 张行动建议卡”。
|
||||
- 第二屏必须进入“今日头条”或“场景推荐”,不能被长段说明占满。
|
||||
- 附录入口必须在前三次滑动内可见,但默认不展开长表格。
|
||||
|
||||
### 11.2 移动端首页高保真说明
|
||||
|
||||
#### 11.2.1 顶部情报头
|
||||
|
||||
固定字段:
|
||||
|
||||
- 左侧:`05-13 Wed`
|
||||
- 右侧:`08:35 更新`
|
||||
- 下方标签:`价格战活跃`、`新模型日`、`免费策略波动`
|
||||
- 最底一行:一句短副标题,控制在 18 个中文字符内
|
||||
|
||||
视觉要求:
|
||||
|
||||
- 顶部区域不使用纯白平板,采用轻雾灰底 + 细粒度渐变。
|
||||
- 标签采用胶囊形,单个标签宽度不超过一行的 40%。
|
||||
- 日期和更新时间用较小字号,但不得低于 14px。
|
||||
|
||||
#### 11.2.2 一句话结论卡
|
||||
|
||||
内容结构固定:
|
||||
|
||||
- 主结论:1 条,28-40 个中文字符
|
||||
- 辅助标签:最多 2 个
|
||||
- 可选证据短句:1 条,10-18 个中文字符
|
||||
|
||||
视觉层级:
|
||||
|
||||
- 主结论字号:22px-26px,字重明显高于正文
|
||||
- 卡片背景优先使用墨蓝浅化渐变或暖灰底叠加高亮描边
|
||||
- 该卡片必须成为首屏最大视觉锚点
|
||||
|
||||
禁止事项:
|
||||
|
||||
- 禁止在该卡片中堆 2 段以上解释文字
|
||||
- 禁止放 4 个以上标签
|
||||
- 禁止把统计数字作为主标题替代结论句
|
||||
|
||||
#### 11.2.3 行动建议卡
|
||||
|
||||
每张卡固定包含 4 行:
|
||||
|
||||
1. 动作标题,例如 `今天先试它`
|
||||
2. 适用人群,例如 `适合低成本代码生成`
|
||||
3. 标签组,最多 3 个
|
||||
4. 证据短句位,必须存在
|
||||
|
||||
卡片高度建议:
|
||||
|
||||
- 默认高度:112px-128px
|
||||
- 标题最多 1 行
|
||||
- 适用人群最多 1 行
|
||||
- 证据短句最多 1 行
|
||||
|
||||
视觉要求:
|
||||
|
||||
- 三张卡必须形成强弱关系,推荐优先级最高的一张使用更高对比色边框或更厚阴影。
|
||||
- 不允许三张卡完全同权展示,否则用户无法一眼分辨首选动作。
|
||||
|
||||
#### 11.2.4 今日头条卡
|
||||
|
||||
每张头条卡固定包含:
|
||||
|
||||
- 事件标签
|
||||
- 标题
|
||||
- 影响短句
|
||||
- 比较基线或关键数字
|
||||
- 来源可信度标签
|
||||
|
||||
内容长度限制:
|
||||
|
||||
- 标题最多 2 行
|
||||
- 影响短句最多 2 行
|
||||
- 关键数字必须大于正文层级
|
||||
|
||||
视觉要求:
|
||||
|
||||
- `新发布` 优先用墨蓝
|
||||
- `价格下调` 优先用祖母绿
|
||||
- `活动 / 促销` 优先用琥珀橙
|
||||
- `来源风险` 优先用朱砂红
|
||||
|
||||
#### 11.2.5 场景推荐区
|
||||
|
||||
每个场景模块固定包含:
|
||||
|
||||
- 场景标题
|
||||
- 1 个主推荐卡
|
||||
- 2 个次推荐条目
|
||||
- 1 个“查看更多”入口
|
||||
|
||||
展示规则:
|
||||
|
||||
- 主推荐卡允许露出模型名、用途、来源类型、价格摘要、证据短句
|
||||
- 次推荐条目只露出模型名 + 1 个标签 + 1 个价格摘要
|
||||
- 同一模型在一个首页中最多出现 2 次
|
||||
|
||||
#### 11.2.6 附录入口
|
||||
|
||||
移动端首页只允许展示附录入口,不允许直接铺开完整表格。
|
||||
|
||||
入口结构:
|
||||
|
||||
- 标题:`完整数据附录`
|
||||
- 三个快捷入口:`完整价格`、`完整免费`、`平台覆盖`
|
||||
- 一条解释短句:`适合深度比价时查看`
|
||||
|
||||
### 11.3 Web 首页线框
|
||||
|
||||
目标设备:
|
||||
|
||||
- 设计基准宽度:1440px
|
||||
- 主内容最大宽度:1280px
|
||||
- 栅格:12 列
|
||||
- 页面左右留白:48px-64px
|
||||
|
||||
布局固定如下:
|
||||
|
||||
- 左 7 列:一句话结论卡、行动建议、今日头条、场景推荐
|
||||
- 右 5 列:关键指标、市场状态、风险提醒、来源说明入口
|
||||
|
||||
线框结构:
|
||||
|
||||
```text
|
||||
+-----------------------------+---------------------------+
|
||||
| 顶部情报头 | 今日市场状态 / 风险摘要 |
|
||||
+-----------------------------+---------------------------+
|
||||
| 一句话结论卡 | 关键指标卡组 |
|
||||
+-----------------------------+---------------------------+
|
||||
| 三条行动建议 | 来源可信度说明 |
|
||||
+-----------------------------+---------------------------+
|
||||
| 今日头条卡组 | 风险提示卡 |
|
||||
+-----------------------------+---------------------------+
|
||||
| 场景推荐矩阵 | 锚点导航 / 附录入口 |
|
||||
+---------------------------------------------------------+
|
||||
| 附录区:完整价格 / 完整免费 / 平台覆盖 / 来源证据 |
|
||||
+---------------------------------------------------------+
|
||||
```
|
||||
|
||||
Web 端目标不是“更花”,而是“更方便横向比较”。因此:
|
||||
|
||||
- 左列承担叙事和推荐。
|
||||
- 右列承担解释和证据。
|
||||
- 附录区承担完整查询,不干扰上半屏结论阅读。
|
||||
|
||||
### 11.4 Web 首页高保真说明
|
||||
|
||||
#### 11.4.1 顶部信息带
|
||||
|
||||
顶部允许比移动端多展示 1 组指标,但仍遵循“短句优先”:
|
||||
|
||||
- 日期
|
||||
- 更新时间
|
||||
- 今日变化摘要:新增模型数、降价数、活动数
|
||||
- 市场状态标签
|
||||
|
||||
要求:
|
||||
|
||||
- 顶部信息带总高度控制在 88px-112px
|
||||
- 不允许做成传统 KPI 仪表盘
|
||||
|
||||
#### 11.4.2 关键指标卡组
|
||||
|
||||
右侧指标卡组只保留 4 张:
|
||||
|
||||
- 今日新增模型
|
||||
- 今日重要降价
|
||||
- 官方免费数量
|
||||
- 聚合免费数量
|
||||
|
||||
展示要求:
|
||||
|
||||
- 数字大,解释短
|
||||
- 每张卡必须有“指标含义”短标签
|
||||
- 禁止在 Web 首屏出现 8 张以上统计块
|
||||
|
||||
#### 11.4.3 场景推荐矩阵
|
||||
|
||||
Web 端场景推荐允许做成 2 x 2 矩阵:
|
||||
|
||||
- 左上:低成本编码
|
||||
- 右上:中文通用
|
||||
- 左下:Agent / 工具调用
|
||||
- 右下:视觉 / 多模态
|
||||
|
||||
每格固定结构:
|
||||
|
||||
- 场景标题
|
||||
- 主推荐 1 条
|
||||
- 次推荐 2 条
|
||||
- 来源说明入口
|
||||
|
||||
#### 11.4.4 来源与风险区
|
||||
|
||||
右侧必须有一个常驻区块,用于解释:
|
||||
|
||||
- 今日哪些“免费”是官方免费
|
||||
- 哪些免费来自聚合平台
|
||||
- 哪些来源待验证
|
||||
- 哪些活动存在截止时间
|
||||
|
||||
该区块默认用短句摘要表达,并允许展开查看证据。
|
||||
|
||||
### 11.5 视觉层级固定规则
|
||||
|
||||
为防止实现时退化成“信息都一样重要”,页面层级固定如下:
|
||||
|
||||
- `P0`:一句话结论卡
|
||||
- `P1`:第一张行动建议卡 + 第一条头条
|
||||
- `P2`:其余行动建议卡 + 其余头条卡
|
||||
- `P3`:场景推荐主卡
|
||||
- `P4`:附录入口、解释性文字、完整表格
|
||||
|
||||
实现要求:
|
||||
|
||||
- 首屏同一时刻只能有 1 个 P0。
|
||||
- 每个区块最多存在 1 个强主色焦点。
|
||||
- P4 信息不能在视觉上压过 P1 / P2。
|
||||
|
||||
### 11.6 状态切换规则
|
||||
|
||||
首页必须支持三种状态,并由数据自动驱动:
|
||||
|
||||
#### 11.6.1 常规日
|
||||
|
||||
触发条件:
|
||||
|
||||
- 有 1-2 条重要变化
|
||||
- 或存在 1 条较强头条但整体事件密度不高
|
||||
|
||||
页面策略:
|
||||
|
||||
- 保持标准六区块结构
|
||||
- 头条区展示 2-3 张卡
|
||||
- 场景推荐正常展开
|
||||
|
||||
#### 11.6.2 平静日
|
||||
|
||||
触发条件建议:
|
||||
|
||||
- 重大变化事件少于 2 条
|
||||
- 且无 `official_release`
|
||||
- 且无显著降价或活动
|
||||
|
||||
页面策略:
|
||||
|
||||
- 结论卡改为“观察重点 + 稳定推荐”语气
|
||||
- 行动建议卡优先展示“稳定商用选择”
|
||||
- 头条区减少到 1-2 张,并允许用“今日无重大上新 / 无显著调价”作为信息性卡片
|
||||
- 场景推荐上移,承担更多首页价值
|
||||
|
||||
禁止事项:
|
||||
|
||||
- 禁止用旧榜单、旧大表格填满头条区
|
||||
- 禁止为了“看起来有内容”重复同一模型三次以上
|
||||
|
||||
#### 11.6.3 热点日
|
||||
|
||||
触发条件建议:
|
||||
|
||||
- `official_release` >= 1
|
||||
- 或重大变化事件 >= 3
|
||||
- 或同日存在“新发布 + 降价 + 活动”组合
|
||||
|
||||
页面策略:
|
||||
|
||||
- 顶部情报头增加“热点日”状态标签
|
||||
- 今日头条区允许扩展到 4 张卡
|
||||
- 第一条头条卡可升级为宽版主头条
|
||||
- 场景推荐保留,但默认折叠次推荐条目,避免首屏过长
|
||||
|
||||
### 11.7 页面状态验收要求
|
||||
|
||||
线框和高保真说明必须同时覆盖以下状态:
|
||||
|
||||
- 移动端常规日
|
||||
- 移动端平静日
|
||||
- 移动端热点日
|
||||
- Web 端常规日
|
||||
- Web 端平静日
|
||||
- Web 端热点日
|
||||
|
||||
验收标准:
|
||||
|
||||
- 任一状态下,用户在 30 秒内都能说出“今天值不值得关注”。
|
||||
- 平静日不会退化成榜单堆砌页。
|
||||
- 热点日不会因为信息过多而丢失主结论。
|
||||
|
||||
## 12. 版本路线图
|
||||
|
||||
### V1 可读版
|
||||
|
||||
目标:
|
||||
|
||||
- 让日报从“数据导出页”变成“人能快速看懂的日报”
|
||||
|
||||
包含:
|
||||
|
||||
- 一句话结论
|
||||
- 3 条行动建议
|
||||
- 今日变化摘要
|
||||
- 免费来源类型标签
|
||||
- 附录后置
|
||||
|
||||
验收:
|
||||
|
||||
- 用户 30 秒内说出今天最重要变化
|
||||
- 免费区 100% 带来源类型标签
|
||||
- 首屏不再是大表格
|
||||
- 存在“平静日状态”首页方案,且不使用重复榜单填充头条区
|
||||
|
||||
### V2 情报版
|
||||
|
||||
目标:
|
||||
|
||||
- 让日报具备“每天值得打开”的新闻价值
|
||||
|
||||
包含:
|
||||
|
||||
- 事件流
|
||||
- 今日头条 3 条
|
||||
- 活动 / 发布 / 降价打标
|
||||
- 来源可信度分级
|
||||
|
||||
验收:
|
||||
|
||||
- 80% 日报至少有 1 条真正变化事件
|
||||
- 头条区每条都含事件类型、可信度、影响对象
|
||||
- 主推荐区不出现待验证来源
|
||||
|
||||
### V3 专业版
|
||||
|
||||
目标:
|
||||
|
||||
- 形成可持续的 AI 模型与价格情报产品
|
||||
|
||||
包含:
|
||||
|
||||
- 事件表 / 来源注册表
|
||||
- 风险分层
|
||||
- 趋势入口
|
||||
- 周报 / 专题扩展能力
|
||||
|
||||
验收:
|
||||
|
||||
- 用户可把日报当作日常选型输入
|
||||
- 变化、活动、来源风险都能稳定进入产品
|
||||
|
||||
## 13. 实施建议
|
||||
|
||||
推荐实施顺序:
|
||||
|
||||
1. 先完成移动端首页高保真设计
|
||||
2. 再扩展 Web 端版式
|
||||
3. 抽出组件规范
|
||||
4. 改造 HTML 日报模板
|
||||
5. 改造 Dashboard 日报入口与摘要视图
|
||||
6. 用真实日报数据回填验收
|
||||
|
||||
推荐工程路径:
|
||||
|
||||
- 优先改造 `scripts/generate_daily_report.go` 中的 HTML 模板和内容编排
|
||||
- 复用现有 `/api/v1/reports/latest` 能力,在前端摘要入口中承载新版视觉模块
|
||||
- 后续再将事件与来源标签能力沉淀到独立模块
|
||||
|
||||
## 14. 结论
|
||||
|
||||
本次 UI 重构的目标不是单纯“美化日报”,而是把日报变成一个:
|
||||
|
||||
- 愿意每天打开的高端移动资讯产品
|
||||
- 能快速做出选型判断的专业工具
|
||||
- 不牺牲底层数据库完整性的情报展示层
|
||||
|
||||
后续实现必须始终围绕三条主线:
|
||||
|
||||
- 信息层级清楚
|
||||
- 来源可信度透明
|
||||
- 读者能快速行动
|
||||
296
docs/plans/2026-05-13-daily-report-v1-implementation-plan.md
Normal file
296
docs/plans/2026-05-13-daily-report-v1-implementation-plan.md
Normal file
@@ -0,0 +1,296 @@
|
||||
# Daily Report V1 Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** 将当前“数据库导出式日报”改造成移动端优先、变化驱动、可快速决策的 V1 日报页面,并同步补齐 Dashboard 摘要入口。
|
||||
|
||||
**Architecture:** 保留现有数据库和采集链路不动,把改造集中在 `scripts/generate_daily_report.go` 的报告语义层与 HTML 模板层。先把“结论、行动建议、头条、免费来源标签、场景推荐”抽成可测试的构建函数,再重写 HTML 模板和前端摘要视图,最后用真实日报生成、Go 测试、前端构建联合验收。
|
||||
|
||||
**Tech Stack:** Go 1.22、html/template、PostgreSQL、React、TypeScript、CSS
|
||||
|
||||
---
|
||||
|
||||
### Task 1: 为日报 V1 语义层补测试
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/generate_daily_report_test.go`
|
||||
- Test: `scripts/generate_daily_report_test.go`
|
||||
|
||||
**Step 1: 写失败测试**
|
||||
|
||||
补 3 组测试:
|
||||
- 免费来源标签分组测试:验证 `official_free`、`aggregator_free`、`unknown_free`
|
||||
- V1 首页摘要测试:验证会输出一句话结论、行动建议、头条、附录快捷入口
|
||||
- 平静日状态测试:验证当事件不足时,首页出现“观察重点 + 稳定推荐”文案
|
||||
|
||||
**Step 2: 运行失败测试**
|
||||
|
||||
Run: `go test -tags llm_script ./scripts -run 'TestGenerate|TestBuild'`
|
||||
|
||||
Expected:
|
||||
- 新增测试失败
|
||||
- 失败原因是缺少语义层函数或 HTML 中不存在新版文案
|
||||
|
||||
**Step 3: 最小实现语义函数**
|
||||
|
||||
在 `scripts/generate_daily_report.go` 中新增:
|
||||
- 免费来源分类辅助类型
|
||||
- 首页摘要结构
|
||||
- 头条 / 建议 / 推荐构建函数
|
||||
|
||||
**Step 4: 重新运行测试**
|
||||
|
||||
Run: `go test -tags llm_script ./scripts -run 'TestGenerate|TestBuild'`
|
||||
|
||||
Expected:
|
||||
- 新增测试通过
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/generate_daily_report.go scripts/generate_daily_report_test.go
|
||||
git commit -m "feat(report): add v1 report summary builders"
|
||||
```
|
||||
|
||||
### Task 2: 重构日报数据结构以支撑 V1 页面
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/generate_daily_report.go`
|
||||
- Test: `scripts/generate_daily_report_test.go`
|
||||
|
||||
**Step 1: 写失败测试**
|
||||
|
||||
为以下内容补断言:
|
||||
- 免费模型按来源可信度分组显示
|
||||
- 推荐卡存在证据短句
|
||||
- 头条卡存在变化基线或“首次出现”标签
|
||||
|
||||
**Step 2: 运行失败测试**
|
||||
|
||||
Run: `go test -tags llm_script ./scripts -run 'TestGenerateHTMLV3|TestBuild'`
|
||||
|
||||
Expected:
|
||||
- HTML 内容不包含新版结构字段
|
||||
|
||||
**Step 3: 最小实现**
|
||||
|
||||
在 `ReportV3` 上新增 V1 所需衍生字段,例如:
|
||||
- `HeroSummary`
|
||||
- `ActionItems`
|
||||
- `HeadlineItems`
|
||||
- `SceneSections`
|
||||
- `FreeBreakdown`
|
||||
- `AppendixLinks`
|
||||
- `PageMode`
|
||||
|
||||
并在 `generateReportDataV3` 末尾统一填充。
|
||||
|
||||
**Step 4: 重新运行测试**
|
||||
|
||||
Run: `go test -tags llm_script ./scripts -run 'TestGenerateHTMLV3|TestBuild'`
|
||||
|
||||
Expected:
|
||||
- 数据结构相关测试通过
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/generate_daily_report.go scripts/generate_daily_report_test.go
|
||||
git commit -m "feat(report): enrich daily report v1 view model"
|
||||
```
|
||||
|
||||
### Task 3: 重写 HTML 模板为移动端优先 V1 首页
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/generate_daily_report.go`
|
||||
- Test: `scripts/generate_daily_report_test.go`
|
||||
|
||||
**Step 1: 写失败测试**
|
||||
|
||||
断言 HTML 包含以下结构关键词:
|
||||
- `今日一句话结论`
|
||||
- `三条行动建议`
|
||||
- `今日头条`
|
||||
- `场景推荐`
|
||||
- `完整数据附录`
|
||||
- 免费来源标签:`官方免费`、`聚合免费`、`待确认`
|
||||
|
||||
**Step 2: 运行失败测试**
|
||||
|
||||
Run: `go test -tags llm_script ./scripts -run 'TestGenerateHTMLV3Includes'`
|
||||
|
||||
Expected:
|
||||
- 失败,说明旧模板仍是统计卡 + 表格
|
||||
|
||||
**Step 3: 最小实现**
|
||||
|
||||
重写 `generateHTMLV3`:
|
||||
- 使用移动端优先布局
|
||||
- 加入结论卡、行动建议卡、头条卡、场景推荐、附录入口
|
||||
- 保留必要的完整表格,但下沉到附录区
|
||||
- 按设计文档落地颜色、字号、标签、平静日/热点日状态
|
||||
|
||||
**Step 4: 重新运行测试**
|
||||
|
||||
Run: `go test -tags llm_script ./scripts -run 'TestGenerateHTMLV3Includes'`
|
||||
|
||||
Expected:
|
||||
- 模板结构测试通过
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/generate_daily_report.go scripts/generate_daily_report_test.go
|
||||
git commit -m "feat(report): redesign html daily report for v1"
|
||||
```
|
||||
|
||||
### Task 4: 调整 Markdown 让结构与新版日报一致
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/generate_daily_report.go`
|
||||
- Test: `scripts/generate_daily_report_test.go`
|
||||
|
||||
**Step 1: 写失败测试**
|
||||
|
||||
补充 Markdown 断言:
|
||||
- 顶部出现“今日结论”“今日行动建议”“今日变化”
|
||||
- 免费区出现来源分类摘要
|
||||
|
||||
**Step 2: 运行失败测试**
|
||||
|
||||
Run: `go test -tags llm_script ./scripts -run 'TestGenerateMarkdownV3'`
|
||||
|
||||
Expected:
|
||||
- 旧 Markdown 不包含新版结构
|
||||
|
||||
**Step 3: 最小实现**
|
||||
|
||||
改写 `generateMarkdownV3` 的章节顺序,至少与 HTML 保持:
|
||||
- 结论
|
||||
- 行动建议
|
||||
- 变化摘要
|
||||
- 场景推荐
|
||||
- 附录
|
||||
|
||||
**Step 4: 重新运行测试**
|
||||
|
||||
Run: `go test -tags llm_script ./scripts -run 'TestGenerateMarkdownV3'`
|
||||
|
||||
Expected:
|
||||
- Markdown 测试通过
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/generate_daily_report.go scripts/generate_daily_report_test.go
|
||||
git commit -m "feat(report): align markdown report with v1 structure"
|
||||
```
|
||||
|
||||
### Task 5: 补 Dashboard 摘要卡以对齐新版日报
|
||||
|
||||
**Files:**
|
||||
- Modify: `frontend/src/pages/Dashboard.tsx`
|
||||
- Modify: `frontend/src/App.css`
|
||||
|
||||
**Step 1: 写失败测试或构建前检查**
|
||||
|
||||
由于当前前端未配置页面测试,先以类型检查和构建作为验收门槛,并在实现前明确 UI 目标:
|
||||
- Dashboard 出现一句话摘要
|
||||
- 显示报告日期、状态、HTML / Markdown 入口
|
||||
- 展示“固定路径回退”提示
|
||||
- 视觉上更接近新版日报入口卡
|
||||
|
||||
**Step 2: 最小实现**
|
||||
|
||||
在 `Dashboard.tsx` 中:
|
||||
- 扩展 `LatestReport` 展示字段
|
||||
- 生成更强的信息层级摘要
|
||||
- 调整布局让入口更接近移动端日报卡片语义
|
||||
|
||||
在 `App.css` 中:
|
||||
- 为日报入口补新版卡片层级
|
||||
- 优化移动端字号和按钮布局
|
||||
|
||||
**Step 3: 运行构建**
|
||||
|
||||
Run: `cd frontend && npm run build`
|
||||
|
||||
Expected:
|
||||
- 构建通过
|
||||
|
||||
**Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add frontend/src/pages/Dashboard.tsx frontend/src/App.css
|
||||
git commit -m "feat(frontend): align dashboard report card with v1 report"
|
||||
```
|
||||
|
||||
### Task 6: 真实生成和联调验证
|
||||
|
||||
**Files:**
|
||||
- Modify: `reports/daily/*`(生成产物)
|
||||
- Verify: `scripts/generate_daily_report.go`
|
||||
- Verify: `scripts/verify_phase3.sh`
|
||||
|
||||
**Step 1: 运行脚本测试**
|
||||
|
||||
Run: `go test -tags llm_script ./scripts`
|
||||
|
||||
Expected:
|
||||
- 脚本相关测试通过
|
||||
|
||||
**Step 2: 运行后端测试**
|
||||
|
||||
Run: `go test ./...`
|
||||
|
||||
Expected:
|
||||
- 全部通过
|
||||
|
||||
**Step 3: 运行前端构建**
|
||||
|
||||
Run: `cd frontend && npm run build`
|
||||
|
||||
Expected:
|
||||
- 构建通过
|
||||
|
||||
**Step 4: 真实生成日报**
|
||||
|
||||
Run: `go run -tags llm_script ./scripts/generate_daily_report.go`
|
||||
|
||||
Expected:
|
||||
- 生成新版 md/html
|
||||
- 主产物与归档产物都更新
|
||||
|
||||
**Step 5: 门禁验证**
|
||||
|
||||
Run: `bash scripts/verify_phase3.sh`
|
||||
|
||||
Expected:
|
||||
- `PHASE_RESULT: PASS`
|
||||
|
||||
**Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/generate_daily_report.go scripts/generate_daily_report_test.go frontend/src/pages/Dashboard.tsx frontend/src/App.css reports/daily
|
||||
git commit -m "feat(report): ship daily report v1 experience"
|
||||
```
|
||||
|
||||
### Task 7: 推送到仓库
|
||||
|
||||
**Files:**
|
||||
- Verify: working tree
|
||||
|
||||
**Step 1: 检查状态**
|
||||
|
||||
Run: `git status --short`
|
||||
|
||||
Expected:
|
||||
- 只剩可接受的既有脏文件或已知产物
|
||||
|
||||
**Step 2: 推送**
|
||||
|
||||
Run: `git push`
|
||||
|
||||
Expected:
|
||||
- 推送成功
|
||||
|
||||
143
docs/plans/2026-05-14-daily-report-v2-closeout-plan.md
Normal file
143
docs/plans/2026-05-14-daily-report-v2-closeout-plan.md
Normal file
@@ -0,0 +1,143 @@
|
||||
# Daily Report V2 Closeout Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** 补齐日报 V2 情报版剩余 3 个收口项,让事件流覆盖营销活动、头条显式展示影响对象,并新增可重复执行的历史日报事件覆盖率验收脚本。
|
||||
|
||||
**Architecture:** 继续复用 `scripts/generate_daily_report.go` 作为唯一日报语义层入口,不改数据库结构。`promo_campaign` 先以最小本地活动源接入,再复用现有 `ModelEvent -> HeadlineItem` 链路;头条影响对象通过扩展 `HeadlineItem` 数据结构和模板完成;覆盖率验收通过单独脚本直接查询数据库并调用现有历史重建入口完成。
|
||||
|
||||
**Tech Stack:** Go 1.22、html/template、PostgreSQL、Bash
|
||||
|
||||
---
|
||||
|
||||
### Task 1: 为 V2 收口项补失败测试
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/generate_daily_report_test.go`
|
||||
|
||||
**Step 1: 写失败测试**
|
||||
|
||||
补 3 组测试:
|
||||
- `promo_campaign` 头条测试:验证会生成 `活动/营销` 类型头条,并保留来源、证据、基线
|
||||
- 头条影响对象测试:验证 `HeadlineItem` 有 `Audience`,HTML/Markdown 都会渲染
|
||||
- 覆盖率汇总测试:如果批量验收逻辑抽成辅助函数,则为通过率计算补单测
|
||||
|
||||
**Step 2: 运行失败测试**
|
||||
|
||||
Run: `go test -tags llm_script scripts/generate_daily_report.go scripts/generate_daily_report_test.go`
|
||||
|
||||
Expected:
|
||||
- 新增断言失败
|
||||
- 失败原因是缺少 `promo_campaign` / `Audience` 相关实现
|
||||
|
||||
**Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/generate_daily_report_test.go
|
||||
git commit -m "test(report): cover v2 closeout requirements"
|
||||
```
|
||||
|
||||
### Task 2: 接入 promo_campaign 事件流并渲染头条影响对象
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/generate_daily_report.go`
|
||||
- Modify: `scripts/generate_daily_report_test.go`
|
||||
|
||||
**Step 1: 最小实现 promo 活动源**
|
||||
|
||||
在 `scripts/generate_daily_report.go` 中新增本地活动源定义,至少包含:
|
||||
- 活动日期
|
||||
- 模型名或匹配键
|
||||
- 标题 / 摘要
|
||||
- 主来源
|
||||
- 证据说明
|
||||
- 影响对象
|
||||
- 优先级
|
||||
|
||||
首批只接入最小样本,够覆盖 V2 能力:
|
||||
- `DeepSeek` 价格活动或发布期活动
|
||||
- 允许未来继续追加
|
||||
|
||||
**Step 2: 把活动源并入事件流**
|
||||
|
||||
在 `loadModelEvents` 中新增:
|
||||
- `loadPromoCampaignEvents(date string)` 或等价辅助函数
|
||||
- 统一走 `ModelEvent`
|
||||
- `EventType = "promo_campaign"`
|
||||
|
||||
**Step 3: 扩 HeadlineItem 影响对象**
|
||||
|
||||
新增:
|
||||
- `HeadlineItem.Audience`
|
||||
- `ModelEvent.Audience`
|
||||
|
||||
并在:
|
||||
- `headlineItemFromModelEvent`
|
||||
- Markdown 头条输出
|
||||
- HTML 头条卡输出
|
||||
|
||||
中渲染“影响对象”。
|
||||
|
||||
**Step 4: 重新运行测试**
|
||||
|
||||
Run: `go test -tags llm_script scripts/generate_daily_report.go scripts/generate_daily_report_test.go`
|
||||
|
||||
Expected:
|
||||
- `promo_campaign`、`Audience` 相关测试通过
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/generate_daily_report.go scripts/generate_daily_report_test.go
|
||||
git commit -m "feat(report): close v2 event and audience gaps"
|
||||
```
|
||||
|
||||
### Task 3: 新增 V2 历史日报事件覆盖率验收脚本
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/verify_v2_event_coverage.sh`
|
||||
- Create or Modify: `scripts/report_event_coverage.go`
|
||||
- Modify: `scripts/generate_daily_report_test.go` 或新增脚本测试文件(仅在有价值时)
|
||||
|
||||
**Step 1: 实现覆盖率统计脚本**
|
||||
|
||||
目标:
|
||||
- 输入日期范围
|
||||
- 统计每个日期是否命中真正变化事件
|
||||
- 统计命中率
|
||||
- 失败阈值:小于 80%
|
||||
|
||||
“真正变化事件”至少包括:
|
||||
- `official_release`
|
||||
- `promo_campaign`
|
||||
- `new_model`
|
||||
- `price_cut`
|
||||
- `price_increase`
|
||||
|
||||
**Step 2: 使用历史日报入口做一次真实验收**
|
||||
|
||||
Run:
|
||||
- `bash scripts/verify_v2_event_coverage.sh 2024-06-01 2026-05-14`
|
||||
|
||||
Expected:
|
||||
- 输出总天数、命中天数、覆盖率
|
||||
- 覆盖率满足或明确暴露当前缺口
|
||||
|
||||
**Step 3: 运行全量验证**
|
||||
|
||||
Run:
|
||||
- `go test ./...`
|
||||
- `go run -tags llm_script scripts/generate_daily_report.go --date=2025-08-07`
|
||||
- `bash scripts/verify_v2_event_coverage.sh 2024-06-01 2026-05-14`
|
||||
|
||||
Expected:
|
||||
- Go 测试通过
|
||||
- 历史日报仍可生成
|
||||
- 覆盖率脚本输出稳定
|
||||
|
||||
**Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/verify_v2_event_coverage.sh scripts/report_event_coverage.go scripts/generate_daily_report.go scripts/generate_daily_report_test.go
|
||||
git commit -m "feat(report): add v2 event coverage verification"
|
||||
```
|
||||
197
docs/plans/2026-05-14-runtime-trust-gap-remediation-plan.md
Normal file
197
docs/plans/2026-05-14-runtime-trust-gap-remediation-plan.md
Normal file
@@ -0,0 +1,197 @@
|
||||
# Runtime Trust Gap Remediation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
|
||||
> **状态更新(2026-05-14 16:23 CST)**:三阶段已按顺序完成并落到仓库;对应提交为 `a8999ab`。
|
||||
> **最新验证**:`go test ./...`、`npm run build`(`frontend/`)、`bash scripts/verify_phase3.sh`、`bash scripts/verify_phase5.sh` 均已通过。
|
||||
|
||||
**Goal:** 系统性修复日报与采集链路中影响真实性和长期可信度的 3 个缺口,确保“每日定时产出”的结果来自真实采集、可审计运行、并覆盖多源数据链路。
|
||||
|
||||
**Architecture:** 不推翻现有 Phase 1/Phase 2 设计,只在运行语义和审计层补强。将“采集是否真实成功”“这次运行是否为正式日常产出/历史重建”“多源数据是否进入定时链路”拆成独立状态,并让 `run_daily.sh`、日报生成器、验证脚本、数据库记录统一使用同一套运行语义。优先修复最容易掩盖真实失败的宽松成功判定,再修复审计分流,最后把多源采集纳入自动调度。
|
||||
|
||||
**Tech Stack:** Bash、Go 1.22、PostgreSQL、cron、html/template
|
||||
|
||||
## 实施结果摘要
|
||||
|
||||
- 阶段 1:`fetch_openrouter.go` 已支持严格真实模式,正式调度不再把 mock、仅写 JSON 或旧数据误判为成功。
|
||||
- 阶段 2:日报写入已统一携带 `run_kind`、`trigger_source`、`is_official_daily`,正式日报与历史重建已分流。
|
||||
- 阶段 3:`fetch_multi_source.go` 已纳入每日调度链,并把 `selected_source_keys` / `failed_source_keys` 写入运行审计摘要。
|
||||
- Phase 5 基线文档已补齐 `.github/workflows/ci.yml`,Phase 5 门禁不再卡在 CI 文件缺失。
|
||||
|
||||
---
|
||||
|
||||
### Task 1: 收紧“采集成功”判定,避免 mock / 写库失败被伪装成成功
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/fetch_openrouter.go`
|
||||
- Modify: `scripts/run_daily.sh`
|
||||
- Modify: `scripts/run_real_pipeline.sh`
|
||||
- Modify: `scripts/verify_phase3.sh`
|
||||
- Test: `scripts/fetch_openrouter_test.go`
|
||||
- Test: `scripts/run_daily` 对应 shell 验证(可先用现有 verify 脚本)
|
||||
|
||||
**Step 1: 写失败测试**
|
||||
|
||||
补 3 个失败场景:
|
||||
- 没有 `OPENROUTER_API_KEY` 时,调度链不应被当作真实采集成功
|
||||
- `summarizeDB` 写库失败时,`fetch_openrouter` 在“真实模式”下应返回非 0
|
||||
- `run_daily.sh` 不能仅凭“数据库里已有旧数据”就通过质量检查
|
||||
|
||||
**Step 2: 跑测试确认当前行为过宽**
|
||||
|
||||
Run:
|
||||
- `go test -tags llm_script scripts/fetch_openrouter.go scripts/fetch_openrouter_test.go`
|
||||
- `bash scripts/verify_phase3.sh`
|
||||
|
||||
Expected:
|
||||
- 能看到 mock / 降级 / 旧数据掩盖真实失败的风险暴露出来
|
||||
|
||||
**Step 3: 最小实现**
|
||||
|
||||
建议分两层收紧:
|
||||
- `fetch_openrouter.go` 增加严格模式或显式运行模式,真实调度默认要求数据库写入成功,否则退出非 0
|
||||
- `run_daily.sh` 在质量检查中引入“本次运行必须产生当天的写入痕迹”而不是只看历史总量
|
||||
- `run_real_pipeline.sh` 明确只把“真实采集 + 真实写库 + 真实日报生成”视为成功
|
||||
|
||||
**Step 4: 重新运行验证**
|
||||
|
||||
Run:
|
||||
- `bash scripts/run_daily.sh`
|
||||
- `bash scripts/run_real_pipeline.sh`
|
||||
- `bash scripts/verify_phase3.sh`
|
||||
|
||||
Expected:
|
||||
- 真实失败会真正失败
|
||||
- mock / 仅写 JSON / 旧数据不会再伪装成已完成
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/fetch_openrouter.go scripts/run_daily.sh scripts/run_real_pipeline.sh scripts/verify_phase3.sh scripts/fetch_openrouter_test.go
|
||||
git commit -m "fix(runtime): harden daily ingestion success checks"
|
||||
```
|
||||
|
||||
### Task 2: 将正式日报与历史重建分流到不同运行语义,修复审计混写
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/generate_daily_report.go`
|
||||
- Modify: `scripts/rebuild_historical_report.sh`
|
||||
- Modify: `scripts/report_utils.sh`
|
||||
- Modify: `scripts/run_daily.sh`
|
||||
- Modify: `scripts/run_real_pipeline.sh`
|
||||
- Modify: `scripts/verify_phase3.sh`
|
||||
- Test: `scripts/generate_daily_report_test.go`
|
||||
|
||||
**Step 1: 写失败测试**
|
||||
|
||||
补测试验证:
|
||||
- 正式日常产出与历史重建会写入不同的运行类型
|
||||
- 历史重建不应冒充“每日定时产出”
|
||||
- `fetchLatestReport` 与前端最新日报读取仍然只面向正式产出口径
|
||||
|
||||
**Step 2: 跑测试确认当前混写**
|
||||
|
||||
Run:
|
||||
- `go test -tags llm_script scripts/generate_daily_report.go scripts/generate_daily_report_test.go`
|
||||
|
||||
Expected:
|
||||
- 当前 `daily_report` / `report_runs` 的运行语义仍不区分正式与重建
|
||||
|
||||
**Step 3: 最小实现**
|
||||
|
||||
建议新增并统一以下语义字段:
|
||||
- `run_kind`: `scheduled` / `historical_rebuild` / `manual`
|
||||
- `trigger_source`: `cron` / `cli` / `rebuild_script`
|
||||
- `is_official_daily`: 是否属于当天定时正式产出
|
||||
|
||||
落点建议:
|
||||
- `generate_daily_report.go` 的数据库写入携带运行类型
|
||||
- `rebuild_historical_report.sh` 强制标记历史重建语义
|
||||
- 前端和 API 默认只读取正式产出作为“最新日报”
|
||||
|
||||
**Step 4: 重新运行验证**
|
||||
|
||||
Run:
|
||||
- `go test ./...`
|
||||
- `bash scripts/rebuild_historical_report.sh 2025-08-07`
|
||||
- `bash scripts/run_daily.sh`
|
||||
|
||||
Expected:
|
||||
- 历史重建和日常产出可以共存,但不会再在审计层混为一类
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/generate_daily_report.go scripts/rebuild_historical_report.sh scripts/report_utils.sh scripts/run_daily.sh scripts/run_real_pipeline.sh scripts/verify_phase3.sh scripts/generate_daily_report_test.go
|
||||
git commit -m "feat(audit): separate scheduled and rebuild report runs"
|
||||
```
|
||||
|
||||
### Task 3: 把多源数据纳入同一条每日自动调度链
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/run_daily.sh`
|
||||
- Modify: `scripts/run_real_pipeline.sh`
|
||||
- Modify: `scripts/fetch_multi_source.go`
|
||||
- Create or Modify: `scripts/fetch_multi_source_test.go`
|
||||
- Modify: `scripts/verify_phase3.sh`
|
||||
- Modify: `scripts/verify_phase5.sh`
|
||||
- 视需要修改:`scripts/import_phase2_data.go`、`scripts/import_zhipu_data.go`、`scripts/import_bytedance_data.go`
|
||||
|
||||
**Step 1: 写失败测试**
|
||||
|
||||
补测试验证:
|
||||
- 调度链能明确知道哪些来源参与了当日同步
|
||||
- 至少 OpenRouter、国内厂商、聚合平台的每日同步在验证层可被看见
|
||||
|
||||
**Step 2: 设计最小调度编排**
|
||||
|
||||
建议把每日调度拆成可枚举阶段:
|
||||
- `openrouter`
|
||||
- `multi_source`
|
||||
- `official_imports`
|
||||
- `daily_report`
|
||||
|
||||
并定义每个阶段的失败策略:
|
||||
- 任一必需来源失败时,日报应标记为降级/失败,不应伪装成完全成功
|
||||
- 允许某些官方导入在单源失败时继续,但必须在运行记录中留下来源级失败痕迹
|
||||
|
||||
**Step 3: 最小实现**
|
||||
|
||||
优先级建议:
|
||||
- 先把 `fetch_multi_source.go` 接入每日调度
|
||||
- 再把已有官方导入脚本接入可选的日常补充同步阶段
|
||||
- 最后统一审计输出,让 `report_runs` 能显示本次触发的来源集合和失败来源集合
|
||||
|
||||
**Step 4: 重新运行验证**
|
||||
|
||||
Run:
|
||||
- `go test -tags llm_script scripts/fetch_multi_source.go scripts/fetch_multi_source_test.go`
|
||||
- `bash scripts/run_daily.sh`
|
||||
- `bash scripts/verify_phase3.sh`
|
||||
- `bash scripts/verify_phase5.sh`
|
||||
|
||||
Expected:
|
||||
- 每日调度不再只证明 OpenRouter 独立刷新
|
||||
- 多源同步在调度和验收层都能被识别
|
||||
|
||||
**Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/run_daily.sh scripts/run_real_pipeline.sh scripts/fetch_multi_source.go scripts/fetch_multi_source_test.go scripts/verify_phase3.sh scripts/verify_phase5.sh
|
||||
git commit -m "feat(runtime): fold multi-source sync into daily pipeline"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 执行顺序建议
|
||||
|
||||
1. 先做 **Task 1**,因为这是最容易把“假成功”伪装成“真成功”的问题,风险最高。
|
||||
2. 再做 **Task 2**,把正式日报与历史重建的审计边界切开。
|
||||
3. 最后做 **Task 3**,把多源同步真正纳入每日调度链。
|
||||
|
||||
### 验收顺序建议
|
||||
|
||||
1. `bash scripts/run_daily.sh`
|
||||
2. `bash scripts/rebuild_historical_report.sh <date>`
|
||||
3. `bash scripts/verify_phase3.sh`
|
||||
4. `bash scripts/verify_phase5.sh`
|
||||
5. `go test ./...`
|
||||
335
docs/plans/2026-05-22-cucloud-pricing-importer-plan.md
Normal file
335
docs/plans/2026-05-22-cucloud-pricing-importer-plan.md
Normal file
@@ -0,0 +1,335 @@
|
||||
# 联通云细颗粒度 Pricing Importer 设计计划
|
||||
|
||||
> For Hermes: Use subagent-driven-development skill to implement this plan task-by-task.
|
||||
|
||||
**Goal:** 在不伪造价格事实的前提下,把联通云从目录级 `import_cucloud_catalog.go` 升级为可验证的细颗粒度 pricing importer。
|
||||
|
||||
**Architecture:** 采用“两层事实源”设计:第一层抓取帮助中心公开文档中已明确结构化披露的 Token Plan 模型价格与区域支持信息并真实落库;第二层对 AISP 按量计费公开文档仅做“计费模式已验证、具体 per-model payg 单价未公开”的 blocker 标注,不把未公开的销售价伪造成 `region_pricing`。实现上优先复用 `official_pricing_import_common.go`,新增 `import_cucloud_pricing.go`,并保留 `import_cucloud_catalog.go` 作为目录入口校验。
|
||||
|
||||
**Tech Stack:** Go llm_script importer、联通云帮助中心 SSR HTML、正则/HTML 表格解析、现有 `officialPricingRecord` / `catalogVerificationRecord`。
|
||||
|
||||
---
|
||||
|
||||
## 已验证事实
|
||||
|
||||
1. 目录入口仍有效:
|
||||
- `https://www.cucloud.cn/act/CloudAI.html`
|
||||
- 已由现有 `import_cucloud_catalog.go` 校验 AICP / AI 应用开发平台存在。
|
||||
|
||||
2. AISP 帮助中心公开页面可直接 `GET`,无需登录,且页面源码内内嵌完整文档内容:
|
||||
- `https://support.cucloud.cn/document/127/591/2357.html?id=2357&folderid=2973`(购买计费)
|
||||
- `https://support.cucloud.cn/document/127/591/2357.html?id=2357&folderid=3237`(Coding Plan)
|
||||
- `https://support.cucloud.cn/document/127/591/2357.html?id=2357&folderid=3236`(Token Plan)
|
||||
|
||||
3. 已从公开源码中确认的结构化价格事实:
|
||||
- Token Plan 个人版:
|
||||
- Lite 15元/月,600万 tokens
|
||||
- Pro 30元/月,1200万 tokens
|
||||
- Max 45元/月,1800万 tokens
|
||||
- Token Plan 团队版:
|
||||
- Lite 198元/月,25,000 credits
|
||||
- Pro 698元/月,100,000 credits
|
||||
- Max 1398元/月,250,000 credits
|
||||
- 团队版 credits 对三种模型公开披露了折算综合单价:
|
||||
- `DeepSeek-V4-Pro`:9.30 元/百万tokens
|
||||
- `DeepSeek-V4-Flash`:0.70 元/百万tokens
|
||||
- `MiniMax-M2.5`:1.10 元/百万tokens
|
||||
|
||||
4. 已从公开源码中确认的结构化区域支持事实:
|
||||
- 区域列:`呼和浩特二区 / 上海二十二区 / 武汉四区 / 济南五区 / 贵阳基地二区`
|
||||
- 可从 `Token Plan概述` 或 `AI服务平台API介绍` 表格中解析模型-区域支持矩阵。
|
||||
|
||||
5. 已确认但不能伪造的边界:
|
||||
- AISP `购买计费/计费项及计费方式`、`按量计费模式` 文档明确说明“按量计费”为官方公开模式,单位为 `元/千 Tokens`,且按所选模型销售价实时累加。
|
||||
- 但当前公开帮助中心页面未披露具体每个模型的按量销售价表。
|
||||
- 因此现阶段不能把 AISP payg 的 per-model 单价写入 `region_pricing`。
|
||||
|
||||
## 设计决策
|
||||
|
||||
### 决策 A:拆成“可落库价格”与“已验证 blocker”两部分
|
||||
|
||||
1. 可直接落库到 `region_pricing` 的只有:
|
||||
- Token Plan 团队版中公开给出的三个模型综合单价。
|
||||
2. 只能记录 blocker、不能写价格的部分:
|
||||
- AISP 按量计费 per-model 销售价。
|
||||
|
||||
理由:
|
||||
- 公开文档对 payg 的计费机制有描述,但没有模型价格表。
|
||||
- 用户明确要求“找不到就标 blocker,不伪造 importer”。
|
||||
- Token Plan 团队版三模型综合单价属于公开结构化价格,足够支撑一个“真实 importer v1”。
|
||||
|
||||
### 决策 B:新增 pricing importer,不覆盖 catalog importer
|
||||
|
||||
保留:
|
||||
- `scripts/import_cucloud_catalog.go`:继续负责 `cucloud-aicp-platform` / `cucloud-ai-app-platform` 目录存在性校验。
|
||||
|
||||
新增:
|
||||
- `scripts/import_cucloud_pricing.go`:负责 AISP Token Plan 公开价格与模型区域支持的结构化导入。
|
||||
|
||||
理由:
|
||||
- catalog importer 与 pricing importer 的事实层级不同。
|
||||
- 后续若官方公开 payg 模型价表,可在 `import_cucloud_pricing.go` 内扩展,不影响目录校验链路。
|
||||
|
||||
### 决策 C:v1 只导入三个模型,价格视为 blended price
|
||||
|
||||
v1 导入模型:
|
||||
- `DeepSeek-V4-Pro`
|
||||
- `DeepSeek-V4-Flash`
|
||||
- `MiniMax-M2.5`
|
||||
|
||||
价格写法:
|
||||
- 因公开文档给的是“综合单价 X 元/百万tokens”,不是 input/output 分拆价;
|
||||
- v1 写入 `officialPricingRecord` 时采用:
|
||||
- `InputPrice = blendedPrice`
|
||||
- `OutputPrice = blendedPrice`
|
||||
- 同时在 `SourceURL` / `notes` / 文档中明确这是 `Token Plan blended price`,不是 AISP payg 的 input/output 拆分价。
|
||||
|
||||
风险:
|
||||
- 这不适合作为严格意义上的 OpenAI-style in/out token 定价比较。
|
||||
- 但比继续停留在目录级“无细颗粒度价格”更真实,且不会伪造不可得的 input/output 拆分。
|
||||
|
||||
### 决策 D:区域粒度以“支持矩阵交集”写入
|
||||
|
||||
建议落库策略:
|
||||
- 仅对公开支持该模型的区域写入 region_pricing 记录。
|
||||
- 例如:
|
||||
- `DeepSeek-V4-Pro` -> `贵阳基地二区`
|
||||
- `DeepSeek-V4-Flash` -> `贵阳基地二区`(团队版表明确)以及个人版支持区域若文档已写明,可谨慎扩展,但 v1 优先用矩阵表而不是 prose。
|
||||
- `MiniMax-M2.5` -> 从矩阵表取支持区域。
|
||||
|
||||
理由:
|
||||
- 同一模型在联通云并非全区域可用。
|
||||
- 使用支持矩阵可避免写出不存在的区域价格。
|
||||
|
||||
## 文件设计
|
||||
|
||||
### 1. 新增 importer
|
||||
|
||||
**Create:** `scripts/import_cucloud_pricing.go`
|
||||
|
||||
职责:
|
||||
1. 获取公开帮助中心页面 HTML
|
||||
2. 修复页面源码中的 UTF-8 / Latin1 混杂问题
|
||||
3. 从页面源码中定位目标文档 `content` 块:
|
||||
- `Token Plan概述`
|
||||
- `AI服务平台API介绍` 或 `各云区域模型支持情况`
|
||||
- `计费项及计费方式`
|
||||
- `按量计费模式`
|
||||
4. 解析:
|
||||
- Token Plan 团队版三模型综合单价表
|
||||
- 模型-区域支持矩阵表
|
||||
5. 生成 `officialPricingRecord`
|
||||
6. dry-run 输出:
|
||||
- 记录数
|
||||
- 模型数
|
||||
- 区域数
|
||||
- 是否检测到 `payg_mode_confirmed=true`
|
||||
- `payg_price_table_public=false`
|
||||
|
||||
建议 CLI 参数:
|
||||
- `-url`:默认购买计费页或 Token Plan 页
|
||||
- `-fixture`:本地样例 HTML
|
||||
- `-dry-run`
|
||||
- `-timeout`
|
||||
|
||||
### 2. 新增测试
|
||||
|
||||
**Create:** `scripts/import_cucloud_pricing_test.go`
|
||||
|
||||
测试覆盖:
|
||||
1. 能从 fixture 中解析三模型 blended 价格
|
||||
2. 能解析区域支持矩阵
|
||||
3. 仅为支持区域生成记录
|
||||
4. dry-run 摘要包含:
|
||||
- `source=cucloud-pricing-import`
|
||||
- `models=3`
|
||||
- `payg_mode_confirmed=true`
|
||||
- `payg_price_table_public=false`
|
||||
5. 若 fixture 缺少三模型价格表,测试应 fail
|
||||
|
||||
### 3. 新增 fixture
|
||||
|
||||
**Create:** `scripts/testdata/cucloud_pricing_sample.html`
|
||||
|
||||
内容最少应覆盖:
|
||||
- Token Plan 团队版价格表
|
||||
- 三模型 `综合单价X元/百万tokens`
|
||||
- 区域支持矩阵表
|
||||
- `按量计费` 文本(用于 blocker 语义断言)
|
||||
|
||||
### 4. runtime 接入
|
||||
|
||||
**Modify:**
|
||||
- `scripts/run_intel_pipeline.sh`
|
||||
- `scripts/run_real_pipeline.sh`
|
||||
- `scripts/run_daily.sh`
|
||||
- `scripts/verify_importer_smoke.sh`
|
||||
- `scripts/importer_smoke_gate_test.sh`
|
||||
- `scripts/pipeline_runtime_alignment_test.sh`
|
||||
|
||||
接入方式:
|
||||
- 保留 `cucloud_catalog`
|
||||
- 新增 `cucloud_pricing`
|
||||
- 失败消息区分:
|
||||
- 目录失败:`联通云目录校验失败`
|
||||
- 价格失败:`联通云 Token Plan 价格导入失败`
|
||||
|
||||
### 5. seed / docs 同步
|
||||
|
||||
**Modify:**
|
||||
- `seeds/plan_catalog_inventory_seed_cn_relays_top20plus.json`
|
||||
- `docs/PLAN_CATALOG_COVERAGE_MATRIX.md`
|
||||
- `docs/NEXT_IMPORTER_RUNTIME_PRIORITY.md`
|
||||
- `docs/PLAN_CATALOG_INVENTORY.md`
|
||||
- `scripts/import_plan_catalog_test.go`
|
||||
|
||||
同步原则:
|
||||
- `cucloud-aicp-platform` / `cucloud-ai-app-platform` 仍指向 `import_cucloud_catalog.go`
|
||||
- 如新增联通云价格型 catalogCode,则新建 seed 项;否则仅在 docs 中注明:
|
||||
- `目录入口已导入`
|
||||
- `Token Plan 三模型 blended price 已导入`
|
||||
- `AISP payg per-model price table 仍未公开`
|
||||
|
||||
## 推荐实现顺序
|
||||
|
||||
### Task 1: 固化 discovery 结果到 fixture 与计划文档
|
||||
|
||||
**Objective:** 把已验证的公开证据固化为可重复测试输入。
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/testdata/cucloud_pricing_sample.html`
|
||||
- Create: `docs/plans/2026-05-22-cucloud-pricing-importer-plan.md`
|
||||
|
||||
**Step 1: 写 fixture**
|
||||
- 从公开页面中裁剪最小必要 HTML 片段:
|
||||
- Token Plan 三模型价格表
|
||||
- 区域支持矩阵表
|
||||
- 按量计费说明段落
|
||||
|
||||
**Step 2: 验证 fixture 可读**
|
||||
Run:
|
||||
- `python3 - <<'PY' ...` 或 importer 单测读取 fixture
|
||||
Expected:
|
||||
- 能定位三张关键表 / 段落
|
||||
|
||||
### Task 2: 先写失败测试
|
||||
|
||||
**Objective:** 先锁定 importer 的真实合同。
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/import_cucloud_pricing_test.go`
|
||||
|
||||
**Step 1: 写 failing tests**
|
||||
至少包括:
|
||||
- `TestParseCUCloudPricingBuildsBlendedRecords`
|
||||
- `TestParseCUCloudPricingBuildsRegionMatrix`
|
||||
- `TestRunCUCloudPricingImportDryRunPrintsSummary`
|
||||
|
||||
**Step 2: 运行测试确认失败**
|
||||
Run:
|
||||
- `go test -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_cucloud_pricing.go ./scripts/import_cucloud_pricing_test.go`
|
||||
Expected:
|
||||
- FAIL,因为 importer 尚不存在或解析逻辑未实现
|
||||
|
||||
### Task 3: 实现最小 importer
|
||||
|
||||
**Objective:** 只实现三模型 blended price + 区域支持矩阵。
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/import_cucloud_pricing.go`
|
||||
|
||||
**实现要点:**
|
||||
1. 获取 HTML
|
||||
2. `latin1 -> utf8` 修正
|
||||
3. 通过最近的 `"content":"...","createBy"` 边界提取目标内容块,而不是依赖简单 title-first regex
|
||||
4. 表格解析:
|
||||
- Table A:团队版三模型综合单价
|
||||
- Table B:模型区域支持矩阵
|
||||
5. 产出 `officialPricingRecord`
|
||||
- `OperatorName`: `Unicom AISP`
|
||||
- `OperatorNameCn`: `联通云 AI服务平台AISP`
|
||||
- `OperatorWebsite`: `https://www.cucloud.cn`
|
||||
- `SourceURL`: 购买计费 / Token Plan 页面
|
||||
- `Currency`: `CNY`
|
||||
- `InputPrice == OutputPrice == blendedPrice`
|
||||
- `Region`: 匹配支持矩阵中的具体云区域
|
||||
6. dry-run 摘要要显式输出:
|
||||
- `payg_mode_confirmed=true`
|
||||
- `payg_price_table_public=false`
|
||||
|
||||
### Task 4: 运行 focused tests
|
||||
|
||||
**Objective:** 验证 importer 合同成立。
|
||||
|
||||
Run:
|
||||
- `go test -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_cucloud_pricing.go ./scripts/import_cucloud_pricing_test.go`
|
||||
Expected:
|
||||
- PASS
|
||||
|
||||
### Task 5: 接入 smoke / pipeline
|
||||
|
||||
**Objective:** 让新 importer 进入日跑链路,但不移除 catalog importer。
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/verify_importer_smoke.sh`
|
||||
- Modify: `scripts/importer_smoke_gate_test.sh`
|
||||
- Modify: `scripts/pipeline_runtime_alignment_test.sh`
|
||||
- Modify: `scripts/run_intel_pipeline.sh`
|
||||
- Modify: `scripts/run_real_pipeline.sh`
|
||||
- Modify: `scripts/run_daily.sh`
|
||||
|
||||
**Step 1: 增加 `cucloud-pricing-fixture` / `cucloud-pricing-live` smoke**
|
||||
**Step 2: 增加 runtime source key `cucloud_pricing`**
|
||||
**Step 3: 保留 `cucloud_catalog`**
|
||||
|
||||
### Task 6: 文档 truth-sync
|
||||
|
||||
**Objective:** 把联通云状态从“只有目录级”升级为“目录+部分结构化价格”。
|
||||
|
||||
**Files:**
|
||||
- Modify: `docs/PLAN_CATALOG_COVERAGE_MATRIX.md`
|
||||
- Modify: `docs/NEXT_IMPORTER_RUNTIME_PRIORITY.md`
|
||||
- Modify: `docs/PLAN_CATALOG_INVENTORY.md`
|
||||
- Modify: `seeds/plan_catalog_inventory_seed_cn_relays_top20plus.json`(如需要)
|
||||
|
||||
**文案要求:**
|
||||
- 明确写:
|
||||
- Token Plan 三模型 blended price 已真实导入
|
||||
- AISP payg per-model 单价未公开,仍属 blocker
|
||||
- 禁止写成“联通云 payg 已完整打通”
|
||||
|
||||
## 验证命令
|
||||
|
||||
### Focused unit tests
|
||||
- `go test -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_cucloud_pricing.go ./scripts/import_cucloud_pricing_test.go`
|
||||
|
||||
### Plan catalog mapping tests
|
||||
- `go test -tags llm_script ./scripts/subscription_import_common.go ./scripts/import_plan_catalog.go ./scripts/import_plan_catalog_test.go`
|
||||
|
||||
### Shell gates
|
||||
- `bash scripts/pipeline_runtime_alignment_test.sh`
|
||||
- `bash scripts/importer_smoke_gate_test.sh`
|
||||
|
||||
### Live dry-run
|
||||
- `go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_cucloud_pricing.go -dry-run`
|
||||
|
||||
**Expected live dry-run truth:**
|
||||
- 只宣称 Token Plan 三模型 blended price 已导入
|
||||
- 同时输出 / 记录 `payg_mode_confirmed=true`
|
||||
- 同时输出 / 记录 `payg_price_table_public=false`
|
||||
|
||||
## 非目标
|
||||
|
||||
1. 不伪造 AISP payg per-model input/output 单价
|
||||
2. 不把 Token Plan blended price 冒充为 OpenAI 风格 input/output split price
|
||||
3. 不删除现有 `import_cucloud_catalog.go`
|
||||
4. 不在未发现公开价表前宣称“联通云细颗粒度价格已完整闭环”
|
||||
|
||||
## 当前最短闭环路径
|
||||
|
||||
1. 先实现 `import_cucloud_pricing.go` v1
|
||||
2. 只导入三模型 Token Plan blended price + 区域支持矩阵
|
||||
3. runtime/smoke 接入
|
||||
4. docs 标明 payg per-model price 仍是 verified blocker
|
||||
|
||||
这条路径能把联通云从“纯目录级”提升到“部分结构化价格已真实落库”,同时保持事实边界清晰。
|
||||
@@ -0,0 +1,420 @@
|
||||
# Intraday Discovery + Verification Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** 在不污染正式日报语义的前提下,为现有日内链路增加“搜索引擎 + 大模型候选发现层”和“官方来源验证层”,让当天的大模型价格新闻、版本发布、活动窗口能更早进入候选池,并只把已验证事实接入现有 `daily_signal_snapshot` / 日报语义链路。
|
||||
|
||||
**Architecture:** 保留现有 `scripts/run_intraday_price_watch.sh` 作为结构化价格事实刷新入口,不改它“只刷新价格/信号、不生成正式日报”的边界。新增一条独立的 `run_intraday_discovery_watch.sh` 发现链路:先用搜索引擎与 LLM 生成候选事件,再通过官方页面 / 价格页 / docs / 公告页做二次验证。候选与验证结果分别落入新表;只有 `official_confirmed` 的事件才允许映射进 `materialize_daily_signals.go` 的 `signalModelEvent`,并由现有 `generate_daily_report.go` 继续消费,不新造第二套日报事实系统。发现层与验证层必须通过仓库内可运行的 provider adapter 落地,不能依赖当前会话专属工具;实现上采用“命令或 HTTP provider 适配层 + fixture 测试”的方式,确保本地 cron 和 CI 环境可执行。已验证 discovery 事件接入现有事件流时必须去重:若同一 `provider + model + event_type + date` 已由 importer / 原生 loader 给出,则以原生事实为准,discovery 事件只补缺,不覆盖。
|
||||
|
||||
**Tech Stack:** Go 1.22、PostgreSQL、Bash、可配置搜索/LLM provider adapter、JSONB
|
||||
|
||||
---
|
||||
|
||||
### Task 1: 为候选发现与验证链路定义持久化结构
|
||||
|
||||
**Files:**
|
||||
- Create: `db/migrations/017_intraday_news_candidates.sql`
|
||||
- Modify: `docs/CONFIGURATION.md`
|
||||
- Modify: `DEPLOYMENT.md`
|
||||
|
||||
**Step 1: 新增候选表与验证表 migration**
|
||||
|
||||
创建两张表:
|
||||
- `intraday_news_candidate`
|
||||
- `intraday_news_verification`
|
||||
|
||||
候选表至少包含:
|
||||
- `candidate_date`
|
||||
- `event_type`
|
||||
- `provider_name`
|
||||
- `model_name`
|
||||
- `provider_country`
|
||||
- `title`
|
||||
- `summary`
|
||||
- `candidate_urls JSONB`
|
||||
- `discovery_source`
|
||||
- `discovery_query`
|
||||
- `discovery_evidence JSONB`
|
||||
- `normalized_key`
|
||||
- `status`
|
||||
- `verification_confidence`
|
||||
- `verification_notes`
|
||||
|
||||
验证表至少包含:
|
||||
- `candidate_id`
|
||||
- `verifier_source`
|
||||
- `verifier_url`
|
||||
- `verifier_status`
|
||||
- `extracted_facts JSONB`
|
||||
- `notes`
|
||||
|
||||
约束:
|
||||
- `intraday_news_candidate.normalized_key` 必须唯一,用于防止同日重复发现
|
||||
- `status` 至少支持:`candidate` / `verifying` / `verified` / `rejected` / `stale`
|
||||
- `verification_confidence` 至少支持:`candidate` / `secondary_confirmed` / `official_confirmed`
|
||||
|
||||
**Step 2: 明确与正式事实层的边界文档**
|
||||
|
||||
在 `docs/CONFIGURATION.md` 和 `DEPLOYMENT.md` 写明:
|
||||
- 候选发现层不会直接写 `daily_report`
|
||||
- 候选发现层不会覆盖 `latest_report`
|
||||
- `daily_signal_snapshot` 只消费已验证事实,不消费 `candidate_only`
|
||||
- `leak_or_rumor` 默认只保留在候选层,不进入正式日报事实
|
||||
|
||||
**Step 3: 运行 migration 验证**
|
||||
|
||||
Run:
|
||||
- `bash scripts/apply_migration.sh`
|
||||
|
||||
Expected:
|
||||
- 新表创建成功
|
||||
- 重复执行 migration 不报错
|
||||
|
||||
**Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add db/migrations/017_intraday_news_candidates.sql docs/CONFIGURATION.md DEPLOYMENT.md
|
||||
git commit -m "feat(intraday): add candidate and verification persistence"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: 实现候选发现层最小闭环
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/discover_intraday_news_candidates.go`
|
||||
- Create: `scripts/discover_intraday_news_candidates_test.go`
|
||||
- Create: `scripts/testdata/intraday_discovery_search_sample.json`
|
||||
- Create: `scripts/testdata/intraday_discovery_llm_sample.json`
|
||||
- Modify: `docs/CONFIGURATION.md`
|
||||
- Create: `scripts/intraday_discovery_provider.go`
|
||||
|
||||
**Step 1: 先写失败测试**
|
||||
|
||||
补 4 组测试:
|
||||
- 搜索结果解析测试:验证能从样例结果提取 title / summary / url / provider 线索
|
||||
- LLM 输出解析测试:验证能把 LLM JSON 输出转成候选事件
|
||||
- 候选归一化测试:验证同一事件经过标题差异改写后仍生成同一 `normalized_key`
|
||||
- URL 过滤测试:验证没有 URL 的候选被丢弃,避免 LLM 空口造线索
|
||||
|
||||
**Step 2: 运行失败测试**
|
||||
|
||||
Run:
|
||||
- `go test -count=1 -tags llm_script ./scripts/discover_intraday_news_candidates.go ./scripts/discover_intraday_news_candidates_test.go`
|
||||
|
||||
Expected:
|
||||
- 新增测试失败
|
||||
- 失败原因是缺少解析、归一化或去重逻辑
|
||||
|
||||
**Step 3: 实现最小候选发现器**
|
||||
|
||||
在 `discover_intraday_news_candidates.go` 中实现:
|
||||
- 固定 provider 查询模板集(中英双语)
|
||||
- 搜索结果抓取适配层
|
||||
- LLM 候选摘要适配层
|
||||
- 去重与归一化逻辑
|
||||
- 写入 `intraday_news_candidate`
|
||||
- provider adapter 抽象层(搜索 / LLM 均可通过命令或 HTTP provider 接入,默认实现不可依赖当前会话专属工具)
|
||||
|
||||
限制:
|
||||
- LLM 只允许输出候选,不允许直接标成 `verified`
|
||||
- 无 URL 候选直接丢弃
|
||||
- 搜索 / LLM provider 未配置时必须以前置条件错误退出,不能伪装成业务无新闻
|
||||
- 默认事件类型至少支持:
|
||||
- `price_cut`
|
||||
- `price_increase`
|
||||
- `official_release`
|
||||
- `promo_campaign`
|
||||
- `leak_or_rumor`
|
||||
- `unknown`
|
||||
|
||||
**Step 4: 重新运行测试**
|
||||
|
||||
Run:
|
||||
- `go test -count=1 -tags llm_script ./scripts/discover_intraday_news_candidates.go ./scripts/discover_intraday_news_candidates_test.go`
|
||||
|
||||
Expected:
|
||||
- 候选解析与归一化测试通过
|
||||
|
||||
**Step 5: 运行一次 dry-run 验证**
|
||||
|
||||
Run:
|
||||
- `go run -tags llm_script ./scripts/discover_intraday_news_candidates.go --date=2026-05-25 --dry-run`
|
||||
|
||||
Expected:
|
||||
- 输出 `candidate_total` / `provider_hit_count` / `event_type_counts`
|
||||
- dry-run 不写 `daily_report`
|
||||
- dry-run 不改 `latest_report`
|
||||
|
||||
**Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/discover_intraday_news_candidates.go scripts/discover_intraday_news_candidates_test.go scripts/testdata/intraday_discovery_search_sample.json scripts/testdata/intraday_discovery_llm_sample.json docs/CONFIGURATION.md
|
||||
git commit -m "feat(intraday): add news candidate discovery pipeline"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: 实现候选验证层并固化“只信官方事实”的规则
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/verify_intraday_news_candidates.go`
|
||||
- Create: `scripts/verify_intraday_news_candidates_test.go`
|
||||
- Create: `scripts/testdata/intraday_verification_official_release.html`
|
||||
- Create: `scripts/testdata/intraday_verification_pricing_page.html`
|
||||
- Create: `scripts/testdata/intraday_verification_secondary_media.html`
|
||||
- Modify: `docs/CONFIGURATION.md`
|
||||
|
||||
**Step 1: 先写失败测试**
|
||||
|
||||
补 5 组测试:
|
||||
- 官方发布页验证测试:命中模型名与发布时间时,产出 `official_confirmed`
|
||||
- 官方价格页验证测试:只有拿到真实价格变化时,才允许产出 `price_cut` / `price_increase`
|
||||
- 活动页验证测试:官方活动页可映射为 `promo_campaign`
|
||||
- 二手媒体降级测试:二手媒体最多得到 `secondary_confirmed`,不能直接进入正式事实层
|
||||
- 泄露类隔离测试:`leak_or_rumor` 即使有外部讨论,也不会升级为正式日报事实
|
||||
|
||||
**Step 2: 运行失败测试**
|
||||
|
||||
Run:
|
||||
- `go test -count=1 -tags llm_script ./scripts/verify_intraday_news_candidates.go ./scripts/verify_intraday_news_candidates_test.go`
|
||||
|
||||
Expected:
|
||||
- 新增测试失败
|
||||
- 失败原因是缺少来源分类与验证状态映射逻辑
|
||||
|
||||
**Step 3: 实现验证器**
|
||||
|
||||
在 `verify_intraday_news_candidates.go` 中实现:
|
||||
- 读取 `candidate` / `verifying` 状态候选
|
||||
- 拉取 `candidate_urls`
|
||||
- 基于域名与页面内容判定:
|
||||
- `official_page`
|
||||
- `pricing_page`
|
||||
- `official_docs`
|
||||
- `official_blog`
|
||||
- `secondary_media`
|
||||
- 把验证轨迹写入 `intraday_news_verification`
|
||||
- 更新 `intraday_news_candidate.status` 与 `verification_confidence`
|
||||
- 验证成功后只更新候选层状态,不直接写 `daily_signal_snapshot`;正式事实仍统一由物化器汇总
|
||||
|
||||
规则:
|
||||
- 只有官方页面 / 价格页 / docs / 公告页可以产出 `official_confirmed`
|
||||
- 价格新闻若无法拿到真实价格事实,只能维持候选或二级确认,不能伪造价格变化事件
|
||||
- `leak_or_rumor` 默认不升级为正式事实
|
||||
|
||||
**Step 4: 重新运行测试**
|
||||
|
||||
Run:
|
||||
- `go test -count=1 -tags llm_script ./scripts/verify_intraday_news_candidates.go ./scripts/verify_intraday_news_candidates_test.go`
|
||||
|
||||
Expected:
|
||||
- 验证规则测试通过
|
||||
|
||||
**Step 5: 运行一次 dry-run 验证**
|
||||
|
||||
Run:
|
||||
- `go run -tags llm_script ./scripts/verify_intraday_news_candidates.go --date=2026-05-25 --dry-run`
|
||||
|
||||
Expected:
|
||||
- 输出 `verified_total` / `official_confirmed_total` / `secondary_confirmed_total`
|
||||
- dry-run 只打印摘要,不写 `daily_report`
|
||||
|
||||
**Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/verify_intraday_news_candidates.go scripts/verify_intraday_news_candidates_test.go scripts/testdata/intraday_verification_official_release.html scripts/testdata/intraday_verification_pricing_page.html scripts/testdata/intraday_verification_secondary_media.html docs/CONFIGURATION.md
|
||||
git commit -m "feat(intraday): add candidate verification pipeline"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: 把已验证事件接入现有 `materialize_daily_signals.go`
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/materialize_daily_signals.go`
|
||||
- Create or Modify: `scripts/materialize_daily_signals_test.go`
|
||||
- Modify: `docs/plans/2026-05-27-intraday-price-watch-plan.md`
|
||||
- Modify: `README.md`
|
||||
- Modify: `docs/PRODUCTION_CHECKLIST.md`
|
||||
|
||||
**Step 1: 先写失败测试**
|
||||
|
||||
补 4 组测试:
|
||||
- 已验证官方发布事件会进入 `daily_signal_snapshot.top_events`
|
||||
- 已验证活动事件会进入 `daily_signal_snapshot.top_events`
|
||||
- `candidate_only` 与 `leak_or_rumor` 不进入正式快照
|
||||
- 未拿到真实价格变化数据的“价格新闻”不会被错误映射为 `price_cut` / `price_increase`
|
||||
|
||||
**Step 2: 运行失败测试**
|
||||
|
||||
Run:
|
||||
- `go test -count=1 -tags llm_script ./scripts/materialize_daily_signals.go ./scripts/materialize_daily_signals_test.go`
|
||||
|
||||
Expected:
|
||||
- 新增测试失败
|
||||
- 失败原因是当前物化器还不会读取已验证候选事件
|
||||
|
||||
**Step 3: 最小实现 verified event loader**
|
||||
|
||||
在 `materialize_daily_signals.go` 中新增:
|
||||
- `loadVerifiedIntradayNewsEvents(db, date string)`
|
||||
- 将 `official_confirmed` 的:
|
||||
- `official_release`
|
||||
- `promo_campaign`
|
||||
- 已确认真实价格变化的 `price_cut` / `price_increase`
|
||||
映射为现有 `signalModelEvent`
|
||||
- 与现有 `loadSignalModelEvents` 结果做去重合并;同日同模型同事件类型若已由 importer / 原生 loader 给出,则 discovery 事件仅补 `SourceURL` / 证据缺口,不抢占优先级
|
||||
|
||||
约束:
|
||||
- 不新造第二套快照表
|
||||
- 不改变 `daily_signal_snapshot` 的正式事实语义
|
||||
- `secondary_confirmed` 默认不进入正式快照
|
||||
|
||||
**Step 4: 重新运行测试**
|
||||
|
||||
Run:
|
||||
- `go test -count=1 -tags llm_script ./scripts/materialize_daily_signals.go ./scripts/materialize_daily_signals_test.go`
|
||||
|
||||
Expected:
|
||||
- verified event 相关测试通过
|
||||
|
||||
**Step 5: 联合验证日内边界**
|
||||
|
||||
Run:
|
||||
- `REPORT_TRIGGER_SOURCE=intraday_discovery go run -tags llm_script ./scripts/materialize_daily_signals.go --date=2026-05-25 --dry-run`
|
||||
|
||||
Expected:
|
||||
- 输出含 `page_mode` / `event_count`
|
||||
- 不写 `daily_report`
|
||||
- 不覆盖 `latest_report`
|
||||
|
||||
**Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/materialize_daily_signals.go scripts/materialize_daily_signals_test.go README.md docs/PRODUCTION_CHECKLIST.md docs/plans/2026-05-27-intraday-price-watch-plan.md
|
||||
git commit -m "feat(intraday): materialize verified discovery events"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: 组装新的日内发现入口并补部署说明
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/run_intraday_discovery_watch.sh`
|
||||
- Modify: `README.md`
|
||||
- Modify: `docs/CONFIGURATION.md`
|
||||
- Modify: `DEPLOYMENT.md`
|
||||
- Modify: `docs/PRODUCTION_CHECKLIST.md`
|
||||
|
||||
**Step 1: 实现独立入口脚本**
|
||||
|
||||
脚本顺序固定为:
|
||||
1. `discover_intraday_news_candidates.go`
|
||||
2. `verify_intraday_news_candidates.go`
|
||||
3. `materialize_daily_signals.go`(仅消费 verified 事件)
|
||||
|
||||
要求:
|
||||
- 明确要求 `DATABASE_URL`
|
||||
- 搜索 / LLM 所需 key 缺失时,输出前置条件错误,不伪装成代码失败
|
||||
- 不执行 `generate_daily_report.go`
|
||||
- 不写 `daily_report`
|
||||
- 不覆盖 `latest_report`
|
||||
|
||||
**Step 2: 更新调度文档**
|
||||
|
||||
文档里明确两条 cron:
|
||||
- 结构化价格刷新:`run_intraday_price_watch.sh`
|
||||
- 新闻发现与验证:`run_intraday_discovery_watch.sh`
|
||||
|
||||
推荐起步频率:
|
||||
- `run_intraday_discovery_watch.sh`:每 2 小时一次
|
||||
- `run_intraday_price_watch.sh`:每 4 小时一次
|
||||
|
||||
**Step 3: 运行脚本级 dry-run**
|
||||
|
||||
Run:
|
||||
- `bash scripts/run_intraday_discovery_watch.sh --dry-run`
|
||||
|
||||
Expected:
|
||||
- 输出候选发现摘要 + 验证摘要 + 信号物化摘要
|
||||
- 不生成正式日报产物
|
||||
|
||||
**Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add scripts/run_intraday_discovery_watch.sh README.md docs/CONFIGURATION.md DEPLOYMENT.md docs/PRODUCTION_CHECKLIST.md
|
||||
git commit -m "feat(intraday): add discovery watch runner"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: 运行最终联合验收并准备本地提交
|
||||
|
||||
**Files:**
|
||||
- Modify: `README.md`(仅在最终说明缺失时)
|
||||
- Modify: `docs/CONFIGURATION.md`(仅在最终说明缺失时)
|
||||
- Modify: `DEPLOYMENT.md`(仅在最终说明缺失时)
|
||||
|
||||
**Step 1: 运行 focused Go tests**
|
||||
|
||||
Run:
|
||||
- `go test -count=1 -tags llm_script ./scripts/discover_intraday_news_candidates.go ./scripts/discover_intraday_news_candidates_test.go`
|
||||
- `go test -count=1 -tags llm_script ./scripts/verify_intraday_news_candidates.go ./scripts/verify_intraday_news_candidates_test.go`
|
||||
- `go test -count=1 -tags llm_script ./scripts/materialize_daily_signals.go ./scripts/materialize_daily_signals_test.go`
|
||||
|
||||
Expected:
|
||||
- 发现层、验证层、信号物化层 focused tests 全通过
|
||||
|
||||
**Step 2: 运行现有日报/前端回归边界**
|
||||
|
||||
Run:
|
||||
- `go test -count=1 -tags llm_script ./scripts/generate_daily_report.go ./scripts/generate_daily_report_test.go ./scripts/official_import_signature_audit_query_lib.go`
|
||||
- `bash scripts/secret_gate_test.sh`
|
||||
- `bash scripts/test_importers.sh`
|
||||
- `cd frontend && npm test -- --run`
|
||||
- `cd frontend && npm run build`
|
||||
|
||||
Expected:
|
||||
- 原有日报与前端链路不回归
|
||||
- discovery 新增能力不污染正式日报边界
|
||||
|
||||
**Step 3: 运行脚本级联合 dry-run**
|
||||
|
||||
Run:
|
||||
- `bash scripts/run_intraday_discovery_watch.sh --dry-run`
|
||||
- `REPORT_TRIGGER_SOURCE=intraday go run -tags llm_script ./scripts/materialize_daily_signals.go --date=2026-05-25 --dry-run`
|
||||
|
||||
Expected:
|
||||
- 不写 `daily_report`
|
||||
- 不覆盖 `latest_report`
|
||||
- 能稳定输出候选数、验证数、事件数、page_mode、source_audit
|
||||
|
||||
**Step 4: 本地提交**
|
||||
|
||||
```bash
|
||||
git add db/migrations/017_intraday_news_candidates.sql scripts/discover_intraday_news_candidates.go scripts/discover_intraday_news_candidates_test.go scripts/verify_intraday_news_candidates.go scripts/verify_intraday_news_candidates_test.go scripts/materialize_daily_signals.go scripts/materialize_daily_signals_test.go scripts/run_intraday_discovery_watch.sh README.md docs/CONFIGURATION.md DEPLOYMENT.md docs/PRODUCTION_CHECKLIST.md docs/plans/2026-05-25-intraday-discovery-verification-implementation-plan.md docs/plans/2026-05-27-intraday-price-watch-plan.md
|
||||
git commit -m "feat(intraday): add discovery and verification watch pipeline"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 验收标准
|
||||
|
||||
实现完成后,必须同时满足:
|
||||
- 搜索 + LLM 只能产生候选事件,不能直接写成正式日报事实
|
||||
- 只有 `official_confirmed` 的事件才能进入正式 `daily_signal_snapshot` 语义链路
|
||||
- `leak_or_rumor` 不进入正式日报事实层
|
||||
- `run_intraday_discovery_watch.sh` 与 `run_intraday_price_watch.sh` 职责分离
|
||||
- 正式日报仍只由 `run_daily.sh` 负责
|
||||
- 新增链路不会写 `daily_report`、不会覆盖 `latest_report`
|
||||
- discovery provider adapter 在无配置时会明确报前置条件错误;有 fixture / dry-run 模式可本地验证
|
||||
- 新增 focused tests、现有日报测试、前端构建全部通过
|
||||
|
||||
## 非目标
|
||||
|
||||
本计划刻意不做:
|
||||
- 不新增第二套正式日报系统
|
||||
- 不让 LLM 直接替代价格 importer 或官方发布 importer
|
||||
- 不把二手媒体新闻直接映射为 `price_cut` / `price_increase`
|
||||
- 不在第一阶段引入新的前端“候选情报面板”复杂交互;若后续需要,单独立计划
|
||||
60
docs/plans/2026-05-27-intraday-price-watch-plan.md
Normal file
60
docs/plans/2026-05-27-intraday-price-watch-plan.md
Normal file
@@ -0,0 +1,60 @@
|
||||
# 日内价格追踪方案(2026-05-27)
|
||||
|
||||
## 目标
|
||||
|
||||
让“日内大降价 / 大涨价 / 泄露 / 活动窗口”不必等到第二天正式日报才出现。
|
||||
|
||||
## 当前限制
|
||||
|
||||
- 正式链路 `scripts/run_daily.sh` 按天运行一次。
|
||||
- `daily_signal_snapshot` 也是按日物化。
|
||||
- 像“小米大模型大降价”这样的日内事件,即使价格页已经变化,也可能错过当天头条和一句话结论。
|
||||
|
||||
## 最小可落地方案
|
||||
|
||||
新增脚本:`scripts/run_intraday_price_watch.sh`
|
||||
|
||||
它复用当前 `run_intel_pipeline.sh` 的“采集 / 导入 / 物化”链路,但刻意不生成正式日报,不写 `daily_report`,也不污染 `latest_report` 语义。
|
||||
|
||||
### 执行内容
|
||||
|
||||
- `fetch_openrouter.go -strict-real`
|
||||
- `fetch_multi_source.go --sources moonshot,deepseek,openai`
|
||||
- 官方导入脚本(套餐 + 价格 importer)
|
||||
- `materialize_daily_signals.go`
|
||||
|
||||
### 不执行
|
||||
|
||||
- `generate_daily_report.go`
|
||||
- `track_report_state` / `daily_report`
|
||||
- 正式 HTML / Markdown 日报归档
|
||||
|
||||
## 推荐调度频率
|
||||
|
||||
推荐两档:
|
||||
|
||||
1. 保守版:每 4 小时一次
|
||||
- `0 */4 * * * bash scripts/run_intraday_price_watch.sh`
|
||||
2. 激进版:每 2 小时一次
|
||||
- `0 */2 * * * bash scripts/run_intraday_price_watch.sh`
|
||||
|
||||
先从每 4 小时开始,观察外部文档源稳定性和数据库写入压力。
|
||||
|
||||
## 结果用途
|
||||
|
||||
- 更快写入 `pricing_history`
|
||||
- 更快刷新 `daily_signal_snapshot`
|
||||
- 为前端查询页/日内快讯卡提供更及时的信号
|
||||
- 第二天正式日报能直接消费更完整的价格变化记录
|
||||
|
||||
## 与正式日报的边界
|
||||
|
||||
- `run_daily.sh`:正式日级产物,决定 `latest_report`
|
||||
- `run_intraday_price_watch.sh`:日内信号刷新,不生成正式日报
|
||||
- `run_real_pipeline.sh`:人工真实复跑,验证全链路
|
||||
|
||||
## 下一步建议
|
||||
|
||||
1. 已补充 `run_intraday_discovery_watch.sh` 与 DeepSeek 官方新闻页结构签名 guard,可继续扩展到 DeepSeek pricing 页面
|
||||
2. 给前端查询页增加“最近一次价格追踪时间 / 最近一次 discovery 验证时间 / 最近一次官方页 drift 检查时间”提示
|
||||
3. 如果日内事件仍不够敏感,再考虑引入独立 `intraday_signal_snapshot` 或候选情报面板
|
||||
108
docs/references/cucloud-token-plan-vs-aisp.md
Normal file
108
docs/references/cucloud-token-plan-vs-aisp.md
Normal file
@@ -0,0 +1,108 @@
|
||||
# 联通云 Token Plan 与 AISP payg 公开价格边界说明
|
||||
|
||||
更新时间:2026-05-23(Asia/Shanghai)
|
||||
|
||||
## 结论
|
||||
|
||||
当前 `cucloud_pricing` 已能真实覆盖:
|
||||
- AISP Token Plan 团队版公开披露的 3 个模型 blended 单价
|
||||
- 公开模型-区域支持矩阵
|
||||
|
||||
但当前**仍不能**宣称联通云 AICP / AI 应用开发平台的 **payg per-model 官方销售价表** 已公开。
|
||||
|
||||
因此当前仓库的正确语义是:
|
||||
- `cucloud_catalog`:目录入口 / 平台存在性校验
|
||||
- `cucloud_pricing`:Token Plan 三模型 blended price + 区域支持矩阵
|
||||
- AISP payg per-model 销售价:**官方公开模式已确认,但具体模型销售价仍未公开**
|
||||
|
||||
## 已验证的官方公开面
|
||||
|
||||
### 1. 平台入口
|
||||
- `https://www.cucloud.cn/act/CloudAI.html`
|
||||
- 实测:可访问
|
||||
- 用途:确认 AICP / AI 应用开发平台官方入口存在
|
||||
- 边界:不是结构化 payg 价格页
|
||||
|
||||
### 2. AISP 帮助中心公开文档
|
||||
已验证可直接 `GET`、无需登录:
|
||||
- `https://support.cucloud.cn/document/127/591/2357.html?id=2357&folderid=2973`(购买计费)
|
||||
- `https://support.cucloud.cn/document/127/591/2357.html?id=2357&folderid=3236`(Token Plan)
|
||||
- `https://support.cucloud.cn/document/127/591/2357.html?id=2357&folderid=3237`(Coding Plan)
|
||||
|
||||
## 当前公开可落库事实
|
||||
|
||||
### Token Plan 团队版三模型 blended price
|
||||
当前 importer 与测试锁定的公开价格事实:
|
||||
- `DeepSeek-V4-Pro`:`9.30 元/百万tokens`
|
||||
- `DeepSeek-V4-Flash`:`0.70 元/百万tokens`
|
||||
- `MiniMax-M2.5`:`1.10 元/百万tokens`
|
||||
|
||||
当前实现写法:
|
||||
- `InputPrice = blendedPrice`
|
||||
- `OutputPrice = blendedPrice`
|
||||
|
||||
这是为了忠实表达公开的 blended 单价,**不是** AISP payg 的 input/output split price。
|
||||
|
||||
### 区域支持矩阵
|
||||
当前公开文档还能解析出模型-区域支持矩阵,因此 importer 可按支持区域落库,而不是伪造全区域价格。
|
||||
|
||||
## 当前已验证但不能伪造的事实
|
||||
|
||||
公开帮助中心文档已说明:
|
||||
- 联通云存在 `按量计费` 模式
|
||||
- 单位可表述为 `元/千 Tokens`
|
||||
- 费用按所选模型销售价实时累加
|
||||
|
||||
但截至本次复查,公开页面**仍未披露**:
|
||||
- AISP payg 的 per-model 销售价表
|
||||
- 可直接写入 `region_pricing` 的逐模型 payg 单价矩阵
|
||||
- 稳定公开的 input/output 单价表
|
||||
|
||||
因此:
|
||||
- `payg_mode_confirmed = true`
|
||||
- `payg_price_table_public = false`
|
||||
|
||||
这也是当前 live dry-run 的预期输出,而不是临时降级。
|
||||
|
||||
## 本次复查验证结果
|
||||
|
||||
执行 live dry-run:
|
||||
- `go run -tags=llm_script scripts/import_cucloud_pricing.go scripts/official_pricing_import_common.go scripts/subscription_import_common.go -dry-run -timeout 60`
|
||||
|
||||
结果:
|
||||
- `source=cucloud-pricing-import models=3 records=4 regions=2 operator=Unicom AISP payg_mode_confirmed=true payg_price_table_public=false dry_run=true`
|
||||
|
||||
含义:
|
||||
- 3 个模型价格事实仍可解析
|
||||
- 4 条区域化记录仍可生成
|
||||
- 官方按量计费模式仍可确认
|
||||
- 公开 payg per-model 价表依然不存在
|
||||
|
||||
## 为什么现在不应扩 `import_cucloud_pricing.go`
|
||||
|
||||
如果在当前公开面下继续扩成“联通云 payg per-model 官方 importer”,会产生三类失真:
|
||||
1. 把 Token Plan blended price 冒充成 payg 模型销售价
|
||||
2. 把“计费模式说明”误写成“结构化价格页”
|
||||
3. 在 docs/runtime 中夸大为“联通云 payg 已完整打通”
|
||||
|
||||
这些都违背当前仓库的 truth-first 约束。
|
||||
|
||||
## 当前正确表述
|
||||
|
||||
可以说:
|
||||
- 联通云已补齐 `Token Plan` 三模型公开 blended price importer
|
||||
- 联通云公开 payg 模式已验证存在
|
||||
- 联通云 AICP / AI 应用开发平台的 payg per-model 官方价表仍缺
|
||||
|
||||
不能说:
|
||||
- 联通云 payg 已完整打通
|
||||
- 联通云所有模型的按量价格都已公开并入库
|
||||
- Token Plan blended price 等同于 payg input/output 单价
|
||||
|
||||
## 后续触发条件
|
||||
|
||||
只有当官方公开面出现以下任一信号,才应重新打开“联通云 payg per-model importer 扩展”任务:
|
||||
1. 公开帮助中心出现逐模型 payg 销售价表
|
||||
2. 文档源码 / 前端 payload 中出现可稳定提取的逐模型价格结构
|
||||
3. 官方 pricing/docs 页面明确披露按量模型价格矩阵
|
||||
4. `import_cucloud_pricing.go` live dry-run 由 `payg_price_table_public=false` 变为 `true`
|
||||
98
docs/references/modelbest-minicpm-public-source-gap.md
Normal file
98
docs/references/modelbest-minicpm-public-source-gap.md
Normal file
@@ -0,0 +1,98 @@
|
||||
# MiniCPM / ModelBest 官方 payg 价格源缺口说明
|
||||
|
||||
更新时间:2026-05-23(Asia/Shanghai)
|
||||
|
||||
## 结论
|
||||
|
||||
当前**不能**为 `modelbest-minicpm-api-payg` 落地真实官方 payg pricing importer。
|
||||
|
||||
原因不是“还没写解析器”,而是**公开可访问的官方价格源未被证明存在**。
|
||||
|
||||
当前应保持:
|
||||
- `importerKey = import_catalog_seed_verification.go`
|
||||
- `sourceKind = official_product_page`
|
||||
- 平台状态 = **目录级官方入口核验**
|
||||
|
||||
## 已验证的官方公开面
|
||||
|
||||
### 1. 历史 seed / docs 使用的 hostname
|
||||
- URL: `https://platform.modelbest.cn/`
|
||||
- 实测结果:`404`
|
||||
- 结论:历史上用于表示“面壁开放平台”的子域当前不可作为稳定公开来源
|
||||
|
||||
### 2. 当前可访问官网
|
||||
- URL: `https://modelbest.cn/`
|
||||
- 实测结果:`200`
|
||||
- 页面性质:公司官网 / 产品介绍页
|
||||
- 已见信息:MiniCPM 系列模型介绍、GitHub / HuggingFace / 飞书 Cookbook 链接
|
||||
- 未见信息:
|
||||
- `元/百万token`
|
||||
- `元/千token`
|
||||
- `按 token`
|
||||
- `token 后付费`
|
||||
- 结构化 input/output 单价表
|
||||
|
||||
### 3. 当前公开飞书文档
|
||||
- URL: `https://modelbest.feishu.cn/wiki/D2tFw8Pcsi5CIzkaHNacLK64npg`
|
||||
- URL: `https://modelbest.feishu.cn/wiki/LZxLwp4Lzi29vXklYLFchwN5nCf`
|
||||
- 实测结果:均可访问
|
||||
- 页面性质:MiniCPM Cookbook / 部署指南
|
||||
- 已见信息:模型部署、推理、训练、示例代码、模型下载链接
|
||||
- 未见信息:稳定公开 payg 价格页或结构化计费表
|
||||
|
||||
## 证据摘要
|
||||
|
||||
### HTTP 可达性
|
||||
- `modelbest.cn` → `200`
|
||||
- `platform.modelbest.cn` → `404`
|
||||
|
||||
### 内容检查结论
|
||||
对官网与公开飞书页面做过页面文本/HTML 检查,未发现可用于 `region_pricing` 的公开定价信号:
|
||||
- 无稳定 payg 价卡
|
||||
- 无 input/output token 单价表
|
||||
- 无可证实的公开 pricing JSON/API
|
||||
|
||||
## 为什么这不是 importer 任务
|
||||
|
||||
根据当前仓库的 importer 验收标准,只有在确认到**稳定公开的官方价格源**后,才应该进入:
|
||||
1. fixture/parser 测试
|
||||
2. importer 实现
|
||||
3. runtime/smoke/gate wiring
|
||||
|
||||
当前 MiniCPM 路径缺的是第 0 步:**上游真相本身不存在或至少未公开可访问**。
|
||||
|
||||
如果此时强行写 importer,只会造成:
|
||||
- 把产品页 / 教程页误当成价格页
|
||||
- 在 docs 中虚报“已有官方 payg importer”
|
||||
- 后续 smoke/runtime 无法形成真实闭环
|
||||
|
||||
## 当前仓库已做同步
|
||||
|
||||
已同步修正:
|
||||
- `seeds/plan_catalog_inventory_seed_cn_vendors_top20.json`
|
||||
- `docs/PLAN_CATALOG_COVERAGE_MATRIX.md`
|
||||
- `docs/NEXT_IMPORTER_RUNTIME_PRIORITY.md`
|
||||
- `docs/PLAN_CATALOG_INVENTORY.md`
|
||||
|
||||
修正点:
|
||||
- 把失效 `platform.modelbest.cn` 来源改为当前可访问的 `modelbest.cn`
|
||||
- 明确标注“当前仅走目录级官方入口核验”
|
||||
- 把“MiniCPM 官方 payg importer”改写为“官方价格源复查 / 跟进”
|
||||
|
||||
## 后续触发条件
|
||||
|
||||
只有出现以下任一条件,才重新打开 MiniCPM 官方 pricing importer 任务:
|
||||
|
||||
1. 出现稳定公开的官方价格页
|
||||
- 例如明确展示 `元/百万token` / `按 token` / input-output 单价表
|
||||
2. 官网或开放平台恢复一个可公开访问的 pricing/docs 子域
|
||||
3. 官网前端 bundle / router payload 中出现可验证的 pricing 结构化数据
|
||||
4. 官方公开 API 文档明确给出按量价格表
|
||||
|
||||
## 下一步建议
|
||||
|
||||
优先转向:
|
||||
- `联通云 payg per-model 价格公开表跟进`
|
||||
|
||||
MiniCPM 这条保留为:
|
||||
- **catalog verification + 周期性复查**
|
||||
@@ -3,7 +3,8 @@
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
font-family:
|
||||
-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
}
|
||||
|
||||
.navbar {
|
||||
@@ -200,6 +201,192 @@
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
.report-section {
|
||||
margin-bottom: 24px;
|
||||
padding: 20px;
|
||||
border: 1px solid #bfdbfe;
|
||||
border-radius: 12px;
|
||||
background: linear-gradient(180deg, #eff6ff 0%, #ffffff 100%);
|
||||
}
|
||||
|
||||
.report-header {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.report-header h3 {
|
||||
margin: 0;
|
||||
color: #111827;
|
||||
}
|
||||
|
||||
.report-header p {
|
||||
margin: 6px 0 0;
|
||||
color: #6b7280;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.report-status {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
padding: 8px 12px;
|
||||
border-radius: 999px;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
text-transform: lowercase;
|
||||
}
|
||||
|
||||
.report-status-generated {
|
||||
background: #dbeafe;
|
||||
color: #1d4ed8;
|
||||
}
|
||||
|
||||
.report-status-other {
|
||||
background: #e5e7eb;
|
||||
color: #374151;
|
||||
}
|
||||
|
||||
.report-card {
|
||||
padding: 18px;
|
||||
border: 1px solid #dbeafe;
|
||||
border-radius: 18px;
|
||||
background:
|
||||
radial-gradient(
|
||||
circle at top left,
|
||||
rgba(37, 99, 235, 0.1),
|
||||
transparent 35%
|
||||
),
|
||||
rgba(255, 255, 255, 0.94);
|
||||
box-shadow: 0 12px 30px rgba(37, 99, 235, 0.08);
|
||||
}
|
||||
|
||||
.report-hero {
|
||||
padding: 18px;
|
||||
border-radius: 16px;
|
||||
background: linear-gradient(135deg, #123c63 0%, #24507a 100%);
|
||||
color: #ffffff;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.report-eyebrow {
|
||||
font-size: 12px;
|
||||
letter-spacing: 0.08em;
|
||||
text-transform: uppercase;
|
||||
opacity: 0.78;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.report-summary {
|
||||
font-size: 22px;
|
||||
line-height: 1.35;
|
||||
font-weight: 800;
|
||||
}
|
||||
|
||||
.report-meta {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 12px;
|
||||
margin-bottom: 14px;
|
||||
color: #4b5563;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.report-highlights {
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
.report-highlight {
|
||||
display: grid;
|
||||
gap: 4px;
|
||||
padding: 12px 14px;
|
||||
border-radius: 12px;
|
||||
background: rgba(239, 246, 255, 0.92);
|
||||
border: 1px solid #dbeafe;
|
||||
}
|
||||
|
||||
.report-highlight strong {
|
||||
color: #1e3a8a;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.report-highlight span {
|
||||
color: #475569;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.report-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.report-link {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 10px 14px;
|
||||
border: 1px solid #93c5fd;
|
||||
border-radius: 8px;
|
||||
color: #1d4ed8;
|
||||
text-decoration: none;
|
||||
font-weight: 600;
|
||||
background: #ffffff;
|
||||
}
|
||||
|
||||
.report-link-primary {
|
||||
background: #2563eb;
|
||||
border-color: #2563eb;
|
||||
color: #ffffff;
|
||||
}
|
||||
|
||||
.report-note {
|
||||
margin-top: 12px;
|
||||
color: #6b7280;
|
||||
font-size: 13px;
|
||||
}
|
||||
.runtime-warning {
|
||||
margin-bottom: 16px;
|
||||
padding: 12px 14px;
|
||||
border: 1px solid #f59e0b;
|
||||
border-radius: 10px;
|
||||
background: #fff7ed;
|
||||
color: #9a3412;
|
||||
font-size: 14px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.runtime-error {
|
||||
margin-bottom: 16px;
|
||||
padding: 12px 14px;
|
||||
border: 1px solid #dc2626;
|
||||
border-radius: 10px;
|
||||
background: #fef2f2;
|
||||
color: #991b1b;
|
||||
font-size: 14px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.runtime-error-inline {
|
||||
color: #991b1b;
|
||||
}
|
||||
|
||||
.data-empty {
|
||||
padding: 18px;
|
||||
border: 1px dashed #d1d5db;
|
||||
border-radius: 10px;
|
||||
background: #f9fafb;
|
||||
color: #6b7280;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.runtime-warning-inline {
|
||||
color: #9a3412;
|
||||
}
|
||||
|
||||
.subscription-section {
|
||||
padding: 20px;
|
||||
border: 1px solid #fde68a;
|
||||
@@ -287,6 +474,293 @@
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.explorer-editorial,
|
||||
.report-priority-section {
|
||||
display: grid;
|
||||
gap: 18px;
|
||||
}
|
||||
|
||||
.explorer-hero,
|
||||
.filters-editorial,
|
||||
.pricing-focus-card,
|
||||
.pricing-board-card,
|
||||
.report-priority-card,
|
||||
.report-news-card,
|
||||
.report-theme-card {
|
||||
border: 1px solid rgba(15, 23, 42, 0.08);
|
||||
border-radius: 20px;
|
||||
background: linear-gradient(
|
||||
180deg,
|
||||
rgba(255, 255, 255, 0.98) 0%,
|
||||
rgba(248, 250, 252, 0.96) 100%
|
||||
);
|
||||
box-shadow: 0 20px 50px rgba(15, 23, 42, 0.06);
|
||||
}
|
||||
|
||||
.explorer-hero,
|
||||
.pricing-focus-card,
|
||||
.report-priority-card {
|
||||
padding: 22px;
|
||||
}
|
||||
|
||||
.explorer-kicker,
|
||||
.report-news-label {
|
||||
font-size: 12px;
|
||||
letter-spacing: 0.12em;
|
||||
text-transform: uppercase;
|
||||
color: #475569;
|
||||
}
|
||||
|
||||
.explorer-hero {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: flex-start;
|
||||
gap: 20px;
|
||||
}
|
||||
|
||||
.explorer-hero h2 {
|
||||
margin: 6px 0 8px;
|
||||
font-size: 34px;
|
||||
line-height: 1.05;
|
||||
color: #0f172a;
|
||||
}
|
||||
|
||||
.explorer-hero p,
|
||||
.pricing-focus-header p,
|
||||
.report-news-summary,
|
||||
.report-theme-card li,
|
||||
.report-evidence {
|
||||
color: #475569;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.explorer-hero-meta {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.explorer-hero-meta span,
|
||||
.report-news-label {
|
||||
padding: 8px 12px;
|
||||
border-radius: 999px;
|
||||
background: #eff6ff;
|
||||
color: #1d4ed8;
|
||||
}
|
||||
|
||||
.filters-editorial {
|
||||
padding: 14px 16px;
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
.pricing-focus-header,
|
||||
.report-news-grid,
|
||||
.report-theme-grid,
|
||||
.pricing-board-grid {
|
||||
display: grid;
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.pricing-focus-header {
|
||||
grid-template-columns: 1fr auto;
|
||||
align-items: start;
|
||||
}
|
||||
|
||||
.pricing-focus-header h3,
|
||||
.report-theme-title,
|
||||
.report-news-title {
|
||||
margin: 8px 0 6px;
|
||||
color: #0f172a;
|
||||
}
|
||||
|
||||
.pricing-focus-prices {
|
||||
margin-top: 18px;
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.pricing-focus-prices div,
|
||||
.pricing-board-card,
|
||||
.report-news-card,
|
||||
.report-theme-card {
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
.pricing-focus-prices span,
|
||||
.pricing-board-meta {
|
||||
display: block;
|
||||
color: #64748b;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.pricing-focus-prices strong,
|
||||
.pricing-board-price {
|
||||
display: block;
|
||||
margin-top: 8px;
|
||||
font-size: 24px;
|
||||
color: #111827;
|
||||
}
|
||||
|
||||
.pricing-board-grid,
|
||||
.report-news-grid,
|
||||
.report-theme-grid {
|
||||
grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
|
||||
}
|
||||
|
||||
.report-priority-card {
|
||||
background: linear-gradient(180deg, #fff 0%, #f8fafc 100%);
|
||||
}
|
||||
|
||||
.report-hero-priority {
|
||||
background: linear-gradient(135deg, #0f172a 0%, #1e3a8a 55%, #2563eb 100%);
|
||||
}
|
||||
|
||||
.report-evidence {
|
||||
margin-top: 12px;
|
||||
color: rgba(255, 255, 255, 0.82);
|
||||
}
|
||||
|
||||
.report-theme-card ul {
|
||||
margin: 12px 0 0;
|
||||
padding-left: 18px;
|
||||
}
|
||||
|
||||
.model-table-editorial {
|
||||
box-shadow: 0 12px 30px rgba(15, 23, 42, 0.04);
|
||||
}
|
||||
|
||||
.theme-news-list {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
margin-top: 12px;
|
||||
}
|
||||
|
||||
.theme-news-item {
|
||||
padding: 14px;
|
||||
border-radius: 14px;
|
||||
background: rgba(248, 250, 252, 0.92);
|
||||
border: 1px solid rgba(148, 163, 184, 0.18);
|
||||
border-left: 6px solid #94a3b8;
|
||||
}
|
||||
|
||||
|
||||
.theme-news-item.tone-success {
|
||||
background: linear-gradient(180deg, rgba(240, 253, 244, 0.98) 0%, rgba(220, 252, 231, 0.92) 100%);
|
||||
border-color: rgba(34, 197, 94, 0.28);
|
||||
border-left-color: #16a34a;
|
||||
}
|
||||
|
||||
.theme-news-item.tone-success .card-title,
|
||||
.theme-news-item.tone-success .trust-line {
|
||||
color: #166534;
|
||||
}
|
||||
|
||||
.theme-news-item.tone-caution {
|
||||
background: linear-gradient(180deg, rgba(254, 242, 242, 0.98) 0%, rgba(254, 226, 226, 0.92) 100%);
|
||||
border-color: rgba(239, 68, 68, 0.26);
|
||||
border-left-color: #dc2626;
|
||||
}
|
||||
|
||||
.theme-news-item.tone-caution .card-title,
|
||||
.theme-news-item.tone-caution .trust-line {
|
||||
color: #991b1b;
|
||||
}
|
||||
|
||||
.theme-news-item.tone-promo {
|
||||
background: linear-gradient(180deg, rgba(255, 247, 237, 0.98) 0%, rgba(250, 245, 255, 0.94) 100%);
|
||||
border-color: rgba(168, 85, 247, 0.22);
|
||||
border-left-color: #f97316;
|
||||
}
|
||||
|
||||
.theme-news-item.tone-promo .card-title,
|
||||
.theme-news-item.tone-promo .trust-line {
|
||||
color: #9a3412;
|
||||
}
|
||||
|
||||
.report-theme-card {
|
||||
background: linear-gradient(180deg, rgba(248, 250, 252, 0.98) 0%, rgba(241, 245, 249, 0.94) 100%);
|
||||
}
|
||||
|
||||
.report-theme-card:first-child {
|
||||
border-color: rgba(34, 197, 94, 0.18);
|
||||
}
|
||||
|
||||
.report-theme-card:first-child .report-theme-title {
|
||||
color: #166534;
|
||||
}
|
||||
|
||||
.report-theme-card:nth-child(2) {
|
||||
border-color: rgba(239, 68, 68, 0.18);
|
||||
}
|
||||
|
||||
.report-theme-card:nth-child(2) .report-theme-title {
|
||||
color: #991b1b;
|
||||
}
|
||||
|
||||
.report-theme-card:nth-child(3) {
|
||||
border-color: rgba(249, 115, 22, 0.2);
|
||||
}
|
||||
|
||||
.report-theme-card:nth-child(3) .report-theme-title {
|
||||
color: #9a3412;
|
||||
}
|
||||
|
||||
.report-theme-badge {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.report-theme-badge-icon {
|
||||
width: 26px;
|
||||
height: 26px;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
border-radius: 999px;
|
||||
background: rgba(15, 23, 42, 0.08);
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.report-theme-badge-label {
|
||||
font-size: 11px;
|
||||
letter-spacing: 0.12em;
|
||||
text-transform: uppercase;
|
||||
color: #475569;
|
||||
}
|
||||
|
||||
.report-theme-card.tone-success .report-theme-badge-icon {
|
||||
background: rgba(34, 197, 94, 0.16);
|
||||
color: #166534;
|
||||
}
|
||||
|
||||
.report-theme-card.tone-caution .report-theme-badge-icon {
|
||||
background: rgba(239, 68, 68, 0.16);
|
||||
color: #991b1b;
|
||||
}
|
||||
|
||||
.report-theme-card.tone-promo .report-theme-badge-icon {
|
||||
background: rgba(249, 115, 22, 0.16);
|
||||
color: #9a3412;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.explorer-hero,
|
||||
.pricing-focus-header {
|
||||
grid-template-columns: 1fr;
|
||||
display: grid;
|
||||
}
|
||||
|
||||
.explorer-hero h2 {
|
||||
font-size: 28px;
|
||||
}
|
||||
|
||||
.pricing-focus-prices {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
@media (max-width: 768px) {
|
||||
.app {
|
||||
padding: 16px;
|
||||
@@ -302,6 +776,22 @@
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.report-header {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.report-summary {
|
||||
font-size: 24px;
|
||||
}
|
||||
|
||||
.report-actions {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.report-link {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.subscription-summary {
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
formatPrice,
|
||||
formatSubscriptionQuota,
|
||||
@@ -7,113 +7,207 @@ import {
|
||||
providerDistribution,
|
||||
summarizeModels,
|
||||
summarizeSubscriptionPlans,
|
||||
} from './models'
|
||||
} from "./models";
|
||||
|
||||
describe('models helpers', () => {
|
||||
it('normalizes fallback pricing and stale flags', () => {
|
||||
describe("models helpers", () => {
|
||||
it("normalizes fallback pricing and stale flags", () => {
|
||||
const model = normalizeModel({
|
||||
id: 'anthropic/claude-sonnet-4.6',
|
||||
provider_cn: 'Anthropic',
|
||||
id: "anthropic/claude-sonnet-4.6",
|
||||
provider_cn: "Anthropic",
|
||||
context_length: 200000,
|
||||
input_price: '3',
|
||||
output_price: '15',
|
||||
data_confidence: 'stale',
|
||||
})
|
||||
input_price: "3",
|
||||
output_price: "15",
|
||||
data_confidence: "stale",
|
||||
});
|
||||
|
||||
expect(model).not.toBeNull()
|
||||
expect(model?.providerCN).toBe('Anthropic')
|
||||
expect(model?.inputPrice).toBe(3)
|
||||
expect(model?.outputPrice).toBe(15)
|
||||
expect(model?.stale).toBe(true)
|
||||
expect(model?.pricingAvailable).toBe(true)
|
||||
})
|
||||
expect(model).not.toBeNull();
|
||||
expect(model?.providerCN).toBe("Anthropic");
|
||||
expect(model?.inputPrice).toBe(3);
|
||||
expect(model?.outputPrice).toBe(15);
|
||||
expect(model?.stale).toBe(true);
|
||||
expect(model?.pricingAvailable).toBe(true);
|
||||
});
|
||||
|
||||
it('marks free models and pricing unavailable correctly', () => {
|
||||
it("marks free models and pricing unavailable correctly", () => {
|
||||
const freeModel = normalizeModel({
|
||||
id: 'qwen/qwen3-coder:free',
|
||||
})
|
||||
id: "qwen/qwen3-coder:free",
|
||||
});
|
||||
const paidModel = normalizeModel({
|
||||
id: 'openai/gpt-4.1',
|
||||
id: "openai/gpt-4.1",
|
||||
pricing: {},
|
||||
})
|
||||
});
|
||||
|
||||
expect(formatPrice(freeModel!, 'input')).toContain('免费')
|
||||
expect(formatPrice(paidModel!, 'input')).toBe('pricing unavailable')
|
||||
})
|
||||
expect(formatPrice(freeModel!, "input")).toContain("免费");
|
||||
expect(formatPrice(paidModel!, "input")).toBe("pricing unavailable");
|
||||
});
|
||||
|
||||
it('summarizes providers and currencies', () => {
|
||||
it("summarizes providers and currencies", () => {
|
||||
const models = [
|
||||
normalizeModel({ id: 'deepseek/deepseek-chat', provider_cn: 'DeepSeek', currency: 'CNY', pricing: { input: 1, output: 2 } }),
|
||||
normalizeModel({ id: 'deepseek/deepseek-reasoner', provider_cn: 'DeepSeek', currency: 'CNY', pricing: { input: 2, output: 4 } }),
|
||||
normalizeModel({ id: 'anthropic/claude-sonnet-4.6', provider_cn: 'Anthropic', currency: 'USD', pricing: { input: 3, output: 15 } }),
|
||||
].filter((model): model is NonNullable<typeof model> => model !== null)
|
||||
normalizeModel({
|
||||
id: "deepseek/deepseek-chat",
|
||||
provider_cn: "DeepSeek",
|
||||
currency: "CNY",
|
||||
pricing: { input: 1, output: 2 },
|
||||
}),
|
||||
normalizeModel({
|
||||
id: "deepseek/deepseek-reasoner",
|
||||
provider_cn: "DeepSeek",
|
||||
currency: "CNY",
|
||||
pricing: { input: 2, output: 4 },
|
||||
}),
|
||||
normalizeModel({
|
||||
id: "anthropic/claude-sonnet-4.6",
|
||||
provider_cn: "Anthropic",
|
||||
currency: "USD",
|
||||
pricing: { input: 3, output: 15 },
|
||||
}),
|
||||
].filter((model): model is NonNullable<typeof model> => model !== null);
|
||||
|
||||
expect(summarizeModels(models)).toEqual({
|
||||
modelCount: 3,
|
||||
providerCount: 2,
|
||||
cnyCount: 2,
|
||||
})
|
||||
});
|
||||
expect(providerDistribution(models)).toEqual([
|
||||
{ name: 'DeepSeek', value: 2 },
|
||||
{ name: 'Anthropic', value: 1 },
|
||||
])
|
||||
})
|
||||
{ name: "DeepSeek", value: 2 },
|
||||
{ name: "Anthropic", value: 1 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('normalizes subscription plans from API payload', () => {
|
||||
it("normalizes subscription plans from API payload", () => {
|
||||
const plan = normalizeSubscriptionPlan({
|
||||
planCode: 'token-plan-lite',
|
||||
planName: '通用 Token Plan Lite',
|
||||
planFamily: 'token_plan',
|
||||
tier: 'Lite',
|
||||
provider: 'Tencent',
|
||||
providerCN: '腾讯',
|
||||
operator: 'Tencent Cloud',
|
||||
operatorCN: '腾讯云',
|
||||
currency: 'CNY',
|
||||
planCode: "token-plan-lite",
|
||||
planName: "通用 Token Plan Lite",
|
||||
planFamily: "token_plan",
|
||||
tier: "Lite",
|
||||
provider: "Tencent",
|
||||
providerCN: "腾讯",
|
||||
operator: "Tencent Cloud",
|
||||
operatorCN: "腾讯云",
|
||||
currency: "CNY",
|
||||
listPrice: 39,
|
||||
priceUnit: 'CNY/month',
|
||||
priceUnit: "CNY/month",
|
||||
quotaValue: 35000000,
|
||||
quotaUnit: 'tokens/month',
|
||||
quotaUnit: "tokens/month",
|
||||
contextWindow: 0,
|
||||
modelScope: ['tc-code-latest', 'glm-5', 'glm-5.1'],
|
||||
})
|
||||
modelScope: ["tc-code-latest", "glm-5", "glm-5.1"],
|
||||
});
|
||||
|
||||
expect(plan).not.toBeNull()
|
||||
expect(plan?.planCode).toBe('token-plan-lite')
|
||||
expect(plan?.providerCN).toBe('腾讯')
|
||||
expect(plan?.modelScope.length).toBe(3)
|
||||
expect(plan?.modelPreview).toBe('tc-code-latest, glm-5, glm-5.1')
|
||||
})
|
||||
expect(plan).not.toBeNull();
|
||||
expect(plan?.planCode).toBe("token-plan-lite");
|
||||
expect(plan?.providerCN).toBe("腾讯");
|
||||
expect(plan?.modelScope.length).toBe(3);
|
||||
expect(plan?.modelPreview).toBe("tc-code-latest, glm-5, glm-5.1");
|
||||
});
|
||||
|
||||
it('formats subscription quotas and summarizes plan stats', () => {
|
||||
it("formats subscription quotas and summarizes plan stats", () => {
|
||||
const plans = [
|
||||
normalizeSubscriptionPlan({
|
||||
planCode: 'token-plan-lite',
|
||||
planName: '通用 Token Plan Lite',
|
||||
providerCN: '腾讯',
|
||||
planCode: "token-plan-lite",
|
||||
planName: "通用 Token Plan Lite",
|
||||
providerCN: "腾讯",
|
||||
listPrice: 39,
|
||||
quotaValue: 35000000,
|
||||
quotaUnit: 'tokens/month',
|
||||
modelScope: ['tc-code-latest', 'glm-5', 'glm-5.1'],
|
||||
quotaUnit: "tokens/month",
|
||||
modelScope: ["tc-code-latest", "glm-5", "glm-5.1"],
|
||||
}),
|
||||
normalizeSubscriptionPlan({
|
||||
planCode: 'hy-token-plan-max',
|
||||
planName: 'Hy Token Plan Max',
|
||||
providerCN: '腾讯',
|
||||
planCode: "hy-token-plan-max",
|
||||
planName: "Hy Token Plan Max",
|
||||
providerCN: "腾讯",
|
||||
listPrice: 468,
|
||||
quotaValue: 650000000,
|
||||
quotaUnit: 'tokens/month',
|
||||
quotaUnit: "tokens/month",
|
||||
contextWindow: 262144,
|
||||
modelScope: ['hy3-preview'],
|
||||
modelScope: ["hy3-preview"],
|
||||
}),
|
||||
].filter((plan): plan is NonNullable<typeof plan> => plan !== null)
|
||||
].filter((plan): plan is NonNullable<typeof plan> => plan !== null);
|
||||
|
||||
expect(formatSubscriptionQuota(plans[0].quotaValue, plans[0].quotaUnit)).toBe('3500万 Tokens/月')
|
||||
expect(formatSubscriptionQuota(plans[1].quotaValue, plans[1].quotaUnit)).toBe('6.5亿 Tokens/月')
|
||||
expect(
|
||||
formatSubscriptionQuota(plans[0].quotaValue, plans[0].quotaUnit),
|
||||
).toBe("3500万 Tokens/月");
|
||||
expect(
|
||||
formatSubscriptionQuota(plans[1].quotaValue, plans[1].quotaUnit),
|
||||
).toBe("6.5亿 Tokens/月");
|
||||
expect(summarizeSubscriptionPlans(plans)).toEqual({
|
||||
planCount: 2,
|
||||
providerCount: 1,
|
||||
minMonthlyPrice: 39,
|
||||
})
|
||||
})
|
||||
})
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("prefers the largest daily price swing model as pricing lead", () => {
|
||||
const models = [
|
||||
normalizeModel({
|
||||
id: "deepseek/deepseek-v4-flash",
|
||||
name: "DeepSeek-V4-Flash",
|
||||
provider_cn: "DeepSeek",
|
||||
pricing: { input: 0.3, output: 1.2 },
|
||||
data_confidence: "official",
|
||||
}),
|
||||
normalizeModel({
|
||||
id: "qwen/qwen-vl-max",
|
||||
name: "Qwen VL Max",
|
||||
provider_cn: "阿里云",
|
||||
pricing: { input: 0.8, output: 2.4 },
|
||||
data_confidence: "official",
|
||||
}),
|
||||
normalizeModel({
|
||||
id: "glm/glm-5",
|
||||
name: "GLM-5",
|
||||
provider_cn: "智谱",
|
||||
pricing: { input: 0, output: 0 },
|
||||
is_free: true,
|
||||
data_confidence: "official",
|
||||
}),
|
||||
].filter((model): model is NonNullable<typeof model> => model !== null);
|
||||
|
||||
const ranked = [...models].sort(
|
||||
(a, b) => b.outputPrice - a.outputPrice || b.inputPrice - a.inputPrice,
|
||||
);
|
||||
expect(ranked[0].name).toBe("Qwen VL Max");
|
||||
expect(formatPrice(ranked[0], "output")).toContain("2.4");
|
||||
});
|
||||
|
||||
it("extracts pricing-first report sections from markdown summary", async () => {
|
||||
const { normalizeLatestReportPayload } = await import("../pages/Dashboard");
|
||||
const report = normalizeLatestReportPayload({
|
||||
reportDate: "2026-05-25",
|
||||
status: "generated",
|
||||
modelCount: 504,
|
||||
summaryMD: [
|
||||
"## 今日结论",
|
||||
"> 今天最值得关注的是 qwen-vl-max 价格下降 18%,优先复查它是否改变默认选型与预算策略。",
|
||||
"- 证据: 主来源:pricing_history;输入价格较昨日下降 18%",
|
||||
"",
|
||||
"## 今日行动建议",
|
||||
"1. **先看 qwen-vl-max** ",
|
||||
"2. **复查 GLM-5** ",
|
||||
"",
|
||||
"## 今日价格新闻",
|
||||
"### 降价机会",
|
||||
"#### qwen-vl-max 成本下调 18%",
|
||||
"- 影响: 视觉模型价格下降已足以影响默认选型。",
|
||||
"### 平台活动",
|
||||
"#### DeepSeek-V4-Flash 进入活动窗口",
|
||||
"- 影响: 平台活动窗口出现后,值得重新评估低成本推理方案。",
|
||||
"",
|
||||
"## 场景推荐",
|
||||
"### 低成本编码",
|
||||
"- 主推荐: DeepSeek-V4-Flash",
|
||||
"### 中文通用",
|
||||
"- 主推荐: GLM-5",
|
||||
].join("\n"),
|
||||
markdownUrl: "/report.md",
|
||||
htmlUrl: "/report.html",
|
||||
updatedAt: "2026-05-25T10:00:00",
|
||||
});
|
||||
|
||||
expect(report.pricingLead).toContain("qwen-vl-max");
|
||||
expect(report.pricingLeadNote).toContain("pricing_history");
|
||||
expect(report.headlines[0].title).toContain("qwen-vl-max");
|
||||
expect(report.themes[0].title).toBe("降价机会");
|
||||
expect(report.themes[0].bullets[0]).toContain("qwen-vl-max");
|
||||
expect(report.themes[1].title).toBe("平台活动");
|
||||
});
|
||||
|
||||
@@ -110,35 +110,31 @@ export function normalizeModel(raw: any): Model | null {
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeModelList(raw: any) {
|
||||
const arr: any[] = Array.isArray(raw) ? raw : (raw?.models || [])
|
||||
return arr
|
||||
.map(normalizeModel)
|
||||
.filter((model: Model | null): model is Model => model !== null)
|
||||
}
|
||||
|
||||
async function loadRuntimeSnapshot() {
|
||||
try {
|
||||
const response = await fetch('/latest_models.json', { cache: 'no-store' })
|
||||
if (!response.ok) {
|
||||
return []
|
||||
}
|
||||
|
||||
const raw = await response.json()
|
||||
return normalizeModelList(raw)
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadFallbackModels() {
|
||||
const snapshot = await loadRuntimeSnapshot()
|
||||
if (snapshot.length > 0) {
|
||||
return snapshot
|
||||
// latest_models.json is a local runtime snapshot when present.
|
||||
// models.json is the committed fixture fallback kept in the repo.
|
||||
const sources = [
|
||||
() => import('../data/latest_models.json'),
|
||||
() => import('../data/models.json'),
|
||||
]
|
||||
|
||||
for (const load of sources) {
|
||||
try {
|
||||
const module = await load()
|
||||
const raw = module.default as any
|
||||
const arr: any[] = Array.isArray(raw) ? raw : (raw.models || [])
|
||||
const normalized = arr
|
||||
.map(normalizeModel)
|
||||
.filter((model: Model | null): model is Model => model !== null)
|
||||
if (normalized.length > 0) {
|
||||
return normalized
|
||||
}
|
||||
} catch {
|
||||
// 继续尝试下一个回退源
|
||||
}
|
||||
}
|
||||
|
||||
const module = await import('../data/models.json')
|
||||
return normalizeModelList(module.default)
|
||||
return []
|
||||
}
|
||||
|
||||
export function formatPrice(model: Model, kind: 'input' | 'output') {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import * as echarts from 'echarts'
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import * as echarts from "echarts";
|
||||
import {
|
||||
formatSubscriptionQuota,
|
||||
loadFallbackModels,
|
||||
@@ -10,112 +10,368 @@ import {
|
||||
summarizeSubscriptionPlans,
|
||||
type Model,
|
||||
type SubscriptionPlan,
|
||||
} from '../lib/models'
|
||||
} from "../lib/models";
|
||||
import {
|
||||
buildApiUnavailableNotice,
|
||||
buildFallbackNotice,
|
||||
detectRuntimeEnvironment,
|
||||
shouldUseLocalFallback,
|
||||
} from "../lib/runtimeVisibility";
|
||||
|
||||
type ReportHeadline = {
|
||||
label: string;
|
||||
title: string;
|
||||
summary: string;
|
||||
};
|
||||
|
||||
type ReportTheme = {
|
||||
title: string;
|
||||
bullets: string[];
|
||||
};
|
||||
|
||||
type LatestReport = {
|
||||
reportDate: string;
|
||||
status: string;
|
||||
modelCount: number;
|
||||
summaryMD: string;
|
||||
markdownUrl: string;
|
||||
htmlUrl: string;
|
||||
updatedAt: string;
|
||||
pricingLead: string;
|
||||
pricingLeadNote: string;
|
||||
headlines: ReportHeadline[];
|
||||
themes: ReportTheme[];
|
||||
};
|
||||
|
||||
function formatLocalReportDate(date: Date) {
|
||||
const year = date.getFullYear();
|
||||
const month = `${date.getMonth() + 1}`.padStart(2, "0");
|
||||
const day = `${date.getDate()}`.padStart(2, "0");
|
||||
return `${year}-${month}-${day}`;
|
||||
}
|
||||
|
||||
function buildFallbackLatestReport(): LatestReport {
|
||||
const reportDate = formatLocalReportDate(new Date());
|
||||
return {
|
||||
reportDate,
|
||||
status: "generated",
|
||||
modelCount: 0,
|
||||
summaryMD: "最新日报入口可用,后端元数据暂未返回摘要。",
|
||||
markdownUrl: `/reports/daily/daily_report_${reportDate}.md`,
|
||||
htmlUrl: `/reports/daily/html/daily_report_${reportDate}.html`,
|
||||
updatedAt: "",
|
||||
pricingLead: "当日价格异动摘要暂不可用",
|
||||
pricingLeadNote: "请直接打开 HTML 日报查看完整价格异动与主题分组。",
|
||||
headlines: [],
|
||||
themes: [],
|
||||
};
|
||||
}
|
||||
|
||||
function extractReportSections(summaryMD: string) {
|
||||
const normalized = summaryMD.replace(/\r\n/g, "\n");
|
||||
const lines = normalized.split("\n");
|
||||
const sections = new Map<string, string[]>();
|
||||
let current = "";
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.startsWith("## ")) {
|
||||
current = trimmed.slice(3).trim();
|
||||
sections.set(current, []);
|
||||
continue;
|
||||
}
|
||||
if (!current || trimmed === "") {
|
||||
continue;
|
||||
}
|
||||
sections.get(current)?.push(trimmed);
|
||||
}
|
||||
|
||||
return sections;
|
||||
}
|
||||
|
||||
function summarizeLatestReport(report: LatestReport) {
|
||||
if (report.pricingLead.trim()) {
|
||||
return report.pricingLead.trim();
|
||||
}
|
||||
if (report.summaryMD.trim()) {
|
||||
return report.summaryMD.trim();
|
||||
}
|
||||
if (report.modelCount > 0) {
|
||||
return `最新日报已生成,覆盖 ${report.modelCount} 个模型,可直接查看完整 HTML 页面。`;
|
||||
}
|
||||
return "最新日报入口已准备好,可直接打开 HTML 或 Markdown 查看。";
|
||||
}
|
||||
|
||||
export function normalizeLatestReportPayload(payload: any): LatestReport {
|
||||
const summaryMD =
|
||||
typeof payload?.summaryMD === "string" ? payload.summaryMD : "";
|
||||
const sections = extractReportSections(summaryMD);
|
||||
const conclusion = sections.get("今日结论") ?? [];
|
||||
const changes = sections.get("今日价格新闻") ?? [];
|
||||
const sceneLines = sections.get("场景推荐") ?? [];
|
||||
const actionLines = sections.get("今日行动建议") ?? [];
|
||||
|
||||
const pricingLead =
|
||||
conclusion[0]?.replace(/^>\s*/, "") || "当日价格异动摘要暂不可用";
|
||||
const pricingLeadNote =
|
||||
changes
|
||||
.find((line) => line.startsWith("- 证据:"))
|
||||
?.replace("- 证据:", "")
|
||||
.trim() ||
|
||||
conclusion
|
||||
.find((line) => line.startsWith("- 证据:"))
|
||||
?.replace("- 证据:", "")
|
||||
.trim() ||
|
||||
"请直接打开 HTML 日报查看完整价格异动与主题分组。";
|
||||
|
||||
const headlines = actionLines
|
||||
.filter((line) => /^\d+\./.test(line))
|
||||
.slice(0, 3)
|
||||
.map((line) => {
|
||||
const title = line
|
||||
.replace(/^\d+\.\s*\*\*/, "")
|
||||
.replace(/\*\*\s*$/, "")
|
||||
.trim();
|
||||
return {
|
||||
label: "今日动作",
|
||||
title,
|
||||
summary: "围绕当天最重要的价格异动与选型影响整理。",
|
||||
};
|
||||
});
|
||||
|
||||
const sceneThemes: ReportTheme[] = [];
|
||||
let currentSceneTheme: ReportTheme | null = null;
|
||||
for (const line of sceneLines) {
|
||||
if (line.startsWith("### ")) {
|
||||
currentSceneTheme = { title: line.slice(4).trim(), bullets: [] };
|
||||
sceneThemes.push(currentSceneTheme);
|
||||
continue;
|
||||
}
|
||||
if (currentSceneTheme && line.startsWith("- ")) {
|
||||
currentSceneTheme.bullets.push(line.slice(2).trim());
|
||||
}
|
||||
}
|
||||
|
||||
const pricingThemes: ReportTheme[] = [];
|
||||
let currentPricingTheme: ReportTheme | null = null;
|
||||
for (const line of changes) {
|
||||
if (line.startsWith("### ")) {
|
||||
currentPricingTheme = { title: line.slice(4).trim(), bullets: [] };
|
||||
pricingThemes.push(currentPricingTheme);
|
||||
continue;
|
||||
}
|
||||
if (currentPricingTheme && line.startsWith("#### ")) {
|
||||
currentPricingTheme.bullets.push(line.slice(5).trim());
|
||||
continue;
|
||||
}
|
||||
if (currentPricingTheme && line.startsWith("- 影响: ")) {
|
||||
currentPricingTheme.bullets.push(line.replace("- 影响: ", ""));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
reportDate: payload?.reportDate,
|
||||
status: payload?.status || "generated",
|
||||
modelCount: Number(payload?.modelCount || 0),
|
||||
summaryMD,
|
||||
markdownUrl: payload?.markdownUrl,
|
||||
htmlUrl: payload?.htmlUrl,
|
||||
updatedAt: payload?.updatedAt || "",
|
||||
pricingLead,
|
||||
pricingLeadNote,
|
||||
headlines,
|
||||
themes: pricingThemes.length > 0 ? pricingThemes : sceneThemes,
|
||||
};
|
||||
}
|
||||
|
||||
function reportThemeBadge(themeTitle: string) {
|
||||
if (themeTitle.includes("降价")) {
|
||||
return { icon: "↓", label: "Opportunity", tone: "tone-success" };
|
||||
}
|
||||
if (themeTitle.includes("涨价")) {
|
||||
return { icon: "↑", label: "Warning", tone: "tone-caution" };
|
||||
}
|
||||
if (themeTitle.includes("活动")) {
|
||||
return { icon: "✦", label: "Campaign", tone: "tone-promo" };
|
||||
}
|
||||
return { icon: "•", label: "Signal", tone: "tone-neutral" };
|
||||
}
|
||||
|
||||
|
||||
function Dashboard() {
|
||||
const chartRef = useRef<HTMLDivElement>(null)
|
||||
const [modelCount, setModelCount] = useState(0)
|
||||
const [providerCount, setProviderCount] = useState(0)
|
||||
const [cnyCount, setCnyCount] = useState(0)
|
||||
const [subscriptionPlans, setSubscriptionPlans] = useState<SubscriptionPlan[]>([])
|
||||
const [planCount, setPlanCount] = useState(0)
|
||||
const [planMinPrice, setPlanMinPrice] = useState(0)
|
||||
const chartRef = useRef<HTMLDivElement>(null);
|
||||
const [modelCount, setModelCount] = useState(0);
|
||||
const [providerCount, setProviderCount] = useState(0);
|
||||
const [cnyCount, setCnyCount] = useState(0);
|
||||
const [subscriptionPlans, setSubscriptionPlans] = useState<
|
||||
SubscriptionPlan[]
|
||||
>([]);
|
||||
const [planCount, setPlanCount] = useState(0);
|
||||
const [planMinPrice, setPlanMinPrice] = useState(0);
|
||||
const [latestReport, setLatestReport] = useState<LatestReport | null>(null);
|
||||
const [modelsFallback, setModelsFallback] = useState(false);
|
||||
const [modelsUnavailable, setModelsUnavailable] = useState("");
|
||||
const [reportFallback, setReportFallback] = useState(false);
|
||||
const [reportUnavailable, setReportUnavailable] = useState("");
|
||||
const runtime = detectRuntimeEnvironment();
|
||||
const modelsFallbackNotice = buildFallbackNotice("models", runtime);
|
||||
const modelsUnavailableNotice = buildApiUnavailableNotice("models", runtime);
|
||||
const reportFallbackNotice = buildFallbackNotice("latestReport", runtime);
|
||||
const reportUnavailableNotice = buildApiUnavailableNotice(
|
||||
"latestReport",
|
||||
runtime,
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
let chart: echarts.ECharts | null = null
|
||||
let disposed = false
|
||||
let chart: echarts.ECharts | null = null;
|
||||
let disposed = false;
|
||||
|
||||
const renderChart = (models: Model[]) => {
|
||||
if (!chartRef.current) {
|
||||
return
|
||||
return;
|
||||
}
|
||||
|
||||
chart = echarts.init(chartRef.current)
|
||||
chart = echarts.init(chartRef.current);
|
||||
const option: echarts.EChartsOption = {
|
||||
title: { text: '厂商模型分布', left: 'center' },
|
||||
tooltip: { trigger: 'item' },
|
||||
series: [{
|
||||
type: 'pie',
|
||||
radius: '60%',
|
||||
data: providerDistribution(models),
|
||||
emphasis: {
|
||||
itemStyle: {
|
||||
shadowBlur: 10,
|
||||
shadowOffsetX: 0,
|
||||
shadowColor: 'rgba(0, 0, 0, 0.5)'
|
||||
}
|
||||
}
|
||||
}]
|
||||
}
|
||||
chart.setOption(option)
|
||||
}
|
||||
title: { text: "厂商模型分布", left: "center" },
|
||||
tooltip: { trigger: "item" },
|
||||
series: [
|
||||
{
|
||||
type: "pie",
|
||||
radius: "60%",
|
||||
data: providerDistribution(models),
|
||||
emphasis: {
|
||||
itemStyle: {
|
||||
shadowBlur: 10,
|
||||
shadowOffsetX: 0,
|
||||
shadowColor: "rgba(0, 0, 0, 0.5)",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
chart.setOption(option);
|
||||
};
|
||||
|
||||
const updateStats = (models: Model[]) => {
|
||||
const summary = summarizeModels(models)
|
||||
setModelCount(summary.modelCount)
|
||||
setProviderCount(summary.providerCount)
|
||||
setCnyCount(summary.cnyCount)
|
||||
renderChart(models)
|
||||
}
|
||||
const summary = summarizeModels(models);
|
||||
setModelCount(summary.modelCount);
|
||||
setProviderCount(summary.providerCount);
|
||||
setCnyCount(summary.cnyCount);
|
||||
renderChart(models);
|
||||
};
|
||||
|
||||
const updatePlans = (plans: SubscriptionPlan[]) => {
|
||||
const summary = summarizeSubscriptionPlans(plans)
|
||||
setSubscriptionPlans(plans)
|
||||
setPlanCount(summary.planCount)
|
||||
setPlanMinPrice(summary.minMonthlyPrice)
|
||||
}
|
||||
const summary = summarizeSubscriptionPlans(plans);
|
||||
setSubscriptionPlans(plans);
|
||||
setPlanCount(summary.planCount);
|
||||
setPlanMinPrice(summary.minMonthlyPrice);
|
||||
};
|
||||
|
||||
const loadModels = async () => {
|
||||
try {
|
||||
const response = await fetch('/api/v1/models')
|
||||
const response = await fetch("/api/v1/models");
|
||||
if (!response.ok) {
|
||||
throw new Error(`models request failed: ${response.status}`)
|
||||
throw new Error(`models request failed: ${response.status}`);
|
||||
}
|
||||
const payload = await response.json()
|
||||
const rawModels: any[] = Array.isArray(payload?.data) ? payload.data : []
|
||||
const payload = await response.json();
|
||||
const rawModels: any[] = Array.isArray(payload?.data)
|
||||
? payload.data
|
||||
: [];
|
||||
const models = rawModels
|
||||
.map(normalizeModel)
|
||||
.filter((model: Model | null): model is Model => model !== null)
|
||||
.filter((model: Model | null): model is Model => model !== null);
|
||||
if (!disposed) {
|
||||
updateStats(models)
|
||||
updateStats(models);
|
||||
setModelsFallback(false);
|
||||
setModelsUnavailable("");
|
||||
}
|
||||
} catch {
|
||||
const fallback = await loadFallbackModels()
|
||||
if (!disposed) {
|
||||
updateStats(fallback)
|
||||
if (shouldUseLocalFallback("models", runtime)) {
|
||||
const fallback = await loadFallbackModels();
|
||||
if (!disposed) {
|
||||
updateStats(fallback);
|
||||
setModelsFallback(fallback.length > 0);
|
||||
setModelsUnavailable(
|
||||
fallback.length === 0 ? modelsUnavailableNotice : "",
|
||||
);
|
||||
}
|
||||
} else if (!disposed) {
|
||||
updateStats([]);
|
||||
setModelsFallback(false);
|
||||
setModelsUnavailable(modelsUnavailableNotice);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const loadSubscriptionPlans = async () => {
|
||||
try {
|
||||
const response = await fetch('/api/v1/subscription-plans')
|
||||
const response = await fetch("/api/v1/subscription-plans");
|
||||
if (!response.ok) {
|
||||
throw new Error(`subscription plans request failed: ${response.status}`)
|
||||
throw new Error(
|
||||
`subscription plans request failed: ${response.status}`,
|
||||
);
|
||||
}
|
||||
const payload = await response.json()
|
||||
const rawPlans: any[] = Array.isArray(payload?.data) ? payload.data : []
|
||||
const payload = await response.json();
|
||||
const rawPlans: any[] = Array.isArray(payload?.data)
|
||||
? payload.data
|
||||
: [];
|
||||
const plans = rawPlans
|
||||
.map(normalizeSubscriptionPlan)
|
||||
.filter((plan: SubscriptionPlan | null): plan is SubscriptionPlan => plan !== null)
|
||||
.filter(
|
||||
(plan: SubscriptionPlan | null): plan is SubscriptionPlan =>
|
||||
plan !== null,
|
||||
);
|
||||
if (!disposed) {
|
||||
updatePlans(plans)
|
||||
updatePlans(plans);
|
||||
}
|
||||
} catch {
|
||||
if (!disposed) {
|
||||
updatePlans([])
|
||||
updatePlans([]);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void loadModels()
|
||||
void loadSubscriptionPlans()
|
||||
const loadLatestReport = async () => {
|
||||
try {
|
||||
const response = await fetch("/api/v1/reports/latest");
|
||||
if (!response.ok) {
|
||||
throw new Error(`latest report request failed: ${response.status}`);
|
||||
}
|
||||
const payload = await response.json();
|
||||
const report = payload?.data;
|
||||
if (!report?.reportDate || !report?.htmlUrl || !report?.markdownUrl) {
|
||||
throw new Error("latest report payload invalid");
|
||||
}
|
||||
if (!disposed) {
|
||||
setLatestReport(normalizeLatestReportPayload(payload?.data));
|
||||
setReportFallback(false);
|
||||
setReportUnavailable("");
|
||||
}
|
||||
} catch {
|
||||
if (shouldUseLocalFallback("latestReport", runtime)) {
|
||||
if (!disposed) {
|
||||
setLatestReport(buildFallbackLatestReport());
|
||||
setReportFallback(true);
|
||||
setReportUnavailable("");
|
||||
}
|
||||
} else if (!disposed) {
|
||||
setLatestReport(null);
|
||||
setReportFallback(false);
|
||||
setReportUnavailable(reportUnavailableNotice);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void loadModels();
|
||||
void loadSubscriptionPlans();
|
||||
void loadLatestReport();
|
||||
|
||||
return () => {
|
||||
disposed = true
|
||||
chart?.dispose()
|
||||
}
|
||||
}, [])
|
||||
disposed = true;
|
||||
chart?.dispose();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="dashboard">
|
||||
@@ -138,9 +394,126 @@ function Dashboard() {
|
||||
<div className="stat-label">腾讯云套餐</div>
|
||||
</div>
|
||||
</div>
|
||||
{modelsFallback && (
|
||||
<div className="runtime-warning" role="alert">
|
||||
{modelsFallbackNotice}
|
||||
</div>
|
||||
)}
|
||||
{modelsUnavailable && (
|
||||
<div className="runtime-error" role="alert">
|
||||
{modelsUnavailable}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="chart-container">
|
||||
<div ref={chartRef} style={{ width: '100%', height: '400px' }} />
|
||||
<div ref={chartRef} style={{ width: "100%", height: "400px" }} />
|
||||
</div>
|
||||
<section className="report-section report-priority-section">
|
||||
<div className="report-header">
|
||||
<div>
|
||||
<h3>📰 今日价格异动日报</h3>
|
||||
<p>
|
||||
先看当天最值得改默认选型的一条价格信息,再按主题浏览价格新闻。
|
||||
</p>
|
||||
</div>
|
||||
{latestReport && (
|
||||
<span
|
||||
className={`report-status ${latestReport.status === "generated" ? "report-status-generated" : "report-status-other"}`}
|
||||
>
|
||||
{latestReport.status}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{latestReport ? (
|
||||
<div className="report-card report-priority-card">
|
||||
<div className="report-hero report-hero-priority">
|
||||
<div className="report-eyebrow">今日首要价格异动</div>
|
||||
<div className="report-summary">
|
||||
{summarizeLatestReport(latestReport)}
|
||||
</div>
|
||||
<div className="report-evidence">
|
||||
{latestReport.pricingLeadNote}
|
||||
</div>
|
||||
</div>
|
||||
<div className="report-meta">
|
||||
<span>报告日期 {latestReport.reportDate}</span>
|
||||
{latestReport.modelCount > 0 && (
|
||||
<span>{latestReport.modelCount} 个模型</span>
|
||||
)}
|
||||
{latestReport.updatedAt && (
|
||||
<span>更新于 {latestReport.updatedAt}</span>
|
||||
)}
|
||||
</div>
|
||||
{latestReport.headlines.length > 0 && (
|
||||
<div className="report-news-grid">
|
||||
{latestReport.headlines.map((item) => (
|
||||
<article key={item.title} className="report-news-card">
|
||||
<div className="report-news-label">{item.label}</div>
|
||||
<div className="report-news-title">{item.title}</div>
|
||||
<div className="report-news-summary">{item.summary}</div>
|
||||
</article>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
{latestReport.themes.length > 0 && (
|
||||
<div className="report-theme-grid">
|
||||
{latestReport.themes.map((theme) => {
|
||||
const badge = reportThemeBadge(theme.title);
|
||||
return (
|
||||
<article
|
||||
key={theme.title}
|
||||
className={`report-theme-card ${badge.tone}`}
|
||||
>
|
||||
<div className="report-theme-badge">
|
||||
<span className="report-theme-badge-icon">{badge.icon}</span>
|
||||
<span className="report-theme-badge-label">{badge.label}</span>
|
||||
</div>
|
||||
<div className="report-theme-title">{theme.title}</div>
|
||||
<ul>
|
||||
{theme.bullets.slice(0, 3).map((bullet) => (
|
||||
<li key={bullet}>{bullet}</li>
|
||||
))}
|
||||
</ul>
|
||||
</article>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
<div className="report-actions">
|
||||
<a
|
||||
className="report-link report-link-primary"
|
||||
href={latestReport.htmlUrl}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
>
|
||||
查看 HTML 日报
|
||||
</a>
|
||||
<a
|
||||
className="report-link"
|
||||
href={latestReport.markdownUrl}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
>
|
||||
查看 Markdown
|
||||
</a>
|
||||
</div>
|
||||
{reportFallback && (
|
||||
<div className="report-note runtime-warning-inline">
|
||||
{reportFallbackNotice}
|
||||
</div>
|
||||
)}
|
||||
{reportUnavailable && (
|
||||
<div className="report-note runtime-error-inline">
|
||||
{reportUnavailable}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<div className="data-empty">
|
||||
最新日报当前不可用,请先恢复后端 API。
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
<section className="subscription-section">
|
||||
<div className="subscription-header">
|
||||
<div>
|
||||
@@ -168,18 +541,28 @@ function Dashboard() {
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{subscriptionPlans.map(plan => (
|
||||
{subscriptionPlans.map((plan) => (
|
||||
<tr key={plan.planCode}>
|
||||
<td>
|
||||
<div className="plan-name">{plan.planName}</div>
|
||||
<div className="plan-meta">{plan.operatorCN || plan.operator}</div>
|
||||
<div className="plan-meta">
|
||||
{plan.operatorCN || plan.operator}
|
||||
</div>
|
||||
</td>
|
||||
<td>¥{plan.listPrice.toFixed(2)}/月</td>
|
||||
<td>{formatSubscriptionQuota(plan.quotaValue, plan.quotaUnit)}</td>
|
||||
<td>{plan.contextWindow > 0 ? `${Math.round(plan.contextWindow / 1024)}K` : '-'}</td>
|
||||
<td>
|
||||
{formatSubscriptionQuota(plan.quotaValue, plan.quotaUnit)}
|
||||
</td>
|
||||
<td>
|
||||
{plan.contextWindow > 0
|
||||
? `${Math.round(plan.contextWindow / 1024)}K`
|
||||
: "-"}
|
||||
</td>
|
||||
<td>
|
||||
<div>{plan.modelCount} 个模型</div>
|
||||
{plan.modelPreview && <div className="plan-meta">{plan.modelPreview}</div>}
|
||||
{plan.modelPreview && (
|
||||
<div className="plan-meta">{plan.modelPreview}</div>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
@@ -188,7 +571,7 @@ function Dashboard() {
|
||||
)}
|
||||
</section>
|
||||
</div>
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export default Dashboard
|
||||
export default Dashboard;
|
||||
|
||||
@@ -1,148 +1,298 @@
|
||||
import { useEffect, useMemo, useState } from 'react'
|
||||
import { formatPrice, loadFallbackModels, normalizeModel, type Model } from '../lib/models'
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import {
|
||||
formatPrice,
|
||||
loadFallbackModels,
|
||||
normalizeModel,
|
||||
type Model,
|
||||
} from "../lib/models";
|
||||
import {
|
||||
buildApiUnavailableNotice,
|
||||
buildFallbackNotice,
|
||||
detectRuntimeEnvironment,
|
||||
shouldUseLocalFallback,
|
||||
} from "../lib/runtimeVisibility";
|
||||
|
||||
type SortField = 'name' | 'inputPrice' | 'outputPrice' | 'contextLength'
|
||||
type SortOrder = 'asc' | 'desc'
|
||||
type SortField = "name" | "inputPrice" | "outputPrice" | "contextLength";
|
||||
|
||||
const PAGE_SIZE = 5
|
||||
type SortOrder = "asc" | "desc";
|
||||
|
||||
const PAGE_SIZE = 5;
|
||||
|
||||
function Explorer() {
|
||||
const [models, setModels] = useState<Model[]>([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [page, setPage] = useState(1)
|
||||
const [sortField, setSortField] = useState<SortField>('inputPrice')
|
||||
const [sortOrder, setSortOrder] = useState<SortOrder>('asc')
|
||||
const [providerFilter, setProviderFilter] = useState<string>('')
|
||||
const [modalityFilter, setModalityFilter] = useState<string>('')
|
||||
const [models, setModels] = useState<Model[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [page, setPage] = useState(1);
|
||||
const [sortField, setSortField] = useState<SortField>("inputPrice");
|
||||
const [sortOrder, setSortOrder] = useState<SortOrder>("asc");
|
||||
const [providerFilter, setProviderFilter] = useState<string>("");
|
||||
const [modalityFilter, setModalityFilter] = useState<string>("");
|
||||
const [modelsFallback, setModelsFallback] = useState(false);
|
||||
const [modelsUnavailable, setModelsUnavailable] = useState("");
|
||||
const runtime = detectRuntimeEnvironment();
|
||||
const fallbackNotice = buildFallbackNotice("models", runtime);
|
||||
const unavailableNotice = buildApiUnavailableNotice("models", runtime);
|
||||
|
||||
useEffect(() => {
|
||||
// 从API加载数据
|
||||
fetch('/api/v1/models')
|
||||
.then(r => r.json())
|
||||
.then(data => {
|
||||
const rawModels: any[] = Array.isArray(data?.data) ? data.data : []
|
||||
fetch("/api/v1/models")
|
||||
.then(async (r) => {
|
||||
if (!r.ok) {
|
||||
throw new Error(`models request failed: ${r.status}`);
|
||||
}
|
||||
return r.json();
|
||||
})
|
||||
.then((data) => {
|
||||
const rawModels: any[] = Array.isArray(data?.data) ? data.data : [];
|
||||
const normalized = rawModels
|
||||
.map(normalizeModel)
|
||||
.filter((model: Model | null): model is Model => model !== null)
|
||||
setModels(normalized)
|
||||
setLoading(false)
|
||||
.filter((model: Model | null): model is Model => model !== null);
|
||||
setModels(normalized);
|
||||
setModelsFallback(false);
|
||||
setModelsUnavailable("");
|
||||
setLoading(false);
|
||||
})
|
||||
.catch(async () => {
|
||||
// 降级:使用本地静态数据
|
||||
const fallback = await loadFallbackModels()
|
||||
setModels(fallback)
|
||||
setLoading(false)
|
||||
})
|
||||
}, [])
|
||||
if (shouldUseLocalFallback("models", runtime)) {
|
||||
const fallback = await loadFallbackModels();
|
||||
setModels(fallback);
|
||||
setModelsFallback(fallback.length > 0);
|
||||
setModelsUnavailable(fallback.length === 0 ? unavailableNotice : "");
|
||||
} else {
|
||||
setModels([]);
|
||||
setModelsFallback(false);
|
||||
setModelsUnavailable(unavailableNotice);
|
||||
}
|
||||
setLoading(false);
|
||||
});
|
||||
}, []);
|
||||
|
||||
// 动态提取厂商列表
|
||||
const providers = useMemo(() => {
|
||||
const set = new Set<string>()
|
||||
models.forEach(m => {
|
||||
if (m.providerCN && m.providerCN !== 'Unknown') {
|
||||
set.add(m.providerCN)
|
||||
const set = new Set<string>();
|
||||
models.forEach((m) => {
|
||||
if (m.providerCN && m.providerCN !== "Unknown") {
|
||||
set.add(m.providerCN);
|
||||
}
|
||||
})
|
||||
return Array.from(set).sort()
|
||||
}, [models])
|
||||
});
|
||||
return Array.from(set).sort();
|
||||
}, [models]);
|
||||
|
||||
// 排序+筛选
|
||||
const filtered = useMemo(() => {
|
||||
let result = [...models]
|
||||
let result = [...models];
|
||||
if (providerFilter) {
|
||||
result = result.filter(m => m.providerCN === providerFilter)
|
||||
result = result.filter((m) => m.providerCN === providerFilter);
|
||||
}
|
||||
if (modalityFilter) {
|
||||
result = result.filter(m => m.modality === modalityFilter)
|
||||
result = result.filter((m) => m.modality === modalityFilter);
|
||||
}
|
||||
result.sort((a, b) => {
|
||||
const aVal = a[sortField]
|
||||
const bVal = b[sortField]
|
||||
if (typeof aVal === 'string') {
|
||||
return sortOrder === 'asc'
|
||||
const aVal = a[sortField];
|
||||
const bVal = b[sortField];
|
||||
if (typeof aVal === "string") {
|
||||
return sortOrder === "asc"
|
||||
? aVal.localeCompare(bVal as string)
|
||||
: (bVal as string).localeCompare(aVal)
|
||||
: (bVal as string).localeCompare(aVal);
|
||||
}
|
||||
return sortOrder === 'asc'
|
||||
return sortOrder === "asc"
|
||||
? (aVal as number) - (bVal as number)
|
||||
: (bVal as number) - (aVal as number)
|
||||
})
|
||||
return result
|
||||
}, [models, sortField, sortOrder, providerFilter, modalityFilter])
|
||||
: (bVal as number) - (aVal as number);
|
||||
});
|
||||
return result;
|
||||
}, [models, sortField, sortOrder, providerFilter, modalityFilter]);
|
||||
|
||||
const totalPages = Math.max(1, Math.ceil(filtered.length / PAGE_SIZE))
|
||||
const paginated = filtered.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE)
|
||||
const totalPages = Math.max(1, Math.ceil(filtered.length / PAGE_SIZE));
|
||||
const paginated = filtered.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE);
|
||||
const pricingFocus = filtered[0] ?? null;
|
||||
const pricingBoard = filtered.slice(0, 3);
|
||||
|
||||
const toggleSort = (field: SortField) => {
|
||||
if (sortField === field) {
|
||||
setSortOrder(o => o === 'asc' ? 'desc' : 'asc')
|
||||
setSortOrder((o) => (o === "asc" ? "desc" : "asc"));
|
||||
} else {
|
||||
setSortField(field)
|
||||
setSortOrder('asc')
|
||||
setSortField(field);
|
||||
setSortOrder("asc");
|
||||
}
|
||||
setPage(1)
|
||||
}
|
||||
setPage(1);
|
||||
};
|
||||
|
||||
if (loading) return <div className="loading">加载中...</div>
|
||||
if (loading) return <div className="loading">加载中...</div>;
|
||||
|
||||
return (
|
||||
<div className="explorer">
|
||||
<h2>🔍 模型 Explorer</h2>
|
||||
<div className="explorer explorer-editorial">
|
||||
<div className="explorer-hero">
|
||||
<div>
|
||||
<div className="explorer-kicker">模型价格查询</div>
|
||||
<h2>今天先看最值得改默认选型的价格</h2>
|
||||
<p>把价格异动、平台来源和上下文能力放到同一屏,先决策,再看全表。</p>
|
||||
</div>
|
||||
<div className="explorer-hero-meta">
|
||||
<span>模型池 {filtered.length}</span>
|
||||
<span>厂商 {providers.length}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="filters">
|
||||
<select value={providerFilter} onChange={e => { setProviderFilter(e.target.value); setPage(1) }}>
|
||||
<div className="filters filters-editorial">
|
||||
<select
|
||||
value={providerFilter}
|
||||
onChange={(e) => {
|
||||
setProviderFilter(e.target.value);
|
||||
setPage(1);
|
||||
}}
|
||||
>
|
||||
<option value="">全部厂商</option>
|
||||
{providers.map(p => <option key={p} value={p}>{p}</option>)}
|
||||
{providers.map((p) => (
|
||||
<option key={p} value={p}>
|
||||
{p}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<select value={modalityFilter} onChange={e => { setModalityFilter(e.target.value); setPage(1) }}>
|
||||
<select
|
||||
value={modalityFilter}
|
||||
onChange={(e) => {
|
||||
setModalityFilter(e.target.value);
|
||||
setPage(1);
|
||||
}}
|
||||
>
|
||||
<option value="">全部类型</option>
|
||||
<option value="text">文本</option>
|
||||
<option value="multimodal">多模态</option>
|
||||
</select>
|
||||
<span className="count">共 {filtered.length} 个模型</span>
|
||||
</div>
|
||||
{modelsFallback && (
|
||||
<div className="runtime-warning" role="alert">
|
||||
{fallbackNotice}
|
||||
</div>
|
||||
)}
|
||||
{modelsUnavailable && (
|
||||
<div className="runtime-error" role="alert">
|
||||
{modelsUnavailable}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<table className="model-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th onClick={() => toggleSort('name')}>模型 {sortField === 'name' && (sortOrder === 'asc' ? '▲' : '▼')}</th>
|
||||
<th>厂商</th>
|
||||
<th>状态</th>
|
||||
<th onClick={() => toggleSort('inputPrice')}>输入价格 {sortField === 'inputPrice' && (sortOrder === 'asc' ? '▲' : '▼')}</th>
|
||||
<th onClick={() => toggleSort('outputPrice')}>输出价格 {sortField === 'outputPrice' && (sortOrder === 'asc' ? '▲' : '▼')}</th>
|
||||
<th onClick={() => toggleSort('contextLength')}>上下文 {sortField === 'contextLength' && (sortOrder === 'asc' ? '▲' : '▼')}</th>
|
||||
<th>类型</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{paginated.map(m => (
|
||||
<tr key={m.id} className={`${m.isFree ? 'free' : ''} ${m.stale ? 'stale' : ''}`.trim()}>
|
||||
<td>
|
||||
<div className="model-name">{m.name || m.id}</div>
|
||||
<div className="model-id">{m.id}</div>
|
||||
</td>
|
||||
<td>{m.providerCN || m.provider}</td>
|
||||
<td>
|
||||
<span className={`status-badge ${m.stale ? 'status-stale' : 'status-fresh'}`}>
|
||||
{m.stale ? 'stale' : m.dataConfidence}
|
||||
</span>
|
||||
</td>
|
||||
<td>{formatPrice(m, 'input')}</td>
|
||||
<td>{formatPrice(m, 'output')}</td>
|
||||
<td>{(m.contextLength / 1000).toFixed(0)}K</td>
|
||||
<td>{m.modality}</td>
|
||||
</tr>
|
||||
{pricingFocus && (
|
||||
<section className="pricing-focus-card">
|
||||
<div className="pricing-focus-header">
|
||||
<div>
|
||||
<div className="explorer-kicker">今日查价优先位</div>
|
||||
<h3>{pricingFocus.name || pricingFocus.id}</h3>
|
||||
<p>
|
||||
{pricingFocus.providerCN || pricingFocus.provider} ·{" "}
|
||||
{pricingFocus.modality} ·{" "}
|
||||
{(pricingFocus.contextLength / 1000).toFixed(0)}K 上下文
|
||||
</p>
|
||||
</div>
|
||||
<span
|
||||
className={`status-badge ${pricingFocus.stale ? "status-stale" : "status-fresh"}`}
|
||||
>
|
||||
{pricingFocus.stale ? "stale" : pricingFocus.dataConfidence}
|
||||
</span>
|
||||
</div>
|
||||
<div className="pricing-focus-prices">
|
||||
<div>
|
||||
<span>输入价格</span>
|
||||
<strong>{formatPrice(pricingFocus, "input")}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span>输出价格</span>
|
||||
<strong>{formatPrice(pricingFocus, "output")}</strong>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
)}
|
||||
|
||||
{pricingBoard.length > 0 && (
|
||||
<section className="pricing-board-grid">
|
||||
{pricingBoard.map((model) => (
|
||||
<article key={model.id} className="pricing-board-card">
|
||||
<div className="pricing-board-title">
|
||||
{model.name || model.id}
|
||||
</div>
|
||||
<div className="pricing-board-meta">
|
||||
{model.providerCN || model.provider} · {model.modality}
|
||||
</div>
|
||||
<div className="pricing-board-price">
|
||||
{formatPrice(model, "input")} / {formatPrice(model, "output")}
|
||||
</div>
|
||||
</article>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
)}
|
||||
|
||||
{paginated.length === 0 ? (
|
||||
<div className="data-empty">当前暂无可展示的模型数据。</div>
|
||||
) : (
|
||||
<table className="model-table model-table-editorial">
|
||||
<thead>
|
||||
<tr>
|
||||
<th onClick={() => toggleSort("name")}>
|
||||
模型 {sortField === "name" && (sortOrder === "asc" ? "▲" : "▼")}
|
||||
</th>
|
||||
<th>厂商</th>
|
||||
<th>状态</th>
|
||||
<th onClick={() => toggleSort("inputPrice")}>
|
||||
输入价格{" "}
|
||||
{sortField === "inputPrice" &&
|
||||
(sortOrder === "asc" ? "▲" : "▼")}
|
||||
</th>
|
||||
<th onClick={() => toggleSort("outputPrice")}>
|
||||
输出价格{" "}
|
||||
{sortField === "outputPrice" &&
|
||||
(sortOrder === "asc" ? "▲" : "▼")}
|
||||
</th>
|
||||
<th onClick={() => toggleSort("contextLength")}>
|
||||
上下文{" "}
|
||||
{sortField === "contextLength" &&
|
||||
(sortOrder === "asc" ? "▲" : "▼")}
|
||||
</th>
|
||||
<th>类型</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{paginated.map((m) => (
|
||||
<tr
|
||||
key={m.id}
|
||||
className={`${m.isFree ? "free" : ""} ${m.stale ? "stale" : ""}`.trim()}
|
||||
>
|
||||
<td>
|
||||
<div className="model-name">{m.name || m.id}</div>
|
||||
<div className="model-id">{m.id}</div>
|
||||
</td>
|
||||
<td>{m.providerCN || m.provider}</td>
|
||||
<td>
|
||||
<span
|
||||
className={`status-badge ${m.stale ? "status-stale" : "status-fresh"}`}
|
||||
>
|
||||
{m.stale ? "stale" : m.dataConfidence}
|
||||
</span>
|
||||
</td>
|
||||
<td>{formatPrice(m, "input")}</td>
|
||||
<td>{formatPrice(m, "output")}</td>
|
||||
<td>{(m.contextLength / 1000).toFixed(0)}K</td>
|
||||
<td>{m.modality}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
|
||||
<div className="pagination">
|
||||
<button disabled={page === 1} onClick={() => setPage(p => p - 1)}>上一页</button>
|
||||
<span>第 {page} / {totalPages} 页</span>
|
||||
<button disabled={page === totalPages} onClick={() => setPage(p => p + 1)}>下一页</button>
|
||||
<button disabled={page === 1} onClick={() => setPage((p) => p - 1)}>
|
||||
上一页
|
||||
</button>
|
||||
<span>
|
||||
第 {page} / {totalPages} 页
|
||||
</span>
|
||||
<button
|
||||
disabled={page === totalPages}
|
||||
onClick={() => setPage((p) => p + 1)}
|
||||
>
|
||||
下一页
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export default Explorer
|
||||
export default Explorer;
|
||||
|
||||
@@ -3,19 +3,13 @@ set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
if [[ -f ".env.local" ]]; then
|
||||
# shellcheck disable=SC1091
|
||||
source ".env.local"
|
||||
fi
|
||||
if [[ -f ".env" ]]; then
|
||||
# shellcheck disable=SC1091
|
||||
source ".env"
|
||||
fi
|
||||
source scripts/report_utils.sh
|
||||
while IFS= read -r kv; do export "$kv"; done < <("$ROOT_DIR/scripts/load_project_env.sh" ".env.local")
|
||||
while IFS= read -r kv; do key="${kv%%=*}"; [[ -n "$key" && -n "${!key:-}" ]] && continue; export "$kv"; done < <("$ROOT_DIR/scripts/load_project_env.sh" ".env")
|
||||
|
||||
DB_URL="${DATABASE_URL:-host=/var/run/postgresql dbname=llm_intelligence user=long sslmode=disable}"
|
||||
TODAY="$(date +%Y-%m-%d)"
|
||||
REPORT_PATH="reports/daily/daily_report_${TODAY}.md"
|
||||
TODAY="$(report_date_value)"
|
||||
REPORT_PATH="$(report_markdown_path "$TODAY")"
|
||||
|
||||
psql "$DB_URL" -Atqc "select 1;" >/dev/null
|
||||
|
||||
@@ -24,7 +18,7 @@ if [[ -f "$REPORT_PATH" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
LATEST_REPORT="$(find reports/daily -maxdepth 1 -type f -name 'daily_report_*.md' | sort | tail -n 1)"
|
||||
LATEST_REPORT="$(find "$(report_output_dir)" -maxdepth 1 -type f -name 'daily_report_*.md' | sort | tail -n 1)"
|
||||
if [[ -n "$LATEST_REPORT" ]]; then
|
||||
echo "healthcheck: ok (db=up latest_report=$LATEST_REPORT)"
|
||||
exit 0
|
||||
|
||||
@@ -4,19 +4,43 @@ package retry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type temporaryError interface {
|
||||
Temporary() bool
|
||||
}
|
||||
|
||||
type timeoutError interface {
|
||||
Timeout() bool
|
||||
}
|
||||
|
||||
type HTTPStatusError struct {
|
||||
StatusCode int
|
||||
Body string
|
||||
}
|
||||
|
||||
func (e HTTPStatusError) Error() string {
|
||||
if e.Body == "" {
|
||||
return fmt.Sprintf("http status %d", e.StatusCode)
|
||||
}
|
||||
return fmt.Sprintf("http status %d: %s", e.StatusCode, e.Body)
|
||||
}
|
||||
|
||||
// Strategy 重试策略
|
||||
type Strategy struct {
|
||||
MaxRetries int // 最大重试次数(0=不重试)
|
||||
BaseDelay time.Duration // 基础延迟
|
||||
MaxDelay time.Duration // 最大延迟上限
|
||||
Multiplier float64 // 乘数(默认2.0)
|
||||
Jitter bool // 是否添加随机抖动
|
||||
Retryable func(error) bool // 判断错误是否可重试
|
||||
MaxRetries int // 最大重试次数(0=不重试)
|
||||
BaseDelay time.Duration // 基础延迟
|
||||
MaxDelay time.Duration // 最大延迟上限
|
||||
Multiplier float64 // 乘数(默认2.0)
|
||||
Jitter bool // 是否添加随机抖动
|
||||
Retryable func(error) bool // 判断错误是否可重试
|
||||
}
|
||||
|
||||
// DefaultStrategy 返回默认重试策略
|
||||
@@ -31,36 +55,89 @@ func DefaultStrategy() Strategy {
|
||||
}
|
||||
}
|
||||
|
||||
// IsRetryable 默认重试判定:网络错误、超时、5xx状态码等可重试
|
||||
// IsRetryable 默认重试判定:仅临时网络错误、429、5xx 等可重试
|
||||
func IsRetryable(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
// 这里可以扩展更多错误类型判定
|
||||
return true
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
|
||||
var statusErr HTTPStatusError
|
||||
if errors.As(err, &statusErr) {
|
||||
return statusErr.StatusCode == 429 || (statusErr.StatusCode >= 500 && statusErr.StatusCode < 600)
|
||||
}
|
||||
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return true
|
||||
}
|
||||
|
||||
message := strings.ToLower(err.Error())
|
||||
if strings.Contains(message, "json 解析失败") ||
|
||||
strings.Contains(message, "invalid character") ||
|
||||
strings.Contains(message, "unmarshal") ||
|
||||
strings.Contains(message, "decode") ||
|
||||
strings.Contains(message, "schema") {
|
||||
return false
|
||||
}
|
||||
|
||||
var tempErr temporaryError
|
||||
if errors.As(err, &tempErr) && tempErr.Temporary() {
|
||||
return true
|
||||
}
|
||||
|
||||
var timeoutErr timeoutError
|
||||
if errors.As(err, &timeoutErr) && timeoutErr.Timeout() {
|
||||
return true
|
||||
}
|
||||
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && (netErr.Timeout() || netErr.Temporary()) {
|
||||
return true
|
||||
}
|
||||
|
||||
retriableMarkers := []string{
|
||||
"transport closed",
|
||||
"connection reset",
|
||||
"connection refused",
|
||||
"tls handshake timeout",
|
||||
"i/o timeout",
|
||||
"no such host",
|
||||
"temporarily unavailable",
|
||||
"too many requests",
|
||||
"rate limit",
|
||||
}
|
||||
for _, marker := range retriableMarkers {
|
||||
if strings.Contains(message, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Do 执行带重试的操作
|
||||
func Do(ctx context.Context, strategy Strategy, fn func() error) error {
|
||||
var lastErr error
|
||||
|
||||
|
||||
for attempt := 0; attempt <= strategy.MaxRetries; attempt++ {
|
||||
if err := fn(); err != nil {
|
||||
lastErr = err
|
||||
|
||||
|
||||
// 不判断最后一次是否需要重试
|
||||
if attempt == strategy.MaxRetries {
|
||||
break
|
||||
}
|
||||
|
||||
|
||||
// 检查是否可重试
|
||||
if strategy.Retryable != nil && !strategy.Retryable(err) {
|
||||
return fmt.Errorf("non-retryable error on attempt %d: %w", attempt+1, err)
|
||||
}
|
||||
|
||||
|
||||
// 计算退避延迟
|
||||
delay := calculateDelay(strategy, attempt)
|
||||
|
||||
|
||||
// 检查上下文是否已取消
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -72,7 +149,7 @@ func Do(ctx context.Context, strategy Strategy, fn func() error) error {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return fmt.Errorf("all %d attempts failed, last error: %w", strategy.MaxRetries+1, lastErr)
|
||||
}
|
||||
|
||||
@@ -80,18 +157,18 @@ func Do(ctx context.Context, strategy Strategy, fn func() error) error {
|
||||
func calculateDelay(s Strategy, attempt int) time.Duration {
|
||||
// 指数退避: base * multiplier^attempt
|
||||
delay := float64(s.BaseDelay) * math.Pow(s.Multiplier, float64(attempt))
|
||||
|
||||
|
||||
// 添加上限
|
||||
if max := float64(s.MaxDelay); delay > max {
|
||||
delay = max
|
||||
}
|
||||
|
||||
|
||||
// 添加抖动(±25%)
|
||||
if s.Jitter {
|
||||
jitter := delay * 0.25
|
||||
delay = delay - jitter + (jitter * 2 * float64(time.Now().Nanosecond()%1000) / 1000)
|
||||
}
|
||||
|
||||
|
||||
return time.Duration(delay)
|
||||
}
|
||||
|
||||
@@ -99,31 +176,31 @@ func calculateDelay(s Strategy, attempt int) time.Duration {
|
||||
func DoWithResult[T any](ctx context.Context, strategy Strategy, fn func() (T, error)) (T, error) {
|
||||
var zero T
|
||||
var lastErr error
|
||||
|
||||
|
||||
for attempt := 0; attempt <= strategy.MaxRetries; attempt++ {
|
||||
result, err := fn()
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
||||
lastErr = err
|
||||
if attempt == strategy.MaxRetries {
|
||||
break
|
||||
}
|
||||
|
||||
|
||||
if strategy.Retryable != nil && !strategy.Retryable(err) {
|
||||
return zero, fmt.Errorf("non-retryable error on attempt %d: %w", attempt+1, err)
|
||||
}
|
||||
|
||||
|
||||
delay := calculateDelay(strategy, attempt)
|
||||
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return zero, fmt.Errorf("context cancelled after attempt %d: %w", attempt+1, ctx.Err())
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return zero, fmt.Errorf("all %d attempts failed, last error: %w", strategy.MaxRetries+1, lastErr)
|
||||
}
|
||||
|
||||
@@ -139,7 +216,7 @@ func DoWithMetrics(ctx context.Context, strategy Strategy, fn func() error) (Met
|
||||
m := Metrics{}
|
||||
var lastErr error
|
||||
start := time.Now()
|
||||
|
||||
|
||||
for attempt := 0; attempt <= strategy.MaxRetries; attempt++ {
|
||||
m.Attempts = attempt + 1
|
||||
if err := fn(); err != nil {
|
||||
@@ -164,7 +241,7 @@ func DoWithMetrics(ctx context.Context, strategy Strategy, fn func() error) (Met
|
||||
return m, nil
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
m.TotalDelay = time.Since(start)
|
||||
return m, fmt.Errorf("all %d attempts failed, last error: %w", strategy.MaxRetries+1, lastErr)
|
||||
}
|
||||
|
||||
@@ -4,19 +4,31 @@ package retry
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func alwaysRetry(error) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func neverRetry(error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func TestDo_Success(t *testing.T) {
|
||||
strategy := DefaultStrategy()
|
||||
callCount := 0
|
||||
|
||||
|
||||
err := Do(context.Background(), strategy, func() error {
|
||||
callCount++
|
||||
return nil
|
||||
})
|
||||
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
@@ -32,10 +44,10 @@ func TestDo_RetryThenSuccess(t *testing.T) {
|
||||
MaxDelay: 100 * time.Millisecond,
|
||||
Multiplier: 2.0,
|
||||
Jitter: false,
|
||||
Retryable: IsRetryable,
|
||||
Retryable: alwaysRetry,
|
||||
}
|
||||
callCount := 0
|
||||
|
||||
|
||||
err := Do(context.Background(), strategy, func() error {
|
||||
callCount++
|
||||
if callCount < 3 {
|
||||
@@ -43,7 +55,7 @@ func TestDo_RetryThenSuccess(t *testing.T) {
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
@@ -59,16 +71,16 @@ func TestDo_MaxRetriesExceeded(t *testing.T) {
|
||||
MaxDelay: 50 * time.Millisecond,
|
||||
Multiplier: 2.0,
|
||||
Jitter: false,
|
||||
Retryable: IsRetryable,
|
||||
Retryable: alwaysRetry,
|
||||
}
|
||||
callCount := 0
|
||||
expectedErr := errors.New("persistent error")
|
||||
|
||||
|
||||
err := Do(context.Background(), strategy, func() error {
|
||||
callCount++
|
||||
return expectedErr
|
||||
})
|
||||
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
@@ -84,15 +96,15 @@ func TestDo_NonRetryableError(t *testing.T) {
|
||||
MaxDelay: 100 * time.Millisecond,
|
||||
Multiplier: 2.0,
|
||||
Jitter: false,
|
||||
Retryable: func(err error) bool { return false }, // 任何错误都不重试
|
||||
Retryable: neverRetry, // 任何错误都不重试
|
||||
}
|
||||
callCount := 0
|
||||
|
||||
|
||||
err := Do(context.Background(), strategy, func() error {
|
||||
callCount++
|
||||
return errors.New("non-retryable")
|
||||
})
|
||||
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
@@ -101,6 +113,48 @@ func TestDo_NonRetryableError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRetryableRejectsContextCancellation(t *testing.T) {
|
||||
if IsRetryable(context.Canceled) {
|
||||
t.Fatal("context.Canceled should not be retryable")
|
||||
}
|
||||
if IsRetryable(context.DeadlineExceeded) {
|
||||
t.Fatal("context.DeadlineExceeded should not be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRetryableRejectsPermanentHTTPStatus(t *testing.T) {
|
||||
err := HTTPStatusError{StatusCode: http.StatusForbidden, Body: "forbidden"}
|
||||
if IsRetryable(err) {
|
||||
t.Fatal("403 should not be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRetryableAllowsTemporaryNetworkAndServerErrors(t *testing.T) {
|
||||
err := HTTPStatusError{StatusCode: http.StatusBadGateway, Body: "bad gateway"}
|
||||
if !IsRetryable(err) {
|
||||
t.Fatal("502 should be retryable")
|
||||
}
|
||||
|
||||
netErr := &net.DNSError{IsTemporary: true}
|
||||
if !IsRetryable(netErr) {
|
||||
t.Fatal("temporary network errors should be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRetryableRejectsJSONParseErrors(t *testing.T) {
|
||||
err := errors.New("JSON 解析失败: invalid character")
|
||||
if IsRetryable(err) {
|
||||
t.Fatal("JSON parse errors should not be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRetryableAllowsUnexpectedEOF(t *testing.T) {
|
||||
err := fmt.Errorf("transport closed: %w", io.ErrUnexpectedEOF)
|
||||
if !IsRetryable(err) {
|
||||
t.Fatal("unexpected EOF should be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDo_ContextCancellation(t *testing.T) {
|
||||
strategy := Strategy{
|
||||
MaxRetries: 3,
|
||||
@@ -108,18 +162,18 @@ func TestDo_ContextCancellation(t *testing.T) {
|
||||
MaxDelay: 5 * time.Second,
|
||||
Multiplier: 2.0,
|
||||
Jitter: false,
|
||||
Retryable: IsRetryable,
|
||||
Retryable: alwaysRetry,
|
||||
}
|
||||
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
|
||||
callCount := 0
|
||||
err := Do(ctx, strategy, func() error {
|
||||
callCount++
|
||||
return errors.New("error")
|
||||
})
|
||||
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
@@ -138,10 +192,10 @@ func TestDoWithResult(t *testing.T) {
|
||||
MaxDelay: 50 * time.Millisecond,
|
||||
Multiplier: 2.0,
|
||||
Jitter: false,
|
||||
Retryable: IsRetryable,
|
||||
Retryable: alwaysRetry,
|
||||
}
|
||||
callCount := 0
|
||||
|
||||
|
||||
result, err := DoWithResult(context.Background(), strategy, func() (string, error) {
|
||||
callCount++
|
||||
if callCount < 2 {
|
||||
@@ -149,7 +203,7 @@ func TestDoWithResult(t *testing.T) {
|
||||
}
|
||||
return "success", nil
|
||||
})
|
||||
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
@@ -168,9 +222,9 @@ func TestDoWithMetrics(t *testing.T) {
|
||||
MaxDelay: 100 * time.Millisecond,
|
||||
Multiplier: 2.0,
|
||||
Jitter: false,
|
||||
Retryable: IsRetryable,
|
||||
Retryable: alwaysRetry,
|
||||
}
|
||||
|
||||
|
||||
// 成功场景
|
||||
m, err := DoWithMetrics(context.Background(), strategy, func() error {
|
||||
return nil
|
||||
@@ -184,7 +238,7 @@ func TestDoWithMetrics(t *testing.T) {
|
||||
if m.Attempts != 1 {
|
||||
t.Errorf("expected 1 attempt, got %d", m.Attempts)
|
||||
}
|
||||
|
||||
|
||||
// 失败场景
|
||||
m2, err := DoWithMetrics(context.Background(), strategy, func() error {
|
||||
return errors.New("always fails")
|
||||
@@ -207,7 +261,7 @@ func TestCalculateDelay(t *testing.T) {
|
||||
Multiplier: 2.0,
|
||||
Jitter: false,
|
||||
}
|
||||
|
||||
|
||||
tests := []struct {
|
||||
attempt int
|
||||
min time.Duration
|
||||
@@ -219,7 +273,7 @@ func TestCalculateDelay(t *testing.T) {
|
||||
{3, 8 * time.Second, 8 * time.Second},
|
||||
{4, 10 * time.Second, 10 * time.Second}, // 达到上限
|
||||
}
|
||||
|
||||
|
||||
for _, tt := range tests {
|
||||
delay := calculateDelay(strategy, tt.attempt)
|
||||
if delay < tt.min || delay > tt.max {
|
||||
@@ -236,7 +290,7 @@ func BenchmarkDo(b *testing.B) {
|
||||
Multiplier: 0,
|
||||
Jitter: false,
|
||||
}
|
||||
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = Do(context.Background(), strategy, func() error {
|
||||
return nil
|
||||
|
||||
128
reports/openclaw/2026-05-13-1510-review.md
Normal file
128
reports/openclaw/2026-05-13-1510-review.md
Normal file
@@ -0,0 +1,128 @@
|
||||
# OpenClaw Review — 2026-05-13 15:10 Asia/Shanghai
|
||||
|
||||
> **Review ID**: `830ba8ca-2026-05-13-1510`
|
||||
> **Trigger**: `cron 830ba8ca-9863-4d4d-9c45-4e30860ea27a llm-intelligence-afternoon-review`
|
||||
> **Reviewer**: 宰相(AI Agent)
|
||||
> **Scope**: 高频真实状态 review,非破坏性,不改业务代码
|
||||
>
|
||||
> **历史快照说明(2026-05-24 更新)**:本文件主体反映的是 `2026-05-13 15:10` 左右的仓库现场,只能作为历史追溯材料,不代表当前 gate 结论。当前真相请先读:[`OPENCLAW_EXECUTION.md`](../../OPENCLAW_EXECUTION.md)、[`OPENCLAW_CAPABILITY_BACKLOG.md`](./OPENCLAW_CAPABILITY_BACKLOG.md)、[`docs/README.md`](../../docs/README.md)。若本文件中的 Phase 5 / Phase 6 阻塞描述与当前执行手册或 backlog 冲突,应以后者为准。
|
||||
|
||||
> **2026-05-13 17:49 校正注记**:本报告主体反映的是 15:10 左右的仓库快照;其后 `scripts/run_real_pipeline.sh` 与 `scripts/verify_phase3.sh` 已修正归档链路。按 17:49 复核,统一口径应为:**功能主链路可运行;Phase 6 当前 FAIL;直接阻塞只剩 Phase 5 CI 缺失。** 不应继续把 Phase 3 归档缺口或“6 项活跃 FAIL”当作当前真相。
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
### Review Frame
|
||||
|
||||
- 本次 review 的时间窗口:基于 2026-05-13 15:10 Asia/Shanghai 的仓库现场状态;重点检查 git 状态、最近提交、核心项目文档、`reports/`、验证入口与综合验收门禁。
|
||||
- 与上一次 review 的间隔:无法仅凭当前仓库精确确认上一次 review 时间;已读取当前 backlog 与模板作为最近 review 上下文。
|
||||
- 与最后一次真实 commit 的间隔:约 1 分钟;最新 commit 为 `6a2cd3f 2026-05-13 15:08:37 +0800 chore(frontend): split fixture and generated model snapshots`。
|
||||
- 本轮是否存在仓库状态变化:存在。最近提交已发生变化,且综合验收结果与今日日报产物共同构成本轮 delta。
|
||||
|
||||
### Stage Judgment
|
||||
|
||||
- 当前真实阶段:项目功能主链路可运行,但不处于“生产级收口完成”状态;17:49 复核后的统一口径是 **Phase 6 当前 FAIL;直接阻塞只剩 Phase 5 CI 缺失**。
|
||||
- 主要判断依据:`verify_phase1.sh`、`verify_phase2.sh`、`verify_phase4.sh` 通过;归档根因已修复后,综合验收剩余阻塞应收敛到 Phase 5 CI 缺失。
|
||||
- 本轮背景说明:本轮为 cron 高频 review,按要求优先审查真实仓库状态,不同步任务状态,不修改业务代码,重点识别 gap 与 OpenClaw 能力优化项。
|
||||
|
||||
## Evidence
|
||||
|
||||
### Evidence Grades
|
||||
|
||||
- `runtime-verified`:`bash scripts/verify_phase6.sh`、`bash scripts/verify_pre_phase6.sh`、`bash scripts/verify_phase5.sh` 的实际输出;`git log --date=iso -1`。
|
||||
- `artifact-present`:`TASKS.md`、`GOALS.md`、`OPENCLAW_EXECUTION.md`、`Makefile`、`reports/daily/daily_report_2026-05-13.md`、`reports/daily/html/daily_report_2026-05-13.html`、`reports/openclaw/*` 文件存在。
|
||||
- `doc-claimed`:`TASKS.md` 中多项“✅ 完成”状态与 `OPENCLAW_EXECUTION.md` 中的阶段描述;这些声明本轮只在必要处被交叉验证,未做逐项全量复验。
|
||||
|
||||
### Verification Commands
|
||||
|
||||
- 命令:`git status --short && git log --oneline -8 && find reports -maxdepth 2 -type f | sort`
|
||||
- 结果:工作区无未提交输出;最近提交包含 `6a2cd3f`、`b19bb7d`、`4974926` 等;`reports/` 下仅见 `reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md` 与 `reports/openclaw/REVIEW_TEMPLATE.md`,`reports/daily/` 另有今日日报与 HTML。
|
||||
- 命令:`bash scripts/verify_phase6.sh`
|
||||
- 结果:`SUMMARY pass=15 fail=1 warn=0`,`PHASE_RESULT: FAIL`。综合验收失败,但输出已指向 `verify_phase5.sh FAIL` 与 pre-phase6 失败。
|
||||
- 命令:`bash scripts/verify_pre_phase6.sh`
|
||||
- 结果:Phase 1、2、4 通过;Phase 3 与 Phase 5 失败;`PRE_PHASE6_RESULT: FAIL`。
|
||||
- 命令:`bash scripts/verify_phase5.sh`
|
||||
- 结果:在 15:10 快照对应的脚本版本下,`.github/workflows/ci.yml` 缺失会级联触发 6 项 FAIL;**但按 16:22 复核当前脚本版本时,真实活跃 FAIL 已收敛为该文件缺失这一项**。
|
||||
- 命令:`git log --date=iso --pretty=format:'%h %ad %s' -1`
|
||||
- 结果:最新 commit 为 `6a2cd3f 2026-05-13 15:08:37 +0800 chore(frontend): split fixture and generated model snapshots`。
|
||||
|
||||
### Completed
|
||||
|
||||
- 已完成项:主仓库处于干净状态,且存在新的真实 commit。
|
||||
- 证据:`runtime-verified`,`git status --short` 无输出;`git log -1` 显示 15:08 新提交。
|
||||
- 已完成项:今日日报 Markdown 与 HTML 产物存在。
|
||||
- 证据:`artifact-present`,`reports/daily/daily_report_2026-05-13.md`、`reports/daily/html/daily_report_2026-05-13.html` 均存在。
|
||||
- 已完成项:Phase 1、Phase 2、Phase 4 当前门禁通过。
|
||||
- 证据:`runtime-verified`,`verify_pre_phase6.sh` 输出对应三阶段 `PASS`。
|
||||
- 已完成项:Phase 6 中 API 与真实采集相关子检查大部分通过。
|
||||
- 证据:`runtime-verified`,`verify_phase6.sh` 中 `/health`、`/api/v1/models`、`/api/v1/subscription-plans`、真实采集与今日日报检查均为 `PASS`。
|
||||
|
||||
### Incomplete
|
||||
|
||||
- 未完成项:Phase 3 归档门禁未满足(15:10 快照);该项已于 17:49 复核时修复。
|
||||
- 影响:15:10 快照下会把综合验收拖成 FAIL,即使今日日报主文件已经生成。
|
||||
- 当前状态:`runtime-verified`;当前已通过修复 `run_real_pipeline.sh` 补齐 `reports/daily/$(date +%Y/%m)/` 归档副本。
|
||||
- 未完成项:Phase 5 的 GitHub Actions CI 文件及其相关门禁未满足。
|
||||
- 影响:生产级收口与自动化验收声明不能成立,综合验收持续 FAIL。
|
||||
- 当前状态:`runtime-verified`,按 16:22 复核,`verify_phase5.sh` 的当前真实活跃 FAIL 为 `.github/workflows/ci.yml` 缺失。
|
||||
- 未完成项:OpenClaw review 输出文件尚不存在于本轮开始时。
|
||||
- 影响:需要本轮补写审查报告与 backlog 更新,保持 review 链路闭环。
|
||||
- 当前状态:`artifact-present`,`reports/openclaw/` 初始只有模板与 backlog。
|
||||
|
||||
### Inconsistencies
|
||||
|
||||
- 伪进展或文档/实现不一致项:`OPENCLAW_EXECUTION.md` 中写有“Phase 6 综合验收通过(verify_phase6.sh PASS)”,但当前仓库现场 `verify_phase6.sh` 为 FAIL。
|
||||
- 证据:`doc-claimed` vs `runtime-verified` 冲突;当前实际输出 `SUMMARY pass=15 fail=1 warn=0 / PHASE_RESULT: FAIL`。
|
||||
- 伪进展或文档/实现不一致项:`TASKS.md` 中 `T-5.5 自动采集与日报调度` 标注完成,但当前 Phase 3 仍缺“今日归档报告存在”这一门禁。
|
||||
- 证据:`doc-claimed` 与 `runtime-verified` 并存;这不证明任务一定错误完成,但至少说明“日报闭环”与“归档闭环”并不完全一致,不能包装成全量完成。
|
||||
- 伪进展或文档/实现不一致项:`TASKS.md` 中部分生产级收口相关完成声明,不能覆盖当前 `.github/workflows/ci.yml` 缺失这一现实 gap。
|
||||
- 证据:`runtime-verified`,`verify_phase5.sh` 明确 FAIL。
|
||||
|
||||
### Key Gaps
|
||||
|
||||
- Gap:日报主文件已生成,但 15:10 快照下 Phase 6 使用的 `run_real_pipeline.sh` 不产出 Phase 3 期望的归档文件;该根因已在 17:49 复核时修复。
|
||||
- 优先级:P0
|
||||
- 影响:修复前会把 Phase 3 与 Phase 6 拖成 FAIL,制造“主链路可跑但收口门禁不通过”的持续噪声。
|
||||
- 证据:`runtime-verified`;当前修复方式是在 `run_real_pipeline.sh` 中补齐 `reports/daily/2026/05/...` 归档文件复制,并将 `verify_phase3.sh` 拆成更清晰的主产物/归档副本检查。
|
||||
- Gap:`.github/workflows/ci.yml` 缺失,Phase 5 自动化验收名存实亡。
|
||||
- 优先级:P0
|
||||
- 影响:至少 CI 工作流存在性这一生产级门禁当前无法成立;当前脚本版本下其余检查项已改写为非 workflow 文件依赖项,但 `.github/workflows/ci.yml` 缺失本身仍是活跃阻塞。
|
||||
- 证据:`runtime-verified`,16:22 复核时 `verify_phase5.sh` 唯一 FAIL 即该文件缺失。
|
||||
- Gap:项目执行说明与任务面板存在过时“已通过/已完成”表达。
|
||||
- 优先级:P1
|
||||
- 影响:review 容易被文档宣称误导,增加误报与判断成本。
|
||||
- 证据:`doc-claimed`,`OPENCLAW_EXECUTION.md` 与当前 runtime 验证结果不一致。
|
||||
- Gap:综合验收虽然能暴露失败,但仍需要人工下钻子 phase 才能快速定位根因。
|
||||
- 优先级:P1
|
||||
- 影响:review 成本高,容易只记住顶层 FAIL 而忽略真正失败点。
|
||||
- 证据:`runtime-verified`,本轮必须继续执行 `verify_pre_phase6.sh` 与 `verify_phase5.sh` 才定位到实际问题。
|
||||
|
||||
## Outcome
|
||||
|
||||
### Executive Summary
|
||||
|
||||
- 本轮执行摘要:仓库在 15:08 有新提交,主工作区干净;主 API、真实采集、今日日报主文件、Phase 1/2/4 均正常;17:49 复核后统一口径应更新为 **功能主链路可运行;Phase 6 当前 FAIL;直接阻塞只剩 Phase 5 CI 缺失**。Phase 3 归档缺口已通过修复 `run_real_pipeline.sh` 关闭;这不是主链路整体崩坏。
|
||||
- 风险判断:高。原因不是“系统不可用”,而是“收口门禁与文档真实状态失配”,会持续制造错误的完成感,并阻塞生产级验收结论。
|
||||
- 阶段结论:当前最准确的阶段判断应为 **功能主链路可运行,生产级治理收口未完成;Phase 6 当前 FAIL,直接阻塞只剩 Phase 5 CI 缺失**,不能继续使用“Phase 6 已通过”叙述。
|
||||
|
||||
### Decisions
|
||||
|
||||
- 本轮最重要的落地结论:后续任何对外或对内结论都应统一为 **功能主链路可运行;Phase 6 当前 FAIL;直接阻塞只剩 Phase 5 CI 缺失**,不要继续沿用旧的 Phase 3 失败面,也不要泛泛宣称“整体已完成”。
|
||||
- 是否需要更新 `OPENCLAW_CAPABILITY_BACKLOG.md`:需要。原因是本轮新增/确认了 3 类 review 能力问题:归档门禁失配仍活跃、CI 缺失已从历史风险变成当前 runtime 失败、综合验收仍需下钻才能定位根因。
|
||||
|
||||
## Next
|
||||
|
||||
### Priority Actions
|
||||
|
||||
1. 动作:恢复或新增 `.github/workflows/ci.yml`,覆盖 Go 测试、前端构建、Docker 构建、覆盖率门禁、产物上传
|
||||
- Owner:集成验收 / 工程治理
|
||||
- 预期证据:`bash scripts/verify_phase5.sh` 返回 `PHASE_RESULT: PASS`
|
||||
2. 动作:修正文档中的过时完成表述,尤其是 `OPENCLAW_EXECUTION.md` 对 Phase 6 的当前状态描述
|
||||
- Owner:产品架构师 / 集成验收
|
||||
- 预期证据:文档改写后重新运行 `bash scripts/verify_phase6.sh`,并确保文档声明与 runtime 结果一致
|
||||
|
||||
### Follow-up Notes
|
||||
|
||||
- 需要人工介入的事项:若 `.github/workflows/ci.yml` 是有意从公开仓库移除,需要明确这是产品策略还是暂时缺口;否则 review 会持续将其判为活跃 gap。
|
||||
- 下轮 review 应重点复核的事项:`verify_phase3.sh` 的归档路径是否已补齐;`.github/workflows/ci.yml` 是否恢复;`OPENCLAW_EXECUTION.md` 是否仍保留过时的“Phase 6 已通过”叙述。
|
||||
131
reports/openclaw/2026-05-20-2106-review.md
Normal file
131
reports/openclaw/2026-05-20-2106-review.md
Normal file
@@ -0,0 +1,131 @@
|
||||
# OpenClaw Review — 2026-05-20 2106 Asia/Shanghai
|
||||
|
||||
> **Review ID**: `175a61b2-2026-05-20-2106`
|
||||
> **Trigger**: `cron 175a61b2-c2e7-4df4-a994-2fcacdbd24c6 llm-intelligence-morning-review`
|
||||
> **Reviewer**: 宰相(AI Agent)
|
||||
> **Scope**: 高频真实状态 review,非破坏性,不改业务代码
|
||||
>
|
||||
> **历史快照说明(2026-05-24 更新)**:本文件只反映 `2026-05-20 21:06` 当时现场,不代表当前 gate 结论。当前真相请先读:[`OPENCLAW_EXECUTION.md`](../../OPENCLAW_EXECUTION.md)、[`OPENCLAW_CAPABILITY_BACKLOG.md`](./OPENCLAW_CAPABILITY_BACKLOG.md)、[`docs/README.md`](../../docs/README.md)。截至 `2026-05-24 19:05`,Phase 6 已恢复通过;若本文件中的 `PHASE_RESULT: FAIL`、Perplexity 超时或窗口失败结论与当前文档冲突,应以当前真相入口为准。
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
### Review Frame
|
||||
|
||||
- 本次 review 的时间窗口:2026-05-20 21:06~21:15 Asia/Shanghai;按 prompt 完成 `git status --short`、最近提交记录、`TASKS.md`、`GOALS.md`、`OPENCLAW_EXECUTION.md`、`reports/`、验证入口检查,并执行非破坏性验证 `bash scripts/verify_phase6.sh`。
|
||||
- 与上一次 review 的间隔:约 23.5 小时;上一次落盘报告为 `reports/openclaw/2026-05-19-2130-review.md`。
|
||||
- 与最后一次真实 commit 的间隔:约 1 天 6 小时;最新 commit 仍为 `42e75e7 docs(runtime): sync execution and backlog status`,本轮前无新增 commit。
|
||||
- 本轮是否存在仓库状态变化:有显著工作区现场变更(19 文件、~900 行新增),但无提交层收敛;runtime 结论有 delta——稳定性窗口从 `85.71%` 回落到 `71.43%`。
|
||||
|
||||
### Stage Judgment
|
||||
|
||||
- 当前真实阶段:项目处于"新增导入器 smoke gate 已准入,但 Phase 6 仍被单一外部依赖 + 历史前置条件窗口门禁阻断,且大量改动未提交收敛"的阶段。
|
||||
- 主要判断依据:`bash scripts/verify_phase6.sh` 完整输出 `SUMMARY pass=15 fail=2 warn=0` 与 `PHASE_RESULT: FAIL`;`importer_smoke_gate_result=PASS`;`live_run_result=FAIL` 仍由 `perplexity_pricing_signature_guard` 抓取 `https://docs.perplexity.ai/docs/agent-api/models.md` 超时触发;`window_gate_result=FAIL`,最近 7 次窗口为 `success_count=5 failure_count=2 success_rate=71.43 threshold=95 precondition_missing=2`,失败分类仍为 `window_failure_class=precondition_missing_only`。
|
||||
- 本轮背景说明:相对 05-19 21:30,本轮有 runtime delta——稳定性窗口进一步回落(85.71% → 71.43%),原因是今日新增一次 `precondition_missing` 失败样本(`2026-05-20 08:00:01` 严格真实模式下未提供 API Key)。工作区有大量未提交改动(19 文件、~900 行),涉及 CoreHub 导入器、天翼云订阅库、日报生成器、验证脚本等核心组件。
|
||||
|
||||
## Evidence
|
||||
|
||||
### Evidence Grades
|
||||
|
||||
- `runtime-verified`:`git status --short`、`git log --oneline -8`、`git diff --stat HEAD`、验证入口检查、`bash scripts/verify_phase6.sh`。
|
||||
- `artifact-present`:`TASKS.md`、`GOALS.md`、`OPENCLAW_EXECUTION.md`、`reports/openclaw/REVIEW_TEMPLATE.md`、`reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md`、`Makefile`、`scripts/verify_importer_smoke.sh`、`scripts/importer_smoke_gate_test.sh`。
|
||||
- `doc-claimed`:`TASKS.md` 与执行手册中的完成态/规则说明;这些都未替代本轮真实验证。
|
||||
|
||||
### Verification Commands
|
||||
|
||||
- 命令:`git status --short`
|
||||
- 结果:tracked 修改包括 `docs/PLAN_CATALOG_INVENTORY.md`、`reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md`、`scripts/coreshub_pricing_lib.go`、`scripts/ctyun_subscription_lib.go`、`scripts/generate_daily_report.go`、`scripts/import_coreshub_pricing.go`、`scripts/import_coreshub_pricing_test.go`、`scripts/import_ctyun_subscription_test.go`、`scripts/importer_smoke_gate_test.sh`、`scripts/report_state_tracking_test.sh`、`scripts/report_utils.sh`、`scripts/run_daily.sh`、`scripts/run_intel_pipeline.sh`、`scripts/run_real_pipeline.sh`、`scripts/testdata/coreshub_pricing_sample.txt`、`scripts/testdata/ctyun_token_plan_sample.txt`、`scripts/verify_importer_smoke.sh`、`scripts/verify_phase6.sh`、`seeds/plan_catalog_inventory_seed_cn_relays_top20plus.json`;untracked 仍有 `memory/.dreams/`。`runtime-verified`
|
||||
- 命令:`git log --oneline -8`
|
||||
- 结果:最新提交仍为 `42e75e7 docs(runtime): sync execution and backlog status`(2026-05-19);本轮前无新 commit。`runtime-verified`
|
||||
- 命令:`git diff --stat HEAD`
|
||||
- 结果:19 个文件变更,+900/-247 行;涉及 CoreHub 导入器(`coreshub_pricing_lib.go` +81、`import_coreshub_pricing.go` +88、`import_coreshub_pricing_test.go` +64)、天翼云订阅库(`ctyun_subscription_lib.go` +201)、日报生成器(`generate_daily_report.go` +78/-)、验证脚本(`verify_phase6.sh` +115/-)等。`runtime-verified`
|
||||
- 命令:`bash scripts/verify_phase6.sh`
|
||||
- 结果:完整输出 `SUMMARY pass=15 fail=2 warn=0`、`PHASE_RESULT: FAIL`;其中 `importer_smoke_gate_result=PASS`(coreshub-fixture/coreshub-live/ctyun-fixture/ctyun-live 全部 PASS);`live_run_result=FAIL`,错误为 `perplexity_pricing_signature_guard: fetch https://docs.perplexity.ai/docs/agent-api/models.md: context deadline exceeded`;`window_gate_result=FAIL`,最近 7 次窗口为 `success_count=5 failure_count=2 success_rate=71.43 threshold=95 precondition_missing=2 external_provider_failure=0 collector_runtime_failure=0 unknown_failure=0`,失败分类为 `window_failure_class=precondition_missing_only`。`runtime-verified`
|
||||
|
||||
### Completed
|
||||
|
||||
- 已完成项:新增导入器 smoke gate 已真实接入 Phase 6 综合门禁并通过。
|
||||
- 证据:`runtime-verified`;`verify_phase6.sh` 输出 `[PASS] importer_smoke_gate_result=PASS`。
|
||||
- 已完成项:Phase 1~5 总门禁、本仓 Go 测试、脚本级采集器单测、API Server 构建、健康检查、模型 API、套餐 API、前端测试入口在本轮仍通过。
|
||||
- 证据:`runtime-verified`;`verify_phase6.sh` 对应 `[PASS]` 项。
|
||||
- 已完成项:当前 live blocker 继续收敛为单一外部文档签名校验超时,而不是新增导入器准入问题。
|
||||
- 证据:`runtime-verified`;四个 smoke 子项全部 PASS,综合失败点只剩 Perplexity 外部超时与窗口门禁。
|
||||
- 已完成项:工作区有大量实质性进展——CoreHub 导入器(lib + 导入器 + 测试)、天翼云订阅库扩展、日报生成器改进、验证脚本增强等已落地到工作区。
|
||||
- 证据:`runtime-verified`;`git diff --stat HEAD` 显示 +900 行变更。
|
||||
|
||||
### Incomplete
|
||||
|
||||
- 未完成项:Phase 6 综合门禁仍未通过。
|
||||
- 影响:项目当前不能被表述为 release-ready 或"生产级综合验收完成"。
|
||||
- 当前状态:`runtime-verified`;`PHASE_RESULT: FAIL`。
|
||||
- 未完成项:`live_run_result` 仍被 Perplexity 外部文档签名校验超时阻断。
|
||||
- 影响:即使 importer smoke、API、测试等已通过,综合门禁仍会因单一外部依赖超时失败。
|
||||
- 当前状态:`runtime-verified`;`context deadline exceeded`。
|
||||
- 未完成项:稳定性窗口门禁本轮继续 FAIL,且窗口成功率进一步回落。
|
||||
- 影响:release 结论继续受历史前置条件纪律问题拖累;本轮从 85.71% 降至 71.43%,新增一次 precondition_missing 失败。
|
||||
- 当前状态:`runtime-verified`;`window_gate_result=FAIL`、`success_rate=71.43`、`window_failure_class=precondition_missing_only`。
|
||||
- 未完成项:19 个文件的实质性改动未提交收敛。
|
||||
- 影响:versioned truth 严重落后于 runtime truth,增加 review 漂移与回归成本;CoreHub 导入器、天翼云订阅库扩展、日报生成器改进等关键变更均未入版本控制。
|
||||
- 当前状态:`runtime-verified`;`git diff --stat HEAD` 显示 +900/-247 行变更,最新 commit 未变化。
|
||||
|
||||
### Inconsistencies
|
||||
|
||||
- 伪进展或文档/实现不一致项:工作区已有 CoreHub 导入器全套实现(lib + 导入器 + 测试 + fixture),但 `TASKS.md` 未反映这些新任务/进展。
|
||||
- 证据:`artifact-present` + `runtime-verified`;`git diff --stat HEAD` 显示新增文件,但 `TASKS.md` 无对应条目。
|
||||
- 伪进展或文档/实现不一致项:`importer_smoke_gate_test.sh` 仍假定"当前 live ctyun smoke 应失败",与本轮 `verify_phase6.sh` 中 `ctyun-live` 已通过直接冲突(同问题 35)。
|
||||
- 证据:`artifact-present` + `runtime-verified`;脚本内容仍保留旧断言。
|
||||
- 伪进展或文档/实现不一致项:如果只看 `TASKS.md`/执行手册的完成态而不看本轮 runtime,容易误把当前状态包装成"基本完成";本轮未做真实验证的完成态只能算 `doc-claimed`,不能替代 `PHASE_RESULT: FAIL`。
|
||||
- 证据:`doc-claimed` + `runtime-verified`;本轮真实综合门禁未通过。
|
||||
|
||||
### Key Gaps
|
||||
|
||||
- Gap:稳定性窗口进一步老化——从 85.71% 降至 71.43%,precondition_missing 样本从 1 增至 2。
|
||||
- 优先级:P1
|
||||
- 影响:窗口门禁持续 FAIL,且失败样本在增长;若继续叠加 precondition_missing 样本,窗口成功率会进一步下降。
|
||||
- 证据:`runtime-verified`;`verify_phase6.sh` 输出 `success_count=5 failure_count=2 success_rate=71.43 precondition_missing=2`。
|
||||
- Gap:外部 provider 失败与主链路成功仍被聚合为单个 `live_run_result=FAIL`,解释层仍不够细。
|
||||
- 优先级:P1
|
||||
- 影响:review 容易把"外部文档抓取超时"误读成"真实采集主链路失败",修复焦点会偏移。
|
||||
- 证据:`runtime-verified`;同一轮中 importer smoke、API 与测试均 PASS,但综合门禁仍因 Perplexity 文档超时失败。
|
||||
- Gap:smoke gate 测试脚本自身已老化,未跟上当前 live 行为(同问题 35)。
|
||||
- 优先级:P1
|
||||
- 影响:测试门禁会传播过时结论,降低 smoke gate 相关验证的可信度。
|
||||
- 证据:`artifact-present` + `runtime-verified`;`scripts/importer_smoke_gate_test.sh` 仍断言 ctyun live smoke 应失败。
|
||||
- Gap:working tree 长期不收敛,且本轮变更量显著增大(+900 行)。
|
||||
- 优先级:P0
|
||||
- 影响:大量核心组件改动(CoreHub 导入器、天翼云订阅库、日报生成器、验证脚本)未入版本控制,一旦工作区丢失则无法恢复;versioned truth 与 runtime truth 持续漂移。
|
||||
- 证据:`runtime-verified`;19 文件、+900/-247 行未提交。
|
||||
|
||||
## Outcome
|
||||
|
||||
### Executive Summary
|
||||
|
||||
- 本轮执行摘要:21:06 review 已按 prompt 完成现场检查,并重跑 `verify_phase6.sh`。相对 05-19 21:30,本轮有 runtime delta——稳定性窗口进一步回落(85.71% → 71.43%),新增一次 precondition_missing 失败样本。工作区变更量显著增大(19 文件、+900 行),涉及 CoreHub 导入器全套实现、天翼云订阅库扩展、日报生成器改进、验证脚本增强等,但全部未提交收敛。
|
||||
- 风险判断:中高。主链路大体可运行,但综合门禁仍未通过;失败同时包含外部依赖超时与历史窗口纪律问题;大量核心改动未入版本控制,工作区丢失风险上升。
|
||||
- 阶段结论:项目当前真实状态是"有实质性进展但未提交收敛,Phase 6 持续卡在单一外部依赖 + 历史窗口纪律"。工作区变更量已大到不能再视为"轻微漂移",需要尽快提交收敛。
|
||||
- 本轮最重要的落地结论:应把"大量核心改动未提交"提升为 P0 风险;稳定性窗口持续回落也需关注;当前 live blocker 仍是 `perplexity_pricing_signature_guard` 外部超时,未切换。
|
||||
|
||||
### Decisions
|
||||
|
||||
- 本轮最重要的落地结论:当前综合门禁的主 blocker 仍是 `perplexity_pricing_signature_guard` 外部超时;新增导入器 smoke gate 不是 current blocker;但工作区未收敛已从"长期存在"升级为"变更量显著增大",需要尽快提交。
|
||||
- 是否需要更新 `OPENCLAW_CAPABILITY_BACKLOG.md`:需要;应追加"稳定性窗口回落 + 工作区变更量增大"的记录,更新相应影响次数,并将 working tree 不收敛提升为 P0。
|
||||
|
||||
## Next
|
||||
|
||||
### Priority Actions
|
||||
|
||||
1. 动作:尽快将当前工作区改动(19 文件、+900 行)提交收敛,至少按逻辑拆分为 2~3 个 commit(如 CoreHub 导入器、天翼云订阅库扩展、日报/验证改进)
|
||||
- Owner:数据后端 / 集成验收
|
||||
- 预期证据:`git log --oneline` 出现新提交,`git diff --stat HEAD` 大幅收缩
|
||||
2. 动作:为 `perplexity_pricing_signature_guard` 增加更清晰的 release 级分类或降级策略,避免单一外部文档超时与主链路失败混写
|
||||
- Owner:数据后端 / 集成验收
|
||||
- 预期证据:`verify_phase6.sh` 输出能把外部依赖失败与主链路结果分开表述
|
||||
3. 动作:修正 `scripts/importer_smoke_gate_test.sh` 的过时断言,使其与当前 smoke gate runtime truth 对齐
|
||||
- Owner:数据后端
|
||||
- 预期证据:脚本断言更新后,相关测试可在当前仓库状态下真实表达 PASS/FAIL 预期
|
||||
|
||||
### Follow-up Notes
|
||||
|
||||
- 需要人工介入的事项:若 Perplexity 文档站波动是外部常态,应明确该签名校验在 release 门禁中的严格性策略;同时应尽快安排工作区提交收敛。
|
||||
- 下轮 review 应重点复核的事项:`live_run_result` 是否仍因外部文档超时失败、`window_gate_result` 成功率是否继续回落、working tree 是否已收敛、`importer_smoke_gate_test.sh` 是否仍与 runtime truth 冲突。
|
||||
@@ -8,109 +8,695 @@
|
||||
- 每个问题都要说明影响
|
||||
- 每个建议都要可执行、可验证
|
||||
|
||||
---
|
||||
## 当前未修复问题速查表(截至 2026-05-30 15:10)
|
||||
|
||||
## 当前未修复问题速查表(截至 2026-05-13 09:30)
|
||||
|
||||
| # | 问题 | 优先级 | 首次暴露 | 修复状态 | 影响次数 |
|
||||
|---|------|--------|----------|----------|----------|
|
||||
| 1 | 验证器 `rg` 依赖误报 | P0 | 05-07 22:50 | ✅ **已修复**(05-10 14:30 确认 `grep` 替换完成) | 10 次 |
|
||||
| 2 | 验证器退出码设计 | P0 | 05-07 22:50 | ⚠️ 部分(`rg` 误报消除,但三级状态仍未实现) | 10 次 |
|
||||
| 3 | session 历史工具/业务错误区分 | P1 | 05-07 22:50 | ❌ 未修复 | 11 次 |
|
||||
| 4 | cron 无主动状态报告机制 | P1 | 05-07 22:50 | ❌ 未修复 | 11 次 |
|
||||
| 5 | subagent spawn 未传递 workspace | P1 | 05-07 22:50 | ❌ 未修复 | 11 次 |
|
||||
| 6 | 验收脚本无法检测构建 | P1 | 05-08 09:05 | ❌ 未修复 | 10 次 |
|
||||
| 7 | 环境变量/API Key 缺失未自动检测 | P1 | 05-08 09:05 | ⚠️ 部分(已写入 review 标准步骤,但未固化到 prompt) | 10 次 |
|
||||
| 8 | 文件修改后未触发 commit 提示 | P2→P1 | 05-08 09:05 | ❌ 未修复 | 12 次 |
|
||||
| 9 | cron review 无 delta 时空转 | P1 | 05-08 09:12 | ❌ 未修复 | 12 次 |
|
||||
| 10 | 验证模式伪进展(artifact_present 局限) | P1 | 05-08 14:30 | ❌ 未修复 | 9 次 |
|
||||
| 11 | **项目提交停滞(commit stagnation)** | **P0** | **05-08 21:30** | **❌ 未修复(最新仍停留 05-08 commit)** | **12 次** |
|
||||
| 12 | review 报告未触发修复动作 | P2→P1 | 05-08 21:30 | ❌ 未修复 | 9 次 |
|
||||
| 13 | BACKLOG 文件膨胀导致 review 成本递增 | P1 | 05-09 09:30 | ⚠️ 部分(已实施分层归档,但主文件仍在增长) | 7 次 |
|
||||
| 14 | **untracked 核心代码未入版本控制** | **P0** | **05-10 21:30** | **❌ 未修复(本轮仍大量 untracked)** | **7 次** |
|
||||
| 15 | **CI 配置存在但未验证运行** | **P1** | **05-10 21:30** | **❌ 未修复(仍仅 artifact-present)** | **7 次** |
|
||||
| 16 | **Phase 6+ 范围未定义** | **P1** | **05-10 21:30** | **❌ 未修复** | **5 次** |
|
||||
| 17 | collection_stats vs collector_stats 表名不一致 | P2 | 05-11 09:30 | ✅ **已澄清为误报**(05-11 14:30 确认 verify_phase2.sh 与 schema 一致) | 1 次 |
|
||||
| 18 | **无 .gitignore 文件** | **P1** | **05-11 14:30** | **❌ 未修复** | **3 次** |
|
||||
| 19 | **review 误报传播** | **P1** | **05-11 14:30** | **❌ 未修复** | **4 次** |
|
||||
| 20 | **untracked 文件统计遗漏** | **P1** | **05-11 14:30** | **❌ 未修复** | **3 次** |
|
||||
| 21 | **验收脚本瞬时回归缺少稳定性标记** | **P1** | **05-12 22:46** | **❌ 未修复(本轮再次证明单次 FAIL 可能下一轮恢复)** | **3 次** |
|
||||
| 22 | **无 delta 场景缺少老化风险优先策略** | **P2** | **05-12 22:46** | **❌ 未修复** | **3 次** |
|
||||
| 23 | **日报归档路径门禁失配** | **P0** | **05-13 00:15** | **⚠️ 待复核(本轮未复现,当前 `verify_phase6.sh` 已 PASS)** | **1 次** |
|
||||
| 24 | **综合验收错误聚合误导根因判断** | **P1** | **05-13 00:15** | **❌ 未修复** | **1 次** |
|
||||
|
||||
---
|
||||
|
||||
## Review 日志
|
||||
|
||||
### 2026-05-13 09:30(第 18 次 review,morning-review)
|
||||
#### 问题 47 状态更新:已修复(从 current 表移除)→ 复发
|
||||
|
||||
> **前置说明**:距上一次 review(05-13 00:15)约 **9 小时 15 分钟**。本轮仓库状态的关键 delta 是:上一轮记录为 FAIL 的 `verify_phase6.sh`,本轮实际执行恢复为 **PASS**。这说明上一轮暴露的归档门禁问题当前未复现;与之相对,版本控制停滞与大量 untracked 仍无 delta,继续是最老化、最真实的系统性风险。**
|
||||
- **首次暴露**:2026-05-29 15:10
|
||||
- **原修复**:治理修复工作区已通过两次本地提交收口:
|
||||
1. `e999d31` — `fix: harden review and verifier governance`
|
||||
2. `d7455b8` — `docs: reconcile openclaw backlog truth`
|
||||
- **05-29 18:52 状态**:`git status --short` 已恢复为空;工作区污染已完全解除。
|
||||
- **05-30 15:10 复发**:工作区重新积累 11 modified + 2 untracked = 13 文件未提交;BACKLOG 本身未在 modified 中,但 scripts 目录多个脚本变更(healthcheck.sh, apply_migration.sh, rebuild_historical_report.sh, restore.sh, run_daily.sh, run_intel_pipeline.sh, run_intraday_discovery_watch.sh, run_intraday_price_watch.sh, run_real_pipeline.sh)未 commit;新增 untracked:scripts/env_precedence_test.sh, scripts/load_project_env.sh
|
||||
- **复发根因**:问题 47 虽然在 05-29 被 commit 收口,但 scripts 目录后续变更未被及时 commit,导致工作区重新污染
|
||||
- **验证证据**:
|
||||
1. `git status --short` → 13 files changed
|
||||
2. `bash scripts/review/backlog_current_freshness_guard.sh reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md` → `stale current truth snapshot`(freshness 超 20 小时阈值)
|
||||
- **结论**:问题 47 以新形式复发;与 05-29 的 121 文件变更相比,本轮 13 文件变更规模更小,但同属未 commit 工作区污染;应与问题 51 联动处置。
|
||||
|
||||
#### 问题 48 状态更新:已恢复(从 current 表移除)
|
||||
|
||||
- **首次暴露**:2026-05-29 15:10
|
||||
- **旧症状**:`xfyun-live` smoke FAIL 导致 `live_run_result=SKIPPED`。
|
||||
- **当前状态**:同类 smoke/live 传导链问题已不再稳定复现;最新 `verify_importer_smoke.sh` 与 `verify_phase6.sh` 均恢复通过。
|
||||
- **验证证据**:
|
||||
1. `bash scripts/verify_importer_smoke.sh` → `IMPORTER_SMOKE_RESULT: PASS`
|
||||
2. `bash scripts/verify_phase6.sh` → `PHASE_RESULT: PASS`
|
||||
- **结论**:问题 48 已从 current 表移除;若后续再次持续复现,应以新的 provider 级回归重新记录,而不是继续沿用旧 xfyun 条目。
|
||||
|
||||
|
||||
#### 问题 49 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **首次暴露**:2026-05-29 15:10
|
||||
- **根因复核**:三次 cron failed(10:32 / 10:58 / 18:44)根因一致,都是 `OPENROUTER_API_KEY` 缺失触发的 strict-real precondition 失败,并不存在“cron 失败但 22:01 又成功且未闭环”的真实矛盾。
|
||||
- **修复**:
|
||||
1. `run_daily.sh` 在 failure 路径现在会把这类错误显式标成 `precondition_missing`
|
||||
2. `cron_status_report.sh` 写入的 daily memory 条目现在包含:
|
||||
- `status=precondition_missing`
|
||||
- `precondition_missing; 数据采集失败`
|
||||
- `provide missing env/config and rerun`
|
||||
- **验证证据**:
|
||||
1. `bash scripts/cron_precondition_integration_test.sh` → PASS
|
||||
2. 手工执行 `bash scripts/run_daily.sh` 后,`memory/2026-05-29.md` 最新 cron 条目已写成 `status=precondition_missing`
|
||||
- **结论**:问题 49 已关闭;cron failed 现在已形成可解释闭环,不再只是笼统的 failed 条目。
|
||||
|
||||
|
||||
#### 问题 50 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **首次暴露**:2026-05-29 15:10
|
||||
- **修复**:
|
||||
1. current 表中的 resolved 行已清理完毕,仅保留当前未修复问题
|
||||
2. current table timestamp 已刷新到最新 review 时间
|
||||
3. backlog current table guard 与 freshness guard 已恢复绿色
|
||||
- **验证证据**:
|
||||
1. `bash scripts/review/backlog_current_table_guard.sh reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md` → `resolved_rows=0`
|
||||
2. `bash scripts/review/backlog_current_freshness_guard.sh reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md` → `status=fresh`
|
||||
- **结论**:问题 50 已关闭;BACKLOG current table 已恢复 current-truth 语义与 freshness。
|
||||
|
||||
### 2026-05-27 15:10(afternoon-review cron)
|
||||
|
||||
> **前置说明**:距上一次 review(05-26 15:10)约 **24 小时**。无新 commit。工作区从 22/+2819/-466 行扩大至 23/+3650/-808 行。scripts 新增 1619 行(主要是 generate_daily_report.go +1032 行及其测试 +567 行)。importer smoke 16 PASS 持续。ECharts FAIL 持续 2+ 天。scripts 目录 go test 出现 redeclared main build failure(新增 P1 gap)。
|
||||
|
||||
#### 本次新增发现
|
||||
|
||||
- **综合验收当前恢复正常**:`bash scripts/verify_phase6.sh` 返回 `SUMMARY pass=14 fail=0 warn=0` 与 `PHASE_RESULT: PASS`,说明主链路当前可运行。
|
||||
- **上一轮 FAIL 更像瞬时状态,不足以直接定性为结构性回归**:至少在本轮时间窗口内,Phase 3/Phase 6 未再失败。
|
||||
- **review 的长期主风险未变**:最后 commit 仍停在 `ba054f0`(2026-05-08),大量 modified/untracked 仍存在,导致“功能已做出但无版本锚点”的风险继续累积。
|
||||
- **CI 证据仍停留在 artifact-present**:`.github/` 虽存在,但仍未进入 git 历史,也没有本轮可引用的真实 workflow run 结果。
|
||||
- **工作区扩大至 23/+3650/-808 行**:scripts 新增 1619 行(generate_daily_report.go +1032 行、generate_daily_report_test.go +567 行);frontend 新增 ~834 行(Dashboard.tsx +534 行、Explorer.tsx +342 行);cmd/server 新增 ~535 行(main.go +274 行、main_test.go +261 行)。
|
||||
- **scripts 目录 go test build failure**:多个脚本(fetch_openrouter.go、fetch_multi_source.go、generate_daily_report.go、fetch_tencent_catalog.go、export_official_seed_json.go、cloudflare_pricing_signature_guard.go)存在 `main redeclared`、`ModelPricing redeclared`、`logger redeclared` 冲突,导致 `go test ./scripts` build FAIL。但 `go build ./cmd/server` 成功,不影响主服务构建。
|
||||
- **importer smoke 16 PASS 持续**:verify_importer_smoke.sh 全 PASS,采集链路健康。
|
||||
- **verify_phase4 ECharts FAIL 持续**:已持续 2+ 天,唯一 FAIL 项是 `[FAIL] Dashboard 已集成 ECharts`。
|
||||
|
||||
#### 问题 21(P1):验收脚本瞬时回归缺少稳定性标记(再次确认)
|
||||
#### 问题 45(新发现):scripts 目录 go test build failure(redeclared main)
|
||||
|
||||
- **09:30 状态**:上一轮 review 记录 `verify_phase6.sh` FAIL,本轮同命令恢复 PASS。
|
||||
- **影响**:
|
||||
- 单次 FAIL 容易被 review 写成结构性故障
|
||||
- backlog 会积累“本轮失败、下轮恢复”的噪声,降低长期可读性
|
||||
- 团队可能误把短时波动当成实现回归,分散精力
|
||||
- **15:10 状态**:`go test ./scripts` 输出大量 `main redeclared in this block` 和 `ModelPricing/logger redeclared` 错误。涉及脚本包括 fetch_openrouter.go、fetch_multi_source.go、generate_daily_report.go、fetch_tencent_catalog.go、export_official_seed_json.go、cloudflare_pricing_signature_guard.go 等。这些脚本在同一 main package 中共享符号。
|
||||
- **问题影响**:`go test ./scripts` 无法执行,scripts 目录的单元测试链路断裂;但 `go build ./cmd/server` 不受影响,主服务可正常构建。
|
||||
- **优化建议**:
|
||||
1. review prompt 中增加“单次 FAIL 先标记为 transient-suspect,连续复现或稳定复现后再升级为结构性问题”
|
||||
2. Phase 验收脚本失败后,若成本允许,自动补跑一次最小复验命令,区分瞬时波动与稳定故障
|
||||
3. backlog 条目增加“复现状态”字段,如 `single-hit / repeated / reproducible`
|
||||
- **建议验证方法**:后续若再次出现单轮 FAIL,要求下一轮或同轮最小复验后再决定是否升级 backlog 严重度
|
||||
1. 为 scripts 目录下的各脚本添加 `// +build ignore` build tag 或移至独立包,使每个脚本可独立构建
|
||||
2. 或者在 go test 命令中使用 `go test -tags ignore` 配合 build tag 排除冲突脚本
|
||||
3. 或者将共享类型(ModelPricing、logger)移至 internal/common 包,各脚本独立引用
|
||||
- **优先级**:P1
|
||||
- **建议验证方法**:修复后执行 `go test ./scripts` 无 build error;或 `go test -tags llm_script ./scripts` 全 PASS。
|
||||
|
||||
#### 问题 23(P0→待复核):日报归档路径门禁失配
|
||||
#### 问题 10 状态更新:项目提交停滞(影响次数 23)
|
||||
|
||||
- **09:30 状态**:本轮未复现。`bash scripts/verify_phase6.sh` 已整体 PASS,说明上一轮的 Phase 3/归档门禁异常当前不是稳定故障。
|
||||
- **影响**:
|
||||
- 若未来复现,仍会级联拖累综合验收判断
|
||||
- 但在本轮证据下,不应继续把它包装成“当前稳定存在的结构性 P0 故障”
|
||||
- **15:10 状态**:23 文件 +3650/-808 行核心组件改动未提交,含 generate_daily_report.go +1032 行大改、main_test.go +261 行、前端 Dashboard +534 行等关键业务代码。
|
||||
- **问题影响**:versioned truth 与 runtime truth 漂移加剧;scripts build failure 在 commit 前必须修复。
|
||||
- **优化建议**:立即按逻辑拆分为 2~3 个 commit(如"server 重构与测试"、"前端 Dashboard/Explorer 扩展"、"日报生成器大改");scripts build failure 需在 commit 前解决。
|
||||
- **优先级**:P0
|
||||
- **建议验证方法**:修复 scripts build failure 后提交;`git diff --stat HEAD` 变更量大幅收缩。
|
||||
|
||||
#### 问题 41 状态更新:live_run SUMMARY 缺失(影响次数 5)
|
||||
|
||||
- **15:10 状态**:verify_phase6.sh 在 30s 内退出,未输出 window_size / success_rate / live_run_result SUMMARY。连续超时问题已解决(连续第三次不超时),但 live_run SUMMARY 仍缺失。
|
||||
- **问题影响**:Phase 6 稳定性窗口 PASS/FAIL 状态无法通过脚本输出确认(但 importer smoke 全 PASS 说明采集链路健康)。
|
||||
- **优化建议**:同 05-26 15:10 记录。
|
||||
- **优先级**:P1(从 P0 降级,本轮连续超时未复现)
|
||||
- **建议验证方法**:修正后执行 verify_phase6.sh,确认输出完整 SUMMARY。
|
||||
|
||||
#### 问题 43 状态更新:verify_phase4 ECharts FAIL(影响次数 2)
|
||||
|
||||
- **15:10 状态**:verify_phase4 ECharts 断言失败已持续 2+ 天,本轮无变化。
|
||||
- **结论**:影响次数从 1 更新为 2 次。
|
||||
|
||||
#### 问题 29 状态更新:已修复(从 current 表移除)
|
||||
|
||||
#### 问题 31 状态更新:已修复(从 current 表移除)
|
||||
|
||||
#### 问题 34 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **旧缺口**:当 importer smoke 这类局部门禁已经恢复 PASS,但 phase 级主 blocker 已经转移到别的 gate(例如 `live_run` 或 `api_server`)时,输出里没有显式提示“全局 blocker 已切换”。结果是:读者容易继续把 smoke gate 当成当前主 blocker,而忽略真正还在阻断主链路的 gate。
|
||||
- **修复**:
|
||||
1. 新增 `scripts/review/global_blocker_switch_guard.sh`
|
||||
2. 新增 `scripts/review/global_blocker_switch_guard_test.sh`
|
||||
3. 新增 `scripts/review/global_blocker_switch_capture_test.sh`
|
||||
4. `verify_phase6.sh` 现在维护并输出:
|
||||
- `BLOCKER_SWITCH class=<...> old=<...> new=<...>`
|
||||
5. 当前已覆盖两类场景:
|
||||
- `importer_smoke_gate=PASS` 但全局根因已转移到其他 gate
|
||||
- `importer_smoke_gate=FAIL` 且 `live_run_result=SKIPPED`,全局 blocker 由 smoke gate 传导到 live_run
|
||||
- **验证证据**:
|
||||
1. `bash scripts/review/global_blocker_switch_guard_test.sh` → PASS
|
||||
2. `bash scripts/review/global_blocker_switch_capture_test.sh` → PASS
|
||||
- **结论**:问题 34 已关闭;局部 smoke 恢复或局部 smoke 传导导致的全局 blocker 切换,现在都会在 phase 级输出中被显式提示,不再靠读者自己脑补。
|
||||
#### 问题 33 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **旧缺口**:问题 12/32 虽然已经分别处理了 resolved 行清理和 same-day blocker 替换,但仍缺一个更直接的自动撤销机制:如果 review 日志里已经明确写出“问题 X 状态更新:已修复(从 current 表移除)”,current 表就不该继续保留这个问题。否则就会出现‘日志层已证伪,current truth 仍保留’的矛盾。
|
||||
- **修复**:
|
||||
1. 新增 `scripts/review/backlog_revocation_guard.sh`
|
||||
2. 新增 `scripts/review/backlog_revocation_guard_test.sh`
|
||||
3. guard 会扫描 backlog 中所有:
|
||||
- `#### 问题 X 状态更新:已修复(从 current 表移除)`
|
||||
并检查 current 表是否仍残留对应 issue id;若残留则直接 FAIL
|
||||
- **验证证据**:
|
||||
1. `bash scripts/review/backlog_revocation_guard_test.sh` → PASS
|
||||
- **结论**:问题 33 已关闭;已证伪/已宣告移除的 blocker 现在有了自动撤销 guard,不会再继续挂在 current truth 上自相矛盾。
|
||||
#### 问题 32 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **旧缺口**:问题 29 解决了 review 文本层的 same-day blocker switch 提示,但 backlog current truth 层仍没有同步约束。结果是:即使 review 已明确写出 `old -> new` 的 blocker 切换,旧 blocker 仍可能继续留在 current 表里,继续伪装成当前未修复项。
|
||||
- **修复**:
|
||||
1. 新增 `scripts/review/backlog_blocker_freshness_guard.sh`
|
||||
2. 新增 `scripts/review/backlog_blocker_freshness_guard_test.sh`
|
||||
3. 规则:一旦 backlog 文本中出现:
|
||||
- `freshness_hint=same-day-blocker-switch old=<...> new=<...>`
|
||||
guard 就会检查 current 表中是否还残留 `old` blocker;若残留则直接 FAIL
|
||||
4. 这样 same-day blocker 切换不只是在 prose 层有提示,也会约束 current truth 层必须同步更新
|
||||
- **验证证据**:
|
||||
1. `bash scripts/review/backlog_blocker_freshness_guard_test.sh` → PASS
|
||||
- **结论**:问题 32 已关闭;同日 blocker 切换后,旧 blocker 不能再继续滞留在 current 表里冒充最新真相。
|
||||
- **旧缺口**:问题 18 已经让 no-delta 场景输出 `aging_focus`,但还没有区分一种更尖锐的停滞态:同一天内没有新的主结论 / 没有新的 blocker 切换。此时 review 不只是“没变化”,而是“今天已经 review 过,但仍没有形成新的主判断”,需要更强的风险优先策略。
|
||||
- **修复**:
|
||||
1. `scripts/review/review_status_summary.sh` 新增:
|
||||
- `same_day_no_decision_focus=`
|
||||
2. 当前输出 top2 形式:
|
||||
- `same_day_no_decision_focus=<issue>:<priority>:<impact>,...`
|
||||
3. 新增 `scripts/review/review_same_day_no_decision_test.sh`
|
||||
4. 这样 no-delta 摘要不再只给一般 aging_focus,还会单独指出“同日无主结论”场景下最值得优先处理的问题
|
||||
- **验证证据**:
|
||||
1. `bash scripts/review/review_status_summary_test.sh` → PASS
|
||||
2. `bash scripts/review/review_aging_priority_test.sh` → PASS
|
||||
3. `bash scripts/review/review_same_day_no_decision_test.sh` → PASS
|
||||
- **结论**:问题 31 已关闭;同日 no-delta 现在不再只是一般 aging,而有独立的 same-day no-decision 风险优先输出。
|
||||
#### 问题 30 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **旧缺口**:当前链路已经能够把 `precondition_missing` 分类出来,但历史 precondition 样本仍会持续占据最近 7 次窗口。这样即使当前链路已经恢复绿色,success_rate 仍可能被很久以前的“缺钥/缺连接串”样本拖低,导致 release 语义长期停留在 degraded。
|
||||
- **修复**:
|
||||
1. 在 `collector_stats_window_audit.sh` 中新增:
|
||||
- `aged_precondition_missing`
|
||||
- `AGED_PRECONDITION_MINUTES=1440`
|
||||
2. 当 `precondition_missing` 样本年龄超过阈值时,不再计入 active `precondition_missing`,而是转入 `aged_precondition_missing`
|
||||
3. `SUCCESS_RATE` 的分母会剔除 aged precondition 样本,因此历史前置条件失败不会继续污染当前 release success-rate
|
||||
- **验证证据**:
|
||||
1. `bash scripts/collector_stats_window_audit_test.sh` → PASS
|
||||
2. aged 样例输出已含:
|
||||
- `precondition_missing=0`
|
||||
- `aged_precondition_missing=1`
|
||||
3. `bash scripts/verify_phase6.sh` → `PHASE_RESULT: PASS`
|
||||
- **结论**:问题 30 已关闭;历史 precondition 样本现在会老化出 active release 窗口,不再持续拖低当前 success-rate 与 release 判断。
|
||||
- **旧缺口**:当同一天内 review 的主 blocker 已经从 A 切换到 B(例如 `xfyun-live` 替换 `sensenova-live`)时,旧 blocker 仍可能继续残留或被复述出去,但报告中没有任何显式 freshness 提示告诉读者“这是同日 blocker 切换,不要继续把旧 blocker 当成当前主 blocker”。
|
||||
- **修复**:
|
||||
1. 新增 `scripts/review/blocker_switch_guard.sh`
|
||||
2. 新增 `scripts/review/blocker_switch_guard_test.sh`
|
||||
3. 规则:一旦 review 文本里出现“替换”语义,就必须同时出现:
|
||||
- `freshness_hint=same-day-blocker-switch old=<...> new=<...>`
|
||||
4. 这样同日 blocker 切换会被显式标记为 freshness 事件,而不再只是自然语言描述
|
||||
- **验证证据**:
|
||||
1. `bash scripts/review/blocker_switch_guard_test.sh` → PASS
|
||||
- **结论**:问题 29 已关闭;同日 blocker 切换现在会带 freshness_hint,旧 blocker 不再能在 review 链里无提示继续传播。
|
||||
### 2026-05-26 15:10(afternoon-review cron)
|
||||
|
||||
> **前置说明**:距上一次 review(05-25 15:10)约 **24 小时**。本轮距上次 afternoon review 无新 commit,工作区变更从 19 文件 +1372/-281 行增长到 22 文件 +2819/-466 行。verify_phase6.sh 连续超时问题(本轮跨三次 review 的 05-25 记录)本轮首次解决,importer smoke 全 PASS;但 live_run SUMMARY 仍缺失。PRE_PHASE6 FAIL(verify_phase4 ECharts 断言失败)。go test 全 PASS。
|
||||
|
||||
#### 本次新增发现
|
||||
|
||||
- **verify_phase6.sh 连续超时问题本轮消失**:本轮执行 `timeout 60 bash scripts/verify_phase6.sh` 在 60s 内完成,importer smoke 8 组全 PASS(coreshub/huawei-maas/baichuan/lingyiwanwu/sensenova/xfyun/bytedance 各 fixture+live PASS),gate PASS。但 live_run 仅触发 smokerun,脚本在 60s 内退出,**未输出 window_size / success_rate / live_run_result SUMMARY**。
|
||||
- **PRE_PHASE6 FAIL,根因是 verify_phase4 ECharts 断言失败**:`verify_pre_phase6.sh` → `PRE_PHASE6_RESULT: FAIL`,唯一 FAIL 项是 `[FAIL] Dashboard 已集成 ECharts`。Phase 1 PASS(9/9)、Phase 2 PASS(9/9)、Phase 3 PASS(17/17)、Phase 5 PASS(15/15)。
|
||||
- **工作区变更量增长**:22 文件 +2819/-466 行(含 cmd/server BasicAuth 重构 +261 行测试、main_test.go +261 行、前端 Dashboard/Explorer +876 行、日报生成器 +229/- 行),BACKLOG 本身也在未提交列表中。
|
||||
- **新增 untracked 项**:scripts/secret_gate_lib.sh(1846 字节)、scripts/secret_gate_test.sh(1823 字节)、scripts/testdata/empty.dockerignore(19 字节)、.agent/、.serena/、.dockerignore,均无门禁覆盖。
|
||||
|
||||
#### 问题 10 状态更新:项目提交停滞(影响次数 22)
|
||||
|
||||
- **15:10 状态**:22 文件 +2819/-466 行核心组件改动未提交,含 cmd/server BasicAuth/IP 限速/apiError 重构、main_test.go +261 行、前端 Dashboard/Explorer 大改(+534/-、+342/- 行)、日报生成器(+229/- 行)。BACKLOG 本身也在未提交列表中。
|
||||
- **问题影响**:versioned truth 与 runtime truth 漂移加剧;一旦工作区丢失则核心组件改动无法恢复;BACKLOG 持续未收敛使 review 成本递增。
|
||||
- **优化建议**:立即按逻辑拆分为 2~3 个 commit(如"server 重构与测试"、"前端 Dashboard/Explorer 扩展"、"日报生成器与门禁改进");review prompt 应在工作区变更量超过阈值时自动提升 commit 停滞优先级。
|
||||
- **优先级**:P0
|
||||
- **建议验证方法**:提交后检查 `git log --oneline` 出现新提交,`git diff --stat HEAD` 变更量大幅收缩。
|
||||
|
||||
#### 问题 41 状态更新:从"连续超时"降级为"live_run SUMMARY 缺失"(影响次数 4)
|
||||
|
||||
- **15:10 状态**:连续超时未在本轮复现(importer smoke 全 PASS,gate PASS),但 live_run SUMMARY(window_size / success_rate / live_run_result)仍未输出,脚本在 smokerun 后 60s 内退出。
|
||||
- **问题影响**:Phase 6 稳定性窗口 PASS/FAIL 状态无法确认;无法判断 05-25 的三次超时是外部文档站卡死还是脚本性能退化。
|
||||
- **优化建议**:
|
||||
1. 保留条目,但状态降级为“待复核/瞬时问题”
|
||||
2. 下次若再触发,必须同时保存失败时的期望路径与实际路径
|
||||
3. 在 review 里区分“当前活跃故障”和“历史单次异常”
|
||||
- **建议验证方法**:未来若再次出现 Phase 3 FAIL,立即单独执行 `bash scripts/verify_phase3.sh` 并采集路径证据;若连续两轮复现,再升回结构性问题
|
||||
1. 调查 verify_phase6.sh live_run 未输出完整 SUMMARY 的根因(60s 内退出但未打印 window / success_rate / live_run_result)
|
||||
2. 为 verify_phase6.sh 增加单次检查的独立超时控制,避免单次检查卡死导致整脚本超时
|
||||
3. 在 verify_phase6.sh 输出中增加"当前检查进度"标记
|
||||
- **优先级**:P0 → P1(本轮 importer smoke 全 PASS 说明不是持续卡死,但 live_run SUMMARY 缺失仍是 P1)
|
||||
- **建议验证方法**:修正后执行 verify_phase6.sh,确认能在 <120s 内输出完整 SUMMARY(含 window_size / success_rate / live_run_result)。
|
||||
|
||||
#### 问题 24(P1):综合验收错误聚合误导根因判断
|
||||
#### 问题 42 状态更新:已修复(从 backlog current 表移除)
|
||||
|
||||
- **09:30 状态**:本轮虽未触发 FAIL,但问题仍未修复,因为顶层脚本的失败聚合可读性并未被专门改进。
|
||||
- **影响**:
|
||||
- 下一次综合验收失败时,review 仍可能被顶层压缩输出误导
|
||||
- 人工下钻成本高,容易产生二次误报
|
||||
- **15:10 状态**:verify_phase6.sh 连续超时未在本轮复现,importer smoke 全 PASS。05-25 的三次连续超时更接近外部文档站临时卡死而非脚本性能退化。
|
||||
- **结论**:问题 42 从 current 表移除,归档至 review 日志。
|
||||
|
||||
#### 问题 43(新发现):verify_phase4 ECharts 集成断言失败(历史遗留 P2)
|
||||
|
||||
- **15:10 状态**:`[FAIL] Dashboard 已集成 ECharts` 是 verify_phase4 的唯一 FAIL 项。Dashboard.tsx 中已引入 `import * as echarts from 'echarts'` 和 `echarts.init()` 逻辑,但 verify 脚本断言逻辑与实际代码行为不匹配。
|
||||
- **问题影响**:导致 PRE_PHASE6 整体 FAIL;但不影响主采集链路(Phase 1/2/3 全 PASS,importer smoke 全 PASS);历史遗留问题(首现于 05-25 15:10 systematic review)。
|
||||
- **优化建议**:
|
||||
1. `verify_phase6.sh` 在调用 `verify_pre_phase6.sh` 失败时直接输出失败 phase 名称
|
||||
2. `verify_pre_phase6.sh` 增加失败 phase 列表摘要
|
||||
3. review prompt 固化“综合门禁 FAIL 必须下钻子 phase”规则
|
||||
- **建议验证方法**:人为制造单个子 phase 失败,确认顶层输出能直接定位到具体失败 phase 与失败项
|
||||
1. 更新 verify_phase4 中 ECharts 集成断言逻辑,使其与当前 Dashboard.tsx 的 echarts 使用方式一致
|
||||
2. 或者确认当前代码是否真正满足"已集成 ECharts"语义,若不满足则完成集成
|
||||
3. 考虑将 ECharts 相关断言降级为 WARNING 而非 FAIL,以区分"历史遗留 P2"与"真实 blocker"
|
||||
- **优先级**:P2
|
||||
- **建议验证方法**:`bash scripts/verify_phase4.sh` → SUMMARY pass=10 fail=0 warn=0,PRE_PHASE6_RESULT: PASS。
|
||||
|
||||
---
|
||||
#### 问题 44(新发现):新增 scripts 无门禁覆盖
|
||||
|
||||
## 已归档问题(修复后移入)
|
||||
- **15:10 状态**:scripts/secret_gate_lib.sh(1846 字节)、scripts/secret_gate_test.sh(1823 字节)、scripts/testdata/empty.dockerignore 为新增 untracked 项,无对应 verify 门禁验证其正确性。
|
||||
- **问题影响**:新增安全类脚本无法确认是否正确落地;一旦工作区切换或代码丢失,这些脚本的存在和正确性无法追溯。
|
||||
- **优化建议**:
|
||||
1. 为 secret_gate_lib.sh / secret_gate_test.sh 建立对应的 smoke gate 或单元测试门禁
|
||||
2. 考虑在 verify_phase5 或 verify_phase6 中增加对新 scripts 目录的覆盖检查
|
||||
- **优先级**:P2
|
||||
- **建议验证方法**:执行 `bash scripts/secret_gate_test.sh` 验证其正确性,并确认门禁已纳入综合验收。
|
||||
|
||||
### 2026-05-10 14:30 — 问题 1 归档:验证器 `rg` 依赖误报
|
||||
#### 问题 13 状态更新:untracked 核心代码重新活跃(影响次数 14)
|
||||
|
||||
- **首次暴露**:2026-05-07 22:50
|
||||
- **修复时间**:2026-05-10 14:30 前
|
||||
- **修复方式**:`TASKS.md` 中 T-1.1 和 T-3.2 的验证命令从 `rg -n` 替换为 `grep -nE`
|
||||
- **验证方法**:`go run scripts/verification_executor.go` 在无 `rg` 环境下返回 PASS
|
||||
- **残余注意**:验证器本身仍未实现 toolchain readiness check 和三级状态
|
||||
- **15:10 状态**:scripts/secret_gate_lib.sh / secret_gate_test.sh 为新增 untracked 安全类脚本;BACKLOG 本身也在未提交列表中;.agent/、.serena/ 等目录长期未治理。
|
||||
- **问题影响**:同问题 10;untracked 列表持续增长增加了 versioned truth 漂移风险。
|
||||
- **优化建议**:同问题 10;尽快提交工作区变更,清理非必要 untracked 项。
|
||||
- **优先级**:P0
|
||||
- **建议验证方法**:提交后 `git status --short` 中 untracked 列表显著收缩。
|
||||
|
||||
### 2026-05-11 14:30 — 问题 17 归档:collection_stats vs collector_stats 表名不一致
|
||||
#### 问题 38 状态更新:PRE_PHASE6_RESULT 标签冲突(影响次数 4)
|
||||
|
||||
- **首次暴露**:2026-05-11 09:30(误报)
|
||||
- **澄清时间**:2026-05-11 14:30
|
||||
- **澄清方式**:二次验证 `grep -n "collector_stats" scripts/verify_phase2.sh` 确认脚本与 schema 一致
|
||||
- **根因**:09:30 review 未实际验证即复制了错误结论
|
||||
- **教训**:review 中的 "不一致" 声称必须二次验证,不能仅凭记忆或旧报告复制
|
||||
- **15:10 状态**:verify_phase4 ECharts 断言失败导致 PRE_PHASE6 FAIL;但 verify_phase4 内部 SUMMARY 显示 pass=9 fail=1 warn=0,说明是单一断言失败而非系统性卡死。
|
||||
- **问题影响**:PRE_PHASE6 FAIL 的根因已明确为 verify_phase4 ECharts 断言问题(历史 P2),不影响主链路;但标签冲突使 reviewer 需要额外下钻才能判断真实阶段。
|
||||
- **优化建议**:将 verify_phase4 中的 ECharts 相关断言降级为 WARNING,或更新断言逻辑使其与当前 Dashboard.tsx echarts 使用方式一致。
|
||||
- **优先级**:P1
|
||||
- **建议验证方法**:verify_phase4 中 ECharts 断言修复后,PRE_PHASE6_RESULT 应回到 PASS。
|
||||
|
||||
---
|
||||
### 2026-05-25 15:10(afternoon-review cron,第 41 次 review)
|
||||
|
||||
*Backlog 最后更新:2026-05-13 09:30 Asia/Shanghai*
|
||||
> **前置说明**:距上一次 review(05-25 08:59)约 **6 小时 11 分钟**。本轮无新 delta:working tree 仍 19 文件未提交(与 08:59 systematic review 完全一致),无新 commit。verify_phase6.sh 第三次连续超时(09:06 morning → 09:06 systematic → 15:10 afternoon),Phase 6 live blocker 状态完全无法确认。Phase 1~5 PASS,go test 全 PASS,日报已生成,但所有 systematic review 修复落地项(.dockerignore、runtimeVisibility、BasicAuth、Explorer.tsx 部分修复)均未 commit。
|
||||
|
||||
#### 本次新增发现
|
||||
|
||||
- **verify_phase6.sh 第三次连续超时**:本轮执行 `timeout 180 bash scripts/verify_phase6.sh`,>200s 无输出,连续第三次(09:06 morning / 09:06 systematic / 15:10 afternoon)。Phase 6 live blocker 状态(Zhipu 403 是否仍活跃、是否已消失或切换到新外部源)完全无法确认。
|
||||
- **Phase 1~5 门禁全 PASS**:`verify_pre_phase6.sh` 输出 `PRE_PHASE6_RESULT: PASS`,SUMMARY pass=15 fail=0 warn=0,与历史一致。
|
||||
- **Working tree 状态与 08:59 systematic review 完全一致**:19 文件 +1372/-281 行仍未提交,包含 .dockerignore、runtimeVisibility.ts、BasicAuth 实现、Explorer.tsx 部分修复等 systematic review 所有 P0/P1 修复落地项。
|
||||
- **systematic review P0-3 修复已落地但未 commit**:`.dockerignore` 已创建(285 字节,12:03 创建,artifact-present),`frontend/src/lib/runtimeVisibility.ts` + `runtimeVisibility.test.ts` 已创建。
|
||||
- **Explorer.tsx fallback 修复尚未完整验证**:runtimeVisibility.ts 已就绪但 Explorer.tsx 中只引入了部分 notice 构建逻辑,未完全实现"禁止静默 fallback"的 P0-2 修复目标。
|
||||
- **整体项目状态无新 delta**:距上次 review 6+ 小时,无新 commit,无新 runtime 证据,主链路健康(API 200,日报已生成)。
|
||||
|
||||
#### 问题 42(新发现):verify_phase6.sh 第三次连续超时,Phase 6 live blocker 状态完全不明
|
||||
|
||||
- **15:10 状态**:连续三次 verify_phase6.sh 超时(09:06 morning / 09:06 systematic / 15:10 afternoon),均无法在 180s 内完成并输出 Phase 6 SUMMARY。这不是偶发性问题,而是持续性卡死——可能存在外部文档站持续卡死或脚本本身性能退化。
|
||||
- **问题影响**:
|
||||
- Phase 6 综合门禁 PASS/FAIL 完全不明,连续三次 review 均无法给出准确的阶段判断
|
||||
- 无法确认 Zhipu 403 blocker 是否仍活跃、是否已消失还是切换到新的外部源
|
||||
- 外部文档站可能存在新的持续卡死,需要立即调查超时根因
|
||||
- **优化建议**:
|
||||
1. 调查 verify_phase6.sh 超时根因:单次外部文档站卡死 vs 整体脚本性能退化
|
||||
2. 为 verify_phase6.sh 增加单次检查的独立超时控制,避免单次检查卡死导致整脚本超时
|
||||
3. 在 verify_phase6.sh 输出中增加"当前检查进度"标记,方便定位卡死环节
|
||||
4. 在 verify_phase6.sh 中为连续超时的外部 URL 建立快速失败策略
|
||||
- **优先级**:P0
|
||||
- **建议验证方法**:修正后执行 verify_phase6.sh,确认能在 <120s 内完成并输出完整 SUMMARY(含 window_size / success_rate / live_run_result)
|
||||
|
||||
#### 问题 40 状态更新:优先级升级,影响次数更新
|
||||
|
||||
- **15:10 状态**:问题 40 自 08:51 首现,已持续 6+ 小时未解决,working tree 仍包含 systematic review 所有 P0/P1 修复落地项。优先级从 P2 升级为 P1(因为现在包含 P0 修复落地项的未 commit 风险);影响次数从 2 更新为 3 次。
|
||||
- **结论**:优先级从 P2 升级为 P1,影响次数从 2 更新为 3 次。
|
||||
|
||||
#### 问题 38 状态更新:PRE_PHASE6_RESULT 标签冲突仍待系统性修复
|
||||
|
||||
- **15:10 状态**:问题 38 影响次数从 2 更新为 3 次。PRE_PHASE6_RESULT 标签逻辑本身仍未系统性修复。
|
||||
- **结论**:影响次数从 2 更新为 3 次。
|
||||
|
||||
#### 问题 39 状态更新:日报时间戳异常仍未修复
|
||||
|
||||
- **15:10 状态**:问题 39 影响次数从 2 更新为 3 次。generated_at 仍显示 2026-05-25T19:03:55+08:00,比实际时间晚约 10 小时,与 08:51 / 08:59 记录一致。
|
||||
- **结论**:影响次数从 2 更新为 3 次。
|
||||
|
||||
### 2026-05-25 09:06(night-review cron,第 40 次 review)
|
||||
|
||||
> **前置说明**:距上一次 review(05-25 08:59)约 **7 分钟**。本轮属于"无新 delta 且 verify_phase6.sh 异常超时":无新 commit,Phase 1~5 门禁仍全 PASS,但 verify_phase6.sh 连续两次执行超时(>180s)导致 Phase 6 live blocker 状态无法确认。BACKLOG 文件 uncommitted 已持续 75 分钟+(08:51 → 08:59 → 09:06)。
|
||||
|
||||
#### 本次新增发现
|
||||
|
||||
- **verify_phase6.sh 连续两次超时**:本轮 review 两次执行 `bash scripts/verify_phase6.sh`,第一次在 90s 内完成了前 30 个 importer smoke 全 PASS 但未输出最终 SUMMARY;第二次直接超时(>180s 无法完成)。Phase 6 live blocker 状态(Zhipu 403 是否仍活跃)无法本轮真实验证。
|
||||
- **Phase 1~5 门禁仍然全 PASS**:`verify_pre_phase6.sh` 输出 `PRE_PHASE6_RESULT: PASS`,与上一轮一致,无变化。
|
||||
- **BACKLOG 文件 uncommitted 已持续 75 分钟+**:问题 40 从 08:51 首现,08:59 仍存在,09:06 仍未解决,已跨三轮 review 无收敛动作。
|
||||
- **日报时间戳异常仍未改善**:`daily_report_2026-05-25.md` 的 `generated_at: 2026-05-25T19:03:55+08:00` 比实际时间(09:06)晚约 10 小时,与 08:51 / 08:59 记录一致。
|
||||
|
||||
#### 问题 41(新发现):verify_phase6.sh 连续超时导致 Phase 6 live blocker 状态无法确认
|
||||
|
||||
- **09:06 状态**:本轮 review 连续两次执行 `bash scripts/verify_phase6.sh`,均无法在合理时间内完成。第一次在前 90s 内完成了 30 个 importer smoke 全 PASS 但未输出最终 SUMMARY;第二次直接超时(>180s 无法完成)。
|
||||
- **问题影响**:
|
||||
- Phase 6 综合门禁 PASS/FAIL 状态无法确认,reviewer 无法给出准确的阶段判断
|
||||
- 上一轮(08:59)记录的 Zhipu 403 blocker 是否仍活跃、是否已切换,本轮无法验证
|
||||
- 超时可能与 Zhipu 403 或其他外部文档站卡死有关,需要调查根因
|
||||
- **优化建议**:
|
||||
1. 调查 verify_phase6.sh 超时根因:单次外部文档站拉取卡死 vs 整体脚本性能退化
|
||||
2. 为 verify_phase6.sh 增加单次检查的独立超时控制,避免单次检查卡死导致整脚本超时
|
||||
3. 在 verify_phase6.sh 输出中增加"当前检查进度"标记,方便定位卡死环节
|
||||
- **优先级**:P1
|
||||
- **建议验证方法**:修正后执行 verify_phase6.sh,确认能在 <120s 内完成并输出完整 SUMMARY(含 window_size / success_rate / live_run_result)
|
||||
|
||||
#### 问题 37 状态更新:外部文档站故障仍无系统化降级
|
||||
|
||||
- **09:06 状态**:问题 37 仍活跃,影响次数从 3 更新为 4 次。本轮 verify_phase6 超时可能与外部文档站卡死有关(可能是 Zhipu 403 或其他源),blocker 在不同外部源之间游走的模式持续。
|
||||
- **结论**:从"3 次"更新为"4 次"。
|
||||
|
||||
#### 问题 39 状态更新:日报时间戳异常仍未改善
|
||||
|
||||
- **09:06 状态**:generated_at 仍显示 2026-05-25T19:03:55+08:00,比实际时间晚约 10 小时,无修复动作。
|
||||
- **结论**:影响次数从 1 更新为 2 次。
|
||||
|
||||
#### 问题 40 状态更新:BACKLOG uncommitted 已持续 75 分钟+
|
||||
|
||||
- **09:06 状态**:问题 40 已从 08:51 首现(morning review 修改 BACKLOG 后未 commit),08:59 仍存在,09:06 仍未解决,跨三轮 review 无收敛动作。
|
||||
- **结论**:影响次数从 1 更新为 2 次。
|
||||
|
||||
|
||||
## Review 日志
|
||||
|
||||
### 2026-05-29 15:10(afternoon-review cron)
|
||||
|
||||
> **前置说明**:距上一次 review(05-28 15:10)约 **24 小时**。距最后一次 commit(88833fa,05-27 22:01)约 17 小时,无新 commit。工作区在 05-28 review 之后重新积累 75 modified + 46 untracked 共 121 个文件变更。Phase 1/2/3/4/5 全 PASS(ECharts FAIL 已消失);Phase 6 FAIL(xfyun-live smoke FAIL);window_gate 全绿(7/7 success_rate=100%);daily report 已生成(22:01)但 cron 两次失败;BACKLOG freshness + table 双 guard FAIL。
|
||||
|
||||
#### 本次新增发现
|
||||
|
||||
- **工作区重新污染 P0**:05-28 review 之后,工作区从干净状态重新积累 75 modified + 46 untracked 共 121 个文件变更,包括 BACKLOG 本身也在 modified 中;BACKLOG freshness guard FAIL(stale current truth snapshot);BACKLOG table guard FAIL(resolved rows=13)
|
||||
- **xfyun-live smoke FAIL 替换 sensenova-live**:本轮 Phase 6 FAIL 根因从 sensenova 切换为 xfyun(chromium render timeout after 45s);与上轮 sensenova 属于同类外部 provider 渲染超时问题
|
||||
- **live_run 被 SKIP 传导链**:xfyun smoke FAIL 导致 `live_run_result=SKIPPED`,即使 window_gate 全绿(7/7),主链路健康状态也无法被本轮验收确认
|
||||
- **ECharts FAIL 已消失**:verify_phase4 恢复 PASS(10/10),问题 38 确认关闭
|
||||
|
||||
#### 问题 47 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **首次暴露**:2026-05-29 15:10
|
||||
- **修复**:治理修复工作区已通过两次本地提交收口:
|
||||
1. `e999d31` — `fix: harden review and verifier governance`
|
||||
2. `d7455b8` — `docs: reconcile openclaw backlog truth`
|
||||
- **当前状态**:`git status --short` 已恢复为空;工作区污染已完全解除。
|
||||
- **验证证据**:
|
||||
1. `git status --short` → 无输出
|
||||
2. `bash scripts/review/backlog_current_table_guard.sh reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md` → `resolved_rows=0`
|
||||
3. `LLM_NOW='2026-05-29 18:52' bash scripts/review/backlog_current_freshness_guard.sh reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md` → `status=fresh`
|
||||
- **结论**:问题 47 已关闭;工作区严重污染 P0 已被提交收口,不再是当前 blocker。
|
||||
|
||||
|
||||
|
||||
#### 问题 48 状态更新:已恢复(从 current 表移除)
|
||||
|
||||
- **首次暴露**:2026-05-29 15:10
|
||||
- **旧症状**:`xfyun-live` smoke FAIL 导致 `live_run_result=SKIPPED`。
|
||||
- **当前状态**:同类 smoke/live 传导链问题已不再稳定复现;最新 `verify_importer_smoke.sh` 与 `verify_phase6.sh` 均恢复通过。
|
||||
- **验证证据**:
|
||||
1. `bash scripts/verify_importer_smoke.sh` → `IMPORTER_SMOKE_RESULT: PASS`
|
||||
2. `bash scripts/verify_phase6.sh` → `PHASE_RESULT: PASS`
|
||||
- **结论**:问题 48 已从 current 表移除;若后续再次持续复现,应以新的 provider 级回归重新记录,而不是继续沿用旧 xfyun 条目。
|
||||
|
||||
|
||||
#### 问题 49 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **首次暴露**:2026-05-29 15:10
|
||||
- **根因复核**:三次 cron failed(10:32 / 10:58 / 18:44)根因一致,都是 `OPENROUTER_API_KEY` 缺失触发的 strict-real precondition 失败,并不存在“cron 失败但 22:01 又成功且未闭环”的真实矛盾。
|
||||
- **修复**:
|
||||
1. `run_daily.sh` 在 failure 路径现在会把这类错误显式标成 `precondition_missing`
|
||||
2. `cron_status_report.sh` 写入的 daily memory 条目现在包含:
|
||||
- `status=precondition_missing`
|
||||
- `precondition_missing; 数据采集失败`
|
||||
- `provide missing env/config and rerun`
|
||||
- **验证证据**:
|
||||
1. `bash scripts/cron_precondition_integration_test.sh` → PASS
|
||||
2. 手工执行 `bash scripts/run_daily.sh` 后,`memory/2026-05-29.md` 最新 cron 条目已写成 `status=precondition_missing`
|
||||
- **结论**:问题 49 已关闭;cron failed 现在已形成可解释闭环,不再只是笼统的 failed 条目。
|
||||
|
||||
|
||||
#### 问题 50 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **首次暴露**:2026-05-29 15:10
|
||||
- **修复**:
|
||||
1. current 表中的 resolved 行已清理完毕,仅保留当前未修复问题
|
||||
2. current table timestamp 已刷新到最新 review 时间
|
||||
3. backlog current table guard 与 freshness guard 已恢复绿色
|
||||
- **验证证据**:
|
||||
1. `bash scripts/review/backlog_current_table_guard.sh reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md` → `resolved_rows=0`
|
||||
2. `bash scripts/review/backlog_current_freshness_guard.sh reports/openclaw/OPENCLAW_CAPABILITY_BACKLOG.md` → `status=fresh`
|
||||
- **结论**:问题 50 已关闭;BACKLOG current table 已恢复 current-truth 语义与 freshness。
|
||||
|
||||
### 2026-05-27 15:10(afternoon-review cron)
|
||||
|
||||
> **前置说明**:距上一次 review(05-26 15:10)约 **24 小时**。无新 commit。工作区从 22/+2819/-466 行扩大至 23/+3650/-808 行。scripts 新增 1619 行(主要是 generate_daily_report.go +1032 行及其测试 +567 行)。importer smoke 16 PASS 持续。ECharts FAIL 持续 2+ 天。scripts 目录 go test 出现 redeclared main build failure(新增 P1 gap)。
|
||||
|
||||
#### 本次新增发现
|
||||
|
||||
- **工作区扩大至 23/+3650/-808 行**:scripts 新增 1619 行(generate_daily_report.go +1032 行、generate_daily_report_test.go +567 行);frontend 新增 ~834 行(Dashboard.tsx +534 行、Explorer.tsx +342 行);cmd/server 新增 ~535 行(main.go +274 行、main_test.go +261 行)。
|
||||
- **scripts 目录 go test build failure**:多个脚本(fetch_openrouter.go、fetch_multi_source.go、generate_daily_report.go、fetch_tencent_catalog.go、export_official_seed_json.go、cloudflare_pricing_signature_guard.go)存在 `main redeclared`、`ModelPricing redeclared`、`logger redeclared` 冲突,导致 `go test ./scripts` build FAIL。但 `go build ./cmd/server` 成功,不影响主服务构建。
|
||||
- **importer smoke 16 PASS 持续**:verify_importer_smoke.sh 全 PASS,采集链路健康。
|
||||
- **verify_phase4 ECharts FAIL 持续**:已持续 2+ 天,唯一 FAIL 项是 `[FAIL] Dashboard 已集成 ECharts`。
|
||||
|
||||
#### 问题 45(新发现):scripts 目录 go test build failure(redeclared main)
|
||||
|
||||
- **15:10 状态**:`go test ./scripts` 输出大量 `main redeclared in this block` 和 `ModelPricing/logger redeclared` 错误。涉及脚本包括 fetch_openrouter.go、fetch_multi_source.go、generate_daily_report.go、fetch_tencent_catalog.go、export_official_seed_json.go、cloudflare_pricing_signature_guard.go 等。这些脚本在同一 main package 中共享符号。
|
||||
- **问题影响**:`go test ./scripts` 无法执行,scripts 目录的单元测试链路断裂;但 `go build ./cmd/server` 不受影响,主服务可正常构建。
|
||||
- **优化建议**:
|
||||
1. 为 scripts 目录下的各脚本添加 `// +build ignore` build tag 或移至独立包,使每个脚本可独立构建
|
||||
2. 或者在 go test 命令中使用 `go test -tags ignore` 配合 build tag 排除冲突脚本
|
||||
3. 或者将共享类型(ModelPricing、logger)移至 internal/common 包,各脚本独立引用
|
||||
- **优先级**:P1
|
||||
- **建议验证方法**:修复后执行 `go test ./scripts` 无 build error;或 `go test -tags llm_script ./scripts` 全 PASS。
|
||||
|
||||
#### 问题 10 状态更新:项目提交停滞(影响次数 23)
|
||||
|
||||
- **15:10 状态**:23 文件 +3650/-808 行核心组件改动未提交,含 generate_daily_report.go +1032 行大改、main_test.go +261 行、前端 Dashboard +534 行等关键业务代码。
|
||||
- **问题影响**:versioned truth 与 runtime truth 漂移加剧;scripts build failure 在 commit 前必须修复。
|
||||
- **优化建议**:立即按逻辑拆分为 2~3 个 commit(如"server 重构与测试"、"前端 Dashboard/Explorer 扩展"、"日报生成器大改");scripts build failure 需在 commit 前解决。
|
||||
- **优先级**:P0
|
||||
- **建议验证方法**:修复 scripts build failure 后提交;`git diff --stat HEAD` 变更量大幅收缩。
|
||||
|
||||
#### 问题 41 状态更新:live_run SUMMARY 缺失(影响次数 5)
|
||||
|
||||
- **15:10 状态**:verify_phase6.sh 在 30s 内退出,未输出 window_size / success_rate / live_run_result SUMMARY。连续超时问题已解决(连续第三次不超时),但 live_run SUMMARY 仍缺失。
|
||||
- **问题影响**:Phase 6 稳定性窗口 PASS/FAIL 状态无法通过脚本输出确认(但 importer smoke 全 PASS 说明采集链路健康)。
|
||||
- **优化建议**:同 05-26 15:10 记录。
|
||||
- **优先级**:P1(从 P0 降级,本轮连续超时未复现)
|
||||
- **建议验证方法**:修正后执行 verify_phase6.sh,确认输出完整 SUMMARY。
|
||||
|
||||
#### 问题 43 状态更新:verify_phase4 ECharts FAIL(影响次数 2)
|
||||
|
||||
- **15:10 状态**:verify_phase4 ECharts 断言失败已持续 2+ 天,本轮无变化。
|
||||
- **结论**:影响次数从 1 更新为 2 次。
|
||||
|
||||
#### 问题 29 状态更新:已修复(从 current 表移除)
|
||||
|
||||
#### 问题 31 状态更新:已修复(从 current 表移除)
|
||||
|
||||
#### 问题 34 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **旧缺口**:当 importer smoke 这类局部门禁已经恢复 PASS,但 phase 级主 blocker 已经转移到别的 gate(例如 `live_run` 或 `api_server`)时,输出里没有显式提示“全局 blocker 已切换”。结果是:读者容易继续把 smoke gate 当成当前主 blocker,而忽略真正还在阻断主链路的 gate。
|
||||
- **修复**:
|
||||
1. 新增 `scripts/review/global_blocker_switch_guard.sh`
|
||||
2. 新增 `scripts/review/global_blocker_switch_guard_test.sh`
|
||||
3. 新增 `scripts/review/global_blocker_switch_capture_test.sh`
|
||||
4. `verify_phase6.sh` 现在维护并输出:
|
||||
- `BLOCKER_SWITCH class=<...> old=<...> new=<...>`
|
||||
5. 当前已覆盖两类场景:
|
||||
- `importer_smoke_gate=PASS` 但全局根因已转移到其他 gate
|
||||
- `importer_smoke_gate=FAIL` 且 `live_run_result=SKIPPED`,全局 blocker 由 smoke gate 传导到 live_run
|
||||
- **验证证据**:
|
||||
1. `bash scripts/review/global_blocker_switch_guard_test.sh` → PASS
|
||||
2. `bash scripts/review/global_blocker_switch_capture_test.sh` → PASS
|
||||
- **结论**:问题 34 已关闭;局部 smoke 恢复或局部 smoke 传导导致的全局 blocker 切换,现在都会在 phase 级输出中被显式提示,不再靠读者自己脑补。
|
||||
#### 问题 33 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **旧缺口**:问题 12/32 虽然已经分别处理了 resolved 行清理和 same-day blocker 替换,但仍缺一个更直接的自动撤销机制:如果 review 日志里已经明确写出“问题 X 状态更新:已修复(从 current 表移除)”,current 表就不该继续保留这个问题。否则就会出现‘日志层已证伪,current truth 仍保留’的矛盾。
|
||||
- **修复**:
|
||||
1. 新增 `scripts/review/backlog_revocation_guard.sh`
|
||||
2. 新增 `scripts/review/backlog_revocation_guard_test.sh`
|
||||
3. guard 会扫描 backlog 中所有:
|
||||
- `#### 问题 X 状态更新:已修复(从 current 表移除)`
|
||||
并检查 current 表是否仍残留对应 issue id;若残留则直接 FAIL
|
||||
- **验证证据**:
|
||||
1. `bash scripts/review/backlog_revocation_guard_test.sh` → PASS
|
||||
- **结论**:问题 33 已关闭;已证伪/已宣告移除的 blocker 现在有了自动撤销 guard,不会再继续挂在 current truth 上自相矛盾。
|
||||
#### 问题 32 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **旧缺口**:问题 29 解决了 review 文本层的 same-day blocker switch 提示,但 backlog current truth 层仍没有同步约束。结果是:即使 review 已明确写出 `old -> new` 的 blocker 切换,旧 blocker 仍可能继续留在 current 表里,继续伪装成当前未修复项。
|
||||
- **修复**:
|
||||
1. 新增 `scripts/review/backlog_blocker_freshness_guard.sh`
|
||||
2. 新增 `scripts/review/backlog_blocker_freshness_guard_test.sh`
|
||||
3. 规则:一旦 backlog 文本中出现:
|
||||
- `freshness_hint=same-day-blocker-switch old=<...> new=<...>`
|
||||
guard 就会检查 current 表中是否还残留 `old` blocker;若残留则直接 FAIL
|
||||
4. 这样 same-day blocker 切换不只是在 prose 层有提示,也会约束 current truth 层必须同步更新
|
||||
- **验证证据**:
|
||||
1. `bash scripts/review/backlog_blocker_freshness_guard_test.sh` → PASS
|
||||
- **结论**:问题 32 已关闭;同日 blocker 切换后,旧 blocker 不能再继续滞留在 current 表里冒充最新真相。
|
||||
- **旧缺口**:问题 18 已经让 no-delta 场景输出 `aging_focus`,但还没有区分一种更尖锐的停滞态:同一天内没有新的主结论 / 没有新的 blocker 切换。此时 review 不只是“没变化”,而是“今天已经 review 过,但仍没有形成新的主判断”,需要更强的风险优先策略。
|
||||
- **修复**:
|
||||
1. `scripts/review/review_status_summary.sh` 新增:
|
||||
- `same_day_no_decision_focus=`
|
||||
2. 当前输出 top2 形式:
|
||||
- `same_day_no_decision_focus=<issue>:<priority>:<impact>,...`
|
||||
3. 新增 `scripts/review/review_same_day_no_decision_test.sh`
|
||||
4. 这样 no-delta 摘要不再只给一般 aging_focus,还会单独指出“同日无主结论”场景下最值得优先处理的问题
|
||||
- **验证证据**:
|
||||
1. `bash scripts/review/review_status_summary_test.sh` → PASS
|
||||
2. `bash scripts/review/review_aging_priority_test.sh` → PASS
|
||||
3. `bash scripts/review/review_same_day_no_decision_test.sh` → PASS
|
||||
- **结论**:问题 31 已关闭;同日 no-delta 现在不再只是一般 aging,而有独立的 same-day no-decision 风险优先输出。
|
||||
#### 问题 30 状态更新:已修复(从 current 表移除)
|
||||
|
||||
- **旧缺口**:当前链路已经能够把 `precondition_missing` 分类出来,但历史 precondition 样本仍会持续占据最近 7 次窗口。这样即使当前链路已经恢复绿色,success_rate 仍可能被很久以前的“缺钥/缺连接串”样本拖低,导致 release 语义长期停留在 degraded。
|
||||
- **修复**:
|
||||
1. 在 `collector_stats_window_audit.sh` 中新增:
|
||||
- `aged_precondition_missing`
|
||||
- `AGED_PRECONDITION_MINUTES=1440`
|
||||
2. 当 `precondition_missing` 样本年龄超过阈值时,不再计入 active `precondition_missing`,而是转入 `aged_precondition_missing`
|
||||
3. `SUCCESS_RATE` 的分母会剔除 aged precondition 样本,因此历史前置条件失败不会继续污染当前 release success-rate
|
||||
- **验证证据**:
|
||||
1. `bash scripts/collector_stats_window_audit_test.sh` → PASS
|
||||
2. aged 样例输出已含:
|
||||
- `precondition_missing=0`
|
||||
- `aged_precondition_missing=1`
|
||||
3. `bash scripts/verify_phase6.sh` → `PHASE_RESULT: PASS`
|
||||
- **结论**:问题 30 已关闭;历史 precondition 样本现在会老化出 active release 窗口,不再持续拖低当前 success-rate 与 release 判断。
|
||||
- **旧缺口**:当同一天内 review 的主 blocker 已经从 A 切换到 B(例如 `xfyun-live` 替换 `sensenova-live`)时,旧 blocker 仍可能继续残留或被复述出去,但报告中没有任何显式 freshness 提示告诉读者“这是同日 blocker 切换,不要继续把旧 blocker 当成当前主 blocker”。
|
||||
- **修复**:
|
||||
1. 新增 `scripts/review/blocker_switch_guard.sh`
|
||||
2. 新增 `scripts/review/blocker_switch_guard_test.sh`
|
||||
3. 规则:一旦 review 文本里出现“替换”语义,就必须同时出现:
|
||||
- `freshness_hint=same-day-blocker-switch old=<...> new=<...>`
|
||||
4. 这样同日 blocker 切换会被显式标记为 freshness 事件,而不再只是自然语言描述
|
||||
- **验证证据**:
|
||||
1. `bash scripts/review/blocker_switch_guard_test.sh` → PASS
|
||||
- **结论**:问题 29 已关闭;同日 blocker 切换现在会带 freshness_hint,旧 blocker 不再能在 review 链里无提示继续传播。
|
||||
### 2026-05-26 15:10(afternoon-review cron)
|
||||
|
||||
> **前置说明**:距上一次 review(05-25 15:10)约 **24 小时**。本轮距上次 afternoon review 无新 commit,工作区变更从 19 文件 +1372/-281 行增长到 22 文件 +2819/-466 行。verify_phase6.sh 连续超时问题(本轮跨三次 review 的 05-25 记录)本轮首次解决,importer smoke 全 PASS;但 live_run SUMMARY 仍缺失。PRE_PHASE6 FAIL(verify_phase4 ECharts 断言失败)。go test 全 PASS。
|
||||
|
||||
#### 本次新增发现
|
||||
|
||||
- **verify_phase6.sh 连续超时问题本轮消失**:本轮执行 `timeout 60 bash scripts/verify_phase6.sh` 在 60s 内完成,importer smoke 8 组全 PASS(coreshub/huawei-maas/baichuan/lingyiwanwu/sensenova/xfyun/bytedance 各 fixture+live PASS),gate PASS。但 live_run 仅触发 smokerun,脚本在 60s 内退出,**未输出 window_size / success_rate / live_run_result SUMMARY**。
|
||||
- **PRE_PHASE6 FAIL,根因是 verify_phase4 ECharts 断言失败**:`verify_pre_phase6.sh` → `PRE_PHASE6_RESULT: FAIL`,唯一 FAIL 项是 `[FAIL] Dashboard 已集成 ECharts`。Phase 1 PASS(9/9)、Phase 2 PASS(9/9)、Phase 3 PASS(17/17)、Phase 5 PASS(15/15)。
|
||||
- **工作区变更量增长**:22 文件 +2819/-466 行(含 cmd/server BasicAuth 重构 +261 行测试、main_test.go +261 行、前端 Dashboard/Explorer +876 行、日报生成器 +229/- 行),BACKLOG 本身也在未提交列表中。
|
||||
- **新增 untracked 项**:scripts/secret_gate_lib.sh(1846 字节)、scripts/secret_gate_test.sh(1823 字节)、scripts/testdata/empty.dockerignore(19 字节)、.agent/、.serena/、.dockerignore,均无门禁覆盖。
|
||||
|
||||
#### 问题 10 状态更新:项目提交停滞(影响次数 22)
|
||||
|
||||
- **15:10 状态**:22 文件 +2819/-466 行核心组件改动未提交,含 cmd/server BasicAuth/IP 限速/apiError 重构、main_test.go +261 行、前端 Dashboard/Explorer 大改(+534/-、+342/- 行)、日报生成器(+229/- 行)。BACKLOG 本身也在未提交列表中。
|
||||
- **问题影响**:versioned truth 与 runtime truth 漂移加剧;一旦工作区丢失则核心组件改动无法恢复;BACKLOG 持续未收敛使 review 成本递增。
|
||||
- **优化建议**:立即按逻辑拆分为 2~3 个 commit(如"server 重构与测试"、"前端 Dashboard/Explorer 扩展"、"日报生成器与门禁改进");review prompt 应在工作区变更量超过阈值时自动提升 commit 停滞优先级。
|
||||
- **优先级**:P0
|
||||
- **建议验证方法**:提交后检查 `git log --oneline` 出现新提交,`git diff --stat HEAD` 变更量大幅收缩。
|
||||
|
||||
#### 问题 41 状态更新:从"连续超时"降级为"live_run SUMMARY 缺失"(影响次数 4)
|
||||
|
||||
- **15:10 状态**:连续超时未在本轮复现(importer smoke 全 PASS,gate PASS),但 live_run SUMMARY(window_size / success_rate / live_run_result)仍未输出,脚本在 smokerun 后 60s 内退出。
|
||||
- **问题影响**:Phase 6 稳定性窗口 PASS/FAIL 状态无法确认;无法判断 05-25 的三次超时是外部文档站卡死还是脚本性能退化。
|
||||
- **优化建议**:
|
||||
1. 调查 verify_phase6.sh live_run 未输出完整 SUMMARY 的根因(60s 内退出但未打印 window / success_rate / live_run_result)
|
||||
2. 为 verify_phase6.sh 增加单次检查的独立超时控制,避免单次检查卡死导致整脚本超时
|
||||
3. 在 verify_phase6.sh 输出中增加"当前检查进度"标记
|
||||
- **优先级**:P0 → P1(本轮 importer smoke 全 PASS 说明不是持续卡死,但 live_run SUMMARY 缺失仍是 P1)
|
||||
- **建议验证方法**:修正后执行 verify_phase6.sh,确认能在 <120s 内输出完整 SUMMARY(含 window_size / success_rate / live_run_result)。
|
||||
|
||||
#### 问题 42 状态更新:已修复(从 backlog current 表移除)
|
||||
|
||||
- **15:10 状态**:verify_phase6.sh 连续超时未在本轮复现,importer smoke 全 PASS。05-25 的三次连续超时更接近外部文档站临时卡死而非脚本性能退化。
|
||||
- **结论**:问题 42 从 current 表移除,归档至 review 日志。
|
||||
|
||||
#### 问题 43(新发现):verify_phase4 ECharts 集成断言失败(历史遗留 P2)
|
||||
|
||||
- **15:10 状态**:`[FAIL] Dashboard 已集成 ECharts` 是 verify_phase4 的唯一 FAIL 项。Dashboard.tsx 中已引入 `import * as echarts from 'echarts'` 和 `echarts.init()` 逻辑,但 verify 脚本断言逻辑与实际代码行为不匹配。
|
||||
- **问题影响**:导致 PRE_PHASE6 整体 FAIL;但不影响主采集链路(Phase 1/2/3 全 PASS,importer smoke 全 PASS);历史遗留问题(首现于 05-25 15:10 systematic review)。
|
||||
- **优化建议**:
|
||||
1. 更新 verify_phase4 中 ECharts 集成断言逻辑,使其与当前 Dashboard.tsx 的 echarts 使用方式一致
|
||||
2. 或者确认当前代码是否真正满足"已集成 ECharts"语义,若不满足则完成集成
|
||||
3. 考虑将 ECharts 相关断言降级为 WARNING 而非 FAIL,以区分"历史遗留 P2"与"真实 blocker"
|
||||
- **优先级**:P2
|
||||
- **建议验证方法**:`bash scripts/verify_phase4.sh` → SUMMARY pass=10 fail=0 warn=0,PRE_PHASE6_RESULT: PASS。
|
||||
|
||||
#### 问题 44(新发现):新增 scripts 无门禁覆盖
|
||||
|
||||
- **15:10 状态**:scripts/secret_gate_lib.sh(1846 字节)、scripts/secret_gate_test.sh(1823 字节)、scripts/testdata/empty.dockerignore 为新增 untracked 项,无对应 verify 门禁验证其正确性。
|
||||
- **问题影响**:新增安全类脚本无法确认是否正确落地;一旦工作区切换或代码丢失,这些脚本的存在和正确性无法追溯。
|
||||
- **优化建议**:
|
||||
1. 为 secret_gate_lib.sh / secret_gate_test.sh 建立对应的 smoke gate 或单元测试门禁
|
||||
2. 考虑在 verify_phase5 或 verify_phase6 中增加对新 scripts 目录的覆盖检查
|
||||
- **优先级**:P2
|
||||
- **建议验证方法**:执行 `bash scripts/secret_gate_test.sh` 验证其正确性,并确认门禁已纳入综合验收。
|
||||
|
||||
#### 问题 13 状态更新:untracked 核心代码重新活跃(影响次数 14)
|
||||
|
||||
- **15:10 状态**:scripts/secret_gate_lib.sh / secret_gate_test.sh 为新增 untracked 安全类脚本;BACKLOG 本身也在未提交列表中;.agent/、.serena/ 等目录长期未治理。
|
||||
- **问题影响**:同问题 10;untracked 列表持续增长增加了 versioned truth 漂移风险。
|
||||
- **优化建议**:同问题 10;尽快提交工作区变更,清理非必要 untracked 项。
|
||||
- **优先级**:P0
|
||||
- **建议验证方法**:提交后 `git status --short` 中 untracked 列表显著收缩。
|
||||
|
||||
#### 问题 38 状态更新:PRE_PHASE6_RESULT 标签冲突(影响次数 4)
|
||||
|
||||
- **15:10 状态**:verify_phase4 ECharts 断言失败导致 PRE_PHASE6 FAIL;但 verify_phase4 内部 SUMMARY 显示 pass=9 fail=1 warn=0,说明是单一断言失败而非系统性卡死。
|
||||
- **问题影响**:PRE_PHASE6 FAIL 的根因已明确为 verify_phase4 ECharts 断言问题(历史 P2),不影响主链路;但标签冲突使 reviewer 需要额外下钻才能判断真实阶段。
|
||||
- **优化建议**:将 verify_phase4 中的 ECharts 相关断言降级为 WARNING,或更新断言逻辑使其与当前 Dashboard.tsx echarts 使用方式一致。
|
||||
- **优先级**:P1
|
||||
- **建议验证方法**:verify_phase4 中 ECharts 断言修复后,PRE_PHASE6_RESULT 应回到 PASS。
|
||||
|
||||
### 2026-05-25 15:10(afternoon-review cron,第 41 次 review)
|
||||
|
||||
> **前置说明**:距上一次 review(05-25 08:59)约 **6 小时 11 分钟**。本轮无新 delta:working tree 仍 19 文件未提交(与 08:59 systematic review 完全一致),无新 commit。verify_phase6.sh 第三次连续超时(09:06 morning → 09:06 systematic → 15:10 afternoon),Phase 6 live blocker 状态完全无法确认。Phase 1~5 PASS,go test 全 PASS,日报已生成,但所有 systematic review 修复落地项(.dockerignore、runtimeVisibility、BasicAuth、Explorer.tsx 部分修复)均未 commit。
|
||||
|
||||
#### 本次新增发现
|
||||
|
||||
- **verify_phase6.sh 第三次连续超时**:本轮执行 `timeout 180 bash scripts/verify_phase6.sh`,>200s 无输出,连续第三次(09:06 morning / 09:06 systematic / 15:10 afternoon)。Phase 6 live blocker 状态(Zhipu 403 是否仍活跃、是否已消失或切换到新外部源)完全无法确认。
|
||||
- **Phase 1~5 门禁全 PASS**:`verify_pre_phase6.sh` 输出 `PRE_PHASE6_RESULT: PASS`,SUMMARY pass=15 fail=0 warn=0,与历史一致。
|
||||
- **Working tree 状态与 08:59 systematic review 完全一致**:19 文件 +1372/-281 行仍未提交,包含 .dockerignore、runtimeVisibility.ts、BasicAuth 实现、Explorer.tsx 部分修复等 systematic review 所有 P0/P1 修复落地项。
|
||||
- **systematic review P0-3 修复已落地但未 commit**:`.dockerignore` 已创建(285 字节,12:03 创建,artifact-present),`frontend/src/lib/runtimeVisibility.ts` + `runtimeVisibility.test.ts` 已创建。
|
||||
- **Explorer.tsx fallback 修复尚未完整验证**:runtimeVisibility.ts 已就绪但 Explorer.tsx 中只引入了部分 notice 构建逻辑,未完全实现"禁止静默 fallback"的 P0-2 修复目标。
|
||||
- **整体项目状态无新 delta**:距上次 review 6+ 小时,无新 commit,无新 runtime 证据,主链路健康(API 200,日报已生成)。
|
||||
|
||||
#### 问题 42(新发现):verify_phase6.sh 第三次连续超时,Phase 6 live blocker 状态完全不明
|
||||
|
||||
- **15:10 状态**:连续三次 verify_phase6.sh 超时(09:06 morning / 09:06 systematic / 15:10 afternoon),均无法在 180s 内完成并输出 Phase 6 SUMMARY。这不是偶发性问题,而是持续性卡死——可能存在外部文档站持续卡死或脚本本身性能退化。
|
||||
- **问题影响**:
|
||||
- Phase 6 综合门禁 PASS/FAIL 完全不明,连续三次 review 均无法给出准确的阶段判断
|
||||
- 无法确认 Zhipu 403 blocker 是否仍活跃、是否已消失还是切换到新的外部源
|
||||
- 外部文档站可能存在新的持续卡死,需要立即调查超时根因
|
||||
- **优化建议**:
|
||||
1. 调查 verify_phase6.sh 超时根因:单次外部文档站卡死 vs 整体脚本性能退化
|
||||
2. 为 verify_phase6.sh 增加单次检查的独立超时控制,避免单次检查卡死导致整脚本超时
|
||||
3. 在 verify_phase6.sh 输出中增加"当前检查进度"标记,方便定位卡死环节
|
||||
4. 在 verify_phase6.sh 中为连续超时的外部 URL 建立快速失败策略
|
||||
- **优先级**:P0
|
||||
- **建议验证方法**:修正后执行 verify_phase6.sh,确认能在 <120s 内完成并输出完整 SUMMARY(含 window_size / success_rate / live_run_result)
|
||||
|
||||
#### 问题 40 状态更新:优先级升级,影响次数更新
|
||||
|
||||
- **15:10 状态**:问题 40 自 08:51 首现,已持续 6+ 小时未解决,working tree 仍包含 systematic review 所有 P0/P1 修复落地项。优先级从 P2 升级为 P1(因为现在包含 P0 修复落地项的未 commit 风险);影响次数从 2 更新为 3 次。
|
||||
- **结论**:优先级从 P2 升级为 P1,影响次数从 2 更新为 3 次。
|
||||
|
||||
#### 问题 38 状态更新:PRE_PHASE6_RESULT 标签冲突仍待系统性修复
|
||||
|
||||
- **15:10 状态**:问题 38 影响次数从 2 更新为 3 次。PRE_PHASE6_RESULT 标签逻辑本身仍未系统性修复。
|
||||
- **结论**:影响次数从 2 更新为 3 次。
|
||||
|
||||
#### 问题 39 状态更新:日报时间戳异常仍未修复
|
||||
|
||||
- **15:10 状态**:问题 39 影响次数从 2 更新为 3 次。generated_at 仍显示 2026-05-25T19:03:55+08:00,比实际时间晚约 10 小时,与 08:51 / 08:59 记录一致。
|
||||
- **结论**:影响次数从 2 更新为 3 次。
|
||||
|
||||
### 2026-05-25 09:06(night-review cron,第 40 次 review)
|
||||
|
||||
> **前置说明**:距上一次 review(05-25 08:59)约 **7 分钟**。本轮属于"无新 delta 且 verify_phase6.sh 异常超时":无新 commit,Phase 1~5 门禁仍全 PASS,但 verify_phase6.sh 连续两次执行超时(>180s)导致 Phase 6 live blocker 状态无法确认。BACKLOG 文件 uncommitted 已持续 75 分钟+(08:51 → 08:59 → 09:06)。
|
||||
|
||||
#### 本次新增发现
|
||||
|
||||
- **verify_phase6.sh 连续两次超时**:本轮 review 两次执行 `bash scripts/verify_phase6.sh`,第一次在 90s 内完成了前 30 个 importer smoke 全 PASS 但未输出最终 SUMMARY;第二次直接超时(>180s 无法完成)。Phase 6 live blocker 状态(Zhipu 403 是否仍活跃)无法本轮真实验证。
|
||||
- **Phase 1~5 门禁仍然全 PASS**:`verify_pre_phase6.sh` 输出 `PRE_PHASE6_RESULT: PASS`,与上一轮一致,无变化。
|
||||
- **BACKLOG 文件 uncommitted 已持续 75 分钟+**:问题 40 从 08:51 首现,08:59 仍存在,09:06 仍未解决,已跨三轮 review 无收敛动作。
|
||||
- **日报时间戳异常仍未改善**:`daily_report_2026-05-25.md` 的 `generated_at: 2026-05-25T19:03:55+08:00` 比实际时间(09:06)晚约 10 小时,与 08:51 / 08:59 记录一致。
|
||||
|
||||
#### 问题 41(新发现):verify_phase6.sh 连续超时导致 Phase 6 live blocker 状态无法确认
|
||||
|
||||
- **09:06 状态**:本轮 review 连续两次执行 `bash scripts/verify_phase6.sh`,均无法在合理时间内完成。第一次在前 90s 内完成了 30 个 importer smoke 全 PASS 但未输出最终 SUMMARY;第二次直接超时(>180s 无法完成)。
|
||||
- **问题影响**:
|
||||
- Phase 6 综合门禁 PASS/FAIL 状态无法确认,reviewer 无法给出准确的阶段判断
|
||||
- 上一轮(08:59)记录的 Zhipu 403 blocker 是否仍活跃、是否已切换,本轮无法验证
|
||||
- 超时可能与 Zhipu 403 或其他外部文档站卡死有关,需要调查根因
|
||||
- **优化建议**:
|
||||
1. 调查 verify_phase6.sh 超时根因:单次外部文档站拉取卡死 vs 整体脚本性能退化
|
||||
2. 为 verify_phase6.sh 增加单次检查的独立超时控制,避免单次检查卡死导致整脚本超时
|
||||
3. 在 verify_phase6.sh 输出中增加"当前检查进度"标记,方便定位卡死环节
|
||||
- **优先级**:P1
|
||||
- **建议验证方法**:修正后执行 verify_phase6.sh,确认能在 <120s 内完成并输出完整 SUMMARY(含 window_size / success_rate / live_run_result)
|
||||
|
||||
#### 问题 37 状态更新:外部文档站故障仍无系统化降级
|
||||
|
||||
- **09:06 状态**:问题 37 仍活跃,影响次数从 3 更新为 4 次。本轮 verify_phase6 超时可能与外部文档站卡死有关(可能是 Zhipu 403 或其他源),blocker 在不同外部源之间游走的模式持续。
|
||||
- **结论**:从"3 次"更新为"4 次"。
|
||||
|
||||
#### 问题 39 状态更新:日报时间戳异常仍未改善
|
||||
|
||||
- **09:06 状态**:generated_at 仍显示 2026-05-25T19:03:55+08:00,比实际时间晚约 10 小时,无修复动作。
|
||||
- **结论**:影响次数从 1 更新为 2 次。
|
||||
|
||||
#### 问题 40 状态更新:BACKLOG uncommitted 已持续 75 分钟+
|
||||
|
||||
- **09:06 状态**:问题 40 已从 08:51 首现(morning review 修改 BACKLOG 后未 commit),08:59 仍存在,09:06 仍未解决,跨三轮 review 无收敛动作。
|
||||
- **结论**:影响次数从 1 更新为 2 次。
|
||||
|
||||
@@ -1,235 +0,0 @@
|
||||
# LLM Intelligence 项目 Review 报告
|
||||
|
||||
- 审查时间:2026-05-20
|
||||
- 审查人:Hermes Agent
|
||||
- 审查对象:D:\project\llm-intelligence
|
||||
- 审查方式:仓库结构盘点 + 关键代码抽样 + 配置/验证链路审查 + counter-evidence/calibration
|
||||
|
||||
## 1. 结论摘要
|
||||
|
||||
总体判断:这是一个“文档/规划活跃,但工程闭环和验证闭环明显不足”的项目。
|
||||
|
||||
成熟度判断:
|
||||
- 当前级别:demo-grade
|
||||
- 不建议给出 production-candidate 或“可稳定上线”的结论
|
||||
|
||||
主导问题:
|
||||
1. 基线不稳定
|
||||
2. 运行/验证环境不自洽
|
||||
3. 文档声称的完成度高于当前可复现度
|
||||
4. 前后端/脚本/部署链路存在多处断裂
|
||||
|
||||
## 2. 审查范围与限制
|
||||
|
||||
已检查:
|
||||
- git 基线状态
|
||||
- 顶层文档与 truth-map 候选
|
||||
- Go 服务端主入口与主要查询逻辑
|
||||
- 前端 Explorer / Dashboard / models 辅助库
|
||||
- docker-compose.yml / Dockerfile / nginx.conf / healthcheck.sh
|
||||
- verify 脚本与 verification_executor.go
|
||||
- 前端测试执行结果
|
||||
|
||||
受限项:
|
||||
- 当前环境中 go 不存在,因此 Go 测试未能实际跑通
|
||||
- 数据库验证未完整复现,因为 verify shell 脚本先被行尾格式问题拦住
|
||||
|
||||
## 3. 基线稳定性
|
||||
|
||||
git status 显示当前工作区存在大面积修改,覆盖:
|
||||
- 顶层文档
|
||||
- Go 服务端
|
||||
- migration
|
||||
- frontend
|
||||
- scripts
|
||||
- tests
|
||||
|
||||
这意味着:
|
||||
- 任何历史“验证通过”“Phase 1 完成”的说法,都不能直接当作当前真相
|
||||
- 当前 review 只能对当前工作区快照负责,不能继承旧报告的高置信结论
|
||||
|
||||
判定:P1 级问题。
|
||||
|
||||
## 4. Truth Map / Source of Truth
|
||||
|
||||
仓库顶层没有 README.md。
|
||||
|
||||
当前 truth candidates 主要包括:
|
||||
- PRD.md
|
||||
- TECHNICAL_DESIGN.md
|
||||
- IMPLEMENTATION_PLAN.md
|
||||
- IMPLEMENTATION_PLAN_v1.1.md
|
||||
- RUNBOOK.md
|
||||
- DEPLOYMENT.md
|
||||
- TASKS.md
|
||||
- GOALS.md
|
||||
- VERIFICATION_REPORT_Sprint1-3.md
|
||||
|
||||
判断:
|
||||
- PRD.md / TECHNICAL_DESIGN.md:更像 target design + 部分当前叙述混合体
|
||||
- RUNBOOK.md / DEPLOYMENT.md:试图充当 current ops truth,但可信度不足
|
||||
- VERIFICATION_REPORT_Sprint1-3.md:更像历史验证叙事,不足以代表当前 truth
|
||||
- 代码与当前可执行环境,优先级高于历史报告
|
||||
|
||||
问题:source-of-truth fragmented。
|
||||
|
||||
## 5. 五层成熟度判断
|
||||
|
||||
### 5.1 文档成熟度
|
||||
|
||||
优点:
|
||||
- 文档密度高,主题覆盖广
|
||||
- 技术设计、产品需求、部署、运维、验收、验证报告较齐全
|
||||
|
||||
问题:
|
||||
- current truth 与 target design 混杂
|
||||
- 顶层缺少统一入口文档
|
||||
- 文档中仍有明显历史/Linux 路径痕迹,如 /home/long/project/llm-intelligence
|
||||
|
||||
结论:
|
||||
- 文档本身:中上
|
||||
- 文档作为当前真相载体:中下
|
||||
|
||||
### 5.2 执行成熟度
|
||||
|
||||
后端锚点:cmd/server/main.go
|
||||
|
||||
优点:
|
||||
- API 入口清晰:/health、/api/v1/models、/api/v1/subscription-plans
|
||||
- 查询结构整体直白
|
||||
|
||||
问题:
|
||||
- 健康检查把“进程活着”和“数据库可用”混在一起
|
||||
- 数据库未配置时整个 API 直接 503
|
||||
- 与前端 fallback 的产品语义不统一
|
||||
- 服务端缺少更完整的超时与边界处理
|
||||
|
||||
前端锚点:
|
||||
- frontend/src/pages/Explorer.tsx
|
||||
- frontend/src/pages/Dashboard.tsx
|
||||
- frontend/src/lib/models.ts
|
||||
|
||||
优点:
|
||||
- Explorer 支持筛选/排序/分页
|
||||
- Dashboard 对模型和套餐做了分开展示
|
||||
- 有静态 fallback 数据方案
|
||||
|
||||
问题:
|
||||
- Explorer 对 fetch 未先检查 response.ok
|
||||
- modality 筛选口径与设计不一致
|
||||
- Dashboard 的“国内厂商”文案与真实统计口径不一致
|
||||
|
||||
结论:执行成熟度中下。
|
||||
|
||||
### 5.3 验证成熟度
|
||||
|
||||
反证非常明显:
|
||||
|
||||
1. Go 测试不可复现
|
||||
- 实测:go test ./...
|
||||
- 结果:go: command not found
|
||||
|
||||
2. 前端测试当前失败
|
||||
- 实测:npm test -- --run
|
||||
- 结果:缺失 @rollup/rollup-linux-x64-gnu
|
||||
|
||||
3. verify shell 脚本当前直接失败
|
||||
- 实测:bash scripts/verify_phase1.sh
|
||||
- 结果:$'\r': command not found、pipefail\r: invalid option name
|
||||
|
||||
结论:
|
||||
- 验证设计意图:中上
|
||||
- 当前可复现性:低
|
||||
- 不能给出“验证闭环成熟”的结论
|
||||
|
||||
### 5.4 运维成熟度
|
||||
|
||||
检查文件:
|
||||
- docker-compose.yml
|
||||
- Dockerfile
|
||||
- nginx.conf
|
||||
- healthcheck.sh
|
||||
- RUNBOOK.md
|
||||
|
||||
问题:
|
||||
- docker-compose.yml 中 DATABASE_URL 看起来像遮罩占位值,不像真实可运行配置
|
||||
- Dockerfile 中前端产物与 compose/nginx 实际消费路径脱节
|
||||
- healthcheck.sh 将“日报存在”混入基础健康判定
|
||||
- RUNBOOK.md 仍带个人化/历史路径
|
||||
|
||||
结论:有雏形,但未形成可信部署闭环。
|
||||
|
||||
### 5.5 生产成熟度
|
||||
|
||||
综合结论:
|
||||
- 文档成熟度:中上
|
||||
- review/治理成熟度:中
|
||||
- 执行成熟度:中下
|
||||
- 验证成熟度:低
|
||||
- 生产成熟度:低
|
||||
|
||||
最终成熟度带:demo-grade
|
||||
|
||||
主导 drift 类型:
|
||||
- validation drift
|
||||
- execution drift
|
||||
- source-of-truth drift
|
||||
|
||||
## 6. 最高风险的假成熟信号
|
||||
|
||||
1. 文档很多、报告很多,但当前环境下基础验证链路并不稳
|
||||
2. 前端 fallback 可能掩盖后端/数据库不可用问题
|
||||
3. RUNBOOK / DEPLOYMENT / compose / healthcheck 存在,但没有形成可一键复现的统一现实
|
||||
4. verification_executor 看起来成熟,但底层 shell 验证资产自身未持续通过
|
||||
|
||||
## 7. 问题清单
|
||||
|
||||
### P1
|
||||
1. 工作区大面积脏修改,导致历史验证/完成度结论失去当前高置信度
|
||||
2. 验证链路不可复现:当前环境无 go,前端测试失败,verify shell 脚本 CRLF 不兼容
|
||||
3. docker-compose.yml 中 app 的 DATABASE_URL 形态可疑,像占位值,不像可运行配置
|
||||
4. Dockerfile 产物路径与 compose/nginx 消费路径脱节,前端部署闭环不完整
|
||||
5. 顶层缺 README,source-of-truth 分散,文档与代码现实存在漂移
|
||||
6. 健康检查、前端 fallback、后端 503 策略未形成一致服务语义
|
||||
|
||||
### P2
|
||||
1. Explorer 未显式检查 response.ok
|
||||
2. modality 筛选与设计模型不一致
|
||||
3. Dashboard 文案“国内厂商”与真实统计口径不符
|
||||
4. writeJSON 错误处理不干净
|
||||
5. 服务端缺少更完整的超时配置
|
||||
6. RUNBOOK.md 中路径/环境信息陈旧
|
||||
|
||||
### P3
|
||||
1. 上下文窗口展示粗糙
|
||||
2. 部分前端/文案细节仍有占位感
|
||||
|
||||
## 8. 建议整改顺序
|
||||
|
||||
第一阶段:先修真相和验证,不要先补新功能
|
||||
1. 补顶层 README.md
|
||||
2. 统一 shell 脚本为 LF,并增加环境 preflight
|
||||
3. 前端依赖重装并跑通 npm test / npm build
|
||||
4. 修复 compose 的数据库配置
|
||||
5. 打通前端构建/运行链路
|
||||
|
||||
第二阶段:修服务语义
|
||||
6. 拆分 liveness / readiness
|
||||
7. 统一“API 不可用时前端是否允许 fallback”的产品语义
|
||||
8. 明确“无 DB 时系统是否仍算部分可用”
|
||||
|
||||
第三阶段:再继续扩展功能
|
||||
9. 修正 modality / 搜索 /指标口径等一致性问题
|
||||
10. 再扩展多源采集与更复杂报告能力
|
||||
|
||||
## 9. 最终 plain-language verdict
|
||||
|
||||
一句话评价:
|
||||
|
||||
这是一个“文档和治理意图明显超前于工程闭环”的项目。
|
||||
|
||||
更直白地说:
|
||||
- 它不像一堆随手拼的代码,说明作者有产品化和治理意识;
|
||||
- 但它还没有进入“可以被高置信度地认定为稳定可运行、稳定可验证、稳定可部署”的阶段。
|
||||
|
||||
最终评级:demo-grade
|
||||
179
scripts/aliyun_subscription_lib.go
Normal file
179
scripts/aliyun_subscription_lib.go
Normal file
@@ -0,0 +1,179 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultAliyunTokenPlanURL = "https://help.aliyun.com/zh/model-studio/token-plan-overview"
|
||||
defaultAliyunCodingPlanURL = "https://help.aliyun.com/zh/model-studio/coding-plan-quickstart"
|
||||
)
|
||||
|
||||
func parseAliyunSubscriptionCatalog(tokenRaw string, codingRaw string) ([]subscriptionImportRecord, error) {
|
||||
publishedAt, known := publishedAtFromText(firstNonEmptyText(tokenRaw, codingRaw))
|
||||
|
||||
tokenRecords, err := parseAliyunTokenPlan(tokenRaw, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
codingRecords, err := parseAliyunCodingPlan(codingRaw, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records := append(tokenRecords, codingRecords...)
|
||||
for i := range records {
|
||||
records[i].PublishedAtKnown = known
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseAliyunTokenPlan(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
seatPattern := regexp.MustCompile(`(?s)(标准坐席|高级坐席|尊享坐席)\s+¥([\d,]+)(?:/坐席/月)?\s+([\d,]+)\s*Credits/坐席/月\s+([^\n]+)`)
|
||||
matches := seatPattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) != 3 {
|
||||
return nil, fmt.Errorf("unexpected aliyun token seat count: %d", len(matches))
|
||||
}
|
||||
|
||||
tierCodes := map[string]string{
|
||||
"标准坐席": "standard-seat",
|
||||
"高级坐席": "advanced-seat",
|
||||
"尊享坐席": "premium-seat",
|
||||
}
|
||||
tierNames := map[string]string{
|
||||
"标准坐席": "Standard",
|
||||
"高级坐席": "Advanced",
|
||||
"尊享坐席": "Premium",
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, 4)
|
||||
for _, match := range matches {
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Alibaba",
|
||||
ProviderNameCn: "阿里巴巴",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.aliyun.com",
|
||||
OperatorName: "Alibaba Bailian",
|
||||
OperatorNameCn: "阿里云百炼",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://help.aliyun.com/zh/model-studio/",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: "aliyun-token-plan-" + tierCodes[match[1]],
|
||||
PlanName: "Token Plan 团队版 " + match[1],
|
||||
Tier: tierNames[match[1]],
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(match[2]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: mustParseSubscriptionInt64(match[3]),
|
||||
QuotaUnit: "credits/month",
|
||||
PlanScope: "Token Plan 团队版",
|
||||
SourceURL: defaultAliyunTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: strings.TrimSpace(match[4]),
|
||||
})
|
||||
}
|
||||
|
||||
sharedPattern := regexp.MustCompile(`共享用量包\s+¥([\d,]+)(?:/个)?\s+([\d,]+)\s*Credits/个`)
|
||||
shared := sharedPattern.FindStringSubmatch(raw)
|
||||
if len(shared) != 3 {
|
||||
return nil, fmt.Errorf("aliyun shared pack not found")
|
||||
}
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Alibaba",
|
||||
ProviderNameCn: "阿里巴巴",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.aliyun.com",
|
||||
OperatorName: "Alibaba Bailian",
|
||||
OperatorNameCn: "阿里云百炼",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://help.aliyun.com/zh/model-studio/",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: "aliyun-token-plan-shared-pack",
|
||||
PlanName: "Token Plan 团队版 共享用量包",
|
||||
Tier: "SharedPack",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(shared[1]),
|
||||
PriceUnit: "CNY/pack",
|
||||
QuotaValue: mustParseSubscriptionInt64(shared[2]),
|
||||
QuotaUnit: "credits/pack",
|
||||
PlanScope: "Token Plan 团队版",
|
||||
SourceURL: defaultAliyunTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: "跨坐席共享的弹性用量包,有效期 1 个月。",
|
||||
})
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseAliyunCodingPlan(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
pricePattern := regexp.MustCompile(`价格\s+¥\s*([\d,]+)\s*/月`)
|
||||
priceMatch := pricePattern.FindStringSubmatch(raw)
|
||||
if len(priceMatch) != 2 {
|
||||
return nil, fmt.Errorf("aliyun coding plan price not found")
|
||||
}
|
||||
|
||||
modelsPattern := regexp.MustCompile(`支持的模型\s+\|\s+推荐模型:([^\n]+)`)
|
||||
modelsMatch := modelsPattern.FindStringSubmatch(raw)
|
||||
modelScope := []string{}
|
||||
if len(modelsMatch) == 2 {
|
||||
for _, item := range strings.Split(modelsMatch[1], "、") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item != "" {
|
||||
modelScope = append(modelScope, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
limitPattern := regexp.MustCompile(`每月\s*([\d,]+)\s*次请求`)
|
||||
limitMatch := limitPattern.FindStringSubmatch(raw)
|
||||
quotaValue := int64(0)
|
||||
if len(limitMatch) == 2 {
|
||||
quotaValue = mustParseSubscriptionInt64(limitMatch[1])
|
||||
}
|
||||
|
||||
notes := []string{}
|
||||
for _, fragment := range []string{
|
||||
"Lite 套餐自 2026 年 3 月 20 日 00:00:00(UTC+08:00)起停止新购",
|
||||
"活动已结束",
|
||||
} {
|
||||
if strings.Contains(raw, fragment) {
|
||||
notes = append(notes, fragment)
|
||||
}
|
||||
}
|
||||
|
||||
return []subscriptionImportRecord{{
|
||||
ProviderName: "Alibaba",
|
||||
ProviderNameCn: "阿里巴巴",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.aliyun.com",
|
||||
OperatorName: "Alibaba Bailian",
|
||||
OperatorNameCn: "阿里云百炼",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://help.aliyun.com/zh/model-studio/",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "aliyun-coding-plan-pro",
|
||||
PlanName: "百炼 Coding Plan Pro",
|
||||
Tier: "Pro",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(priceMatch[1]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: quotaValue,
|
||||
QuotaUnit: "requests/month",
|
||||
PlanScope: "Coding Plan",
|
||||
ModelScope: modelScope,
|
||||
SourceURL: defaultAliyunCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: strings.Join(notes, ";"),
|
||||
}}, nil
|
||||
}
|
||||
@@ -4,14 +4,8 @@ set -euo pipefail
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
if [[ -f ".env.local" ]]; then
|
||||
# shellcheck disable=SC1091
|
||||
source ".env.local"
|
||||
fi
|
||||
if [[ -f ".env" ]]; then
|
||||
# shellcheck disable=SC1091
|
||||
source ".env"
|
||||
fi
|
||||
while IFS= read -r kv; do export "$kv"; done < <("$ROOT_DIR/scripts/load_project_env.sh" ".env.local")
|
||||
while IFS= read -r kv; do key="${kv%%=*}"; [[ -n "$key" && -n "${!key:-}" ]] && continue; export "$kv"; done < <("$ROOT_DIR/scripts/load_project_env.sh" ".env")
|
||||
|
||||
if [[ -z "${DATABASE_URL:-}" ]]; then
|
||||
echo "DATABASE_URL 未设置" >&2
|
||||
|
||||
69
scripts/apply_script_test_tags.py
Normal file
69
scripts/apply_script_test_tags.py
Normal file
@@ -0,0 +1,69 @@
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
KEEP_UNTAGGED = {
|
||||
'official_pricing_import_common.go',
|
||||
'subscription_import_common.go',
|
||||
'catalog_verification_common.go',
|
||||
'cloudflare_pricing_signature_guard_lib.go',
|
||||
'cloudflare_pricing_snapshot_lib.go',
|
||||
'cloudflare_pricing_import_runner.go',
|
||||
'cloudflare_pricing_lib.go',
|
||||
'coreshub_pricing_lib.go',
|
||||
'ctyun_subscription_lib.go',
|
||||
'deepseek_news_signature_guard_lib.go',
|
||||
'deepseek_news_snapshot_lib.go',
|
||||
'deepseek_pricing_signature_guard_lib.go',
|
||||
'deepseek_pricing_snapshot_lib.go',
|
||||
'intraday_discovery_common.go',
|
||||
'intraday_discovery_provider.go',
|
||||
'official_import_signature_audit_lib.go',
|
||||
'official_import_signature_audit_query_lib.go',
|
||||
'perplexity_pricing_signature_guard_lib.go',
|
||||
'perplexity_pricing_snapshot_lib.go',
|
||||
'perplexity_pricing_import_runner.go',
|
||||
'perplexity_pricing_lib.go',
|
||||
'pricing_markdown_snapshot_lib.go',
|
||||
'ppio_pricing_lib.go',
|
||||
'report_event_coverage.go',
|
||||
'signature_guard_common.go',
|
||||
'siliconflow_pricing_lib.go',
|
||||
'tencent_catalog_lib.go',
|
||||
'ucloud_pricing_lib.go',
|
||||
'vertex_pricing_signature_guard_lib.go',
|
||||
'vertex_pricing_snapshot_lib.go',
|
||||
'vertex_pricing_import_runner.go',
|
||||
'vertex_pricing_lib.go',
|
||||
'youdao_pricing_lib.go',
|
||||
'huawei_package_lib.go',
|
||||
'bytedance_subscription_lib.go',
|
||||
'baidu_subscription_lib.go',
|
||||
'aliyun_subscription_lib.go',
|
||||
'azure_openai_pricing_lib.go',
|
||||
'baichuan_pricing_lib.go',
|
||||
'bedrock_pricing_lib.go',
|
||||
'platform360_pricing_lib.go',
|
||||
'minimax_subscription_lib.go',
|
||||
'mobile_cloud_pricing_lib.go',
|
||||
'qwen_pricing_lib.go',
|
||||
'hunyuan_pricing_lib.go',
|
||||
'lingyiwanwu_pricing_lib.go',
|
||||
'huawei_maas_pricing_lib.go',
|
||||
'xfyun_pricing_lib.go',
|
||||
'sensenova_pricing_lib.go',
|
||||
}
|
||||
# only keep files that actually exist / are referenced; missing names are harmless in the set
|
||||
|
||||
for path in sorted(ROOT.glob('*.go')):
|
||||
if path.name.endswith('_test.go') or path.name in KEEP_UNTAGGED:
|
||||
continue
|
||||
text = path.read_text()
|
||||
if 'func main()' not in text:
|
||||
continue
|
||||
lines = text.splitlines()
|
||||
if lines and lines[0].strip() == '//go:build llm_script':
|
||||
lines[0] = '//go:build llm_script && !scripts_pkg'
|
||||
else:
|
||||
# only adjust files that already participate in llm_script flows
|
||||
continue
|
||||
path.write_text('\n'.join(lines) + ('\n' if text.endswith('\n') else ''))
|
||||
225
scripts/azure_openai_pricing_lib.go
Normal file
225
scripts/azure_openai_pricing_lib.go
Normal file
@@ -0,0 +1,225 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultAzureOpenAIPricingURL = "https://prices.azure.com/api/retail/prices?api-version=2023-01-01-preview¤cyCode='USD'&$filter=contains(productName,'OpenAI')"
|
||||
|
||||
type azureRetailPriceResponse struct {
|
||||
Items []azureRetailPriceItem `json:"Items"`
|
||||
NextPageLink string `json:"NextPageLink"`
|
||||
}
|
||||
|
||||
type azureRetailPriceItem struct {
|
||||
CurrencyCode string `json:"currencyCode"`
|
||||
RetailPrice float64 `json:"retailPrice"`
|
||||
UnitPrice float64 `json:"unitPrice"`
|
||||
Location string `json:"location"`
|
||||
MeterName string `json:"meterName"`
|
||||
ProductName string `json:"productName"`
|
||||
SkuName string `json:"skuName"`
|
||||
ServiceName string `json:"serviceName"`
|
||||
UnitOfMeasure string `json:"unitOfMeasure"`
|
||||
Type string `json:"type"`
|
||||
ArmSkuName string `json:"armSkuName"`
|
||||
ArmRegionName string `json:"armRegionName"`
|
||||
IsPrimaryMeter bool `json:"isPrimaryMeterRegion"`
|
||||
}
|
||||
|
||||
type azurePricingPair struct {
|
||||
ModelName string
|
||||
Region string
|
||||
Currency string
|
||||
InputPrice float64
|
||||
OutputPrice float64
|
||||
}
|
||||
|
||||
var azureKindPattern = regexp.MustCompile(`(?i)\b(inp|inpt|input|out|outp|outpt|output|opt)\b`)
|
||||
|
||||
func fetchAzureOpenAIPricingCatalog(url string, fixture string, client *http.Client) (string, error) {
|
||||
if strings.TrimSpace(fixture) != "" {
|
||||
return fetchRawPricingPage(url, fixture, client)
|
||||
}
|
||||
|
||||
aggregated := azureRetailPriceResponse{}
|
||||
seenPages := map[string]struct{}{}
|
||||
nextURL := url
|
||||
for strings.TrimSpace(nextURL) != "" {
|
||||
if _, exists := seenPages[nextURL]; exists {
|
||||
return "", fmt.Errorf("azure retail pricing pagination loop detected: %s", nextURL)
|
||||
}
|
||||
seenPages[nextURL] = struct{}{}
|
||||
|
||||
raw, err := fetchRawPricingPage(nextURL, "", client)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var page azureRetailPriceResponse
|
||||
if err := json.Unmarshal([]byte(raw), &page); err != nil {
|
||||
return "", fmt.Errorf("unmarshal azure retail pricing page: %w", err)
|
||||
}
|
||||
aggregated.Items = append(aggregated.Items, page.Items...)
|
||||
nextURL = page.NextPageLink
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(aggregated)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal azure retail pricing aggregate: %w", err)
|
||||
}
|
||||
return string(payload), nil
|
||||
}
|
||||
|
||||
func parseAzureOpenAIPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
var response azureRetailPriceResponse
|
||||
if err := json.Unmarshal([]byte(raw), &response); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal azure retail pricing: %w", err)
|
||||
}
|
||||
pairs := make(map[string]*azurePricingPair)
|
||||
for _, item := range response.Items {
|
||||
kind, modelName, ok := classifyAzureRetailPrice(item)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
region := strings.TrimSpace(item.Location)
|
||||
if region == "" {
|
||||
region = "global"
|
||||
}
|
||||
currency := strings.TrimSpace(item.CurrencyCode)
|
||||
if currency == "" {
|
||||
currency = "USD"
|
||||
}
|
||||
key := strings.Join([]string{modelName, region, currency}, "|")
|
||||
pair := pairs[key]
|
||||
if pair == nil {
|
||||
pair = &azurePricingPair{
|
||||
ModelName: modelName,
|
||||
Region: region,
|
||||
Currency: currency,
|
||||
}
|
||||
pairs[key] = pair
|
||||
}
|
||||
price := item.UnitPrice
|
||||
if strings.EqualFold(strings.TrimSpace(item.UnitOfMeasure), "1K") {
|
||||
price *= 1000
|
||||
}
|
||||
if kind == "input" {
|
||||
pair.InputPrice = price
|
||||
} else {
|
||||
pair.OutputPrice = price
|
||||
}
|
||||
}
|
||||
|
||||
records := make([]officialPricingRecord, 0, len(pairs))
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata("OpenAI")
|
||||
for _, pair := range pairs {
|
||||
if pair.InputPrice == 0 || pair.OutputPrice == 0 {
|
||||
continue
|
||||
}
|
||||
record := officialPricingRecord{
|
||||
ModelID: normalizeExternalID("azure-openai", pair.ModelName),
|
||||
ModelName: pair.ModelName,
|
||||
ProviderName: "OpenAI",
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Microsoft Azure",
|
||||
OperatorNameCn: "微软 Azure",
|
||||
OperatorCountry: "US",
|
||||
OperatorWebsite: "https://azure.microsoft.com",
|
||||
OperatorType: "cloud",
|
||||
Region: pair.Region,
|
||||
Currency: pair.Currency,
|
||||
InputPrice: pair.InputPrice,
|
||||
OutputPrice: pair.OutputPrice,
|
||||
SourceURL: defaultAzureOpenAIPricingURL,
|
||||
ModelSourceURL: defaultAzureOpenAIPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(pair.ModelName),
|
||||
}
|
||||
record.IsFree = false
|
||||
records = append(records, record)
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("no azure openai token prices found")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func classifyAzureRetailPrice(item azureRetailPriceItem) (string, string, bool) {
|
||||
if item.ServiceName != "Foundry Models" || item.Type != "Consumption" {
|
||||
return "", "", false
|
||||
}
|
||||
productLower := strings.ToLower(item.ProductName)
|
||||
if !strings.Contains(productLower, "openai") || strings.Contains(productLower, "media") {
|
||||
return "", "", false
|
||||
}
|
||||
name := strings.ToLower(strings.TrimSpace(strings.Join([]string{item.SkuName, item.MeterName, item.ArmSkuName}, " ")))
|
||||
if !azureKindPattern.MatchString(name) {
|
||||
return "", "", false
|
||||
}
|
||||
for _, blocked := range []string{
|
||||
"batch",
|
||||
"cache",
|
||||
"cchd",
|
||||
"prty",
|
||||
" pp ",
|
||||
"hosting",
|
||||
"training",
|
||||
" ft ",
|
||||
"ft ",
|
||||
" mdl ",
|
||||
"grdr",
|
||||
"file-search",
|
||||
"code-interpreter",
|
||||
"session",
|
||||
"transcribe",
|
||||
" aud ",
|
||||
"audio",
|
||||
" img ",
|
||||
"image",
|
||||
"voice",
|
||||
"rt ",
|
||||
"realtime",
|
||||
"tool",
|
||||
} {
|
||||
if strings.Contains(name, blocked) {
|
||||
return "", "", false
|
||||
}
|
||||
}
|
||||
kind := "output"
|
||||
if strings.Contains(name, "inp") || strings.Contains(name, "input") || strings.Contains(name, "inpt") {
|
||||
kind = "input"
|
||||
}
|
||||
modelName := normalizeAzureModelName(item)
|
||||
if modelName == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return kind, modelName, true
|
||||
}
|
||||
|
||||
func normalizeAzureModelName(item azureRetailPriceItem) string {
|
||||
base := strings.ToLower(strings.TrimSpace(item.MeterName))
|
||||
replacer := strings.NewReplacer("-", " ", ".", ".", "_", " ")
|
||||
base = replacer.Replace(base)
|
||||
base = regexp.MustCompile(`(?i)\s+(inp|inpt|input|out|outp|outpt|output|opt)\b.*$`).ReplaceAllString(base, "")
|
||||
base = strings.TrimSpace(base)
|
||||
if base == "" {
|
||||
return ""
|
||||
}
|
||||
if regexp.MustCompile(`^\d`).MatchString(base) {
|
||||
base = "gpt " + base
|
||||
}
|
||||
base = regexp.MustCompile(`\s+`).ReplaceAllString(base, " ")
|
||||
if strings.HasPrefix(base, "gpt ") {
|
||||
return "GPT-" + strings.TrimSpace(strings.TrimPrefix(base, "gpt "))
|
||||
}
|
||||
return strings.ToUpper(base[:1]) + base[1:]
|
||||
}
|
||||
113
scripts/baidu_subscription_lib.go
Normal file
113
scripts/baidu_subscription_lib.go
Normal file
@@ -0,0 +1,113 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultBaiduCodingPlanURL = "https://cloud.baidu.com/doc/qianfan/s/imlg0beiu"
|
||||
defaultBaiduTokenPlanURL = "https://cloud.baidu.com/doc/qianfan/s/Smoghsq3g"
|
||||
)
|
||||
|
||||
func parseBaiduSubscriptionCatalog(codingRaw string, tokenRaw string) ([]subscriptionImportRecord, error) {
|
||||
publishedAt, known := publishedAtFromText(firstNonEmptyText(codingRaw, tokenRaw))
|
||||
codingRecords, err := parseBaiduCodingPlan(codingRaw, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tokenRecords, err := parseBaiduTokenBenefitPack(tokenRaw, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records := append(codingRecords, tokenRecords...)
|
||||
for i := range records {
|
||||
records[i].PublishedAtKnown = known
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseBaiduCodingPlan(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
pattern := regexp.MustCompile(`Coding Plan (Lite|Pro)\s+¥\s*([\d,]+)\s*/\s*月\s+每 5 小时:最多约 [\d,]+ 次请求\s+每周:最多约 [\d,]+ 次请求\s+每订阅月:最多约 ([\d,]+) 次请求`)
|
||||
matches := pattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) != 2 {
|
||||
return nil, fmt.Errorf("unexpected baidu coding plan count: %d", len(matches))
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
tier := match[1]
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Baidu",
|
||||
ProviderNameCn: "百度",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://cloud.baidu.com",
|
||||
OperatorName: "Baidu Qianfan",
|
||||
OperatorNameCn: "百度千帆",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://cloud.baidu.com/doc/qianfan/index.html",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "baidu-coding-plan-" + strings.ToLower(tier),
|
||||
PlanName: "千帆 Coding Plan " + tier,
|
||||
Tier: tier,
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(match[2]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: mustParseSubscriptionInt64(match[3]),
|
||||
QuotaUnit: "requests/month",
|
||||
PlanScope: "Coding Plan",
|
||||
SourceURL: defaultBaiduCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: "额度按 5 小时、每周、每订阅月三重窗口刷新。",
|
||||
})
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseBaiduTokenBenefitPack(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
pattern := regexp.MustCompile(`(\d{2,3},\d{3})\s+1个月\s+¥(\d+)\s+¥(\d+)`)
|
||||
matches := pattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) != 5 {
|
||||
return nil, fmt.Errorf("unexpected baidu token benefit pack count: %d", len(matches))
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
quota := mustParseSubscriptionInt64(match[1])
|
||||
originalPrice := strings.TrimSpace(match[2])
|
||||
promoPrice := strings.TrimSpace(match[3])
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Baidu",
|
||||
ProviderNameCn: "百度",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://cloud.baidu.com",
|
||||
OperatorName: "Baidu Qianfan",
|
||||
OperatorNameCn: "百度千帆",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://cloud.baidu.com/doc/qianfan/index.html",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: fmt.Sprintf("baidu-token-benefit-pack-%d", quota),
|
||||
PlanName: fmt.Sprintf("千帆 Token 福利包 %d", quota),
|
||||
Tier: fmt.Sprintf("%d", quota),
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(promoPrice),
|
||||
PriceUnit: "CNY/pack",
|
||||
QuotaValue: quota,
|
||||
QuotaUnit: "credits/pack",
|
||||
PlanScope: "Token 福利包",
|
||||
SourceURL: defaultBaiduTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: fmt.Sprintf("首购优惠价 ¥%s,原价 ¥%s,有效期 1 个月。", promoPrice, originalPrice),
|
||||
})
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
323
scripts/bedrock_pricing_lib.go
Normal file
323
scripts/bedrock_pricing_lib.go
Normal file
@@ -0,0 +1,323 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultBedrockPricingURL = "https://aws.amazon.com/bedrock/pricing/"
|
||||
|
||||
var (
|
||||
bedrockRegionPattern = regexp.MustCompile(`(?s)<p><b>Regions?: ([^<]+)</b></p>`)
|
||||
bedrockTablePattern = regexp.MustCompile(`(?s)<table[^>]*>(.*?)</table>`)
|
||||
bedrockRowPattern = regexp.MustCompile(`(?s)<tr>(.*?)</tr>`)
|
||||
bedrockCellPattern = regexp.MustCompile(`(?s)<t[dh][^>]*>(.*?)</t[dh]>`)
|
||||
)
|
||||
|
||||
func parseBedrockPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
section := extractBetween(raw, `<h3 id="Model_Pricing"`, `<h2 id="Pricing_examples"`)
|
||||
if strings.TrimSpace(section) == "" {
|
||||
section = raw
|
||||
}
|
||||
|
||||
blocks := splitBedrockProviderBlocks(section)
|
||||
records := make([]officialPricingRecord, 0)
|
||||
for _, block := range blocks {
|
||||
records = append(records, parseBedrockProviderBlock(block.providerLabel, block.content)...)
|
||||
}
|
||||
if len(records) == 0 {
|
||||
records = append(records, parseBedrockPricingTextFallback(cleanHTMLText(section))...)
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("no bedrock pricing rows found")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseBedrockProviderBlock(providerLabel string, raw string) []officialPricingRecord {
|
||||
providerName := normalizeBedrockProvider(providerLabel)
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
|
||||
regionMatches := bedrockRegionPattern.FindAllStringSubmatchIndex(raw, -1)
|
||||
tables := bedrockTablePattern.FindAllStringSubmatchIndex(raw, -1)
|
||||
records := make([]officialPricingRecord, 0)
|
||||
seenModelRegion := make(map[string]struct{})
|
||||
for _, tableIndex := range tables {
|
||||
tableHTML := raw[tableIndex[2]:tableIndex[3]]
|
||||
if !strings.Contains(tableHTML, "Price per 1M input tokens") || !strings.Contains(tableHTML, "$") {
|
||||
continue
|
||||
}
|
||||
region := "global"
|
||||
for _, regionIndex := range regionMatches {
|
||||
if regionIndex[0] < tableIndex[0] {
|
||||
region = cleanHTMLText(raw[regionIndex[2]:regionIndex[3]])
|
||||
}
|
||||
}
|
||||
rows := parseBedrockTableRows(tableHTML)
|
||||
for _, row := range rows {
|
||||
dedupeKey := strings.Join([]string{region, row.ModelName}, "|")
|
||||
if _, exists := seenModelRegion[dedupeKey]; exists {
|
||||
continue
|
||||
}
|
||||
record := officialPricingRecord{
|
||||
ModelID: normalizeExternalID("bedrock", providerName, row.ModelName),
|
||||
ModelName: row.ModelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Amazon Bedrock",
|
||||
OperatorNameCn: "Amazon Bedrock",
|
||||
OperatorCountry: "US",
|
||||
OperatorWebsite: "https://aws.amazon.com/bedrock/",
|
||||
OperatorType: "cloud",
|
||||
Region: region,
|
||||
Currency: "USD",
|
||||
InputPrice: row.InputPrice,
|
||||
OutputPrice: row.OutputPrice,
|
||||
SourceURL: defaultBedrockPricingURL,
|
||||
ModelSourceURL: defaultBedrockPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(row.ModelName),
|
||||
}
|
||||
record.IsFree = false
|
||||
seenModelRegion[dedupeKey] = struct{}{}
|
||||
records = append(records, record)
|
||||
}
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
type bedrockProviderBlock struct {
|
||||
providerLabel string
|
||||
content string
|
||||
}
|
||||
|
||||
func splitBedrockProviderBlocks(raw string) []bedrockProviderBlock {
|
||||
marker := `<h2 id="`
|
||||
indices := make([]int, 0)
|
||||
for offset := 0; ; {
|
||||
next := strings.Index(raw[offset:], marker)
|
||||
if next == -1 {
|
||||
break
|
||||
}
|
||||
indices = append(indices, offset+next)
|
||||
offset += next + len(marker)
|
||||
}
|
||||
blocks := make([]bedrockProviderBlock, 0, len(indices))
|
||||
for i, start := range indices {
|
||||
end := len(raw)
|
||||
if i+1 < len(indices) {
|
||||
end = indices[i+1]
|
||||
}
|
||||
chunk := raw[start:end]
|
||||
h2End := strings.Index(chunk, "</h2>")
|
||||
if h2End == -1 {
|
||||
continue
|
||||
}
|
||||
openEnd := strings.Index(chunk, ">")
|
||||
if openEnd == -1 || openEnd >= h2End {
|
||||
continue
|
||||
}
|
||||
label := cleanHTMLText(chunk[openEnd+1 : h2End])
|
||||
if strings.TrimSpace(label) == "" {
|
||||
continue
|
||||
}
|
||||
blocks = append(blocks, bedrockProviderBlock{
|
||||
providerLabel: label,
|
||||
content: chunk,
|
||||
})
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
func extractBetween(raw string, startMarker string, endMarker string) string {
|
||||
start := strings.Index(raw, startMarker)
|
||||
if start == -1 {
|
||||
return ""
|
||||
}
|
||||
segment := raw[start:]
|
||||
if endMarker == "" {
|
||||
return segment
|
||||
}
|
||||
end := strings.Index(segment, endMarker)
|
||||
if end == -1 {
|
||||
return segment
|
||||
}
|
||||
return segment[:end]
|
||||
}
|
||||
|
||||
type bedrockPriceRow struct {
|
||||
ModelName string
|
||||
InputPrice float64
|
||||
OutputPrice float64
|
||||
}
|
||||
|
||||
func parseBedrockTableRows(tableHTML string) []bedrockPriceRow {
|
||||
rows := bedrockRowPattern.FindAllStringSubmatch(tableHTML, -1)
|
||||
parsed := make([]bedrockPriceRow, 0)
|
||||
for _, row := range rows {
|
||||
cells := bedrockCellPattern.FindAllStringSubmatch(row[1], -1)
|
||||
if len(cells) < 3 {
|
||||
continue
|
||||
}
|
||||
values := make([]string, 0, len(cells))
|
||||
for _, cell := range cells {
|
||||
values = append(values, cleanHTMLText(cell[1]))
|
||||
}
|
||||
if strings.Contains(strings.ToLower(values[0]), "models") {
|
||||
continue
|
||||
}
|
||||
|
||||
modelName := values[0]
|
||||
inputCell := values[1]
|
||||
outputCell := values[2]
|
||||
if len(values) >= 6 && strings.Contains(strings.ToLower(values[5]), "$") {
|
||||
outputCell = values[5]
|
||||
}
|
||||
inputPrice, ok := firstDollarPrice(inputCell)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
outputPrice, ok := firstDollarPrice(outputCell)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
parsed = append(parsed, bedrockPriceRow{
|
||||
ModelName: modelName,
|
||||
InputPrice: inputPrice,
|
||||
OutputPrice: outputPrice,
|
||||
})
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func normalizeBedrockProvider(raw string) string {
|
||||
switch strings.TrimSpace(raw) {
|
||||
case "Amazon Nova":
|
||||
return "Amazon"
|
||||
case "Anthropic":
|
||||
return "Anthropic"
|
||||
case "Cohere":
|
||||
return "Cohere"
|
||||
case "DeepSeek":
|
||||
return "DeepSeek"
|
||||
case "Meta":
|
||||
return "Meta"
|
||||
case "Mistral AI":
|
||||
return "Mistral AI"
|
||||
case "Moonshot AI":
|
||||
return "Moonshot AI"
|
||||
case "Kimi":
|
||||
return "Moonshot AI"
|
||||
case "NVIDIA":
|
||||
return "NVIDIA"
|
||||
case "OpenAI OSS Models":
|
||||
return "OpenAI"
|
||||
case "Qwen":
|
||||
return "Qwen"
|
||||
case "Writer":
|
||||
return "Writer"
|
||||
case "Z AI":
|
||||
return "Zhipu AI"
|
||||
default:
|
||||
return strings.TrimSpace(raw)
|
||||
}
|
||||
}
|
||||
|
||||
var bedrockTextProviderHeaderPattern = regexp.MustCompile(`([A-Za-z][A-Za-z0-9 .&-]+)\s+models\s+Pr(?:i)?ce per 1M input tokens`)
|
||||
var bedrockTextRowPattern = regexp.MustCompile(`([A-Za-z0-9 .:+-]+?)\s+\$\s*([0-9.]+)\s+\$\s*([0-9.]+)`)
|
||||
|
||||
func parseBedrockPricingTextFallback(raw string) []officialPricingRecord {
|
||||
matches := bedrockTextProviderHeaderPattern.FindAllStringSubmatchIndex(raw, -1)
|
||||
records := make([]officialPricingRecord, 0)
|
||||
seen := make(map[string]struct{})
|
||||
for i, match := range matches {
|
||||
if len(match) < 4 {
|
||||
continue
|
||||
}
|
||||
start := match[0]
|
||||
end := len(raw)
|
||||
if i+1 < len(matches) {
|
||||
end = matches[i+1][0]
|
||||
}
|
||||
block := raw[start:end]
|
||||
region := normalizeBedrockRegionText(findBedrockTextRegion(raw, start))
|
||||
providerName := normalizeBedrockProvider(raw[match[2]:match[3]])
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
rows := bedrockTextRowPattern.FindAllStringSubmatch(block, -1)
|
||||
for _, row := range rows {
|
||||
if len(row) != 4 {
|
||||
continue
|
||||
}
|
||||
modelName := strings.TrimSpace(row[1])
|
||||
key := strings.Join([]string{providerName, region, modelName}, "|")
|
||||
if _, exists := seen[key]; exists {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
records = append(records, officialPricingRecord{
|
||||
ModelID: normalizeExternalID("bedrock", providerName, modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Amazon Bedrock",
|
||||
OperatorNameCn: "Amazon Bedrock",
|
||||
OperatorCountry: "US",
|
||||
OperatorWebsite: "https://aws.amazon.com/bedrock/",
|
||||
OperatorType: "cloud",
|
||||
Region: region,
|
||||
Currency: "USD",
|
||||
InputPrice: mustParseSubscriptionPrice(row[2]),
|
||||
OutputPrice: mustParseSubscriptionPrice(row[3]),
|
||||
SourceURL: defaultBedrockPricingURL,
|
||||
ModelSourceURL: defaultBedrockPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(modelName),
|
||||
})
|
||||
}
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func findBedrockTextRegion(raw string, headerStart int) string {
|
||||
prefixStart := headerStart - 300
|
||||
if prefixStart < 0 {
|
||||
prefixStart = 0
|
||||
}
|
||||
prefix := raw[prefixStart:headerStart]
|
||||
lastPlural := strings.LastIndex(prefix, "Regions:")
|
||||
lastSingular := strings.LastIndex(prefix, "Region:")
|
||||
lastIndex := lastPlural
|
||||
marker := "Regions:"
|
||||
if lastSingular > lastIndex {
|
||||
lastIndex = lastSingular
|
||||
marker = "Region:"
|
||||
}
|
||||
if lastIndex == -1 {
|
||||
return ""
|
||||
}
|
||||
region := strings.TrimSpace(prefix[lastIndex+len(marker):])
|
||||
for _, stopMarker := range []string{" Priority ", " Flex ", " Batch ", " models "} {
|
||||
if stop := strings.Index(region, stopMarker); stop != -1 {
|
||||
region = strings.TrimSpace(region[:stop])
|
||||
}
|
||||
}
|
||||
return region
|
||||
}
|
||||
|
||||
func normalizeBedrockRegionText(raw string) string {
|
||||
trimmed := strings.TrimSpace(raw)
|
||||
if trimmed == "" {
|
||||
return "global"
|
||||
}
|
||||
trimmed = strings.TrimSuffix(trimmed, ",")
|
||||
return strings.Join(strings.Fields(trimmed), " ")
|
||||
}
|
||||
119
scripts/bytedance_subscription_lib.go
Normal file
119
scripts/bytedance_subscription_lib.go
Normal file
@@ -0,0 +1,119 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultBytedanceCodingPlanURL = "https://developer.volcengine.com/articles/7574419773204004906"
|
||||
defaultBytedanceCodingPlanNotice = "https://developer.volcengine.com/articles/7604465649330749490"
|
||||
)
|
||||
|
||||
func parseBytedanceSubscriptionCatalog(pricingRaw string, noticeRaw string) ([]subscriptionImportRecord, error) {
|
||||
publishedAt, known := publishedAtFromText(firstNonEmptyText(pricingRaw, noticeRaw))
|
||||
liteSection := sliceSection(pricingRaw, "Lite 套餐", "Pro 套餐")
|
||||
proSection := sliceSection(pricingRaw, "Pro 套餐", "")
|
||||
if strings.TrimSpace(liteSection) == "" || strings.TrimSpace(proSection) == "" {
|
||||
return nil, fmt.Errorf("unexpected bytedance coding plan sections")
|
||||
}
|
||||
|
||||
promoNote := "新用户首购优惠。"
|
||||
if strings.Contains(noticeRaw, "每日 10:30") {
|
||||
promoNote = "新用户首购优惠,每日 10:30 限量开放。"
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, 4)
|
||||
for _, tierSection := range []struct {
|
||||
Tier string
|
||||
Content string
|
||||
}{
|
||||
{Tier: "Lite", Content: liteSection},
|
||||
{Tier: "Pro", Content: proSection},
|
||||
} {
|
||||
tier := tierSection.Tier
|
||||
section := strings.TrimSpace(tierSection.Content)
|
||||
lines := strings.Split(section, "\n")
|
||||
scene := ""
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" {
|
||||
scene = line
|
||||
break
|
||||
}
|
||||
}
|
||||
pricePattern := regexp.MustCompile(`([\d.]+)\s+元\s+([\d.]+)\s+元/月`)
|
||||
priceMatch := pricePattern.FindStringSubmatch(section)
|
||||
if len(priceMatch) != 3 {
|
||||
return nil, fmt.Errorf("missing bytedance %s prices", tier)
|
||||
}
|
||||
quotaPattern := regexp.MustCompile(`每月约\s+([\d,]+)\s+次请求`)
|
||||
quotaMatch := quotaPattern.FindStringSubmatch(section)
|
||||
if len(quotaMatch) != 2 {
|
||||
return nil, fmt.Errorf("missing bytedance %s monthly quota", tier)
|
||||
}
|
||||
promoPrice := mustParseSubscriptionPrice(priceMatch[1])
|
||||
standardPrice := mustParseSubscriptionPrice(priceMatch[2])
|
||||
monthlyQuota := mustParseSubscriptionInt64(quotaMatch[1])
|
||||
tierCode := strings.ToLower(tier)
|
||||
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "ByteDance",
|
||||
ProviderNameCn: "字节跳动",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.volcengine.com",
|
||||
OperatorName: "ByteDance Volcano",
|
||||
OperatorNameCn: "火山引擎",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://developer.volcengine.com",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "bytedance-coding-plan-" + tierCode,
|
||||
PlanName: "方舟 Coding Plan " + tier,
|
||||
Tier: tier,
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: standardPrice,
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: monthlyQuota,
|
||||
QuotaUnit: "requests/month",
|
||||
PlanScope: "方舟 Coding Plan",
|
||||
SourceURL: defaultBytedanceCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: scene + ";续费标准价。",
|
||||
PublishedAtKnown: known,
|
||||
})
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "ByteDance",
|
||||
ProviderNameCn: "字节跳动",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.volcengine.com",
|
||||
OperatorName: "ByteDance Volcano",
|
||||
OperatorNameCn: "火山引擎",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://developer.volcengine.com",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "bytedance-coding-plan-" + tierCode + "-first-month",
|
||||
PlanName: "方舟 Coding Plan " + tier + " 首月活动版",
|
||||
Tier: tier + " Promo",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: promoPrice,
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: monthlyQuota,
|
||||
QuotaUnit: "requests/month",
|
||||
PlanScope: "方舟 Coding Plan",
|
||||
SourceURL: defaultBytedanceCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: scene + ";" + promoNote,
|
||||
PublishedAtKnown: known,
|
||||
})
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
56
scripts/catalog_verification_common.go
Normal file
56
scripts/catalog_verification_common.go
Normal file
@@ -0,0 +1,56 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
type catalogVerificationRecord struct {
|
||||
CatalogCode string
|
||||
SourceURL string
|
||||
SourceTitle string
|
||||
PlanStatus string
|
||||
Notes string
|
||||
}
|
||||
|
||||
type catalogVerificationImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func upsertCatalogVerificationRecords(db *sql.DB, records []catalogVerificationRecord) error {
|
||||
if len(records) == 0 {
|
||||
return fmt.Errorf("catalog verification records are empty")
|
||||
}
|
||||
for _, record := range records {
|
||||
result, err := db.Exec(
|
||||
`UPDATE plan_catalog_inventory
|
||||
SET source_url = $2,
|
||||
source_title = $3,
|
||||
plan_status = $4,
|
||||
notes = $5,
|
||||
last_checked_at = CURRENT_TIMESTAMP,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE catalog_code = $1`,
|
||||
record.CatalogCode, record.SourceURL, record.SourceTitle, record.PlanStatus, record.Notes,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update plan_catalog_inventory %s: %w", record.CatalogCode, err)
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("rows affected for %s: %w", record.CatalogCode, err)
|
||||
}
|
||||
if rowsAffected == 0 {
|
||||
return fmt.Errorf("catalog_code %s not found in plan_catalog_inventory", record.CatalogCode)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
66
scripts/cloudflare_pricing_import_runner.go
Normal file
66
scripts/cloudflare_pricing_import_runner.go
Normal file
@@ -0,0 +1,66 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type cloudflarePricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
SnapshotOnly bool
|
||||
SnapshotOut string
|
||||
SignatureOut string
|
||||
}
|
||||
|
||||
func runCloudflarePricingImport(cfg cloudflarePricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchRawPricingPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.SnapshotOnly || strings.TrimSpace(cfg.SnapshotOut) != "" || strings.TrimSpace(cfg.SignatureOut) != "" {
|
||||
snapshotPath, signaturePath := resolveCloudflarePricingSnapshotPaths(cfg.SnapshotOut, cfg.SignatureOut, "", time.Now())
|
||||
signature, err := writeCloudflarePricingSnapshotArtifacts(raw, cfg.URL, snapshotPath, signaturePath, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.SnapshotOnly {
|
||||
_, err = fmt.Fprintf(out,
|
||||
"source=cloudflare-pricing-snapshot snapshot_only=true byte_size=%d sha256=%s structure_sha256=%s snapshot_out=%s signature_out=%s\n",
|
||||
signature.ByteSize, signature.SHA256, signature.StructureSHA256, snapshotPath, signaturePath,
|
||||
)
|
||||
return err
|
||||
}
|
||||
}
|
||||
records, err := parseCloudflarePricingCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=cloudflare-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "cloudflare-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=cloudflare-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
108
scripts/cloudflare_pricing_lib.go
Normal file
108
scripts/cloudflare_pricing_lib.go
Normal file
@@ -0,0 +1,108 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultCloudflarePricingFetchURL = "https://developers.cloudflare.com/workers-ai/platform/pricing/index.md"
|
||||
defaultCloudflarePricingSourceURL = "https://developers.cloudflare.com/workers-ai/platform/pricing/"
|
||||
)
|
||||
|
||||
func parseCloudflarePricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
section, ok := extractCloudflareLLMPricingSection(raw)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected cloudflare pricing content")
|
||||
}
|
||||
|
||||
lines := strings.Split(section, "\n")
|
||||
records := make([]officialPricingRecord, 0)
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "| @cf/") {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(line, "|")
|
||||
if len(parts) < 4 {
|
||||
continue
|
||||
}
|
||||
modelPath := strings.Trim(strings.TrimSpace(parts[1]), "`")
|
||||
priceCell := strings.TrimSpace(parts[2])
|
||||
prices := extractCloudflarePrices(priceCell)
|
||||
if len(prices) < 2 {
|
||||
continue
|
||||
}
|
||||
providerName := providerFromModelPath(strings.TrimPrefix(modelPath, "@cf/"))
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
record := officialPricingRecord{
|
||||
ModelID: normalizeExternalID("cloudflare", modelPath),
|
||||
ModelName: modelPath,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Cloudflare Workers AI",
|
||||
OperatorNameCn: "Cloudflare Workers AI",
|
||||
OperatorCountry: "US",
|
||||
OperatorWebsite: "https://developers.cloudflare.com/workers-ai/",
|
||||
OperatorType: "cloud",
|
||||
Region: "global",
|
||||
Currency: "USD",
|
||||
InputPrice: prices[0],
|
||||
OutputPrice: prices[1],
|
||||
SourceURL: defaultCloudflarePricingSourceURL,
|
||||
ModelSourceURL: defaultCloudflarePricingSourceURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(modelPath),
|
||||
}
|
||||
record.IsFree = record.InputPrice == 0 && record.OutputPrice == 0
|
||||
records = append(records, record)
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("no cloudflare llm pricing rows found")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func extractCloudflarePrices(raw string) []float64 {
|
||||
fields := strings.Split(raw, "$")
|
||||
prices := make([]float64, 0, 3)
|
||||
for _, field := range fields[1:] {
|
||||
value := strings.TrimSpace(field)
|
||||
end := strings.Index(value, " per ")
|
||||
if end == -1 {
|
||||
continue
|
||||
}
|
||||
prices = append(prices, mustParseSubscriptionPrice(value[:end]))
|
||||
}
|
||||
return prices
|
||||
}
|
||||
|
||||
func extractCloudflareLLMPricingSection(raw string) (string, bool) {
|
||||
lines := strings.Split(raw, "\n")
|
||||
start := -1
|
||||
end := len(lines)
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(trimmed, "## ") {
|
||||
continue
|
||||
}
|
||||
title := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(trimmed, "## ")))
|
||||
if start == -1 {
|
||||
if strings.Contains(title, "llm") && strings.Contains(title, "pricing") {
|
||||
start = i
|
||||
}
|
||||
continue
|
||||
}
|
||||
end = i
|
||||
break
|
||||
}
|
||||
if start == -1 {
|
||||
return "", false
|
||||
}
|
||||
return strings.Join(lines[start:end], "\n"), true
|
||||
}
|
||||
51
scripts/cloudflare_pricing_signature_guard.go
Normal file
51
scripts/cloudflare_pricing_signature_guard.go
Normal file
@@ -0,0 +1,51 @@
|
||||
//go:build llm_script && !scripts_pkg
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var snapshotDir string
|
||||
var baselinePath string
|
||||
var timeoutSeconds int
|
||||
var allowBootstrap bool
|
||||
|
||||
flag.StringVar(&url, "url", defaultCloudflarePricingFetchURL, "Cloudflare Workers AI 官方价格 markdown")
|
||||
flag.StringVar(&fixture, "fixture", "", "Cloudflare Workers AI 价格样例文件")
|
||||
flag.StringVar(&snapshotDir, "snapshot-dir", "", "Cloudflare snapshot 输出目录")
|
||||
flag.StringVar(&baselinePath, "baseline-path", "", "Cloudflare 结构基线签名路径")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.BoolVar(&allowBootstrap, "allow-bootstrap", true, "当 baseline 缺失时自动初始化")
|
||||
flag.Parse()
|
||||
|
||||
now := time.Now()
|
||||
cfg := cloudflarePricingSignatureGuardConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
SnapshotDir: snapshotDir,
|
||||
BaselinePath: baselinePath,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
AllowBootstrap: allowBootstrap,
|
||||
}
|
||||
result, err := runCloudflarePricingSignatureGuard(cfg, now)
|
||||
if auditErr := persistCloudflarePricingSignatureAuditIfConfigured(cfg, result, now, err); auditErr != nil {
|
||||
fmt.Fprintf(os.Stderr, "cloudflare_pricing_signature_guard audit: %v\n", auditErr)
|
||||
if err == nil {
|
||||
err = auditErr
|
||||
}
|
||||
}
|
||||
fmt.Println(formatCloudflarePricingSignatureGuardSummary(result))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cloudflare_pricing_signature_guard: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
136
scripts/cloudflare_pricing_signature_guard_lib.go
Normal file
136
scripts/cloudflare_pricing_signature_guard_lib.go
Normal file
@@ -0,0 +1,136 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type cloudflarePricingSignatureGuardConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
SnapshotDir string
|
||||
BaselinePath string
|
||||
Timeout time.Duration
|
||||
AllowBootstrap bool
|
||||
}
|
||||
|
||||
type cloudflarePricingSignatureGuardResult struct {
|
||||
SnapshotPath string
|
||||
SignaturePath string
|
||||
BaselinePath string
|
||||
DriftDetected bool
|
||||
BaselineInitialized bool
|
||||
PreviousBaselineHash string
|
||||
CurrentSignature markdownPricingStructureSignature
|
||||
}
|
||||
|
||||
func runCloudflarePricingSignatureGuard(cfg cloudflarePricingSignatureGuardConfig, now time.Time) (cloudflarePricingSignatureGuardResult, error) {
|
||||
snapshotDir := cfg.SnapshotDir
|
||||
if snapshotDir == "" {
|
||||
snapshotDir = filepath.Join("logs", "cloudflare-pricing-snapshots")
|
||||
}
|
||||
if err := os.MkdirAll(snapshotDir, 0o755); err != nil {
|
||||
return cloudflarePricingSignatureGuardResult{}, fmt.Errorf("mkdir snapshot dir: %w", err)
|
||||
}
|
||||
|
||||
snapshotPath, signaturePath := resolveCloudflarePricingSnapshotPaths("", "", snapshotDir, now)
|
||||
baselinePath := cfg.BaselinePath
|
||||
if baselinePath == "" {
|
||||
baselinePath = filepath.Join(snapshotDir, "baseline.signature.json")
|
||||
}
|
||||
|
||||
clientCfg := cloudflarePricingImportConfig{
|
||||
URL: cfg.URL,
|
||||
Fixture: cfg.Fixture,
|
||||
DryRun: true,
|
||||
Timeout: cfg.Timeout,
|
||||
SnapshotOnly: true,
|
||||
SnapshotOut: snapshotPath,
|
||||
SignatureOut: signaturePath,
|
||||
}
|
||||
if err := runCloudflarePricingImport(clientCfg, nil, ioDiscard{}); err != nil {
|
||||
return cloudflarePricingSignatureGuardResult{}, err
|
||||
}
|
||||
|
||||
current, err := readMarkdownPricingStructureSignature(signaturePath)
|
||||
if err != nil {
|
||||
return cloudflarePricingSignatureGuardResult{}, err
|
||||
}
|
||||
result := cloudflarePricingSignatureGuardResult{
|
||||
SnapshotPath: snapshotPath,
|
||||
SignaturePath: signaturePath,
|
||||
BaselinePath: baselinePath,
|
||||
CurrentSignature: current,
|
||||
}
|
||||
|
||||
previous, err := readMarkdownPricingStructureSignature(baselinePath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
if !cfg.AllowBootstrap {
|
||||
return result, fmt.Errorf("cloudflare pricing baseline missing: %s", baselinePath)
|
||||
}
|
||||
if err := copyFileCommon(signaturePath, baselinePath); err != nil {
|
||||
return result, fmt.Errorf("initialize baseline: %w", err)
|
||||
}
|
||||
result.BaselineInitialized = true
|
||||
return result, nil
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
|
||||
result.PreviousBaselineHash = previous.StructureSHA256
|
||||
if previous.StructureSHA256 != current.StructureSHA256 {
|
||||
result.DriftDetected = true
|
||||
return result, fmt.Errorf(
|
||||
"cloudflare pricing structure drift detected: baseline=%s current=%s baseline_path=%s signature_path=%s snapshot_path=%s",
|
||||
previous.StructureSHA256, current.StructureSHA256, baselinePath, signaturePath, snapshotPath,
|
||||
)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func formatCloudflarePricingSignatureGuardSummary(result cloudflarePricingSignatureGuardResult) string {
|
||||
return fmt.Sprintf(
|
||||
"source=cloudflare-pricing-signature-guard drift=%t baseline_initialized=%t structure_sha256=%s previous_baseline_sha256=%s snapshot_out=%s signature_out=%s baseline_path=%s",
|
||||
result.DriftDetected,
|
||||
result.BaselineInitialized,
|
||||
result.CurrentSignature.StructureSHA256,
|
||||
emptyIfBlank(result.PreviousBaselineHash),
|
||||
result.SnapshotPath,
|
||||
result.SignaturePath,
|
||||
result.BaselinePath,
|
||||
)
|
||||
}
|
||||
|
||||
func buildCloudflarePricingSignatureAuditRecord(cfg cloudflarePricingSignatureGuardConfig, result cloudflarePricingSignatureGuardResult, checkedAt time.Time, runErr error) officialImportSignatureAuditRecord {
|
||||
record := officialImportSignatureAuditRecord{
|
||||
SourceKey: "cloudflare_pricing_signature",
|
||||
CheckedAt: checkedAt,
|
||||
Status: officialImportSignatureAuditStatus(result.DriftDetected, result.BaselineInitialized, runErr),
|
||||
DriftDetected: result.DriftDetected,
|
||||
BaselineInitialized: result.BaselineInitialized,
|
||||
SourceURL: strings.TrimSpace(cfg.URL),
|
||||
FixturePath: strings.TrimSpace(cfg.Fixture),
|
||||
SnapshotPath: strings.TrimSpace(result.SnapshotPath),
|
||||
SignaturePath: strings.TrimSpace(result.SignaturePath),
|
||||
BaselinePath: strings.TrimSpace(result.BaselinePath),
|
||||
StructureSHA256: strings.TrimSpace(result.CurrentSignature.StructureSHA256),
|
||||
PreviousStructureSHA256: strings.TrimSpace(result.PreviousBaselineHash),
|
||||
ByteSize: result.CurrentSignature.ByteSize,
|
||||
ErrorMessage: errorMessageText(runErr),
|
||||
}
|
||||
if hasMarkdownPricingStructureSignature(result.CurrentSignature) {
|
||||
signatureCopy := result.CurrentSignature
|
||||
record.SignaturePayload = &signatureCopy
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
func persistCloudflarePricingSignatureAuditIfConfigured(cfg cloudflarePricingSignatureGuardConfig, result cloudflarePricingSignatureGuardResult, checkedAt time.Time, runErr error) error {
|
||||
return persistOfficialImportSignatureAuditIfConfigured(buildCloudflarePricingSignatureAuditRecord(cfg, result, checkedAt, runErr))
|
||||
}
|
||||
102
scripts/cloudflare_pricing_signature_guard_test.go
Normal file
102
scripts/cloudflare_pricing_signature_guard_test.go
Normal file
@@ -0,0 +1,102 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRunCloudflarePricingSignatureGuardInitializesBaseline(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
baselinePath := filepath.Join(tempDir, "baseline.signature.json")
|
||||
|
||||
result, err := runCloudflarePricingSignatureGuard(cloudflarePricingSignatureGuardConfig{
|
||||
URL: defaultCloudflarePricingFetchURL,
|
||||
Fixture: filepath.Join("testdata", "cloudflare_pricing_sample.md"),
|
||||
SnapshotDir: tempDir,
|
||||
BaselinePath: baselinePath,
|
||||
Timeout: time.Second,
|
||||
AllowBootstrap: true,
|
||||
}, time.Date(2026, 5, 15, 20, 30, 0, 0, time.FixedZone("CST", 8*3600)))
|
||||
if err != nil {
|
||||
t.Fatalf("runCloudflarePricingSignatureGuard 返回错误: %v", err)
|
||||
}
|
||||
if !result.BaselineInitialized {
|
||||
t.Fatalf("期望初始化 baseline")
|
||||
}
|
||||
if result.DriftDetected {
|
||||
t.Fatalf("首次初始化不应判定为漂移")
|
||||
}
|
||||
if _, err := os.Stat(baselinePath); err != nil {
|
||||
t.Fatalf("baseline 未写入: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCloudflarePricingSignatureGuardDetectsDrift(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
baselinePath := filepath.Join(tempDir, "baseline.signature.json")
|
||||
|
||||
_, err := runCloudflarePricingSignatureGuard(cloudflarePricingSignatureGuardConfig{
|
||||
URL: defaultCloudflarePricingFetchURL,
|
||||
Fixture: filepath.Join("testdata", "cloudflare_pricing_sample.md"),
|
||||
SnapshotDir: tempDir,
|
||||
BaselinePath: baselinePath,
|
||||
Timeout: time.Second,
|
||||
AllowBootstrap: true,
|
||||
}, time.Date(2026, 5, 15, 20, 31, 0, 0, time.FixedZone("CST", 8*3600)))
|
||||
if err != nil {
|
||||
t.Fatalf("初始化 baseline 失败: %v", err)
|
||||
}
|
||||
|
||||
driftFixture := "## Text model pricing\n\n| Model | Price |\n| --- | --- |\n| @cf/meta/llama-3.1-8b-instruct | $1 |\n"
|
||||
driftPath := filepath.Join(tempDir, "cloudflare-drift.md")
|
||||
if err := os.WriteFile(driftPath, []byte(driftFixture), 0o644); err != nil {
|
||||
t.Fatalf("写入 drift fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
result, err := runCloudflarePricingSignatureGuard(cloudflarePricingSignatureGuardConfig{
|
||||
URL: defaultCloudflarePricingFetchURL,
|
||||
Fixture: driftPath,
|
||||
SnapshotDir: tempDir,
|
||||
BaselinePath: baselinePath,
|
||||
Timeout: time.Second,
|
||||
AllowBootstrap: false,
|
||||
}, time.Date(2026, 5, 15, 20, 32, 0, 0, time.FixedZone("CST", 8*3600)))
|
||||
if err == nil {
|
||||
t.Fatalf("期望结构漂移时报错")
|
||||
}
|
||||
if !result.DriftDetected {
|
||||
t.Fatalf("期望 driftDetected=true")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cloudflare pricing structure drift detected") {
|
||||
t.Fatalf("期望返回 drift 错误,实际: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatCloudflarePricingSignatureGuardSummary(t *testing.T) {
|
||||
result := cloudflarePricingSignatureGuardResult{
|
||||
SnapshotPath: "/tmp/cloudflare.md",
|
||||
SignaturePath: "/tmp/cloudflare.signature.json",
|
||||
BaselinePath: "/tmp/baseline.signature.json",
|
||||
DriftDetected: false,
|
||||
BaselineInitialized: true,
|
||||
CurrentSignature: markdownPricingStructureSignature{
|
||||
StructureSHA256: "abc123",
|
||||
},
|
||||
}
|
||||
summary := formatCloudflarePricingSignatureGuardSummary(result)
|
||||
for _, want := range []string{
|
||||
"source=cloudflare-pricing-signature-guard",
|
||||
"drift=false",
|
||||
"baseline_initialized=true",
|
||||
"structure_sha256=abc123",
|
||||
} {
|
||||
if !strings.Contains(summary, want) {
|
||||
t.Fatalf("summary 缺少 %q,实际: %q", want, summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
24
scripts/cloudflare_pricing_snapshot_lib.go
Normal file
24
scripts/cloudflare_pricing_snapshot_lib.go
Normal file
@@ -0,0 +1,24 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import "time"
|
||||
|
||||
var cloudflarePricingSignatureContainsNeedles = map[string]string{
|
||||
"llm": "llm",
|
||||
"pricing": "pricing",
|
||||
"cf_model_prefix": "@cf/",
|
||||
"price_tokens": "price in tokens",
|
||||
}
|
||||
|
||||
func buildCloudflarePricingStructureSignature(raw string) markdownPricingStructureSignature {
|
||||
return buildMarkdownPricingStructureSignature(raw, cloudflarePricingSignatureContainsNeedles)
|
||||
}
|
||||
|
||||
func writeCloudflarePricingSnapshotArtifacts(raw string, sourceURL string, snapshotPath string, signaturePath string, now time.Time) (markdownPricingStructureSignature, error) {
|
||||
return writeMarkdownPricingSnapshotArtifacts(raw, sourceURL, snapshotPath, signaturePath, now, cloudflarePricingSignatureContainsNeedles)
|
||||
}
|
||||
|
||||
func resolveCloudflarePricingSnapshotPaths(snapshotPath string, signaturePath string, snapshotDir string, now time.Time) (string, string) {
|
||||
return resolveMarkdownPricingSnapshotPaths(snapshotPath, signaturePath, snapshotDir, "cloudflare-pricing", now)
|
||||
}
|
||||
90
scripts/cloudflare_pricing_snapshot_test.go
Normal file
90
scripts/cloudflare_pricing_snapshot_test.go
Normal file
@@ -0,0 +1,90 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildCloudflarePricingStructureSignatureCapturesShape(t *testing.T) {
|
||||
raw := `
|
||||
## LLM pricing
|
||||
|
||||
| Model | Price in Tokens | Price in Neurons |
|
||||
| --- | --- | --- |
|
||||
| @cf/meta/llama-3.1-8b-instruct | $0.20 per M input tokens $1.00 per M output tokens | ignored |
|
||||
`
|
||||
|
||||
signature := buildCloudflarePricingStructureSignature(raw)
|
||||
if signature.ByteSize == 0 {
|
||||
t.Fatalf("期望 byte_size 非 0")
|
||||
}
|
||||
if signature.SHA256 == "" || signature.StructureSHA256 == "" {
|
||||
t.Fatalf("期望生成 sha256 签名: %+v", signature)
|
||||
}
|
||||
if len(signature.Headings) == 0 || signature.Headings[0] != "LLM pricing" {
|
||||
t.Fatalf("标题提取错误: %+v", signature.Headings)
|
||||
}
|
||||
if len(signature.TableHeaders) == 0 || !strings.Contains(signature.TableHeaders[0], "Price in Tokens") {
|
||||
t.Fatalf("表头提取错误: %+v", signature.TableHeaders)
|
||||
}
|
||||
if !signature.Contains["llm"] || !signature.Contains["pricing"] || !signature.Contains["cf_model_prefix"] {
|
||||
t.Fatalf("期望识别 Cloudflare 关键结构: %+v", signature.Contains)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCloudflarePricingImportSnapshotOnlyWritesArtifacts(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
snapshotPath := filepath.Join(tempDir, "cloudflare-live.md")
|
||||
signaturePath := filepath.Join(tempDir, "cloudflare-live.signature.json")
|
||||
|
||||
var out bytes.Buffer
|
||||
err := runCloudflarePricingImport(cloudflarePricingImportConfig{
|
||||
URL: defaultCloudflarePricingFetchURL,
|
||||
Fixture: filepath.Join("testdata", "cloudflare_pricing_sample.md"),
|
||||
DryRun: true,
|
||||
SnapshotOnly: true,
|
||||
SnapshotOut: snapshotPath,
|
||||
SignatureOut: signaturePath,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runCloudflarePricingImport 返回错误: %v", err)
|
||||
}
|
||||
|
||||
snapshotBytes, err := os.ReadFile(snapshotPath)
|
||||
if err != nil {
|
||||
t.Fatalf("读取 snapshot 失败: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(snapshotBytes), "@cf/meta/llama-3.2-1b-instruct") {
|
||||
t.Fatalf("snapshot 内容错误")
|
||||
}
|
||||
|
||||
signatureBytes, err := os.ReadFile(signaturePath)
|
||||
if err != nil {
|
||||
t.Fatalf("读取 signature 失败: %v", err)
|
||||
}
|
||||
var signature markdownPricingStructureSignature
|
||||
if err := json.Unmarshal(signatureBytes, &signature); err != nil {
|
||||
t.Fatalf("signature JSON 解析失败: %v", err)
|
||||
}
|
||||
if !signature.Contains["cf_model_prefix"] {
|
||||
t.Fatalf("期望 signature 含 cf_model_prefix: %+v", signature.Contains)
|
||||
}
|
||||
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=cloudflare-pricing-snapshot",
|
||||
"snapshot_only=true",
|
||||
"signature_out=" + signaturePath,
|
||||
"snapshot_out=" + snapshotPath,
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
192
scripts/collector_stats_window_audit.sh
Normal file
192
scripts/collector_stats_window_audit.sh
Normal file
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
LIMIT=7
|
||||
DB_URL="${DATABASE_URL:-}"
|
||||
INPUT_PATH=""
|
||||
THRESHOLD=""
|
||||
FIELD_SEP=$'\x1f'
|
||||
NOW_RAW="${LLM_NOW:-}"
|
||||
AGED_PRECONDITION_COUNT=0
|
||||
AGED_PRECONDITION_MINUTES=1440
|
||||
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
用法:
|
||||
bash scripts/collector_stats_window_audit.sh --db <DATABASE_URL> [--limit N] [--assert-success-rate PCT]
|
||||
bash scripts/collector_stats_window_audit.sh --input <tsv-file> [--limit N] [--assert-success-rate PCT]
|
||||
|
||||
输入 TSV 列顺序:
|
||||
source<TAB>success<TAB>error_message<TAB>created_at
|
||||
EOF
|
||||
}
|
||||
|
||||
classify_failure() {
|
||||
local message normalized
|
||||
message="${1:-}"
|
||||
normalized="$(printf '%s' "$message" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
if [[ -z "${normalized// }" ]]; then
|
||||
printf '%s\n' "collector_runtime_failure"
|
||||
return
|
||||
fi
|
||||
|
||||
case "$normalized" in
|
||||
*"api key"*|*"openrouter_api_key"*|*"database_url"*|*"strict real mode"*|*"password authentication failed"*|*"permission denied"*|*"role does not exist"*|*"relation does not exist"*|*"must provide"*|*"未设置"*)
|
||||
printf '%s\n' "precondition_missing"
|
||||
;;
|
||||
*"429"*|*"rate limit"*|*"too many requests"*|*"timeout"*|*"temporarily unavailable"*|*"transport closed"*|*"connection reset"*|*"connection refused"*|*"eof"*|*"tls handshake timeout"*|*"no such host"*|*"i/o timeout"*|*"unexpected status 403"*|*"unexpected status 502"*|*"unexpected status 503"*|*"unexpected status 504"*|*"signature drift"*|*"no pricing cards found"*|*"no model rows parsed"*|*"no model overview cards parsed"*|*"unexpected * pricing content"*)
|
||||
printf '%s\n' "external_provider_failure"
|
||||
;;
|
||||
*)
|
||||
printf '%s\n' "collector_runtime_failure"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
minutes_since_created() {
|
||||
local created_at="$1"
|
||||
python3 - <<'PY' "$created_at" "$NOW_RAW"
|
||||
from datetime import datetime
|
||||
import sys
|
||||
created = datetime.strptime(sys.argv[1], '%Y-%m-%d %H:%M:%S')
|
||||
raw_now = sys.argv[2].strip()
|
||||
now = datetime.strptime(raw_now, '%Y-%m-%d %H:%M') if raw_now else datetime.now()
|
||||
print(int((now - created).total_seconds() // 60))
|
||||
PY
|
||||
}
|
||||
|
||||
|
||||
fetch_rows_from_db() {
|
||||
if [[ -z "${DB_URL:-}" ]]; then
|
||||
echo "missing --db / DATABASE_URL" >&2
|
||||
return 1
|
||||
fi
|
||||
psql "$DB_URL" -F "$FIELD_SEP" -Atqc "
|
||||
SELECT
|
||||
COALESCE(source, ''),
|
||||
CASE WHEN success THEN 't' ELSE 'f' END,
|
||||
COALESCE(error_message, ''),
|
||||
TO_CHAR(created_at, 'YYYY-MM-DD HH24:MI:SS')
|
||||
FROM collector_stats
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ${LIMIT};
|
||||
"
|
||||
}
|
||||
|
||||
fetch_rows_from_file() {
|
||||
if [[ -z "${INPUT_PATH:-}" ]]; then
|
||||
echo "missing --input" >&2
|
||||
return 1
|
||||
fi
|
||||
head -n "$LIMIT" "$INPUT_PATH"
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--db)
|
||||
DB_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--input)
|
||||
INPUT_PATH="$2"
|
||||
shift 2
|
||||
;;
|
||||
--limit)
|
||||
LIMIT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--assert-success-rate)
|
||||
THRESHOLD="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help|-h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown arg: $1" >&2
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -n "$INPUT_PATH" ]]; then
|
||||
ROWS="$(fetch_rows_from_file)"
|
||||
else
|
||||
ROWS="$(fetch_rows_from_db)"
|
||||
fi
|
||||
|
||||
SUCCESS_COUNT=0
|
||||
FAILURE_COUNT=0
|
||||
PRECONDITION_COUNT=0
|
||||
EXTERNAL_COUNT=0
|
||||
RUNTIME_COUNT=0
|
||||
UNKNOWN_COUNT=0
|
||||
ROW_COUNT=0
|
||||
DETAIL_LINES=""
|
||||
|
||||
while IFS= read -r raw_line; do
|
||||
[[ -z "${raw_line}" ]] && continue
|
||||
normalized_line="${raw_line//$'\t'/$FIELD_SEP}"
|
||||
IFS="$FIELD_SEP" read -r source success error_message created_at <<< "$normalized_line"
|
||||
ROW_COUNT=$((ROW_COUNT + 1))
|
||||
if [[ "$success" == "t" || "$success" == "true" ]]; then
|
||||
SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
|
||||
category="success"
|
||||
rendered_error="-"
|
||||
else
|
||||
FAILURE_COUNT=$((FAILURE_COUNT + 1))
|
||||
category="$(classify_failure "$error_message")"
|
||||
rendered_error="${error_message:-unknown}"
|
||||
if [[ "$category" == "precondition_missing" ]]; then
|
||||
age_minutes="$(minutes_since_created "${created_at:-1970-01-01 00:00:00}")"
|
||||
if [[ "$age_minutes" -gt "$AGED_PRECONDITION_MINUTES" ]]; then
|
||||
category="aged_precondition_missing"
|
||||
AGED_PRECONDITION_COUNT=$((AGED_PRECONDITION_COUNT + 1))
|
||||
fi
|
||||
fi
|
||||
case "$category" in
|
||||
precondition_missing)
|
||||
PRECONDITION_COUNT=$((PRECONDITION_COUNT + 1))
|
||||
;;
|
||||
aged_precondition_missing)
|
||||
;;
|
||||
external_provider_failure)
|
||||
EXTERNAL_COUNT=$((EXTERNAL_COUNT + 1))
|
||||
;;
|
||||
collector_runtime_failure)
|
||||
RUNTIME_COUNT=$((RUNTIME_COUNT + 1))
|
||||
;;
|
||||
*)
|
||||
UNKNOWN_COUNT=$((UNKNOWN_COUNT + 1))
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
DETAIL_LINES+=$'sample_'"${ROW_COUNT}"$' created_at='"${created_at:-unknown}"$' source='"${source:-unknown}"$' outcome='"$([[ "$category" == "success" ]] && printf '%s' "success" || printf '%s' "failure")"$' category='"${category}"$' error='"${rendered_error}"$'\n'
|
||||
done <<< "$ROWS"
|
||||
|
||||
if [[ "$ROW_COUNT" -eq 0 ]]; then
|
||||
echo "window_size=0 success_count=0 failure_count=0 success_rate=0.00 threshold=${THRESHOLD:-n/a} precondition_missing=0 aged_precondition_missing=0 external_provider_failure=0 collector_runtime_failure=0 unknown_failure=0"
|
||||
|
||||
echo "sample_window=empty"
|
||||
if [[ -n "$THRESHOLD" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
SUCCESS_RATE="$(awk -v success="$SUCCESS_COUNT" -v aged="$AGED_PRECONDITION_COUNT" -v total="$ROW_COUNT" 'BEGIN { effective_total = total - aged; if (effective_total <= 0) { printf "0.00" } else { printf "%.2f", (success * 100) / effective_total } }')"
|
||||
echo "window_size=${ROW_COUNT} success_count=${SUCCESS_COUNT} failure_count=${FAILURE_COUNT} success_rate=${SUCCESS_RATE} threshold=${THRESHOLD:-n/a} precondition_missing=${PRECONDITION_COUNT} aged_precondition_missing=${AGED_PRECONDITION_COUNT} external_provider_failure=${EXTERNAL_COUNT} collector_runtime_failure=${RUNTIME_COUNT} unknown_failure=${UNKNOWN_COUNT}"
|
||||
printf '%s' "$DETAIL_LINES"
|
||||
|
||||
if [[ -n "$THRESHOLD" ]]; then
|
||||
if awk -v actual="$SUCCESS_RATE" -v threshold="$THRESHOLD" 'BEGIN { exit !(actual >= threshold) }'; then
|
||||
exit 0
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
88
scripts/collector_stats_window_audit_test.sh
Normal file
88
scripts/collector_stats_window_audit_test.sh
Normal file
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
FIXTURE_FAIL="$TMP_DIR/collector_stats_fail.tsv"
|
||||
cat > "$FIXTURE_FAIL" <<'EOF'
|
||||
openrouter f 严格真实模式下必须提供 API Key 2026-05-15 20:00:00
|
||||
openrouter f 429 Too Many Requests 2026-05-15 19:59:00
|
||||
openrouter t 2026-05-15 19:58:00
|
||||
openrouter t 2026-05-15 19:57:00
|
||||
openrouter t 2026-05-15 19:56:00
|
||||
openrouter t 2026-05-15 19:55:00
|
||||
openrouter f insert models failed 2026-05-15 19:54:00
|
||||
EOF
|
||||
|
||||
set +e
|
||||
FAIL_OUTPUT="$(bash scripts/collector_stats_window_audit.sh --input "$FIXTURE_FAIL" --limit 7 --assert-success-rate 95 2>&1)"
|
||||
FAIL_RC=$?
|
||||
set -e
|
||||
|
||||
if [[ "$FAIL_RC" -eq 0 ]]; then
|
||||
echo "expected failing fixture to exit non-zero"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '%s' "$FAIL_OUTPUT" | grep -q 'success_rate=66.67'
|
||||
printf '%s' "$FAIL_OUTPUT" | grep -q 'precondition_missing=0'
|
||||
printf '%s' "$FAIL_OUTPUT" | grep -q 'aged_precondition_missing=1'
|
||||
printf '%s' "$FAIL_OUTPUT" | grep -q 'external_provider_failure=1'
|
||||
printf '%s' "$FAIL_OUTPUT" | grep -q 'collector_runtime_failure=1'
|
||||
printf '%s' "$FAIL_OUTPUT" | grep -q 'sample_1 created_at=2026-05-15 20:00:00'
|
||||
|
||||
FIXTURE_PASS="$TMP_DIR/collector_stats_pass.tsv"
|
||||
cat > "$FIXTURE_PASS" <<'EOF'
|
||||
openrouter t 2026-05-15 20:00:00
|
||||
openrouter t 2026-05-15 19:59:00
|
||||
openrouter t 2026-05-15 19:58:00
|
||||
openrouter t 2026-05-15 19:57:00
|
||||
openrouter t 2026-05-15 19:56:00
|
||||
openrouter t 2026-05-15 19:55:00
|
||||
openrouter t 2026-05-15 19:54:00
|
||||
EOF
|
||||
|
||||
PASS_OUTPUT="$(bash scripts/collector_stats_window_audit.sh --input "$FIXTURE_PASS" --limit 7 --assert-success-rate 95 2>&1)"
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q 'success_rate=100.00'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q 'failure_count=0'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q 'sample_7 created_at=2026-05-15 19:54:00'
|
||||
|
||||
|
||||
FIXTURE_AGED_PRECONDITION="$TMP_DIR/collector_stats_aged_precondition.tsv"
|
||||
cat > "$FIXTURE_AGED_PRECONDITION" <<'EOF'
|
||||
openrouter f OPENROUTER_API_KEY 未设置 2026-05-10 08:00:00
|
||||
openrouter t 2026-05-15 20:00:00
|
||||
openrouter t 2026-05-15 19:59:00
|
||||
openrouter t 2026-05-15 19:58:00
|
||||
openrouter t 2026-05-15 19:57:00
|
||||
openrouter t 2026-05-15 19:56:00
|
||||
openrouter t 2026-05-15 19:55:00
|
||||
EOF
|
||||
|
||||
AGED_OUTPUT="$(LLM_NOW='2026-05-15 20:00' bash scripts/collector_stats_window_audit.sh --input "$FIXTURE_AGED_PRECONDITION" --limit 7 --assert-success-rate 95 2>&1)"
|
||||
printf '%s' "$AGED_OUTPUT" | grep -q 'aged_precondition_missing=1'
|
||||
printf '%s' "$AGED_OUTPUT" | grep -q 'precondition_missing=0'
|
||||
FIXTURE_EXTERNAL_ONLY="$TMP_DIR/collector_stats_external_only.tsv"
|
||||
cat > "$FIXTURE_EXTERNAL_ONLY" <<'EOF'
|
||||
perplexity f unexpected perplexity pricing content: no model rows parsed 2026-05-15 20:00:00
|
||||
vertex f fetch https://example.com: unexpected status 403 2026-05-15 19:59:00
|
||||
cloudflare t 2026-05-15 19:58:00
|
||||
EOF
|
||||
|
||||
set +e
|
||||
EXTERNAL_OUTPUT="$(bash scripts/collector_stats_window_audit.sh --input "$FIXTURE_EXTERNAL_ONLY" --limit 3 --assert-success-rate 95 2>&1)"
|
||||
EXTERNAL_RC=$?
|
||||
set -e
|
||||
|
||||
if [[ "$EXTERNAL_RC" -eq 0 ]]; then
|
||||
echo "expected external-only fixture to exit non-zero"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '%s' "$EXTERNAL_OUTPUT" | grep -q 'external_provider_failure=2'
|
||||
printf '%s' "$EXTERNAL_OUTPUT" | grep -q 'collector_runtime_failure=0'
|
||||
81
scripts/coreshub_pricing_lib.go
Normal file
81
scripts/coreshub_pricing_lib.go
Normal file
@@ -0,0 +1,81 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultCoresHubPricingURL = "https://docs.coreshub.cn/console/big_model_server/introduce/model_choose"
|
||||
|
||||
var coreshubPricingPattern = regexp.MustCompile(`(DeepSeek-[A-Za-z0-9.\-]+)\s+(限时免费|¥\s*[\d.]+\s*/\s*千\s*tokens)\s+(限时免费|¥\s*[\d.]+\s*/\s*千\s*tokens)`)
|
||||
var coreshubPricingHTMLRowPattern = regexp.MustCompile(`(?is)<tr>\s*<td[^>]*>\s*<p[^>]*>(DeepSeek-[^<]+)</p>\s*</td>\s*<td[^>]*>\s*<p[^>]*>(限时免费|¥\s*[\d.]+\s*/\s*千\s*tokens)</p>\s*</td>\s*<td[^>]*>\s*<p[^>]*>(限时免费|¥\s*[\d.]+\s*/\s*千\s*tokens)</p>\s*</td>\s*</tr>`)
|
||||
var coreshubPriceValuePattern = regexp.MustCompile(`([\d.]+)`)
|
||||
|
||||
func parseCoresHubPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
raw = strings.ReplaceAll(raw, "¥", "¥")
|
||||
matches := coreshubPricingHTMLRowPattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) == 0 {
|
||||
normalized := cleanHTMLText(raw)
|
||||
normalized = strings.ReplaceAll(normalized, "¥", "¥")
|
||||
matches = coreshubPricingPattern.FindAllStringSubmatch(normalized, -1)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("no coreshub pricing rows found")
|
||||
}
|
||||
|
||||
records := make([]officialPricingRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
modelName := strings.TrimSpace(match[1])
|
||||
providerName := "DeepSeek"
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
inputPrice, inputFree, err := parseCoresHubPrice(match[2])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse input price for %s: %w", modelName, err)
|
||||
}
|
||||
outputPrice, outputFree, err := parseCoresHubPrice(match[3])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse output price for %s: %w", modelName, err)
|
||||
}
|
||||
record := officialPricingRecord{
|
||||
ModelID: normalizeExternalID("coreshub", modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "CoresHub",
|
||||
OperatorNameCn: "CoresHub",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.qingcloud.com/products/coreshub",
|
||||
OperatorType: "cloud",
|
||||
Region: "CN",
|
||||
Currency: "CNY",
|
||||
InputPrice: inputPrice,
|
||||
OutputPrice: outputPrice,
|
||||
SourceURL: defaultCoresHubPricingURL,
|
||||
ModelSourceURL: defaultCoresHubPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_product_page",
|
||||
Modality: detectModality(modelName),
|
||||
IsFree: inputFree && outputFree,
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseCoresHubPrice(raw string) (float64, bool, error) {
|
||||
value := strings.TrimSpace(raw)
|
||||
if strings.Contains(value, "免费") {
|
||||
return 0, true, nil
|
||||
}
|
||||
match := coreshubPriceValuePattern.FindStringSubmatch(value)
|
||||
if len(match) != 2 {
|
||||
return 0, false, fmt.Errorf("price value not found in %q", raw)
|
||||
}
|
||||
price := mustParseSubscriptionPrice(match[1]) * 1000
|
||||
return price, false, nil
|
||||
}
|
||||
16
scripts/cron_precondition_integration_test.sh
Normal file
16
scripts/cron_precondition_integration_test.sh
Normal file
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
export LLM_DAILY_MEMORY_PATH="$TMP_DIR/2026-05-29.md"
|
||||
export REPORT_DATE='2026-05-29'
|
||||
|
||||
bash scripts/cron_status_report.sh cron precondition_missing 'run_daily.sh failed' 'precondition_missing; 严格真实模式下必须提供 API Key' 'provide missing env/config and rerun' >/tmp/cron_precondition_test.out 2>&1
|
||||
|
||||
grep -q 'status=precondition_missing' "$LLM_DAILY_MEMORY_PATH"
|
||||
grep -q 'precondition_missing; 严格真实模式下必须提供 API Key' "$LLM_DAILY_MEMORY_PATH"
|
||||
grep -q 'provide missing env/config and rerun' "$LLM_DAILY_MEMORY_PATH"
|
||||
68
scripts/cron_status_report.sh
Executable file
68
scripts/cron_status_report.sh
Executable file
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
ACTOR="${1:-cron}"
|
||||
STATUS="${2:-unknown}"
|
||||
TOPIC="${3:-cron status report}"
|
||||
EVIDENCE_LINE="${4:-}"
|
||||
NEXT_LINE="${5:-}"
|
||||
|
||||
if [[ "$ACTOR" != "cron" ]]; then
|
||||
echo "unsupported actor: $ACTOR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
report_date="${REPORT_DATE:-$(date +%F)}"
|
||||
daily_memory_path="${LLM_DAILY_MEMORY_PATH:-memory/${report_date}.md}"
|
||||
now_hm="$(date +%H:%M)"
|
||||
|
||||
header="# llm-intelligence Daily Memory - ${report_date}
|
||||
|
||||
> 项目单日归档文件,不是实时 WAL。
|
||||
|
||||
## Entries
|
||||
"
|
||||
if [[ -f "$daily_memory_path" ]]; then
|
||||
existing="$(python3 - <<'PY' "$daily_memory_path"
|
||||
from pathlib import Path
|
||||
import sys
|
||||
p=Path(sys.argv[1])
|
||||
print(p.read_text(encoding='utf-8'), end='')
|
||||
PY
|
||||
)"
|
||||
else
|
||||
mkdir -p "$(dirname "$daily_memory_path")"
|
||||
existing="$header"
|
||||
fi
|
||||
|
||||
entry="
|
||||
## ${now_hm} - cron - cron status report
|
||||
### Context
|
||||
status=${STATUS}
|
||||
topic=${TOPIC}
|
||||
|
||||
### Evidence
|
||||
${EVIDENCE_LINE:-none}
|
||||
|
||||
### Outcome
|
||||
status=${STATUS}
|
||||
${TOPIC}
|
||||
|
||||
### Next
|
||||
${NEXT_LINE:-none}
|
||||
"
|
||||
|
||||
python3 - <<'PY' "$daily_memory_path" "$existing" "$entry"
|
||||
from pathlib import Path
|
||||
import sys
|
||||
path=Path(sys.argv[1])
|
||||
existing=sys.argv[2]
|
||||
entry=sys.argv[3]
|
||||
content=existing.rstrip() + "\n" + entry
|
||||
path.write_text(content, encoding='utf-8')
|
||||
PY
|
||||
|
||||
echo "CRON_STATUS actor=cron status=${STATUS} file=${daily_memory_path}"
|
||||
32
scripts/cron_status_report_test.sh
Normal file
32
scripts/cron_status_report_test.sh
Normal file
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
DAILY_MEMORY="$TMP_DIR/2026-05-29.md"
|
||||
export LLM_DAILY_MEMORY_PATH="$DAILY_MEMORY"
|
||||
|
||||
bash scripts/cron_status_report.sh cron success 'run_daily.sh completed' 'verify_phase6 PASS' 'next=none'
|
||||
|
||||
grep -q '^# llm-intelligence Daily Memory - 2026-05-29$' "$DAILY_MEMORY"
|
||||
grep -q '^## Entries$' "$DAILY_MEMORY"
|
||||
grep -q '## .* - cron - cron status report' "$DAILY_MEMORY"
|
||||
grep -q '### Context' "$DAILY_MEMORY"
|
||||
grep -q '### Evidence' "$DAILY_MEMORY"
|
||||
grep -q '### Outcome' "$DAILY_MEMORY"
|
||||
grep -q '### Next' "$DAILY_MEMORY"
|
||||
grep -q 'status=success' "$DAILY_MEMORY"
|
||||
grep -q 'run_daily.sh completed' "$DAILY_MEMORY"
|
||||
grep -q 'verify_phase6 PASS' "$DAILY_MEMORY"
|
||||
|
||||
PRECONDITION_MEMORY="$TMP_DIR/2026-05-30.md"
|
||||
export LLM_DAILY_MEMORY_PATH="$PRECONDITION_MEMORY"
|
||||
|
||||
bash scripts/cron_status_report.sh cron precondition_missing 'run_daily.sh failed' 'missing OPENROUTER_API_KEY' 'next=provide key'
|
||||
|
||||
grep -q 'status=precondition_missing' "$PRECONDITION_MEMORY"
|
||||
grep -q 'missing OPENROUTER_API_KEY' "$PRECONDITION_MEMORY"
|
||||
409
scripts/ctyun_subscription_lib.go
Normal file
409
scripts/ctyun_subscription_lib.go
Normal file
@@ -0,0 +1,409 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultCTYunCodingPlanURL = "https://www.ctyun.cn/document/11061839/11092368"
|
||||
defaultCTYunTokenPlanURL = "https://www.ctyun.cn/act/AI/zhuanxiang"
|
||||
)
|
||||
|
||||
func parseCTYunSubscriptionCatalog(codingRaw string, tokenRaw string) ([]subscriptionImportRecord, error) {
|
||||
publishedAt, known := publishedAtFromText(firstNonEmptyText(codingRaw, tokenRaw))
|
||||
codingRecords, err := parseCTYunCodingPlan(codingRaw, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tokenRecords, err := parseCTYunTokenPlan(tokenRaw, publishedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records := append(codingRecords, tokenRecords...)
|
||||
for i := range records {
|
||||
records[i].PublishedAtKnown = known
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseCTYunCodingPlan(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
if !strings.Contains(raw, "GLM Lite") || !strings.Contains(raw, "GLM Max") {
|
||||
return nil, fmt.Errorf("ctyun coding plan tiers not found")
|
||||
}
|
||||
pricePattern := regexp.MustCompile(`包月价格\s+(\d+)元/月\s+(\d+)元/月\s+(\d+)元/月`)
|
||||
priceMatch := pricePattern.FindStringSubmatch(raw)
|
||||
if len(priceMatch) != 4 {
|
||||
return nil, fmt.Errorf("ctyun coding plan monthly prices not found")
|
||||
}
|
||||
limitPattern := regexp.MustCompile(`每月最多约([\d,]+)次prompts`)
|
||||
limitMatches := limitPattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(limitMatches) < 3 {
|
||||
return nil, fmt.Errorf("ctyun coding plan monthly limits not found")
|
||||
}
|
||||
modelScope := extractCTYunCodingModels(raw)
|
||||
|
||||
records := []subscriptionImportRecord{
|
||||
{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.ctyun.cn",
|
||||
OperatorName: "CTYun",
|
||||
OperatorNameCn: "天翼云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ctyun.cn",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "ctyun-coding-plan-lite-monthly",
|
||||
PlanName: "天翼云 Coding Plan Lite(月付)",
|
||||
Tier: "Lite",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(priceMatch[1]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: mustParseSubscriptionInt64(limitMatches[0][1]),
|
||||
QuotaUnit: "prompts/month",
|
||||
PlanScope: "Coding Plan",
|
||||
ModelScope: modelScope,
|
||||
SourceURL: defaultCTYunCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: "每 5 小时约 80 次 prompts;每周约 400 次 prompts。",
|
||||
},
|
||||
{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.ctyun.cn",
|
||||
OperatorName: "CTYun",
|
||||
OperatorNameCn: "天翼云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ctyun.cn",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "ctyun-coding-plan-pro-monthly",
|
||||
PlanName: "天翼云 Coding Plan Pro(月付)",
|
||||
Tier: "Pro",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(priceMatch[2]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: mustParseSubscriptionInt64(limitMatches[1][1]),
|
||||
QuotaUnit: "prompts/month",
|
||||
PlanScope: "Coding Plan",
|
||||
ModelScope: modelScope,
|
||||
SourceURL: defaultCTYunCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: "每 5 小时约 400 次 prompts;每周约 2,000 次 prompts。",
|
||||
},
|
||||
{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.ctyun.cn",
|
||||
OperatorName: "CTYun",
|
||||
OperatorNameCn: "天翼云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ctyun.cn",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "coding_plan",
|
||||
PlanCode: "ctyun-coding-plan-max-monthly",
|
||||
PlanName: "天翼云 Coding Plan Max(月付)",
|
||||
Tier: "Max",
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(priceMatch[3]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: mustParseSubscriptionInt64(limitMatches[2][1]),
|
||||
QuotaUnit: "prompts/month",
|
||||
PlanScope: "Coding Plan",
|
||||
ModelScope: modelScope,
|
||||
SourceURL: defaultCTYunCodingPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: "每 5 小时约 1,600 次 prompts;每周约 8,000 次 prompts。",
|
||||
},
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseCTYunTokenPlan(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
if records, ok := parseCTYunTokenPlanNormalizedLayout(raw, publishedAt); ok {
|
||||
return records, nil
|
||||
}
|
||||
if records, ok := parseCTYunTokenPlanCardLayout(raw, publishedAt); ok {
|
||||
return records, nil
|
||||
}
|
||||
return parseCTYunTokenPlanLegacyLayout(raw, publishedAt)
|
||||
}
|
||||
|
||||
func parseCTYunTokenPlanNormalizedLayout(raw string, publishedAt string) ([]subscriptionImportRecord, bool) {
|
||||
lines := strings.Split(raw, "\n")
|
||||
codeByTier := map[string]string{
|
||||
"基础版": "basic",
|
||||
"专业版": "pro",
|
||||
"旗舰版": "flagship",
|
||||
"轻享版": "starter",
|
||||
"畅享版": "plus",
|
||||
"尊享版": "vip",
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, 6)
|
||||
for i := 0; i < len(lines); i++ {
|
||||
line := strings.TrimSpace(lines[i])
|
||||
if !strings.HasPrefix(line, "Token Plan") {
|
||||
continue
|
||||
}
|
||||
rawTier := strings.TrimSpace(strings.TrimPrefix(line, "Token Plan"))
|
||||
tierCode, ok := codeByTier[rawTier]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
j := i + 1
|
||||
block := make([]string, 0, 12)
|
||||
for ; j < len(lines); j++ {
|
||||
next := strings.TrimSpace(lines[j])
|
||||
if strings.HasPrefix(next, "Token Plan") {
|
||||
break
|
||||
}
|
||||
if next != "" {
|
||||
block = append(block, next)
|
||||
}
|
||||
}
|
||||
model := ""
|
||||
quota := ""
|
||||
price := ""
|
||||
notesParts := make([]string, 0, 4)
|
||||
for k := 0; k < len(block); k++ {
|
||||
item := block[k]
|
||||
switch {
|
||||
case strings.HasPrefix(item, "支持模型:"):
|
||||
model = strings.TrimSpace(strings.TrimPrefix(item, "支持模型:"))
|
||||
case strings.Contains(item, "Tokens"):
|
||||
quota = strings.TrimSpace(strings.TrimSuffix(item, "Tokens"))
|
||||
case regexp.MustCompile(`^[0-9]+$`).MatchString(item) && k+2 < len(block) && regexp.MustCompile(`^\.[0-9]+$`).MatchString(block[k+1]) && block[k+2] == "元/个/月":
|
||||
price = item + block[k+1]
|
||||
case item == "产品优势", item == "立即订购", strings.HasPrefix(item, "支持工具:"), strings.HasPrefix(item, "已抢购"), strings.HasSuffix(item, "用户"), item == "展开更多", item == "免费领取Tokens":
|
||||
continue
|
||||
default:
|
||||
notesParts = append(notesParts, item)
|
||||
}
|
||||
}
|
||||
if model == "" || quota == "" || price == "" {
|
||||
continue
|
||||
}
|
||||
notes := "天翼云大模型 AI 专项活动页套餐。"
|
||||
if len(notesParts) > 0 {
|
||||
notes = strings.Join(notesParts, ";")
|
||||
}
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.ctyun.cn",
|
||||
OperatorName: "CTYun",
|
||||
OperatorNameCn: "天翼云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ctyun.cn",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: "ctyun-token-plan-" + tierCode,
|
||||
PlanName: "天翼云 Token Plan " + rawTier,
|
||||
Tier: rawTier,
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(price),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: parseChineseTokenQuota(quota),
|
||||
QuotaUnit: "tokens/month",
|
||||
PlanScope: "Token Plan",
|
||||
ModelScope: []string{model},
|
||||
SourceURL: defaultCTYunTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: notes,
|
||||
})
|
||||
i = j - 1
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
return records, true
|
||||
}
|
||||
|
||||
func parseCTYunTokenPlanCardLayout(raw string, publishedAt string) ([]subscriptionImportRecord, bool) {
|
||||
cardPattern := regexp.MustCompile(`(?s)<span title="(Token Plan [^"]+)" class="card-header-title-text".*?</span>(.*?)<div class="card-btns-wrap"`)
|
||||
cards := cardPattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(cards) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
codeByTier := map[string]string{
|
||||
"基础版": "basic",
|
||||
"专业版": "pro",
|
||||
"旗舰版": "flagship",
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, len(cards))
|
||||
for _, card := range cards {
|
||||
title := strings.TrimSpace(card[1])
|
||||
body := card[2]
|
||||
rawTier := strings.TrimSpace(strings.TrimPrefix(title, "Token Plan "))
|
||||
tierCode, ok := codeByTier[rawTier]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
modelMatch := regexp.MustCompile(`支持模型:([^<]+)</span>`).FindStringSubmatch(body)
|
||||
if len(modelMatch) != 2 {
|
||||
return nil, false
|
||||
}
|
||||
quotaMatch := regexp.MustCompile(`([0-9]+(?:\.[0-9]+)?亿|[0-9]+万)Tokens`).FindStringSubmatch(body)
|
||||
if len(quotaMatch) != 2 {
|
||||
return nil, false
|
||||
}
|
||||
priceMatch := regexp.MustCompile(`<span class="price-new-big"[^>]*>\s*([0-9]+)\s*</span>\s*<span class="price-new-big"[^>]*>\s*\.([0-9]+)\s*</span>\s*<span class="price-new-unit"[^>]*>元/个/月</span>`).FindStringSubmatch(body)
|
||||
if len(priceMatch) != 3 {
|
||||
return nil, false
|
||||
}
|
||||
notes := "天翼云大模型 AI 专项活动页套餐。"
|
||||
if featureLines := regexp.MustCompile(`card-content-gou-content"[^>]*>([^<]+)</span>`).FindAllStringSubmatch(body, -1); len(featureLines) > 0 {
|
||||
parts := make([]string, 0, len(featureLines))
|
||||
for _, line := range featureLines {
|
||||
text := strings.TrimSpace(line[1])
|
||||
if text == "" || strings.HasPrefix(text, "支持模型:") || strings.Contains(text, "Tokens") {
|
||||
continue
|
||||
}
|
||||
parts = append(parts, text)
|
||||
}
|
||||
if len(parts) > 0 {
|
||||
notes = strings.Join(parts, ";")
|
||||
}
|
||||
}
|
||||
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.ctyun.cn",
|
||||
OperatorName: "CTYun",
|
||||
OperatorNameCn: "天翼云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ctyun.cn",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: "ctyun-token-plan-" + tierCode,
|
||||
PlanName: "天翼云 " + title,
|
||||
Tier: rawTier,
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(priceMatch[1] + "." + priceMatch[2]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: parseChineseTokenQuota(quotaMatch[1]),
|
||||
QuotaUnit: "tokens/month",
|
||||
PlanScope: "Token Plan",
|
||||
ModelScope: []string{strings.TrimSpace(modelMatch[1])},
|
||||
SourceURL: defaultCTYunTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: notes,
|
||||
})
|
||||
}
|
||||
return records, true
|
||||
}
|
||||
|
||||
func parseCTYunTokenPlanLegacyLayout(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
pattern := regexp.MustCompile(`Token Plan ([^\n]+?)(\d+(?:\.\d+)?亿|\d+万)Tokens包[\s\S]*?支持模型:([^\n]+)[\s\S]*?(\d+\s*\.\s*\d+)\s*元/个`)
|
||||
matches := pattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("unexpected ctyun token plan count: 0")
|
||||
}
|
||||
|
||||
codeByTier := map[string]string{
|
||||
"Lite": "lite",
|
||||
"Pro": "pro",
|
||||
"Max": "max",
|
||||
"轻享包": "starter",
|
||||
"畅享包": "plus",
|
||||
"尊享包": "vip",
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
rawTier := strings.TrimSpace(match[1])
|
||||
tierCode, ok := codeByTier[rawTier]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected ctyun token plan tier: %s", rawTier)
|
||||
}
|
||||
quotaValue := parseChineseTokenQuota(match[2])
|
||||
price := mustParseSubscriptionPrice(strings.ReplaceAll(match[4], " ", ""))
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.ctyun.cn",
|
||||
OperatorName: "CTYun",
|
||||
OperatorNameCn: "天翼云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ctyun.cn",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: "ctyun-token-plan-" + tierCode,
|
||||
PlanName: "天翼云 Token Plan " + rawTier,
|
||||
Tier: rawTier,
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: price,
|
||||
PriceUnit: "CNY/pack",
|
||||
QuotaValue: quotaValue,
|
||||
QuotaUnit: "tokens/pack",
|
||||
PlanScope: "Token Plan",
|
||||
ModelScope: []string{strings.TrimSpace(match[3])},
|
||||
SourceURL: defaultCTYunTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: "天翼云大模型 AI 专项活动页套餐。",
|
||||
})
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func parseChineseTokenQuota(raw string) int64 {
|
||||
cleaned := strings.TrimSpace(strings.TrimSuffix(raw, "Tokens包"))
|
||||
cleaned = strings.ReplaceAll(cleaned, " ", "")
|
||||
switch {
|
||||
case strings.Contains(cleaned, "亿"):
|
||||
return parseDecimalMultiplier(strings.TrimSuffix(cleaned, "亿"), 100000000)
|
||||
case strings.Contains(cleaned, "万"):
|
||||
return parseDecimalMultiplier(strings.TrimSuffix(cleaned, "万"), 10000)
|
||||
default:
|
||||
return mustParseSubscriptionInt64(cleaned)
|
||||
}
|
||||
}
|
||||
|
||||
func extractCTYunCodingModels(raw string) []string {
|
||||
lines := strings.Split(raw, "\n")
|
||||
models := make([]string, 0, 8)
|
||||
capturing := false
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
switch {
|
||||
case line == "支持模型":
|
||||
capturing = true
|
||||
continue
|
||||
case line == "用量限制":
|
||||
return models
|
||||
case !capturing || line == "":
|
||||
continue
|
||||
default:
|
||||
models = append(models, line)
|
||||
}
|
||||
}
|
||||
return models
|
||||
}
|
||||
51
scripts/deepseek_news_signature_guard.go
Normal file
51
scripts/deepseek_news_signature_guard.go
Normal file
@@ -0,0 +1,51 @@
|
||||
//go:build llm_script && !scripts_pkg
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var snapshotDir string
|
||||
var baselinePath string
|
||||
var timeoutSeconds int
|
||||
var allowBootstrap bool
|
||||
|
||||
flag.StringVar(&url, "url", defaultDeepSeekNewsFetchURL, "DeepSeek 官方新闻页")
|
||||
flag.StringVar(&fixture, "fixture", "", "DeepSeek 新闻页样例文件")
|
||||
flag.StringVar(&snapshotDir, "snapshot-dir", "", "DeepSeek news snapshot 输出目录")
|
||||
flag.StringVar(&baselinePath, "baseline-path", "", "DeepSeek news 结构基线签名路径")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.BoolVar(&allowBootstrap, "allow-bootstrap", true, "当 baseline 缺失时自动初始化")
|
||||
flag.Parse()
|
||||
|
||||
now := time.Now()
|
||||
cfg := deepseekNewsSignatureGuardConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
SnapshotDir: snapshotDir,
|
||||
BaselinePath: baselinePath,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
AllowBootstrap: allowBootstrap,
|
||||
}
|
||||
result, err := runDeepSeekNewsSignatureGuard(cfg, now)
|
||||
if auditErr := persistDeepSeekNewsSignatureAuditIfConfigured(cfg, result, now, err); auditErr != nil {
|
||||
fmt.Fprintf(os.Stderr, "deepseek_news_signature_guard audit: %v\n", auditErr)
|
||||
if err == nil {
|
||||
err = auditErr
|
||||
}
|
||||
}
|
||||
fmt.Println(formatDeepSeekNewsSignatureGuardSummary(result))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "deepseek_news_signature_guard: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
127
scripts/deepseek_news_signature_guard_lib.go
Normal file
127
scripts/deepseek_news_signature_guard_lib.go
Normal file
@@ -0,0 +1,127 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type deepseekNewsSignatureGuardConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
SnapshotDir string
|
||||
BaselinePath string
|
||||
Timeout time.Duration
|
||||
AllowBootstrap bool
|
||||
}
|
||||
|
||||
type deepseekNewsSignatureGuardResult struct {
|
||||
SnapshotPath string
|
||||
SignaturePath string
|
||||
BaselinePath string
|
||||
DriftDetected bool
|
||||
BaselineInitialized bool
|
||||
PreviousBaselineHash string
|
||||
CurrentSignature deepseekNewsStructureSignature
|
||||
}
|
||||
|
||||
const defaultDeepSeekNewsFetchURL = "https://api-docs.deepseek.com/news/news250120"
|
||||
|
||||
func runDeepSeekNewsSignatureGuard(cfg deepseekNewsSignatureGuardConfig, now time.Time) (deepseekNewsSignatureGuardResult, error) {
|
||||
snapshotDir := cfg.SnapshotDir
|
||||
if snapshotDir == "" {
|
||||
snapshotDir = filepath.Join("logs", "deepseek-news-snapshots")
|
||||
}
|
||||
if err := os.MkdirAll(snapshotDir, 0o755); err != nil {
|
||||
return deepseekNewsSignatureGuardResult{}, fmt.Errorf("mkdir snapshot dir: %w", err)
|
||||
}
|
||||
snapshotPath, signaturePath := resolveDeepSeekNewsSnapshotPaths("", "", snapshotDir, now)
|
||||
baselinePath := cfg.BaselinePath
|
||||
if baselinePath == "" {
|
||||
baselinePath = filepath.Join(snapshotDir, "baseline.signature.json")
|
||||
}
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return deepseekNewsSignatureGuardResult{}, err
|
||||
}
|
||||
current, err := writeDeepSeekNewsSnapshotArtifacts(raw, cfg.URL, snapshotPath, signaturePath, now)
|
||||
if err != nil {
|
||||
return deepseekNewsSignatureGuardResult{}, err
|
||||
}
|
||||
result := deepseekNewsSignatureGuardResult{
|
||||
SnapshotPath: snapshotPath,
|
||||
SignaturePath: signaturePath,
|
||||
BaselinePath: baselinePath,
|
||||
CurrentSignature: current,
|
||||
}
|
||||
previous, err := readDeepSeekNewsStructureSignature(baselinePath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
if !cfg.AllowBootstrap {
|
||||
return result, fmt.Errorf("deepseek news baseline missing: %s", baselinePath)
|
||||
}
|
||||
if err := copyFileCommon(signaturePath, baselinePath); err != nil {
|
||||
return result, fmt.Errorf("initialize baseline: %w", err)
|
||||
}
|
||||
result.BaselineInitialized = true
|
||||
return result, nil
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
result.PreviousBaselineHash = previous.StructureSHA256
|
||||
if previous.StructureSHA256 != current.StructureSHA256 {
|
||||
result.DriftDetected = true
|
||||
return result, fmt.Errorf(
|
||||
"deepseek news structure drift detected: baseline=%s current=%s baseline_path=%s signature_path=%s snapshot_path=%s",
|
||||
previous.StructureSHA256, current.StructureSHA256, baselinePath, signaturePath, snapshotPath,
|
||||
)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func formatDeepSeekNewsSignatureGuardSummary(result deepseekNewsSignatureGuardResult) string {
|
||||
return fmt.Sprintf(
|
||||
"source=deepseek-news-signature-guard drift=%t baseline_initialized=%t structure_sha256=%s previous_baseline_sha256=%s snapshot_out=%s signature_out=%s baseline_path=%s",
|
||||
result.DriftDetected,
|
||||
result.BaselineInitialized,
|
||||
result.CurrentSignature.StructureSHA256,
|
||||
emptyIfBlank(result.PreviousBaselineHash),
|
||||
result.SnapshotPath,
|
||||
result.SignaturePath,
|
||||
result.BaselinePath,
|
||||
)
|
||||
}
|
||||
|
||||
func buildDeepSeekNewsSignatureAuditRecord(cfg deepseekNewsSignatureGuardConfig, result deepseekNewsSignatureGuardResult, checkedAt time.Time, runErr error) officialImportSignatureAuditRecord {
|
||||
record := officialImportSignatureAuditRecord{
|
||||
SourceKey: "deepseek_news_signature",
|
||||
CheckedAt: checkedAt,
|
||||
Status: officialImportSignatureAuditStatus(result.DriftDetected, result.BaselineInitialized, runErr),
|
||||
DriftDetected: result.DriftDetected,
|
||||
BaselineInitialized: result.BaselineInitialized,
|
||||
SourceURL: strings.TrimSpace(cfg.URL),
|
||||
FixturePath: strings.TrimSpace(cfg.Fixture),
|
||||
SnapshotPath: strings.TrimSpace(result.SnapshotPath),
|
||||
SignaturePath: strings.TrimSpace(result.SignaturePath),
|
||||
BaselinePath: strings.TrimSpace(result.BaselinePath),
|
||||
StructureSHA256: strings.TrimSpace(result.CurrentSignature.StructureSHA256),
|
||||
PreviousStructureSHA256: strings.TrimSpace(result.PreviousBaselineHash),
|
||||
ByteSize: result.CurrentSignature.ByteSize,
|
||||
ErrorMessage: errorMessageText(runErr),
|
||||
}
|
||||
if hasDeepSeekNewsStructureSignature(result.CurrentSignature) {
|
||||
signatureCopy := result.CurrentSignature
|
||||
record.SignaturePayload = &signatureCopy
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
func persistDeepSeekNewsSignatureAuditIfConfigured(cfg deepseekNewsSignatureGuardConfig, result deepseekNewsSignatureGuardResult, checkedAt time.Time, runErr error) error {
|
||||
return persistOfficialImportSignatureAuditIfConfigured(buildDeepSeekNewsSignatureAuditRecord(cfg, result, checkedAt, runErr))
|
||||
}
|
||||
88
scripts/deepseek_news_signature_guard_test.go
Normal file
88
scripts/deepseek_news_signature_guard_test.go
Normal file
@@ -0,0 +1,88 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRunDeepSeekNewsSignatureGuardInitializesBaseline(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
baselinePath := filepath.Join(tempDir, "baseline.signature.json")
|
||||
result, err := runDeepSeekNewsSignatureGuard(deepseekNewsSignatureGuardConfig{
|
||||
URL: defaultDeepSeekNewsFetchURL,
|
||||
Fixture: filepath.Join("testdata", "intraday_verification_official_release.html"),
|
||||
SnapshotDir: tempDir,
|
||||
BaselinePath: baselinePath,
|
||||
Timeout: time.Second,
|
||||
AllowBootstrap: true,
|
||||
}, time.Date(2026, 5, 27, 21, 0, 0, 0, time.FixedZone("CST", 8*3600)))
|
||||
if err != nil {
|
||||
t.Fatalf("runDeepSeekNewsSignatureGuard 返回错误: %v", err)
|
||||
}
|
||||
if !result.BaselineInitialized {
|
||||
t.Fatal("期望初始化 baseline")
|
||||
}
|
||||
if _, err := os.Stat(baselinePath); err != nil {
|
||||
t.Fatalf("baseline 未写入: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDeepSeekNewsSignatureGuardDetectsDrift(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
baselinePath := filepath.Join(tempDir, "baseline.signature.json")
|
||||
_, err := runDeepSeekNewsSignatureGuard(deepseekNewsSignatureGuardConfig{
|
||||
URL: defaultDeepSeekNewsFetchURL,
|
||||
Fixture: filepath.Join("testdata", "intraday_verification_official_release.html"),
|
||||
SnapshotDir: tempDir,
|
||||
BaselinePath: baselinePath,
|
||||
Timeout: time.Second,
|
||||
AllowBootstrap: true,
|
||||
}, time.Date(2026, 5, 27, 21, 1, 0, 0, time.FixedZone("CST", 8*3600)))
|
||||
if err != nil {
|
||||
t.Fatalf("初始化 baseline 失败: %v", err)
|
||||
}
|
||||
driftFixture := filepath.Join(tempDir, "drift.html")
|
||||
if err := os.WriteFile(driftFixture, []byte("<html><head><title>DeepSeek-V4 Release</title><meta name=\"description\" content=\"DeepSeek V4 pricing release\"></head><body><h1>DeepSeek V4 Release</h1></body></html>"), 0o644); err != nil {
|
||||
t.Fatalf("写入 drift fixture 失败: %v", err)
|
||||
}
|
||||
result, err := runDeepSeekNewsSignatureGuard(deepseekNewsSignatureGuardConfig{
|
||||
URL: defaultDeepSeekNewsFetchURL,
|
||||
Fixture: driftFixture,
|
||||
SnapshotDir: tempDir,
|
||||
BaselinePath: baselinePath,
|
||||
Timeout: time.Second,
|
||||
AllowBootstrap: false,
|
||||
}, time.Date(2026, 5, 27, 21, 2, 0, 0, time.FixedZone("CST", 8*3600)))
|
||||
if err == nil {
|
||||
t.Fatal("期望结构漂移时报错")
|
||||
}
|
||||
if !result.DriftDetected {
|
||||
t.Fatal("期望 driftDetected=true")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "deepseek news structure drift detected") {
|
||||
t.Fatalf("期望返回 drift 错误,实际: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatDeepSeekNewsSignatureGuardSummary(t *testing.T) {
|
||||
result := deepseekNewsSignatureGuardResult{
|
||||
SnapshotPath: "/tmp/deepseek-news.html",
|
||||
SignaturePath: "/tmp/deepseek-news.signature.json",
|
||||
BaselinePath: "/tmp/baseline.signature.json",
|
||||
BaselineInitialized: true,
|
||||
CurrentSignature: deepseekNewsStructureSignature{
|
||||
StructureSHA256: "abc123",
|
||||
},
|
||||
}
|
||||
summary := formatDeepSeekNewsSignatureGuardSummary(result)
|
||||
for _, want := range []string{"source=deepseek-news-signature-guard", "baseline_initialized=true", "structure_sha256=abc123"} {
|
||||
if !strings.Contains(summary, want) {
|
||||
t.Fatalf("summary 缺少 %q,实际: %q", want, summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
196
scripts/deepseek_news_snapshot_lib.go
Normal file
196
scripts/deepseek_news_snapshot_lib.go
Normal file
@@ -0,0 +1,196 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type deepseekNewsStructureSignature struct {
|
||||
ByteSize int `json:"byte_size"`
|
||||
SHA256 string `json:"sha256"`
|
||||
StructureSHA256 string `json:"structure_sha256"`
|
||||
Title string `json:"title"`
|
||||
MetaDescription string `json:"meta_description"`
|
||||
Headings []string `json:"headings"`
|
||||
Contains map[string]bool `json:"contains"`
|
||||
GeneratedAt string `json:"generated_at,omitempty"`
|
||||
SourceURL string `json:"source_url,omitempty"`
|
||||
SnapshotPath string `json:"snapshot_path,omitempty"`
|
||||
}
|
||||
|
||||
var deepseekNewsContainsNeedles = map[string]string{
|
||||
"deepseek": "deepseek",
|
||||
"release": "release",
|
||||
"news": "news",
|
||||
"api_docs": "api docs",
|
||||
}
|
||||
|
||||
var htmlTagRe = regexp.MustCompile(`(?s)<[^>]+>`)
|
||||
var titleRe = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
|
||||
var metaDescRe = regexp.MustCompile(`(?is)<meta[^>]+name=["']description["'][^>]+content=["']([^"']+)["']`)
|
||||
var h1Re = regexp.MustCompile(`(?is)<h1[^>]*>(.*?)</h1>`)
|
||||
|
||||
func buildDeepSeekNewsStructureSignature(raw string) deepseekNewsStructureSignature {
|
||||
title := firstHTMLMatch(titleRe, raw)
|
||||
meta := firstHTMLMatch(metaDescRe, raw)
|
||||
h1Matches := h1Re.FindAllStringSubmatch(raw, -1)
|
||||
headings := make([]string, 0, len(h1Matches))
|
||||
seen := make(map[string]struct{})
|
||||
for _, match := range h1Matches {
|
||||
if len(match) < 2 {
|
||||
continue
|
||||
}
|
||||
clean := cleanHTMLText(match[1])
|
||||
if clean == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[clean]; exists {
|
||||
continue
|
||||
}
|
||||
seen[clean] = struct{}{}
|
||||
headings = append(headings, clean)
|
||||
}
|
||||
contains := make(map[string]bool, len(deepseekNewsContainsNeedles))
|
||||
lower := strings.ToLower(raw)
|
||||
for key, needle := range deepseekNewsContainsNeedles {
|
||||
contains[key] = strings.Contains(lower, strings.ToLower(needle))
|
||||
}
|
||||
signature := deepseekNewsStructureSignature{
|
||||
ByteSize: len([]byte(raw)),
|
||||
SHA256: deepseekNewsSHA256Hex(raw),
|
||||
Title: title,
|
||||
MetaDescription: meta,
|
||||
Headings: headings,
|
||||
Contains: contains,
|
||||
}
|
||||
signature.StructureSHA256 = deepseekNewsSHA256Hex(deepseekNewsStructureDigestPayload(signature))
|
||||
return signature
|
||||
}
|
||||
|
||||
func writeDeepSeekNewsSnapshotArtifacts(raw string, sourceURL string, snapshotPath string, signaturePath string, now time.Time) (deepseekNewsStructureSignature, error) {
|
||||
if strings.TrimSpace(snapshotPath) == "" {
|
||||
return deepseekNewsStructureSignature{}, fmt.Errorf("snapshot path is required")
|
||||
}
|
||||
if strings.TrimSpace(signaturePath) == "" {
|
||||
return deepseekNewsStructureSignature{}, fmt.Errorf("signature path is required")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(snapshotPath), 0o755); err != nil {
|
||||
return deepseekNewsStructureSignature{}, fmt.Errorf("mkdir snapshot dir: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(signaturePath), 0o755); err != nil {
|
||||
return deepseekNewsStructureSignature{}, fmt.Errorf("mkdir signature dir: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(snapshotPath, []byte(raw), 0o644); err != nil {
|
||||
return deepseekNewsStructureSignature{}, fmt.Errorf("write snapshot: %w", err)
|
||||
}
|
||||
signature := buildDeepSeekNewsStructureSignature(raw)
|
||||
signature.GeneratedAt = now.Format(time.RFC3339)
|
||||
signature.SourceURL = sourceURL
|
||||
signature.SnapshotPath = snapshotPath
|
||||
payload, err := json.MarshalIndent(signature, "", " ")
|
||||
if err != nil {
|
||||
return deepseekNewsStructureSignature{}, fmt.Errorf("marshal signature: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(signaturePath, payload, 0o644); err != nil {
|
||||
return deepseekNewsStructureSignature{}, fmt.Errorf("write signature: %w", err)
|
||||
}
|
||||
return signature, nil
|
||||
}
|
||||
|
||||
func resolveDeepSeekNewsSnapshotPaths(snapshotPath string, signaturePath string, snapshotDir string, now time.Time) (string, string) {
|
||||
if strings.TrimSpace(snapshotDir) == "" {
|
||||
snapshotDir = filepath.Join("logs", "deepseek-news-snapshots")
|
||||
}
|
||||
if strings.TrimSpace(snapshotPath) == "" {
|
||||
base := filepath.Join(snapshotDir, fmt.Sprintf("deepseek-news-%s", now.Format("20060102-150405")))
|
||||
snapshotPath = base + ".html"
|
||||
if strings.TrimSpace(signaturePath) == "" {
|
||||
signaturePath = base + ".signature.json"
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(signaturePath) == "" {
|
||||
signaturePath = strings.TrimSuffix(snapshotPath, filepath.Ext(snapshotPath)) + ".signature.json"
|
||||
}
|
||||
return snapshotPath, signaturePath
|
||||
}
|
||||
|
||||
func readDeepSeekNewsStructureSignature(path string) (deepseekNewsStructureSignature, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return deepseekNewsStructureSignature{}, err
|
||||
}
|
||||
var signature deepseekNewsStructureSignature
|
||||
if err := json.Unmarshal(data, &signature); err != nil {
|
||||
return deepseekNewsStructureSignature{}, fmt.Errorf("unmarshal signature %s: %w", path, err)
|
||||
}
|
||||
return signature, nil
|
||||
}
|
||||
|
||||
func hasDeepSeekNewsStructureSignature(signature deepseekNewsStructureSignature) bool {
|
||||
return signature.ByteSize > 0 ||
|
||||
strings.TrimSpace(signature.StructureSHA256) != "" ||
|
||||
strings.TrimSpace(signature.SHA256) != "" ||
|
||||
strings.TrimSpace(signature.Title) != "" ||
|
||||
len(signature.Headings) > 0 ||
|
||||
len(signature.Contains) > 0
|
||||
}
|
||||
|
||||
func deepseekNewsStructureDigestPayload(signature deepseekNewsStructureSignature) string {
|
||||
type containsEntry struct {
|
||||
Name string `json:"name"`
|
||||
Value bool `json:"value"`
|
||||
}
|
||||
keys := make([]string, 0, len(signature.Contains))
|
||||
for key := range signature.Contains {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
entries := make([]containsEntry, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
entries = append(entries, containsEntry{Name: key, Value: signature.Contains[key]})
|
||||
}
|
||||
payload := struct {
|
||||
Title string `json:"title"`
|
||||
MetaDescription string `json:"meta_description"`
|
||||
Headings []string `json:"headings"`
|
||||
Contains []containsEntry `json:"contains"`
|
||||
}{
|
||||
Title: signature.Title,
|
||||
MetaDescription: signature.MetaDescription,
|
||||
Headings: signature.Headings,
|
||||
Contains: entries,
|
||||
}
|
||||
bytes, _ := json.Marshal(payload)
|
||||
return string(bytes)
|
||||
}
|
||||
|
||||
func deepseekNewsSHA256Hex(raw string) string {
|
||||
sum := sha256.Sum256([]byte(raw))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func firstHTMLMatch(re *regexp.Regexp, raw string) string {
|
||||
match := re.FindStringSubmatch(raw)
|
||||
if len(match) < 2 {
|
||||
return ""
|
||||
}
|
||||
return cleanHTMLText(match[1])
|
||||
}
|
||||
|
||||
func cleanHTMLText(raw string) string {
|
||||
text := htmlTagRe.ReplaceAllString(raw, " ")
|
||||
text = strings.ReplaceAll(text, "&", "&")
|
||||
text = strings.ReplaceAll(text, " ", " ")
|
||||
text = strings.Join(strings.Fields(text), " ")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
57
scripts/deepseek_pricing_signature_guard.go
Normal file
57
scripts/deepseek_pricing_signature_guard.go
Normal file
@@ -0,0 +1,57 @@
|
||||
//go:build llm_script && !scripts_pkg
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var snapshotDir string
|
||||
var baselinePath string
|
||||
var timeoutSeconds int
|
||||
var allowBootstrap bool
|
||||
var sourceKey string
|
||||
var snapshotBase string
|
||||
|
||||
flag.StringVar(&sourceKey, "source-key", "deepseek_pricing_signature", "审计 source_key")
|
||||
flag.StringVar(&snapshotBase, "snapshot-base", "deepseek-pricing", "snapshot 文件名前缀")
|
||||
flag.StringVar(&url, "url", defaultDeepSeekPricingFetchURL, "DeepSeek 官方价格页")
|
||||
flag.StringVar(&fixture, "fixture", "", "DeepSeek 价格页样例文件")
|
||||
flag.StringVar(&snapshotDir, "snapshot-dir", "", "DeepSeek pricing snapshot 输出目录")
|
||||
flag.StringVar(&baselinePath, "baseline-path", "", "DeepSeek pricing 结构基线签名路径")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.BoolVar(&allowBootstrap, "allow-bootstrap", true, "当 baseline 缺失时自动初始化")
|
||||
flag.Parse()
|
||||
|
||||
now := time.Now()
|
||||
cfg := deepseekPricingSignatureGuardConfig{
|
||||
SourceKey: sourceKey,
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
SnapshotDir: snapshotDir,
|
||||
BaselinePath: baselinePath,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
AllowBootstrap: allowBootstrap,
|
||||
SnapshotBase: snapshotBase,
|
||||
}
|
||||
result, err := runDeepSeekPricingSignatureGuard(cfg, now)
|
||||
if auditErr := persistDeepSeekPricingSignatureAuditIfConfigured(cfg, result, now, err); auditErr != nil {
|
||||
fmt.Fprintf(os.Stderr, "deepseek_pricing_signature_guard audit: %v\n", auditErr)
|
||||
if err == nil {
|
||||
err = auditErr
|
||||
}
|
||||
}
|
||||
fmt.Println(formatDeepSeekPricingSignatureGuardSummary(sourceKey, result))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "deepseek_pricing_signature_guard: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
132
scripts/deepseek_pricing_signature_guard_lib.go
Normal file
132
scripts/deepseek_pricing_signature_guard_lib.go
Normal file
@@ -0,0 +1,132 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type deepseekPricingSignatureGuardConfig struct {
|
||||
SourceKey string
|
||||
URL string
|
||||
Fixture string
|
||||
SnapshotDir string
|
||||
BaselinePath string
|
||||
Timeout time.Duration
|
||||
AllowBootstrap bool
|
||||
SnapshotBase string
|
||||
SourceKindLabel string
|
||||
}
|
||||
|
||||
type deepseekPricingSignatureGuardResult struct {
|
||||
SnapshotPath string
|
||||
SignaturePath string
|
||||
BaselinePath string
|
||||
DriftDetected bool
|
||||
BaselineInitialized bool
|
||||
PreviousBaselineHash string
|
||||
CurrentSignature deepseekPricingStructureSignature
|
||||
}
|
||||
|
||||
const defaultDeepSeekPricingFetchURL = "https://platform.deepseek.com/pricing"
|
||||
const defaultDeepSeekAPIPricingFetchURL = "https://platform.deepseek.com/docs/api-pricing"
|
||||
|
||||
func runDeepSeekPricingSignatureGuard(cfg deepseekPricingSignatureGuardConfig, now time.Time) (deepseekPricingSignatureGuardResult, error) {
|
||||
snapshotDir := cfg.SnapshotDir
|
||||
if snapshotDir == "" {
|
||||
snapshotDir = filepath.Join("logs", cfg.SnapshotBase+"-snapshots")
|
||||
}
|
||||
if err := os.MkdirAll(snapshotDir, 0o755); err != nil {
|
||||
return deepseekPricingSignatureGuardResult{}, fmt.Errorf("mkdir snapshot dir: %w", err)
|
||||
}
|
||||
snapshotPath, signaturePath := resolveDeepSeekPricingSnapshotPaths("", "", snapshotDir, cfg.SnapshotBase, now)
|
||||
baselinePath := cfg.BaselinePath
|
||||
if baselinePath == "" {
|
||||
baselinePath = filepath.Join(snapshotDir, "baseline.signature.json")
|
||||
}
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return deepseekPricingSignatureGuardResult{}, err
|
||||
}
|
||||
current, err := writeDeepSeekPricingSnapshotArtifacts(raw, cfg.URL, snapshotPath, signaturePath, now)
|
||||
if err != nil {
|
||||
return deepseekPricingSignatureGuardResult{}, err
|
||||
}
|
||||
result := deepseekPricingSignatureGuardResult{
|
||||
SnapshotPath: snapshotPath,
|
||||
SignaturePath: signaturePath,
|
||||
BaselinePath: baselinePath,
|
||||
CurrentSignature: current,
|
||||
}
|
||||
previous, err := readDeepSeekPricingStructureSignature(baselinePath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
if !cfg.AllowBootstrap {
|
||||
return result, fmt.Errorf("%s baseline missing: %s", cfg.SourceKey, baselinePath)
|
||||
}
|
||||
if err := copyFileCommon(signaturePath, baselinePath); err != nil {
|
||||
return result, fmt.Errorf("initialize baseline: %w", err)
|
||||
}
|
||||
result.BaselineInitialized = true
|
||||
return result, nil
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
result.PreviousBaselineHash = previous.StructureSHA256
|
||||
if previous.StructureSHA256 != current.StructureSHA256 {
|
||||
result.DriftDetected = true
|
||||
return result, fmt.Errorf(
|
||||
"%s structure drift detected: baseline=%s current=%s baseline_path=%s signature_path=%s snapshot_path=%s",
|
||||
cfg.SourceKey, previous.StructureSHA256, current.StructureSHA256, baselinePath, signaturePath, snapshotPath,
|
||||
)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func formatDeepSeekPricingSignatureGuardSummary(sourceKey string, result deepseekPricingSignatureGuardResult) string {
|
||||
return fmt.Sprintf(
|
||||
"source=%s drift=%t baseline_initialized=%t structure_sha256=%s previous_baseline_sha256=%s snapshot_out=%s signature_out=%s baseline_path=%s",
|
||||
sourceKey,
|
||||
result.DriftDetected,
|
||||
result.BaselineInitialized,
|
||||
result.CurrentSignature.StructureSHA256,
|
||||
emptyIfBlank(result.PreviousBaselineHash),
|
||||
result.SnapshotPath,
|
||||
result.SignaturePath,
|
||||
result.BaselinePath,
|
||||
)
|
||||
}
|
||||
|
||||
func buildDeepSeekPricingSignatureAuditRecord(cfg deepseekPricingSignatureGuardConfig, result deepseekPricingSignatureGuardResult, checkedAt time.Time, runErr error) officialImportSignatureAuditRecord {
|
||||
record := officialImportSignatureAuditRecord{
|
||||
SourceKey: cfg.SourceKey,
|
||||
CheckedAt: checkedAt,
|
||||
Status: officialImportSignatureAuditStatus(result.DriftDetected, result.BaselineInitialized, runErr),
|
||||
DriftDetected: result.DriftDetected,
|
||||
BaselineInitialized: result.BaselineInitialized,
|
||||
SourceURL: strings.TrimSpace(cfg.URL),
|
||||
FixturePath: strings.TrimSpace(cfg.Fixture),
|
||||
SnapshotPath: strings.TrimSpace(result.SnapshotPath),
|
||||
SignaturePath: strings.TrimSpace(result.SignaturePath),
|
||||
BaselinePath: strings.TrimSpace(result.BaselinePath),
|
||||
StructureSHA256: strings.TrimSpace(result.CurrentSignature.StructureSHA256),
|
||||
PreviousStructureSHA256: strings.TrimSpace(result.PreviousBaselineHash),
|
||||
ByteSize: result.CurrentSignature.ByteSize,
|
||||
ErrorMessage: errorMessageText(runErr),
|
||||
}
|
||||
if hasDeepSeekPricingStructureSignature(result.CurrentSignature) {
|
||||
signatureCopy := result.CurrentSignature
|
||||
record.SignaturePayload = &signatureCopy
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
func persistDeepSeekPricingSignatureAuditIfConfigured(cfg deepseekPricingSignatureGuardConfig, result deepseekPricingSignatureGuardResult, checkedAt time.Time, runErr error) error {
|
||||
return persistOfficialImportSignatureAuditIfConfigured(buildDeepSeekPricingSignatureAuditRecord(cfg, result, checkedAt, runErr))
|
||||
}
|
||||
96
scripts/deepseek_pricing_signature_guard_test.go
Normal file
96
scripts/deepseek_pricing_signature_guard_test.go
Normal file
@@ -0,0 +1,96 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRunDeepSeekPricingSignatureGuardInitializesBaseline(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
baselinePath := filepath.Join(tempDir, "baseline.signature.json")
|
||||
fixture := filepath.Join(tempDir, "pricing.html")
|
||||
if err := os.WriteFile(fixture, []byte(`<html><head><title>DeepSeek</title><meta name="description" content="Join DeepSeek API platform"><meta name="commit-id" content="abc123"><meta property="og:url" content="https://platform.deepseek.com/pricing"></head><body>pricing</body></html>`), 0o644); err != nil {
|
||||
t.Fatalf("写入 fixture 失败: %v", err)
|
||||
}
|
||||
result, err := runDeepSeekPricingSignatureGuard(deepseekPricingSignatureGuardConfig{
|
||||
SourceKey: "deepseek_pricing_signature",
|
||||
URL: defaultDeepSeekPricingFetchURL,
|
||||
Fixture: fixture,
|
||||
SnapshotDir: tempDir,
|
||||
BaselinePath: baselinePath,
|
||||
Timeout: time.Second,
|
||||
AllowBootstrap: true,
|
||||
SnapshotBase: "deepseek-pricing",
|
||||
}, time.Date(2026, 5, 27, 22, 0, 0, 0, time.FixedZone("CST", 8*3600)))
|
||||
if err != nil {
|
||||
t.Fatalf("runDeepSeekPricingSignatureGuard 返回错误: %v", err)
|
||||
}
|
||||
if !result.BaselineInitialized {
|
||||
t.Fatal("期望初始化 baseline")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDeepSeekPricingSignatureGuardDetectsDrift(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
baselinePath := filepath.Join(tempDir, "baseline.signature.json")
|
||||
fixture := filepath.Join(tempDir, "pricing.html")
|
||||
if err := os.WriteFile(fixture, []byte(`<html><head><title>DeepSeek</title><meta name="description" content="Join DeepSeek API platform"><meta name="commit-id" content="abc123"><meta property="og:url" content="https://platform.deepseek.com/pricing"></head><body>pricing</body></html>`), 0o644); err != nil {
|
||||
t.Fatalf("写入 fixture 失败: %v", err)
|
||||
}
|
||||
_, err := runDeepSeekPricingSignatureGuard(deepseekPricingSignatureGuardConfig{
|
||||
SourceKey: "deepseek_pricing_signature",
|
||||
URL: defaultDeepSeekPricingFetchURL,
|
||||
Fixture: fixture,
|
||||
SnapshotDir: tempDir,
|
||||
BaselinePath: baselinePath,
|
||||
Timeout: time.Second,
|
||||
AllowBootstrap: true,
|
||||
SnapshotBase: "deepseek-pricing",
|
||||
}, time.Date(2026, 5, 27, 22, 1, 0, 0, time.FixedZone("CST", 8*3600)))
|
||||
if err != nil {
|
||||
t.Fatalf("初始化 baseline 失败: %v", err)
|
||||
}
|
||||
driftFixture := filepath.Join(tempDir, "pricing-drift.html")
|
||||
if err := os.WriteFile(driftFixture, []byte(`<html><head><title>DeepSeek Pricing</title><meta name="description" content="Updated DeepSeek pricing"><meta name="commit-id" content="def456"><meta property="og:url" content="https://platform.deepseek.com/pricing"></head><body>pricing update</body></html>`), 0o644); err != nil {
|
||||
t.Fatalf("写入 drift fixture 失败: %v", err)
|
||||
}
|
||||
result, err := runDeepSeekPricingSignatureGuard(deepseekPricingSignatureGuardConfig{
|
||||
SourceKey: "deepseek_pricing_signature",
|
||||
URL: defaultDeepSeekPricingFetchURL,
|
||||
Fixture: driftFixture,
|
||||
SnapshotDir: tempDir,
|
||||
BaselinePath: baselinePath,
|
||||
Timeout: time.Second,
|
||||
AllowBootstrap: false,
|
||||
SnapshotBase: "deepseek-pricing",
|
||||
}, time.Date(2026, 5, 27, 22, 2, 0, 0, time.FixedZone("CST", 8*3600)))
|
||||
if err == nil {
|
||||
t.Fatal("期望结构漂移时报错")
|
||||
}
|
||||
if !result.DriftDetected {
|
||||
t.Fatal("期望 driftDetected=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatDeepSeekPricingSignatureGuardSummary(t *testing.T) {
|
||||
result := deepseekPricingSignatureGuardResult{
|
||||
SnapshotPath: "/tmp/deepseek-pricing.html",
|
||||
SignaturePath: "/tmp/deepseek-pricing.signature.json",
|
||||
BaselinePath: "/tmp/baseline.signature.json",
|
||||
BaselineInitialized: true,
|
||||
CurrentSignature: deepseekPricingStructureSignature{
|
||||
StructureSHA256: "abc123",
|
||||
},
|
||||
}
|
||||
summary := formatDeepSeekPricingSignatureGuardSummary("deepseek_pricing_signature", result)
|
||||
for _, want := range []string{"source=deepseek_pricing_signature", "baseline_initialized=true", "structure_sha256=abc123"} {
|
||||
if !strings.Contains(summary, want) {
|
||||
t.Fatalf("summary 缺少 %q,实际: %q", want, summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
183
scripts/deepseek_pricing_snapshot_lib.go
Normal file
183
scripts/deepseek_pricing_snapshot_lib.go
Normal file
@@ -0,0 +1,183 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type deepseekPricingStructureSignature struct {
|
||||
ByteSize int `json:"byte_size"`
|
||||
SHA256 string `json:"sha256"`
|
||||
StructureSHA256 string `json:"structure_sha256"`
|
||||
Title string `json:"title"`
|
||||
MetaDescription string `json:"meta_description"`
|
||||
CommitID string `json:"commit_id"`
|
||||
CanonicalURL string `json:"canonical_url"`
|
||||
Contains map[string]bool `json:"contains"`
|
||||
GeneratedAt string `json:"generated_at,omitempty"`
|
||||
SourceURL string `json:"source_url,omitempty"`
|
||||
SnapshotPath string `json:"snapshot_path,omitempty"`
|
||||
}
|
||||
|
||||
var deepseekPricingContainsNeedles = map[string]string{
|
||||
"deepseek": "deepseek",
|
||||
"platform": "platform",
|
||||
"pricing": "pricing",
|
||||
"api_docs": "api",
|
||||
"developer": "developer resources",
|
||||
}
|
||||
|
||||
var deepseekPricingTitleRe = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
|
||||
var deepseekPricingMetaDescRe = regexp.MustCompile(`(?is)<meta[^>]+name=["']description["'][^>]+content=["']([^"']+)["']`)
|
||||
var deepseekPricingCommitRe = regexp.MustCompile(`(?is)<meta[^>]+name=["']commit-id["'][^>]+content=["']([^"']+)["']`)
|
||||
var deepseekPricingCanonicalRe = regexp.MustCompile(`(?is)<meta[^>]+property=["']og:url["'][^>]+content=["']([^"']+)["']`)
|
||||
var deepseekPricingHTMLTagRe = regexp.MustCompile(`(?s)<[^>]+>`)
|
||||
|
||||
func buildDeepSeekPricingStructureSignature(raw string) deepseekPricingStructureSignature {
|
||||
title := firstDeepSeekPricingHTMLMatch(deepseekPricingTitleRe, raw)
|
||||
meta := firstDeepSeekPricingHTMLMatch(deepseekPricingMetaDescRe, raw)
|
||||
commitID := firstDeepSeekPricingHTMLMatch(deepseekPricingCommitRe, raw)
|
||||
canonicalURL := firstDeepSeekPricingHTMLMatch(deepseekPricingCanonicalRe, raw)
|
||||
contains := make(map[string]bool, len(deepseekPricingContainsNeedles))
|
||||
lower := strings.ToLower(raw)
|
||||
for key, needle := range deepseekPricingContainsNeedles {
|
||||
contains[key] = strings.Contains(lower, strings.ToLower(needle))
|
||||
}
|
||||
signature := deepseekPricingStructureSignature{
|
||||
ByteSize: len([]byte(raw)),
|
||||
SHA256: deepseekPricingSHA256Hex(raw),
|
||||
Title: title,
|
||||
MetaDescription: meta,
|
||||
CommitID: commitID,
|
||||
CanonicalURL: canonicalURL,
|
||||
Contains: contains,
|
||||
}
|
||||
signature.StructureSHA256 = deepseekPricingSHA256Hex(deepseekPricingStructureDigestPayload(signature))
|
||||
return signature
|
||||
}
|
||||
|
||||
func writeDeepSeekPricingSnapshotArtifacts(raw string, sourceURL string, snapshotPath string, signaturePath string, now time.Time) (deepseekPricingStructureSignature, error) {
|
||||
if strings.TrimSpace(snapshotPath) == "" {
|
||||
return deepseekPricingStructureSignature{}, fmt.Errorf("snapshot path is required")
|
||||
}
|
||||
if strings.TrimSpace(signaturePath) == "" {
|
||||
return deepseekPricingStructureSignature{}, fmt.Errorf("signature path is required")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(snapshotPath), 0o755); err != nil {
|
||||
return deepseekPricingStructureSignature{}, fmt.Errorf("mkdir snapshot dir: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(signaturePath), 0o755); err != nil {
|
||||
return deepseekPricingStructureSignature{}, fmt.Errorf("mkdir signature dir: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(snapshotPath, []byte(raw), 0o644); err != nil {
|
||||
return deepseekPricingStructureSignature{}, fmt.Errorf("write snapshot: %w", err)
|
||||
}
|
||||
signature := buildDeepSeekPricingStructureSignature(raw)
|
||||
signature.GeneratedAt = now.Format(time.RFC3339)
|
||||
signature.SourceURL = sourceURL
|
||||
signature.SnapshotPath = snapshotPath
|
||||
payload, err := json.MarshalIndent(signature, "", " ")
|
||||
if err != nil {
|
||||
return deepseekPricingStructureSignature{}, fmt.Errorf("marshal signature: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(signaturePath, payload, 0o644); err != nil {
|
||||
return deepseekPricingStructureSignature{}, fmt.Errorf("write signature: %w", err)
|
||||
}
|
||||
return signature, nil
|
||||
}
|
||||
|
||||
func resolveDeepSeekPricingSnapshotPaths(snapshotPath string, signaturePath string, snapshotDir string, baseName string, now time.Time) (string, string) {
|
||||
if strings.TrimSpace(snapshotDir) == "" {
|
||||
snapshotDir = filepath.Join("logs", baseName+"-snapshots")
|
||||
}
|
||||
if strings.TrimSpace(snapshotPath) == "" {
|
||||
base := filepath.Join(snapshotDir, fmt.Sprintf("%s-%s", baseName, now.Format("20060102-150405")))
|
||||
snapshotPath = base + ".html"
|
||||
if strings.TrimSpace(signaturePath) == "" {
|
||||
signaturePath = base + ".signature.json"
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(signaturePath) == "" {
|
||||
signaturePath = strings.TrimSuffix(snapshotPath, filepath.Ext(snapshotPath)) + ".signature.json"
|
||||
}
|
||||
return snapshotPath, signaturePath
|
||||
}
|
||||
|
||||
func readDeepSeekPricingStructureSignature(path string) (deepseekPricingStructureSignature, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return deepseekPricingStructureSignature{}, err
|
||||
}
|
||||
var signature deepseekPricingStructureSignature
|
||||
if err := json.Unmarshal(data, &signature); err != nil {
|
||||
return deepseekPricingStructureSignature{}, fmt.Errorf("unmarshal signature %s: %w", path, err)
|
||||
}
|
||||
return signature, nil
|
||||
}
|
||||
|
||||
func hasDeepSeekPricingStructureSignature(signature deepseekPricingStructureSignature) bool {
|
||||
return signature.ByteSize > 0 ||
|
||||
strings.TrimSpace(signature.StructureSHA256) != "" ||
|
||||
strings.TrimSpace(signature.SHA256) != "" ||
|
||||
strings.TrimSpace(signature.Title) != "" ||
|
||||
strings.TrimSpace(signature.CommitID) != "" ||
|
||||
len(signature.Contains) > 0
|
||||
}
|
||||
|
||||
func deepseekPricingStructureDigestPayload(signature deepseekPricingStructureSignature) string {
|
||||
type containsEntry struct {
|
||||
Name string `json:"name"`
|
||||
Value bool `json:"value"`
|
||||
}
|
||||
keys := make([]string, 0, len(signature.Contains))
|
||||
for key := range signature.Contains {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
entries := make([]containsEntry, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
entries = append(entries, containsEntry{Name: key, Value: signature.Contains[key]})
|
||||
}
|
||||
payload := struct {
|
||||
Title string `json:"title"`
|
||||
MetaDescription string `json:"meta_description"`
|
||||
CommitID string `json:"commit_id"`
|
||||
CanonicalURL string `json:"canonical_url"`
|
||||
Contains []containsEntry `json:"contains"`
|
||||
}{
|
||||
Title: signature.Title,
|
||||
MetaDescription: signature.MetaDescription,
|
||||
CommitID: signature.CommitID,
|
||||
CanonicalURL: signature.CanonicalURL,
|
||||
Contains: entries,
|
||||
}
|
||||
bytes, _ := json.Marshal(payload)
|
||||
return string(bytes)
|
||||
}
|
||||
|
||||
func deepseekPricingSHA256Hex(raw string) string {
|
||||
sum := sha256.Sum256([]byte(raw))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func firstDeepSeekPricingHTMLMatch(re *regexp.Regexp, raw string) string {
|
||||
match := re.FindStringSubmatch(raw)
|
||||
if len(match) < 2 {
|
||||
return ""
|
||||
}
|
||||
text := deepseekPricingHTMLTagRe.ReplaceAllString(match[1], " ")
|
||||
text = strings.ReplaceAll(text, "&", "&")
|
||||
text = strings.ReplaceAll(text, " ", " ")
|
||||
text = strings.Join(strings.Fields(text), " ")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
449
scripts/discover_intraday_news_candidates.go
Normal file
449
scripts/discover_intraday_news_candidates.go
Normal file
@@ -0,0 +1,449 @@
|
||||
//go:build llm_script && !scripts_pkg
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
type intradayNewsCandidate struct {
|
||||
CandidateDate string
|
||||
EventType string
|
||||
ProviderName string
|
||||
ModelName string
|
||||
ProviderCountry string
|
||||
Title string
|
||||
Summary string
|
||||
CandidateURLs []string
|
||||
DiscoverySource string
|
||||
DiscoveryQuery string
|
||||
DiscoveryEvidence map[string]any
|
||||
NormalizedKey string
|
||||
Status string
|
||||
VerificationConfidence string
|
||||
VerificationNotes string
|
||||
}
|
||||
|
||||
type intradayDiscoveryConfig struct {
|
||||
Date string
|
||||
DryRun bool
|
||||
Search intradayProviderConfig
|
||||
LLM intradayProviderConfig
|
||||
DatabaseURL string
|
||||
Timeout time.Duration
|
||||
ProviderLimit int
|
||||
}
|
||||
|
||||
type intradayDiscoverySummary struct {
|
||||
CandidateTotal int `json:"candidate_total"`
|
||||
ProviderHitCount int `json:"provider_hit_count"`
|
||||
EventTypeCounts map[string]int `json:"event_type_counts"`
|
||||
DiscoverySourceSet []string `json:"discovery_source_set"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
}
|
||||
|
||||
var intradayDiscoveryLogger *slog.Logger
|
||||
|
||||
func init() {
|
||||
intradayDiscoveryLogger = slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadIntradayEnv()
|
||||
cfg := loadIntradayDiscoveryConfig()
|
||||
if err := runIntradayCandidateDiscovery(cfg); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "discover_intraday_news_candidates: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func loadIntradayDiscoveryConfig() intradayDiscoveryConfig {
|
||||
var cfg intradayDiscoveryConfig
|
||||
flag.StringVar(&cfg.Date, "date", intradayDateValue(), "候选发现日期,格式 YYYY-MM-DD")
|
||||
flag.BoolVar(&cfg.DryRun, "dry-run", false, "仅输出摘要,不写数据库")
|
||||
flag.IntVar(&cfg.ProviderLimit, "provider-limit", 10, "最大 provider 数")
|
||||
flag.Parse()
|
||||
|
||||
cfg.DatabaseURL = intradayDefaultDSN()
|
||||
cfg.Timeout = discoveryTimeoutFromEnv()
|
||||
cfg.Search = intradayProviderConfig{
|
||||
Mode: strings.TrimSpace(os.Getenv("INTRADAY_DISCOVERY_SEARCH_PROVIDER")),
|
||||
Command: strings.TrimSpace(os.Getenv("INTRADAY_DISCOVERY_SEARCH_COMMAND")),
|
||||
URL: strings.TrimSpace(os.Getenv("INTRADAY_DISCOVERY_SEARCH_URL")),
|
||||
Fixture: strings.TrimSpace(os.Getenv("INTRADAY_DISCOVERY_SEARCH_FIXTURE")),
|
||||
Timeout: cfg.Timeout,
|
||||
}
|
||||
cfg.LLM = intradayProviderConfig{
|
||||
Mode: strings.TrimSpace(os.Getenv("INTRADAY_DISCOVERY_LLM_PROVIDER")),
|
||||
Command: strings.TrimSpace(os.Getenv("INTRADAY_DISCOVERY_LLM_COMMAND")),
|
||||
URL: strings.TrimSpace(os.Getenv("INTRADAY_DISCOVERY_LLM_URL")),
|
||||
Fixture: strings.TrimSpace(os.Getenv("INTRADAY_DISCOVERY_LLM_FIXTURE")),
|
||||
Timeout: cfg.Timeout,
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func runIntradayCandidateDiscovery(cfg intradayDiscoveryConfig) error {
|
||||
if strings.TrimSpace(cfg.Date) == "" {
|
||||
return fmt.Errorf("date 未设置")
|
||||
}
|
||||
if err := validateIntradayProviderConfig("search", cfg.Search); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateIntradayProviderConfig("llm", cfg.LLM); err != nil {
|
||||
return err
|
||||
}
|
||||
queries := buildIntradayQueries(cfg.Date, cfg.ProviderLimit)
|
||||
searchRecords, err := loadIntradaySearchRecords(cfg.Search, cfg.Date, queries)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
llmRecords, err := loadIntradayLLMRecords(cfg.LLM, cfg.Date, searchRecords)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
candidates := normalizeIntradayCandidates(cfg.Date, searchRecords, llmRecords)
|
||||
summary := summarizeIntradayCandidates(candidates, cfg.DryRun)
|
||||
if cfg.DryRun {
|
||||
return printIntradayDiscoverySummary(summary)
|
||||
}
|
||||
|
||||
db, err := sql.Open("postgres", cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open db: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := upsertIntradayCandidates(context.Background(), db, candidates); err != nil {
|
||||
return err
|
||||
}
|
||||
return printIntradayDiscoverySummary(summary)
|
||||
}
|
||||
|
||||
func validateIntradayProviderConfig(name string, cfg intradayProviderConfig) error {
|
||||
if strings.TrimSpace(cfg.Mode) == "" {
|
||||
return fmt.Errorf("%s provider 未设置", name)
|
||||
}
|
||||
switch cfg.Mode {
|
||||
case "fixture":
|
||||
if strings.TrimSpace(cfg.Fixture) == "" {
|
||||
return fmt.Errorf("%s provider fixture 未设置", name)
|
||||
}
|
||||
case "command_json":
|
||||
if strings.TrimSpace(cfg.Command) == "" {
|
||||
return fmt.Errorf("%s provider command 未设置", name)
|
||||
}
|
||||
case "http_json":
|
||||
if strings.TrimSpace(cfg.URL) == "" {
|
||||
return fmt.Errorf("%s provider url 未设置", name)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("%s provider mode 不支持: %s", name, cfg.Mode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildIntradayQueries(date string, providerLimit int) []string {
|
||||
queries := []string{
|
||||
"site:platform.deepseek.com DeepSeek pricing",
|
||||
"site:api-docs.deepseek.com DeepSeek release news",
|
||||
"site:docs.anthropic.com Claude Sonnet 4 announcement",
|
||||
"site:openrouter.ai OpenRouter models",
|
||||
}
|
||||
if providerLimit > 0 && providerLimit < len(queries) {
|
||||
return queries[:providerLimit]
|
||||
}
|
||||
return queries
|
||||
}
|
||||
|
||||
func normalizeIntradayCandidates(date string, searchRecords []intradaySearchRecord, llmRecords []intradayLLMRecord) []intradayNewsCandidate {
|
||||
searchIndex := indexSearchRecordsByURL(searchRecords)
|
||||
candidatesByKey := map[string]intradayNewsCandidate{}
|
||||
for _, record := range llmRecords {
|
||||
candidate := candidateFromLLMRecord(date, record, searchIndex)
|
||||
if len(candidate.CandidateURLs) == 0 {
|
||||
continue
|
||||
}
|
||||
if candidate.ProviderName == "" {
|
||||
candidate.ProviderName = inferProviderFromTitle(candidate.Title)
|
||||
}
|
||||
candidate.EventType = normalizeIntradayEventType(candidate.EventType)
|
||||
candidate.NormalizedKey = buildIntradayNormalizedKey(candidate)
|
||||
mergeIntradayCandidate(candidatesByKey, candidate)
|
||||
}
|
||||
result := make([]intradayNewsCandidate, 0, len(candidatesByKey))
|
||||
for _, candidate := range candidatesByKey {
|
||||
result = append(result, candidate)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].ProviderName != result[j].ProviderName {
|
||||
return result[i].ProviderName < result[j].ProviderName
|
||||
}
|
||||
if result[i].EventType != result[j].EventType {
|
||||
return result[i].EventType < result[j].EventType
|
||||
}
|
||||
return result[i].NormalizedKey < result[j].NormalizedKey
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func candidateFromLLMRecord(date string, record intradayLLMRecord, searchIndex map[string]intradaySearchRecord) intradayNewsCandidate {
|
||||
candidate := intradayNewsCandidate{
|
||||
CandidateDate: date,
|
||||
EventType: record.EventType,
|
||||
ProviderName: strings.TrimSpace(record.ProviderName),
|
||||
ModelName: strings.TrimSpace(record.ModelName),
|
||||
ProviderCountry: strings.TrimSpace(record.ProviderCountry),
|
||||
Title: strings.TrimSpace(record.Title),
|
||||
Summary: strings.TrimSpace(record.Summary),
|
||||
CandidateURLs: dedupeStrings(record.CandidateURLs),
|
||||
DiscoverySource: "llm_answer",
|
||||
DiscoveryEvidence: map[string]any{"llm_record": record},
|
||||
Status: "candidate",
|
||||
VerificationConfidence: "candidate",
|
||||
}
|
||||
matchedSearch := false
|
||||
filteredURLs := make([]string, 0, len(candidate.CandidateURLs))
|
||||
for _, url := range candidate.CandidateURLs {
|
||||
searchRecord, ok := searchIndex[url]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if !searchRecordMatchesDate(searchRecord, date) {
|
||||
continue
|
||||
}
|
||||
matchedSearch = true
|
||||
filteredURLs = append(filteredURLs, url)
|
||||
candidate.DiscoverySource = "web_search+llm"
|
||||
candidate.DiscoveryQuery = searchRecord.Title
|
||||
candidate.DiscoveryEvidence["search_record"] = searchRecord
|
||||
if candidate.ProviderName == "" {
|
||||
candidate.ProviderName = strings.TrimSpace(searchRecord.Provider)
|
||||
}
|
||||
if candidate.Title == "" {
|
||||
candidate.Title = strings.TrimSpace(searchRecord.Title)
|
||||
}
|
||||
if candidate.Summary == "" {
|
||||
candidate.Summary = strings.TrimSpace(searchRecord.Summary)
|
||||
}
|
||||
}
|
||||
if !matchedSearch {
|
||||
candidate.CandidateURLs = nil
|
||||
return candidate
|
||||
}
|
||||
candidate.CandidateURLs = dedupeStrings(filteredURLs)
|
||||
return candidate
|
||||
}
|
||||
|
||||
func indexSearchRecordsByURL(records []intradaySearchRecord) map[string]intradaySearchRecord {
|
||||
indexed := make(map[string]intradaySearchRecord, len(records))
|
||||
for _, record := range records {
|
||||
url := strings.TrimSpace(record.URL)
|
||||
if url == "" {
|
||||
continue
|
||||
}
|
||||
indexed[url] = record
|
||||
}
|
||||
return indexed
|
||||
}
|
||||
|
||||
func mergeIntradayCandidate(target map[string]intradayNewsCandidate, candidate intradayNewsCandidate) {
|
||||
if candidate.NormalizedKey == "" {
|
||||
return
|
||||
}
|
||||
existing, ok := target[candidate.NormalizedKey]
|
||||
if !ok {
|
||||
target[candidate.NormalizedKey] = candidate
|
||||
return
|
||||
}
|
||||
merged := existing
|
||||
merged.CandidateURLs = dedupeStrings(append(existing.CandidateURLs, candidate.CandidateURLs...))
|
||||
if strings.TrimSpace(merged.Summary) == "" {
|
||||
merged.Summary = candidate.Summary
|
||||
}
|
||||
if strings.TrimSpace(merged.ProviderCountry) == "" {
|
||||
merged.ProviderCountry = candidate.ProviderCountry
|
||||
}
|
||||
if merged.DiscoverySource != candidate.DiscoverySource && candidate.DiscoverySource != "" {
|
||||
merged.DiscoverySource = "web_search+llm"
|
||||
}
|
||||
if merged.DiscoveryEvidence == nil {
|
||||
merged.DiscoveryEvidence = map[string]any{}
|
||||
}
|
||||
if llmRecord, ok := candidate.DiscoveryEvidence["llm_record"]; ok {
|
||||
merged.DiscoveryEvidence["llm_record"] = llmRecord
|
||||
}
|
||||
if searchRecord, ok := candidate.DiscoveryEvidence["search_record"]; ok {
|
||||
merged.DiscoveryEvidence["search_record"] = searchRecord
|
||||
}
|
||||
target[candidate.NormalizedKey] = merged
|
||||
}
|
||||
|
||||
func buildIntradayNormalizedKey(candidate intradayNewsCandidate) string {
|
||||
provider := normalizeWord(candidate.ProviderName)
|
||||
model := normalizeWord(candidate.ModelName)
|
||||
if model == "" {
|
||||
model = normalizeWord(candidate.Title)
|
||||
}
|
||||
return strings.Join([]string{
|
||||
candidate.CandidateDate,
|
||||
normalizeWord(candidate.EventType),
|
||||
provider,
|
||||
model,
|
||||
}, "|")
|
||||
}
|
||||
|
||||
func searchRecordMatchesDate(record intradaySearchRecord, date string) bool {
|
||||
published := strings.TrimSpace(record.PublishedAt)
|
||||
if published == "" {
|
||||
return false
|
||||
}
|
||||
if ts, ok := parseSearchPublishedAt(published); ok {
|
||||
return ts == date
|
||||
}
|
||||
return strings.Contains(published, date)
|
||||
}
|
||||
|
||||
func parseSearchPublishedAt(value string) (string, bool) {
|
||||
for _, layout := range []string{time.RFC3339, "2006-01-02", "Mon, 02 Jan 2006 15:04:05 MST", "Mon, 2 Jan 2006 15:04:05 MST"} {
|
||||
if ts, err := time.Parse(layout, value); err == nil {
|
||||
return ts.Format("2006-01-02"), true
|
||||
}
|
||||
}
|
||||
localized := strings.NewReplacer(
|
||||
"周一", "Mon", "周二", "Tue", "周三", "Wed", "周四", "Thu", "周五", "Fri", "周六", "Sat", "周日", "Sun",
|
||||
"1月", "Jan", "2月", "Feb", "3月", "Mar", "4月", "Apr", "5月", "May", "6月", "Jun",
|
||||
"7月", "Jul", "8月", "Aug", "9月", "Sep", "10月", "Oct", "11月", "Nov", "12月", "Dec",
|
||||
).Replace(value)
|
||||
for _, layout := range []string{"Mon, 2 Jan 2006 15:04:05 MST", "Mon, 02 Jan 2006 15:04:05 MST"} {
|
||||
if ts, err := time.Parse(layout, localized); err == nil {
|
||||
return ts.Format("2006-01-02"), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
|
||||
func summarizeIntradayCandidates(candidates []intradayNewsCandidate, dryRun bool) intradayDiscoverySummary {
|
||||
eventTypeCounts := make(map[string]int)
|
||||
providerSet := map[string]struct{}{}
|
||||
sourceSet := map[string]struct{}{}
|
||||
for _, candidate := range candidates {
|
||||
eventTypeCounts[candidate.EventType]++
|
||||
if candidate.ProviderName != "" {
|
||||
providerSet[candidate.ProviderName] = struct{}{}
|
||||
}
|
||||
if candidate.DiscoverySource != "" {
|
||||
sourceSet[candidate.DiscoverySource] = struct{}{}
|
||||
}
|
||||
}
|
||||
sources := make([]string, 0, len(sourceSet))
|
||||
for source := range sourceSet {
|
||||
sources = append(sources, source)
|
||||
}
|
||||
sort.Strings(sources)
|
||||
return intradayDiscoverySummary{
|
||||
CandidateTotal: len(candidates),
|
||||
ProviderHitCount: len(providerSet),
|
||||
EventTypeCounts: eventTypeCounts,
|
||||
DiscoverySourceSet: sources,
|
||||
DryRun: dryRun,
|
||||
}
|
||||
}
|
||||
|
||||
func printIntradayDiscoverySummary(summary intradayDiscoverySummary) error {
|
||||
payload, err := json.Marshal(summary)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(payload))
|
||||
return nil
|
||||
}
|
||||
|
||||
func upsertIntradayCandidates(ctx context.Context, db *sql.DB, candidates []intradayNewsCandidate) error {
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is nil")
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
urls, err := json.Marshal(candidate.CandidateURLs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal candidate urls: %w", err)
|
||||
}
|
||||
evidence, err := json.Marshal(candidate.DiscoveryEvidence)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal discovery evidence: %w", err)
|
||||
}
|
||||
_, err = db.ExecContext(ctx, `
|
||||
INSERT INTO intraday_news_candidate (
|
||||
candidate_date, event_type, provider_name, model_name, provider_country,
|
||||
title, summary, candidate_urls, discovery_source, discovery_query,
|
||||
discovery_evidence, normalized_key, status, verification_confidence, verification_notes
|
||||
) VALUES (
|
||||
$1::date, $2, $3, NULLIF($4, ''), NULLIF($5, ''),
|
||||
$6, NULLIF($7, ''), $8::jsonb, $9, NULLIF($10, ''),
|
||||
$11::jsonb, $12, $13, $14, NULLIF($15, '')
|
||||
)
|
||||
ON CONFLICT (normalized_key) DO UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
summary = COALESCE(NULLIF(EXCLUDED.summary, ''), intraday_news_candidate.summary),
|
||||
candidate_urls = EXCLUDED.candidate_urls,
|
||||
discovery_source = EXCLUDED.discovery_source,
|
||||
discovery_query = COALESCE(NULLIF(EXCLUDED.discovery_query, ''), intraday_news_candidate.discovery_query),
|
||||
discovery_evidence = EXCLUDED.discovery_evidence,
|
||||
provider_country = COALESCE(NULLIF(EXCLUDED.provider_country, ''), intraday_news_candidate.provider_country),
|
||||
updated_at = CURRENT_TIMESTAMP`,
|
||||
candidate.CandidateDate,
|
||||
candidate.EventType,
|
||||
candidate.ProviderName,
|
||||
candidate.ModelName,
|
||||
candidate.ProviderCountry,
|
||||
candidate.Title,
|
||||
candidate.Summary,
|
||||
string(urls),
|
||||
candidate.DiscoverySource,
|
||||
candidate.DiscoveryQuery,
|
||||
string(evidence),
|
||||
candidate.NormalizedKey,
|
||||
candidate.Status,
|
||||
candidate.VerificationConfidence,
|
||||
candidate.VerificationNotes,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert intraday candidate %s: %w", candidate.NormalizedKey, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func inferProviderFromTitle(title string) string {
|
||||
lower := strings.ToLower(title)
|
||||
for _, pair := range []struct{ match, provider string }{
|
||||
{"openai", "OpenAI"},
|
||||
{"anthropic", "Anthropic"},
|
||||
{"gemini", "Google"},
|
||||
{"deepseek", "DeepSeek"},
|
||||
{"qwen", "Qwen"},
|
||||
{"dashscope", "DashScope"},
|
||||
{"xai", "xAI"},
|
||||
{"minimax", "MiniMax"},
|
||||
{"智谱", "智谱"},
|
||||
{"百度", "百度"},
|
||||
{"腾讯", "腾讯"},
|
||||
} {
|
||||
if strings.Contains(lower, pair.match) {
|
||||
return pair.provider
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
158
scripts/discover_intraday_news_candidates_test.go
Normal file
158
scripts/discover_intraday_news_candidates_test.go
Normal file
@@ -0,0 +1,158 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoadIntradaySearchRecordsFromFixture(t *testing.T) {
|
||||
cfg := intradayProviderConfig{
|
||||
Mode: "fixture",
|
||||
Fixture: filepath.Join("testdata", "intraday_discovery_search_sample.json"),
|
||||
}
|
||||
records, err := loadIntradaySearchRecords(cfg, "2026-05-25", []string{"OpenAI pricing release"})
|
||||
if err != nil {
|
||||
t.Fatalf("loadIntradaySearchRecords 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 2 {
|
||||
t.Fatalf("搜索样例条数错误: got=%d", len(records))
|
||||
}
|
||||
if records[0].URL == "" || records[0].Provider == "" {
|
||||
t.Fatalf("搜索样例未保留 URL/provider: %+v", records[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadIntradayLLMRecordsFromFixture(t *testing.T) {
|
||||
cfg := intradayProviderConfig{
|
||||
Mode: "fixture",
|
||||
Fixture: filepath.Join("testdata", "intraday_discovery_llm_sample.json"),
|
||||
}
|
||||
records, err := loadIntradayLLMRecords(cfg, "2026-05-25", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("loadIntradayLLMRecords 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 2 {
|
||||
t.Fatalf("LLM 样例条数错误: got=%d", len(records))
|
||||
}
|
||||
if records[0].EventType != "official_release" {
|
||||
t.Fatalf("LLM 事件类型错误: %+v", records[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeIntradayCandidatesDedupesEquivalentEvents(t *testing.T) {
|
||||
searchRecords := []intradaySearchRecord{{
|
||||
Title: "OpenAI announces GPT-5.6 preview pricing update",
|
||||
Summary: "Search summary",
|
||||
URL: "https://openai.example.com/news/gpt-5-6-pricing",
|
||||
Provider: "OpenAI",
|
||||
PublishedAt: "2026-05-25",
|
||||
}}
|
||||
llmRecords := []intradayLLMRecord{
|
||||
{
|
||||
EventType: "official_release",
|
||||
ProviderName: "OpenAI",
|
||||
ModelName: "GPT-5.6",
|
||||
ProviderCountry: "US",
|
||||
Title: "GPT-5.6 preview pricing update",
|
||||
Summary: "First summary",
|
||||
CandidateURLs: []string{"https://openai.example.com/news/gpt-5-6-pricing"},
|
||||
},
|
||||
{
|
||||
EventType: "official_release",
|
||||
ProviderName: "OpenAI",
|
||||
ModelName: "GPT 5.6",
|
||||
ProviderCountry: "US",
|
||||
Title: "OpenAI GPT 5.6 preview pricing update",
|
||||
Summary: "Second summary",
|
||||
CandidateURLs: []string{"https://openai.example.com/news/gpt-5-6-pricing"},
|
||||
},
|
||||
}
|
||||
candidates := normalizeIntradayCandidates("2026-05-25", searchRecords, llmRecords)
|
||||
if len(candidates) != 1 {
|
||||
t.Fatalf("期望去重后只剩 1 条候选, got=%d", len(candidates))
|
||||
}
|
||||
if candidates[0].DiscoverySource != "web_search+llm" {
|
||||
t.Fatalf("期望 discovery source 合并, got=%q", candidates[0].DiscoverySource)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeIntradayCandidatesDropsOutdatedSearchMatches(t *testing.T) {
|
||||
searchRecords := []intradaySearchRecord{{
|
||||
Title: "Old DeepSeek pricing article",
|
||||
Summary: "Yesterday record",
|
||||
URL: "https://deepseek.example.com/pricing",
|
||||
Provider: "DeepSeek",
|
||||
PublishedAt: "2026-05-24",
|
||||
}}
|
||||
llmRecords := []intradayLLMRecord{{
|
||||
EventType: "price_cut",
|
||||
ProviderName: "DeepSeek",
|
||||
ModelName: "DeepSeek-V4-Flash",
|
||||
ProviderCountry: "CN",
|
||||
Title: "DeepSeek V4 Flash price cut",
|
||||
Summary: "Should be dropped because search evidence is stale",
|
||||
CandidateURLs: []string{"https://deepseek.example.com/pricing"},
|
||||
}}
|
||||
candidates := normalizeIntradayCandidates("2026-05-25", searchRecords, llmRecords)
|
||||
if len(candidates) != 0 {
|
||||
t.Fatalf("旧闻搜索结果不应进入候选池, got=%d", len(candidates))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeIntradayCandidatesDropsURLlessRecords(t *testing.T) {
|
||||
llmRecords := []intradayLLMRecord{{
|
||||
EventType: "promo_campaign",
|
||||
ProviderName: "DeepSeek",
|
||||
ModelName: "DeepSeek-V4-Flash",
|
||||
Title: "No URL candidate",
|
||||
Summary: "Should be dropped",
|
||||
}}
|
||||
candidates := normalizeIntradayCandidates("2026-05-25", nil, llmRecords)
|
||||
if len(candidates) != 0 {
|
||||
t.Fatalf("无 URL 候选应被丢弃, got=%d", len(candidates))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchRecordMatchesLocalizedBingDate(t *testing.T) {
|
||||
record := intradaySearchRecord{PublishedAt: "周一, 25 5月 2026 14:08:00 GMT"}
|
||||
if !searchRecordMatchesDate(record, "2026-05-25") {
|
||||
t.Fatal("应识别本地化 Bing pubDate 为当天")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateIntradayProviderConfigRequiresCommandOrURLOrFixture(t *testing.T) {
|
||||
if err := validateIntradayProviderConfig("search", intradayProviderConfig{Mode: "command_json"}); err == nil {
|
||||
t.Fatal("缺少 command 时应报错")
|
||||
}
|
||||
if err := validateIntradayProviderConfig("llm", intradayProviderConfig{Mode: "http_json"}); err == nil {
|
||||
t.Fatal("缺少 url 时应报错")
|
||||
}
|
||||
if err := validateIntradayProviderConfig("search", intradayProviderConfig{Mode: "fixture", Fixture: "fixture.json"}); err != nil {
|
||||
t.Fatalf("fixture provider 不应报错: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildIntradayNormalizedKeyUsesProviderModelAndDate(t *testing.T) {
|
||||
key := buildIntradayNormalizedKey(intradayNewsCandidate{
|
||||
CandidateDate: "2026-05-25",
|
||||
EventType: "official_release",
|
||||
ProviderName: "OpenAI",
|
||||
ModelName: "GPT-5.6",
|
||||
})
|
||||
if !strings.Contains(key, "2026-05-25") || !strings.Contains(key, "openai") || !strings.Contains(key, "gpt-5-6") {
|
||||
t.Fatalf("normalized key 不符合预期: %q", key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertIntradayCandidatesRequiresDB(t *testing.T) {
|
||||
var db *sql.DB
|
||||
err := upsertIntradayCandidates(context.Background(), db, nil)
|
||||
if err == nil {
|
||||
t.Fatal("nil db 时应报错")
|
||||
}
|
||||
}
|
||||
25
scripts/env_precedence_test.sh
Executable file
25
scripts/env_precedence_test.sh
Executable file
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
cat > "$TMP_DIR/.env.local" <<'EOF'
|
||||
LOCAL_ALPHA=test-placeholder-value
|
||||
LOCAL_BETA=real-db
|
||||
EOF
|
||||
cat > "$TMP_DIR/.env" <<'EOF'
|
||||
LOCAL_ALPHA=
|
||||
LOCAL_BETA=
|
||||
EOF
|
||||
|
||||
|
||||
|
||||
unset LOCAL_ALPHA LOCAL_BETA || true
|
||||
while IFS= read -r kv; do export "$kv"; done < <(scripts/load_project_env.sh "$TMP_DIR/.env.local")
|
||||
while IFS= read -r kv; do key="${kv%%=*}"; [[ -n "$key" && -n "${!key:-}" ]] && continue; export "$kv"; done < <(scripts/load_project_env.sh "$TMP_DIR/.env")
|
||||
[[ "$LOCAL_ALPHA" == "test-placeholder-value" ]]
|
||||
[[ "$LOCAL_BETA" == "real-db" ]]
|
||||
195
scripts/export_official_seed_json.go
Normal file
195
scripts/export_official_seed_json.go
Normal file
@@ -0,0 +1,195 @@
|
||||
//go:build llm_script && !scripts_pkg
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
type bytedanceSeedRow struct {
|
||||
Model string `json:"model"`
|
||||
InputPrice float64 `json:"inputPrice"`
|
||||
OutputPrice float64 `json:"outputPrice"`
|
||||
ContextLength int `json:"contextLength"`
|
||||
Operator string `json:"operator"`
|
||||
Region string `json:"region"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
|
||||
type baiduSeedRow struct {
|
||||
Model string `json:"model"`
|
||||
Type string `json:"type"`
|
||||
InputPrice *float64 `json:"inputPrice"`
|
||||
OutputPrice *float64 `json:"outputPrice"`
|
||||
Operator string `json:"operator"`
|
||||
Region string `json:"region"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
dsn := os.Getenv("DATABASE_URL")
|
||||
if dsn == "" {
|
||||
dsn = "postgres://long@/llm_intelligence?host=/var/run/postgresql"
|
||||
}
|
||||
|
||||
db, err := sql.Open("postgres", dsn)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
bytedanceRows, err := loadBytedanceSeedRows(db)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
baiduRows, err := loadBaiduSeedRows(db)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
if err := writeJSON("/tmp/bytedance_raw.json", map[string]any{"bytedance": bytedanceRows}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := writeJSON("/tmp/phase2_raw_data.json", map[string]any{"baidu": baiduRows, "zhipu": []any{}}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
log.Printf("Exported %d ByteDance rows to /tmp/bytedance_raw.json", len(bytedanceRows))
|
||||
log.Printf("Exported %d Baidu rows to /tmp/phase2_raw_data.json", len(baiduRows))
|
||||
}
|
||||
|
||||
func loadBytedanceSeedRows(db *sql.DB) ([]bytedanceSeedRow, error) {
|
||||
rows, err := db.Query(`
|
||||
WITH latest AS (
|
||||
SELECT DISTINCT ON (rp.model_id, rp.operator_id, rp.region, rp.currency)
|
||||
rp.model_id,
|
||||
rp.operator_id,
|
||||
rp.region,
|
||||
rp.currency,
|
||||
rp.input_price_per_mtok,
|
||||
rp.output_price_per_mtok,
|
||||
rp.effective_date
|
||||
FROM region_pricing rp
|
||||
JOIN models m ON m.id = rp.model_id
|
||||
JOIN operator o ON o.id = rp.operator_id
|
||||
WHERE m.external_id LIKE 'bytedance-%'
|
||||
AND o.name = 'ByteDance Volcano'
|
||||
ORDER BY rp.model_id, rp.operator_id, rp.region, rp.currency, rp.effective_date DESC, rp.updated_at DESC, rp.id DESC
|
||||
)
|
||||
SELECT REPLACE(m.external_id, 'bytedance-', ''),
|
||||
COALESCE(latest.input_price_per_mtok, 0),
|
||||
COALESCE(latest.output_price_per_mtok, 0),
|
||||
COALESCE(m.context_length, 0),
|
||||
o.name,
|
||||
latest.region,
|
||||
latest.currency
|
||||
FROM latest
|
||||
JOIN models m ON m.id = latest.model_id
|
||||
JOIN operator o ON o.id = latest.operator_id
|
||||
ORDER BY m.external_id
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []bytedanceSeedRow
|
||||
for rows.Next() {
|
||||
var row bytedanceSeedRow
|
||||
if err := rows.Scan(
|
||||
&row.Model,
|
||||
&row.InputPrice,
|
||||
&row.OutputPrice,
|
||||
&row.ContextLength,
|
||||
&row.Operator,
|
||||
&row.Region,
|
||||
&row.Currency,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, row)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func loadBaiduSeedRows(db *sql.DB) ([]baiduSeedRow, error) {
|
||||
rows, err := db.Query(`
|
||||
WITH latest AS (
|
||||
SELECT DISTINCT ON (rp.model_id, rp.operator_id, rp.region, rp.currency)
|
||||
rp.model_id,
|
||||
rp.operator_id,
|
||||
rp.region,
|
||||
rp.currency,
|
||||
rp.input_price_per_mtok,
|
||||
rp.output_price_per_mtok,
|
||||
rp.effective_date
|
||||
FROM region_pricing rp
|
||||
JOIN models m ON m.id = rp.model_id
|
||||
JOIN operator o ON o.id = rp.operator_id
|
||||
WHERE m.external_id LIKE 'baidu-%'
|
||||
AND o.name = 'Baidu Qianfan'
|
||||
ORDER BY rp.model_id, rp.operator_id, rp.region, rp.currency, rp.effective_date DESC, rp.updated_at DESC, rp.id DESC
|
||||
)
|
||||
SELECT m.name,
|
||||
COALESCE(latest.input_price_per_mtok, 0),
|
||||
COALESCE(latest.output_price_per_mtok, 0),
|
||||
o.name,
|
||||
latest.region,
|
||||
latest.currency
|
||||
FROM latest
|
||||
JOIN models m ON m.id = latest.model_id
|
||||
JOIN operator o ON o.id = latest.operator_id
|
||||
ORDER BY m.external_id
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []baiduSeedRow
|
||||
for rows.Next() {
|
||||
var (
|
||||
model string
|
||||
inputPrice float64
|
||||
outputPrice float64
|
||||
operator string
|
||||
region string
|
||||
currency string
|
||||
)
|
||||
if err := rows.Scan(&model, &inputPrice, &outputPrice, &operator, ®ion, ¤cy); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inputPerToken := inputPrice / 1000000
|
||||
outputPerToken := outputPrice / 1000000
|
||||
result = append(result, baiduSeedRow{
|
||||
Model: model,
|
||||
Type: "输入",
|
||||
InputPrice: &inputPerToken,
|
||||
Operator: operator,
|
||||
Region: region,
|
||||
Currency: currency,
|
||||
})
|
||||
result = append(result, baiduSeedRow{
|
||||
Model: model,
|
||||
Type: "输出",
|
||||
OutputPrice: &outputPerToken,
|
||||
Operator: operator,
|
||||
Region: region,
|
||||
Currency: currency,
|
||||
})
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func writeJSON(path string, value any) error {
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0644)
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build llm_script
|
||||
//go:build llm_script && !scripts_pkg
|
||||
|
||||
// fetch_multi_source.go - 多源 LLM 定价采集器
|
||||
// 支持: OpenRouter, Moonshot, DeepSeek, OpenAI 等
|
||||
@@ -64,11 +64,14 @@ type sourceDefinition struct {
|
||||
}
|
||||
|
||||
type runSummary struct {
|
||||
SelectedSources int
|
||||
SuccessfulSources int
|
||||
TotalModels int
|
||||
DomesticModels int
|
||||
CurrencyCounts map[string]int
|
||||
SelectedSources int
|
||||
SelectedSourceKeys []string
|
||||
SuccessfulSources int
|
||||
SuccessfulSourceKeys []string
|
||||
FailedSourceKeys []string
|
||||
TotalModels int
|
||||
DomesticModels int
|
||||
CurrencyCounts map[string]int
|
||||
}
|
||||
|
||||
type pricingMetadataFields struct {
|
||||
@@ -256,12 +259,15 @@ func listSourceKeys(apiKey string) []string {
|
||||
return keys
|
||||
}
|
||||
|
||||
func summarizePrices(selectedSources int, successfulSources int, prices []ModelPricing) runSummary {
|
||||
func summarizePrices(selectedSourceKeys []string, successfulSourceKeys []string, failedSourceKeys []string, prices []ModelPricing) runSummary {
|
||||
summary := runSummary{
|
||||
SelectedSources: selectedSources,
|
||||
SuccessfulSources: successfulSources,
|
||||
TotalModels: len(prices),
|
||||
CurrencyCounts: make(map[string]int),
|
||||
SelectedSources: len(selectedSourceKeys),
|
||||
SelectedSourceKeys: append([]string(nil), selectedSourceKeys...),
|
||||
SuccessfulSources: len(successfulSourceKeys),
|
||||
SuccessfulSourceKeys: append([]string(nil), successfulSourceKeys...),
|
||||
FailedSourceKeys: append([]string(nil), failedSourceKeys...),
|
||||
TotalModels: len(prices),
|
||||
CurrencyCounts: make(map[string]int),
|
||||
}
|
||||
for _, price := range prices {
|
||||
if strings.EqualFold(price.ProviderCountry, "CN") {
|
||||
@@ -272,6 +278,21 @@ func summarizePrices(selectedSources int, successfulSources int, prices []ModelP
|
||||
return summary
|
||||
}
|
||||
|
||||
func sourceKey(src DataSource) string {
|
||||
switch strings.ToLower(strings.TrimSpace(src.Name())) {
|
||||
case "openrouter":
|
||||
return "openrouter"
|
||||
case "moonshot":
|
||||
return "moonshot"
|
||||
case "deepseek":
|
||||
return "deepseek"
|
||||
case "openai":
|
||||
return "openai"
|
||||
default:
|
||||
return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(src.Name()), " ", "_"))
|
||||
}
|
||||
}
|
||||
|
||||
func formatCountMap(counts map[string]int) string {
|
||||
if len(counts) == 0 {
|
||||
return "none"
|
||||
@@ -289,17 +310,27 @@ func formatCountMap(counts map[string]int) string {
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func formatKeyList(keys []string) string {
|
||||
if len(keys) == 0 {
|
||||
return "none"
|
||||
}
|
||||
return strings.Join(keys, ",")
|
||||
}
|
||||
|
||||
func printSummary(w io.Writer, summary runSummary) error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
_, err := fmt.Fprintf(
|
||||
w,
|
||||
"sources=%d successful_sources=%d models=%d domestic_models=%d currencies=%s\n",
|
||||
"sources=%d successful_sources=%d models=%d domestic_models=%d selected_source_keys=%s successful_source_keys=%s failed_source_keys=%s currencies=%s\n",
|
||||
summary.SelectedSources,
|
||||
summary.SuccessfulSources,
|
||||
summary.TotalModels,
|
||||
summary.DomesticModels,
|
||||
formatKeyList(summary.SelectedSourceKeys),
|
||||
formatKeyList(summary.SuccessfulSourceKeys),
|
||||
formatKeyList(summary.FailedSourceKeys),
|
||||
formatCountMap(summary.CurrencyCounts),
|
||||
)
|
||||
return err
|
||||
@@ -564,23 +595,29 @@ func defaultDSN() string {
|
||||
|
||||
func runCollector(cfg runConfig, sources []DataSource, saveFn func([]ModelPricing) error, out io.Writer) error {
|
||||
allPrices := make([]ModelPricing, 0)
|
||||
successfulSources := 0
|
||||
selectedSourceKeys := make([]string, 0, len(sources))
|
||||
successfulSourceKeys := make([]string, 0, len(sources))
|
||||
failedSourceKeys := make([]string, 0)
|
||||
|
||||
for _, src := range sources {
|
||||
key := sourceKey(src)
|
||||
selectedSourceKeys = append(selectedSourceKeys, key)
|
||||
|
||||
prices, err := src.FetchPricing()
|
||||
if err != nil {
|
||||
logger.Error("采集失败", "source", src.Name(), "error", err)
|
||||
failedSourceKeys = append(failedSourceKeys, key)
|
||||
continue
|
||||
}
|
||||
successfulSources++
|
||||
successfulSourceKeys = append(successfulSourceKeys, key)
|
||||
allPrices = append(allPrices, prices...)
|
||||
}
|
||||
|
||||
summary := summarizePrices(len(sources), successfulSources, allPrices)
|
||||
summary := summarizePrices(selectedSourceKeys, successfulSourceKeys, failedSourceKeys, allPrices)
|
||||
if err := printSummary(out, summary); err != nil {
|
||||
return err
|
||||
}
|
||||
if successfulSources == 0 {
|
||||
if summary.SuccessfulSources == 0 {
|
||||
return fmt.Errorf("no data source collected successfully")
|
||||
}
|
||||
if cfg.DryRun {
|
||||
@@ -593,7 +630,7 @@ func runCollector(cfg runConfig, sources []DataSource, saveFn func([]ModelPricin
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Info("多源采集完成", "total_models", len(allPrices), "sources", successfulSources)
|
||||
logger.Info("多源采集完成", "total_models", len(allPrices), "sources", summary.SuccessfulSources)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -90,6 +90,49 @@ func TestRunCollectorDryRunSkipsDatabaseWrite(t *testing.T) {
|
||||
if !bytes.Contains(out.Bytes(), []byte("currencies=CNY:2,USD:1")) {
|
||||
t.Fatalf("expected currency summary, got %q", output)
|
||||
}
|
||||
if !bytes.Contains(out.Bytes(), []byte("selected_source_keys=moonshot,openai")) {
|
||||
t.Fatalf("expected selected source keys in summary, got %q", output)
|
||||
}
|
||||
if !bytes.Contains(out.Bytes(), []byte("successful_source_keys=moonshot,openai")) {
|
||||
t.Fatalf("expected successful source keys in summary, got %q", output)
|
||||
}
|
||||
if !bytes.Contains(out.Bytes(), []byte("failed_source_keys=none")) {
|
||||
t.Fatalf("expected failed source keys in summary, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCollectorReportsFailedSourceKeys(t *testing.T) {
|
||||
cfg := runConfig{DryRun: true}
|
||||
var out bytes.Buffer
|
||||
|
||||
err := runCollector(
|
||||
cfg,
|
||||
[]DataSource{
|
||||
fakeSource{
|
||||
name: "Moonshot",
|
||||
prices: []ModelPricing{
|
||||
{ModelID: "kimi-k2.6", ProviderCountry: "CN", Currency: "CNY"},
|
||||
},
|
||||
},
|
||||
fakeSource{
|
||||
name: "OpenAI",
|
||||
err: bytes.ErrTooLarge,
|
||||
},
|
||||
},
|
||||
nil,
|
||||
&out,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("runCollector returned error: %v", err)
|
||||
}
|
||||
|
||||
output := out.String()
|
||||
if !bytes.Contains(out.Bytes(), []byte("successful_source_keys=moonshot")) {
|
||||
t.Fatalf("expected successful source keys in summary, got %q", output)
|
||||
}
|
||||
if !bytes.Contains(out.Bytes(), []byte("failed_source_keys=openai")) {
|
||||
t.Fatalf("expected failed source keys in summary, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPricingMetadataClassifiesSourceType(t *testing.T) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build llm_script
|
||||
//go:build llm_script && !scripts_pkg
|
||||
|
||||
// fetch_openrouter.go - OpenRouter 模型数据采集器 v2.0
|
||||
// Sprint 2 增强版:指数退避重试 + 批量插入 + ProviderMapper + audit_log + 价格变动检测 + slog
|
||||
@@ -33,6 +33,7 @@ type Config struct {
|
||||
TimeoutSec int
|
||||
BatchSize int
|
||||
DBConn string
|
||||
StrictReal bool
|
||||
}
|
||||
|
||||
// ModelInfo 模型信息(与 collectors 包兼容)
|
||||
@@ -92,13 +93,14 @@ func main() {
|
||||
|
||||
func parseArgs() Config {
|
||||
loadProjectEnv()
|
||||
apiKey := flag.String("api-key", "", "OpenRouter API Key")
|
||||
apiKey := flag.String("api-key", os.Getenv("OPENROUTER_API_KEY"), "OpenRouter API Key")
|
||||
apiURL := flag.String("api-url", "https://openrouter.ai/api/v1/models", "API 地址")
|
||||
outPath := flag.String("out", "models.json", "输出文件路径")
|
||||
maxRetries := flag.Int("retry", 3, "最大重试次数")
|
||||
timeoutSec := flag.Int("timeout", 30, "请求超时(秒)")
|
||||
batchSize := flag.Int("batch", 100, "批量插入批次大小")
|
||||
dbConn := flag.String("db", os.Getenv("DATABASE_URL"), "PostgreSQL 连接字符串")
|
||||
strictReal := flag.Bool("strict-real", false, "严格真实模式:缺少 API Key 或数据库写入失败时返回错误")
|
||||
flag.Parse()
|
||||
return Config{
|
||||
APIKey: *apiKey,
|
||||
@@ -108,6 +110,7 @@ func parseArgs() Config {
|
||||
TimeoutSec: *timeoutSec,
|
||||
BatchSize: *batchSize,
|
||||
DBConn: *dbConn,
|
||||
StrictReal: *strictReal,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -158,6 +161,9 @@ func run(cfg Config) error {
|
||||
if cfg.DBConn != "" {
|
||||
if err := summarizeDB(cfg.DBConn, models, cfg.BatchSize); err != nil {
|
||||
logger.Error("PostgreSQL 写入失败", "error", err)
|
||||
if cfg.StrictReal {
|
||||
return fmt.Errorf("PostgreSQL 写入失败: %w", err)
|
||||
}
|
||||
logger.Warn("降级为仅写入 JSON")
|
||||
} else {
|
||||
logger.Info("PostgreSQL 写入完成", "records", len(models))
|
||||
@@ -169,6 +175,9 @@ func run(cfg Config) error {
|
||||
// fetchModels 抓取 OpenRouter 模型列表(集成指数退避重试)
|
||||
func fetchModels(cfg Config) ([]ModelInfo, error) {
|
||||
if cfg.APIKey == "" {
|
||||
if cfg.StrictReal {
|
||||
return nil, fmt.Errorf("严格真实模式下必须提供 API Key")
|
||||
}
|
||||
logger.Warn("未提供 API Key,使用模拟数据")
|
||||
return []ModelInfo{
|
||||
{ID: "openai/gpt-4o", ContextLength: 128000, Pricing: ModelPricing{Input: 2.5, Output: 10.0}},
|
||||
@@ -206,7 +215,7 @@ func fetchModels(cfg Config) ([]ModelInfo, error) {
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
lastErr = fmt.Errorf("非 200 响应: %d %s", resp.StatusCode, string(body))
|
||||
lastErr = retry.HTTPStatusError{StatusCode: resp.StatusCode, Body: string(body)}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
@@ -278,6 +287,38 @@ func parseModels(raw []byte) ([]ModelInfo, error) {
|
||||
return models, nil
|
||||
}
|
||||
|
||||
func deriveModality(model ModelInfo) string {
|
||||
for _, capability := range model.Capabilities {
|
||||
normalized := strings.ToLower(capability)
|
||||
switch {
|
||||
case strings.Contains(normalized, "vision"), strings.Contains(normalized, "image"):
|
||||
return "multimodal"
|
||||
case strings.Contains(normalized, "audio"):
|
||||
return "audio"
|
||||
case strings.Contains(normalized, "video"):
|
||||
return "video"
|
||||
case strings.Contains(normalized, "code"):
|
||||
return "code"
|
||||
}
|
||||
}
|
||||
|
||||
hints := strings.ToLower(strings.Join([]string{model.ID, model.Name, model.Description}, " "))
|
||||
switch {
|
||||
case strings.Contains(hints, "video") && (strings.Contains(hints, "omni") || strings.Contains(hints, "vision") || strings.Contains(hints, "multimodal")):
|
||||
return "multimodal"
|
||||
case strings.Contains(hints, "vision") || strings.Contains(hints, "image") || strings.Contains(hints, "vl") || strings.Contains(hints, "omni") || strings.Contains(hints, "multimodal"):
|
||||
return "multimodal"
|
||||
case strings.Contains(hints, "audio") || strings.Contains(hints, "speech") || strings.Contains(hints, "voice"):
|
||||
return "audio"
|
||||
case strings.Contains(hints, "video"):
|
||||
return "video"
|
||||
case strings.Contains(hints, "code"):
|
||||
return "code"
|
||||
default:
|
||||
return "text"
|
||||
}
|
||||
}
|
||||
|
||||
func getString(m map[string]any, key string) string {
|
||||
if v, ok := m[key].(string); ok {
|
||||
return v
|
||||
@@ -434,7 +475,7 @@ func summarizeDB(connStr string, models []ModelInfo, batchSize int) error {
|
||||
`,
|
||||
"openrouter", m.ID, m.Name, m.Description, m.ContextLength,
|
||||
jsonCapabilities(m.Capabilities), m.Created, isFree, "active",
|
||||
rawPayload(m), providerID, "", "text",
|
||||
rawPayload(m), providerID, "", deriveModality(m),
|
||||
"official", now, batchID, collectorVersion,
|
||||
"https://openrouter.ai/api/v1/models", now).Scan(&modelID)
|
||||
if err != nil {
|
||||
|
||||
@@ -4,9 +4,13 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"llm-intelligence/internal/retry"
|
||||
)
|
||||
|
||||
// Test 1: parseModels 正确解析 name、context_length、capabilities、pricing input/prompt 和 output/completion
|
||||
@@ -46,6 +50,10 @@ func TestParseModels(t *testing.T) {
|
||||
if m.Pricing.Output != 10.0 {
|
||||
t.Errorf("Pricing.Output 错误: %f", m.Pricing.Output)
|
||||
}
|
||||
if modality := deriveModality(m); modality != "multimodal" {
|
||||
t.Errorf("deriveModality = %q, want %q", modality, "multimodal")
|
||||
|
||||
}
|
||||
|
||||
// 第二条:pricing 用 prompt/completion 别名回退
|
||||
m2 := models[1]
|
||||
@@ -63,6 +71,68 @@ func TestParseModels(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveModality(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
capabilities []string
|
||||
want string
|
||||
}{
|
||||
{name: "vision first", capabilities: []string{"vision", "json_mode"}, want: "multimodal"},
|
||||
{name: "audio", capabilities: []string{"audio_generation"}, want: "audio"},
|
||||
{name: "code", capabilities: []string{"code_interpreter"}, want: "code"},
|
||||
{name: "text fallback", capabilities: []string{"function_calling"}, want: "text"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := deriveModality(ModelInfo{Capabilities: tt.capabilities}); got != tt.want {
|
||||
t.Fatalf("deriveModality() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveModalityInfersFromModelIdentityWithoutCapabilities(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
model ModelInfo
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "omni id maps to multimodal",
|
||||
model: ModelInfo{
|
||||
ID: "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free",
|
||||
Description: "accepts text, image, video, and audio inputs",
|
||||
},
|
||||
want: "multimodal",
|
||||
},
|
||||
{
|
||||
name: "audio id maps to audio",
|
||||
model: ModelInfo{
|
||||
ID: "openai/gpt-audio",
|
||||
Description: "audio model for natural sounding voices",
|
||||
},
|
||||
want: "audio",
|
||||
},
|
||||
{
|
||||
name: "vl id maps to multimodal",
|
||||
model: ModelInfo{
|
||||
ID: "qwen/qwen3-vl-32b-instruct",
|
||||
Description: "vision-language model for text, images, and video",
|
||||
},
|
||||
want: "multimodal",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := deriveModality(tt.model); got != tt.want {
|
||||
t.Fatalf("deriveModality(%+v) = %q, want %q", tt.model, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Test 2: run 无 API Key 时写入临时文件,JSON 含 total 和 models 字段
|
||||
func TestRunNoAPIKey(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
@@ -98,3 +168,96 @@ func TestRunNoAPIKey(t *testing.T) {
|
||||
t.Error("models 为空")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchModelsFailsInStrictRealModeWithoutAPIKey(t *testing.T) {
|
||||
_, err := fetchModels(Config{StrictReal: true})
|
||||
if err == nil {
|
||||
t.Fatal("strict real mode should fail without API key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchModelsDoesNotRetryPermanentHTTPErrors(t *testing.T) {
|
||||
attempts := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
http.Error(w, "forbidden", http.StatusForbidden)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
_, err := fetchModels(Config{
|
||||
APIKey: "test-key",
|
||||
APIURL: server.URL,
|
||||
MaxRetries: 3,
|
||||
TimeoutSec: 1,
|
||||
StrictReal: true,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected fetchModels to fail on 403")
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Fatalf("expected 1 attempt for permanent HTTP error, got %d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchModelsRetriesServerErrors(t *testing.T) {
|
||||
attempts := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
if attempts < 3 {
|
||||
http.Error(w, "temporary", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[{"id":"openai/gpt-4o","name":"GPT-4o","context_length":128000,"pricing":{"input":2.5,"output":10.0}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
models, err := fetchModels(Config{
|
||||
APIKey: "test-key",
|
||||
APIURL: server.URL,
|
||||
MaxRetries: 3,
|
||||
TimeoutSec: 1,
|
||||
StrictReal: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected retry success, got %v", err)
|
||||
}
|
||||
if len(models) != 1 {
|
||||
t.Fatalf("expected 1 model, got %d", len(models))
|
||||
}
|
||||
if attempts != 3 {
|
||||
t.Fatalf("expected 3 attempts for temporary server error, got %d", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFailsInStrictRealModeWhenDBWriteFails(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
outPath := filepath.Join(tmpDir, "models.json")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[{"id":"openai/gpt-4o","name":"GPT-4o","context_length":128000,"pricing":{"input":2.5,"output":10.0}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
err := run(Config{
|
||||
APIKey: "test-key",
|
||||
APIURL: server.URL,
|
||||
OutPath: outPath,
|
||||
DBConn: "postgres://invalid@127.0.0.1:1/invalid?sslmode=disable",
|
||||
BatchSize: 10,
|
||||
TimeoutSec: 1,
|
||||
StrictReal: true,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("strict real mode should fail when database write fails")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryHTTPStatusErrorClassification(t *testing.T) {
|
||||
if retry.IsRetryable(retry.HTTPStatusError{StatusCode: http.StatusForbidden}) {
|
||||
t.Fatal("403 should not be retryable")
|
||||
}
|
||||
if !retry.IsRetryable(retry.HTTPStatusError{StatusCode: http.StatusBadGateway}) {
|
||||
t.Fatal("502 should be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build llm_script
|
||||
//go:build llm_script && !scripts_pkg
|
||||
|
||||
package main
|
||||
|
||||
|
||||
@@ -96,3 +96,33 @@ func TestRunTencentCatalogDryRunPrintsSummary(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTencentCatalogExtractsPromotionalPlans(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "tencent_token_plan_promo_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取促销样例失败: %v", err)
|
||||
}
|
||||
|
||||
catalog, err := parseTencentCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseTencentCatalog 失败: %v", err)
|
||||
}
|
||||
|
||||
if len(catalog.Plans) != 2 {
|
||||
t.Fatalf("期望 2 个套餐,实际 %d", len(catalog.Plans))
|
||||
}
|
||||
|
||||
first := catalog.Plans[0]
|
||||
if first.Series != "通用 Token Plan" {
|
||||
t.Fatalf("套餐系列错误: %q", first.Series)
|
||||
}
|
||||
if first.Tier != "首月活动版" {
|
||||
t.Fatalf("促销套餐档位错误: %q", first.Tier)
|
||||
}
|
||||
if first.Price != "19元/月" {
|
||||
t.Fatalf("促销套餐价格错误: %q", first.Price)
|
||||
}
|
||||
if first.Scene != "首购用户首月优惠,次月恢复标准价。" {
|
||||
t.Fatalf("促销套餐说明错误: %q", first.Scene)
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
//go:build llm_script
|
||||
//go:build llm_script && !scripts_pkg
|
||||
|
||||
package main
|
||||
|
||||
|
||||
21
scripts/git_commit_status_report.sh
Executable file
21
scripts/git_commit_status_report.sh
Executable file
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
LABEL="${1:-worktree}"
|
||||
STATUS_OUTPUT="$(git status --short 2>/dev/null || true)"
|
||||
BLOCKER_THRESHOLD="${WORKTREE_BLOCKER_THRESHOLD:-50}"
|
||||
|
||||
if [[ -z "$STATUS_OUTPUT" ]]; then
|
||||
echo "WORKTREE_STATUS label=${LABEL} state=clean tracked_modified=0 untracked=0 total=0 commit_hint=none severity=normal"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TRACKED_MODIFIED=$(printf '%s\n' "$STATUS_OUTPUT" | awk 'NF && $1 !~ /^\?\?/ { count++ } END { print count+0 }')
|
||||
UNTRACKED=$(printf '%s\n' "$STATUS_OUTPUT" | awk '$1 ~ /^\?\?/ { count++ } END { print count+0 }')
|
||||
TOTAL=$((TRACKED_MODIFIED + UNTRACKED))
|
||||
SEVERITY="warning"
|
||||
if [[ "$TOTAL" -gt "$BLOCKER_THRESHOLD" ]]; then
|
||||
SEVERITY="blocker"
|
||||
fi
|
||||
|
||||
echo "WORKTREE_STATUS label=${LABEL} state=dirty tracked_modified=${TRACKED_MODIFIED} untracked=${UNTRACKED} total=${TOTAL} commit_hint=needed severity=${SEVERITY}"
|
||||
36
scripts/git_commit_status_test.sh
Normal file
36
scripts/git_commit_status_test.sh
Normal file
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
CLEAN_REPO="$TMP_DIR/repo"
|
||||
mkdir -p "$CLEAN_REPO"
|
||||
cd "$CLEAN_REPO"
|
||||
git init -q
|
||||
printf 'tracked\n' > tracked.txt
|
||||
git add tracked.txt
|
||||
git config user.email test@example.com
|
||||
git config user.name test
|
||||
|
||||
git commit -qm 'init'
|
||||
|
||||
CLEAN_OUTPUT="$(bash /home/long/project/llm-intelligence/scripts/git_commit_status_report.sh clean)"
|
||||
printf '%s' "$CLEAN_OUTPUT" | grep -q 'state=clean'
|
||||
printf '%s' "$CLEAN_OUTPUT" | grep -q 'severity=normal'
|
||||
|
||||
printf 'dirty\n' >> tracked.txt
|
||||
DIRTY_OUTPUT="$(bash /home/long/project/llm-intelligence/scripts/git_commit_status_report.sh dirty)"
|
||||
printf '%s' "$DIRTY_OUTPUT" | grep -q 'state=dirty'
|
||||
printf '%s' "$DIRTY_OUTPUT" | grep -q 'tracked_modified=1'
|
||||
printf '%s' "$DIRTY_OUTPUT" | grep -q 'commit_hint=needed'
|
||||
printf '%s' "$DIRTY_OUTPUT" | grep -q 'severity=warning'
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
printf 'u%d\n' "$i" > "untracked_$i.txt"
|
||||
done
|
||||
BLOCKER_OUTPUT="$(WORKTREE_BLOCKER_THRESHOLD=50 bash /home/long/project/llm-intelligence/scripts/git_commit_status_report.sh blocker)"
|
||||
printf '%s' "$BLOCKER_OUTPUT" | grep -q 'severity=blocker'
|
||||
157
scripts/huawei_package_lib.go
Normal file
157
scripts/huawei_package_lib.go
Normal file
@@ -0,0 +1,157 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultHuaweiPackagePlanURL = "https://support.huaweicloud.com/price-maas/price-maas-0002.html"
|
||||
|
||||
func parseHuaweiPackageCatalog(raw string) ([]subscriptionImportRecord, error) {
|
||||
publishedAt, known := publishedAtFromText(raw)
|
||||
type packDef struct {
|
||||
QuotaRaw string
|
||||
BillingCycle string
|
||||
CodeSuffix string
|
||||
ReadableCycle string
|
||||
}
|
||||
packs := []packDef{
|
||||
{QuotaRaw: "100万", BillingCycle: "monthly", CodeSuffix: "100w-1m", ReadableCycle: "1个月"},
|
||||
{QuotaRaw: "1000万", BillingCycle: "monthly", CodeSuffix: "1000w-1m", ReadableCycle: "1个月"},
|
||||
{QuotaRaw: "1亿", BillingCycle: "quarterly", CodeSuffix: "1y-3m", ReadableCycle: "3个月"},
|
||||
{QuotaRaw: "10亿", BillingCycle: "quarterly", CodeSuffix: "10y-3m", ReadableCycle: "3个月"},
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, 8)
|
||||
for _, modelVersion := range []string{"1", "2"} {
|
||||
modelLabel := "DeepSeek-V3." + modelVersion
|
||||
versionCode := strings.ReplaceAll("v3."+modelVersion, ".", "-")
|
||||
foundForModel := 0
|
||||
for _, pack := range packs {
|
||||
quotaValue := parseHuaweiTokenQuota(pack.QuotaRaw)
|
||||
price, found := findHuaweiPackPrice(raw, modelLabel, pack.QuotaRaw, pack.ReadableCycle)
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Huawei",
|
||||
ProviderNameCn: "华为",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.huaweicloud.com",
|
||||
OperatorName: "Huawei Cloud",
|
||||
OperatorNameCn: "华为云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://support.huaweicloud.com",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "package_plan",
|
||||
PlanCode: fmt.Sprintf("huawei-deepseek-%s-package-%s", versionCode, pack.CodeSuffix),
|
||||
PlanName: fmt.Sprintf("华为云 MaaS %s 套餐包 %s", modelLabel, pack.QuotaRaw),
|
||||
Tier: modelLabel,
|
||||
BillingCycle: pack.BillingCycle,
|
||||
Currency: "CNY",
|
||||
ListPrice: price,
|
||||
PriceUnit: "CNY/pack",
|
||||
QuotaValue: quotaValue,
|
||||
QuotaUnit: "tokens/pack",
|
||||
PlanScope: "MaaS 文本生成模型套餐包",
|
||||
ModelScope: []string{modelLabel},
|
||||
SourceURL: defaultHuaweiPackagePlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: fmt.Sprintf("官方套餐包,有效期 %s,仅抵扣 %s Token 用量。", pack.ReadableCycle, modelLabel),
|
||||
PublishedAtKnown: known,
|
||||
})
|
||||
foundForModel++
|
||||
}
|
||||
_ = foundForModel
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("no huawei package plan matched from source page")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func fallbackHuaweiPackageCatalog() []subscriptionImportRecord {
|
||||
publishedAt := "2026-05-14 00:00:00"
|
||||
effectiveDate := "2026-05-14"
|
||||
type packRow struct {
|
||||
ModelScope string
|
||||
VersionCode string
|
||||
QuotaValue int64
|
||||
QuotaLabel string
|
||||
BillingCycle string
|
||||
CodeSuffix string
|
||||
Price float64
|
||||
ReadableCycle string
|
||||
}
|
||||
packs := []packRow{
|
||||
{ModelScope: "DeepSeek-V3.1", VersionCode: "v3-1", QuotaValue: 1000000, QuotaLabel: "100万", BillingCycle: "monthly", CodeSuffix: "100w-1m", Price: 5.6, ReadableCycle: "1个月"},
|
||||
{ModelScope: "DeepSeek-V3.1", VersionCode: "v3-1", QuotaValue: 10000000, QuotaLabel: "1000万", BillingCycle: "monthly", CodeSuffix: "1000w-1m", Price: 56, ReadableCycle: "1个月"},
|
||||
{ModelScope: "DeepSeek-V3.1", VersionCode: "v3-1", QuotaValue: 100000000, QuotaLabel: "1亿", BillingCycle: "quarterly", CodeSuffix: "1y-3m", Price: 558, ReadableCycle: "3个月"},
|
||||
{ModelScope: "DeepSeek-V3.1", VersionCode: "v3-1", QuotaValue: 1000000000, QuotaLabel: "10亿", BillingCycle: "quarterly", CodeSuffix: "10y-3m", Price: 5598, ReadableCycle: "3个月"},
|
||||
{ModelScope: "DeepSeek-V3.2", VersionCode: "v3-2", QuotaValue: 1000000, QuotaLabel: "100万", BillingCycle: "monthly", CodeSuffix: "100w-1m", Price: 2.2, ReadableCycle: "1个月"},
|
||||
{ModelScope: "DeepSeek-V3.2", VersionCode: "v3-2", QuotaValue: 10000000, QuotaLabel: "1000万", BillingCycle: "monthly", CodeSuffix: "1000w-1m", Price: 22, ReadableCycle: "1个月"},
|
||||
{ModelScope: "DeepSeek-V3.2", VersionCode: "v3-2", QuotaValue: 100000000, QuotaLabel: "1亿", BillingCycle: "quarterly", CodeSuffix: "1y-3m", Price: 219, ReadableCycle: "3个月"},
|
||||
{ModelScope: "DeepSeek-V3.2", VersionCode: "v3-2", QuotaValue: 1000000000, QuotaLabel: "10亿", BillingCycle: "quarterly", CodeSuffix: "10y-3m", Price: 2199, ReadableCycle: "3个月"},
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, len(packs))
|
||||
for _, pack := range packs {
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Huawei",
|
||||
ProviderNameCn: "华为",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.huaweicloud.com",
|
||||
OperatorName: "Huawei Cloud",
|
||||
OperatorNameCn: "华为云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://support.huaweicloud.com",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "package_plan",
|
||||
PlanCode: fmt.Sprintf("huawei-deepseek-%s-package-%s", pack.VersionCode, pack.CodeSuffix),
|
||||
PlanName: fmt.Sprintf("华为云 MaaS %s 套餐包 %s", pack.ModelScope, pack.QuotaLabel),
|
||||
Tier: pack.ModelScope,
|
||||
BillingCycle: pack.BillingCycle,
|
||||
Currency: "CNY",
|
||||
ListPrice: pack.Price,
|
||||
PriceUnit: "CNY/pack",
|
||||
QuotaValue: pack.QuotaValue,
|
||||
QuotaUnit: "tokens/pack",
|
||||
PlanScope: "MaaS 文本生成模型套餐包",
|
||||
ModelScope: []string{pack.ModelScope},
|
||||
SourceURL: defaultHuaweiPackagePlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDate,
|
||||
Notes: fmt.Sprintf("官方价格页动态渲染,当前回退至最近核验的官方快照;有效期 %s,仅抵扣 %s Token 用量。", pack.ReadableCycle, pack.ModelScope),
|
||||
PublishedAtKnown: true,
|
||||
})
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func findHuaweiPackPrice(raw string, modelLabel string, quotaRaw string, cycle string) (float64, bool) {
|
||||
pattern := regexp.MustCompile(`(?s)` + regexp.QuoteMeta(modelLabel) + `.*?` + regexp.QuoteMeta(quotaRaw) + `.*?` + regexp.QuoteMeta(cycle) + `.*?([\d.]+)`)
|
||||
match := pattern.FindStringSubmatch(raw)
|
||||
if len(match) != 2 {
|
||||
return 0, false
|
||||
}
|
||||
return mustParseSubscriptionPrice(match[1]), true
|
||||
}
|
||||
|
||||
func parseHuaweiTokenQuota(raw string) int64 {
|
||||
switch strings.TrimSpace(raw) {
|
||||
case "100万":
|
||||
return 1000000
|
||||
case "1000万":
|
||||
return 10000000
|
||||
case "1亿":
|
||||
return 100000000
|
||||
case "10亿":
|
||||
return 1000000000
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
88
scripts/import_360_pricing.go
Normal file
88
scripts/import_360_pricing.go
Normal file
@@ -0,0 +1,88 @@
|
||||
//go:build llm_script && !scripts_pkg
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type platform360PricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", default360PricingURL, "360 智脑开放平台模型价格页")
|
||||
flag.StringVar(&fixture, "fixture", "", "360 智脑开放平台价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := platform360PricingImportConfig{
|
||||
URL: url,
|
||||
Fixture: fixture,
|
||||
DryRun: dryRun,
|
||||
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||||
}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := run360PricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_360_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run360PricingImport(cfg platform360PricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchSubscriptionPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parse360PricingCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=360-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "360-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=360-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
55
scripts/import_360_pricing_test.go
Normal file
55
scripts/import_360_pricing_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParse360PricingCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "platform360_pricing_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parse360PricingCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parse360PricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 4 {
|
||||
t.Fatalf("期望 4 条 360 价格记录,实际 %d", len(records))
|
||||
}
|
||||
if records[0].ModelID != "360-deepseek-deepseek-v4-flash" {
|
||||
t.Fatalf("首条 modelID 错误: %q", records[0].ModelID)
|
||||
}
|
||||
if records[1].ContextLength != 1000000 {
|
||||
t.Fatalf("第二条上下文长度错误: %d", records[1].ContextLength)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun360PricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := run360PricingImport(platform360PricingImportConfig{
|
||||
URL: default360PricingURL,
|
||||
Fixture: filepath.Join("testdata", "platform360_pricing_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("run360PricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=360-pricing-import",
|
||||
"models=4",
|
||||
"operator=360 ZhiNao",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user