feat(pipeline): wire collectors into real pipeline gates
Wire the new subscription and official pricing collectors into the daily, real, and intel pipeline entrypoints. This commit also upgrades Phase 6 verification with recent-window collector classification so gate failures distinguish preconditions from true runtime or provider issues.
This commit is contained in:
phamnazage-jpg
@@ -14,14 +14,15 @@ if [[ -f "$PROJECT_DIR/.env" ]]; then
|
||||
source "$PROJECT_DIR/.env"
|
||||
fi
|
||||
DB_URL="${DATABASE_URL:-host=/var/run/postgresql dbname=llm_intelligence user=long sslmode=disable}"
|
||||
export DATABASE_URL="$DB_URL"
|
||||
REPORT_DATE="$(report_date_value)"
|
||||
LOG_FILE="/tmp/llm_hub_daily_${REPORT_DATE}.log"
|
||||
FEISHU_WEBHOOK="${FEISHU_WEBHOOK:-}"
|
||||
MODEL_COUNT=""
|
||||
FETCH_OUT="${PROJECT_DIR}/models.json"
|
||||
FETCH_TOTAL="0"
|
||||
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_report"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance"
|
||||
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot,daily_report"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,catalog_seed_verification"
|
||||
PIPELINE_FAILED_SOURCE_SET="none"
|
||||
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
|
||||
PIPELINE_AUDIT_SUMMARY=""
|
||||
@@ -181,6 +182,210 @@ if ! go run -tags llm_script scripts/import_bytedance_data.go >> "$LOG_FILE" 2>&
|
||||
merge_failed_source_keys "bytedance"
|
||||
error_exit "字节官方导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/aliyun_subscription_lib.go \
|
||||
scripts/import_aliyun_subscription.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "aliyun_subscription"
|
||||
error_exit "阿里云套餐导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/baidu_subscription_lib.go \
|
||||
scripts/import_baidu_subscription.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "baidu_subscription"
|
||||
error_exit "百度套餐导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/ctyun_subscription_lib.go \
|
||||
scripts/import_ctyun_subscription.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "ctyun_subscription"
|
||||
error_exit "天翼云套餐导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/bytedance_subscription_lib.go \
|
||||
scripts/import_bytedance_subscription.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "bytedance_subscription"
|
||||
error_exit "火山方舟套餐导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/huawei_package_lib.go \
|
||||
scripts/import_huawei_package.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "huawei_package"
|
||||
error_exit "华为云套餐包导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/zhipu_coding_plan_lib.go \
|
||||
scripts/import_zhipu_coding_plan.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "zhipu_coding_plan"
|
||||
error_exit "智谱 Coding Plan 导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/minimax_subscription_lib.go \
|
||||
scripts/import_minimax_subscription.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "minimax_subscription"
|
||||
error_exit "MiniMax Token Plan 导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/catalog_verification_common.go \
|
||||
scripts/import_cucloud_catalog.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "cucloud_catalog"
|
||||
error_exit "联通云目录校验失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/catalog_verification_common.go \
|
||||
scripts/import_mobile_cloud_catalog.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "mobile_cloud_catalog"
|
||||
error_exit "移动云目录校验失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/youdao_pricing_lib.go \
|
||||
scripts/import_youdao_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "youdao_pricing"
|
||||
error_exit "网易有道价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/platform360_pricing_lib.go \
|
||||
scripts/import_360_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "platform360_pricing"
|
||||
error_exit "360 智脑价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/siliconflow_pricing_lib.go \
|
||||
scripts/import_siliconflow_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "siliconflow_pricing"
|
||||
error_exit "硅基流动价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/ppio_pricing_lib.go \
|
||||
scripts/import_ppio_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "ppio_pricing"
|
||||
error_exit "PPIO 价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/ucloud_pricing_lib.go \
|
||||
scripts/import_ucloud_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "ucloud_pricing"
|
||||
error_exit "UCloud 价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/pricing_markdown_snapshot_lib.go \
|
||||
scripts/cloudflare_pricing_snapshot_lib.go \
|
||||
scripts/signature_guard_common.go \
|
||||
scripts/official_import_signature_audit_lib.go \
|
||||
scripts/cloudflare_pricing_signature_guard_lib.go \
|
||||
scripts/cloudflare_pricing_import_runner.go \
|
||||
scripts/cloudflare_pricing_lib.go \
|
||||
scripts/cloudflare_pricing_signature_guard.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "cloudflare_pricing_signature"
|
||||
error_exit "Cloudflare Workers AI 价格页结构签名漂移"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/pricing_markdown_snapshot_lib.go \
|
||||
scripts/cloudflare_pricing_snapshot_lib.go \
|
||||
scripts/cloudflare_pricing_import_runner.go \
|
||||
scripts/cloudflare_pricing_lib.go \
|
||||
scripts/import_cloudflare_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "cloudflare_pricing"
|
||||
error_exit "Cloudflare Workers AI 价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/pricing_markdown_snapshot_lib.go \
|
||||
scripts/perplexity_pricing_snapshot_lib.go \
|
||||
scripts/signature_guard_common.go \
|
||||
scripts/official_import_signature_audit_lib.go \
|
||||
scripts/perplexity_pricing_signature_guard_lib.go \
|
||||
scripts/perplexity_pricing_import_runner.go \
|
||||
scripts/perplexity_pricing_lib.go \
|
||||
scripts/perplexity_pricing_signature_guard.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "perplexity_pricing_signature"
|
||||
error_exit "Perplexity API 价格页结构签名漂移"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/pricing_markdown_snapshot_lib.go \
|
||||
scripts/perplexity_pricing_snapshot_lib.go \
|
||||
scripts/perplexity_pricing_import_runner.go \
|
||||
scripts/perplexity_pricing_lib.go \
|
||||
scripts/import_perplexity_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "perplexity_pricing"
|
||||
error_exit "Perplexity API 价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/pricing_markdown_snapshot_lib.go \
|
||||
scripts/signature_guard_common.go \
|
||||
scripts/official_import_signature_audit_lib.go \
|
||||
scripts/vertex_pricing_snapshot_lib.go \
|
||||
scripts/vertex_pricing_signature_guard_lib.go \
|
||||
scripts/vertex_pricing_import_runner.go \
|
||||
scripts/vertex_pricing_lib.go \
|
||||
scripts/vertex_pricing_signature_guard.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "vertex_pricing_signature"
|
||||
error_exit "Vertex AI 价格页结构签名漂移"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/vertex_pricing_snapshot_lib.go \
|
||||
scripts/vertex_pricing_import_runner.go \
|
||||
scripts/vertex_pricing_lib.go \
|
||||
scripts/import_vertex_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "vertex_pricing"
|
||||
error_exit "Vertex AI 价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/bedrock_pricing_lib.go \
|
||||
scripts/import_bedrock_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "bedrock_pricing"
|
||||
error_exit "Amazon Bedrock 价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/azure_openai_pricing_lib.go \
|
||||
scripts/import_azure_openai_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "azure_openai_pricing"
|
||||
error_exit "Azure OpenAI 价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/import_catalog_seed_verification.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "catalog_seed_verification"
|
||||
error_exit "目录级官方入口核验失败"
|
||||
fi
|
||||
if ! SIGNAL_SOURCE_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run -tags llm_script \
|
||||
scripts/materialize_daily_signals.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "daily_signal_snapshot"
|
||||
error_exit "每日关键信号物化失败"
|
||||
fi
|
||||
refresh_pipeline_audit
|
||||
log "✅ 多源补充同步完成"
|
||||
|
||||
@@ -195,7 +400,7 @@ log "✅ 数据质量检查通过 (模型数: ${MODEL_COUNT})"
|
||||
# 3. 生成日报
|
||||
log "3️⃣ 生成日报..."
|
||||
export DATABASE_URL="$DB_URL"
|
||||
if ! REPORT_RUN_KIND="scheduled" REPORT_TRIGGER_SOURCE="cron" REPORT_IS_OFFICIAL_DAILY="true" REPORT_RUNTIME_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run scripts/generate_daily_report.go >> "$LOG_FILE" 2>&1; then
|
||||
if ! REPORT_RUN_KIND="scheduled" REPORT_TRIGGER_SOURCE="cron" REPORT_IS_OFFICIAL_DAILY="true" REPORT_RUNTIME_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run scripts/generate_daily_report.go scripts/official_import_signature_audit_query_lib.go >> "$LOG_FILE" 2>&1; then
|
||||
error_exit "日报生成失败"
|
||||
fi
|
||||
log "✅ 日报生成完成"
|
||||
|
||||
Reference in New Issue
Block a user