llm-intelligence/scripts/run_intel_pipeline.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT_DIR"

if [[ -f ".env.local" ]]; then
  # shellcheck disable=SC1091
  source ".env.local"
fi
if [[ -f ".env" ]]; then
  # shellcheck disable=SC1091
  source ".env"
fi

if [[ -z "${DATABASE_URL:-}" ]]; then
  echo "DATABASE_URL 未设置" >&2
  exit 1
fi

if [[ -z "${OPENROUTER_API_KEY:-}" ]]; then
  echo "OPENROUTER_API_KEY 未设置，无法执行真实采集" >&2
  exit 1
fi

REPORT_DATE="${REPORT_DATE:-$(date +%F)}"
FETCH_OUT="$ROOT_DIR/models.json"
FETCH_TOTAL="0"
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot"
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,catalog_seed_verification"
PIPELINE_FAILED_SOURCE_SET="none"
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
PIPELINE_AUDIT_SUMMARY=""

normalize_summary_file() {
  local path="$1"
  if [[ ! -f "$path" ]]; then
    return
  fi
  tr '\n' ' ' < "$path" | sed 's/[[:space:]]\+/ /g; s/^ //; s/ $//'
}

extract_failed_source_keys() {
  local summary="$1"
  printf '%s\n' "$summary" | sed -n 's/.*failed_source_keys=\([^ ]*\).*/\1/p'
}

merge_failed_source_keys() {
  local keys="$1"
  if [[ -z "$keys" || "$keys" == "none" ]]; then
    return
  fi
  if [[ "$PIPELINE_FAILED_SOURCE_SET" == "none" ]]; then
    PIPELINE_FAILED_SOURCE_SET="$keys"
    return
  fi
  PIPELINE_FAILED_SOURCE_SET="${PIPELINE_FAILED_SOURCE_SET},${keys}"
}

refresh_pipeline_audit() {
  PIPELINE_AUDIT_SUMMARY="runtime_audit stage_set=${PIPELINE_STAGE_SET} selected_source_keys=${PIPELINE_SOURCE_SET} failed_source_keys=${PIPELINE_FAILED_SOURCE_SET} openrouter_total=${FETCH_TOTAL:-0} ${MULTI_SOURCE_AUDIT}"
}

run_or_fail() {
  local source_key="$1"
  local error_message="$2"
  shift 2
  if ! "$@"; then
    merge_failed_source_keys "$source_key"
    refresh_pipeline_audit
    echo "$error_message" >&2
    exit 1
  fi
}

refresh_pipeline_audit

bash "$ROOT_DIR/scripts/apply_migration.sh"

run_or_fail "openrouter" "OpenRouter 真实采集失败" \
  go run "./scripts/fetch_openrouter.go" -api-key "$OPENROUTER_API_KEY" -db "$DATABASE_URL" -out "$FETCH_OUT" -strict-real

FETCH_TOTAL=$(python3 - <<'PY' "$FETCH_OUT"
import json, sys
path = sys.argv[1]
with open(path, 'r', encoding='utf-8') as f:
    data = json.load(f)
print(int(data.get("total", 0)))
PY
)
if [[ "${FETCH_TOTAL:-0}" -lt 10 ]]; then
  merge_failed_source_keys "openrouter"
  refresh_pipeline_audit
  echo "本次采集结果异常: total=${FETCH_TOTAL:-0} < 10" >&2
  exit 1
fi
refresh_pipeline_audit

MULTI_SOURCE_OUTPUT="$(mktemp)"
if ! go run "./scripts/fetch_multi_source.go" --sources moonshot,deepseek,openai > "$MULTI_SOURCE_OUTPUT"; then
  MULTI_SOURCE_SUMMARY="$(normalize_summary_file "$MULTI_SOURCE_OUTPUT")"
  if [[ -n "$MULTI_SOURCE_SUMMARY" ]]; then
    MULTI_SOURCE_AUDIT="multi_source_audit=${MULTI_SOURCE_SUMMARY}"
    merge_failed_source_keys "$(extract_failed_source_keys "$MULTI_SOURCE_SUMMARY")"
  else
    MULTI_SOURCE_AUDIT="multi_source_audit=stage_failed"
    merge_failed_source_keys "moonshot,deepseek,openai"
  fi
  cat "$MULTI_SOURCE_OUTPUT"
  rm -f "$MULTI_SOURCE_OUTPUT"
  refresh_pipeline_audit
  echo "多源补充同步失败" >&2
  exit 1
fi
MULTI_SOURCE_SUMMARY="$(normalize_summary_file "$MULTI_SOURCE_OUTPUT")"
MULTI_SOURCE_AUDIT="multi_source_audit=${MULTI_SOURCE_SUMMARY:-none}"
merge_failed_source_keys "$(extract_failed_source_keys "$MULTI_SOURCE_SUMMARY")"
refresh_pipeline_audit
cat "$MULTI_SOURCE_OUTPUT"
rm -f "$MULTI_SOURCE_OUTPUT"

run_or_fail "zhipu" "智谱官方导入失败" go run -tags llm_script "./scripts/import_zhipu_data.go"
run_or_fail "official_seed_export" "官方种子导出失败" go run -tags llm_script "./scripts/export_official_seed_json.go"
run_or_fail "baidu" "百度官方导入失败" go run -tags llm_script "./scripts/import_phase2_data.go"
run_or_fail "bytedance" "字节官方导入失败" go run -tags llm_script "./scripts/import_bytedance_data.go"

run_or_fail "aliyun_subscription" "阿里云套餐导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/aliyun_subscription_lib.go ./scripts/import_aliyun_subscription.go
run_or_fail "baidu_subscription" "百度套餐导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/baidu_subscription_lib.go ./scripts/import_baidu_subscription.go
run_or_fail "ctyun_subscription" "天翼云套餐导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/ctyun_subscription_lib.go ./scripts/import_ctyun_subscription.go
run_or_fail "bytedance_subscription" "火山方舟套餐导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/bytedance_subscription_lib.go ./scripts/import_bytedance_subscription.go
run_or_fail "huawei_package" "华为云套餐包导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/huawei_package_lib.go ./scripts/import_huawei_package.go
run_or_fail "zhipu_coding_plan" "智谱 Coding Plan 导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/zhipu_coding_plan_lib.go ./scripts/import_zhipu_coding_plan.go
run_or_fail "minimax_subscription" "MiniMax Token Plan 导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/minimax_subscription_lib.go ./scripts/import_minimax_subscription.go
run_or_fail "cucloud_catalog" "联通云目录校验失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/catalog_verification_common.go ./scripts/import_cucloud_catalog.go
run_or_fail "mobile_cloud_catalog" "移动云目录校验失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/catalog_verification_common.go ./scripts/import_mobile_cloud_catalog.go
run_or_fail "tencent_subscription" "腾讯云套餐导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/tencent_catalog_lib.go ./scripts/import_tencent_subscription.go
run_or_fail "youdao_pricing" "网易有道价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/youdao_pricing_lib.go ./scripts/import_youdao_pricing.go
run_or_fail "platform360_pricing" "360 智脑价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/platform360_pricing_lib.go ./scripts/import_360_pricing.go
run_or_fail "siliconflow_pricing" "硅基流动价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/siliconflow_pricing_lib.go ./scripts/import_siliconflow_pricing.go
run_or_fail "ppio_pricing" "PPIO 价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/ppio_pricing_lib.go ./scripts/import_ppio_pricing.go
run_or_fail "ucloud_pricing" "UCloud 价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/ucloud_pricing_lib.go ./scripts/import_ucloud_pricing.go
run_or_fail "coreshub_pricing" "CoresHub 价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/coreshub_pricing_lib.go ./scripts/import_coreshub_pricing.go
run_or_fail "cloudflare_pricing_signature" "Cloudflare Workers AI 价格页结构签名漂移" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/pricing_markdown_snapshot_lib.go ./scripts/cloudflare_pricing_snapshot_lib.go ./scripts/signature_guard_common.go ./scripts/official_import_signature_audit_lib.go ./scripts/cloudflare_pricing_signature_guard_lib.go ./scripts/cloudflare_pricing_import_runner.go ./scripts/cloudflare_pricing_lib.go ./scripts/cloudflare_pricing_signature_guard.go
run_or_fail "cloudflare_pricing" "Cloudflare Workers AI 价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/pricing_markdown_snapshot_lib.go ./scripts/cloudflare_pricing_snapshot_lib.go ./scripts/cloudflare_pricing_import_runner.go ./scripts/cloudflare_pricing_lib.go ./scripts/import_cloudflare_pricing.go
run_or_fail "perplexity_pricing_signature" "Perplexity API 价格页结构签名漂移" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/pricing_markdown_snapshot_lib.go ./scripts/perplexity_pricing_snapshot_lib.go ./scripts/signature_guard_common.go ./scripts/official_import_signature_audit_lib.go ./scripts/perplexity_pricing_signature_guard_lib.go ./scripts/perplexity_pricing_import_runner.go ./scripts/perplexity_pricing_lib.go ./scripts/perplexity_pricing_signature_guard.go
run_or_fail "perplexity_pricing" "Perplexity API 价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/pricing_markdown_snapshot_lib.go ./scripts/perplexity_pricing_snapshot_lib.go ./scripts/perplexity_pricing_import_runner.go ./scripts/perplexity_pricing_lib.go ./scripts/import_perplexity_pricing.go
run_or_fail "vertex_pricing_signature" "Vertex AI 价格页结构签名漂移" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/pricing_markdown_snapshot_lib.go ./scripts/signature_guard_common.go ./scripts/official_import_signature_audit_lib.go ./scripts/vertex_pricing_snapshot_lib.go ./scripts/vertex_pricing_signature_guard_lib.go ./scripts/vertex_pricing_import_runner.go ./scripts/vertex_pricing_lib.go ./scripts/vertex_pricing_signature_guard.go

run_or_fail "vertex_pricing" "Vertex AI 价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/vertex_pricing_snapshot_lib.go ./scripts/vertex_pricing_import_runner.go ./scripts/vertex_pricing_lib.go ./scripts/import_vertex_pricing.go
run_or_fail "bedrock_pricing" "Amazon Bedrock 价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/bedrock_pricing_lib.go ./scripts/import_bedrock_pricing.go
run_or_fail "azure_openai_pricing" "Azure OpenAI 价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/azure_openai_pricing_lib.go ./scripts/import_azure_openai_pricing.go
run_or_fail "qwen_pricing" "通义千问价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_qwen_pricing.go
run_or_fail "hunyuan_pricing" "腾讯混元价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_hunyuan_pricing.go
run_or_fail "huawei_maas_pricing" "华为云 MaaS 价格导入失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_huawei_maas_pricing.go

refresh_pipeline_audit
run_or_fail "catalog_seed_verification" "目录级官方入口核验失败" \
  go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/import_catalog_seed_verification.go

refresh_pipeline_audit
run_or_fail "daily_signal_snapshot" "每日关键信号物化失败" \
  env SIGNAL_SOURCE_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run -tags llm_script "./scripts/materialize_daily_signals.go"

echo "$PIPELINE_AUDIT_SUMMARY"