feat(pricing): add cucloud and bytedance payg importers
- Add import_cucloud_pricing.go for 联通云 payg 公开价抓取 - Add import_bytedance_pricing.go for 火山引擎/ByteDance Ark 定价导入 - Include test files and sample testdata for both importers - Update plan catalog inventory docs and seeds - Add cucloud pricing importer implementation plan - Align pipeline scripts and smoke gate tests
This commit is contained in:
284
scripts/import_bytedance_pricing.go
Normal file
284
scripts/import_bytedance_pricing.go
Normal file
@@ -0,0 +1,284 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultBytedanceArkPricingURL = "https://www.volcengine.com/docs/82379/1544106"
|
||||
|
||||
type bytedanceArkPricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultBytedanceArkPricingURL, "火山方舟官方模型价格页")
|
||||
flag.StringVar(&fixture, "fixture", "", "火山方舟价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := bytedanceArkPricingImportConfig{URL: url, Fixture: fixture, DryRun: dryRun, Timeout: time.Duration(timeoutSeconds) * time.Second}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runBytedanceArkPricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_bytedance_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runBytedanceArkPricingImport(cfg bytedanceArkPricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchRawPricingPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parseBytedanceArkPricingCatalog(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=bytedance-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "bytedance-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=bytedance-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
|
||||
func parseBytedanceArkPricingCatalog(raw string) ([]officialPricingRecord, error) {
|
||||
markdown, err := extractBytedanceArkPricingMarkdown(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := extractMarkdownTableRowsForHeading(markdown, "## 在线推理(常规)")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(rows) < 2 {
|
||||
return nil, fmt.Errorf("unexpected bytedance ark pricing table")
|
||||
}
|
||||
|
||||
records := make([]officialPricingRecord, 0, len(rows)-1)
|
||||
for _, row := range rows[1:] {
|
||||
if len(row) < 6 {
|
||||
continue
|
||||
}
|
||||
modelName := cleanBytedanceArkCell(row[0])
|
||||
if modelName == "" || isBytedanceArkConditionRow(modelName) {
|
||||
continue
|
||||
}
|
||||
inputPrice := bytedanceArkPriceValue(row[2])
|
||||
outputPrice := bytedanceArkPriceValue(row[5])
|
||||
if inputPrice <= 0 || outputPrice <= 0 {
|
||||
continue
|
||||
}
|
||||
providerName := bytedanceArkProviderName(modelName)
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
records = append(records, officialPricingRecord{
|
||||
ModelID: normalizeExternalID("bytedance", modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "ByteDance Volcano",
|
||||
OperatorNameCn: "火山引擎",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.volcengine.com/product/ark",
|
||||
OperatorType: "official",
|
||||
Region: "CN",
|
||||
Currency: "CNY",
|
||||
InputPrice: inputPrice,
|
||||
OutputPrice: outputPrice,
|
||||
SourceURL: defaultBytedanceArkPricingURL,
|
||||
ModelSourceURL: defaultBytedanceArkPricingURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(modelName),
|
||||
})
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("no bytedance ark input/output pricing rows found")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func extractBytedanceArkPricingMarkdown(raw string) (string, error) {
|
||||
if !strings.Contains(raw, "window._ROUTER_DATA = ") {
|
||||
return raw, nil
|
||||
}
|
||||
jsonText, err := extractJSONAfterMarker(raw, "window._ROUTER_DATA = ")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var envelope map[string]any
|
||||
if err := json.Unmarshal([]byte(jsonText), &envelope); err != nil {
|
||||
return "", fmt.Errorf("parse bytedance router json: %w", err)
|
||||
}
|
||||
loaderData, _ := envelope["loaderData"].(map[string]any)
|
||||
page, _ := loaderData["docs/(libid)/(docid$)/page"].(map[string]any)
|
||||
curDoc, _ := page["curDoc"].(map[string]any)
|
||||
markdown, _ := curDoc["MDContent"].(string)
|
||||
if strings.TrimSpace(markdown) == "" {
|
||||
return "", fmt.Errorf("missing bytedance pricing markdown content")
|
||||
}
|
||||
return markdown, nil
|
||||
}
|
||||
|
||||
func extractJSONAfterMarker(raw string, marker string) (string, error) {
|
||||
start := strings.Index(raw, marker)
|
||||
if start < 0 {
|
||||
return "", fmt.Errorf("marker %q not found", marker)
|
||||
}
|
||||
start += len(marker)
|
||||
braceDepth := 0
|
||||
inString := false
|
||||
escaped := false
|
||||
end := -1
|
||||
for i := start; i < len(raw); i++ {
|
||||
ch := raw[i]
|
||||
if inString {
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
switch ch {
|
||||
case '\\':
|
||||
escaped = true
|
||||
case '"':
|
||||
inString = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch ch {
|
||||
case '"':
|
||||
inString = true
|
||||
case '{':
|
||||
braceDepth++
|
||||
case '}':
|
||||
braceDepth--
|
||||
if braceDepth == 0 {
|
||||
end = i + 1
|
||||
i = len(raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
if end <= start {
|
||||
return "", fmt.Errorf("unable to locate router json boundary")
|
||||
}
|
||||
return raw[start:end], nil
|
||||
}
|
||||
|
||||
func extractMarkdownTableRowsForHeading(markdown string, heading string) ([][]string, error) {
|
||||
lines := strings.Split(markdown, "\n")
|
||||
capturing := false
|
||||
rows := make([][]string, 0)
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
switch {
|
||||
case trimmed == heading:
|
||||
capturing = true
|
||||
case capturing && strings.HasPrefix(trimmed, "#") && trimmed != heading:
|
||||
if len(rows) > 0 {
|
||||
return rows, nil
|
||||
}
|
||||
capturing = false
|
||||
}
|
||||
if !capturing || !strings.HasPrefix(trimmed, "|") || strings.Contains(trimmed, "|---") {
|
||||
continue
|
||||
}
|
||||
cells := strings.Split(strings.Trim(trimmed, "|"), "|")
|
||||
for i := range cells {
|
||||
cells[i] = strings.TrimSpace(cells[i])
|
||||
}
|
||||
rows = append(rows, cells)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, fmt.Errorf("missing markdown table for heading %s", heading)
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func cleanBytedanceArkCell(raw string) string {
|
||||
cleaned := html.UnescapeString(strings.TrimSpace(raw))
|
||||
cleaned = strings.ReplaceAll(cleaned, `\-`, "-")
|
||||
cleaned = strings.ReplaceAll(cleaned, `\`, "")
|
||||
cleaned = strings.ReplaceAll(cleaned, "<br><br>", " ")
|
||||
cleaned = strings.ReplaceAll(cleaned, "<br />", " ")
|
||||
cleaned = strings.ReplaceAll(cleaned, "<br/>", " ")
|
||||
cleaned = strings.ReplaceAll(cleaned, "<br>", " ")
|
||||
cleaned = regexp.MustCompile(`(?is)<[^>]+>`).ReplaceAllString(cleaned, " ")
|
||||
cleaned = regexp.MustCompile(`\s+`).ReplaceAllString(cleaned, " ")
|
||||
return strings.TrimSpace(cleaned)
|
||||
}
|
||||
|
||||
func bytedanceArkPriceValue(raw string) float64 {
|
||||
cleaned := cleanBytedanceArkCell(raw)
|
||||
if cleaned == "" || strings.Contains(cleaned, "不支持") {
|
||||
return 0
|
||||
}
|
||||
match := regexp.MustCompile(`([0-9]+(?:\.[0-9]+)?)`).FindStringSubmatch(cleaned)
|
||||
if len(match) != 2 {
|
||||
return 0
|
||||
}
|
||||
return mustParseSubscriptionPrice(match[1])
|
||||
}
|
||||
|
||||
func isBytedanceArkConditionRow(value string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(value))
|
||||
return lower == "" || strings.HasPrefix(lower, "输入长度")
|
||||
}
|
||||
|
||||
func bytedanceArkProviderName(modelName string) string {
|
||||
lower := strings.ToLower(strings.TrimSpace(modelName))
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "deepseek"):
|
||||
return "DeepSeek"
|
||||
case strings.HasPrefix(lower, "glm"):
|
||||
return "Zhipu AI"
|
||||
default:
|
||||
return "ByteDance"
|
||||
}
|
||||
}
|
||||
82
scripts/import_bytedance_pricing_test.go
Normal file
82
scripts/import_bytedance_pricing_test.go
Normal file
@@ -0,0 +1,82 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseBytedanceArkPricingCatalogBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "bytedance_ark_pricing_sample.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, err := parseBytedanceArkPricingCatalog(string(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("parseBytedanceArkPricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 20 {
|
||||
t.Fatalf("期望 20 条火山方舟价格记录,实际 %d", len(records))
|
||||
}
|
||||
recordMap := make(map[string]officialPricingRecord, len(records))
|
||||
for _, record := range records {
|
||||
recordMap[record.ModelID] = record
|
||||
}
|
||||
if recordMap["bytedance-doubao-seed-2-0-pro"].InputPrice != 3.2 || recordMap["bytedance-doubao-seed-2-0-pro"].OutputPrice != 16.0 {
|
||||
t.Fatalf("doubao-seed-2.0-pro 基线价格错误: %+v", recordMap["bytedance-doubao-seed-2-0-pro"])
|
||||
}
|
||||
if recordMap["bytedance-doubao-seed-1-8"].OutputPrice != 2.0 {
|
||||
t.Fatalf("doubao-seed-1.8 应取首个阶梯输出价 2.0,实际 %v", recordMap["bytedance-doubao-seed-1-8"].OutputPrice)
|
||||
}
|
||||
if recordMap["bytedance-glm-4-7"].ProviderName != "Zhipu AI" || recordMap["bytedance-glm-4-7"].InputPrice != 2.0 || recordMap["bytedance-glm-4-7"].OutputPrice != 8.0 {
|
||||
t.Fatalf("glm-4.7 解析错误: %+v", recordMap["bytedance-glm-4-7"])
|
||||
}
|
||||
if recordMap["bytedance-deepseek-v3-2"].ProviderName != "DeepSeek" {
|
||||
t.Fatalf("deepseek-v3.2 provider 归一化错误: %+v", recordMap["bytedance-deepseek-v3-2"])
|
||||
}
|
||||
if recordMap["bytedance-doubao-1-5-vision-pro"].Modality != "multimodal" {
|
||||
t.Fatalf("doubao-1.5-vision-pro modality 错误: %+v", recordMap["bytedance-doubao-1-5-vision-pro"])
|
||||
}
|
||||
if _, ok := recordMap["bytedance-doubao-embedding-vision"]; ok {
|
||||
t.Fatalf("当前 schema 不支持仅输入计费的向量模型,不应误入库")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractBytedanceArkPricingMarkdownFromRouterData(t *testing.T) {
|
||||
html := `<script>window._ROUTER_DATA = {"loaderData":{"docs/(libid)/(docid$)/page":{"curDoc":{"MDContent":"## 在线推理(常规)\n|模型名称 |条件 |输入 |缓存存储 |缓存输入 |输出 |\n|---|---|---|---|---|---|\n|doubao\\-seed\\-2.0\\-mini |输入长度 [0, 32] |0.2 |0.017 |0.04 |2.0 |"}}}}</script>`
|
||||
markdown, err := extractBytedanceArkPricingMarkdown(html)
|
||||
if err != nil {
|
||||
t.Fatalf("extractBytedanceArkPricingMarkdown 返回错误: %v", err)
|
||||
}
|
||||
if !strings.Contains(markdown, "doubao\\-seed\\-2.0\\-mini") {
|
||||
t.Fatalf("提取后的 markdown 缺少模型行: %q", markdown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBytedanceArkPricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runBytedanceArkPricingImport(bytedanceArkPricingImportConfig{
|
||||
URL: defaultBytedanceArkPricingURL,
|
||||
Fixture: filepath.Join("testdata", "bytedance_ark_pricing_sample.txt"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runBytedanceArkPricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=bytedance-pricing-import",
|
||||
"models=20",
|
||||
"operator=ByteDance Volcano",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
320
scripts/import_cucloud_pricing.go
Normal file
320
scripts/import_cucloud_pricing.go
Normal file
@@ -0,0 +1,320 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultCUCloudPricingURL = "https://support.cucloud.cn/document/127/591/2357.html?id=2357&folderid=3236"
|
||||
|
||||
type cucloudPricingImportConfig struct {
|
||||
URL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type cucloudPricingSummary struct {
|
||||
Models int
|
||||
Records int
|
||||
Regions int
|
||||
PaygModeConfirmed bool
|
||||
PaygPriceTablePublic bool
|
||||
}
|
||||
|
||||
var cucloudRequiredModels = []string{"DeepSeek-V4-Pro", "DeepSeek-V4-Flash", "MiniMax-M2.5"}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var url string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&url, "url", defaultCUCloudPricingURL, "联通云 AISP Token Plan 页面")
|
||||
flag.StringVar(&fixture, "fixture", "", "联通云价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := cucloudPricingImportConfig{URL: url, Fixture: fixture, DryRun: dryRun, Timeout: time.Duration(timeoutSeconds) * time.Second}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runCUCloudPricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_cucloud_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runCUCloudPricingImport(cfg cucloudPricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
raw, err := fetchRawPricingPage(cfg.URL, cfg.Fixture, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, summary, err := parseCUCloudPricingCatalog(raw, cfg.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=cucloud-pricing-import models=%d records=%d regions=%d operator=%s payg_mode_confirmed=%t payg_price_table_public=%t dry_run=true\n",
|
||||
summary.Models, summary.Records, summary.Regions, records[0].OperatorName, summary.PaygModeConfirmed, summary.PaygPriceTablePublic)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "cucloud-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=cucloud-pricing-import models=%d records=%d regions=%d operator=%s table_rows=%d payg_mode_confirmed=%t payg_price_table_public=%t dry_run=false\n",
|
||||
summary.Models, summary.Records, summary.Regions, records[0].OperatorName, tableRows, summary.PaygModeConfirmed, summary.PaygPriceTablePublic)
|
||||
return err
|
||||
}
|
||||
|
||||
func parseCUCloudPricingCatalog(raw string, sourceURL string) ([]officialPricingRecord, cucloudPricingSummary, error) {
|
||||
normalized := normalizeCUCloudRaw(raw)
|
||||
priceMap, err := extractCUCloudBlendedPrices(normalized)
|
||||
if err != nil {
|
||||
return nil, cucloudPricingSummary{}, err
|
||||
}
|
||||
regionMap, err := extractCUCloudRegionSupport(normalized)
|
||||
if err != nil {
|
||||
return nil, cucloudPricingSummary{}, err
|
||||
}
|
||||
|
||||
records := make([]officialPricingRecord, 0)
|
||||
modelSet := make(map[string]struct{})
|
||||
regionSet := make(map[string]struct{})
|
||||
for _, modelName := range cucloudRequiredModels {
|
||||
price, ok := priceMap[modelName]
|
||||
if !ok {
|
||||
return nil, cucloudPricingSummary{}, fmt.Errorf("missing blended price for %s", modelName)
|
||||
}
|
||||
regions := regionMap[modelName]
|
||||
if len(regions) == 0 {
|
||||
return nil, cucloudPricingSummary{}, fmt.Errorf("missing supported regions for %s", modelName)
|
||||
}
|
||||
providerName := cucloudProviderName(modelName)
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
for _, region := range regions {
|
||||
records = append(records, officialPricingRecord{
|
||||
ModelID: normalizeExternalID("cucloud", "aisp", modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Unicom AISP",
|
||||
OperatorNameCn: "联通云 AI服务平台AISP",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.cucloud.cn",
|
||||
OperatorType: "official",
|
||||
Region: region,
|
||||
Currency: "CNY",
|
||||
InputPrice: price,
|
||||
OutputPrice: price,
|
||||
SourceURL: sourceURL,
|
||||
ModelSourceURL: sourceURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(modelName),
|
||||
})
|
||||
regionSet[region] = struct{}{}
|
||||
}
|
||||
modelSet[modelName] = struct{}{}
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, cucloudPricingSummary{}, fmt.Errorf("no cucloud pricing records found")
|
||||
}
|
||||
summary := cucloudPricingSummary{
|
||||
Models: len(modelSet),
|
||||
Records: len(records),
|
||||
Regions: len(regionSet),
|
||||
PaygModeConfirmed: cucloudPaygModeConfirmed(normalized),
|
||||
PaygPriceTablePublic: cucloudHasPublicPaygPriceTable(normalized),
|
||||
}
|
||||
return records, summary, nil
|
||||
}
|
||||
|
||||
func normalizeCUCloudRaw(raw string) string {
|
||||
raw = strings.ReplaceAll(raw, `\u003c`, "<")
|
||||
raw = strings.ReplaceAll(raw, `\u003e`, ">")
|
||||
raw = strings.ReplaceAll(raw, `\u0026nbsp;`, " ")
|
||||
raw = strings.ReplaceAll(raw, `\n`, "\n")
|
||||
raw = strings.ReplaceAll(raw, `\t`, " ")
|
||||
raw = strings.ReplaceAll(raw, `\r`, "\n")
|
||||
raw = html.UnescapeString(raw)
|
||||
return raw
|
||||
}
|
||||
|
||||
func extractCUCloudBlendedPrices(raw string) (map[string]float64, error) {
|
||||
for _, table := range cucloudTableBlocks(raw) {
|
||||
rows := cucloudTableRows(table)
|
||||
if len(rows) == 0 {
|
||||
continue
|
||||
}
|
||||
prices := make(map[string]float64)
|
||||
for _, cell := range rows[0] {
|
||||
modelName, price, ok := cucloudBlendedPriceCell(cell)
|
||||
if ok {
|
||||
prices[modelName] = price
|
||||
}
|
||||
}
|
||||
if cucloudHasAllRequiredModels(prices) {
|
||||
return prices, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected cucloud blended price table")
|
||||
}
|
||||
|
||||
func cucloudBlendedPriceCell(raw string) (string, float64, bool) {
|
||||
cleaned := strings.TrimSpace(cleanHTMLText(raw))
|
||||
match := regexp.MustCompile(`^(.*?)\s*综合单价\s*([0-9]+(?:\.[0-9]+)?)元/百万tokens$`).FindStringSubmatch(cleaned)
|
||||
if len(match) != 3 {
|
||||
return "", 0, false
|
||||
}
|
||||
modelName := strings.TrimSpace(match[1])
|
||||
if modelName == "" {
|
||||
return "", 0, false
|
||||
}
|
||||
return modelName, mustParseSubscriptionPrice(match[2]), true
|
||||
}
|
||||
|
||||
func extractCUCloudRegionSupport(raw string) (map[string][]string, error) {
|
||||
for _, table := range cucloudTableBlocks(raw) {
|
||||
rows := cucloudTableRows(table)
|
||||
if len(rows) < 2 {
|
||||
continue
|
||||
}
|
||||
headers := rows[0]
|
||||
if len(headers) < 2 || strings.TrimSpace(headers[0]) != "模型" {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(strings.Join(headers, "|"), "贵阳基地二区") {
|
||||
continue
|
||||
}
|
||||
regionMap := make(map[string][]string)
|
||||
regions := headers[1:]
|
||||
for _, row := range rows[1:] {
|
||||
if len(row) < len(regions)+1 {
|
||||
continue
|
||||
}
|
||||
modelName := strings.TrimSpace(row[0])
|
||||
if modelName == "" {
|
||||
continue
|
||||
}
|
||||
supported := make([]string, 0)
|
||||
for idx, region := range regions {
|
||||
if strings.Contains(strings.TrimSpace(row[idx+1]), "支持") {
|
||||
supported = append(supported, strings.TrimSpace(region))
|
||||
}
|
||||
}
|
||||
if len(supported) > 0 {
|
||||
regionMap[modelName] = supported
|
||||
}
|
||||
}
|
||||
if cucloudHasAllRequiredRegionRows(regionMap) {
|
||||
return regionMap, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected cucloud region support table")
|
||||
}
|
||||
|
||||
func cucloudTableBlocks(raw string) []string {
|
||||
pattern := regexp.MustCompile(`(?is)<table[^>]*>.*?</table>`)
|
||||
return pattern.FindAllString(raw, -1)
|
||||
}
|
||||
|
||||
func cucloudTableRows(table string) [][]string {
|
||||
rowPattern := regexp.MustCompile(`(?is)<tr[^>]*>(.*?)</tr>`)
|
||||
cellPattern := regexp.MustCompile(`(?is)<t[dh][^>]*>(.*?)</t[dh]>`)
|
||||
matches := rowPattern.FindAllStringSubmatch(table, -1)
|
||||
rows := make([][]string, 0, len(matches))
|
||||
for _, rowMatch := range matches {
|
||||
cells := cellPattern.FindAllStringSubmatch(rowMatch[1], -1)
|
||||
if len(cells) == 0 {
|
||||
continue
|
||||
}
|
||||
row := make([]string, 0, len(cells))
|
||||
for _, cell := range cells {
|
||||
row = append(row, strings.TrimSpace(cleanHTMLText(cell[1])))
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func cucloudPaygModeConfirmed(raw string) bool {
|
||||
text := cleanHTMLText(raw)
|
||||
return strings.Contains(text, "按量计费") && (strings.Contains(text, "元/千 Tokens") || strings.Contains(text, "元/千Tokens"))
|
||||
}
|
||||
|
||||
func cucloudHasPublicPaygPriceTable(raw string) bool {
|
||||
for _, table := range cucloudTableBlocks(raw) {
|
||||
text := cleanHTMLText(table)
|
||||
if !(strings.Contains(text, "元/千 Tokens") || strings.Contains(text, "元/千Tokens")) {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(text, "DeepSeek-V4-Pro") || strings.Contains(text, "MiniMax-M2.5") || strings.Contains(text, "DeepSeek-V4-Flash") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func cucloudProviderName(modelName string) string {
|
||||
lower := strings.ToLower(strings.TrimSpace(modelName))
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "deepseek"):
|
||||
return "DeepSeek"
|
||||
case strings.HasPrefix(lower, "minimax"):
|
||||
return "MiniMax"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
func cucloudHasAllRequiredModels(prices map[string]float64) bool {
|
||||
for _, modelName := range cucloudRequiredModels {
|
||||
if _, ok := prices[modelName]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func cucloudHasAllRequiredRegionRows(regionMap map[string][]string) bool {
|
||||
for _, modelName := range cucloudRequiredModels {
|
||||
if len(regionMap[modelName]) == 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
95
scripts/import_cucloud_pricing_test.go
Normal file
95
scripts/import_cucloud_pricing_test.go
Normal file
@@ -0,0 +1,95 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseCUCloudPricingBuildsBlendedRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "cucloud_pricing_sample.html"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
|
||||
records, summary, err := parseCUCloudPricingCatalog(string(raw), defaultCUCloudPricingURL)
|
||||
if err != nil {
|
||||
t.Fatalf("parseCUCloudPricingCatalog 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 4 {
|
||||
t.Fatalf("期望 4 条联通云价格记录,实际 %d", len(records))
|
||||
}
|
||||
if summary.Models != 3 || summary.Records != 4 || summary.Regions != 2 {
|
||||
t.Fatalf("summary 计数错误: %+v", summary)
|
||||
}
|
||||
if !summary.PaygModeConfirmed || summary.PaygPriceTablePublic {
|
||||
t.Fatalf("payg 摘要错误: %+v", summary)
|
||||
}
|
||||
|
||||
recordMap := make(map[string]officialPricingRecord, len(records))
|
||||
for _, record := range records {
|
||||
recordMap[record.ModelName+"|"+record.Region] = record
|
||||
}
|
||||
|
||||
pro := recordMap["DeepSeek-V4-Pro|贵阳基地二区"]
|
||||
if pro.InputPrice != 9.3 || pro.OutputPrice != 9.3 || pro.ProviderName != "DeepSeek" {
|
||||
t.Fatalf("DeepSeek-V4-Pro 记录错误: %+v", pro)
|
||||
}
|
||||
flash := recordMap["DeepSeek-V4-Flash|贵阳基地二区"]
|
||||
if flash.InputPrice != 0.7 || flash.OutputPrice != 0.7 || flash.ProviderName != "DeepSeek" {
|
||||
t.Fatalf("DeepSeek-V4-Flash 记录错误: %+v", flash)
|
||||
}
|
||||
miniWuhan := recordMap["MiniMax-M2.5|武汉四区"]
|
||||
if miniWuhan.InputPrice != 1.1 || miniWuhan.OutputPrice != 1.1 || miniWuhan.ProviderName != "MiniMax" {
|
||||
t.Fatalf("MiniMax-M2.5 武汉记录错误: %+v", miniWuhan)
|
||||
}
|
||||
miniGuiyang := recordMap["MiniMax-M2.5|贵阳基地二区"]
|
||||
if miniGuiyang.InputPrice != 1.1 || miniGuiyang.OutputPrice != 1.1 {
|
||||
t.Fatalf("MiniMax-M2.5 贵阳记录错误: %+v", miniGuiyang)
|
||||
}
|
||||
if _, ok := recordMap["MiniMax-M2.5|上海二十二区"]; ok {
|
||||
t.Fatalf("不支持的区域不应生成记录")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCUCloudPricingRequiresBlendedPriceTable(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "cucloud_pricing_sample.html"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
broken := strings.Replace(string(raw), "综合单价9.30元/百万tokens", "综合单价元/百万tokens", 1)
|
||||
if _, _, err := parseCUCloudPricingCatalog(broken, defaultCUCloudPricingURL); err == nil {
|
||||
t.Fatalf("缺少 blended 价格时应返回错误")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCUCloudPricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runCUCloudPricingImport(cucloudPricingImportConfig{
|
||||
URL: defaultCUCloudPricingURL,
|
||||
Fixture: filepath.Join("testdata", "cucloud_pricing_sample.html"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runCUCloudPricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=cucloud-pricing-import",
|
||||
"models=3",
|
||||
"records=4",
|
||||
"regions=2",
|
||||
"operator=Unicom AISP",
|
||||
"payg_mode_confirmed=true",
|
||||
"payg_price_table_public=false",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -24,8 +24,8 @@ func TestBuildPlanCatalogRows(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("buildPlanCatalogRows 失败: %v", err)
|
||||
}
|
||||
if len(rows) != 70 {
|
||||
t.Fatalf("期望 70 条基础目录记录,实际 %d", len(rows))
|
||||
if len(rows) != 71 {
|
||||
t.Fatalf("期望 71 条基础目录记录,实际 %d", len(rows))
|
||||
}
|
||||
|
||||
foundVendorTop20 := false
|
||||
@@ -38,6 +38,7 @@ func TestBuildPlanCatalogRows(t *testing.T) {
|
||||
"aliyun-bailian-coding-plan": "import_aliyun_subscription.go",
|
||||
"baidu-qianfan-token-benefit-pack": "import_baidu_subscription.go",
|
||||
"baidu-qianfan-coding-plan": "import_baidu_subscription.go",
|
||||
"bytedance-doubao-api-payg": "import_bytedance_pricing.go",
|
||||
"zhipu-glm-coding-plan": "import_zhipu_coding_plan.go",
|
||||
"minimax-token-plan": "import_minimax_subscription.go",
|
||||
"volcengine-ark-coding-plan": "import_bytedance_subscription.go",
|
||||
@@ -46,7 +47,8 @@ func TestBuildPlanCatalogRows(t *testing.T) {
|
||||
"ctyun-coding-plan": "import_ctyun_subscription.go",
|
||||
"cucloud-aicp-platform": "import_cucloud_catalog.go",
|
||||
"cucloud-ai-app-platform": "import_cucloud_catalog.go",
|
||||
"mobile-cloud-ai-market": "import_mobile_cloud_catalog.go",
|
||||
"cucloud-aisp-token-plan-pricing": "import_cucloud_pricing.go",
|
||||
"mobile-cloud-ai-market": "import_mobile_cloud_pricing.go",
|
||||
"aliyun-modelscope-api-inference": "import_catalog_seed_verification.go",
|
||||
"youdao-zhiyun-maas": "import_youdao_pricing.go",
|
||||
"ctyun-model-inference-payg": "import_catalog_seed_verification.go",
|
||||
@@ -135,13 +137,13 @@ func TestRunPlanCatalogImportDryRunPrintsSummary(t *testing.T) {
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=plan-catalog-import",
|
||||
"rows=70",
|
||||
"rows=71",
|
||||
"coding_plan:7",
|
||||
"package_plan:1",
|
||||
"pay_as_you_go:51",
|
||||
"token_plan:7",
|
||||
"unknown:4",
|
||||
"confirmed:70",
|
||||
"pay_as_you_go:52",
|
||||
"token_plan:8",
|
||||
"unknown:3",
|
||||
"confirmed:71",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
|
||||
@@ -30,7 +30,13 @@ printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=qwen-fixture'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=qwen-live'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=hunyuan-fixture'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=hunyuan-live'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=mobile-cloud-fixture'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=mobile-cloud-live'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=cucloud-pricing-fixture'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=cucloud-pricing-live'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=huawei-maas-fixture'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=huawei-maas-live'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=bytedance-fixture'
|
||||
printf '%s' "$PASS_OUTPUT" | grep -q '\[PASS\] importer_smoke=bytedance-live'
|
||||
|
||||
echo "importer_smoke_gate_test: PASS"
|
||||
|
||||
@@ -25,18 +25,30 @@ check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "tencent_subscri
|
||||
check_contains "scripts/run_daily.sh" 'error_exit "腾讯云套餐导入失败"'
|
||||
check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "qwen_pricing" "通义千问价格导入失败"'
|
||||
check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "hunyuan_pricing" "腾讯混元价格导入失败"'
|
||||
check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "mobile_cloud_pricing" "移动云 MoMA 价格导入失败"'
|
||||
check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "cucloud_pricing" "联通云 Token Plan 价格导入失败"'
|
||||
check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "huawei_maas_pricing" "华为云 MaaS 价格导入失败"'
|
||||
check_contains "scripts/run_intel_pipeline.sh" 'run_or_fail "bytedance_pricing" "火山方舟价格导入失败"'
|
||||
check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "qwen_pricing"'
|
||||
check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "hunyuan_pricing"'
|
||||
check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "mobile_cloud_pricing"'
|
||||
check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "cucloud_pricing"'
|
||||
check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "huawei_maas_pricing"'
|
||||
check_contains "scripts/run_real_pipeline.sh" 'merge_failed_source_keys "bytedance_pricing"'
|
||||
check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "qwen_pricing"'
|
||||
check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "hunyuan_pricing"'
|
||||
check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "mobile_cloud_pricing"'
|
||||
check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "cucloud_pricing"'
|
||||
check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "huawei_maas_pricing"'
|
||||
check_contains "scripts/run_daily.sh" 'merge_failed_source_keys "bytedance_pricing"'
|
||||
|
||||
|
||||
check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "tencent-live"'
|
||||
check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "qwen-fixture"'
|
||||
check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "hunyuan-fixture"'
|
||||
check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "mobile-cloud-fixture"'
|
||||
check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "cucloud-pricing-fixture"'
|
||||
check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "huawei-maas-fixture"'
|
||||
check_contains "scripts/verify_importer_smoke.sh" 'run_smoke "bytedance-fixture"'
|
||||
|
||||
echo "pipeline_runtime_alignment_test: PASS"
|
||||
|
||||
@@ -22,7 +22,7 @@ MODEL_COUNT=""
|
||||
FETCH_OUT="${PROJECT_DIR}/models.json"
|
||||
FETCH_TOTAL="0"
|
||||
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot,daily_report"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,catalog_seed_verification"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,cucloud_pricing,mobile_cloud_pricing,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,bytedance_pricing,catalog_seed_verification"
|
||||
PIPELINE_FAILED_SOURCE_SET="none"
|
||||
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
|
||||
PIPELINE_AUDIT_SUMMARY=""
|
||||
@@ -240,10 +240,17 @@ if ! go run -tags llm_script \
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/catalog_verification_common.go \
|
||||
scripts/import_mobile_cloud_catalog.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "mobile_cloud_catalog"
|
||||
error_exit "移动云目录校验失败"
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/import_cucloud_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "cucloud_pricing"
|
||||
error_exit "联通云 Token Plan 价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/import_mobile_cloud_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "mobile_cloud_pricing"
|
||||
error_exit "移动云 MoMA 价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
@@ -412,6 +419,13 @@ if ! go run -tags llm_script \
|
||||
merge_failed_source_keys "huawei_maas_pricing"
|
||||
error_exit "华为云 MaaS 价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/official_pricing_import_common.go \
|
||||
scripts/import_bytedance_pricing.go >> "$LOG_FILE" 2>&1; then
|
||||
merge_failed_source_keys "bytedance_pricing"
|
||||
error_exit "火山方舟价格导入失败"
|
||||
fi
|
||||
if ! go run -tags llm_script \
|
||||
scripts/subscription_import_common.go \
|
||||
scripts/import_catalog_seed_verification.go >> "$LOG_FILE" 2>&1; then
|
||||
|
||||
@@ -27,7 +27,7 @@ REPORT_DATE="${REPORT_DATE:-$(date +%F)}"
|
||||
FETCH_OUT="$ROOT_DIR/models.json"
|
||||
FETCH_TOTAL="0"
|
||||
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,catalog_seed_verification"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,cucloud_pricing,mobile_cloud_pricing,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,bytedance_pricing,catalog_seed_verification"
|
||||
PIPELINE_FAILED_SOURCE_SET="none"
|
||||
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
|
||||
PIPELINE_AUDIT_SUMMARY=""
|
||||
@@ -140,8 +140,10 @@ run_or_fail "minimax_subscription" "MiniMax Token Plan 导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/minimax_subscription_lib.go ./scripts/import_minimax_subscription.go
|
||||
run_or_fail "cucloud_catalog" "联通云目录校验失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/catalog_verification_common.go ./scripts/import_cucloud_catalog.go
|
||||
run_or_fail "mobile_cloud_catalog" "移动云目录校验失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/catalog_verification_common.go ./scripts/import_mobile_cloud_catalog.go
|
||||
run_or_fail "cucloud_pricing" "联通云 Token Plan 价格导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_cucloud_pricing.go
|
||||
run_or_fail "mobile_cloud_pricing" "移动云 MoMA 价格导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_mobile_cloud_pricing.go
|
||||
run_or_fail "tencent_subscription" "腾讯云套餐导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/tencent_catalog_lib.go ./scripts/import_tencent_subscription.go
|
||||
run_or_fail "youdao_pricing" "网易有道价格导入失败" \
|
||||
@@ -179,6 +181,8 @@ run_or_fail "hunyuan_pricing" "腾讯混元价格导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_hunyuan_pricing.go
|
||||
run_or_fail "huawei_maas_pricing" "华为云 MaaS 价格导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_huawei_maas_pricing.go
|
||||
run_or_fail "bytedance_pricing" "火山方舟价格导入失败" \
|
||||
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_bytedance_pricing.go
|
||||
|
||||
refresh_pipeline_audit
|
||||
run_or_fail "catalog_seed_verification" "目录级官方入口核验失败" \
|
||||
|
||||
@@ -28,7 +28,7 @@ REPORT_DATE="$(report_date_value)"
|
||||
FETCH_OUT="$ROOT_DIR/models.json"
|
||||
FETCH_TOTAL="0"
|
||||
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot,daily_report"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,catalog_seed_verification"
|
||||
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,tencent_subscription,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,cucloud_pricing,mobile_cloud_pricing,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,coreshub_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,qwen_pricing,hunyuan_pricing,huawei_maas_pricing,bytedance_pricing,catalog_seed_verification"
|
||||
PIPELINE_FAILED_SOURCE_SET="none"
|
||||
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
|
||||
PIPELINE_AUDIT_SUMMARY=""
|
||||
@@ -189,9 +189,14 @@ if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./script
|
||||
record_failure "联通云目录校验失败"
|
||||
exit 1
|
||||
fi
|
||||
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/catalog_verification_common.go" "./scripts/import_mobile_cloud_catalog.go"; then
|
||||
merge_failed_source_keys "mobile_cloud_catalog"
|
||||
record_failure "移动云目录校验失败"
|
||||
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_cucloud_pricing.go"; then
|
||||
merge_failed_source_keys "cucloud_pricing"
|
||||
record_failure "联通云 Token Plan 价格导入失败"
|
||||
exit 1
|
||||
fi
|
||||
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_mobile_cloud_pricing.go"; then
|
||||
merge_failed_source_keys "mobile_cloud_pricing"
|
||||
record_failure "移动云 MoMA 价格导入失败"
|
||||
exit 1
|
||||
fi
|
||||
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/tencent_catalog_lib.go" "./scripts/import_tencent_subscription.go"; then
|
||||
@@ -284,6 +289,11 @@ if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./script
|
||||
record_failure "华为云 MaaS 价格导入失败"
|
||||
exit 1
|
||||
fi
|
||||
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/import_bytedance_pricing.go"; then
|
||||
merge_failed_source_keys "bytedance_pricing"
|
||||
record_failure "火山方舟价格导入失败"
|
||||
exit 1
|
||||
fi
|
||||
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/import_catalog_seed_verification.go"; then
|
||||
merge_failed_source_keys "catalog_seed_verification"
|
||||
record_failure "目录级官方入口核验失败"
|
||||
|
||||
59
scripts/testdata/bytedance_ark_pricing_sample.txt
vendored
Normal file
59
scripts/testdata/bytedance_ark_pricing_sample.txt
vendored
Normal file
@@ -0,0 +1,59 @@
|
||||
# 大语言模型
|
||||
|
||||
## 在线推理(常规)
|
||||
|
||||
|模型名称 |条件<br><br>千 token |输入<br><br>元/百万token |缓存存储<br><br>元/百万 token /小时 |缓存输入<br><br>元/百万token |输出<br><br>元/百万token |
|
||||
|---|---|---|---|---|---|
|
||||
|doubao\-seed\-2.0\-pro |输入长度 [0, 32] |3.2 |0.017 |0.64 |16.0 |
|
||||
||输入长度 (32, 128] |4.8 |0.017 |0.96 |24.0 |
|
||||
||输入长度 (128, 256] |9.6 |0.017 |1.92 |48.0 |
|
||||
|doubao\-seed\-2.0\-lite |输入长度 [0, 32] |0.6 |0.017 |0.12 |3.6 |
|
||||
||输入长度 (32, 128] |0.9 |0.017 |0.18 |5.4 |
|
||||
||输入长度 (128, 256] |1.8 |0.017 |0.36 |10.8 |
|
||||
|doubao\-seed\-2.0\-mini |输入长度 [0, 32] |0.2 |0.017 |0.04 |2.0 |
|
||||
||输入长度 (32, 128] |0.4 |0.017 |0.08 |4.0 |
|
||||
||输入长度 (128, 256] |0.8 |0.017 |0.16 |8.0 |
|
||||
|doubao\-seed\-2.0\-code |输入长度 [0, 32] |3.2 |0.017 |0.64 |16.0 |
|
||||
||输入长度 (32, 128] |4.8 |0.017 |0.96 |24.0 |
|
||||
||输入长度 (128, 256] |9.6 |0.017 |1.92 |48.0 |
|
||||
|doubao\-seed\-1.8 |输入长度 [0, 32]<br><br>且输出长度 [0, 0.2] |0.80 |0.017 |0.16 |2.00 |
|
||||
||输入长度 [0, 32]<br><br>且输出长度 (0.2,+∞) |0.80 |0.017 |0.16 |8.00 |
|
||||
||输入长度 (32, 128] |1.20 |0.017 |0.16 |16.00 |
|
||||
||输入长度 (128, 256] |2.40 |0.017 |0.16 |24.00 |
|
||||
|doubao\-seed\-character |输入长度 [0, 32] |0.80 |0.017 |0.16 |2.00 |
|
||||
||输入长度 (32, 128] |1.20 |0.017 |0.16 |6.00 |
|
||||
|doubao\-seed\-code |输入长度 [0, 32] |1.20 |0.017 |0.24 |8.00 |
|
||||
||输入长度 (32, 128] |1.40 |0.017 |0.24 |12.00 |
|
||||
||输入长度 (128, 256] |2.80 |0.017 |0.24 |16.00 |
|
||||
|doubao\-seed\-1.6 |输入长度 [0, 32]<br><br>且输出长度 [0, 0.2] |0.80 |0.017 |0.16 |2.00 |
|
||||
||输入长度 [0, 32]<br><br>且输出长度 (0.2,+∞) |0.80 |0.017 |0.16 |8.00 |
|
||||
||输入长度 (32, 128] |1.20 |0.017 |0.16 |16.00 |
|
||||
||输入长度 (128, 256] |2.40 |0.017 |0.16 |24.00 |
|
||||
|doubao\-seed\-1.6\-lite |输入长度 [0, 32]<br><br>且输出长度 [0, 0.2] |0.30 |0.017 |0.06 |0.60 |
|
||||
||输入长度 [0, 32]<br><br>且输出长度 (0.2,+∞) |0.30 |0.017 |0.06 |2.40 |
|
||||
||输入长度 (32, 128] |0.60 |0.017 |0.06 |4.00 |
|
||||
||输入长度 (128, 256] |1.20 |0.017 |0.06 |12.00 |
|
||||
|doubao\-seed\-1.6\-flash |输入长度 [0, 32] |0.15 |0.017 |0.03 |1.50 |
|
||||
||输入长度 (32, 128] |0.30 |0.017 |0.03 |3.00 |
|
||||
||输入长度 (128, 256] |0.60 |0.017 |0.03 |6.00 |
|
||||
|doubao\-seed\-1.6\-vision |输入长度 [0, 32] |0.80 |0.017 |0.16 |8.00 |
|
||||
||输入长度 (32, 128] |1.20 |0.017 |0.16 |16.00 |
|
||||
||输入长度 (128, 256] |2.40 |0.017 |0.16 |24.00 |
|
||||
|doubao\-seed\-translation |\- |1.20 |不支持 |不支持 |3.60 |
|
||||
|doubao\-1.5\-pro\-32k |\- |0.80 |0.017 |0.16 |2.00 |
|
||||
|doubao\-1.5\-lite\-32k |\- |0.30 |0.017 |0.06 |0.60 |
|
||||
|doubao\-1.5\-vision\-pro |\- |3.00 |不支持 |不支持 |9.00 |
|
||||
|glm\-4.7 |输入长度 [0, 32]<br><br>且输出长度 [0, 0.2] |2.0 |0.017 |0.4 |8.0 |
|
||||
||输入长度 [0, 32]<br><br>且输出长度 (0.2,+∞) |3.0 |0.017 |0.6 |14.0 |
|
||||
||输入长度 (32, 200] |4.0 |0.017 |0.8 |16.0 |
|
||||
|deepseek\-v3.2 |输入长度 [0, 32] |2.00 |0.017 |0.4 |3.00 |
|
||||
||输入长度 (32, 128] |4.00 |0.017 |0.4 |6.00 |
|
||||
|deepseek\-v3.1 |\- |4.00 |0.017 |0.80 |12.00 |
|
||||
|deepseek\-v3 |\- |2.00 |0.017 |0.40 |8.00 |
|
||||
|deepseek\-r1 |\- |4.00 |0.017 |0.80 |16.00 |
|
||||
|
||||
# 向量模型
|
||||
|
||||
|模型 |文本输入<br><br>元/百万 token |图片输入<br><br>元/百万 token |
|
||||
|---|---|---|
|
||||
|doubao\-embedding\-vision |0.70 |1.80 |
|
||||
62
scripts/testdata/cucloud_pricing_sample.html
vendored
Normal file
62
scripts/testdata/cucloud_pricing_sample.html
vendored
Normal file
@@ -0,0 +1,62 @@
|
||||
<section>
|
||||
<h1>Token Plan概述</h1>
|
||||
<table cellspacing="0">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>套餐额度</td>
|
||||
<td><p><strong>DeepSeek-V4-Pro</strong></p><p>综合单价9.30元/百万tokens</p></td>
|
||||
<td><p><strong>DeepSeek-V4-Flash</strong></p><p>综合单价0.70元/百万tokens</p></td>
|
||||
<td><p><strong>MiniMax-M2.5</strong></p><p>综合单价1.10元/百万tokens</p></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>25,000 credits</td>
|
||||
<td>约 27 百万tokens</td>
|
||||
<td>约 357 百万tokens</td>
|
||||
<td>约 227 百万tokens</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
<section>
|
||||
<h1>各云区域模型支持情况</h1>
|
||||
<table cellspacing="0">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>模型</td>
|
||||
<td>呼和浩特二区</td>
|
||||
<td>上海二十二区</td>
|
||||
<td>武汉四区</td>
|
||||
<td>济南五区</td>
|
||||
<td>贵阳基地二区</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>DeepSeek-V4-Pro</td>
|
||||
<td><br/></td>
|
||||
<td><br/></td>
|
||||
<td><br/></td>
|
||||
<td><br/></td>
|
||||
<td>支持</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>DeepSeek-V4-Flash</td>
|
||||
<td><br/></td>
|
||||
<td><br/></td>
|
||||
<td><br/></td>
|
||||
<td><br/></td>
|
||||
<td>支持</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>MiniMax-M2.5</td>
|
||||
<td><br/></td>
|
||||
<td><br/></td>
|
||||
<td>支持</td>
|
||||
<td><br/></td>
|
||||
<td>支持</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
<section>
|
||||
<h1>按量计费模式</h1>
|
||||
<p>按量计费按模型销售价实时累加,计费单位为元/千 Tokens。</p>
|
||||
</section>
|
||||
@@ -11,6 +11,9 @@ CTYUN_TOKEN_FIXTURE_PATH="${CTYUN_TOKEN_FIXTURE_PATH:-./scripts/testdata/ctyun_t
|
||||
TENCENT_FIXTURE_PATH="${TENCENT_FIXTURE_PATH:-./scripts/testdata/tencent_token_plan_sample.txt}"
|
||||
QWEN_FIXTURE_PATH="${QWEN_FIXTURE_PATH:-./scripts/testdata/qwen_pricing_sample.txt}"
|
||||
HUNYUAN_FIXTURE_PATH="${HUNYUAN_FIXTURE_PATH:-./scripts/testdata/hunyuan_pricing_sample.txt}"
|
||||
MOBILE_CLOUD_FIXTURE_PATH="${MOBILE_CLOUD_FIXTURE_PATH:-./scripts/testdata/mobile_cloud_pricing_sample.html}"
|
||||
CUCLOUD_FIXTURE_PATH="${CUCLOUD_FIXTURE_PATH:-./scripts/testdata/cucloud_pricing_sample.html}"
|
||||
BYTEDANCE_FIXTURE_PATH="${BYTEDANCE_FIXTURE_PATH:-./scripts/testdata/bytedance_ark_pricing_sample.txt}"
|
||||
HUAWEI_MAAS_FIXTURE_PATH="${HUAWEI_MAAS_FIXTURE_PATH:-./scripts/testdata/huawei_maas_pricing_sample.json}"
|
||||
|
||||
last_meaningful_line() {
|
||||
@@ -48,7 +51,13 @@ run_smoke "qwen-fixture" "go run -tags llm_script ./scripts/subscription_import_
|
||||
run_smoke "qwen-live" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_qwen_pricing.go -dry-run"
|
||||
run_smoke "hunyuan-fixture" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_hunyuan_pricing.go -fixture ${HUNYUAN_FIXTURE_PATH@Q} -dry-run"
|
||||
run_smoke "hunyuan-live" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_hunyuan_pricing.go -dry-run"
|
||||
run_smoke "mobile-cloud-fixture" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_mobile_cloud_pricing.go -fixture ${MOBILE_CLOUD_FIXTURE_PATH@Q} -dry-run"
|
||||
run_smoke "mobile-cloud-live" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_mobile_cloud_pricing.go -dry-run"
|
||||
run_smoke "cucloud-pricing-fixture" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_cucloud_pricing.go -fixture ${CUCLOUD_FIXTURE_PATH@Q} -dry-run"
|
||||
run_smoke "cucloud-pricing-live" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_cucloud_pricing.go -dry-run"
|
||||
run_smoke "huawei-maas-fixture" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_huawei_maas_pricing.go -fixture ${HUAWEI_MAAS_FIXTURE_PATH@Q} -dry-run"
|
||||
run_smoke "huawei-maas-live" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_huawei_maas_pricing.go -dry-run"
|
||||
run_smoke "bytedance-fixture" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_bytedance_pricing.go -fixture ${BYTEDANCE_FIXTURE_PATH@Q} -dry-run"
|
||||
run_smoke "bytedance-live" "go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/import_bytedance_pricing.go -dry-run"
|
||||
|
||||
echo "IMPORTER_SMOKE_RESULT: PASS"
|
||||
|
||||
Reference in New Issue
Block a user