feat(region_pricing): 扩展非 token 统一计费字段,支持语音按字符/秒计费
- 新增 region_pricing.pricing_mode / price_unit / flat_price 字段 - 新增 migration 016_region_pricing_non_token_units.sql - officialPricingRecord 新增 PricingMode/PriceUnit/FlatPrice 字段 - detectModality 新增 audio 模态检测(voice/audio/speech) - providerMetadata 新增 BAAI/ByteDance/China Mobile 元数据 - import_mobile_cloud_pricing.go: 解析语音计费表(CosyVoice/SenseVoice) - CosyVoice: 2元/万字符 → pricingMode=flat, priceUnit=10k_characters - SenseVoice: 0.0007元/秒 → pricingMode=flat, priceUnit=second - mobileCloudProviderName 新增 cosyvoice/sensevoice → Alibaba 映射 - cmd/server: modelResponse 新增 pricingMode/priceUnit/flatPrice,API 字段说明同步更新 - 新增 TestModelsHandlerReturnsFlatPricingFields 测试
This commit is contained in:
@@ -22,6 +22,9 @@ type modelResponse struct {
|
||||
ProviderCN string `json:"providerCN"`
|
||||
Modality string `json:"modality"`
|
||||
ContextLength int `json:"contextLength"`
|
||||
PricingMode string `json:"pricingMode,omitempty"`
|
||||
PriceUnit string `json:"priceUnit,omitempty"`
|
||||
FlatPrice float64 `json:"flatPrice,omitempty"`
|
||||
InputPrice float64 `json:"inputPrice"`
|
||||
OutputPrice float64 `json:"outputPrice"`
|
||||
Currency string `json:"currency"`
|
||||
@@ -171,6 +174,9 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) {
|
||||
WITH latest_prices AS (
|
||||
SELECT
|
||||
rp.model_id,
|
||||
rp.pricing_mode,
|
||||
rp.price_unit,
|
||||
rp.flat_price,
|
||||
rp.input_price_per_mtok,
|
||||
rp.output_price_per_mtok,
|
||||
rp.currency,
|
||||
@@ -188,6 +194,9 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) {
|
||||
COALESCE(mp.name, split_part(m.external_id, '/', 1)),
|
||||
COALESCE(m.modality, 'text'),
|
||||
COALESCE(m.context_length, 0),
|
||||
COALESCE(lp.pricing_mode, 'input_output'),
|
||||
COALESCE(lp.price_unit, 'million_tokens'),
|
||||
COALESCE(lp.flat_price, 0),
|
||||
lp.input_price_per_mtok,
|
||||
lp.output_price_per_mtok,
|
||||
COALESCE(lp.currency, 'USD'),
|
||||
@@ -207,6 +216,7 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) {
|
||||
var models []modelResponse
|
||||
for rows.Next() {
|
||||
var model modelResponse
|
||||
var flatPrice sql.NullFloat64
|
||||
var inputPrice sql.NullFloat64
|
||||
var outputPrice sql.NullFloat64
|
||||
if err := rows.Scan(
|
||||
@@ -216,6 +226,9 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) {
|
||||
&model.Provider,
|
||||
&model.Modality,
|
||||
&model.ContextLength,
|
||||
&model.PricingMode,
|
||||
&model.PriceUnit,
|
||||
&flatPrice,
|
||||
&inputPrice,
|
||||
&outputPrice,
|
||||
&model.Currency,
|
||||
@@ -232,6 +245,9 @@ func fetchModels(ctx context.Context, db *sql.DB) ([]modelResponse, error) {
|
||||
if outputPrice.Valid {
|
||||
model.OutputPrice = outputPrice.Float64
|
||||
}
|
||||
if flatPrice.Valid {
|
||||
model.FlatPrice = flatPrice.Float64
|
||||
}
|
||||
model.Stale = model.DataConfidence == "stale"
|
||||
models = append(models, model)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,55 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestModelsHandlerReturnsFlatPricingFields(t *testing.T) {
|
||||
mux := newMux(
|
||||
&sql.DB{},
|
||||
func(context.Context, *sql.DB) ([]modelResponse, error) {
|
||||
return []modelResponse{{
|
||||
ID: "mobile-cloud-huabei-huhehaote-cosyvoice",
|
||||
Name: "CosyVoice",
|
||||
Provider: "Alibaba",
|
||||
ProviderCN: "阿里云",
|
||||
Modality: "audio",
|
||||
PricingMode: "flat",
|
||||
PriceUnit: "10k_characters",
|
||||
FlatPrice: 2,
|
||||
Currency: "CNY",
|
||||
IsFree: false,
|
||||
DataConfidence: "official",
|
||||
}}, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) ([]subscriptionPlanResponse, error) {
|
||||
return nil, nil
|
||||
},
|
||||
func(context.Context, *sql.DB) (*latestReportResponse, error) {
|
||||
return nil, sql.ErrNoRows
|
||||
},
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/models", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("expected status 200, got %d", rec.Code)
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
Data []modelResponse `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("unmarshal response: %v", err)
|
||||
}
|
||||
if len(payload.Data) != 1 {
|
||||
t.Fatalf("expected 1 model, got %d", len(payload.Data))
|
||||
}
|
||||
got := payload.Data[0]
|
||||
if got.PricingMode != "flat" || got.PriceUnit != "10k_characters" || got.FlatPrice != 2 {
|
||||
t.Fatalf("unexpected flat pricing payload: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubscriptionPlansHandlerReturnsEnvelope(t *testing.T) {
|
||||
mux := newMux(
|
||||
&sql.DB{},
|
||||
|
||||
31
db/migrations/016_region_pricing_non_token_units.sql
Normal file
31
db/migrations/016_region_pricing_non_token_units.sql
Normal file
@@ -0,0 +1,31 @@
|
||||
-- Phase 2: region_pricing 扩展非 token 统一计费字段(字符/秒等)
|
||||
|
||||
ALTER TABLE region_pricing
|
||||
ADD COLUMN IF NOT EXISTS pricing_mode TEXT NOT NULL DEFAULT 'input_output',
|
||||
ADD COLUMN IF NOT EXISTS price_unit TEXT NOT NULL DEFAULT 'million_tokens',
|
||||
ADD COLUMN IF NOT EXISTS flat_price REAL;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_constraint
|
||||
WHERE conname = 'chk_region_pricing_pricing_mode'
|
||||
) THEN
|
||||
ALTER TABLE region_pricing
|
||||
ADD CONSTRAINT chk_region_pricing_pricing_mode
|
||||
CHECK (pricing_mode IN ('input_output', 'flat'));
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
UPDATE region_pricing
|
||||
SET pricing_mode = 'input_output'
|
||||
WHERE coalesce(pricing_mode, '') = '';
|
||||
|
||||
UPDATE region_pricing
|
||||
SET price_unit = 'million_tokens'
|
||||
WHERE coalesce(price_unit, '') = '';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_region_pricing_pricing_mode ON region_pricing(pricing_mode);
|
||||
CREATE INDEX IF NOT EXISTS idx_region_pricing_price_unit ON region_pricing(price_unit);
|
||||
517
scripts/import_mobile_cloud_pricing.go
Normal file
517
scripts/import_mobile_cloud_pricing.go
Normal file
@@ -0,0 +1,517 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultMobileCloudOutlineTreeURL = "https://ecloud.10086.cn/op-help-center/request-api/service-api/outline/tree?outlineId=972"
|
||||
defaultMobileCloudArticleInfoURL = "https://ecloud.10086.cn/op-help-center/request-api/service-api/article/info/%d"
|
||||
defaultMobileCloudArticleContentURL = "https://ecloud.10086.cn/op-help-center/request-api/service-api/article/content/%s"
|
||||
defaultMobileCloudDocURLPattern = "https://ecloud.10086.cn/op-help-center/doc/article/%d"
|
||||
mobileCloudPricingArticleTitle = "预置模型服务-token按量计费"
|
||||
)
|
||||
|
||||
type mobileCloudPricingImportConfig struct {
|
||||
OutlineTreeURL string
|
||||
Fixture string
|
||||
DryRun bool
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type mobileCloudOutlineEnvelope struct {
|
||||
Code int `json:"code"`
|
||||
Data mobileCloudOutlineNode `json:"data"`
|
||||
}
|
||||
|
||||
type mobileCloudOutlineNode struct {
|
||||
ArticleID int `json:"articleId"`
|
||||
ArticleTitle string `json:"articleTitle"`
|
||||
ArticleContentPublished string `json:"articleContentPublished"`
|
||||
Children []mobileCloudOutlineNode `json:"children"`
|
||||
}
|
||||
|
||||
type mobileCloudArticleInfoEnvelope struct {
|
||||
Code int `json:"code"`
|
||||
Data mobileCloudArticleInfo `json:"data"`
|
||||
}
|
||||
|
||||
type mobileCloudArticleInfo struct {
|
||||
ID int `json:"id"`
|
||||
Title string `json:"title"`
|
||||
ContentPublished string `json:"contentPublished"`
|
||||
}
|
||||
|
||||
type mobileCloudArticlePayload struct {
|
||||
ArticleID int
|
||||
Title string
|
||||
ContentPublished string
|
||||
DocURL string
|
||||
ContentHTML string
|
||||
}
|
||||
|
||||
func main() {
|
||||
loadSubscriptionImportEnv()
|
||||
|
||||
var outlineTreeURL string
|
||||
var fixture string
|
||||
var dryRun bool
|
||||
var timeoutSeconds int
|
||||
|
||||
flag.StringVar(&outlineTreeURL, "outline-tree-url", defaultMobileCloudOutlineTreeURL, "移动云 MoMA 文档大纲树接口")
|
||||
flag.StringVar(&fixture, "fixture", "", "移动云 MoMA 价格样例文件")
|
||||
flag.BoolVar(&dryRun, "dry-run", false, "仅解析并打印摘要,不写入数据库")
|
||||
flag.IntVar(&timeoutSeconds, "timeout", 20, "请求超时(秒)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := mobileCloudPricingImportConfig{OutlineTreeURL: outlineTreeURL, Fixture: fixture, DryRun: dryRun, Timeout: time.Duration(timeoutSeconds) * time.Second}
|
||||
|
||||
var db *sql.DB
|
||||
var err error
|
||||
if !cfg.DryRun {
|
||||
db, err = subscriptionImportDB()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open db: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
if err := runMobileCloudPricingImport(cfg, db, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "import_mobile_cloud_pricing: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func runMobileCloudPricingImport(cfg mobileCloudPricingImportConfig, db *sql.DB, out io.Writer) error {
|
||||
client := &http.Client{Timeout: cfg.Timeout}
|
||||
payload, err := fetchMobileCloudArticlePayload(cfg, client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records, err := parseMobileCloudPricingHTML(payload.ContentHTML, payload.DocURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
records = dedupeOfficialPricingRecords(records)
|
||||
if cfg.DryRun {
|
||||
_, err = fmt.Fprintf(out, "source=mobile-cloud-pricing-import models=%d operator=%s dry_run=true\n", len(records), records[0].OperatorName)
|
||||
return err
|
||||
}
|
||||
if db == nil {
|
||||
return fmt.Errorf("db is required when dry-run=false")
|
||||
}
|
||||
if err := upsertOfficialPricingRecords(db, records, "mobile-cloud-pricing-import"); err != nil {
|
||||
return err
|
||||
}
|
||||
var tableRows int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM region_pricing`).Scan(&tableRows); err != nil {
|
||||
return fmt.Errorf("count region_pricing: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "source=mobile-cloud-pricing-import models=%d operator=%s table_rows=%d dry_run=false\n", len(records), records[0].OperatorName, tableRows)
|
||||
return err
|
||||
}
|
||||
|
||||
func fetchMobileCloudArticlePayload(cfg mobileCloudPricingImportConfig, client *http.Client) (mobileCloudArticlePayload, error) {
|
||||
if cfg.Fixture != "" {
|
||||
data, err := os.ReadFile(cfg.Fixture)
|
||||
if err != nil {
|
||||
return mobileCloudArticlePayload{}, fmt.Errorf("read fixture %s: %w", cfg.Fixture, err)
|
||||
}
|
||||
return mobileCloudArticlePayload{
|
||||
ArticleID: 91592,
|
||||
Title: mobileCloudPricingArticleTitle,
|
||||
DocURL: fmt.Sprintf(defaultMobileCloudDocURLPattern, 91592),
|
||||
ContentHTML: string(data),
|
||||
}, nil
|
||||
}
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 20 * time.Second}
|
||||
}
|
||||
outlineRaw, err := fetchRawPricingPage(cfg.OutlineTreeURL, "", client)
|
||||
if err != nil {
|
||||
return mobileCloudArticlePayload{}, err
|
||||
}
|
||||
articleID, contentPublished, err := resolveMobileCloudPricingArticle(outlineRaw)
|
||||
if err != nil {
|
||||
return mobileCloudArticlePayload{}, err
|
||||
}
|
||||
infoURL := fmt.Sprintf(defaultMobileCloudArticleInfoURL, articleID)
|
||||
infoRaw, err := fetchRawPricingPage(infoURL, "", client)
|
||||
if err != nil {
|
||||
return mobileCloudArticlePayload{}, err
|
||||
}
|
||||
articleInfo, err := parseMobileCloudArticleInfo(infoRaw)
|
||||
if err != nil {
|
||||
return mobileCloudArticlePayload{}, err
|
||||
}
|
||||
if strings.TrimSpace(contentPublished) == "" {
|
||||
contentPublished = articleInfo.ContentPublished
|
||||
}
|
||||
contentURL := fmt.Sprintf(defaultMobileCloudArticleContentURL, contentPublished)
|
||||
contentHTML, err := fetchRawPricingPage(contentURL, "", client)
|
||||
if err != nil {
|
||||
return mobileCloudArticlePayload{}, err
|
||||
}
|
||||
return mobileCloudArticlePayload{
|
||||
ArticleID: articleInfo.ID,
|
||||
Title: articleInfo.Title,
|
||||
ContentPublished: contentPublished,
|
||||
DocURL: fmt.Sprintf(defaultMobileCloudDocURLPattern, articleInfo.ID),
|
||||
ContentHTML: contentHTML,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveMobileCloudPricingArticle(raw string) (int, string, error) {
|
||||
var envelope mobileCloudOutlineEnvelope
|
||||
if err := json.Unmarshal([]byte(raw), &envelope); err != nil {
|
||||
return 0, "", fmt.Errorf("parse mobile cloud outline tree: %w", err)
|
||||
}
|
||||
articleID, contentPublished, ok := findMobileCloudPricingArticle(envelope.Data)
|
||||
if !ok {
|
||||
return 0, "", fmt.Errorf("mobile cloud pricing article %q not found in outline tree", mobileCloudPricingArticleTitle)
|
||||
}
|
||||
return articleID, contentPublished, nil
|
||||
}
|
||||
|
||||
func findMobileCloudPricingArticle(node mobileCloudOutlineNode) (int, string, bool) {
|
||||
if strings.TrimSpace(node.ArticleTitle) == mobileCloudPricingArticleTitle && node.ArticleID > 0 {
|
||||
return node.ArticleID, strings.TrimSpace(node.ArticleContentPublished), true
|
||||
}
|
||||
for _, child := range node.Children {
|
||||
if articleID, contentPublished, ok := findMobileCloudPricingArticle(child); ok {
|
||||
return articleID, contentPublished, true
|
||||
}
|
||||
}
|
||||
return 0, "", false
|
||||
}
|
||||
|
||||
func parseMobileCloudArticleInfo(raw string) (mobileCloudArticleInfo, error) {
|
||||
var envelope mobileCloudArticleInfoEnvelope
|
||||
if err := json.Unmarshal([]byte(raw), &envelope); err != nil {
|
||||
return mobileCloudArticleInfo{}, fmt.Errorf("parse mobile cloud article info: %w", err)
|
||||
}
|
||||
if envelope.Data.ID == 0 {
|
||||
return mobileCloudArticleInfo{}, fmt.Errorf("unexpected mobile cloud article info content")
|
||||
}
|
||||
return envelope.Data, nil
|
||||
}
|
||||
|
||||
func parseMobileCloudPricingHTML(raw string, docURL string) ([]officialPricingRecord, error) {
|
||||
sections := mobileCloudRegionSections(raw)
|
||||
if len(sections) == 0 {
|
||||
return nil, fmt.Errorf("no mobile cloud pricing regions found")
|
||||
}
|
||||
records := make([]officialPricingRecord, 0)
|
||||
for _, section := range sections {
|
||||
for _, table := range mobileCloudTableBlocks(section.Body) {
|
||||
rows := mobileCloudTableRows(table)
|
||||
if len(rows) < 2 {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case isMobileCloudTokenPricingHeader(rows[0]):
|
||||
records = append(records, buildMobileCloudRecordsFromTable(section.Region, rows[1:], docURL)...)
|
||||
case isMobileCloudVoicePricingHeader(rows[0]):
|
||||
records = append(records, buildMobileCloudVoiceRecordsFromTable(section.Region, rows[1:], docURL)...)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, fmt.Errorf("no mobile cloud token pricing rows found")
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
type mobileCloudRegionSection struct {
|
||||
Region string
|
||||
Body string
|
||||
}
|
||||
|
||||
func mobileCloudRegionSections(raw string) []mobileCloudRegionSection {
|
||||
headingPattern := regexp.MustCompile(`(?is)<h2[^>]*>(.*?)</h2>`)
|
||||
matches := headingPattern.FindAllStringSubmatchIndex(raw, -1)
|
||||
sections := make([]mobileCloudRegionSection, 0, len(matches))
|
||||
for i, match := range matches {
|
||||
heading := cleanMobileCloudHTMLText(raw[match[2]:match[3]])
|
||||
if !strings.Contains(heading, "支持订购模型") {
|
||||
continue
|
||||
}
|
||||
start := match[1]
|
||||
end := len(raw)
|
||||
if i+1 < len(matches) {
|
||||
end = matches[i+1][0]
|
||||
}
|
||||
region := strings.TrimSpace(strings.TrimSuffix(heading, "资源池支持订购模型"))
|
||||
if region == heading {
|
||||
region = strings.TrimSpace(strings.TrimSuffix(heading, "支持订购模型"))
|
||||
}
|
||||
sections = append(sections, mobileCloudRegionSection{Region: region, Body: raw[start:end]})
|
||||
}
|
||||
return sections
|
||||
}
|
||||
|
||||
func mobileCloudTableBlocks(raw string) []string {
|
||||
return regexp.MustCompile(`(?is)<table.*?</table>`).FindAllString(raw, -1)
|
||||
}
|
||||
|
||||
func mobileCloudTableRows(raw string) [][]string {
|
||||
rowMatches := regexp.MustCompile(`(?is)<tr[^>]*>(.*?)</tr>`).FindAllStringSubmatch(raw, -1)
|
||||
rows := make([][]string, 0, len(rowMatches))
|
||||
for _, rowMatch := range rowMatches {
|
||||
cellMatches := regexp.MustCompile(`(?is)<t[dh][^>]*>(.*?)</t[dh]>`).FindAllStringSubmatch(rowMatch[1], -1)
|
||||
cells := make([]string, 0, len(cellMatches))
|
||||
for _, cellMatch := range cellMatches {
|
||||
cells = append(cells, cleanMobileCloudHTMLText(cellMatch[1]))
|
||||
}
|
||||
if len(cells) > 0 {
|
||||
rows = append(rows, cells)
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func cleanMobileCloudHTMLText(raw string) string {
|
||||
raw = strings.ReplaceAll(raw, "<br>", " ")
|
||||
raw = strings.ReplaceAll(raw, "<br/>", " ")
|
||||
raw = strings.ReplaceAll(raw, "<br />", " ")
|
||||
raw = regexp.MustCompile(`(?is)<[^>]+>`).ReplaceAllString(raw, " ")
|
||||
raw = html.UnescapeString(raw)
|
||||
raw = regexp.MustCompile(`\s+`).ReplaceAllString(raw, " ")
|
||||
return strings.TrimSpace(raw)
|
||||
}
|
||||
|
||||
func isMobileCloudTokenPricingHeader(cells []string) bool {
|
||||
if len(cells) < 4 {
|
||||
return false
|
||||
}
|
||||
return cells[0] == "规格名称" && cells[1] == "输入/输出tokens" && cells[2] == "单价(元/百万tokens)" && cells[3] == "包含模型"
|
||||
}
|
||||
|
||||
func isMobileCloudVoicePricingHeader(cells []string) bool {
|
||||
if len(cells) < 5 {
|
||||
return false
|
||||
}
|
||||
return cells[0] == "规格名称" && cells[1] == "模型类别" && cells[2] == "资费场景" && cells[3] == "单价" && cells[4] == "包含模型"
|
||||
}
|
||||
|
||||
func buildMobileCloudRecordsFromTable(region string, rows [][]string, docURL string) []officialPricingRecord {
|
||||
records := make([]officialPricingRecord, 0)
|
||||
currentModels := make([]string, 0)
|
||||
currentInputPrice := 0.0
|
||||
for _, row := range rows {
|
||||
switch {
|
||||
case len(row) >= 4:
|
||||
billingKind := strings.TrimSpace(row[1])
|
||||
price := mustParseSubscriptionPrice(row[2])
|
||||
currentModels = mobileCloudModelNames(row[3])
|
||||
switch billingKind {
|
||||
case "输入tokens":
|
||||
currentInputPrice = price
|
||||
case "tokens资费":
|
||||
records = append(records, buildMobileCloudFlatTokenRecords(region, currentModels, price, docURL)...)
|
||||
currentInputPrice = 0
|
||||
default:
|
||||
currentInputPrice = 0
|
||||
}
|
||||
case len(row) >= 2 && strings.TrimSpace(row[0]) == "输出tokens":
|
||||
if currentInputPrice <= 0 || len(currentModels) == 0 {
|
||||
continue
|
||||
}
|
||||
outputPrice := mustParseSubscriptionPrice(row[1])
|
||||
records = append(records, buildMobileCloudInputOutputRecords(region, currentModels, currentInputPrice, outputPrice, docURL)...)
|
||||
currentInputPrice = 0
|
||||
}
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func buildMobileCloudInputOutputRecords(region string, modelNames []string, inputPrice float64, outputPrice float64, docURL string) []officialPricingRecord {
|
||||
records := make([]officialPricingRecord, 0, len(modelNames))
|
||||
for _, modelName := range modelNames {
|
||||
providerName := mobileCloudProviderName(modelName)
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
records = append(records, officialPricingRecord{
|
||||
ModelID: normalizeExternalID("mobile-cloud", mobileCloudRegionCode(region), modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Mobile Cloud",
|
||||
OperatorNameCn: "移动云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://ecloud.10086.cn/portal/product/MaaS",
|
||||
OperatorType: "official",
|
||||
Region: region,
|
||||
Currency: "CNY",
|
||||
InputPrice: inputPrice,
|
||||
OutputPrice: outputPrice,
|
||||
SourceURL: docURL,
|
||||
ModelSourceURL: docURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(modelName),
|
||||
})
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func buildMobileCloudFlatTokenRecords(region string, modelNames []string, price float64, docURL string) []officialPricingRecord {
|
||||
records := make([]officialPricingRecord, 0, len(modelNames))
|
||||
for _, modelName := range modelNames {
|
||||
providerName := mobileCloudProviderName(modelName)
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
records = append(records, officialPricingRecord{
|
||||
ModelID: normalizeExternalID("mobile-cloud", mobileCloudRegionCode(region), modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Mobile Cloud",
|
||||
OperatorNameCn: "移动云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://ecloud.10086.cn/portal/product/MaaS",
|
||||
OperatorType: "official",
|
||||
Region: region,
|
||||
Currency: "CNY",
|
||||
InputPrice: price,
|
||||
OutputPrice: price,
|
||||
SourceURL: docURL,
|
||||
ModelSourceURL: docURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: detectModality(modelName),
|
||||
})
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func buildMobileCloudVoiceRecordsFromTable(region string, rows [][]string, docURL string) []officialPricingRecord {
|
||||
records := make([]officialPricingRecord, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if len(row) < 5 {
|
||||
continue
|
||||
}
|
||||
modelNames := mobileCloudModelNames(row[4])
|
||||
if len(modelNames) == 0 {
|
||||
modelNames = []string{strings.TrimSpace(row[0])}
|
||||
}
|
||||
flatPrice := mobileCloudInlinePrice(row[3])
|
||||
priceUnit := mobileCloudVoicePriceUnit(row[2], row[3])
|
||||
for _, modelName := range modelNames {
|
||||
providerName := mobileCloudProviderName(modelName)
|
||||
providerNameCn, providerCountry, providerWebsite := providerMetadata(providerName)
|
||||
records = append(records, officialPricingRecord{
|
||||
ModelID: normalizeExternalID("mobile-cloud", mobileCloudRegionCode(region), modelName),
|
||||
ModelName: modelName,
|
||||
ProviderName: providerName,
|
||||
ProviderNameCn: providerNameCn,
|
||||
ProviderCountry: providerCountry,
|
||||
ProviderWebsite: providerWebsite,
|
||||
OperatorName: "Mobile Cloud",
|
||||
OperatorNameCn: "移动云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://ecloud.10086.cn/portal/product/MaaS",
|
||||
OperatorType: "official",
|
||||
Region: region,
|
||||
Currency: "CNY",
|
||||
PricingMode: "flat",
|
||||
PriceUnit: priceUnit,
|
||||
FlatPrice: flatPrice,
|
||||
SourceURL: docURL,
|
||||
ModelSourceURL: docURL,
|
||||
DateConfidence: "unknown",
|
||||
DateSourceKind: "official_pricing",
|
||||
Modality: "audio",
|
||||
})
|
||||
}
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func mobileCloudVoicePriceUnit(scene string, price string) string {
|
||||
text := strings.ToLower(strings.TrimSpace(scene + " " + price))
|
||||
switch {
|
||||
case strings.Contains(text, "万字符"), strings.Contains(text, "字符"):
|
||||
return "10k_characters"
|
||||
case strings.Contains(text, "元/秒"), strings.Contains(text, "秒"):
|
||||
return "second"
|
||||
default:
|
||||
return "flat"
|
||||
}
|
||||
}
|
||||
|
||||
func mobileCloudInlinePrice(raw string) float64 {
|
||||
matches := regexp.MustCompile(`([0-9]+(?:\.[0-9]+)?)`).FindStringSubmatch(raw)
|
||||
if len(matches) != 2 {
|
||||
return 0
|
||||
}
|
||||
return mustParseSubscriptionPrice(matches[1])
|
||||
}
|
||||
|
||||
func mobileCloudModelNames(raw string) []string {
|
||||
parts := strings.Fields(strings.TrimSpace(raw))
|
||||
models := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
cleaned := strings.TrimSpace(strings.TrimSuffix(part, "、"))
|
||||
if cleaned != "" {
|
||||
models = append(models, cleaned)
|
||||
}
|
||||
}
|
||||
return models
|
||||
}
|
||||
|
||||
func mobileCloudProviderName(modelName string) string {
|
||||
lower := strings.ToLower(strings.TrimSpace(modelName))
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "minimax"):
|
||||
return "MiniMax"
|
||||
case strings.HasPrefix(lower, "deepseek"):
|
||||
return "DeepSeek"
|
||||
case strings.HasPrefix(lower, "qwen"), strings.HasPrefix(lower, "qwq"):
|
||||
return "Qwen"
|
||||
case strings.HasPrefix(lower, "bge"):
|
||||
return "BAAI"
|
||||
case strings.HasPrefix(lower, "cosyvoice"), strings.HasPrefix(lower, "sensevoice"):
|
||||
return "Alibaba"
|
||||
default:
|
||||
return "China Mobile"
|
||||
}
|
||||
}
|
||||
|
||||
func mobileCloudRegionCode(region string) string {
|
||||
switch strings.TrimSpace(region) {
|
||||
case "华北-呼和浩特":
|
||||
return "huabei-huhehaote"
|
||||
case "东北-哈尔滨":
|
||||
return "dongbei-haerbin"
|
||||
case "华中-郑州":
|
||||
return "huazhong-zhengzhou"
|
||||
case "黑龙江-哈尔滨":
|
||||
return "heilongjiang-haerbin"
|
||||
case "华东-上海5":
|
||||
return "huadong-shanghai5"
|
||||
case "江西-南昌":
|
||||
return "jiangxi-nanchang"
|
||||
case "湖北-武汉":
|
||||
return "hubei-wuhan"
|
||||
case "华南-广州8":
|
||||
return "huanan-guangzhou8"
|
||||
default:
|
||||
return normalizeExternalID(region)
|
||||
}
|
||||
}
|
||||
89
scripts/import_mobile_cloud_pricing_test.go
Normal file
89
scripts/import_mobile_cloud_pricing_test.go
Normal file
@@ -0,0 +1,89 @@
|
||||
//go:build llm_script
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolveMobileCloudPricingArticle(t *testing.T) {
|
||||
raw := `{"code":200,"data":{"children":[{"articleTitle":"其他文档","articleId":1,"articleContentPublished":"x"},{"children":[{"articleTitle":"预置模型服务-token按量计费","articleId":91592,"articleContentPublished":"64ec46cbfd7c535db501aff43df5a788"}]}]}}`
|
||||
articleID, contentPublished, err := resolveMobileCloudPricingArticle(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveMobileCloudPricingArticle 返回错误: %v", err)
|
||||
}
|
||||
if articleID != 91592 || contentPublished != "64ec46cbfd7c535db501aff43df5a788" {
|
||||
t.Fatalf("解析价格文章失败: articleID=%d contentPublished=%q", articleID, contentPublished)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMobileCloudPricingHTMLBuildsRecords(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "mobile_cloud_pricing_sample.html"))
|
||||
if err != nil {
|
||||
t.Fatalf("读取 fixture 失败: %v", err)
|
||||
}
|
||||
records, err := parseMobileCloudPricingHTML(string(raw), "https://ecloud.10086.cn/op-help-center/doc/article/91592")
|
||||
if err != nil {
|
||||
t.Fatalf("parseMobileCloudPricingHTML 返回错误: %v", err)
|
||||
}
|
||||
if len(records) != 8 {
|
||||
t.Fatalf("期望 8 条移动云价格记录,实际 %d", len(records))
|
||||
}
|
||||
recordMap := make(map[string]officialPricingRecord, len(records))
|
||||
for _, record := range records {
|
||||
recordMap[record.ModelID] = record
|
||||
}
|
||||
if recordMap["mobile-cloud-huabei-huhehaote-minimax-m2-5"].Region != "华北-呼和浩特" {
|
||||
t.Fatalf("华北 MiniMax region 错误: %+v", recordMap["mobile-cloud-huabei-huhehaote-minimax-m2-5"])
|
||||
}
|
||||
if recordMap["mobile-cloud-huabei-huhehaote-deepseek-v3-0324"].InputPrice != 2 || recordMap["mobile-cloud-huabei-huhehaote-deepseek-v3-0324"].OutputPrice != 8 {
|
||||
t.Fatalf("DeepSeek-V3-0324 价格错误: %+v", recordMap["mobile-cloud-huabei-huhehaote-deepseek-v3-0324"])
|
||||
}
|
||||
if recordMap["mobile-cloud-huabei-huhehaote-qwq-32b"].ProviderName != "Qwen" || recordMap["mobile-cloud-huabei-huhehaote-qwq-32b"].OutputPrice != 6 {
|
||||
t.Fatalf("QwQ-32B provider/价格错误: %+v", recordMap["mobile-cloud-huabei-huhehaote-qwq-32b"])
|
||||
}
|
||||
if recordMap["mobile-cloud-huabei-huhehaote-bge-m3"].ProviderName != "BAAI" || recordMap["mobile-cloud-huabei-huhehaote-bge-m3"].InputPrice != 0.5 || recordMap["mobile-cloud-huabei-huhehaote-bge-m3"].OutputPrice != 0.5 {
|
||||
t.Fatalf("bge-m3 平价 token 记录错误: %+v", recordMap["mobile-cloud-huabei-huhehaote-bge-m3"])
|
||||
}
|
||||
cosyVoice, ok := recordMap["mobile-cloud-huabei-huhehaote-cosyvoice"]
|
||||
if !ok {
|
||||
t.Fatalf("缺少 CosyVoice 语音计费记录")
|
||||
}
|
||||
if cosyVoice.ProviderName != "Alibaba" || cosyVoice.Modality != "audio" || cosyVoice.PricingMode != "flat" || cosyVoice.FlatPrice != 2 || cosyVoice.PriceUnit != "10k_characters" {
|
||||
t.Fatalf("CosyVoice 语音计费记录错误: %+v", cosyVoice)
|
||||
}
|
||||
senseVoice, ok := recordMap["mobile-cloud-huabei-huhehaote-sensevoice"]
|
||||
if !ok {
|
||||
t.Fatalf("缺少 SenseVoice 语音计费记录")
|
||||
}
|
||||
if senseVoice.ProviderName != "Alibaba" || senseVoice.Modality != "audio" || senseVoice.PricingMode != "flat" || senseVoice.FlatPrice != 0.0007 || senseVoice.PriceUnit != "second" {
|
||||
t.Fatalf("SenseVoice 语音计费记录错误: %+v", senseVoice)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMobileCloudPricingImportDryRunPrintsSummary(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
err := runMobileCloudPricingImport(mobileCloudPricingImportConfig{
|
||||
OutlineTreeURL: defaultMobileCloudOutlineTreeURL,
|
||||
Fixture: filepath.Join("testdata", "mobile_cloud_pricing_sample.html"),
|
||||
DryRun: true,
|
||||
}, nil, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runMobileCloudPricingImport 返回错误: %v", err)
|
||||
}
|
||||
output := out.String()
|
||||
for _, want := range []string{
|
||||
"source=mobile-cloud-pricing-import",
|
||||
"models=8",
|
||||
"operator=Mobile Cloud",
|
||||
"dry_run=true",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("输出缺少 %q,实际: %q", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,9 @@ type officialPricingRecord struct {
|
||||
OperatorType string
|
||||
Region string
|
||||
Currency string
|
||||
PricingMode string
|
||||
PriceUnit string
|
||||
FlatPrice float64
|
||||
InputPrice float64
|
||||
OutputPrice float64
|
||||
ContextLength int
|
||||
@@ -81,16 +84,21 @@ func upsertOfficialPricingRecords(db *sql.DB, records []officialPricingRecord, b
|
||||
_, err = db.Exec(
|
||||
`INSERT INTO region_pricing (
|
||||
model_id, operator_id, region, currency,
|
||||
pricing_mode, price_unit, flat_price,
|
||||
input_price_per_mtok, output_price_per_mtok,
|
||||
is_free, effective_date, source_url, source_type,
|
||||
free_quota, free_limitations, rate_limit
|
||||
) VALUES (
|
||||
$1, $2, $3, $4,
|
||||
$5, $6, $7, CURRENT_DATE, $8, $9,
|
||||
$10, $11, $12
|
||||
$5, $6, $7,
|
||||
$8, $9, $10, CURRENT_DATE, $11, $12,
|
||||
$13, $14, $15
|
||||
)
|
||||
ON CONFLICT (model_id, operator_id, region, currency, effective_date)
|
||||
DO UPDATE SET
|
||||
pricing_mode = EXCLUDED.pricing_mode,
|
||||
price_unit = EXCLUDED.price_unit,
|
||||
flat_price = EXCLUDED.flat_price,
|
||||
input_price_per_mtok = EXCLUDED.input_price_per_mtok,
|
||||
output_price_per_mtok = EXCLUDED.output_price_per_mtok,
|
||||
is_free = EXCLUDED.is_free,
|
||||
@@ -101,6 +109,7 @@ func upsertOfficialPricingRecords(db *sql.DB, records []officialPricingRecord, b
|
||||
rate_limit = EXCLUDED.rate_limit,
|
||||
updated_at = CURRENT_TIMESTAMP`,
|
||||
modelID, operatorID, record.Region, record.Currency,
|
||||
fallbackPricingMode(record.PricingMode), fallbackPriceUnit(record.PriceUnit), nullIfZeroFloat(record.FlatPrice),
|
||||
record.InputPrice, record.OutputPrice, record.IsFree, record.SourceURL, sourceType,
|
||||
nullIfBlank(freeQuota), freeLimitations, rateLimit,
|
||||
)
|
||||
@@ -260,6 +269,29 @@ func fallbackModality(raw string) string {
|
||||
return value
|
||||
}
|
||||
|
||||
func fallbackPricingMode(raw string) string {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
return "input_output"
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func fallbackPriceUnit(raw string) string {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
return "million_tokens"
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func nullIfZeroFloat(value float64) any {
|
||||
if value == 0 {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func fetchRawPricingPage(url string, fixture string, client *http.Client) (string, error) {
|
||||
return fetchRawPricingPageWithOptions(url, fixture, client, officialPricingFetchOptions{
|
||||
AcceptLanguage: "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
@@ -399,6 +431,8 @@ func detectModality(modelName string) string {
|
||||
switch {
|
||||
case strings.Contains(lower, "coder"), strings.Contains(lower, "code"):
|
||||
return "code"
|
||||
case strings.Contains(lower, "voice"), strings.Contains(lower, "audio"), strings.Contains(lower, "speech"):
|
||||
return "audio"
|
||||
case strings.Contains(lower, "vision"), strings.Contains(lower, "vl"), strings.Contains(lower, "omni"), strings.Contains(lower, "multi"), strings.Contains(lower, "live"):
|
||||
return "multimodal"
|
||||
default:
|
||||
@@ -414,8 +448,14 @@ func providerMetadata(providerName string) (string, string, string) {
|
||||
return "亚马逊", "US", "https://aws.amazon.com"
|
||||
case "Anthropic":
|
||||
return "Anthropic", "US", "https://www.anthropic.com"
|
||||
case "BAAI":
|
||||
return "智源", "CN", "https://www.baai.ac.cn"
|
||||
case "Baidu":
|
||||
return "百度", "CN", "https://cloud.baidu.com"
|
||||
case "ByteDance":
|
||||
return "字节跳动", "CN", "https://www.volcengine.com"
|
||||
case "China Mobile":
|
||||
return "中国移动", "CN", "https://ecloud.10086.cn"
|
||||
case "Cloudflare":
|
||||
return "Cloudflare", "US", "https://www.cloudflare.com"
|
||||
case "Cohere":
|
||||
|
||||
32
scripts/testdata/mobile_cloud_pricing_sample.html
vendored
Normal file
32
scripts/testdata/mobile_cloud_pricing_sample.html
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
<div id="doc-content-details">
|
||||
<div class="article-section">
|
||||
<div data-cms-section-item data-field-type="heading"><h2>华北-呼和浩特资源池支持订购模型</h2></div>
|
||||
<div data-cms-section-item data-field-type="richText">
|
||||
<table>
|
||||
<tr><th>规格名称</th><th>输入/输出tokens</th><th>单价(元/百万tokens)</th><th>包含模型</th></tr>
|
||||
<tr><td>MiniMax-M2.5</td><td>输入tokens</td><td>2.1</td><td><p>MiniMax-M2.5</p></td></tr>
|
||||
<tr><td>输出tokens</td><td>8.4</td></tr>
|
||||
<tr><td>DeepSeek-V3</td><td>输入tokens</td><td>2</td><td><p>DeepSeek-V3</p><p>DeepSeek-V3-0324</p></td></tr>
|
||||
<tr><td>输出tokens</td><td>8</td></tr>
|
||||
<tr><td>QwQ-32B(深度思考)</td><td>输入tokens</td><td>2</td><td><p>QwQ-32B</p></td></tr>
|
||||
<tr><td>输出tokens</td><td>6</td></tr>
|
||||
<tr><td>bge-m3</td><td>tokens资费</td><td>0.5</td><td><p>bge-m3</p></td></tr>
|
||||
</table>
|
||||
<table>
|
||||
<tr><th>规格名称</th><th>模型类别</th><th>资费场景</th><th>单价</th><th>包含模型</th></tr>
|
||||
<tr><td>CosyVoice</td><td>语音生成</td><td>按照处理字符数收费</td><td>2元/万字符</td><td>CosyVoice</td></tr>
|
||||
<tr><td>SenseVoice</td><td>语音识别</td><td>按照处理的语音时长秒数收费</td><td>0.0007 元/秒</td><td>SenseVoice</td></tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<div class="article-section">
|
||||
<div data-cms-section-item data-field-type="heading"><h2>华南-广州8支持订购模型</h2></div>
|
||||
<div data-cms-section-item data-field-type="richText">
|
||||
<table>
|
||||
<tr><th>规格名称</th><th>输入/输出tokens</th><th>单价(元/百万tokens)</th><th>包含模型</th></tr>
|
||||
<tr><td>MiniMax-M2.5</td><td>输入tokens</td><td>2.1</td><td><p>MiniMax-M2.5</p></td></tr>
|
||||
<tr><td>输出tokens</td><td>8.4</td></tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
Reference in New Issue
Block a user