feat(import): extend CTYun subscription collector
- ctyun_subscription_lib.go: extend CTYun subscription data extraction - import_ctyun_subscription_test.go: update tests for CTYun - ctyun_token_plan_sample.txt: updated test fixture
This commit is contained in:
phamnazage-jpg
@@ -133,10 +133,196 @@ func parseCTYunCodingPlan(raw string, publishedAt string) ([]subscriptionImportR
|
||||
}
|
||||
|
||||
func parseCTYunTokenPlan(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
if records, ok := parseCTYunTokenPlanNormalizedLayout(raw, publishedAt); ok {
|
||||
return records, nil
|
||||
}
|
||||
if records, ok := parseCTYunTokenPlanCardLayout(raw, publishedAt); ok {
|
||||
return records, nil
|
||||
}
|
||||
return parseCTYunTokenPlanLegacyLayout(raw, publishedAt)
|
||||
}
|
||||
|
||||
func parseCTYunTokenPlanNormalizedLayout(raw string, publishedAt string) ([]subscriptionImportRecord, bool) {
|
||||
lines := strings.Split(raw, "\n")
|
||||
codeByTier := map[string]string{
|
||||
"基础版": "basic",
|
||||
"专业版": "pro",
|
||||
"旗舰版": "flagship",
|
||||
"轻享版": "starter",
|
||||
"畅享版": "plus",
|
||||
"尊享版": "vip",
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, 6)
|
||||
for i := 0; i < len(lines); i++ {
|
||||
line := strings.TrimSpace(lines[i])
|
||||
if !strings.HasPrefix(line, "Token Plan") {
|
||||
continue
|
||||
}
|
||||
rawTier := strings.TrimSpace(strings.TrimPrefix(line, "Token Plan"))
|
||||
tierCode, ok := codeByTier[rawTier]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
j := i + 1
|
||||
block := make([]string, 0, 12)
|
||||
for ; j < len(lines); j++ {
|
||||
next := strings.TrimSpace(lines[j])
|
||||
if strings.HasPrefix(next, "Token Plan") {
|
||||
break
|
||||
}
|
||||
if next != "" {
|
||||
block = append(block, next)
|
||||
}
|
||||
}
|
||||
model := ""
|
||||
quota := ""
|
||||
price := ""
|
||||
notesParts := make([]string, 0, 4)
|
||||
for k := 0; k < len(block); k++ {
|
||||
item := block[k]
|
||||
switch {
|
||||
case strings.HasPrefix(item, "支持模型："):
|
||||
model = strings.TrimSpace(strings.TrimPrefix(item, "支持模型："))
|
||||
case strings.Contains(item, "Tokens"):
|
||||
quota = strings.TrimSpace(strings.TrimSuffix(item, "Tokens"))
|
||||
case regexp.MustCompile(`^[0-9]+$`).MatchString(item) && k+2 < len(block) && regexp.MustCompile(`^\.[0-9]+$`).MatchString(block[k+1]) && block[k+2] == "元/个/月":
|
||||
price = item + block[k+1]
|
||||
case item == "产品优势", item == "立即订购", strings.HasPrefix(item, "支持工具："), strings.HasPrefix(item, "已抢购"), strings.HasSuffix(item, "用户"), item == "展开更多", item == "免费领取Tokens":
|
||||
continue
|
||||
default:
|
||||
notesParts = append(notesParts, item)
|
||||
}
|
||||
}
|
||||
if model == "" || quota == "" || price == "" {
|
||||
continue
|
||||
}
|
||||
notes := "天翼云大模型 AI 专项活动页套餐。"
|
||||
if len(notesParts) > 0 {
|
||||
notes = strings.Join(notesParts, "；")
|
||||
}
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.ctyun.cn",
|
||||
OperatorName: "CTYun",
|
||||
OperatorNameCn: "天翼云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ctyun.cn",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: "ctyun-token-plan-" + tierCode,
|
||||
PlanName: "天翼云 Token Plan " + rawTier,
|
||||
Tier: rawTier,
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(price),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: parseChineseTokenQuota(quota),
|
||||
QuotaUnit: "tokens/month",
|
||||
PlanScope: "Token Plan",
|
||||
ModelScope: []string{model},
|
||||
SourceURL: defaultCTYunTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: notes,
|
||||
})
|
||||
i = j - 1
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
return records, true
|
||||
}
|
||||
|
||||
func parseCTYunTokenPlanCardLayout(raw string, publishedAt string) ([]subscriptionImportRecord, bool) {
|
||||
cardPattern := regexp.MustCompile(`(?s)<span title="(Token Plan [^"]+)" class="card-header-title-text".*?</span>(.*?)<div class="card-btns-wrap"`)
|
||||
cards := cardPattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(cards) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
codeByTier := map[string]string{
|
||||
"基础版": "basic",
|
||||
"专业版": "pro",
|
||||
"旗舰版": "flagship",
|
||||
}
|
||||
|
||||
records := make([]subscriptionImportRecord, 0, len(cards))
|
||||
for _, card := range cards {
|
||||
title := strings.TrimSpace(card[1])
|
||||
body := card[2]
|
||||
rawTier := strings.TrimSpace(strings.TrimPrefix(title, "Token Plan "))
|
||||
tierCode, ok := codeByTier[rawTier]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
modelMatch := regexp.MustCompile(`支持模型：([^<]+)</span>`).FindStringSubmatch(body)
|
||||
if len(modelMatch) != 2 {
|
||||
return nil, false
|
||||
}
|
||||
quotaMatch := regexp.MustCompile(`([0-9]+(?:\.[0-9]+)?亿|[0-9]+万)Tokens`).FindStringSubmatch(body)
|
||||
if len(quotaMatch) != 2 {
|
||||
return nil, false
|
||||
}
|
||||
priceMatch := regexp.MustCompile(`<span class="price-new-big"[^>]*>\s*([0-9]+)\s*</span>\s*<span class="price-new-big"[^>]*>\s*\.([0-9]+)\s*</span>\s*<span class="price-new-unit"[^>]*>元/个/月</span>`).FindStringSubmatch(body)
|
||||
if len(priceMatch) != 3 {
|
||||
return nil, false
|
||||
}
|
||||
notes := "天翼云大模型 AI 专项活动页套餐。"
|
||||
if featureLines := regexp.MustCompile(`card-content-gou-content"[^>]*>([^<]+)</span>`).FindAllStringSubmatch(body, -1); len(featureLines) > 0 {
|
||||
parts := make([]string, 0, len(featureLines))
|
||||
for _, line := range featureLines {
|
||||
text := strings.TrimSpace(line[1])
|
||||
if text == "" || strings.HasPrefix(text, "支持模型：") || strings.Contains(text, "Tokens") {
|
||||
continue
|
||||
}
|
||||
parts = append(parts, text)
|
||||
}
|
||||
if len(parts) > 0 {
|
||||
notes = strings.Join(parts, "；")
|
||||
}
|
||||
}
|
||||
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
ProviderCountry: "CN",
|
||||
ProviderWebsite: "https://www.ctyun.cn",
|
||||
OperatorName: "CTYun",
|
||||
OperatorNameCn: "天翼云",
|
||||
OperatorCountry: "CN",
|
||||
OperatorWebsite: "https://www.ctyun.cn",
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: "ctyun-token-plan-" + tierCode,
|
||||
PlanName: "天翼云 " + title,
|
||||
Tier: rawTier,
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
ListPrice: mustParseSubscriptionPrice(priceMatch[1] + "." + priceMatch[2]),
|
||||
PriceUnit: "CNY/month",
|
||||
QuotaValue: parseChineseTokenQuota(quotaMatch[1]),
|
||||
QuotaUnit: "tokens/month",
|
||||
PlanScope: "Token Plan",
|
||||
ModelScope: []string{strings.TrimSpace(modelMatch[1])},
|
||||
SourceURL: defaultCTYunTokenPlanURL,
|
||||
PublishedAt: publishedAt,
|
||||
EffectiveDate: effectiveDateFromPublishedAt(publishedAt),
|
||||
Notes: notes,
|
||||
})
|
||||
}
|
||||
return records, true
|
||||
}
|
||||
|
||||
func parseCTYunTokenPlanLegacyLayout(raw string, publishedAt string) ([]subscriptionImportRecord, error) {
|
||||
pattern := regexp.MustCompile(`Token Plan ([^\n]+?)(\d+(?:\.\d+)?亿|\d+万)Tokens包[\s\S]*?支持模型：([^\n]+)[\s\S]*?(\d+\s*\.\s*\d+)\s*元/个`)
|
||||
matches := pattern.FindAllStringSubmatch(raw, -1)
|
||||
if len(matches) != 6 {
|
||||
return nil, fmt.Errorf("unexpected ctyun token plan count: %d", len(matches))
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("unexpected ctyun token plan count: 0")
|
||||
}
|
||||
|
||||
codeByTier := map[string]string{
|
||||
@@ -151,13 +337,12 @@ func parseCTYunTokenPlan(raw string, publishedAt string) ([]subscriptionImportRe
|
||||
records := make([]subscriptionImportRecord, 0, len(matches))
|
||||
for _, match := range matches {
|
||||
rawTier := strings.TrimSpace(match[1])
|
||||
tierCode := codeByTier[rawTier]
|
||||
tierCode, ok := codeByTier[rawTier]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected ctyun token plan tier: %s", rawTier)
|
||||
}
|
||||
quotaValue := parseChineseTokenQuota(match[2])
|
||||
price := mustParseSubscriptionPrice(strings.ReplaceAll(match[4], " ", ""))
|
||||
planName := "天翼云 Token Plan " + rawTier
|
||||
if rawTier == "Lite" || rawTier == "Pro" || rawTier == "Max" {
|
||||
planName = "天翼云 Token Plan " + rawTier
|
||||
}
|
||||
records = append(records, subscriptionImportRecord{
|
||||
ProviderName: "Telecom",
|
||||
ProviderNameCn: "中国电信",
|
||||
@@ -170,7 +355,7 @@ func parseCTYunTokenPlan(raw string, publishedAt string) ([]subscriptionImportRe
|
||||
OperatorType: "cloud",
|
||||
PlanFamily: "token_plan",
|
||||
PlanCode: "ctyun-token-plan-" + tierCode,
|
||||
PlanName: planName,
|
||||
PlanName: "天翼云 Token Plan " + rawTier,
|
||||
Tier: rawTier,
|
||||
BillingCycle: "monthly",
|
||||
Currency: "CNY",
|
||||
|
||||
Reference in New Issue
Block a user