From dfb54092b7f9c7fcd1ffbf5f34f093b92fc46075 Mon Sep 17 00:00:00 2001 From: phamnazage-jpg Date: Thu, 14 May 2026 09:16:12 +0800 Subject: [PATCH] feat(import): classify source-only official families --- scripts/import_bytedance_data.go | 20 ++++++++++ scripts/import_bytedance_data_test.go | 16 +++++++- scripts/import_phase2_data.go | 35 +++++++++++++++++ scripts/import_phase2_data_test.go | 16 +++++++- scripts/import_zhipu_data.go | 55 +++++++++++++++++++++++++++ scripts/import_zhipu_data_test.go | 28 ++++++++++++++ 6 files changed, 168 insertions(+), 2 deletions(-) diff --git a/scripts/import_bytedance_data.go b/scripts/import_bytedance_data.go index d973ef8..c585053 100644 --- a/scripts/import_bytedance_data.go +++ b/scripts/import_bytedance_data.go @@ -131,6 +131,26 @@ var bytedanceModelMetadataRules = []bytedanceModelMetadata{ DateConfidence: "official_primary", DateSourceKind: "official_announcement", }, + { + Prefix: "bytedance-doubao-pro", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "bytedance-doubao-seed-character", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "bytedance-glm-4.7", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "bytedance-deepseek-", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, { Prefix: "bytedance-seedance-1.0-lite", ReleaseDate: "2025-05-13", diff --git a/scripts/import_bytedance_data_test.go b/scripts/import_bytedance_data_test.go index 82d9cd3..cc90a0c 100644 --- a/scripts/import_bytedance_data_test.go +++ b/scripts/import_bytedance_data_test.go @@ -79,7 +79,7 @@ func TestEnrichBytedanceModelMetadataUsesSpecificFamilyRules(t *testing.T) { func TestEnrichBytedanceModelMetadataFallsBackToPricingSource(t *testing.T) { enriched := enrichBytedanceModelMetadata(ModelPricing{ - ModelID: "bytedance-deepseek-r1", + ModelID: "bytedance-unknown-preview", SourceURL: "https://www.volcengine.com/docs/82379/1099320", }) @@ -111,6 +111,20 @@ func TestEnrichBytedanceModelMetadataSupportsSourceOnlyRules(t *testing.T) { } } +func TestEnrichBytedanceModelMetadataSupportsCatalogBackfillRules(t *testing.T) { + enriched := enrichBytedanceModelMetadata(ModelPricing{ + ModelID: "bytedance-doubao-seed-character-128k", + SourceURL: "https://www.volcengine.com/docs/82379/1099320", + }) + + if enriched.ReleaseDate != "" { + t.Fatalf("unexpected release date: %q", enriched.ReleaseDate) + } + if enriched.DateConfidence != "unknown" || enriched.DateSourceKind != "catalog_backfill" { + t.Fatalf("unexpected catalog metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind) + } +} + func TestEnrichBytedanceModelMetadataUsesTwoPointZeroReleaseDate(t *testing.T) { enriched := enrichBytedanceModelMetadata(ModelPricing{ ModelID: "bytedance-doubao-seed-2.0-pro-256k", diff --git a/scripts/import_phase2_data.go b/scripts/import_phase2_data.go index d1f7506..17cffff 100644 --- a/scripts/import_phase2_data.go +++ b/scripts/import_phase2_data.go @@ -152,6 +152,41 @@ var baiduModelMetadataRules = []baiduModelMetadata{ DateConfidence: "official_primary", DateSourceKind: "official_announcement", }, + { + Prefix: "baidu-qianfan-", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "baidu-deepseek-", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "baidu-glm-", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "baidu-qwen", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "baidu-minimax-", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "baidu-kimi-", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "baidu-internvl", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, } func enrichBaiduModelMetadata(model ModelPricing) ModelPricing { diff --git a/scripts/import_phase2_data_test.go b/scripts/import_phase2_data_test.go index e65c1ce..4e45ab1 100644 --- a/scripts/import_phase2_data_test.go +++ b/scripts/import_phase2_data_test.go @@ -86,7 +86,7 @@ func TestEnrichBaiduModelMetadataUsesSpecificFamilyRules(t *testing.T) { func TestEnrichBaiduModelMetadataFallsBackToPricingSource(t *testing.T) { enriched := enrichBaiduModelMetadata(ModelPricing{ - ModelID: "baidu-deepseek-r1", + ModelID: "baidu-unknown-preview", SourceURL: "https://cloud.baidu.com/doc/qianfan/s/wmh4sv6ya", }) @@ -118,6 +118,20 @@ func TestEnrichBaiduModelMetadataSupportsSourceOnlyRules(t *testing.T) { } } +func TestEnrichBaiduModelMetadataSupportsCatalogBackfillRules(t *testing.T) { + enriched := enrichBaiduModelMetadata(ModelPricing{ + ModelID: "baidu-qianfan-vl-70b", + SourceURL: "https://cloud.baidu.com/doc/qianfan/s/wmh4sv6ya", + }) + + if enriched.ReleaseDate != "" { + t.Fatalf("unexpected release date: %q", enriched.ReleaseDate) + } + if enriched.DateConfidence != "unknown" || enriched.DateSourceKind != "catalog_backfill" { + t.Fatalf("unexpected catalog metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind) + } +} + func TestBaiduReleaseDateValueReturnsNilForUnknownDate(t *testing.T) { if value := releaseDateValue(""); value != nil { t.Fatalf("blank release date should stay nil, got %v", value) diff --git a/scripts/import_zhipu_data.go b/scripts/import_zhipu_data.go index c5ecff4..f685527 100644 --- a/scripts/import_zhipu_data.go +++ b/scripts/import_zhipu_data.go @@ -103,6 +103,61 @@ var zhipuModelMetadataRules = []zhipuModelMetadata{ DateConfidence: "official_primary", DateSourceKind: "official_announcement", }, + { + Prefix: "glm-4.5-air", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "glm-4.5v", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "glm-4-air", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "glm-4-long", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "glm-4v", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "chatglm3", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "glm-4-9b", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "glm-4-0520", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "glm-realtime", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "glm-asr", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, + { + Prefix: "glm-4-voice", + DateConfidence: "unknown", + DateSourceKind: "catalog_backfill", + }, } func enrichZhipuModelMetadata(model ModelPricing) ModelPricing { diff --git a/scripts/import_zhipu_data_test.go b/scripts/import_zhipu_data_test.go index ca7198d..1ada7ae 100644 --- a/scripts/import_zhipu_data_test.go +++ b/scripts/import_zhipu_data_test.go @@ -94,6 +94,34 @@ func TestEnrichZhipuModelMetadataFallsBackToPricingSource(t *testing.T) { } } +func TestEnrichZhipuModelMetadataSupportsCatalogBackfillRules(t *testing.T) { + enriched := enrichZhipuModelMetadata(ModelPricing{ + ModelID: "glm-realtime-air", + SourceURL: "https://open.bigmodel.cn/pricing", + }) + + if enriched.ReleaseDate != "" { + t.Fatalf("unexpected release date: %q", enriched.ReleaseDate) + } + if enriched.DateConfidence != "unknown" || enriched.DateSourceKind != "catalog_backfill" { + t.Fatalf("unexpected catalog metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind) + } +} + +func TestEnrichZhipuModelMetadataSupportsLegacyCatalogBackfillRules(t *testing.T) { + enriched := enrichZhipuModelMetadata(ModelPricing{ + ModelID: "glm-4-0520", + SourceURL: "https://open.bigmodel.cn/pricing", + }) + + if enriched.ReleaseDate != "" { + t.Fatalf("unexpected release date: %q", enriched.ReleaseDate) + } + if enriched.DateConfidence != "unknown" || enriched.DateSourceKind != "catalog_backfill" { + t.Fatalf("unexpected legacy catalog metadata: confidence=%q kind=%q", enriched.DateConfidence, enriched.DateSourceKind) + } +} + func TestZhipuReleaseDateValueReturnsNilForUnknownDate(t *testing.T) { if value := releaseDateValue(""); value != nil { t.Fatalf("blank release date should stay nil, got %v", value)