niuniu/llm-intelligence
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(import): track release date evidence tiers

Browse Source
This commit is contained in:
phamnazage-jpg
2026-05-13 23:27:47 +08:00
parent 569b94cb73
commit f2f68b85c1
10 changed files with 599 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
scripts/import_phase2_data.go
View File

@@ -51,6 +51,8 @@ type ModelPricing struct {
SourceURL string
ModelSourceURL string
ReleaseDate string
DateConfidence string
DateSourceKind string
Modality string
SceneTags []string
}
@@ -70,6 +72,8 @@ type baiduModelMetadata struct {
Prefix string
ReleaseDate string
ModelSourceURL string
DateConfidence string
DateSourceKind string
}
var baiduModelMetadataRules = []baiduModelMetadata{
@@ -77,54 +81,76 @@ var baiduModelMetadataRules = []baiduModelMetadata{
Prefix: "baidu-ernie-5.0",
ReleaseDate: "2026-01-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_eacd0f0b-0ca3-4963-aec8-5e6b9ebef9ba",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-x1.1",
ReleaseDate: "2025-09-09",
ModelSourceURL: "https://cloud.baidu.com/news/news_be713ff4-8477-4852-88f1-9cc56c406d6a",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-5.1",
ModelSourceURL: "https://cloud.baidu.com/product/wenxinworkshop.html",
DateConfidence: "unknown",
DateSourceKind: "official_product_page",
},
{
Prefix: "baidu-ernie-4.5-turbo-vl",
ModelSourceURL: "https://cloud.baidu.com/product/wenxinworkshop.html",
DateConfidence: "unknown",
DateSourceKind: "official_product_page",
},
{
Prefix: "baidu-ernie-4.5-turbo",
ReleaseDate: "2025-04-25",
ModelSourceURL: "https://cloud.baidu.com/article/3887765",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-x1-turbo",
ReleaseDate: "2025-04-25",
ModelSourceURL: "https://cloud.baidu.com/article/3887765",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-4.5",
ReleaseDate: "2025-03-16",
ModelSourceURL: "https://cloud.baidu.com/article/3835921",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-x1",
ReleaseDate: "2025-03-16",
ModelSourceURL: "https://cloud.baidu.com/article/3835921",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-character",
ReleaseDate: "2024-03-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-lite-pro",
ReleaseDate: "2024-03-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
{
Prefix: "baidu-ernie-speed-pro",
ReleaseDate: "2024-03-22",
ModelSourceURL: "https://cloud.baidu.com/news/news_667c065f-0bd7-475d-98c2-901763d0ee77",
DateConfidence: "official_primary",
DateSourceKind: "official_announcement",
},
}
@@ -138,17 +164,32 @@ func enrichBaiduModelMetadata(model ModelPricing) ModelPricing {
if metadata.ModelSourceURL != "" {
model.ModelSourceURL = metadata.ModelSourceURL
}
if metadata.DateConfidence != "" {
model.DateConfidence = metadata.DateConfidence
}
if metadata.DateSourceKind != "" {
model.DateSourceKind = metadata.DateSourceKind
}
return model
}
}
if model.ModelSourceURL == "" {
model.ModelSourceURL = model.SourceURL
}
if model.DateConfidence == "" {
model.DateConfidence = "unknown"
}
if model.DateSourceKind == "" {
model.DateSourceKind = "unknown"
}
return model
}
func hasExplicitModelMetadata(model ModelPricing) bool {
return strings.TrimSpace(model.ReleaseDate) != "" || firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL
return strings.TrimSpace(model.ReleaseDate) != "" ||
firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL ||
strings.TrimSpace(model.DateConfidence) != "" && model.DateConfidence != "unknown" ||
strings.TrimSpace(model.DateSourceKind) != "" && model.DateSourceKind != "unknown"
}
func parseZhipuPrice(s string) float64 {
@@ -288,9 +329,9 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows {
err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date, date_confidence, date_source_kind)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9, $10, $11) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), p.DateConfidence, p.DateSourceKind,
).Scan(&modelID)
}
if err != nil {
@@ -304,12 +345,20 @@ func main() {
ELSE COALESCE(NULLIF(source_url, ''), $2)
END,
release_date = CASE
WHEN $4 AND $3::date IS NOT NULL THEN $3::date
WHEN $4 THEN $3::date
ELSE COALESCE(release_date, $3::date)
END,
date_confidence = CASE
WHEN $4 THEN $5
ELSE COALESCE(NULLIF(date_confidence, ''), $5, 'unknown')
END,
date_source_kind = CASE
WHEN $4 THEN $6
ELSE COALESCE(NULLIF(date_source_kind, ''), $6, 'unknown')
END,
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p),
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p), p.DateConfidence, p.DateSourceKind,
); err != nil {
log.Printf("Model metadata update error for %s: %v", p.ModelID, err)
}