feat(import): add explicit zhipu release metadata
This commit is contained in:
@@ -6,6 +6,7 @@ import (
|
|||||||
"database/sql"
|
"database/sql"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
_ "github.com/lib/pq"
|
_ "github.com/lib/pq"
|
||||||
@@ -25,6 +26,7 @@ type ModelPricing struct {
|
|||||||
ContextLength int
|
ContextLength int
|
||||||
IsFree bool
|
IsFree bool
|
||||||
SourceURL string
|
SourceURL string
|
||||||
|
ModelSourceURL string
|
||||||
ReleaseDate string
|
ReleaseDate string
|
||||||
Modality string
|
Modality string
|
||||||
SceneTags []string
|
SceneTags []string
|
||||||
@@ -41,6 +43,68 @@ func releaseDateValue(raw string) time.Time {
|
|||||||
return parsed
|
return parsed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type zhipuModelMetadata struct {
|
||||||
|
Prefix string
|
||||||
|
ReleaseDate string
|
||||||
|
ModelSourceURL string
|
||||||
|
}
|
||||||
|
|
||||||
|
var zhipuModelMetadataRules = []zhipuModelMetadata{
|
||||||
|
{
|
||||||
|
Prefix: "glm-5-turbo",
|
||||||
|
ReleaseDate: "2026-03-15",
|
||||||
|
ModelSourceURL: "https://www.zhipuai.cn/en/research/155",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Prefix: "glm-5.1",
|
||||||
|
ReleaseDate: "2026-04-07",
|
||||||
|
ModelSourceURL: "https://www.zhipuai.cn/zh/research",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Prefix: "glm-5",
|
||||||
|
ReleaseDate: "2026-02-11",
|
||||||
|
ModelSourceURL: "https://www.zhipuai.cn/zh/research/154",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Prefix: "glm-4.7-flash",
|
||||||
|
ReleaseDate: "2026-01-19",
|
||||||
|
ModelSourceURL: "https://www.zhipuai.cn/zh/news/148",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Prefix: "glm-4.7",
|
||||||
|
ReleaseDate: "2025-12-21",
|
||||||
|
ModelSourceURL: "https://www.zhipuai.cn/zh/research",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Prefix: "glm-4.6v",
|
||||||
|
ReleaseDate: "2025-12-07",
|
||||||
|
ModelSourceURL: "https://www.zhipuai.cn/zh/research/144",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Prefix: "glm-tts",
|
||||||
|
ReleaseDate: "2025-12-10",
|
||||||
|
ModelSourceURL: "https://www.zhipuai.cn/zh/research",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func enrichZhipuModelMetadata(model ModelPricing) ModelPricing {
|
||||||
|
for _, metadata := range zhipuModelMetadataRules {
|
||||||
|
if strings.HasPrefix(model.ModelID, metadata.Prefix) {
|
||||||
|
if metadata.ReleaseDate != "" {
|
||||||
|
model.ReleaseDate = metadata.ReleaseDate
|
||||||
|
}
|
||||||
|
if metadata.ModelSourceURL != "" {
|
||||||
|
model.ModelSourceURL = metadata.ModelSourceURL
|
||||||
|
}
|
||||||
|
return model
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if model.ModelSourceURL == "" {
|
||||||
|
model.ModelSourceURL = model.SourceURL
|
||||||
|
}
|
||||||
|
return model
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
dsn := os.Getenv("DATABASE_URL")
|
dsn := os.Getenv("DATABASE_URL")
|
||||||
if dsn == "" {
|
if dsn == "" {
|
||||||
@@ -125,7 +189,8 @@ func main() {
|
|||||||
log.Printf("Importing %d Zhipu AI models...", len(prices))
|
log.Printf("Importing %d Zhipu AI models...", len(prices))
|
||||||
|
|
||||||
// Save to database
|
// Save to database
|
||||||
for _, p := range prices {
|
for _, rawPricing := range prices {
|
||||||
|
p := enrichZhipuModelMetadata(rawPricing)
|
||||||
// Find or create provider
|
// Find or create provider
|
||||||
var providerID int64
|
var providerID int64
|
||||||
err := db.QueryRow("SELECT id FROM model_provider WHERE name = $1", p.ProviderName).Scan(&providerID)
|
err := db.QueryRow("SELECT id FROM model_provider WHERE name = $1", p.ProviderName).Scan(&providerID)
|
||||||
@@ -161,7 +226,7 @@ func main() {
|
|||||||
err = db.QueryRow(
|
err = db.QueryRow(
|
||||||
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
|
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
|
||||||
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
|
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
|
||||||
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.SourceURL, releaseDateValue(p.ReleaseDate),
|
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
|
||||||
).Scan(&modelID)
|
).Scan(&modelID)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -174,7 +239,7 @@ func main() {
|
|||||||
release_date = COALESCE(release_date, $3),
|
release_date = COALESCE(release_date, $3),
|
||||||
updated_at = CURRENT_TIMESTAMP
|
updated_at = CURRENT_TIMESTAMP
|
||||||
WHERE id = $1`,
|
WHERE id = $1`,
|
||||||
modelID, p.SourceURL, releaseDateValue(p.ReleaseDate),
|
modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
|
||||||
)
|
)
|
||||||
|
|
||||||
// Insert pricing
|
// Insert pricing
|
||||||
@@ -211,3 +276,12 @@ func main() {
|
|||||||
|
|
||||||
log.Printf("Successfully imported %d Zhipu AI models", len(prices))
|
log.Printf("Successfully imported %d Zhipu AI models", len(prices))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func firstNonEmpty(values ...string) string {
|
||||||
|
for _, value := range values {
|
||||||
|
if value != "" {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|||||||
72
scripts/import_zhipu_data_test.go
Normal file
72
scripts/import_zhipu_data_test.go
Normal file
@@ -0,0 +1,72 @@
|
|||||||
|
//go:build llm_script
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestEnrichZhipuModelMetadataUsesSpecificFamilyRules(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
modelID string
|
||||||
|
wantReleaseDate string
|
||||||
|
wantSourceURL string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
modelID: "glm-5.1-32k",
|
||||||
|
wantReleaseDate: "2026-04-07",
|
||||||
|
wantSourceURL: "https://www.zhipuai.cn/zh/research",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
modelID: "glm-5-turbo-32k",
|
||||||
|
wantReleaseDate: "2026-03-15",
|
||||||
|
wantSourceURL: "https://www.zhipuai.cn/en/research/155",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
modelID: "glm-5-32k",
|
||||||
|
wantReleaseDate: "2026-02-11",
|
||||||
|
wantSourceURL: "https://www.zhipuai.cn/zh/research/154",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
modelID: "glm-4.7-flash",
|
||||||
|
wantReleaseDate: "2026-01-19",
|
||||||
|
wantSourceURL: "https://www.zhipuai.cn/zh/news/148",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
modelID: "glm-4.6v-flashx",
|
||||||
|
wantReleaseDate: "2025-12-07",
|
||||||
|
wantSourceURL: "https://www.zhipuai.cn/zh/research/144",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
modelID: "glm-tts-clone",
|
||||||
|
wantReleaseDate: "2025-12-10",
|
||||||
|
wantSourceURL: "https://www.zhipuai.cn/zh/research",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
enriched := enrichZhipuModelMetadata(ModelPricing{
|
||||||
|
ModelID: tc.modelID,
|
||||||
|
SourceURL: "https://open.bigmodel.cn/pricing",
|
||||||
|
})
|
||||||
|
|
||||||
|
if enriched.ReleaseDate != tc.wantReleaseDate {
|
||||||
|
t.Fatalf("%s release date = %q, want %q", tc.modelID, enriched.ReleaseDate, tc.wantReleaseDate)
|
||||||
|
}
|
||||||
|
if enriched.ModelSourceURL != tc.wantSourceURL {
|
||||||
|
t.Fatalf("%s source url = %q, want %q", tc.modelID, enriched.ModelSourceURL, tc.wantSourceURL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnrichZhipuModelMetadataFallsBackToPricingSource(t *testing.T) {
|
||||||
|
enriched := enrichZhipuModelMetadata(ModelPricing{
|
||||||
|
ModelID: "glm-unknown-preview",
|
||||||
|
SourceURL: "https://open.bigmodel.cn/pricing",
|
||||||
|
})
|
||||||
|
|
||||||
|
if enriched.ReleaseDate != "" {
|
||||||
|
t.Fatalf("unexpected release date: %q", enriched.ReleaseDate)
|
||||||
|
}
|
||||||
|
if enriched.ModelSourceURL != "https://open.bigmodel.cn/pricing" {
|
||||||
|
t.Fatalf("model source url = %q, want pricing source fallback", enriched.ModelSourceURL)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user