From efc3d5cdbd49ffa083d5e36bd127c4b9d372a710 Mon Sep 17 00:00:00 2001 From: phamnazage-jpg Date: Wed, 13 May 2026 21:46:30 +0800 Subject: [PATCH] feat(import): persist official model release metadata --- scripts/import_bytedance_data.go | 27 +++++++++++++-- scripts/import_phase2_data.go | 31 ++++++++++++++--- scripts/import_zhipu_data.go | 27 +++++++++++++-- scripts/official_import_scripts_test.go | 46 +++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 11 deletions(-) create mode 100644 scripts/official_import_scripts_test.go diff --git a/scripts/import_bytedance_data.go b/scripts/import_bytedance_data.go index a680a41..3e21838 100644 --- a/scripts/import_bytedance_data.go +++ b/scripts/import_bytedance_data.go @@ -7,6 +7,7 @@ import ( "encoding/json" "log" "os" + "time" _ "github.com/lib/pq" ) @@ -25,9 +26,21 @@ type ModelPricing struct { ContextLength int IsFree bool SourceURL string + ReleaseDate string Modality string } +func releaseDateValue(raw string) time.Time { + if raw == "" { + return time.Now() + } + parsed, err := time.Parse("2006-01-02", raw) + if err != nil { + return time.Now() + } + return parsed +} + func main() { dsn := os.Getenv("DATABASE_URL") if dsn == "" { @@ -116,15 +129,23 @@ func main() { err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID) if err == sql.ErrNoRows { err = db.QueryRow( - `INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id) - VALUES ($1, $2, $3, $4, $5, 'active', $6, $7) RETURNING id`, - p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, + `INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date) + VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`, + p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.SourceURL, releaseDateValue(p.ReleaseDate), ).Scan(&modelID) } if err != nil { log.Printf("Model error for %s: %v", p.ModelID, err) continue } + _, _ = db.Exec( + `UPDATE models + SET source_url = COALESCE(NULLIF(source_url, ''), $2), + release_date = COALESCE(release_date, $3), + updated_at = CURRENT_TIMESTAMP + WHERE id = $1`, + modelID, p.SourceURL, releaseDateValue(p.ReleaseDate), + ) // Insert pricing sourceType := p.OperatorType diff --git a/scripts/import_phase2_data.go b/scripts/import_phase2_data.go index 77f61c6..5622324 100644 --- a/scripts/import_phase2_data.go +++ b/scripts/import_phase2_data.go @@ -9,6 +9,7 @@ import ( "log" "os" "strings" + "time" _ "github.com/lib/pq" ) @@ -48,10 +49,22 @@ type ModelPricing struct { ContextLength int IsFree bool SourceURL string + ReleaseDate string Modality string SceneTags []string } +func releaseDateValue(raw string) time.Time { + if strings.TrimSpace(raw) == "" { + return time.Now() + } + parsed, err := time.Parse("2006-01-02", raw) + if err != nil { + return time.Now() + } + return parsed +} + func parseZhipuPrice(s string) float64 { // Extract price from strings like "6元", "免费", "限时免费" if strings.Contains(s, "免费") { @@ -188,16 +201,24 @@ func main() { var modelID int64 err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID) if err == sql.ErrNoRows { - err = db.QueryRow( - `INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id) - VALUES ($1, $2, $3, $4, $5, 'active', $6, $7) RETURNING id`, - p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, - ).Scan(&modelID) + err = db.QueryRow( + `INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date) + VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`, + p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.SourceURL, releaseDateValue(p.ReleaseDate), + ).Scan(&modelID) } if err != nil { log.Printf("Model error: %v", err) continue } + _, _ = db.Exec( + `UPDATE models + SET source_url = COALESCE(NULLIF(source_url, ''), $2), + release_date = COALESCE(release_date, $3), + updated_at = CURRENT_TIMESTAMP + WHERE id = $1`, + modelID, p.SourceURL, releaseDateValue(p.ReleaseDate), + ) // Insert pricing sourceType := p.OperatorType diff --git a/scripts/import_zhipu_data.go b/scripts/import_zhipu_data.go index 0c86e8f..5bc0f1d 100644 --- a/scripts/import_zhipu_data.go +++ b/scripts/import_zhipu_data.go @@ -6,6 +6,7 @@ import ( "database/sql" "log" "os" + "time" _ "github.com/lib/pq" ) @@ -24,10 +25,22 @@ type ModelPricing struct { ContextLength int IsFree bool SourceURL string + ReleaseDate string Modality string SceneTags []string } +func releaseDateValue(raw string) time.Time { + if raw == "" { + return time.Now() + } + parsed, err := time.Parse("2006-01-02", raw) + if err != nil { + return time.Now() + } + return parsed +} + func main() { dsn := os.Getenv("DATABASE_URL") if dsn == "" { @@ -146,15 +159,23 @@ func main() { err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID) if err == sql.ErrNoRows { err = db.QueryRow( - `INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id) - VALUES ($1, $2, $3, $4, $5, 'active', $6, $7) RETURNING id`, - p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, + `INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date) + VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`, + p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.SourceURL, releaseDateValue(p.ReleaseDate), ).Scan(&modelID) } if err != nil { log.Printf("Model error for %s: %v", p.ModelID, err) continue } + _, _ = db.Exec( + `UPDATE models + SET source_url = COALESCE(NULLIF(source_url, ''), $2), + release_date = COALESCE(release_date, $3), + updated_at = CURRENT_TIMESTAMP + WHERE id = $1`, + modelID, p.SourceURL, releaseDateValue(p.ReleaseDate), + ) // Insert pricing sourceType := p.OperatorType diff --git a/scripts/official_import_scripts_test.go b/scripts/official_import_scripts_test.go new file mode 100644 index 0000000..98574a6 --- /dev/null +++ b/scripts/official_import_scripts_test.go @@ -0,0 +1,46 @@ +//go:build llm_script + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestOfficialImportScriptsWriteModelSourceURLAndReleaseDate(t *testing.T) { + projectRoot, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + + scriptDir := projectRoot + if filepath.Base(projectRoot) != "scripts" { + scriptDir = filepath.Join(projectRoot, "scripts") + } + + scripts := []string{ + "import_phase2_data.go", + "import_zhipu_data.go", + "import_bytedance_data.go", + } + + for _, relativePath := range scripts { + contentBytes, err := os.ReadFile(filepath.Join(scriptDir, relativePath)) + if err != nil { + t.Fatalf("read %s: %v", relativePath, err) + } + content := string(contentBytes) + + if !strings.Contains(content, "INSERT INTO models") { + t.Fatalf("%s missing models insert statement", relativePath) + } + if !strings.Contains(content, "source_url") { + t.Fatalf("%s missing source_url in models write path", relativePath) + } + if !strings.Contains(content, "release_date") { + t.Fatalf("%s missing release_date in models write path", relativePath) + } + } +}