feat(import): persist official model release metadata

This commit is contained in:
phamnazage-jpg
2026-05-13 21:46:30 +08:00
parent b9ca312366
commit efc3d5cdbd
4 changed files with 120 additions and 11 deletions

View File

@@ -7,6 +7,7 @@ import (
"encoding/json" "encoding/json"
"log" "log"
"os" "os"
"time"
_ "github.com/lib/pq" _ "github.com/lib/pq"
) )
@@ -25,9 +26,21 @@ type ModelPricing struct {
ContextLength int ContextLength int
IsFree bool IsFree bool
SourceURL string SourceURL string
ReleaseDate string
Modality string Modality string
} }
func releaseDateValue(raw string) time.Time {
if raw == "" {
return time.Now()
}
parsed, err := time.Parse("2006-01-02", raw)
if err != nil {
return time.Now()
}
return parsed
}
func main() { func main() {
dsn := os.Getenv("DATABASE_URL") dsn := os.Getenv("DATABASE_URL")
if dsn == "" { if dsn == "" {
@@ -116,15 +129,23 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID) err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
err = db.QueryRow( err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id) `INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7) RETURNING id`, VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.SourceURL, releaseDateValue(p.ReleaseDate),
).Scan(&modelID) ).Scan(&modelID)
} }
if err != nil { if err != nil {
log.Printf("Model error for %s: %v", p.ModelID, err) log.Printf("Model error for %s: %v", p.ModelID, err)
continue continue
} }
_, _ = db.Exec(
`UPDATE models
SET source_url = COALESCE(NULLIF(source_url, ''), $2),
release_date = COALESCE(release_date, $3),
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, p.SourceURL, releaseDateValue(p.ReleaseDate),
)
// Insert pricing // Insert pricing
sourceType := p.OperatorType sourceType := p.OperatorType

View File

@@ -9,6 +9,7 @@ import (
"log" "log"
"os" "os"
"strings" "strings"
"time"
_ "github.com/lib/pq" _ "github.com/lib/pq"
) )
@@ -48,10 +49,22 @@ type ModelPricing struct {
ContextLength int ContextLength int
IsFree bool IsFree bool
SourceURL string SourceURL string
ReleaseDate string
Modality string Modality string
SceneTags []string SceneTags []string
} }
func releaseDateValue(raw string) time.Time {
if strings.TrimSpace(raw) == "" {
return time.Now()
}
parsed, err := time.Parse("2006-01-02", raw)
if err != nil {
return time.Now()
}
return parsed
}
func parseZhipuPrice(s string) float64 { func parseZhipuPrice(s string) float64 {
// Extract price from strings like "6元", "免费", "限时免费" // Extract price from strings like "6元", "免费", "限时免费"
if strings.Contains(s, "免费") { if strings.Contains(s, "免费") {
@@ -189,15 +202,23 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID) err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
err = db.QueryRow( err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id) `INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7) RETURNING id`, VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.SourceURL, releaseDateValue(p.ReleaseDate),
).Scan(&modelID) ).Scan(&modelID)
} }
if err != nil { if err != nil {
log.Printf("Model error: %v", err) log.Printf("Model error: %v", err)
continue continue
} }
_, _ = db.Exec(
`UPDATE models
SET source_url = COALESCE(NULLIF(source_url, ''), $2),
release_date = COALESCE(release_date, $3),
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, p.SourceURL, releaseDateValue(p.ReleaseDate),
)
// Insert pricing // Insert pricing
sourceType := p.OperatorType sourceType := p.OperatorType

View File

@@ -6,6 +6,7 @@ import (
"database/sql" "database/sql"
"log" "log"
"os" "os"
"time"
_ "github.com/lib/pq" _ "github.com/lib/pq"
) )
@@ -24,10 +25,22 @@ type ModelPricing struct {
ContextLength int ContextLength int
IsFree bool IsFree bool
SourceURL string SourceURL string
ReleaseDate string
Modality string Modality string
SceneTags []string SceneTags []string
} }
func releaseDateValue(raw string) time.Time {
if raw == "" {
return time.Now()
}
parsed, err := time.Parse("2006-01-02", raw)
if err != nil {
return time.Now()
}
return parsed
}
func main() { func main() {
dsn := os.Getenv("DATABASE_URL") dsn := os.Getenv("DATABASE_URL")
if dsn == "" { if dsn == "" {
@@ -146,15 +159,23 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID) err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
err = db.QueryRow( err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id) `INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7) RETURNING id`, VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.SourceURL, releaseDateValue(p.ReleaseDate),
).Scan(&modelID) ).Scan(&modelID)
} }
if err != nil { if err != nil {
log.Printf("Model error for %s: %v", p.ModelID, err) log.Printf("Model error for %s: %v", p.ModelID, err)
continue continue
} }
_, _ = db.Exec(
`UPDATE models
SET source_url = COALESCE(NULLIF(source_url, ''), $2),
release_date = COALESCE(release_date, $3),
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, p.SourceURL, releaseDateValue(p.ReleaseDate),
)
// Insert pricing // Insert pricing
sourceType := p.OperatorType sourceType := p.OperatorType

View File

@@ -0,0 +1,46 @@
//go:build llm_script
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestOfficialImportScriptsWriteModelSourceURLAndReleaseDate(t *testing.T) {
projectRoot, err := os.Getwd()
if err != nil {
t.Fatalf("getwd: %v", err)
}
scriptDir := projectRoot
if filepath.Base(projectRoot) != "scripts" {
scriptDir = filepath.Join(projectRoot, "scripts")
}
scripts := []string{
"import_phase2_data.go",
"import_zhipu_data.go",
"import_bytedance_data.go",
}
for _, relativePath := range scripts {
contentBytes, err := os.ReadFile(filepath.Join(scriptDir, relativePath))
if err != nil {
t.Fatalf("read %s: %v", relativePath, err)
}
content := string(contentBytes)
if !strings.Contains(content, "INSERT INTO models") {
t.Fatalf("%s missing models insert statement", relativePath)
}
if !strings.Contains(content, "source_url") {
t.Fatalf("%s missing source_url in models write path", relativePath)
}
if !strings.Contains(content, "release_date") {
t.Fatalf("%s missing release_date in models write path", relativePath)
}
}
}