niuniu/llm-intelligence
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(import): persist official model release metadata

Browse Source
This commit is contained in:
phamnazage-jpg
2026-05-13 21:46:30 +08:00
parent b9ca312366
commit efc3d5cdbd
4 changed files with 120 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
scripts/import_bytedance_data.go
View File

@@ -7,6 +7,7 @@ import (
"encoding/json"
"log"
"os"
"time"
_ "github.com/lib/pq"
)
@@ -25,9 +26,21 @@ type ModelPricing struct {
ContextLength int
IsFree bool
SourceURL string
ReleaseDate string
Modality string
}
func releaseDateValue(raw string) time.Time {
if raw == "" {
return time.Now()
}
parsed, err := time.Parse("2006-01-02", raw)
if err != nil {
return time.Now()
}
return parsed
}
func main() {
dsn := os.Getenv("DATABASE_URL")
if dsn == "" {
@@ -116,15 +129,23 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows {
err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID,
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.SourceURL, releaseDateValue(p.ReleaseDate),
).Scan(&modelID)
}
if err != nil {
log.Printf("Model error for %s: %v", p.ModelID, err)
continue
}
_, _ = db.Exec(
`UPDATE models
SET source_url = COALESCE(NULLIF(source_url, ''), $2),
release_date = COALESCE(release_date, $3),
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, p.SourceURL, releaseDateValue(p.ReleaseDate),
)
// Insert pricing
sourceType := p.OperatorType

31
scripts/import_phase2_data.go
View File

@@ -9,6 +9,7 @@ import (
"log"
"os"
"strings"
"time"
_ "github.com/lib/pq"
)
@@ -48,10 +49,22 @@ type ModelPricing struct {
ContextLength int
IsFree bool
SourceURL string
ReleaseDate string
Modality string
SceneTags []string
}
func releaseDateValue(raw string) time.Time {
if strings.TrimSpace(raw) == "" {
return time.Now()
}
parsed, err := time.Parse("2006-01-02", raw)
if err != nil {
return time.Now()
}
return parsed
}
func parseZhipuPrice(s string) float64 {
// Extract price from strings like "6元", "免费", "限时免费"
if strings.Contains(s, "免费") {
@@ -188,16 +201,24 @@ func main() {
var modelID int64
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows {
err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID,
).Scan(&modelID)
err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.SourceURL, releaseDateValue(p.ReleaseDate),
).Scan(&modelID)
}
if err != nil {
log.Printf("Model error: %v", err)
continue
}
_, _ = db.Exec(
`UPDATE models
SET source_url = COALESCE(NULLIF(source_url, ''), $2),
release_date = COALESCE(release_date, $3),
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, p.SourceURL, releaseDateValue(p.ReleaseDate),
)
// Insert pricing
sourceType := p.OperatorType

27
scripts/import_zhipu_data.go
View File

@@ -6,6 +6,7 @@ import (
"database/sql"
"log"
"os"
"time"
_ "github.com/lib/pq"
)
@@ -24,10 +25,22 @@ type ModelPricing struct {
ContextLength int
IsFree bool
SourceURL string
ReleaseDate string
Modality string
SceneTags []string
}
func releaseDateValue(raw string) time.Time {
if raw == "" {
return time.Now()
}
parsed, err := time.Parse("2006-01-02", raw)
if err != nil {
return time.Now()
}
return parsed
}
func main() {
dsn := os.Getenv("DATABASE_URL")
if dsn == "" {
@@ -146,15 +159,23 @@ func main() {
err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
if err == sql.ErrNoRows {
err = db.QueryRow(
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID,
`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date)
VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.SourceURL, releaseDateValue(p.ReleaseDate),
).Scan(&modelID)
}
if err != nil {
log.Printf("Model error for %s: %v", p.ModelID, err)
continue
}
_, _ = db.Exec(
`UPDATE models
SET source_url = COALESCE(NULLIF(source_url, ''), $2),
release_date = COALESCE(release_date, $3),
updated_at = CURRENT_TIMESTAMP
WHERE id = $1`,
modelID, p.SourceURL, releaseDateValue(p.ReleaseDate),
)
// Insert pricing
sourceType := p.OperatorType

46
scripts/official_import_scripts_test.go Normal file
View File

@@ -0,0 +1,46 @@
//go:build llm_script
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestOfficialImportScriptsWriteModelSourceURLAndReleaseDate(t *testing.T) {
projectRoot, err := os.Getwd()
if err != nil {
t.Fatalf("getwd: %v", err)
}
scriptDir := projectRoot
if filepath.Base(projectRoot) != "scripts" {
scriptDir = filepath.Join(projectRoot, "scripts")
}
scripts := []string{
"import_phase2_data.go",
"import_zhipu_data.go",
"import_bytedance_data.go",
}
for _, relativePath := range scripts {
contentBytes, err := os.ReadFile(filepath.Join(scriptDir, relativePath))
if err != nil {
t.Fatalf("read %s: %v", relativePath, err)
}
content := string(contentBytes)
if !strings.Contains(content, "INSERT INTO models") {
t.Fatalf("%s missing models insert statement", relativePath)
}
if !strings.Contains(content, "source_url") {
t.Fatalf("%s missing source_url in models write path", relativePath)
}
if !strings.Contains(content, "release_date") {
t.Fatalf("%s missing release_date in models write path", relativePath)
}
}
}