feat(import): enrich baidu and bytedance release metadata

scripts/import_bytedance_data.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"log"
 	"os"
+	"strings"
 	"time"
 
 	_ "github.com/lib/pq"
@@ -26,21 +27,65 @@ type ModelPricing struct {
 	ContextLength   int
 	IsFree          bool
 	SourceURL       string
+	ModelSourceURL  string
 	ReleaseDate     string
 	Modality        string
 }
 
-func releaseDateValue(raw string) time.Time {
+func releaseDateValue(raw string) any {
 	if raw == "" {
-		return time.Now()
+		return nil
 	}
 	parsed, err := time.Parse("2006-01-02", raw)
 	if err != nil {
-		return time.Now()
+		return nil
 	}
 	return parsed
 }
 
+type bytedanceModelMetadata struct {
+	Prefix         string
+	ReleaseDate    string
+	ModelSourceURL string
+}
+
+var bytedanceModelMetadataRules = []bytedanceModelMetadata{
+	{
+		Prefix:         "bytedance-doubao-1.5-thinking",
+		ReleaseDate:    "2025-04-17",
+		ModelSourceURL: "https://developer.volcengine.com/articles/7496718897794039827",
+	},
+	{
+		Prefix:         "bytedance-doubao-seed-1.6",
+		ReleaseDate:    "2025-06-11",
+		ModelSourceURL: "https://developer.volcengine.com/articles/7517188354606104612",
+	},
+	{
+		Prefix:         "bytedance-seedance-1.0-lite",
+		ReleaseDate:    "2025-05-13",
+		ModelSourceURL: "https://developer.volcengine.com/articles/7504284064976502823",
+	},
+}
+
+func enrichBytedanceModelMetadata(model ModelPricing) ModelPricing {
+	normalizedID := strings.ToLower(model.ModelID)
+	for _, metadata := range bytedanceModelMetadataRules {
+		if strings.HasPrefix(normalizedID, metadata.Prefix) {
+			if metadata.ReleaseDate != "" {
+				model.ReleaseDate = metadata.ReleaseDate
+			}
+			if metadata.ModelSourceURL != "" {
+				model.ModelSourceURL = metadata.ModelSourceURL
+			}
+			return model
+		}
+	}
+	if model.ModelSourceURL == "" {
+		model.ModelSourceURL = model.SourceURL
+	}
+	return model
+}
+
 func main() {
 	dsn := os.Getenv("DATABASE_URL")
 	if dsn == "" {
@@ -79,7 +124,7 @@ func main() {
 	batchID := "manual-seed"
 
 	for _, b := range raw.Bytedance {
-		p := ModelPricing{
+		p := enrichBytedanceModelMetadata(ModelPricing{
 			ModelID:         "bytedance-" + b.Model,
 			ModelName:       b.Model,
 			ProviderName:    "ByteDance",
@@ -94,7 +139,7 @@ func main() {
 			IsFree:          b.InputPrice == 0,
 			SourceURL:       "https://www.volcengine.com/docs/82379/1099320",
 			Modality:        "text",
-		}
+		})
 
 		// Find or create provider
 		var providerID int64
@@ -131,7 +176,7 @@ func main() {
 			err = db.QueryRow(
 				`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date) 
 				 VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9) RETURNING id`,
-				p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, p.SourceURL, releaseDateValue(p.ReleaseDate),
+				p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
 			).Scan(&modelID)
 		}
 		if err != nil {
@@ -144,7 +189,7 @@ func main() {
 			     release_date = COALESCE(release_date, $3),
 			     updated_at = CURRENT_TIMESTAMP
 			 WHERE id = $1`,
-			modelID, p.SourceURL, releaseDateValue(p.ReleaseDate),
+			modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate),
 		)
 
 		// Insert pricing
@@ -181,3 +226,12 @@ func main() {
 
 	log.Printf("Successfully imported %d ByteDance models", len(raw.Bytedance))
 }
+
+func firstNonEmpty(values ...string) string {
+	for _, value := range values {
+		if value != "" {
+			return value
+		}
+	}
+	return ""
+}