llm-intelligence/scripts/import_bytedance_data.go

//go:build llm_script && !scripts_pkg

package main

import (
	"database/sql"
	"encoding/json"
	"log"
	"os"
	"strings"
	"time"

	_ "github.com/lib/pq"
)

type ModelPricing struct {
	ModelID         string
	ModelName       string
	ProviderName    string
	ProviderCountry string
	OperatorName    string
	OperatorType    string
	Region          string
	Currency        string
	InputPrice      float64
	OutputPrice     float64
	ContextLength   int
	IsFree          bool
	SourceURL       string
	ModelSourceURL  string
	ReleaseDate     string
	DateConfidence  string
	DateSourceKind  string
	Modality        string
}

func releaseDateValue(raw string) any {
	if raw == "" {
		return nil
	}
	parsed, err := time.Parse("2006-01-02", raw)
	if err != nil {
		return nil
	}
	return parsed
}

type bytedanceModelMetadata struct {
	Prefix         string
	ReleaseDate    string
	ModelSourceURL string
	DateConfidence string
	DateSourceKind string
}

var bytedanceModelMetadataRules = []bytedanceModelMetadata{
	{
		Prefix:         "bytedance-doubao-1.5-vision-pro",
		ReleaseDate:    "2025-01-22",
		ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
		DateConfidence: "official_primary",
		DateSourceKind: "official_announcement",
	},
	{
		Prefix:         "bytedance-doubao-1.5-pro",
		ReleaseDate:    "2025-01-22",
		ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
		DateConfidence: "official_primary",
		DateSourceKind: "official_announcement",
	},
	{
		Prefix:         "bytedance-doubao-1.5-lite",
		ReleaseDate:    "2025-01-22",
		ModelSourceURL: "https://developer.volcengine.com/articles/7462939272262189083",
		DateConfidence: "official_primary",
		DateSourceKind: "official_announcement",
	},
	{
		Prefix:         "bytedance-doubao-1.5-thinking",
		ReleaseDate:    "2025-04-17",
		ModelSourceURL: "https://developer.volcengine.com/articles/7496718897794039827",
		DateConfidence: "official_primary",
		DateSourceKind: "official_announcement",
	},
	{
		Prefix:         "bytedance-doubao-seed-1.6",
		ReleaseDate:    "2025-06-11",
		ModelSourceURL: "https://developer.volcengine.com/articles/7517188354606104612",
		DateConfidence: "official_primary",
		DateSourceKind: "official_announcement",
	},
	{
		Prefix:         "bytedance-doubao-seed-1.8",
		ReleaseDate:    "2025-12-18",
		ModelSourceURL: "https://developer.volcengine.com/articles/7601918680544641034",
		DateConfidence: "secondary_authoritative",
		DateSourceKind: "secondary_authoritative_report",
	},
	{
		Prefix:         "bytedance-doubao-seed-2.0-code",
		ReleaseDate:    "2026-02-14",
		ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
		DateConfidence: "official_primary",
		DateSourceKind: "official_announcement",
	},
	{
		Prefix:         "bytedance-doubao-seed-2.0-pro",
		ReleaseDate:    "2026-02-14",
		ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
		DateConfidence: "official_primary",
		DateSourceKind: "official_announcement",
	},
	{
		Prefix:         "bytedance-doubao-seed-2.0-mini",
		ReleaseDate:    "2026-02-14",
		ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
		DateConfidence: "official_primary",
		DateSourceKind: "official_announcement",
	},
	{
		Prefix:         "bytedance-doubao-seed-2.0-lite",
		ReleaseDate:    "2026-02-14",
		ModelSourceURL: "https://developer.volcengine.com/articles/7610285824933445675",
		DateConfidence: "official_primary",
		DateSourceKind: "official_announcement",
	},
	{
		Prefix:         "bytedance-doubao-seed-code",
		ReleaseDate:    "2024-06-26",
		ModelSourceURL: "https://developer.volcengine.com/articles/7383101327527641125",
		DateConfidence: "official_primary",
		DateSourceKind: "official_announcement",
	},
	{
		Prefix:         "bytedance-doubao-pro",
		ReleaseDate:    "2024-05-15",
		DateConfidence: "secondary_authoritative",
		DateSourceKind: "secondary_authoritative_report",
	},
	{
		Prefix:         "bytedance-doubao-seed-character",
		ReleaseDate:    "2024-05-15",
		DateConfidence: "secondary_authoritative",
		DateSourceKind: "secondary_authoritative_report",
	},
	{
		Prefix:         "bytedance-glm-4.7",
		DateConfidence: "unknown",
		DateSourceKind: "catalog_backfill",
	},
	{
		Prefix:         "bytedance-deepseek-",
		DateConfidence: "unknown",
		DateSourceKind: "catalog_backfill",
	},
	{
		Prefix:         "bytedance-seedance-1.0-lite",
		ReleaseDate:    "2025-05-13",
		ModelSourceURL: "https://developer.volcengine.com/articles/7504284064976502823",
		DateConfidence: "official_primary",
		DateSourceKind: "official_announcement",
	},
}

func enrichBytedanceModelMetadata(model ModelPricing) ModelPricing {
	normalizedID := strings.ToLower(model.ModelID)
	for _, metadata := range bytedanceModelMetadataRules {
		if strings.HasPrefix(normalizedID, metadata.Prefix) {
			if metadata.ReleaseDate != "" {
				model.ReleaseDate = metadata.ReleaseDate
			}
			if metadata.ModelSourceURL != "" {
				model.ModelSourceURL = metadata.ModelSourceURL
			}
			if metadata.DateConfidence != "" {
				model.DateConfidence = metadata.DateConfidence
			}
			if metadata.DateSourceKind != "" {
				model.DateSourceKind = metadata.DateSourceKind
			}
			return model
		}
	}
	if model.ModelSourceURL == "" {
		model.ModelSourceURL = model.SourceURL
	}
	if model.DateConfidence == "" {
		model.DateConfidence = "unknown"
	}
	if model.DateSourceKind == "" {
		model.DateSourceKind = "unknown"
	}
	return model
}

func hasExplicitModelMetadata(model ModelPricing) bool {
	return strings.TrimSpace(model.ReleaseDate) != "" ||
		firstNonEmpty(model.ModelSourceURL) != "" && model.ModelSourceURL != model.SourceURL ||
		strings.TrimSpace(model.DateConfidence) != "" && model.DateConfidence != "unknown" ||
		strings.TrimSpace(model.DateSourceKind) != "" && model.DateSourceKind != "unknown"
}

func main() {
	dsn := os.Getenv("DATABASE_URL")
	if dsn == "" {
		dsn = "postgres://long@/llm_intelligence?host=/var/run/postgresql"
	}

	db, err := sql.Open("postgres", dsn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	// Read raw data
	data, err := os.ReadFile("/tmp/bytedance_raw.json")
	if err != nil {
		log.Fatal("Failed to read raw data:", err)
	}

	var raw struct {
		Bytedance []struct {
			Model         string  `json:"model"`
			InputPrice    float64 `json:"inputPrice"`
			OutputPrice   float64 `json:"outputPrice"`
			ContextLength int     `json:"contextLength"`
			Operator      string  `json:"operator"`
			Region        string  `json:"region"`
			Currency      string  `json:"currency"`
		} `json:"bytedance"`
	}

	if err := json.Unmarshal(data, &raw); err != nil {
		log.Fatal("Failed to parse raw data:", err)
	}

	log.Printf("Importing %d ByteDance models...", len(raw.Bytedance))
	batchID := "manual-seed"

	for _, b := range raw.Bytedance {
		p := enrichBytedanceModelMetadata(ModelPricing{
			ModelID:         "bytedance-" + b.Model,
			ModelName:       b.Model,
			ProviderName:    "ByteDance",
			ProviderCountry: "CN",
			OperatorName:    "ByteDance Volcano",
			OperatorType:    "official",
			Region:          "CN",
			Currency:        "CNY",
			InputPrice:      b.InputPrice,
			OutputPrice:     b.OutputPrice,
			ContextLength:   b.ContextLength,
			IsFree:          b.InputPrice == 0,
			SourceURL:       "https://www.volcengine.com/docs/82379/1099320",
			Modality:        "text",
		})

		// Find or create provider
		var providerID int64
		err := db.QueryRow("SELECT id FROM model_provider WHERE name = $1", p.ProviderName).Scan(&providerID)
		if err == sql.ErrNoRows {
			err = db.QueryRow(
				"INSERT INTO model_provider (name, country, website, status) VALUES ($1, $2, $3, 'active') RETURNING id",
				p.ProviderName, p.ProviderCountry, "",
			).Scan(&providerID)
		}
		if err != nil {
			log.Printf("Provider error: %v", err)
			continue
		}

		// Find or create operator
		var operatorID int64
		err = db.QueryRow("SELECT id FROM operator WHERE name = $1", p.OperatorName).Scan(&operatorID)
		if err == sql.ErrNoRows {
			err = db.QueryRow(
				"INSERT INTO operator (name, country, status) VALUES ($1, $2, 'active') RETURNING id",
				p.OperatorName, p.ProviderCountry,
			).Scan(&operatorID)
		}
		if err != nil {
			log.Printf("Operator error: %v", err)
			continue
		}

		// Find or create model
		var modelID int64
		err = db.QueryRow("SELECT id FROM models WHERE external_id = $1", p.ModelID).Scan(&modelID)
		if err == sql.ErrNoRows {
			err = db.QueryRow(
				`INSERT INTO models (external_id, name, provider_id, modality, context_length, status, source, batch_id, source_url, release_date, date_confidence, date_source_kind) 
				 VALUES ($1, $2, $3, $4, $5, 'active', $6, $7, $8, $9, $10, $11) RETURNING id`,
				p.ModelID, p.ModelName, providerID, p.Modality, p.ContextLength, p.OperatorName, batchID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), p.DateConfidence, p.DateSourceKind,
			).Scan(&modelID)
		}
		if err != nil {
			log.Printf("Model error for %s: %v", p.ModelID, err)
			continue
		}
		if _, err := db.Exec(
			`UPDATE models
			 SET source_url = CASE
			                    WHEN $4 THEN $2
			                    ELSE COALESCE(NULLIF(source_url, ''), $2)
			                  END,
			     release_date = CASE
			                      WHEN $4 THEN $3::date
			                      ELSE COALESCE(release_date, $3::date)
			                    END,
			     date_confidence = CASE
			                         WHEN $4 THEN $5
			                         ELSE COALESCE(NULLIF(date_confidence, ''), $5, 'unknown')
			                       END,
			     date_source_kind = CASE
			                          WHEN $4 THEN $6
			                          ELSE COALESCE(NULLIF(date_source_kind, ''), $6, 'unknown')
			                        END,
			     updated_at = CURRENT_TIMESTAMP
			 WHERE id = $1`,
			modelID, firstNonEmpty(p.ModelSourceURL, p.SourceURL), releaseDateValue(p.ReleaseDate), hasExplicitModelMetadata(p), p.DateConfidence, p.DateSourceKind,
		); err != nil {
			log.Printf("Model metadata update error for %s: %v", p.ModelID, err)
		}

		// Insert pricing
		sourceType := p.OperatorType
		freeQuota := ""
		freeLimitations := "[]"
		rateLimit := "{}"
		if p.IsFree {
			sourceType = "free_tier"
			freeQuota = "Imported free-tier pricing entry"
			freeLimitations = `["See source_url for current quota and policy"]`
		}
		_, err = db.Exec(
			`INSERT INTO region_pricing 
			 (model_id, operator_id, region, currency, input_price_per_mtok, output_price_per_mtok, is_free, effective_date, source_url, source_type, free_quota, free_limitations, rate_limit)
			 VALUES ($1, $2, $3, $4, $5, $6, $7, CURRENT_DATE, $8, $9, $10, $11, $12)
			 ON CONFLICT (model_id, operator_id, region, currency, effective_date) 
			 DO UPDATE SET input_price_per_mtok = EXCLUDED.input_price_per_mtok,
			               output_price_per_mtok = EXCLUDED.output_price_per_mtok,
			               is_free = EXCLUDED.is_free,
			               source_type = EXCLUDED.source_type,
			               free_quota = EXCLUDED.free_quota,
			               free_limitations = EXCLUDED.free_limitations,
			               rate_limit = EXCLUDED.rate_limit,
			               updated_at = CURRENT_TIMESTAMP`,
			modelID, operatorID, p.Region, p.Currency, p.InputPrice, p.OutputPrice, p.IsFree, p.SourceURL,
			sourceType, freeQuota, freeLimitations, rateLimit,
		)
		if err != nil {
			log.Printf("Pricing error for %s: %v", p.ModelID, err)
			continue
		}
	}

	log.Printf("Successfully imported %d ByteDance models", len(raw.Bytedance))
}

func firstNonEmpty(values ...string) string {
	for _, value := range values {
		if value != "" {
			return value
		}
	}
	return ""
}