feat(runtime): harden daily pipeline audit and verification
Tighten real-ingestion success rules, separate scheduled reports from historical rebuilds, and persist source-level runtime audit across daily pipeline runs. Also add the Phase 5 CI workflow contract plus verification updates and supporting docs so the full uncommitted change set can be validated together.
This commit is contained in:
@@ -64,11 +64,14 @@ type sourceDefinition struct {
|
||||
}
|
||||
|
||||
type runSummary struct {
|
||||
SelectedSources int
|
||||
SuccessfulSources int
|
||||
TotalModels int
|
||||
DomesticModels int
|
||||
CurrencyCounts map[string]int
|
||||
SelectedSources int
|
||||
SelectedSourceKeys []string
|
||||
SuccessfulSources int
|
||||
SuccessfulSourceKeys []string
|
||||
FailedSourceKeys []string
|
||||
TotalModels int
|
||||
DomesticModels int
|
||||
CurrencyCounts map[string]int
|
||||
}
|
||||
|
||||
type pricingMetadataFields struct {
|
||||
@@ -256,12 +259,15 @@ func listSourceKeys(apiKey string) []string {
|
||||
return keys
|
||||
}
|
||||
|
||||
func summarizePrices(selectedSources int, successfulSources int, prices []ModelPricing) runSummary {
|
||||
func summarizePrices(selectedSourceKeys []string, successfulSourceKeys []string, failedSourceKeys []string, prices []ModelPricing) runSummary {
|
||||
summary := runSummary{
|
||||
SelectedSources: selectedSources,
|
||||
SuccessfulSources: successfulSources,
|
||||
TotalModels: len(prices),
|
||||
CurrencyCounts: make(map[string]int),
|
||||
SelectedSources: len(selectedSourceKeys),
|
||||
SelectedSourceKeys: append([]string(nil), selectedSourceKeys...),
|
||||
SuccessfulSources: len(successfulSourceKeys),
|
||||
SuccessfulSourceKeys: append([]string(nil), successfulSourceKeys...),
|
||||
FailedSourceKeys: append([]string(nil), failedSourceKeys...),
|
||||
TotalModels: len(prices),
|
||||
CurrencyCounts: make(map[string]int),
|
||||
}
|
||||
for _, price := range prices {
|
||||
if strings.EqualFold(price.ProviderCountry, "CN") {
|
||||
@@ -272,6 +278,21 @@ func summarizePrices(selectedSources int, successfulSources int, prices []ModelP
|
||||
return summary
|
||||
}
|
||||
|
||||
func sourceKey(src DataSource) string {
|
||||
switch strings.ToLower(strings.TrimSpace(src.Name())) {
|
||||
case "openrouter":
|
||||
return "openrouter"
|
||||
case "moonshot":
|
||||
return "moonshot"
|
||||
case "deepseek":
|
||||
return "deepseek"
|
||||
case "openai":
|
||||
return "openai"
|
||||
default:
|
||||
return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(src.Name()), " ", "_"))
|
||||
}
|
||||
}
|
||||
|
||||
func formatCountMap(counts map[string]int) string {
|
||||
if len(counts) == 0 {
|
||||
return "none"
|
||||
@@ -289,17 +310,27 @@ func formatCountMap(counts map[string]int) string {
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func formatKeyList(keys []string) string {
|
||||
if len(keys) == 0 {
|
||||
return "none"
|
||||
}
|
||||
return strings.Join(keys, ",")
|
||||
}
|
||||
|
||||
func printSummary(w io.Writer, summary runSummary) error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
_, err := fmt.Fprintf(
|
||||
w,
|
||||
"sources=%d successful_sources=%d models=%d domestic_models=%d currencies=%s\n",
|
||||
"sources=%d successful_sources=%d models=%d domestic_models=%d selected_source_keys=%s successful_source_keys=%s failed_source_keys=%s currencies=%s\n",
|
||||
summary.SelectedSources,
|
||||
summary.SuccessfulSources,
|
||||
summary.TotalModels,
|
||||
summary.DomesticModels,
|
||||
formatKeyList(summary.SelectedSourceKeys),
|
||||
formatKeyList(summary.SuccessfulSourceKeys),
|
||||
formatKeyList(summary.FailedSourceKeys),
|
||||
formatCountMap(summary.CurrencyCounts),
|
||||
)
|
||||
return err
|
||||
@@ -564,23 +595,29 @@ func defaultDSN() string {
|
||||
|
||||
func runCollector(cfg runConfig, sources []DataSource, saveFn func([]ModelPricing) error, out io.Writer) error {
|
||||
allPrices := make([]ModelPricing, 0)
|
||||
successfulSources := 0
|
||||
selectedSourceKeys := make([]string, 0, len(sources))
|
||||
successfulSourceKeys := make([]string, 0, len(sources))
|
||||
failedSourceKeys := make([]string, 0)
|
||||
|
||||
for _, src := range sources {
|
||||
key := sourceKey(src)
|
||||
selectedSourceKeys = append(selectedSourceKeys, key)
|
||||
|
||||
prices, err := src.FetchPricing()
|
||||
if err != nil {
|
||||
logger.Error("采集失败", "source", src.Name(), "error", err)
|
||||
failedSourceKeys = append(failedSourceKeys, key)
|
||||
continue
|
||||
}
|
||||
successfulSources++
|
||||
successfulSourceKeys = append(successfulSourceKeys, key)
|
||||
allPrices = append(allPrices, prices...)
|
||||
}
|
||||
|
||||
summary := summarizePrices(len(sources), successfulSources, allPrices)
|
||||
summary := summarizePrices(selectedSourceKeys, successfulSourceKeys, failedSourceKeys, allPrices)
|
||||
if err := printSummary(out, summary); err != nil {
|
||||
return err
|
||||
}
|
||||
if successfulSources == 0 {
|
||||
if summary.SuccessfulSources == 0 {
|
||||
return fmt.Errorf("no data source collected successfully")
|
||||
}
|
||||
if cfg.DryRun {
|
||||
@@ -593,7 +630,7 @@ func runCollector(cfg runConfig, sources []DataSource, saveFn func([]ModelPricin
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Info("多源采集完成", "total_models", len(allPrices), "sources", successfulSources)
|
||||
logger.Info("多源采集完成", "total_models", len(allPrices), "sources", summary.SuccessfulSources)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user