62 lines
1.4 KiB
Go
62 lines
1.4 KiB
Go
|
//go:build llm_script
|
||
|
|
|
||
|
|
package main
|
||
|
|
|
||
|
|
import (
|
||
|
|
"flag"
|
||
|
|
"fmt"
|
||
|
|
"io"
|
||
|
|
"net/http"
|
||
|
|
"os"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
func main() {
|
||
|
|
var rawURL string
|
||
|
|
var dryRun bool
|
||
|
|
var timeoutSeconds int
|
||
|
|
var fixturePath string
|
||
|
|
|
||
|
|
flag.StringVar(&rawURL, "url", defaultTencentCatalogURL, "腾讯云公开目录 URL")
|
||
|
|
flag.BoolVar(&dryRun, "dry-run", false, "仅抓取并打印摘要，不写入数据库")
|
||
|
|
flag.IntVar(&timeoutSeconds, "timeout", int(defaultTencentCatalogTimeout/time.Second), "请求超时（秒）")
|
||
|
|
flag.StringVar(&fixturePath, "fixture", "", "本地 HTML/Text 样例文件，优先用于离线 dry-run")
|
||
|
|
flag.Parse()
|
||
|
|
|
||
|
|
cfg := fetchTencentCatalogConfig{
|
||
|
|
URL: rawURL,
|
||
|
|
DryRun: dryRun,
|
||
|
|
Timeout: time.Duration(timeoutSeconds) * time.Second,
|
||
|
|
Fixture: fixturePath,
|
||
|
|
}
|
||
|
|
|
||
|
|
client := &http.Client{Timeout: cfg.Timeout}
|
||
|
|
if err := runTencentCatalog(cfg, client, os.Stdout); err != nil {
|
||
|
|
fmt.Fprintf(os.Stderr, "fetch_tencent_catalog: %v\n", err)
|
||
|
|
os.Exit(1)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func runTencentCatalog(cfg fetchTencentCatalogConfig, client *http.Client, out io.Writer) error {
|
||
|
|
raw, err := fetchTencentCatalogContent(cfg, client)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
catalog, err := parseTencentCatalog(raw)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
_, err = fmt.Fprintf(
|
||
|
|
out,
|
||
|
|
"source=tencent-public-catalog updated_at=%s plans=%d models=%d series=%s dry_run=%t\n",
|
||
|
|
catalog.UpdatedAt,
|
||
|
|
len(catalog.Plans),
|
||
|
|
len(catalog.Models),
|
||
|
|
formatSeriesSummary(catalog.Plans),
|
||
|
|
cfg.DryRun,
|
||
|
|
)
|
||
|
|
return err
|
||
|
|
}
|