niuniu/sub2api-cn-relay-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Complete batch import v2 runtime and host capability recovery

Browse Source
This commit is contained in:
phamnazage-jpg
2026-05-23 09:18:02 +08:00
parent e50c292c7f
commit cfa1eaa904
60 changed files with 3718 additions and 530 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
internal/store/sqlite/import_batches_repo.go
View File

@@ -182,6 +182,36 @@ func (r *ImportBatchesRepo) ListByProviderIDAndHostID(ctx context.Context, provi
return batches, nil
}
func (r *ImportBatchesRepo) ListLatestReconcilable(ctx context.Context) ([]ImportBatch, error) {
rows, err := r.db.QueryContext(ctx, `
SELECT ib.id, ib.host_id, ib.pack_id, ib.provider_id, ib.mode, ib.batch_status, ib.access_status
FROM import_batches ib
INNER JOIN (
SELECT provider_id, host_id, MAX(id) AS latest_id
FROM import_batches
GROUP BY provider_id, host_id
) latest ON latest.latest_id = ib.id
WHERE ib.batch_status IN ('succeeded', 'partially_succeeded')
ORDER BY ib.id DESC`)
if err != nil {
return nil, fmt.Errorf("query latest reconcilable import batches: %w", err)
}
defer rows.Close()
batches := make([]ImportBatch, 0)
for rows.Next() {
var batch ImportBatch
if err := rows.Scan(&batch.ID, &batch.HostID, &batch.PackID, &batch.ProviderID, &batch.Mode, &batch.BatchStatus, &batch.AccessStatus); err != nil {
return nil, fmt.Errorf("scan latest reconcilable import batch: %w", err)
}
batches = append(batches, batch)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate latest reconcilable import batches: %w", err)
}
return batches, nil
}
func (r *ImportBatchItemsRepo) GetByBatchID(ctx context.Context, batchID int64) ([]ImportBatchItem, error) {
if batchID <= 0 {
return nil, fmt.Errorf("batch_id is required")

42
internal/store/sqlite/import_run_items_repo.go
View File

@@ -5,6 +5,7 @@ import (
"database/sql"
"fmt"
"strings"
"time"
)
type ImportRunItem struct {
@@ -236,6 +237,47 @@ func (r *ImportRunItemsRepo) ListByRunID(ctx context.Context, runID string) ([]I
return items, nil
}
func (r *ImportRunItemsRepo) TryAcquireConfirmationLease(ctx context.Context, itemID, workerID string, now time.Time, leaseDuration time.Duration) (ImportRunItem, bool, error) {
itemID = strings.TrimSpace(itemID)
workerID = strings.TrimSpace(workerID)
if itemID == "" {
return ImportRunItem{}, false, fmt.Errorf("item_id is required")
}
if workerID == "" {
return ImportRunItem{}, false, fmt.Errorf("worker_id is required")
}
if leaseDuration <= 0 {
leaseDuration = time.Minute
}
nowText := now.UTC().Format(time.RFC3339)
leaseUntil := now.UTC().Add(leaseDuration).Format(time.RFC3339)
result, err := r.db.ExecContext(ctx, `UPDATE import_run_items
SET lease_owner = ?, lease_until = ?, confirmation_attempts = confirmation_attempts + 1, updated_at = CURRENT_TIMESTAMP
WHERE item_id = ?
AND current_stage = 'confirm'
AND confirmation_status = 'pending'
AND (next_retry_at IS NULL OR next_retry_at = '' OR next_retry_at <= ?)
AND (lease_until IS NULL OR lease_until = '' OR lease_until < ?)`,
workerID, leaseUntil, itemID, nowText, nowText)
if err != nil {
return ImportRunItem{}, false, fmt.Errorf("acquire confirmation lease for %q: %w", itemID, err)
}
rows, err := result.RowsAffected()
if err != nil {
return ImportRunItem{}, false, err
}
if rows == 0 {
return ImportRunItem{}, false, nil
}
item, err := r.GetByItemID(ctx, itemID)
if err != nil {
return ImportRunItem{}, false, err
}
return item, true, nil
}
func boolToInt(value bool) int {
if value {
return 1

58
internal/store/sqlite/import_runs_repo.go
View File

@@ -7,19 +7,23 @@ import (
)
type ImportRun struct {
RunID string
Mode string
AccessMode string
State string
TotalItems int
CompletedItems int
ActiveItems int
DegradedItems int
BrokenItems int
WarningItems int
StartedAt string
UpdatedAt string
FinishedAt string
RunID string
HostID string
Mode string
AccessMode string
SubscriptionUsersJSON string
SubscriptionDays int
ProbeAPIKey string
State string
TotalItems int
CompletedItems int
ActiveItems int
DegradedItems int
BrokenItems int
WarningItems int
StartedAt string
UpdatedAt string
FinishedAt string
}
type ImportRunsRepo struct {
@@ -32,13 +36,18 @@ func newImportRunsRepo(db execQuerier) *ImportRunsRepo {
func (r *ImportRunsRepo) Create(ctx context.Context, run ImportRun) error {
runID := strings.TrimSpace(run.RunID)
hostID := strings.TrimSpace(run.HostID)
mode := strings.TrimSpace(run.Mode)
accessMode := strings.TrimSpace(run.AccessMode)
subscriptionUsersJSON := defaultJSON(run.SubscriptionUsersJSON, "[]")
probeAPIKey := strings.TrimSpace(run.ProbeAPIKey)
state := strings.TrimSpace(run.State)
switch {
case runID == "":
return fmt.Errorf("run_id is required")
case hostID == "":
return fmt.Errorf("host_id is required")
case mode == "":
return fmt.Errorf("mode is required")
case accessMode == "":
@@ -47,8 +56,8 @@ func (r *ImportRunsRepo) Create(ctx context.Context, run ImportRun) error {
return fmt.Errorf("state is required")
}
if _, err := r.db.ExecContext(ctx, `INSERT INTO import_runs (run_id, mode, access_mode, state, total_items, completed_items, active_items, degraded_items, broken_items, warning_items) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
runID, mode, accessMode, state, run.TotalItems, run.CompletedItems, run.ActiveItems, run.DegradedItems, run.BrokenItems, run.WarningItems); err != nil {
if _, err := r.db.ExecContext(ctx, `INSERT INTO import_runs (run_id, host_id, mode, access_mode, subscription_users_json, subscription_days, probe_api_key, state, total_items, completed_items, active_items, degraded_items, broken_items, warning_items) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
runID, hostID, mode, accessMode, subscriptionUsersJSON, run.SubscriptionDays, probeAPIKey, state, run.TotalItems, run.CompletedItems, run.ActiveItems, run.DegradedItems, run.BrokenItems, run.WarningItems); err != nil {
return fmt.Errorf("insert import run %q: %w", runID, err)
}
return nil
@@ -56,14 +65,19 @@ func (r *ImportRunsRepo) Create(ctx context.Context, run ImportRun) error {
func (r *ImportRunsRepo) Update(ctx context.Context, run ImportRun) error {
runID := strings.TrimSpace(run.RunID)
hostID := strings.TrimSpace(run.HostID)
mode := strings.TrimSpace(run.Mode)
accessMode := strings.TrimSpace(run.AccessMode)
subscriptionUsersJSON := defaultJSON(run.SubscriptionUsersJSON, "[]")
probeAPIKey := strings.TrimSpace(run.ProbeAPIKey)
state := strings.TrimSpace(run.State)
finishedAt := strings.TrimSpace(run.FinishedAt)
switch {
case runID == "":
return fmt.Errorf("run_id is required")
case hostID == "":
return fmt.Errorf("host_id is required")
case mode == "":
return fmt.Errorf("mode is required")
case accessMode == "":
@@ -73,9 +87,9 @@ func (r *ImportRunsRepo) Update(ctx context.Context, run ImportRun) error {
}
if _, err := r.db.ExecContext(ctx, `UPDATE import_runs
SET mode = ?, access_mode = ?, state = ?, total_items = ?, completed_items = ?, active_items = ?, degraded_items = ?, broken_items = ?, warning_items = ?, finished_at = ?, updated_at = CURRENT_TIMESTAMP
WHERE run_id = ?`,
mode, accessMode, state, run.TotalItems, run.CompletedItems, run.ActiveItems, run.DegradedItems, run.BrokenItems, run.WarningItems, nullableString(finishedAt), runID); err != nil {
SET host_id = ?, mode = ?, access_mode = ?, subscription_users_json = ?, subscription_days = ?, probe_api_key = ?, state = ?, total_items = ?, completed_items = ?, active_items = ?, degraded_items = ?, broken_items = ?, warning_items = ?, finished_at = ?, updated_at = CURRENT_TIMESTAMP
WHERE run_id = ?`,
hostID, mode, accessMode, subscriptionUsersJSON, run.SubscriptionDays, probeAPIKey, state, run.TotalItems, run.CompletedItems, run.ActiveItems, run.DegradedItems, run.BrokenItems, run.WarningItems, nullableString(finishedAt), runID); err != nil {
return fmt.Errorf("update import run %q: %w", runID, err)
}
return nil
@@ -88,8 +102,8 @@ func (r *ImportRunsRepo) GetByRunID(ctx context.Context, runID string) (ImportRu
}
var run ImportRun
if err := r.db.QueryRowContext(ctx, `SELECT run_id, mode, access_mode, state, total_items, completed_items, active_items, degraded_items, broken_items, warning_items, started_at, updated_at, COALESCE(finished_at, '') FROM import_runs WHERE run_id = ?`, runID).
Scan(&run.RunID, &run.Mode, &run.AccessMode, &run.State, &run.TotalItems, &run.CompletedItems, &run.ActiveItems, &run.DegradedItems, &run.BrokenItems, &run.WarningItems, &run.StartedAt, &run.UpdatedAt, &run.FinishedAt); err != nil {
if err := r.db.QueryRowContext(ctx, `SELECT run_id, host_id, mode, access_mode, subscription_users_json, subscription_days, COALESCE(probe_api_key, ''), state, total_items, completed_items, active_items, degraded_items, broken_items, warning_items, started_at, updated_at, COALESCE(finished_at, '') FROM import_runs WHERE run_id = ?`, runID).
Scan(&run.RunID, &run.HostID, &run.Mode, &run.AccessMode, &run.SubscriptionUsersJSON, &run.SubscriptionDays, &run.ProbeAPIKey, &run.State, &run.TotalItems, &run.CompletedItems, &run.ActiveItems, &run.DegradedItems, &run.BrokenItems, &run.WarningItems, &run.StartedAt, &run.UpdatedAt, &run.FinishedAt); err != nil {
return ImportRun{}, err
}
return run, nil
@@ -100,7 +114,7 @@ func (r *ImportRunsRepo) List(ctx context.Context, limit int) ([]ImportRun, erro
limit = 50
}
rows, err := r.db.QueryContext(ctx, `SELECT run_id, mode, access_mode, state, total_items, completed_items, active_items, degraded_items, broken_items, warning_items, started_at, updated_at, COALESCE(finished_at, '') FROM import_runs ORDER BY started_at DESC LIMIT ?`, limit)
rows, err := r.db.QueryContext(ctx, `SELECT run_id, host_id, mode, access_mode, subscription_users_json, subscription_days, COALESCE(probe_api_key, ''), state, total_items, completed_items, active_items, degraded_items, broken_items, warning_items, started_at, updated_at, COALESCE(finished_at, '') FROM import_runs ORDER BY started_at DESC LIMIT ?`, limit)
if err != nil {
return nil, fmt.Errorf("list import runs: %w", err)
}
@@ -109,7 +123,7 @@ func (r *ImportRunsRepo) List(ctx context.Context, limit int) ([]ImportRun, erro
runs := make([]ImportRun, 0)
for rows.Next() {
var run ImportRun
if err := rows.Scan(&run.RunID, &run.Mode, &run.AccessMode, &run.State, &run.TotalItems, &run.CompletedItems, &run.ActiveItems, &run.DegradedItems, &run.BrokenItems, &run.WarningItems, &run.StartedAt, &run.UpdatedAt, &run.FinishedAt); err != nil {
if err := rows.Scan(&run.RunID, &run.HostID, &run.Mode, &run.AccessMode, &run.SubscriptionUsersJSON, &run.SubscriptionDays, &run.ProbeAPIKey, &run.State, &run.TotalItems, &run.CompletedItems, &run.ActiveItems, &run.DegradedItems, &run.BrokenItems, &run.WarningItems, &run.StartedAt, &run.UpdatedAt, &run.FinishedAt); err != nil {
return nil, fmt.Errorf("scan import run: %w", err)
}
runs = append(runs, run)

26
internal/store/sqlite/import_runs_repo_test.go
View File

@@ -13,11 +13,15 @@ func TestRunStateStore(t *testing.T) {
store := openTestDB(t)
run := ImportRun{
RunID: "run-1",
Mode: "strict",
AccessMode: "subscription",
State: "running",
TotalItems: 1,
RunID: "run-1",
HostID: "host-1",
Mode: "strict",
AccessMode: "subscription",
SubscriptionUsersJSON: `["user-1"]`,
SubscriptionDays: 30,
ProbeAPIKey: "probe-key",
State: "running",
TotalItems: 1,
}
if err := store.ImportRuns().Create(ctx, run); err != nil {
t.Fatalf("ImportRuns().Create() error = %v", err)
@@ -42,6 +46,18 @@ func TestRunStateStore(t *testing.T) {
if gotRun.WarningItems != 1 {
t.Fatalf("run.WarningItems = %d, want 1", gotRun.WarningItems)
}
if gotRun.HostID != "host-1" {
t.Fatalf("run.HostID = %q, want host-1", gotRun.HostID)
}
if gotRun.SubscriptionUsersJSON != `["user-1"]` {
t.Fatalf("run.SubscriptionUsersJSON = %q, want persisted subscription users", gotRun.SubscriptionUsersJSON)
}
if gotRun.SubscriptionDays != 30 {
t.Fatalf("run.SubscriptionDays = %d, want 30", gotRun.SubscriptionDays)
}
if gotRun.ProbeAPIKey != "probe-key" {
t.Fatalf("run.ProbeAPIKey = %q, want probe-key", gotRun.ProbeAPIKey)
}
legacyBatchID := int64(88)
reusedAccountID := int64(321)

9
internal/store/sqlite/reconcile_runs_repo.go
View File

@@ -13,6 +13,7 @@ type ReconcileRun struct {
ProviderID int64
Status string
SummaryJSON string
CreatedAt string
}
type ReconcileRunsRepo struct {
@@ -58,7 +59,7 @@ func (r *ReconcileRunsRepo) GetByBatchID(ctx context.Context, batchID int64) ([]
return nil, fmt.Errorf("batch_id is required")
}
rows, err := r.db.QueryContext(ctx, `SELECT id, batch_id, host_id, provider_id, status, summary_json FROM reconcile_runs WHERE batch_id = ? ORDER BY id DESC`, batchID)
rows, err := r.db.QueryContext(ctx, `SELECT id, batch_id, host_id, provider_id, status, summary_json, created_at FROM reconcile_runs WHERE batch_id = ? ORDER BY id DESC`, batchID)
if err != nil {
return nil, fmt.Errorf("query reconcile runs by batch_id: %w", err)
}
@@ -67,7 +68,7 @@ func (r *ReconcileRunsRepo) GetByBatchID(ctx context.Context, batchID int64) ([]
runs := make([]ReconcileRun, 0)
for rows.Next() {
var run ReconcileRun
if err := rows.Scan(&run.ID, &run.BatchID, &run.HostID, &run.ProviderID, &run.Status, &run.SummaryJSON); err != nil {
if err := rows.Scan(&run.ID, &run.BatchID, &run.HostID, &run.ProviderID, &run.Status, &run.SummaryJSON, &run.CreatedAt); err != nil {
return nil, fmt.Errorf("scan reconcile run by batch_id: %w", err)
}
runs = append(runs, run)
@@ -86,7 +87,7 @@ func (r *ReconcileRunsRepo) GetByProviderIDAndHostID(ctx context.Context, provid
return nil, fmt.Errorf("host_id is required")
}
rows, err := r.db.QueryContext(ctx, `SELECT id, batch_id, host_id, provider_id, status, summary_json FROM reconcile_runs WHERE provider_id = ? AND host_id = ? ORDER BY id DESC`, providerID, hostID)
rows, err := r.db.QueryContext(ctx, `SELECT id, batch_id, host_id, provider_id, status, summary_json, created_at FROM reconcile_runs WHERE provider_id = ? AND host_id = ? ORDER BY id DESC`, providerID, hostID)
if err != nil {
return nil, fmt.Errorf("query reconcile runs: %w", err)
}
@@ -95,7 +96,7 @@ func (r *ReconcileRunsRepo) GetByProviderIDAndHostID(ctx context.Context, provid
runs := make([]ReconcileRun, 0)
for rows.Next() {
var run ReconcileRun
if err := rows.Scan(&run.ID, &run.BatchID, &run.HostID, &run.ProviderID, &run.Status, &run.SummaryJSON); err != nil {
if err := rows.Scan(&run.ID, &run.BatchID, &run.HostID, &run.ProviderID, &run.Status, &run.SummaryJSON, &run.CreatedAt); err != nil {
return nil, fmt.Errorf("scan reconcile run: %w", err)
}
runs = append(runs, run)

56
internal/store/sqlite/reconcile_runs_repo_test.go Normal file
View File

@@ -0,0 +1,56 @@
package sqlite
import (
"context"
"testing"
)
func TestReconcileRunsRepoPersistsCreatedAt(t *testing.T) {
t.Parallel()
store := openTestDB(t)
hostID := createTestHostWithBaseURL(t, store, "host-1", "https://host.example")
packID := createTestPack(t, store)
providerID, err := store.Providers().Create(context.Background(), Provider{
PackID: packID,
ProviderID: "provider-1",
DisplayName: "Provider 1",
BaseURL: "https://provider.example",
Platform: "openai",
})
if err != nil {
t.Fatalf("Providers().Create() error = %v", err)
}
batchID, err := store.ImportBatches().Create(context.Background(), ImportBatch{
HostID: hostID,
PackID: packID,
ProviderID: providerID,
Mode: "partial",
BatchStatus: "succeeded",
AccessStatus: "self_service_ready",
})
if err != nil {
t.Fatalf("ImportBatches().Create() error = %v", err)
}
if _, err := store.ReconcileRuns().Create(context.Background(), ReconcileRun{
BatchID: batchID,
HostID: hostID,
ProviderID: providerID,
Status: "active",
SummaryJSON: `{"ok":true}`,
}); err != nil {
t.Fatalf("Create() error = %v", err)
}
runs, err := store.ReconcileRuns().GetByProviderIDAndHostID(context.Background(), providerID, hostID)
if err != nil {
t.Fatalf("GetByProviderIDAndHostID() error = %v", err)
}
if len(runs) != 1 {
t.Fatalf("reconcile runs = %d, want 1", len(runs))
}
if runs[0].CreatedAt == "" {
t.Fatal("CreatedAt = empty, want timestamp")
}
}